Added Migration Command

This management command can be used to migrate data from the LMS to Oscar. Note that it does NOT save SKUs back to LMS.

XCOM-193

ecommerce/courses/admin.py

+from django.contrib import admin
+
+from ecommerce.courses.models import Course
+
+
+class CourseAdmin(admin.ModelAdmin):
+    list_display = ('id', 'name',)
+
+
+admin.site.register(Course, CourseAdmin)

ecommerce/courses/migrations/0001_initial.py

+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Course',
+            fields=[
+                ('id', models.CharField(max_length=255, serialize=False, verbose_name=b'ID', primary_key=True)),
+                ('name', models.CharField(max_length=255)),
+            ],
+            options={
+            },
+            bases=(models.Model,),
+        ),
+    ]

ecommerce/courses/models.py

+import logging
+
+from django.db import models
+from oscar.core.loading import get_model
+
+logger = logging.getLogger(__name__)
+ProductClass = get_model('catalogue', 'ProductClass')
+
+
+class Course(models.Model):
+    id = models.CharField(null=False, max_length=255, primary_key=True, verbose_name='ID')
+    name = models.CharField(null=False, max_length=255)
+
+    @classmethod
+    def is_mode_verified(cls, mode):
+        """ Returns True if the mode is verified, otherwise False. """
+        return mode.lower() in ('verified', 'professional', 'credit')
+
+    @classmethod
+    def certificate_type_for_mode(cls, mode):
+        mode = mode.lower()
+
+        if mode == 'no-id-professional':
+            return 'professional'
+
+        return mode
+
+    @property
+    def seat_products(self):
+        """ Returns a list of course seat Products related to this course. """
+        seat_product_class = ProductClass.objects.get(slug='seat')
+        products = set()
+
+        for product in self.products.all():
+            if product.get_product_class() == seat_product_class:
+                if product.is_parent:
+                    products.update(product.children.all())
+                else:
+                    products.add(product)
+
+        return list(products)

ecommerce/courses/tests.py

+import ddt
+from django.test import TestCase
+from django_dynamic_fixture import G
+
+from ecommerce.courses.models import Course
+from ecommerce.extensions.catalogue.tests.mixins import CourseCatalogTestMixin
+
+
+@ddt.ddt
+class CourseTests(CourseCatalogTestMixin, TestCase):
+    def test_seat_products(self):
+        """
+        Verify the method returns a list containing purchasable course seats.
+
+        These seats should be the child products.
+        """
+        # Create a new course and verify it has no existing products.
+        course = G(Course)
+        self.assertEqual(course.products.count(), 0)
+        self.assertEqual(len(course.seat_products), 0)
+
+        # Create the seat products
+        seats = self.create_course_seats(course.id, ('honor', 'verified'))
+
+        # Associate the parent and child products with the course. The method should be able to filter out the parent.
+        parent = seats.values()[0].parent
+        parent.course = course
+        parent.save()
+
+        for seat in seats.itervalues():
+            seat.course = course
+            seat.save()
+
+        self.assertEqual(course.products.count(), 3)
+
+        # The method should return only the child seats.
+        # We do not necessarily care about the order, but must sort to check equality.
+        expected = seats.values().sort()
+        self.assertEqual(course.seat_products.sort(), expected)
+
+    @ddt.data(
+        ('verified', True),
+        ('credit', True),
+        ('professional', True),
+        ('honor', False),
+        ('no-id-professional', False),
+        ('audit', False),
+        ('unknown', False),
+    )
+    @ddt.unpack
+    def test_is_mode_verified(self, mode, expected):
+        """ Verify the method returns True only for verified modes. """
+        self.assertEqual(Course.is_mode_verified(mode), expected)
+
+    @ddt.data(
+        ('Verified', 'verified'),
+        ('credit', 'credit'),
+        ('professional', 'professional'),
+        ('honor', 'honor'),
+        ('no-id-professional', 'professional'),
+        ('audit', 'audit'),
+        ('unknown', 'unknown'),
+    )
+    @ddt.unpack
+    def test_certificate_type_for_mode(self, mode, expected):
+        """ Verify the method returns the correct certificate type for a given mode. """
+        self.assertEqual(Course.certificate_type_for_mode(mode), expected)

ecommerce/extensions/catalogue/admin.py

-import oscar.apps.catalogue.admin  # pragma: no cover pylint: disable=unused-import
+from oscar.apps.catalogue.admin import *  # pylint: disable=unused-import,wildcard-import,unused-wildcard-import
+
+
+class ProductAdminExtended(ProductAdmin):
+    list_display = ('get_title', 'upc', 'get_product_class', 'structure',
+                    'attribute_summary', 'date_created', 'course')
+
+
+admin.site.unregister(Product)
+admin.site.register(Product, ProductAdminExtended)

ecommerce/extensions/catalogue/management/commands/migrate_course.py

+import logging
+from optparse import make_option
+from urlparse import urljoin
+
+from django.conf import settings
+from django.core.management import BaseCommand
+from django.db import transaction
+from django.utils.text import slugify
+from oscar.core.loading import get_model
+import requests
+
+from ecommerce.courses.models import Course
+from ecommerce.extensions.catalogue.utils import generate_sku
+
+logger = logging.getLogger(__name__)
+
+Category = get_model('catalogue', 'Category')
+Partner = get_model('partner', 'Partner')
+Product = get_model('catalogue', 'Product')
+ProductCategory = get_model('catalogue', 'ProductCategory')
+ProductClass = get_model('catalogue', 'ProductClass')
+StockRecord = get_model('partner', 'StockRecord')
+
+
+class MigratedCourse(object):
+    def __init__(self, course_id):
+        # Ensure this value is Unicode to avoid issues with slugify.
+        self.course_id = unicode(course_id)
+
+        self.parent_seat = None
+        self.child_seats = {}
+        self._unsaved_stock_records = {}
+
+        try:
+            self.course = Course.objects.get(id=self.course_id)
+        except Course.DoesNotExist:
+            self.course = Course(id=self.course_id)
+
+    @transaction.atomic
+    def save(self):
+        self.course.save()
+
+        self.parent_seat.course = self.course
+        self.parent_seat.save()
+        category = Category.objects.get(name='Seats')
+        ProductCategory.objects.get_or_create(category=category, product=self.parent_seat)
+
+        for product in self.child_seats.values():
+            product.parent = self.parent_seat
+            product.course = self.course
+            product.save()
+
+        for seat_type, stock_record in self._unsaved_stock_records.iteritems():
+            stock_record.product = self.child_seats[seat_type]
+            stock_record.save()
+
+        self._unsaved_stock_records = {}
+
+    def load_from_lms(self):
+        """
+        Loads course products from the LMS.
+
+        Loaded data is NOT persisted until the save() method is called.
+        """
+        name, modes = self._retrieve_data_from_lms()
+        self.course.name = name
+        self._get_products(name, modes)
+
+    def _retrieve_data_from_lms(self):
+        """
+        Retrieves the course name and modes from the LMS.
+        """
+        headers = {
+            'Accept': 'application/json',
+            'X-Edx-Api-Key': settings.EDX_API_KEY
+        }
+
+        # Get course name from Course Structure API
+        url = urljoin(settings.LMS_URL_ROOT, 'api/course_structure/v0/courses/{}/'.format(self.course_id))
+        response = requests.get(url, headers=headers)
+        data = response.json()
+        logger.debug(data)
+        course_name = data['name']
+
+        # TODO Handle non-200 responses and other errors
+
+        # Get modes and pricing from Enrollment API
+        url = urljoin(settings.LMS_URL_ROOT, 'api/enrollment/v1/course/{}'.format(self.course_id))
+        response = requests.get(url, headers=headers)
+        data = response.json()
+        logger.debug(data)
+        modes = data['course_modes']
+
+        # TODO Handle non-200 responses and other errors
+
+        return course_name, modes
+
+    def _get_product_name(self, course_name, mode):
+        name = u'Seat in {course_name} with {certificate_type} certificate'.format(
+            course_name=course_name,
+            certificate_type=Course.certificate_type_for_mode(mode))
+
+        if Course.is_mode_verified(mode):
+            name += u' (and ID verification)'
+
+        return name
+
+    def _get_products(self, course_name, modes):
+        """
+        Creates course seat products.
+
+        Returns:
+            seats (dict):  Mapping of seat types to seat Products
+            stock_records (dict):  Mapping of seat types to StockRecords
+        """
+
+        course_id = self.course_id
+
+        seats = {}
+        stock_records = {}
+        slug = u'parent-cs-{}'.format(slugify(course_id))
+        partner = Partner.objects.get(code='edx')
+
+        try:
+            parent = Product.objects.get(slug=slug)
+            logger.info(u'Retrieved parent seat product for [%s] from database.', course_id)
+        except Product.DoesNotExist:
+            product_class = ProductClass.objects.get(slug='seat')
+            parent = Product(slug=slug, is_discountable=True, structure=Product.PARENT, product_class=product_class)
+            logger.info(u'Parent seat product for [%s] does not exist. Instantiated a new instance.', course_id)
+
+        parent.title = u'Seat in {}'.format(course_name)
+        parent.attr.course_key = course_id
+
+        # Create the child products
+        for mode in modes:
+            seat_type = mode['slug']
+            slug = u'child-cs-{}-{}'.format(seat_type, slugify(course_id))
+            try:
+                seat = Product.objects.get(slug=slug)
+                logger.info(u'Retrieved [%s] course seat child product for [%s] from database.', seat_type, course_id)
+            except Product.DoesNotExist:
+                seat = Product(slug=slug)
+                logger.info(u'[%s] course seat product for [%s] does not exist. Instantiated a new instance.',
+                            seat_type, course_id)
+
+            seat.parent = parent
+            seat.is_discountable = True
+            seat.structure = Product.CHILD
+            seat.title = self._get_product_name(course_name, seat_type)
+            seat.attr.certificate_type = seat_type
+            seat.attr.course_key = course_id
+            seat.attr.id_verification_required = Course.is_mode_verified(seat_type)
+
+            seats[seat_type] = seat
+
+            try:
+                stock_record = StockRecord.objects.get(product=seat, partner=partner)
+                logger.info(u'Retrieved [%s] course seat child product stock record for [%s] from database.',
+                            seat_type, course_id)
+            except StockRecord.DoesNotExist:
+                partner_sku = generate_sku(seat)
+                stock_record = StockRecord(product=seat, partner=partner, partner_sku=partner_sku)
+                logger.info(
+                    u'[%s] course seat product stock record for [%s] does not exist. Instantiated a new instance.',
+                    seat_type, course_id)
+
+            stock_record.price_excl_tax = mode['min_price']
+            stock_record.price_currency = 'USD'
+
+            stock_records[seat_type] = stock_record
+
+        self.parent_seat = parent
+        self.child_seats = seats
+        self._unsaved_stock_records = stock_records
+
+
+class Command(BaseCommand):
+    help = 'Migrate course modes and pricing from LMS to Oscar.'
+
+    option_list = BaseCommand.option_list + (
+        make_option('--commit',
+                    action='store_true',
+                    dest='commit',
+                    default=False,
+                    help='Save the migrated data to the database. If this is not set, '
+                         'migrated data will NOT be saved to the database.'),
+    )
+
+    def handle(self, *args, **options):
+        course_ids = args
+
+        for course_id in course_ids:
+            course_id = unicode(course_id)
+            try:
+                migrated_course = MigratedCourse(course_id)
+                migrated_course.load_from_lms()
+
+                course = migrated_course.course
+                msg = 'Retrieved info for {0} ({1}):\n'.format(course.id, course.name)
+
+                for seat_type, seat in migrated_course.child_seats.iteritems():
+                    stock_record = migrated_course._unsaved_stock_records[seat_type]  # pylint: disable=protected-access
+                    data = (seat_type, seat.attr.id_verification_required,
+                            '{0} {1}'.format(stock_record.price_currency, stock_record.price_excl_tax),
+                            stock_record.partner_sku)
+                    msg += '\t{}\n'.format(data)
+
+                logger.info(msg)
+
+                if options.get('commit', False):
+                    migrated_course.save()
+            except Exception:  # pylint: disable=broad-except
+                logger.exception('Failed to migrate [%s]!', course_id)

ecommerce/extensions/catalogue/migrations/0003_product_course.py

+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('courses', '0001_initial'),
+        ('catalogue', '0002_auto_20150223_1052'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='product',
+            name='course',
+            field=models.ForeignKey(related_name='products', blank=True, to='courses.Course', null=True),
+            preserve_default=True,
+        ),
+    ]

ecommerce/extensions/catalogue/models.py

-from oscar.apps.catalogue.models import *  # pragma: no cover pylint: disable=wildcard-import,unused-wildcard-import
+# noinspection PyUnresolvedReferences
+from django.db import models
+from oscar.apps.catalogue.abstract_models import AbstractProduct
+
+
+class Product(AbstractProduct):
+    course = models.ForeignKey('courses.Course', null=True, blank=True, related_name='products')
+
+# noinspection PyUnresolvedReferences
+from oscar.apps.catalogue.models import *  # noqa pylint: disable=wildcard-import,unused-wildcard-import

ecommerce/extensions/catalogue/tests/mixins.py

 from oscar.core.loading import get_model
 from oscar.test import factories
 
+Category = get_model('catalogue', 'Category')
+Partner = get_model('partner', 'Partner')
 ProductClass = get_model('catalogue', 'ProductClass')
 ProductAttribute = get_model('catalogue', 'ProductAttribute')
 
 
 class CourseCatalogTestMixin(object):
+    def setUp(self):
+        super(CourseCatalogTestMixin, self).setUp()
+
+        # Force the creation of a seat ProductClass
+        self.seat_product_class  # pylint: disable=pointless-statement
+        self.partner, _created = Partner.objects.get_or_create(code='edx')
+        self.category, _created = Category.objects.get_or_create(name='Seats', defaults={'depth': 1})
+
     @property
     def seat_product_class(self):
         defaults = {'requires_shipping': False, 'track_stock': False, 'name': 'Seat'}
-        product_class, created = ProductClass.objects.get_or_create(slug='seat', defaults=defaults)
+        pc, created = ProductClass.objects.get_or_create(slug='seat', defaults=defaults)
 
         if created:
-            factories.ProductAttributeFactory(code='certificate_type', product_class=product_class, type='text')
-            factories.ProductAttributeFactory(code='course_key', product_class=product_class, type='text')
+            factories.ProductAttributeFactory(code='certificate_type', product_class=pc, type='text')
+            factories.ProductAttributeFactory(code='course_key', product_class=pc, type='text')
+            factories.ProductAttributeFactory(code='id_verification_required', product_class=pc, type='boolean')
 
-        return product_class
+        return pc
 
     def create_course_seats(self, course_id, certificate_types):
         title = 'Seat in {}'.format(course_id)

ecommerce/extensions/catalogue/tests/test_migrate_course.py

+import json
+import logging
+from urlparse import urljoin
+
+from django.conf import settings
+from django.core.management import call_command
+from django.test import TestCase
+import httpretty
+from oscar.core.loading import get_model
+
+from ecommerce.courses.models import Course
+from ecommerce.extensions.catalogue.management.commands.migrate_course import MigratedCourse
+from ecommerce.extensions.catalogue.tests.mixins import CourseCatalogTestMixin
+from ecommerce.extensions.catalogue.utils import generate_sku
+
+JSON = 'application/json'
+
+logger = logging.getLogger(__name__)
+
+Category = get_model('catalogue', 'Category')
+Product = get_model('catalogue', 'Product')
+StockRecord = get_model('partner', 'StockRecord')
+
+
+class CourseMigrationTestMixin(CourseCatalogTestMixin):
+    course_id = 'aaa/bbb/ccc'
+
+    prices = {
+        'honor': 0,
+        'verified': 10,
+        'no-id-professional': 100,
+        'professional': 1000
+    }
+
+    def _mock_lms_api(self):
+        self.assertTrue(httpretty.is_enabled, 'httpretty must be enabled to mock LMS API calls.')
+
+        # Mock Course Structure API
+        url = urljoin(settings.LMS_URL_ROOT, 'api/course_structure/v0/courses/{}/'.format(self.course_id))
+        httpretty.register_uri(httpretty.GET, url, body='{"name": "A Test Course"}', content_type=JSON)
+
+        # Mock Enrollment API
+        url = urljoin(settings.LMS_URL_ROOT, 'api/enrollment/v1/course/{}'.format(self.course_id))
+        body = {
+            'course_id': self.course_id,
+            'course_modes': [{'slug': seat_type, 'min_price': price} for seat_type, price in self.prices.iteritems()]
+        }
+        httpretty.register_uri(httpretty.GET, url, body=json.dumps(body), content_type=JSON)
+
+    def assert_stock_record_valid(self, stock_record, seat, price):
+        """ Verify the given StockRecord is configured correctly. """
+        self.assertEqual(stock_record.partner, self.partner)
+        self.assertEqual(stock_record.price_excl_tax, price)
+        self.assertEqual(stock_record.price_currency, 'USD')
+        self.assertEqual(stock_record.partner_sku, generate_sku(seat))
+
+    def assert_seat_valid(self, seat, seat_type):
+        """ Verify the given seat is configured correctly. """
+        certificate_type = Course.certificate_type_for_mode(seat_type)
+
+        expected_title = u'Seat in A Test Course with {} certificate'.format(certificate_type)
+        if Course.is_mode_verified(seat_type):
+            expected_title += u' (and ID verification)'
+
+        self.assertEqual(seat.title, expected_title)
+        self.assertEqual(seat.attr.certificate_type, seat_type)
+        self.assertEqual(seat.attr.course_key, self.course_id)
+        self.assertEqual(seat.attr.id_verification_required, Course.is_mode_verified(seat_type))
+
+    def assert_course_migrated(self):
+        """ Verify the course was migrated and saved to the database. """
+        course = Course.objects.get(id=self.course_id)
+        seats = course.seat_products
+        self.assertEqual(len(seats), 4)
+        parent = course.products.get(structure=Product.PARENT)
+        self.assertEqual(list(parent.categories.all()), [self.category])
+        for seat in seats:
+            seat_type = seat.attr.certificate_type
+            logger.info('Validating objects for %s certificate type...', seat_type)
+
+            stock_record = self.partner.stockrecords.get(product=seat)
+            self.assert_seat_valid(seat, seat_type)
+            self.assert_stock_record_valid(stock_record, seat, self.prices[seat_type])
+
+
+class MigratedCourseTests(CourseMigrationTestMixin, TestCase):
+    def _migrate_course_from_lms(self):
+        """ Create a new MigratedCourse and simulate the loading of data from LMS. """
+        self._mock_lms_api()
+        migrated_course = MigratedCourse(self.course_id)
+        migrated_course.load_from_lms()
+        return migrated_course
+
+    def test_constructor(self):
+        """Verify the constructor instantiates a new object with default property values."""
+        migrated_course = MigratedCourse(self.course_id)
+        self.assertEqual(migrated_course.course_id, unicode(self.course_id))
+        self.assertEqual(migrated_course.course.id, self.course_id)
+        self.assertIsNone(migrated_course.parent_seat)
+        self.assertEqual(migrated_course.child_seats, {})
+        self.assertEqual(migrated_course._unsaved_stock_records, {})  # pylint: disable=protected-access
+
+    @httpretty.activate
+    def test_load_from_lms(self):
+        """ Verify the method creates new objects based on data loaded from the LMS. """
+        initial_product_count = Product.objects.count()
+        initial_stock_record_count = StockRecord.objects.count()
+
+        migrated_course = self._migrate_course_from_lms()
+
+        # Ensure LMS was called with the correct headers
+        for request in httpretty.httpretty.latest_requests:
+            self.assertEqual(request.headers['Accept'], JSON)
+            self.assertEqual(request.headers['X-Edx-Api-Key'], settings.EDX_API_KEY)
+
+        # Verify created objects match mocked data
+        parent_seat = migrated_course.parent_seat
+        self.assertEqual(parent_seat.title, 'Seat in A Test Course')
+
+        for seat_type, price in self.prices.iteritems():
+            logger.info('Validating objects for %s certificate type...', seat_type)
+
+            seat = migrated_course.child_seats[seat_type]
+            self.assert_seat_valid(seat, seat_type)
+
+            stock_record = migrated_course._unsaved_stock_records[seat_type]  # pylint: disable=protected-access
+            self.assert_stock_record_valid(stock_record, seat, price)
+
+        self.assertEqual(Product.objects.count(), initial_product_count, 'No new Products should have been saved.')
+        self.assertEqual(StockRecord.objects.count(), initial_stock_record_count,
+                         'No new StockRecords should have been saved.')
+
+    @httpretty.activate
+    def test_save(self):
+        """ Verify the method saves the data to the database. """
+        migrated_course = self._migrate_course_from_lms()
+        migrated_course.save()
+        self.assert_course_migrated()
+
+
+class CommandTests(CourseMigrationTestMixin, TestCase):
+    @httpretty.activate
+    def test_handle(self):
+        """ Verify the management command retrieves data, but does not save it to the database. """
+        initial_product_count = Product.objects.count()
+        initial_stock_record_count = StockRecord.objects.count()
+
+        self._mock_lms_api()
+        call_command('migrate_course', self.course_id)
+
+        # Ensure LMS was called with the correct headers
+        for request in httpretty.httpretty.latest_requests:
+            self.assertEqual(request.headers['Accept'], JSON)
+            self.assertEqual(request.headers['X-Edx-Api-Key'], settings.EDX_API_KEY)
+
+        self.assertEqual(Product.objects.count(), initial_product_count, 'No new Products should have been saved.')
+        self.assertEqual(StockRecord.objects.count(), initial_stock_record_count,
+                         'No new StockRecords should have been saved.')
+
+    @httpretty.activate
+    def test_handle_with_commit(self):
+        """ Verify the management command retrieves data, and saves it to the database. """
+        self._mock_lms_api()
+        call_command('migrate_course', self.course_id, commit=True)
+        self.assert_course_migrated()

ecommerce/extensions/catalogue/tests/test_utils.py

+from hashlib import md5
+
+from django.test import TestCase
+
+from ecommerce.extensions.catalogue.tests.mixins import CourseCatalogTestMixin
+from ecommerce.extensions.catalogue.utils import generate_sku
+
+
+class UtilsTests(CourseCatalogTestMixin, TestCase):
+    def test_generate_sku_for_course_seat(self):
+        """Verify the method generates a SKU for a course seat."""
+        course_id = 'sku/test/course'
+        certificate_type = 'honor'
+        product = self.create_course_seats(course_id, [certificate_type])[certificate_type]
+
+        _hash = md5(u'{} {}'.format(certificate_type, course_id)).hexdigest()[-7:]
+        expected = _hash.upper()
+        actual = generate_sku(product)
+        self.assertEqual(actual, expected)

ecommerce/extensions/catalogue/utils.py

+from hashlib import md5
+
+
+def generate_sku(product):
+    """
+    Generates a SKU for the given partner and and product combination.
+
+    Example: 76E4E71
+    """
+    # Note: This currently supports seats. In the future, this should
+    # be updated to accommodate other product classes.
+    _hash = u' '.join((product.attr.certificate_type.lower(), product.attr.course_key.lower()))
+    _hash = md5(_hash)
+    _hash = _hash.hexdigest()[-7:]
+    return _hash.upper()

ecommerce/settings/base.py

@@ -251,6 +251,7 @@ DJANGO_APPS = [
 LOCAL_APPS = [
     'ecommerce.user',
     'ecommerce.health',
+    'ecommerce.courses'
 ]
 
 # See: https://docs.djangoproject.com/en/dev/ref/settings/#installed-apps