No longer persist XModule templates

Instead, we use XModule field default values when creating an empty
XModule. Driven by this use case, we also allow for XModules to be
created in memory without being persisted to the database at all. This
necessitates a change to the Modulestore api, replacing clone_item with
create_draft and save_xmodule.

CHANGELOG.rst

@@ -43,6 +43,13 @@ history of background tasks for a given problem and student.
 Blades: Small UX fix on capa multiple-choice problems.  Make labels only
 as wide as the text to reduce accidental choice selections.
 
+Studio: 
+- use xblock field defaults to initialize all new instances' fields and
+only use templates as override samples.
+- create new instances via in memory create_xmodule and related methods rather
+than cloning a db record.
+- have an explicit method for making a draft copy as distinct from making a new module.
+
 Studio: Remove XML from the video component editor. All settings are
 moved to be edited as metadata.
 

README.md

@@ -239,7 +239,6 @@ CMS templates. Fortunately, `rake` will do all of this for you! Just run:
 
     $ rake django-admin[syncdb]
     $ rake django-admin[migrate]
-    $ rake cms:update_templates
 
 If you are running these commands using the [`zsh`](http://www.zsh.org/) shell,
 zsh will assume that you are doing

cms/djangoapps/contentstore/course_info_model.py

@@ -20,8 +20,8 @@ def get_course_updates(location):
     try:
         course_updates = modulestore('direct').get_item(location)
     except ItemNotFoundError:
-        template = Location(['i4x', 'edx', "templates", 'course_info', "Empty"])
-        course_updates = modulestore('direct').clone_item(template, Location(location))
+        modulestore('direct').create_and_save_xmodule(location)
+        course_updates = modulestore('direct').get_item(location)
 
     # current db rep: {"_id" : locationjson, "definition" : { "data" : "<ol>[<li><h2>date</h2>content</li>]</ol>"} "metadata" : ignored}
     location_base = course_updates.location.url()

cms/djangoapps/contentstore/features/common.py

@@ -208,7 +208,7 @@ def set_date_and_time(date_css, desired_date, time_css, desired_time):
 def i_created_a_video_component(step):
     world.create_component_instance(
         step, '.large-video-icon',
-        'i4x://edx/templates/video/default',
+        'video',
         '.xmodule_VideoModule'
     )
 

cms/djangoapps/contentstore/features/component_settings_editor_helpers.py

@@ -7,9 +7,9 @@ from terrain.steps import reload_the_page
 
 
 @world.absorb
-def create_component_instance(step, component_button_css, instance_id, expected_css):
+def create_component_instance(step, component_button_css, category, expected_css, boilerplate=None):
     click_new_component_button(step, component_button_css)
-    click_component_from_menu(instance_id, expected_css)
+    click_component_from_menu(category, boilerplate, expected_css)
 
 
 @world.absorb
@@ -19,7 +19,7 @@ def click_new_component_button(step, component_button_css):
 
 
 @world.absorb
-def click_component_from_menu(instance_id, expected_css):
+def click_component_from_menu(category, boilerplate, expected_css):
     """
     Creates a component from `instance_id`. For components with more
     than one template, clicks on `elem_css` to create the new
@@ -27,11 +27,13 @@ def click_component_from_menu(instance_id, expected_css):
     as the user clicks the appropriate button, so we assert that the
     expected component is present.
     """
-    elem_css = "a[data-location='%s']" % instance_id
+    if boilerplate:
+        elem_css = "a[data-category='{}'][data-boilerplate='{}']".format(category, boilerplate)
+    else:
+        elem_css = "a[data-category='{}']:not([data-boilerplate])".format(category)
     elements = world.css_find(elem_css)
-    assert(len(elements) == 1)
-    if elements[0]['id'] == instance_id:  # If this is a component with multiple templates
-        world.css_click(elem_css)
+    assert_equal(len(elements), 1)
+    world.css_click(elem_css)
     assert_equal(1, len(world.css_find(expected_css)))
 
 

cms/djangoapps/contentstore/features/discussion-editor.py

@@ -8,7 +8,7 @@ from lettuce import world, step
 def i_created_discussion_tag(step):
     world.create_component_instance(
         step, '.large-discussion-icon',
-        'i4x://edx/templates/discussion/Discussion_Tag',
+        'discussion',
         '.xmodule_DiscussionModule'
     )
 
@@ -26,5 +26,5 @@ def i_see_only_the_settings_and_values(step):
 @step('creating a discussion takes a single click')
 def discussion_takes_a_single_click(step):
     assert(not world.is_css_present('.xmodule_DiscussionModule'))
-    world.css_click("a[data-location='i4x://edx/templates/discussion/Discussion_Tag']")
+    world.css_click("a[data-category='discussion']")
     assert(world.is_css_present('.xmodule_DiscussionModule'))

cms/djangoapps/contentstore/features/html-editor.py

@@ -7,7 +7,7 @@ from lettuce import world, step
 @step('I have created a Blank HTML Page$')
 def i_created_blank_html_page(step):
     world.create_component_instance(
-        step, '.large-html-icon', 'i4x://edx/templates/html/Blank_HTML_Page',
+        step, '.large-html-icon', 'html',
         '.xmodule_HtmlModule'
     )
 

cms/djangoapps/contentstore/features/problem-editor.py

@@ -158,7 +158,7 @@ def create_latex_problem(step):
     world.click_new_component_button(step, '.large-problem-icon')
     # Go to advanced tab.
     world.css_click('#ui-id-2')
-    world.click_component_from_menu("i4x://edx/templates/problem/Problem_Written_in_LaTeX", '.xmodule_CapaModule')
+    world.click_component_from_menu("problem", "latex_problem.yaml", '.xmodule_CapaModule')
 
 
 @step('I edit and compile the High Level Source')

cms/djangoapps/contentstore/features/studio-overview-togglesection.py

@@ -22,7 +22,7 @@ def have_a_course_with_1_section(step):
     section = world.ItemFactory.create(parent_location=course.location)
     subsection1 = world.ItemFactory.create(
         parent_location=section.location,
-        template='i4x://edx/templates/sequential/Empty',
+        category='sequential',
         display_name='Subsection One',)
 
 
@@ -33,18 +33,18 @@ def have_a_course_with_two_sections(step):
     section = world.ItemFactory.create(parent_location=course.location)
     subsection1 = world.ItemFactory.create(
         parent_location=section.location,
-        template='i4x://edx/templates/sequential/Empty',
+        category='sequential',
         display_name='Subsection One',)
     section2 = world.ItemFactory.create(
         parent_location=course.location,
         display_name='Section Two',)
     subsection2 = world.ItemFactory.create(
         parent_location=section2.location,
-        template='i4x://edx/templates/sequential/Empty',
+        category='sequential',
         display_name='Subsection Alpha',)
     subsection3 = world.ItemFactory.create(
         parent_location=section2.location,
-        template='i4x://edx/templates/sequential/Empty',
+        category='sequential',
         display_name='Subsection Beta',)
 
 

cms/djangoapps/contentstore/features/video.py

@@ -14,7 +14,7 @@ def does_not_autoplay(_step):
 @step('creating a video takes a single click')
 def video_takes_a_single_click(_step):
     assert(not world.is_css_present('.xmodule_VideoModule'))
-    world.css_click("a[data-location='i4x://edx/templates/video/default']")
+    world.css_click("a[data-category='video']")
     assert(world.is_css_present('.xmodule_VideoModule'))
 
 

cms/djangoapps/contentstore/management/commands/update_templates.py

-from xmodule.templates import update_templates
-from xmodule.modulestore.django import modulestore
-from django.core.management.base import BaseCommand
-
-
-class Command(BaseCommand):
-    help = 'Imports and updates the Studio component templates from the code pack and put in the DB'
-
-    def handle(self, *args, **options):
-        update_templates(modulestore('direct'))

cms/djangoapps/contentstore/module_info_model.py

@@ -3,13 +3,13 @@ from xmodule.modulestore.exceptions import ItemNotFoundError
 from xmodule.modulestore import Location
 
 
-def get_module_info(store, location, parent_location=None, rewrite_static_links=False):
+def get_module_info(store, location, rewrite_static_links=False):
     try:
         module = store.get_item(location)
     except ItemNotFoundError:
         # create a new one
-        template_location = Location(['i4x', 'edx', 'templates', location.category, 'Empty'])
-        module = store.clone_item(template_location, location)
+        store.create_and_save_xmodule(location)
+        module = store.get_item(location)
 
     data = module.data
     if rewrite_static_links:
@@ -29,7 +29,8 @@ def get_module_info(store, location, parent_location=None, rewrite_static_links=
         'id': module.location.url(),
         'data': data,
         # TODO (cpennington): This really shouldn't have to do this much reaching in to get the metadata
-        'metadata': module._model_data._kvs._metadata
+        # what's the intent here? all metadata incl inherited & namespaced?
+        'metadata': module.xblock_kvs._metadata
     }
 
 
@@ -37,14 +38,11 @@ def set_module_info(store, location, post_data):
     module = None
     try:
         module = store.get_item(location)
-    except:
-        pass
-
-    if module is None:
-        # new module at this location
-        # presume that we have an 'Empty' template
-        template_location = Location(['i4x', 'edx', 'templates', location.category, 'Empty'])
-        module = store.clone_item(template_location, location)
+    except ItemNotFoundError:
+        # new module at this location: almost always used for the course about pages; thus, no parent. (there
+        # are quite a handful of about page types available for a course and only the overview is pre-created)
+        store.create_and_save_xmodule(location)
+        module = store.get_item(location)
 
     if post_data.get('data') is not None:
         data = post_data['data']
@@ -79,4 +77,4 @@ def set_module_info(store, location, post_data):
 
         # commit to datastore
         # TODO (cpennington): This really shouldn't have to do this much reaching in to get the metadata
-        store.update_metadata(location, module._model_data._kvs._metadata)
+        store.update_metadata(location, module.xblock_kvs._metadata)

cms/djangoapps/contentstore/tests/test_course_settings.py

@@ -19,6 +19,7 @@ from xmodule.modulestore.tests.factories import CourseFactory
 
 from models.settings.course_metadata import CourseMetadata
 from xmodule.modulestore.xml_importer import import_from_xml
+from xmodule.modulestore.django import modulestore
 from xmodule.fields import Date
 
 from .utils import CourseTestCase

cms/djangoapps/contentstore/tests/test_i18n.py

@@ -35,7 +35,6 @@ class InternationalizationTest(ModuleStoreTestCase):
         self.user.save()
 
         self.course_data = {
-            'template': 'i4x://edx/templates/course/Empty',
             'org': 'MITx',
             'number': '999',
             'display_name': 'Robot Super Course',

cms/djangoapps/contentstore/tests/test_item.py

 from contentstore.tests.test_course_settings import CourseTestCase
 from xmodule.modulestore.tests.factories import CourseFactory
 from django.core.urlresolvers import reverse
+from xmodule.capa_module import CapaDescriptor
+import json
+from xmodule.modulestore.django import modulestore
 
 
 class DeleteItem(CourseTestCase):
@@ -11,14 +14,199 @@ class DeleteItem(CourseTestCase):
 
     def testDeleteStaticPage(self):
         # Add static tab
-        data = {
+        data = json.dumps({
             'parent_location': 'i4x://mitX/333/course/Dummy_Course',
-            'template': 'i4x://edx/templates/static_tab/Empty'
-        }
+            'category': 'static_tab'
+        })
 
-        resp = self.client.post(reverse('clone_item'), data)
+        resp = self.client.post(reverse('create_item'), data,
+            content_type="application/json")
         self.assertEqual(resp.status_code, 200)
 
         # Now delete it. There was a bug that the delete was failing (static tabs do not exist in draft modulestore).
         resp = self.client.post(reverse('delete_item'), resp.content, "application/json")
         self.assertEqual(resp.status_code, 200)
+
+
+class TestCreateItem(CourseTestCase):
+    """
+    Test the create_item handler thoroughly
+    """
+    def response_id(self, response):
+        """
+        Get the id from the response payload
+        :param response:
+        """
+        parsed = json.loads(response.content)
+        return parsed['id']
+
+    def test_create_nicely(self):
+        """
+        Try the straightforward use cases
+        """
+        # create a chapter
+        display_name = 'Nicely created'
+        resp = self.client.post(
+            reverse('create_item'),
+            json.dumps({
+                'parent_location': self.course_location.url(),
+                'display_name': display_name,
+                'category': 'chapter'
+            }),
+            content_type="application/json"
+        )
+        self.assertEqual(resp.status_code, 200)
+
+        # get the new item and check its category and display_name
+        chap_location = self.response_id(resp)
+        new_obj = modulestore().get_item(chap_location)
+        self.assertEqual(new_obj.category, 'chapter')
+        self.assertEqual(new_obj.display_name, display_name)
+        self.assertEqual(new_obj.location.org, self.course_location.org)
+        self.assertEqual(new_obj.location.course, self.course_location.course)
+
+        # get the course and ensure it now points to this one
+        course = modulestore().get_item(self.course_location)
+        self.assertIn(chap_location, course.children)
+
+        # use default display name
+        resp = self.client.post(
+            reverse('create_item'),
+            json.dumps({
+                'parent_location': chap_location,
+                'category': 'vertical'
+            }),
+            content_type="application/json"
+        )
+        self.assertEqual(resp.status_code, 200)
+
+        vert_location = self.response_id(resp)
+
+        # create problem w/ boilerplate
+        template_id = 'multiplechoice.yaml'
+        resp = self.client.post(
+            reverse('create_item'),
+            json.dumps({
+                'parent_location': vert_location,
+                'category': 'problem',
+                'boilerplate': template_id
+            }),
+            content_type="application/json"
+        )
+        self.assertEqual(resp.status_code, 200)
+        prob_location = self.response_id(resp)
+        problem = modulestore('draft').get_item(prob_location)
+        # ensure it's draft
+        self.assertTrue(problem.is_draft)
+        # check against the template
+        template = CapaDescriptor.get_template(template_id)
+        self.assertEqual(problem.data, template['data'])
+        self.assertEqual(problem.display_name, template['metadata']['display_name'])
+        self.assertEqual(problem.markdown, template['metadata']['markdown'])
+
+    def test_create_item_negative(self):
+        """
+        Negative tests for create_item
+        """
+        # non-existent boilerplate: creates a default
+        resp = self.client.post(
+            reverse('create_item'),
+            json.dumps(
+                {'parent_location': self.course_location.url(),
+                 'category': 'problem',
+                 'boilerplate': 'nosuchboilerplate.yaml'
+                 }),
+            content_type="application/json"
+        )
+        self.assertEqual(resp.status_code, 200)
+
+class TestEditItem(CourseTestCase):
+    """
+    Test contentstore.views.item.save_item
+    """
+    def response_id(self, response):
+        """
+        Get the id from the response payload
+        :param response:
+        """
+        parsed = json.loads(response.content)
+        return parsed['id']
+
+    def setUp(self):
+        """ Creates the test course structure and a couple problems to 'edit'. """
+        super(TestEditItem, self).setUp()
+        # create a chapter
+        display_name = 'chapter created'
+        resp = self.client.post(
+            reverse('create_item'),
+            json.dumps(
+                {'parent_location': self.course_location.url(),
+                 'display_name': display_name,
+                 'category': 'chapter'
+                 }),
+            content_type="application/json"
+        )
+        chap_location = self.response_id(resp)
+        resp = self.client.post(
+            reverse('create_item'),
+            json.dumps(
+                {'parent_location': chap_location,
+                 'category': 'vertical'
+                }),
+            content_type="application/json"
+        )
+        vert_location = self.response_id(resp)
+        # create problem w/ boilerplate
+        template_id = 'multiplechoice.yaml'
+        resp = self.client.post(
+            reverse('create_item'),
+            json.dumps({'parent_location': vert_location,
+             'category': 'problem',
+             'boilerplate': template_id
+            }),
+            content_type="application/json"
+        )
+        self.problems = [self.response_id(resp)]
+
+    def test_delete_field(self):
+        """
+        Sending null in for a field 'deletes' it
+        """
+        self.client.post(
+            reverse('save_item'),
+            json.dumps({
+                'id': self.problems[0],
+                'metadata': {'rerandomize': 'onreset'}
+            }),
+            content_type="application/json"
+        )
+        problem = modulestore('draft').get_item(self.problems[0])
+        self.assertEqual(problem.rerandomize, 'onreset')
+        self.client.post(
+            reverse('save_item'),
+            json.dumps({
+                'id': self.problems[0],
+                'metadata': {'rerandomize': None}
+            }),
+            content_type="application/json"
+        )
+        problem = modulestore('draft').get_item(self.problems[0])
+        self.assertEqual(problem.rerandomize, 'never')
+
+
+    def test_null_field(self):
+        """
+        Sending null in for a field 'deletes' it
+        """
+        problem = modulestore('draft').get_item(self.problems[0])
+        self.assertIsNotNone(problem.markdown)
+        self.client.post(
+            reverse('save_item'),
+            json.dumps({
+                'id': self.problems[0],
+                'nullout': ['markdown']
+            }),
+            content_type="application/json"
+        )
+        problem = modulestore('draft').get_item(self.problems[0])
+        self.assertIsNo/ne(problem.markdown)

cms/djangoapps/contentstore/tests/utils.py

@@ -54,7 +54,6 @@ class CourseTestCase(ModuleStoreTestCase):
         self.client.login(username=uname, password=password)
 
         self.course = CourseFactory.create(
-            template='i4x://edx/templates/course/Empty',
             org='MITx',
             number='999',
             display_name='Robot Super Course',

cms/djangoapps/contentstore/utils.py

@@ -19,14 +19,14 @@ NOTES_PANEL = {"name": _("My Notes"), "type": "notes"}
 EXTRA_TAB_PANELS = dict([(p['type'], p) for p in [OPEN_ENDED_PANEL, NOTES_PANEL]])
 
 
-def get_modulestore(location):
+def get_modulestore(category_or_location):
     """
     Returns the correct modulestore to use for modifying the specified location
     """
-    if not isinstance(location, Location):
-        location = Location(location)
+    if isinstance(category_or_location, Location):
+        category_or_location = category_or_location.category
 
-    if location.category in DIRECT_ONLY_CATEGORIES:
+    if category_or_location in DIRECT_ONLY_CATEGORIES:
         return modulestore('direct')
     else:
         return modulestore()

cms/djangoapps/contentstore/views/checklist.py

@@ -7,11 +7,11 @@ from django.views.decorators.http import require_http_methods
 from django_future.csrf import ensure_csrf_cookie
 from mitxmako.shortcuts import render_to_response
 
-from xmodule.modulestore import Location
 from xmodule.modulestore.inheritance import own_metadata
 
 from ..utils import get_modulestore, get_url_reverse
 from .access import get_location_and_verify_access
+from xmodule.course_module import CourseDescriptor
 
 __all__ = ['get_checklists', 'update_checklist']
 
@@ -28,13 +28,11 @@ def get_checklists(request, org, course, name):
 
     modulestore = get_modulestore(location)
     course_module = modulestore.get_item(location)
-    new_course_template = Location('i4x', 'edx', 'templates', 'course', 'Empty')
-    template_module = modulestore.get_item(new_course_template)
 
     # If course was created before checklists were introduced, copy them over from the template.
     copied = False
     if not course_module.checklists:
-        course_module.checklists = template_module.checklists
+        course_module.checklists = CourseDescriptor.checklists.default
         copied = True
 
     checklists, modified = expand_checklist_action_urls(course_module)

cms/djangoapps/contentstore/views/component.py

@@ -26,6 +26,8 @@ from models.settings.course_grading import CourseGradingModel
 
 from .requests import _xmodule_recurse
 from .access import has_access
+from xmodule.x_module import XModuleDescriptor
+from xblock.plugin import PluginMissingError
 
 __all__ = ['OPEN_ENDED_COMPONENT_TYPES',
            'ADVANCED_COMPONENT_POLICY_KEY',
@@ -101,7 +103,7 @@ def edit_subsection(request, location):
     return render_to_response('edit_subsection.html',
                               {'subsection': item,
                                'context_course': course,
-                               'create_new_unit_template': Location('i4x', 'edx', 'templates', 'vertical', 'Empty'),
+                               'new_unit_category': 'vertical',
                                'lms_link': lms_link,
                                'preview_link': preview_link,
                                'course_graders': json.dumps(CourseGradingModel.fetch(course.location).graders),
@@ -134,10 +136,27 @@ def edit_unit(request, location):
         item = modulestore().get_item(location, depth=1)
     except ItemNotFoundError:
         return HttpResponseBadRequest()
-
     lms_link = get_lms_link_for_item(item.location, course_id=course.location.course_id)
 
     component_templates = defaultdict(list)
+    for category in COMPONENT_TYPES:
+        component_class = XModuleDescriptor.load_class(category)
+        # add the default template
+        has_markdown = hasattr(component_class, 'markdown') and component_class.markdown is not None
+        component_templates[category].append((
+            component_class.display_name.default or 'Blank',
+            category,
+            has_markdown,
+            None  # no boilerplate for overrides
+        ))
+        # add boilerplates
+        for template in component_class.templates():
+            component_templates[category].append((
+                template['metadata'].get('display_name'),
+                category,
+                template['metadata'].get('markdown') is not None,
+                template.get('template_id')
+            ))
 
     # Check if there are any advanced modules specified in the course policy. These modules
     # should be specified as a list of strings, where the strings are the names of the modules
@@ -145,29 +164,29 @@ def edit_unit(request, location):
     course_advanced_keys = course.advanced_modules
 
     # Set component types according to course policy file
-    component_types = list(COMPONENT_TYPES)
     if isinstance(course_advanced_keys, list):
-        course_advanced_keys = [c for c in course_advanced_keys if c in ADVANCED_COMPONENT_TYPES]
-        if len(course_advanced_keys) > 0:
-            component_types.append(ADVANCED_COMPONENT_CATEGORY)
+        for category in course_advanced_keys:
+            if category in ADVANCED_COMPONENT_TYPES:
+                # Do I need to allow for boilerplates or just defaults on the class? i.e., can an advanced
+                # have more than one entry in the menu? one for default and others for prefilled boilerplates?
+                try:
+                    component_class = XModuleDescriptor.load_class(category)
+
+                    component_templates['advanced'].append((
+                        component_class.display_name.default or category,
+                        category,
+                        hasattr(component_class, 'markdown') and component_class.markdown is not None,
+                        None  # don't override default data
+                        ))
+                except PluginMissingError:
+                    # dhm: I got this once but it can happen any time the course author configures
+                    # an advanced component which does not exist on the server. This code here merely
+                    # prevents any authors from trying to instantiate the non-existent component type
+                    # by not showing it in the menu
+                    pass
     else:
         log.error("Improper format for course advanced keys! {0}".format(course_advanced_keys))
 
-    templates = modulestore().get_items(Location('i4x', 'edx', 'templates'))
-    for template in templates:
-        category = template.location.category
-
-        if category in course_advanced_keys:
-            category = ADVANCED_COMPONENT_CATEGORY
-
-        if category in component_types:
-            # This is a hack to create categories for different xmodules
-            component_templates[category].append((
-                template.display_name_with_default,
-                template.location.url(),
-                hasattr(template, 'markdown') and template.markdown is not None
-            ))
-
     components = [
         component.location.url()
         for component
@@ -219,7 +238,7 @@ def edit_unit(request, location):
         'subsection': containing_subsection,
         'release_date': get_default_time_display(containing_subsection.lms.start) if containing_subsection.lms.start is not None else None,
         'section': containing_section,
-        'create_new_unit_template': Location('i4x', 'edx', 'templates', 'vertical', 'Empty'),
+        'new_unit_category': 'vertical',
         'unit_state': unit_state,
         'published_date': get_default_time_display(item.cms.published_date) if item.cms.published_date is not None else None
     })
@@ -253,7 +272,7 @@ def create_draft(request):
 
     # This clones the existing item location to a draft location (the draft is implicit,
     # because modulestore is a Draft modulestore)
-    modulestore().clone_item(location, location)
+    modulestore().convert_to_draft(location)
 
     return HttpResponse()
 

cms/djangoapps/contentstore/views/course.py

@@ -82,10 +82,11 @@ def course_index(request, org, course, name):
         'sections': sections,
         'course_graders': json.dumps(CourseGradingModel.fetch(course.location).graders),
         'parent_location': course.location,
-        'new_section_template': Location('i4x', 'edx', 'templates', 'chapter', 'Empty'),
-        'new_subsection_template': Location('i4x', 'edx', 'templates', 'sequential', 'Empty'),  # for now they are the same, but the could be different at some point...
+        'new_section_category': 'chapter',
+        'new_subsection_category': 'sequential',
         'upload_asset_callback_url': upload_asset_callback_url,
-        'create_new_unit_template': Location('i4x', 'edx', 'templates', 'vertical', 'Empty')
+        'new_unit_category': 'vertical',
+        'category': 'vertical'
     })
 
 
@@ -98,12 +99,6 @@ def create_new_course(request):
     if not is_user_in_creator_group(request.user):
         raise PermissionDenied()
 
-    # This logic is repeated in xmodule/modulestore/tests/factories.py
-    # so if you change anything here, you need to also change it there.
-    # TODO: write a test that creates two courses, one with the factory and
-    # the other with this method, then compare them to make sure they are
-    # equivalent.
-    template = Location(request.POST['template'])
     org = request.POST.get('org')
     number = request.POST.get('number')
     display_name = request.POST.get('display_name')
@@ -121,29 +116,26 @@ def create_new_course(request):
         existing_course = modulestore('direct').get_item(dest_location)
     except ItemNotFoundError:
         pass
-
     if existing_course is not None:
         return JsonResponse({'ErrMsg': 'There is already a course defined with this name.'})
 
     course_search_location = ['i4x', dest_location.org, dest_location.course, 'course', None]
     courses = modulestore().get_items(course_search_location)
-
     if len(courses) > 0:
         return JsonResponse({'ErrMsg': 'There is already a course defined with the same organization and course number.'})
 
-    new_course = modulestore('direct').clone_item(template, dest_location)
+    # instantiate the CourseDescriptor and then persist it
+    # note: no system to pass
+    if display_name is None:
+        metadata = {}
+    else:
+        metadata = {'display_name': display_name}
+    modulestore('direct').create_and_save_xmodule(dest_location, metadata=metadata)
+    new_course = modulestore('direct').get_item(dest_location)
 
     # clone a default 'about' module as well
-
-    about_template_location = Location(['i4x', 'edx', 'templates', 'about', 'overview'])
     dest_about_location = dest_location._replace(category='about', name='overview')
-    modulestore('direct').clone_item(about_template_location, dest_about_location)
-
-    if display_name is not None:
-        new_course.display_name = display_name
-
-    # set a default start date to now
-    new_course.start = datetime.datetime.now(UTC())
+    modulestore('direct').create_and_save_xmodule(dest_about_location, system=new_course.system)
 
     initialize_course_tabs(new_course)
 

cms/djangoapps/contentstore/views/item.py

@@ -13,16 +13,26 @@ from util.json_request import expect_json
 from ..utils import get_modulestore
 from .access import has_access
 from .requests import _xmodule_recurse
+from xmodule.x_module import XModuleDescriptor
 
-__all__ = ['save_item', 'clone_item', 'delete_item']
+__all__ = ['save_item', 'create_item', 'delete_item']
 
 # cdodge: these are categories which should not be parented, they are detached from the hierarchy
 DETACHED_CATEGORIES = ['about', 'static_tab', 'course_info']
 
-
 @login_required
 @expect_json
 def save_item(request):
+    """
+    Will carry a json payload with these possible fields
+    :id (required): the id
+    :data (optional): the new value for the data
+    :metadata (optional): new values for the metadata fields.
+        Any whose values are None will be deleted not set to None! Absent ones will be left alone
+    :nullout (optional): which metadata fields to set to None
+    """
+    # The nullout is a bit of a temporary copout until we can make module_edit.coffee and the metadata editors a
+    # little smarter and able to pass something more akin to {unset: [field, field]}
     item_location = request.POST['id']
 
     # check permissions for this user within this course
@@ -42,30 +52,25 @@ def save_item(request):
         children = request.POST['children']
         store.update_children(item_location, children)
 
-    # cdodge: also commit any metadata which might have been passed along in the
-    # POST from the client, if it is there
-    # NOTE, that the postback is not the complete metadata, as there's system metadata which is
-    # not presented to the end-user for editing. So let's fetch the original and
-    # 'apply' the submitted metadata, so we don't end up deleting system metadata
-    if request.POST.get('metadata') is not None:
-        posted_metadata = request.POST['metadata']
-        # fetch original
+    # cdodge: also commit any metadata which might have been passed along
+    if request.POST.get('nullout') is not None or request.POST.get('metadata') is not None:
+        # the postback is not the complete metadata, as there's system metadata which is
+        # not presented to the end-user for editing. So let's fetch the original and
+        # 'apply' the submitted metadata, so we don't end up deleting system metadata
         existing_item = modulestore().get_item(item_location)
+        for metadata_key in request.POST.get('nullout', []):
+            setattr(existing_item, metadata_key, None)
 
         # update existing metadata with submitted metadata (which can be partial)
-        # IMPORTANT NOTE: if the client passed pack 'null' (None) for a piece of metadata that means 'remove it'
-        for metadata_key, value in posted_metadata.items():
-
-            if posted_metadata[metadata_key] is None:
-                # remove both from passed in collection as well as the collection read in from the modulestore
-                if metadata_key in existing_item._model_data:
-                    del existing_item._model_data[metadata_key]
-                del posted_metadata[metadata_key]
-            else:
-                existing_item._model_data[metadata_key] = value
+        # IMPORTANT NOTE: if the client passed 'null' (None) for a piece of metadata that means 'remove it'. If
+        # the intent is to make it None, use the nullout field
+        for metadata_key, value in request.POST.get('metadata', {}).items():
 
+            if value is None:
+                delattr(existing_item, metadata_key)
+            else:
+                setattr(existing_item, metadata_key, value)
         # commit to datastore
-        # TODO (cpennington): This really shouldn't have to do this much reaching in to get the metadata
         store.update_metadata(item_location, own_metadata(existing_item))
 
     return HttpResponse()
@@ -73,28 +78,38 @@ def save_item(request):
 
 @login_required
 @expect_json
-def clone_item(request):
+def create_item(request):
     parent_location = Location(request.POST['parent_location'])
-    template = Location(request.POST['template'])
+    category = request.POST['category']
 
     display_name = request.POST.get('display_name')
 
     if not has_access(request.user, parent_location):
         raise PermissionDenied()
 
-    parent = get_modulestore(template).get_item(parent_location)
-    dest_location = parent_location._replace(category=template.category, name=uuid4().hex)
-
-    new_item = get_modulestore(template).clone_item(template, dest_location)
+    parent = get_modulestore(category).get_item(parent_location)
+    dest_location = parent_location.replace(category=category, name=uuid4().hex)
+
+    # get the metadata, display_name, and definition from the request
+    metadata = {}
+    data = None
+    template_id = request.POST.get('boilerplate')
+    if template_id is not None:
+        clz = XModuleDescriptor.load_class(category)
+        if clz is not None:
+            template = clz.get_template(template_id)
+            if template is not None:
+                metadata = template.get('metadata', {})
+                data = template.get('data')
 
-    # replace the display name with an optional parameter passed in from the caller
     if display_name is not None:
-        new_item.display_name = display_name
+        metadata['display_name'] = display_name
 
-    get_modulestore(template).update_metadata(new_item.location.url(), own_metadata(new_item))
+    get_modulestore(category).create_and_save_xmodule(dest_location, definition_data=data,
+        metadata=metadata, system=parent.system)
 
-    if new_item.location.category not in DETACHED_CATEGORIES:
-        get_modulestore(parent.location).update_children(parent_location, parent.children + [new_item.location.url()])
+    if category not in DETACHED_CATEGORIES:
+        get_modulestore(parent.location).update_children(parent_location, parent.children + [dest_location.url()])
 
     return HttpResponse(json.dumps({'id': dest_location.url()}))
 

cms/djangoapps/contentstore/views/user.py

@@ -27,6 +27,7 @@ def index(request):
     # filter out courses that we don't have access too
     def course_filter(course):
         return (has_access(request.user, course.location)
+                # TODO remove this condition when templates purged from db
                 and course.location.course != 'templates'
                 and course.location.org != ''
                 and course.location.course != ''
@@ -34,7 +35,6 @@ def index(request):
     courses = filter(course_filter, courses)
 
     return render_to_response('index.html', {
-        'new_course_template': Location('i4x', 'edx', 'templates', 'course', 'Empty'),
         'courses': [(course.display_name,
                     get_url_reverse('CourseOutline', course),
                     get_lms_link_for_item(course.location, course_id=course.location.course_id))

cms/djangoapps/models/settings/course_grading.py

@@ -9,7 +9,7 @@ class CourseGradingModel(object):
     """
     def __init__(self, course_descriptor):
         self.course_location = course_descriptor.location
-        self.graders = [CourseGradingModel.jsonize_grader(i, grader) for i, grader in enumerate(course_descriptor.raw_grader)]   # weights transformed to ints [0..100]
+        self.graders = [CourseGradingModel.jsonize_grader(i, grader) for i, grader in enumerate(course_descriptor.raw_grader)]  # weights transformed to ints [0..100]
         self.grade_cutoffs = course_descriptor.grade_cutoffs
         self.grace_period = CourseGradingModel.convert_set_grace_period(course_descriptor)
 
@@ -81,7 +81,7 @@ class CourseGradingModel(object):
         Decode the json into CourseGradingModel and save any changes. Returns the modified model.
         Probably not the usual path for updates as it's too coarse grained.
         """
-        course_location = jsondict['course_location']
+        course_location = Location(jsondict['course_location'])
         descriptor = get_modulestore(course_location).get_item(course_location)
 
         graders_parsed = [CourseGradingModel.parse_grader(jsonele) for jsonele in jsondict['graders']]
@@ -89,7 +89,7 @@ class CourseGradingModel(object):
         descriptor.raw_grader = graders_parsed
         descriptor.grade_cutoffs = jsondict['grade_cutoffs']
 
-        get_modulestore(course_location).update_item(course_location, descriptor._model_data._kvs._data)
+        get_modulestore(course_location).update_item(course_location, descriptor.xblock_kvs._data)
         CourseGradingModel.update_grace_period_from_json(course_location, jsondict['grace_period'])
 
         return CourseGradingModel.fetch(course_location)
@@ -209,7 +209,7 @@ class CourseGradingModel(object):
         descriptor = get_modulestore(location).get_item(location)
         return {"graderType": descriptor.lms.format if descriptor.lms.format is not None else 'Not Graded',
                 "location": location,
-                "id": 99   # just an arbitrary value to
+                "id": 99  # just an arbitrary value to
                 }
 
     @staticmethod
@@ -232,7 +232,7 @@ class CourseGradingModel(object):
         # 5 hours 59 minutes 59 seconds => converted to iso format
         rawgrace = descriptor.lms.graceperiod
         if rawgrace:
-            hours_from_days = rawgrace.days*24
+            hours_from_days = rawgrace.days * 24
             seconds = rawgrace.seconds
             hours_from_seconds = int(seconds / 3600)
             hours = hours_from_days + hours_from_seconds

cms/static/coffee/src/views/module_edit.coffee

@@ -56,14 +56,15 @@ class CMS.Views.ModuleEdit extends Backbone.View
   changedMetadata: ->
     return _.extend(@metadataEditor.getModifiedMetadataValues(), @customMetadata())
 
-  cloneTemplate: (parent, template) ->
-    $.post("/clone_item", {
-      parent_location: parent
-      template: template
-    }, (data) => 
-      @model.set(id: data.id)
-      @$el.data('id', data.id)
-      @render()
+  createItem: (parent, payload) ->
+    payload.parent_location = parent
+    $.post(
+        "/create_item"
+        payload 
+        (data) => 
+            @model.set(id: data.id)
+            @$el.data('id', data.id)
+            @render()
     )
 
   render: ->

cms/static/coffee/src/views/tabs.coffee

@@ -55,9 +55,9 @@ class CMS.Views.TabsEdit extends Backbone.View
       editor.$el.removeClass('new')
     , 500)
 
-    editor.cloneTemplate(
+    editor.createItem(
       @model.get('id'),
-      'i4x://edx/templates/static_tab/Empty'
+      {category: 'static_tab'}
     )
 
     analytics.track "Added Static Page",

cms/static/coffee/src/views/unit.coffee

@@ -89,9 +89,9 @@ class CMS.Views.UnitEdit extends Backbone.View
 
     @$newComponentItem.before(editor.$el)
 
-    editor.cloneTemplate(
+    editor.createItem(
       @$el.data('id'),
-      $(event.currentTarget).data('location')
+      $(event.currentTarget).data()
     )
 
     analytics.track "Added a Component",

cms/static/js/base.js

@@ -338,7 +338,7 @@ function createNewUnit(e) {
     e.preventDefault();
 
     var parent = $(this).data('parent');
-    var template = $(this).data('template');
+    var category = $(this).data('category');
 
     analytics.track('Created a Unit', {
         'course': course_location_analytics,
@@ -346,9 +346,9 @@ function createNewUnit(e) {
     });
 
 
-    $.post('/clone_item', {
+    $.post('/create_item', {
         'parent_location': parent,
-        'template': template,
+        'category': category,
         'display_name': 'New Unit'
     },
 
@@ -551,7 +551,7 @@ function saveNewSection(e) {
 
     var $saveButton = $(this).find('.new-section-name-save');
     var parent = $saveButton.data('parent');
-    var template = $saveButton.data('template');
+    var category = $saveButton.data('category');
     var display_name = $(this).find('.new-section-name').val();
 
     analytics.track('Created a Section', {
@@ -559,9 +559,9 @@ function saveNewSection(e) {
         'display_name': display_name
     });
 
-    $.post('/clone_item', {
+    $.post('/create_item', {
         'parent_location': parent,
-        'template': template,
+        'category': category,
         'display_name': display_name,
     },
 
@@ -595,7 +595,6 @@ function saveNewCourse(e) {
     e.preventDefault();
 
     var $newCourse = $(this).closest('.new-course');
-    var template = $(this).find('.new-course-save').data('template');
     var org = $newCourse.find('.new-course-org').val();
     var number = $newCourse.find('.new-course-number').val();
     var display_name = $newCourse.find('.new-course-name').val();
@@ -612,7 +611,6 @@ function saveNewCourse(e) {
     });
 
     $.post('/create_new_course', {
-        'template': template,
         'org': org,
         'number': number,
         'display_name': display_name
@@ -646,7 +644,7 @@ function addNewSubsection(e) {
     var parent = $(this).parents("section.branch").data("id");
 
     $saveButton.data('parent', parent);
-    $saveButton.data('template', $(this).data('template'));
+    $saveButton.data('category', $(this).data('category'));
 
     $newSubsection.find('.new-subsection-form').bind('submit', saveNewSubsection);
     $cancelButton.bind('click', cancelNewSubsection);
@@ -659,7 +657,7 @@ function saveNewSubsection(e) {
     e.preventDefault();
 
     var parent = $(this).find('.new-subsection-name-save').data('parent');
-    var template = $(this).find('.new-subsection-name-save').data('template');
+    var category = $(this).find('.new-subsection-name-save').data('category');
     var display_name = $(this).find('.new-subsection-name-input').val();
 
     analytics.track('Created a Subsection', {
@@ -668,9 +666,9 @@ function saveNewSubsection(e) {
     });
 
 
-    $.post('/clone_item', {
+    $.post('/create_item', {
         'parent_location': parent,
-        'template': template,
+        'category': category,
         'display_name': display_name
     },
 

cms/templates/index.html

@@ -25,8 +25,8 @@
             </div>
           </div>
           <div class="row">
-	    <input type="submit" value="${_('Save')}" class="new-course-save" data-template="${new_course_template}" />
-	    <input type="button" value="${_('Cancel')}" class="new-course-cancel" />
+            <input type="submit" value="${_('Save')}" class="new-course-save"/>
+            <input type="button" value="${_('Cancel')}" class="new-course-cancel" />
           </div>
         </form>
       </div>

cms/templates/overview.html

-<%! from django.utils.translation import ugettext as _ %>
+/<%! from django.utils.translation import ugettext as _ %>
 <%inherit file="base.html" />
 <%!
    import logging
@@ -66,7 +66,8 @@
           <h3 class="section-name">
           <form class="section-name-form">
             <input type="text" value="${_('New Section Name')}" class="new-section-name" />
-            <input type="submit" class="new-section-name-save" data-parent="${parent_location}" data-template="${new_section_template}" value="${_('Save')}" />
+            <input type="submit" class="new-section-name-save" data-parent="${parent_location}"
+                   data-category="${new_section_category}" value="${_('Save')}" />
             <input type="button" class="new-section-name-cancel" value="${_('Cancel')}" /></h3>
           </form>
         </div>
@@ -83,8 +84,9 @@
           <span class="section-name-span">Click here to set the section name</span>
           <form class="section-name-form">
             <input type="text" value="${_('New Section Name')}" class="new-section-name" />
-            <input type="submit" class="new-section-name-save" data-parent="${parent_location}" data-template="${new_section_template}" value="${_('Save')}" />
-            <input type="button" class="new-section-name-cancel" value="${_('Cancel')}" /></h3>
+            <input type="submit" class="new-section-name-save" data-parent="${parent_location}"
+                   data-category="${new_section_category}" value="${_('Save')}" />
+            <input type="button" class="new-section-name-cancel" value="$(_('Cancel')}" /></h3>
           </form>
         </div>
         <div class="item-actions">
@@ -181,7 +183,7 @@
           </header>
           <div class="subsection-list">
             <div class="list-header">
-              <a href="#" class="new-subsection-item" data-template="${new_subsection_template}">
+              <a href="#" class="new-subsection-item" data-category="${new_subsection_category}">
                 <span class="new-folder-icon"></span>${_("New Subsection")}
               </a>
             </div>

cms/templates/unit.html

@@ -59,8 +59,8 @@
                   % if type == 'advanced' or len(templates) > 1:
                     <a href="#" class="multiple-templates" data-type="${type}">
                   % else:
-                    % for __, location, __ in templates:
-                    <a href="#" class="single-template" data-type="${type}" data-location="${location}">
+                    % for __, category, __, __ in templates:
+                    <a href="#" class="single-template" data-type="${type}" data-category="${category}">
                     % endfor
                   % endif
                       <span class="large-template-icon large-${type}-icon"></span>
@@ -74,49 +74,60 @@
                   % if len(templates) > 1 or type == 'advanced':
                 <div class="new-component-templates new-component-${type}">
                     % if type == "problem":
-                      <div class="tab-group tabs">
-                              <ul class="problem-type-tabs nav-tabs">
-                                <li class="current">
-                                  <a class="link-tab" href="#tab1">${_("Common Problem Types")}</a>
-                                </li>
-                                <li>
-                                  <a class="link-tab" href="#tab2">${_("Advanced")}</a>
-                                </li>
-                              </ul>
-                      % endif
-                          <div class="tab current" id="tab1">
-                            <ul class="new-component-template">
-                                % for name, location, has_markdown in templates:
-                                  % if has_markdown or type != "problem":
-                                    <li class="editor-md">
-                                      <a href="#" id="${location}" data-location="${location}">
-                                        <span class="name"> ${name}</span>
-                                      </a>
-                                    </li>
-                                  % endif
-
-                                %endfor
+                    <div class="tab-group tabs">
+                            <ul class="problem-type-tabs nav-tabs">
+                              <li class="current">
+                                <a class="link-tab" href="#tab1">${_("Common Problem Types")}</a>
+                              </li>
+                              <li>
+                                <a class="link-tab" href="#tab2">${_("Advanced")}</a>
+                              </li>
                             </ul>
-                          </div>
-                      % if type == "problem":
-                              <div class="tab" id="tab2">
-                                <ul class="new-component-template">
-                                    % for name, location, has_markdown in templates:
-                                      % if not has_markdown:
-                                        <li class="editor-manual">
-                                          <a href="#" id="${location}" data-location="${location}">
-                                            <span class="name"> ${name}</span>
-                                          </a>
-                                        </li>
-                                      % endif
-                                    % endfor
-                                </ul>
-                              </div>
-                      </div>
-                      % endif
-                      <a href="#" class="cancel-button">${_("Cancel")}</a>
-                  </div>
-                  % endif
+                    % endif
+                    <div class="tab current" id="tab1">
+                      <ul class="new-component-template">
+                          % for name, category, has_markdown, boilerplate_name in sorted(templates):
+                            % if has_markdown or type != "problem":
+                                % if boilerplate_name is None:
+                                  <li class="editor-md empty">
+                                    <a href="#" data-category="${category}">
+                                      <span class="name">${name}</span>
+                                    </a>
+                                  </li>
+
+                                % else:
+                                  <li class="editor-md">
+                                    <a href="#" data-category="${category}"
+                                       data-boilerplate="${boilerplate_name}">
+                                      <span class="name">${name}</span>
+                                    </a>
+                                  </li>
+                                % endif
+                            % endif
+
+                          %endfor
+                      </ul>
+                    </div>
+                    % if type == "problem":
+                        <div class="tab" id="tab2">
+                          <ul class="new-component-template">
+                              % for name, category, has_markdown, boilerplate_name in sorted(templates):
+                                % if not has_markdown:
+                                  <li class="editor-manual">
+                                    <a href="#" data-category="${category}"
+                                       data-boilerplate="${boilerplate_name}">
+                                      <span class="name">${name}</span>
+                                    </a>
+                                  </li>
+                                % endif
+                              % endfor
+                          </ul>
+                        </div>
+                    </div>
+                    % endif
+                    <a href="#" class="cancel-button">Cancel</a>
+                </div>
+                % endif
                 % endfor
             </li>
           </ol>

cms/templates/widgets/units.html

@@ -34,7 +34,7 @@ This def will enumerate through a passed in subsection and list all of the units
   </li>
   % endfor
   <li>
-    <a href="#" class="new-unit-item" data-template="${create_new_unit_template}" data-parent="${subsection.location}">
+    <a href="#" class="new-unit-item" data-category="${new_unit_category}" data-parent="${subsection.location}">
       <span class="new-unit-icon"></span>New Unit
     </a>
   </li>

cms/urls.py

@@ -17,7 +17,7 @@ urlpatterns = ('',  # nopep8
     url(r'^preview_component/(?P<location>.*?)$', 'contentstore.views.preview_component', name='preview_component'),
     url(r'^save_item$', 'contentstore.views.save_item', name='save_item'),
     url(r'^delete_item$', 'contentstore.views.delete_item', name='delete_item'),
-    url(r'^clone_item$', 'contentstore.views.clone_item', name='clone_item'),
+    url(r'^create_item$', 'contentstore.views.create_item', name='create_item'),
     url(r'^create_draft$', 'contentstore.views.create_draft', name='create_draft'),
     url(r'^publish_draft$', 'contentstore.views.publish_draft', name='publish_draft'),
     url(r'^unpublish_unit$', 'contentstore.views.unpublish_unit', name='unpublish_unit'),

common/djangoapps/terrain/course_helpers.py

@@ -12,7 +12,6 @@ from django.contrib.sessions.middleware import SessionMiddleware
 from student.models import CourseEnrollment
 from xmodule.modulestore.django import modulestore
 from xmodule.contentstore.django import contentstore
-from xmodule.templates import update_templates
 from urllib import quote_plus
 
 
@@ -84,5 +83,4 @@ def clear_courses():
     # from the bash shell to drop it:
     # $ mongo test_xmodule --eval "db.dropDatabase()"
     modulestore().collection.drop()
-    update_templates(modulestore('direct'))
     contentstore().fs_files.drop()

common/djangoapps/tests.py

@@ -23,15 +23,15 @@ class TestXmoduleModfiers(ModuleStoreTestCase):
             number='313', display_name='histogram test')
         section = ItemFactory.create(
             parent_location=course.location, display_name='chapter hist',
-            template='i4x://edx/templates/chapter/Empty')
+            category='chapter')
         problem = ItemFactory.create(
             parent_location=section.location, display_name='problem hist 1',
-            template='i4x://edx/templates/problem/Blank_Common_Problem')
+            category='problem')
         problem.has_score = False  # don't trip trying to retrieve db data
 
         late_problem = ItemFactory.create(
             parent_location=section.location, display_name='problem hist 2',
-            template='i4x://edx/templates/problem/Blank_Common_Problem')
+            category='problem')
         late_problem.lms.start = datetime.datetime.now(UTC) + datetime.timedelta(days=32)
         late_problem.has_score = False
 

common/lib/xmodule/xmodule/abtest_module.py

@@ -80,8 +80,6 @@ class ABTestModule(ABTestFields, XModule):
 class ABTestDescriptor(ABTestFields, RawDescriptor, XmlDescriptor):
     module_class = ABTestModule
 
-    template_dir_name = "abtest"
-
     @classmethod
     def definition_from_xml(cls, xml_object, system):
         """

common/lib/xmodule/xmodule/annotatable_module.py

@@ -150,5 +150,4 @@ class AnnotatableModule(AnnotatableFields, XModule):
 
 class AnnotatableDescriptor(AnnotatableFields, RawDescriptor):
     module_class = AnnotatableModule
-    template_dir_name = "annotatable"
     mako_template = "widgets/raw-edit.html"

common/lib/xmodule/xmodule/combined_open_ended_module.py

@@ -290,7 +290,6 @@ class CombinedOpenEndedDescriptor(CombinedOpenEndedFields, RawDescriptor):
 
     has_score = True
     always_recalculate_grades = True
-    template_dir_name = "combinedopenended"
 
     #Specify whether or not to pass in S3 interface
     needs_s3_interface = True

common/lib/xmodule/xmodule/course_module.py

@@ -366,8 +366,6 @@ class CourseFields(object):
 class CourseDescriptor(CourseFields, SequenceDescriptor):
     module_class = SequenceModule
 
-    template_dir_name = 'course'
-
     def __init__(self, *args, **kwargs):
         """
         Expects the same arguments as XModuleDescriptor.__init__

common/lib/xmodule/xmodule/error_module.py

@@ -96,6 +96,7 @@ class ErrorDescriptor(ErrorFields, JSONEditingDescriptor):
             'contents': contents,
             'display_name': 'Error: ' + location.name,
             'location': location,
+            'category': 'error'
         }
         return cls(
             system,
@@ -109,12 +110,12 @@ class ErrorDescriptor(ErrorFields, JSONEditingDescriptor):
         }
 
     @classmethod
-    def from_json(cls, json_data, system, error_msg='Error not available'):
+    def from_json(cls, json_data, system, location, error_msg='Error not available'):
         return cls._construct(
             system,
-            json.dumps(json_data, indent=4),
+            json.dumps(json_data, skipkeys=False, indent=4),
             error_msg,
-            location=Location(json_data['location']),
+            location=location
         )
 
     @classmethod

common/lib/xmodule/xmodule/foldit_module.py

@@ -184,7 +184,6 @@ class FolditDescriptor(FolditFields, XmlDescriptor, EditingDescriptor):
     filename_extension = "xml"
 
     has_score = True
-    template_dir_name = "foldit"
 
     js = {'coffee': [resource_string(__name__, 'js/src/html/edit.coffee')]}
     js_module_name = "HTMLEditingDescriptor"

common/lib/xmodule/xmodule/gst_module.py

@@ -141,7 +141,6 @@ class GraphicalSliderToolModule(GraphicalSliderToolFields, XModule):
 
 class GraphicalSliderToolDescriptor(GraphicalSliderToolFields, MakoModuleDescriptor, XmlDescriptor):
     module_class = GraphicalSliderToolModule
-    template_dir_name = 'graphical_slider_tool'
 
     @classmethod
     def definition_from_xml(cls, xml_object, system):

common/lib/xmodule/xmodule/html_module.py

@@ -234,7 +234,7 @@ class StaticTabDescriptor(StaticTabFields, HtmlDescriptor):
     These pieces of course content are treated as HtmlModules but we need to overload where the templates are located
     in order to be able to create new ones
     """
-    template_dir_name = "statictab"
+    template_dir_name = None
     module_class = StaticTabModule
 
 
@@ -261,5 +261,5 @@ class CourseInfoDescriptor(CourseInfoFields, HtmlDescriptor):
     These pieces of course content are treated as HtmlModules but we need to overload where the templates are located
     in order to be able to create new ones
     """
-    template_dir_name = "courseinfo"
+    template_dir_name = None
     module_class = CourseInfoModule

common/lib/xmodule/xmodule/js/spec/combinedopenended/edit_spec.coffee

@@ -11,13 +11,13 @@ describe 'OpenEndedMarkdownEditingDescriptor', ->
       @descriptor = new OpenEndedMarkdownEditingDescriptor($('.combinedopenended-editor'))
       @descriptor.createXMLEditor('replace with markdown')
       saveResult = @descriptor.save()
-      expect(saveResult.metadata.markdown).toEqual(null)
+      expect(saveResult.nullout).toEqual(['markdown'])
       expect(saveResult.data).toEqual('replace with markdown')
     it 'saves xml from the xml editor', ->
       loadFixtures 'combinedopenended-without-markdown.html'
       @descriptor = new OpenEndedMarkdownEditingDescriptor($('.combinedopenended-editor'))
       saveResult = @descriptor.save()
-      expect(saveResult.metadata.markdown).toEqual(null)
+      expect(saveResult.nullout).toEqual(['markdown'])
       expect(saveResult.data).toEqual('xml only')
 
   describe 'insertPrompt', ->

common/lib/xmodule/xmodule/js/spec/problem/edit_spec.coffee

@@ -11,13 +11,13 @@ describe 'MarkdownEditingDescriptor', ->
       @descriptor = new MarkdownEditingDescriptor($('.problem-editor'))
       @descriptor.createXMLEditor('replace with markdown')
       saveResult = @descriptor.save()
-      expect(saveResult.metadata.markdown).toEqual(null)
+      expect(saveResult.nullout).toEqual(['markdown'])
       expect(saveResult.data).toEqual('replace with markdown')
     it 'saves xml from the xml editor', ->
       loadFixtures 'problem-without-markdown.html'
       @descriptor = new MarkdownEditingDescriptor($('.problem-editor'))
       saveResult = @descriptor.save()
-      expect(saveResult.metadata.markdown).toEqual(null)
+      expect(saveResult.nullout).toEqual(['markdown'])
       expect(saveResult.data).toEqual('xml only')
 
   describe 'insertMultipleChoice', ->

common/lib/xmodule/xmodule/js/src/combinedopenended/edit.coffee

@@ -153,8 +153,7 @@ Write a persuasive essay to a newspaper reflecting your vies on censorship in li
     else
       {
       data: @xml_editor.getValue()
-      metadata:
-        markdown: null
+      nullout: ['markdown']
       }
 
   @insertRubric: (selectedText) ->

common/lib/xmodule/xmodule/js/src/problem/edit.coffee

@@ -123,9 +123,8 @@ class @MarkdownEditingDescriptor extends XModule.Descriptor
         }
     else
        {
-            data: @xml_editor.getValue()
-            metadata:
-               markdown: null
+          data: @xml_editor.getValue()
+          nullout: ['markdown']
        }
 
   @insertMultipleChoice: (selectedText) ->

common/lib/xmodule/xmodule/modulestore/__init__.py

@@ -310,14 +310,7 @@ class ModuleStore(object):
         """
         raise NotImplementedError
 
-    def clone_item(self, source, location):
-        """
-        Clone a new item that is a copy of the item at the location `source`
-        and writes it to `location`
-        """
-        raise NotImplementedError
-
-    def update_item(self, location, data):
+    def update_item(self, location, data, allow_not_found=False):
         """
         Set the data in the item specified by the location to
         data

common/lib/xmodule/xmodule/modulestore/mongo/draft.py

@@ -8,11 +8,12 @@ and otherwise returns i4x://org/course/cat/name).
 
 from datetime import datetime
 
+from xmodule.exceptions import InvalidVersionError
 from xmodule.modulestore import Location, namedtuple_to_son
-from xmodule.modulestore.exceptions import ItemNotFoundError
+from xmodule.modulestore.exceptions import ItemNotFoundError, DuplicateItemError
 from xmodule.modulestore.inheritance import own_metadata
-from xmodule.exceptions import InvalidVersionError
-from xmodule.modulestore.mongo.base import MongoModuleStore
+from xmodule.modulestore.mongo.base import location_to_query, get_course_id_no_run, MongoModuleStore
+import pymongo
 from pytz import UTC
 
 DRAFT = 'draft'
@@ -92,6 +93,21 @@ class DraftModuleStore(MongoModuleStore):
         except ItemNotFoundError:
             return wrap_draft(super(DraftModuleStore, self).get_instance(course_id, location, depth=depth))
 
+    def create_xmodule(self, location, definition_data=None, metadata=None, system=None):
+        """
+        Create the new xmodule but don't save it. Returns the new module with a draft locator
+
+        :param location: a Location--must have a category
+        :param definition_data: can be empty. The initial definition_data for the kvs
+        :param metadata: can be empty, the initial metadata for the kvs
+        :param system: if you already have an xmodule from the course, the xmodule.system value
+        """
+        draft_loc = as_draft(location)
+        if draft_loc.category in DIRECT_ONLY_CATEGORIES:
+            raise InvalidVersionError(location)
+        return super(DraftModuleStore, self).create_xmodule(draft_loc, definition_data, metadata, system)
+
+
     def get_items(self, location, course_id=None, depth=0):
         """
         Returns a list of XModuleDescriptor instances for the items
@@ -119,14 +135,26 @@ class DraftModuleStore(MongoModuleStore):
         ]
         return [wrap_draft(item) for item in draft_items + non_draft_items]
 
-    def clone_item(self, source, location):
+    def convert_to_draft(self, source_location):
         """
-        Clone a new item that is a copy of the item at the location `source`
-        and writes it to `location`
+        Create a copy of the source and mark its revision as draft.
+
+        :param source: the location of the source (its revision must be None)
         """
-        if Location(location).category in DIRECT_ONLY_CATEGORIES:
-            raise InvalidVersionError(location)
-        return wrap_draft(super(DraftModuleStore, self).clone_item(source, as_draft(location)))
+        original = self.collection.find_one(location_to_query(source_location))
+        draft_location = as_draft(source_location)
+        if draft_location.category in DIRECT_ONLY_CATEGORIES:
+            raise InvalidVersionError(source_location)
+        original['_id'] = draft_location.dict()
+        try:
+            self.collection.insert(original)
+        except pymongo.errors.DuplicateKeyError:
+            raise DuplicateItemError(original['_id'])
+
+        self.refresh_cached_metadata_inheritance_tree(draft_location)
+        self.fire_updated_modulestore_signal(get_course_id_no_run(draft_location), draft_location)
+
+        return self._load_items([original])[0]
 
     def update_item(self, location, data, allow_not_found=False):
         """
@@ -140,7 +168,7 @@ class DraftModuleStore(MongoModuleStore):
         try:
             draft_item = self.get_item(location)
             if not getattr(draft_item, 'is_draft', False):
-                self.clone_item(location, draft_loc)
+                self.convert_to_draft(location)
         except ItemNotFoundError, e:
             if not allow_not_found:
                 raise e
@@ -158,7 +186,7 @@ class DraftModuleStore(MongoModuleStore):
         draft_loc = as_draft(location)
         draft_item = self.get_item(location)
         if not getattr(draft_item, 'is_draft', False):
-            self.clone_item(location, draft_loc)
+            self.convert_to_draft(location)
 
         return super(DraftModuleStore, self).update_children(draft_loc, children)
 
@@ -174,7 +202,7 @@ class DraftModuleStore(MongoModuleStore):
         draft_item = self.get_item(location)
 
         if not getattr(draft_item, 'is_draft', False):
-            self.clone_item(location, draft_loc)
+            self.convert_to_draft(location)
 
         if 'is_draft' in metadata:
             del metadata['is_draft']
@@ -218,9 +246,7 @@ class DraftModuleStore(MongoModuleStore):
         """
         Turn the published version into a draft, removing the published version
         """
-        if Location(location).category in DIRECT_ONLY_CATEGORIES:
-            raise InvalidVersionError(location)
-        super(DraftModuleStore, self).clone_item(location, as_draft(location))
+        self.convert_to_draft(location)
         super(DraftModuleStore, self).delete_item(location)
 
     def _query_children_for_cache_children(self, items):

common/lib/xmodule/xmodule/modulestore/tests/django_utils.py

@@ -5,7 +5,6 @@ from django.test import TestCase
 
 from django.conf import settings
 import xmodule.modulestore.django
-from xmodule.templates import update_templates
 from unittest.util import safe_repr
 
 
@@ -110,22 +109,6 @@ class ModuleStoreTestCase(TestCase):
         modulestore.collection.remove(query)
         modulestore.collection.drop()
 
-    @staticmethod
-    def load_templates_if_necessary():
-        """
-        Load templates into the direct modulestore only if they do not already exist.
-        We need the templates, because they are copied to create
-        XModules such as sections and problems.
-        """
-        modulestore = xmodule.modulestore.django.modulestore('direct')
-
-        # Count the number of templates
-        query = {"_id.course": "templates"}
-        num_templates = modulestore.collection.find(query).count()
-
-        if num_templates < 1:
-            update_templates(modulestore)
-
     @classmethod
     def setUpClass(cls):
         """
@@ -169,9 +152,6 @@ class ModuleStoreTestCase(TestCase):
         # Flush anything that is not a template
         ModuleStoreTestCase.flush_mongo_except_templates()
 
-        # Check that we have templates loaded; if not, load them
-        ModuleStoreTestCase.load_templates_if_necessary()
-
         # Call superclass implementation
         super(ModuleStoreTestCase, self)._pre_setup()
 

common/lib/xmodule/xmodule/modulestore/tests/factories.py

-from factory import Factory, lazy_attribute_sequence, lazy_attribute
-from uuid import uuid4
 import datetime
 
-from xmodule.modulestore import Location
-from xmodule.modulestore.django import modulestore
-from xmodule.modulestore.inheritance import own_metadata
-from xmodule.x_module import ModuleSystem
-from mitxmako.shortcuts import render_to_string
-from xblock.runtime import InvalidScopeError
+from factory import Factory, LazyAttributeSequence
+from uuid import uuid4
 from pytz import UTC
 
+from xmodule.modulestore import Location
+from xmodule.modulestore.django import modulestore
+from xmodule.course_module import CourseDescriptor
+from xblock.core import Scope
 
 class XModuleCourseFactory(Factory):
     """
@@ -21,9 +19,8 @@ class XModuleCourseFactory(Factory):
     @classmethod
     def _create(cls, target_class, **kwargs):
 
-        template = Location('i4x', 'edx', 'templates', 'course', 'Empty')
         org = kwargs.pop('org', None)
-        number = kwargs.pop('number', None)
+        number = kwargs.pop('number', kwargs.pop('course', None))
         display_name = kwargs.pop('display_name', None)
         location = Location('i4x', org, number, 'course', Location.clean(display_name))
 
@@ -33,7 +30,7 @@ class XModuleCourseFactory(Factory):
             store = modulestore()
 
         # Write the data to the mongo datastore
-        new_course = store.clone_item(template, location)
+        new_course = store.create_xmodule(location)
 
         # This metadata code was copied from cms/djangoapps/contentstore/views.py
         if display_name is not None:
@@ -56,13 +53,7 @@ class XModuleCourseFactory(Factory):
             setattr(new_course, k, v)
 
         # Update the data in the mongo datastore
-        store.update_metadata(new_course.location, own_metadata(new_course))
-        store.update_item(new_course.location, new_course._model_data._kvs._data)
-
-        # update_item updates the the course as it exists in the modulestore, but doesn't
-        # update the instance we are working with, so have to refetch the course after updating it.
-        new_course = store.get_instance(new_course.id, new_course.location)
-
+        store.save_xmodule(new_course)
         return new_course
 
 
@@ -73,7 +64,6 @@ class Course:
 class CourseFactory(XModuleCourseFactory):
     FACTORY_FOR = Course
 
-    template = 'i4x://edx/templates/course/Empty'
     org = 'MITx'
     number = '999'
     display_name = 'Robot Super Course'
@@ -86,18 +76,14 @@ class XModuleItemFactory(Factory):
 
     ABSTRACT_FACTORY = True
 
-    display_name = None
-
-    @lazy_attribute
-    def category(attr):
-        template = Location(attr.template)
-        return template.category
+    parent_location = 'i4x://MITx/999/course/Robot_Super_Course'
+    category = 'problem'
+    display_name = LazyAttributeSequence(lambda o, n: "{} {}".format(o.category, n))
 
-    @lazy_attribute
-    def location(attr):
-        parent = Location(attr.parent_location)
-        dest_name = attr.display_name.replace(" ", "_") if attr.display_name is not None else uuid4().hex
-        return parent._replace(category=attr.category, name=dest_name)
+    @staticmethod
+    def location(parent, category, display_name):
+        dest_name = display_name.replace(" ", "_") if display_name is not None else uuid4().hex
+        return Location(parent).replace(category=category, name=dest_name)
 
     @classmethod
     def _create(cls, target_class, **kwargs):
@@ -107,8 +93,7 @@ class XModuleItemFactory(Factory):
         *parent_location* (required): the location of the parent module
             (e.g. the parent course or section)
 
-        *template* (required): the template to create the item from
-            (e.g. i4x://templates/section/Empty)
+        category: the category of the resulting item.
 
         *data* (optional): the data for the item
             (e.g. XML problem definition for a problem item)
@@ -121,41 +106,32 @@ class XModuleItemFactory(Factory):
         """
 
         DETACHED_CATEGORIES = ['about', 'static_tab', 'course_info']
-
+        # catch any old style users before they get into trouble
+        assert not 'template' in kwargs
         parent_location = Location(kwargs.get('parent_location'))
-        template = Location(kwargs.get('template'))
         data = kwargs.get('data')
+        category = kwargs.get('category')
         display_name = kwargs.get('display_name')
         metadata = kwargs.get('metadata', {})
+        location = kwargs.get('location', XModuleItemFactory.location(parent_location, category, display_name))
+        assert location != parent_location
 
         store = modulestore('direct')
 
         # This code was based off that in cms/djangoapps/contentstore/views.py
         parent = store.get_item(parent_location)
 
-        new_item = store.clone_item(template, kwargs.get('location'))
-
         # replace the display name with an optional parameter passed in from the caller
         if display_name is not None:
-            new_item.display_name = display_name
+            metadata['display_name'] = display_name
+        # note that location comes from above lazy_attribute
+        store.create_and_save_xmodule(location, metadata=metadata, definition_data=data)
 
-        # Add additional metadata or override current metadata
-        item_metadata = own_metadata(new_item)
-        item_metadata.update(metadata)
-        store.update_metadata(new_item.location.url(), item_metadata)
+        if location.category not in DETACHED_CATEGORIES:
+            parent.children.append(location.url())
+            store.update_children(parent_location, parent.children)
 
-        # replace the data with the optional *data* parameter
-        if data is not None:
-            store.update_item(new_item.location, data)
-
-        if new_item.location.category not in DETACHED_CATEGORIES:
-            store.update_children(parent_location, parent.children + [new_item.location.url()])
-
-        # update_children updates the the item as it exists in the modulestore, but doesn't
-        # update the instance we are working with, so have to refetch the item after updating it.
-        new_item = store.get_item(new_item.location)
-
-        return new_item
+        return store.get_item(location)
 
 
 class Item:
@@ -164,40 +140,4 @@ class Item:
 
 class ItemFactory(XModuleItemFactory):
     FACTORY_FOR = Item
-
-    parent_location = 'i4x://MITx/999/course/Robot_Super_Course'
-    template = 'i4x://edx/templates/chapter/Empty'
-
-    @lazy_attribute_sequence
-    def display_name(attr, n):
-        return "{} {}".format(attr.category.title(), n)
-
-
-def get_test_xmodule_for_descriptor(descriptor):
-    """
-    Attempts to create an xmodule which responds usually correctly from the descriptor. Not guaranteed.
-
-    :param descriptor:
-    """
-    module_sys = ModuleSystem(
-        ajax_url='',
-        track_function=None,
-        get_module=None,
-        render_template=render_to_string,
-        replace_urls=None,
-        xblock_model_data=_test_xblock_model_data_accessor(descriptor)
-    )
-    return descriptor.xmodule(module_sys)
-
-
-def _test_xblock_model_data_accessor(descriptor):
-    simple_map = {}
-    for field in descriptor.fields:
-        try:
-            simple_map[field.name] = getattr(descriptor, field.name)
-        except InvalidScopeError:
-            simple_map[field.name] = field.default
-    for field in descriptor.module_class.fields:
-        if field.name not in simple_map:
-            simple_map[field.name] = field.default
-    return lambda o: simple_map
+    category = 'chapter'

common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py

@@ -9,7 +9,6 @@ from xblock.runtime import KeyValueStore, InvalidScopeError
 from xmodule.modulestore import Location
 from xmodule.modulestore.mongo import MongoModuleStore, MongoKeyValueStore
 from xmodule.modulestore.xml_importer import import_from_xml
-from xmodule.templates import update_templates
 
 from .test_modulestore import check_path_to_location
 from . import DATA_DIR
@@ -51,7 +50,6 @@ class TestMongoModuleStore(object):
         # Explicitly list the courses to load (don't want the big one)
         courses = ['toy', 'simple']
         import_from_xml(store, DATA_DIR, courses)
-        update_templates(store)
         return store
 
     @staticmethod
@@ -126,7 +124,7 @@ class TestMongoKeyValueStore(object):
         self.location = Location('i4x://org/course/category/name@version')
         self.children = ['i4x://org/course/child/a', 'i4x://org/course/child/b']
         self.metadata = {'meta': 'meta_val'}
-        self.kvs = MongoKeyValueStore(self.data, self.children, self.metadata, self.location)
+        self.kvs = MongoKeyValueStore(self.data, self.children, self.metadata, self.location, 'category')
 
     def _check_read(self, key, expected_value):
         assert_equals(expected_value, self.kvs.get(key))

common/lib/xmodule/xmodule/modulestore/xml.py

@@ -463,7 +463,10 @@ class XMLModuleStore(ModuleStoreBase):
                     # tabs are referenced in policy.json through a 'slug' which is just the filename without the .html suffix
                     slug = os.path.splitext(os.path.basename(filepath))[0]
                     loc = Location('i4x', course_descriptor.location.org, course_descriptor.location.course, category, slug)
-                    module = HtmlDescriptor(system, {'data': html, 'location': loc})
+                    module = HtmlDescriptor(
+                        system,
+                        {'data': html, 'location': loc, 'category': category}
+                    )
                     # VS[compat]:
                     # Hack because we need to pull in the 'display_name' for static tabs (because we need to edit them)
                     # from the course policy

common/lib/xmodule/xmodule/open_ended_grading_classes/combined_open_ended_modulev1.py

@@ -810,7 +810,6 @@ class CombinedOpenEndedV1Descriptor():
     filename_extension = "xml"
 
     has_score = True
-    template_dir_name = "combinedopenended"
 
     def __init__(self, system):
         self.system = system

common/lib/xmodule/xmodule/open_ended_grading_classes/open_ended_module.py

@@ -730,7 +730,6 @@ class OpenEndedDescriptor():
     filename_extension = "xml"
 
     has_score = True
-    template_dir_name = "openended"
 
     def __init__(self, system):
         self.system = system

common/lib/xmodule/xmodule/open_ended_grading_classes/self_assessment_module.py

@@ -287,7 +287,6 @@ class SelfAssessmentDescriptor():
     filename_extension = "xml"
 
     has_score = True
-    template_dir_name = "selfassessment"
 
     def __init__(self, system):
         self.system = system

common/lib/xmodule/xmodule/peer_grading_module.py

@@ -613,7 +613,6 @@ class PeerGradingDescriptor(PeerGradingFields, RawDescriptor):
 
     has_score = True
     always_recalculate_grades = True
-    template_dir_name = "peer_grading"
 
     #Specify whether or not to pass in open ended interface
     needs_open_ended_interface = True

common/lib/xmodule/xmodule/poll_module.py

@@ -140,7 +140,6 @@ class PollDescriptor(PollFields, MakoModuleDescriptor, XmlDescriptor):
     _child_tag_name = 'answer'
 
     module_class = PollModule
-    template_dir_name = 'poll'
 
     @classmethod
     def definition_from_xml(cls, xml_object, system):

common/lib/xmodule/xmodule/templates.py

 """
-This module handles loading xmodule templates from disk into the modulestore.
-These templates are used by the CMS to provide baseline content that
-can be cloned when adding new modules to a course.
+This module handles loading xmodule templates
+These templates are used by the CMS to provide content that overrides xmodule defaults for
+samples.
 
-`Template`s are defined in x_module. They contain 3 attributes:
-    metadata: A dictionary with the template metadata. This should contain
-        any values for fields
-            * with scope Scope.settings
-            * that have values different than the field defaults
-            * and that are to be editable in Studio
-    data: A JSON value that defines the template content. This should be a dictionary
-        containing values for fields
-            * with scope Scope.content
-            * that have values different than the field defaults
-            * and that are to be editable in Studio
-        or, if the module uses a single Scope.content String field named `data`, this
-        should be a string containing the contents of that field
-    children: A list of Location urls that define the template children
-
-Templates are defined on XModuleDescriptor types, in the template attribute.
+``Template``s are defined in x_module. They contain 2 attributes:
+    :metadata: A dictionary with the template metadata
+    :data: A JSON value that defines the template content
 """
 
-
+# should this move to cms since it's really only for module crud?
 import logging
-from fs.memoryfs import MemoryFS
 
 from collections import defaultdict
 from .x_module import XModuleDescriptor
-from .mako_module import MakoDescriptorSystem
-from .modulestore import Location
 
 log = logging.getLogger(__name__)
 
@@ -37,73 +21,9 @@ def all_templates():
     """
     Returns all templates for enabled modules, grouped by descriptor type
     """
-
+    # TODO use memcache to memoize w/ expiration
     templates = defaultdict(list)
     for category, descriptor in XModuleDescriptor.load_classes():
         templates[category] = descriptor.templates()
 
     return templates
-
-
-class TemplateTestSystem(MakoDescriptorSystem):
-    """
-    This system exists to help verify that XModuleDescriptors can be instantiated
-    from their defined templates before we load the templates into the modulestore.
-    """
-    def __init__(self):
-        super(TemplateTestSystem, self).__init__(
-            lambda *a, **k: None,
-            MemoryFS(),
-            lambda msg: None,
-            render_template=lambda *a, **k: None,
-        )
-
-
-def update_templates(modulestore):
-    """
-    Updates the set of templates in the modulestore with all templates currently
-    available from the installed plugins
-    """
-
-    # cdodge: build up a list of all existing templates. This will be used to determine which
-    # templates have been removed from disk - and thus we need to remove from the DB
-    templates_to_delete = modulestore.get_items(['i4x', 'edx', 'templates', None, None, None])
-
-    for category, templates in all_templates().items():
-        for template in templates:
-            if 'display_name' not in template.metadata:
-                log.warning('No display_name specified in template {0}, skipping'.format(template))
-                continue
-
-            template_location = Location('i4x', 'edx', 'templates', category, Location.clean_for_url_name(template.metadata['display_name']))
-
-            try:
-                json_data = {
-                    'definition': {
-                        'data': template.data,
-                        'children': template.children
-                    },
-                    'metadata': template.metadata
-                }
-                json_data['location'] = template_location.dict()
-
-                XModuleDescriptor.load_from_json(json_data, TemplateTestSystem())
-            except:
-                log.warning('Unable to instantiate {cat} from template {template}, skipping'.format(
-                    cat=category,
-                    template=template
-                ), exc_info=True)
-                continue
-
-            modulestore.update_item(template_location, template.data)
-            modulestore.update_children(template_location, template.children)
-            modulestore.update_metadata(template_location, template.metadata)
-
-            # remove template from list of templates to delete
-            templates_to_delete = [t for t in templates_to_delete if t.location != template_location]
-
-    # now remove all templates which appear to have removed from disk
-    if len(templates_to_delete) > 0:
-        logging.debug('deleting dangling templates = {0}'.format(templates_to_delete))
-        for template in templates_to_delete:
-            modulestore.delete_item(template.location)

common/lib/xmodule/xmodule/templates/about/empty.yaml

-{}

common/lib/xmodule/xmodule/templates/annotatable/default.yaml

-{}

common/lib/xmodule/xmodule/templates/combinedopenended/default.yaml

-{}

common/lib/xmodule/xmodule/templates/course/empty.yaml

-{}

common/lib/xmodule/xmodule/templates/courseinfo/empty.yaml

-{}

common/lib/xmodule/xmodule/templates/default/empty.yaml

-{}

common/lib/xmodule/xmodule/templates/discussion/default.yaml

-{}

common/lib/xmodule/xmodule/templates/html/announcement.yaml

@@ -21,4 +21,3 @@ data: |
       </section>
       </li>
       </ol>
-children: []

common/lib/xmodule/xmodule/templates/html/empty.yaml

-{}

common/lib/xmodule/xmodule/templates/html/everything.yaml

----
-metadata:
-      display_name: Announcement
-
-data: |
-      <h1>Heading of document</h1>
-      <h2>First subheading</h2>
-      <p>This is a paragraph. It will take care of line breaks for you.</p><p>HTML only parses the location
-      
-      of tags for inserting line breaks into your doc, not 
-      line
-      breaks
-      you
-      add
-      yourself.
-      </p>
-      <h2>Links</h2>
-      <p>You can refer to other parts of the internet with a <a href="http://www.wikipedia.org/"> link</a>, to other parts of your course by prepending your link with <a href="/course/Week_0">/course/</a></p>
-      <p>Now a list:</p>
-      <ul>
-      <li>An item</li>
-      <li>Another item</li>
-      <li>And yet another</li>
-      </ul>
-      <p>This list has an ordering </p>
-      <ol>
-      <li>An item</li>
-      <li>Another item</li>
-      <li>Yet another item</li>
-      </ol>
-      <p> Note, we have a lot of standard edX styles, so please try to avoid any custom styling, and make sure that you make a note of any custom styling that you do yourself so that we can incorporate it into 
-      tools that other people can use. </p>
-children: []

common/lib/xmodule/xmodule/templates/html/latex_html.yaml

@@ -19,4 +19,3 @@ data: |
           It is very convenient to write complex equations in LaTeX.
         </p>
     </html>
-children: []

common/lib/xmodule/xmodule/templates/peer_grading/default.yaml

-{}

common/lib/xmodule/xmodule/templates/problem/circuitschematic.yaml

-
 ---
 metadata:
     display_name: Circuit Schematic Builder
     rerandomize: never
     showanswer: finished
+    markdown: !!null
 data: |
       <problem >
         Please make a voltage divider that splits the provided voltage evenly.

common/lib/xmodule/xmodule/templates/problem/customgrader.yaml

 ---
 metadata:
     display_name: Custom Python-Evaluated Input
-    rerandomize: never
-    showanswer: finished
+    markdown: !!null
 data: |
-  <problem>
-  <p>
-  A custom python-evaluated input problem accepts one or more lines of text input from the
-  student, and evaluates the inputs for correctness based on evaluation using a 
-  python script embedded within the problem.
-  </p>
-
-  <script type="loncapa/python">
-
-  def test_add(expect, ans):
-      try:
-          a1=int(ans[0])
-          a2=int(ans[1])
-          return (a1+a2) == int(expect)
-      except ValueError:
-          return False
-
-  def test_add_to_ten(expect, ans):
-      return test_add(10, ans)
-
-  </script>
-
-  <p>Enter two integers which sum to 10: </p>
-  <customresponse cfn="test_add_to_ten">
-          <textline size="40" correct_answer="3"/><br/>
-          <textline size="40" correct_answer="7"/>
-  </customresponse>
-
-  <p>Enter two integers which sum to 20: </p>
-  <customresponse cfn="test_add" expect="20">
-          <textline size="40" correct_answer="11"/><br/>
-          <textline size="40" correct_answer="9"/>
-  </customresponse>
-
-      <solution>
-          <div class="detailed-solution">
-              <p>Explanation</p>
-              <p>Any set of integers on the line \(y = 10 - x\) and \(y = 20 - x\) satisfy these constraints.</p>
-              <img src="/static/images/simple_graph.png"/>
-          </div>
-      </solution>
-  </problem>
-
-children: []
+      <problem>
+      <p>
+      A custom python-evaluated input problem accepts one or more lines of text input from the
+      student, and evaluates the inputs for correctness based on evaluation using a 
+      python script embedded within the problem.
+      </p>
+
+      <script type="loncapa/python">
+
+      def test_add(expect, ans):
+          try:
+              a1=int(ans[0])
+              a2=int(ans[1])
+              return (a1+a2) == int(expect)
+          except ValueError:
+              return False
+
+      def test_add_to_ten(expect, ans):
+          return test_add(10, ans)
+
+      </script>
+
+      <p>Enter two integers which sum to 10: </p>
+      <customresponse cfn="test_add_to_ten">
+              <textline size="40" correct_answer="3"/><br/>
+              <textline size="40" correct_answer="7"/>
+      </customresponse>
+
+      <p>Enter two integers which sum to 20: </p>
+      <customresponse cfn="test_add" expect="20">
+              <textline size="40" correct_answer="11"/><br/>
+              <textline size="40" correct_answer="9"/>
+      </customresponse>
+    
+          <solution>
+              <div class="detailed-solution">
+                  <p>Explanation</p>
+                  <p>Any set of integers on the line \(y = 10 - x\) and \(y = 20 - x\) satisfy these constraints.</p>
+                  <img src="/static/images/simple_graph.png"/>
+              </div>
+          </solution>
+      </problem>

common/lib/xmodule/xmodule/templates/problem/empty.yaml

-{}

common/lib/xmodule/xmodule/templates/problem/emptyadvanced.yaml

----
-metadata:
-    display_name: Blank Advanced Problem
-    rerandomize: never
-    showanswer: finished
-data: |
-  <problem>
-  </problem>
-
-children: []

common/lib/xmodule/xmodule/templates/problem/forumularesponse.yaml

@@ -3,6 +3,7 @@ metadata:
     display_name: Math Expression Input
     rerandomize: never
     showanswer: finished
+    markdown: !!null
 data: |
   <problem>
   <p>
@@ -43,5 +44,3 @@ data: |
           </div>
       </solution>
   </problem>
-
-children: []

common/lib/xmodule/xmodule/templates/problem/imageresponse.yaml

@@ -3,6 +3,7 @@ metadata:
     display_name: Image Mapped Input
     rerandomize: never
     showanswer: finished
+    markdown: !!null
 data: |
   <problem>
   <p>
@@ -21,6 +22,3 @@ data: |
           </div>
       </solution>
   </problem>
-
-    
-children: []

common/lib/xmodule/xmodule/templates/problem/latex_problem.yaml

@@ -85,6 +85,7 @@ metadata:
         can contain equations: $\alpha = \frac{2}{\sqrt{1+\gamma}}$ }
         
         This is some text after the showhide example.
+    markdown: !!null
 
 data: |
     <?xml version="1.0"?>
@@ -214,4 +215,3 @@ data: |
         </p>
       </text>
     </problem>
-children: []

common/lib/xmodule/xmodule/templates/problem/multiplechoice.yaml

@@ -3,33 +3,25 @@ metadata:
     display_name: Multiple Choice
     rerandomize: never
     showanswer: finished
-    markdown: 
-       "A multiple choice problem presents radio buttons for student input. Students can only select a single 
+    markdown: |
+       A multiple choice problem presents radio buttons for student input. Students can only select a single 
        option presented. Multiple Choice questions have been the subject of many areas of research due to the early 
        invention and adoption of bubble sheets.
        
-       
        One of the main elements that goes into a good multiple choice question is the existence of good distractors. 
        That is, each of the alternate responses presented to the student should be the result of a plausible mistake 
        that a student might make.
-        
     
        What Apple device competed with the portable CD player?
-       
             ( ) The iPad
-            
             ( ) Napster
-            
             (x) The iPod
-            
             ( ) The vegetable peeler
             
-            
        [explanation]
        The release of the iPod allowed consumers to carry their entire music library with them in a 
        format that did not rely on fragile and energy-intensive spinning disks.
        [explanation]
-       "
 data: |
    <problem>
    <p>
@@ -54,4 +46,3 @@ data: |
            </div>
        </solution>
    </problem>
-children: []

common/lib/xmodule/xmodule/templates/problem/numericalresponse.yaml

@@ -3,43 +3,33 @@ metadata:
     display_name: Numerical Input
     rerandomize: never
     showanswer: finished
-    markdown: 
-       "A numerical input problem accepts a line of text input from the
+    markdown: |
+       A numerical input problem accepts a line of text input from the
        student, and evaluates the input for correctness based on its
        numerical value.
-       
 
        The answer is correct if it is within a specified numerical tolerance
        of the expected answer.
-       
 
        Enter the numerical value of Pi:
-        
        = 3.14159 +- .02
        
-       
        Enter the approximate value of 502*9:
-        
        = 4518 +- 15%
-       
-    
+
        Enter the number of fingers on a human hand:
-        
        = 5
-       
 
        [explanation]
        Pi, or the the ratio between a circle's circumference to its diameter, is an irrational number 
        known to extreme precision. It is value is approximately equal to 3.14.
-               
+
        Although you can get an exact value by typing 502*9 into a calculator, the result will be close to 
        500*10, or 5,000. The grader accepts any response within 15% of the true value, 4518, so that you 
        can use any estimation technique that you like.
-       
+
        If you look at your hand, you can count that you have five fingers.
        [explanation]
-       "
-
 data: |
    <problem>
    <p>
@@ -83,5 +73,3 @@ data: |
            </div>
        </solution>
    </problem>
-
-children: []

common/lib/xmodule/xmodule/templates/problem/optionresponse.yaml

@@ -3,19 +3,16 @@ metadata:
     display_name: Dropdown
     rerandomize: never
     showanswer: finished
-    markdown: 
-        "Dropdown problems give a limited set of options for students to respond with, and present those options
+    markdown: |
+        Dropdown problems give a limited set of options for students to respond with, and present those options
         in a format that encourages them to search for a specific answer rather than being immediately presented 
         with options from which to recognize the correct answer.
-        
 
         The answer options and the identification of the correct answer is defined in the <b>optioninput</b> tag.
-        
 
         Translation between Dropdown and __________ is extremely straightforward:
-        
-        [[(Multiple Choice), Text Input, Numerical Input, External Response, Image Response]]
 
+        [[(Multiple Choice), Text Input, Numerical Input, External Response, Image Response]]
 
         [explanation]
         Multiple Choice also allows students to select from a variety of pre-written responses, although the 
@@ -23,7 +20,6 @@ metadata:
         slightly because students are more likely to think of an answer and then search for it rather than 
         relying purely on recognition to answer the question.
         [explanation]
-        "
 data: |
   <problem>
   <p>Dropdown problems give a limited set of options for students to respond with, and present those options
@@ -45,4 +41,3 @@ data: |
           </div>
       </solution>
   </problem>
-children: []

common/lib/xmodule/xmodule/templates/problem/problem_with_hint.yaml

@@ -46,7 +46,7 @@ metadata:
         enter your answer in upper or lower case, with or without quotes.
         
         \edXabox{type="custom" cfn='test_str' expect='python' hintfn='hint_fn'}
-        
+    markdown: !!null
 data: |
     <?xml version="1.0"?>
     <problem>
@@ -92,4 +92,3 @@ data: |
         </p>
       </text>
     </problem>
-children: []

common/lib/xmodule/xmodule/templates/problem/string_response.yaml

@@ -3,16 +3,13 @@ metadata:
     display_name: Text Input
     rerandomize: never
     showanswer: finished
-    # Note, the extra newlines are needed to make the yaml parser add blank lines instead of folding
-    markdown: 
-        "A text input problem accepts a line of text from the 
+    markdown: |
+        A text input problem accepts a line of text from the 
         student, and evaluates the input for correctness based on an expected 
         answer.
-        
     
         The answer is correct if it matches every character of the expected answer. This can be a problem with  
         international spelling, dates, or anything where the format of the answer is not clear.
-         
     
         Which US state has Lansing as its capital?
 
@@ -23,9 +20,8 @@ metadata:
         Lansing is the capital of Michigan, although it is not Michgan's largest city, 
         or even the seat of the county in which it resides.
         [explanation]
-        "
 data: |
-    <problem showanswer="always">
+    <problem>
     <p>
 
     A text input problem accepts a line of text from the
@@ -46,4 +42,3 @@ data: |
             </div>
         </solution>
     </problem>
-children: []

common/lib/xmodule/xmodule/templates/sequence/with_video.yaml

----
-metadata:
-    display_name: Sequence with Video
-    data_dir: a_made_up_name
-data: ''
-children:
-    - 'i4x://edx/templates/video/default'

common/lib/xmodule/xmodule/templates/statictab/empty.yaml

-{}
\ No newline at end of file

common/lib/xmodule/xmodule/templates/video/default.yaml

-{}

common/lib/xmodule/xmodule/templates/videoalpha/default.yaml

-{}

common/lib/xmodule/xmodule/templates/word_cloud/default.yaml

-{}

common/lib/xmodule/xmodule/tests/test_logic.py

@@ -28,7 +28,8 @@ class LogicTest(unittest.TestCase):
     def setUp(self):
         class EmptyClass:
             """Empty object."""
-            pass
+            url_name = ''
+            category = 'test'
 
         self.system = get_test_system()
         self.descriptor = EmptyClass()

common/lib/xmodule/xmodule/tests/test_xml_module.py

@@ -141,6 +141,7 @@ class EditableMetadataFieldsTest(unittest.TestCase):
     def get_xml_editable_fields(self, model_data):
         system = get_test_system()
         system.render_template = Mock(return_value="<div>Test Template HTML</div>")
+        model_data['category'] = 'test'
         return XmlDescriptor(runtime=system, model_data=model_data).editable_metadata_fields
 
     def get_descriptor(self, model_data):

common/lib/xmodule/xmodule/video_module.py

@@ -97,7 +97,6 @@ class VideoDescriptor(VideoFields,
                       MetadataOnlyEditingDescriptor,
                       RawDescriptor):
     module_class = VideoModule
-    template_dir_name = "video"
 
     def __init__(self, *args, **kwargs):
         super(VideoDescriptor, self).__init__(*args, **kwargs)

common/lib/xmodule/xmodule/videoalpha_module.py

@@ -179,4 +179,3 @@ class VideoAlphaModule(VideoAlphaFields, XModule):
 class VideoAlphaDescriptor(VideoAlphaFields, RawDescriptor):
     """Descriptor for `VideoAlphaModule`."""
     module_class = VideoAlphaModule
-    template_dir_name = "videoalpha"

common/lib/xmodule/xmodule/xml_module.py

@@ -356,6 +356,7 @@ class XmlDescriptor(XModuleDescriptor):
             if key not in set(f.name for f in cls.fields + cls.lms.fields):
                 model_data['xml_attributes'][key] = value
         model_data['location'] = location
+        model_data['category'] = xml_object.tag
 
         return cls(
             system,

lms/djangoapps/courseware/features/common.py

@@ -10,7 +10,6 @@ from django.contrib.auth.models import User
 from student.models import CourseEnrollment
 from xmodule.modulestore import Location
 from xmodule.modulestore.django import _MODULESTORES, modulestore
-from xmodule.templates import update_templates
 from xmodule.course_module import CourseDescriptor
 from courseware.courses import get_course_by_id
 from xmodule import seq_module, vertical_module
@@ -39,7 +38,7 @@ def create_course(step, course):
                                        display_name='Test Section')
 
     problem_section = world.ItemFactory.create(parent_location=world.scenario_dict['SECTION'].location,
-                                               template='i4x://edx/templates/sequential/Empty',
+                                               category='sequential'
                                                display_name='Test Section')
 
 
@@ -62,7 +61,7 @@ def i_am_registered_for_the_course(step, course):
 @step(u'The course "([^"]*)" has extra tab "([^"]*)"$')
 def add_tab_to_course(step, course, extra_tab_name):
     section_item = world.ItemFactory.create(parent_location=course_location(course),
-                                            template="i4x://edx/templates/static_tab/Empty",
+                                            category="static_tab",
                                             display_name=str(extra_tab_name))
 
 

lms/djangoapps/courseware/features/navigation.py

@@ -24,11 +24,11 @@ def view_course_multiple_sections(step):
                                        display_name=section_name(2))
 
     place1 = world.ItemFactory.create(parent_location=section1.location,
-                                               template='i4x://edx/templates/sequential/Empty',
+                                               category='sequential',
                                                display_name=subsection_name(1))
 
     place2 = world.ItemFactory.create(parent_location=section2.location,
-                                               template='i4x://edx/templates/sequential/Empty',
+                                               category='sequential',
                                                display_name=subsection_name(2))
 
     add_problem_to_course_section('model_course', 'multiple choice', place1.location)
@@ -46,7 +46,7 @@ def view_course_multiple_subsections(step):
                                        display_name=section_name(1))
 
     place1 = world.ItemFactory.create(parent_location=section1.location,
-                                               template='i4x://edx/templates/sequential/Empty',
+                                               category='sequential',
                                                display_name=subsection_name(1))
 
     place2 = world.ItemFactory.create(parent_location=section1.location,
@@ -66,7 +66,7 @@ def view_course_multiple_sequences(step):
                                        display_name=section_name(1))
 
     place1 = world.ItemFactory.create(parent_location=section1.location,
-                                               template='i4x://edx/templates/sequential/Empty',
+                                               category='sequential',
                                                display_name=subsection_name(1))
 
     add_problem_to_course_section('model_course', 'multiple choice', place1.location)
@@ -177,9 +177,8 @@ def add_problem_to_course_section(course, problem_type, parent_location, extraMe
     # Create a problem item using our generated XML
     # We set rerandomize=always in the metadata so that the "Reset" button
     # will appear.
-    template_name = "i4x://edx/templates/problem/Blank_Common_Problem"
     world.ItemFactory.create(parent_location=parent_location,
-                            template=template_name,
+                            category='problem',
                             display_name=str(problem_type),
                             data=problem_xml,
                             metadata=metadata)

lms/djangoapps/courseware/features/problems.py

@@ -17,7 +17,7 @@ def view_problem_with_attempts(step, problem_type, attempts):
     i_am_registered_for_the_course(step, 'model_course')
 
     # Ensure that the course has this problem type
-    add_problem_to_course(world.scenario_dict['COURSE'].number, problem_type, {'attempts': attempts})
+    add_problem_to_course(world.scenario_dict['COURSE'].number, problem_type, {'max_attempts': attempts})
 
     # Go to the one section in the factory-created course
     # which should be loaded with the correct problem

lms/djangoapps/courseware/features/problems_setup.py

@@ -273,9 +273,9 @@ def add_problem_to_course(course, problem_type, extraMeta=None):
     # Create a problem item using our generated XML
     # We set rerandomize=always in the metadata so that the "Reset" button
     # will appear.
-    template_name = "i4x://edx/templates/problem/Blank_Common_Problem"
-    world.ItemFactory.create(parent_location=section_location(course),
-                            template=template_name,
+    category_name = "problem"
+    return world.ItemFactory.create(parent_location=section_location(course),
+                            category=category_name,
                             display_name=str(problem_type),
                             data=problem_xml,
                             metadata=metadata)

lms/djangoapps/courseware/features/video.py

@@ -43,14 +43,13 @@ def view_videoalpha(step):
 
 
 def add_video_to_course(course):
-    template_name = 'i4x://edx/templates/video/default'
     world.ItemFactory.create(parent_location=section_location(course),
-                             template=template_name,
+                             category='video',
                              display_name='Video')
 
 
 def add_videoalpha_to_course(course):
-    template_name = 'i4x://edx/templates/videoalpha/Video_Alpha'
+    category = 'videoalpha'
     world.ItemFactory.create(parent_location=section_location(course),
-                             template=template_name,
+                             category=category,
                              display_name='Video Alpha')

lms/djangoapps/courseware/tests/__init__.py

@@ -29,17 +29,17 @@ class BaseTestXmodule(ModuleStoreTestCase):
         2. create, enrol and login users for this course;
 
     Any xmodule should overwrite only next parameters for test:
-        1. TEMPLATE_NAME
+        1. CATEGORY
         2. DATA
         3. MODEL_DATA
 
-    This class should not contain any tests, because TEMPLATE_NAME
+    This class should not contain any tests, because CATEGORY
     should be defined in child class.
     """
     USER_COUNT = 2
 
     # Data from YAML common/lib/xmodule/xmodule/templates/NAME/default.yaml
-    TEMPLATE_NAME = ""
+    CATEGORY = ""
     DATA = ''
     MODEL_DATA = {'data': '<some_module></some_module>'}
 
@@ -53,11 +53,11 @@ class BaseTestXmodule(ModuleStoreTestCase):
 
         chapter = ItemFactory.create(
             parent_location=self.course.location,
-            template="i4x://edx/templates/sequential/Empty",
+            category="sequential",
         )
         section = ItemFactory.create(
             parent_location=chapter.location,
-            template="i4x://edx/templates/sequential/Empty"
+            category="sequential"
         )
 
         # username = robot{0}, password = 'test'
@@ -71,7 +71,7 @@ class BaseTestXmodule(ModuleStoreTestCase):
 
         self.item_descriptor = ItemFactory.create(
             parent_location=section.location,
-            template=self.TEMPLATE_NAME,
+            category=self.CATEGORY,
             data=self.DATA
         )