Add datadog counters everywhere a "VS[compat]" comment exists.

Conflicts:
	common/lib/xmodule/xmodule/xml_module.py

cms/djangoapps/contentstore/views/helpers.py

@@ -15,8 +15,10 @@ from django.utils.translation import ugettext as _
 from edxmako.shortcuts import render_to_string, render_to_response
 from opaque_keys.edx.keys import UsageKey
 from xblock.core import XBlock
+import dogstats_wrapper as dog_stats_api
 from xmodule.modulestore.django import modulestore
 from xmodule.tabs import StaticTab
+from xmodule.x_module import DEPRECATION_VSCOMPAT_EVENT
 
 from contentstore.utils import reverse_course_url, reverse_library_url, reverse_usage_url
 from models.settings.course_grading import CourseGradingModel
@@ -251,6 +253,15 @@ def create_xblock(parent_locator, user, category, display_name, boilerplate=None
         # if we add one then we need to also add it to the policy information (i.e. metadata)
         # we should remove this once we can break this reference from the course to static tabs
         if category == 'static_tab':
+
+            dog_stats_api.increment(
+                DEPRECATION_VSCOMPAT_EVENT,
+                tags=(
+                    "location:create_xblock_static_tab",
+                    u"course:{}".format(unicode(dest_usage_key.course_key)),
+                )
+            )
+
             display_name = display_name or _("Empty")  # Prevent name being None
             course = store.get_course(dest_usage_key.course_key)
             course.tabs.append(

cms/djangoapps/contentstore/views/item.py

@@ -13,6 +13,7 @@ from functools import partial
 from static_replace import replace_static_urls
 from xmodule_modifiers import wrap_xblock, request_token
 
+import dogstats_wrapper as dog_stats_api
 from django.conf import settings
 from django.core.exceptions import PermissionDenied
 from django.contrib.auth.decorators import login_required
@@ -30,7 +31,7 @@ from xmodule.modulestore.django import modulestore
 from xmodule.modulestore.exceptions import ItemNotFoundError, InvalidLocationError
 from xmodule.modulestore.inheritance import own_metadata
 from xmodule.modulestore.draft_and_published import DIRECT_ONLY_CATEGORIES
-from xmodule.x_module import PREVIEW_VIEWS, STUDIO_VIEW, STUDENT_VIEW
+from xmodule.x_module import PREVIEW_VIEWS, STUDIO_VIEW, STUDENT_VIEW, DEPRECATION_VSCOMPAT_EVENT
 
 from xmodule.course_module import DEFAULT_START_DATE
 from django.contrib.auth.models import User
@@ -637,6 +638,15 @@ def _delete_item(usage_key, user):
         # if we add one then we need to also add it to the policy information (i.e. metadata)
         # we should remove this once we can break this reference from the course to static tabs
         if usage_key.category == 'static_tab':
+
+            dog_stats_api.increment(
+                DEPRECATION_VSCOMPAT_EVENT,
+                tags=(
+                    "location:_delete_item_static_tab",
+                    u"course:{}".format(unicode(usage_key.course_key)),
+                )
+            )
+
             course = store.get_course(usage_key.course_key)
             existing_tabs = course.tabs or []
             course.tabs = [tab for tab in existing_tabs if tab.get('url_slug') != usage_key.name]

common/lib/xmodule/xmodule/capa_module.py

@@ -6,10 +6,11 @@ from lxml import etree
 
 from pkg_resources import resource_string
 
+import dogstats_wrapper as dog_stats_api
 from .capa_base import CapaMixin, CapaFields, ComplexEncoder
 from capa import responsetypes
 from .progress import Progress
-from xmodule.x_module import XModule, module_attr
+from xmodule.x_module import XModule, module_attr, DEPRECATION_VSCOMPAT_EVENT
 from xmodule.raw_module import RawDescriptor
 from xmodule.exceptions import NotFoundError, ProcessingError
 
@@ -156,6 +157,10 @@ class CapaDescriptor(CapaFields, RawDescriptor):
     # edited in the cms
     @classmethod
     def backcompat_paths(cls, path):
+        dog_stats_api.increment(
+            DEPRECATION_VSCOMPAT_EVENT,
+            tags=["location:capa_descriptor_backcompat_paths"]
+        )
         return [
             'problems/' + path[8:],
             path[8:],

common/lib/xmodule/xmodule/html_module.py

-import copy
-from fs.errors import ResourceNotFoundError
-import logging
 import os
 import sys
+import re
+import copy
+import logging
+import textwrap
 from lxml import etree
 from path import path
-
+from fs.errors import ResourceNotFoundError
 from pkg_resources import resource_string
-from xblock.fields import Scope, String, Boolean, List
+
+import dogstats_wrapper as dog_stats_api
+from xmodule.annotator_mixin import html_to_text
+from xmodule.contentstore.content import StaticContent
 from xmodule.editing_module import EditingDescriptor
+from xmodule.edxnotes_utils import edxnotes
 from xmodule.html_checker import check_html
 from xmodule.stringify import stringify_children
-from xmodule.x_module import XModule
+from xmodule.x_module import XModule, DEPRECATION_VSCOMPAT_EVENT
 from xmodule.xml_module import XmlDescriptor, name_to_pathname
-import textwrap
-from xmodule.contentstore.content import StaticContent
 from xblock.core import XBlock
-from xmodule.edxnotes_utils import edxnotes
-from xmodule.annotator_mixin import html_to_text
-import re
+from xblock.fields import Scope, String, Boolean, List
 
 log = logging.getLogger("edx.courseware")
 
@@ -103,6 +104,12 @@ class HtmlDescriptor(HtmlFields, XmlDescriptor, EditingDescriptor):
     # are being edited in the cms
     @classmethod
     def backcompat_paths(cls, path):
+
+        dog_stats_api.increment(
+            DEPRECATION_VSCOMPAT_EVENT,
+            tags=["location:html_descriptor_backcompat_paths"]
+        )
+
         if path.endswith('.html.xml'):
             path = path[:-9] + '.html'  # backcompat--look for html instead of xml
         if path.endswith('.html.html'):
@@ -192,6 +199,12 @@ class HtmlDescriptor(HtmlFields, XmlDescriptor, EditingDescriptor):
             # again in the correct format.  This should go away once the CMS is
             # online and has imported all current (fall 2012) courses from xml
             if not system.resources_fs.exists(filepath):
+
+                dog_stats_api.increment(
+                    DEPRECATION_VSCOMPAT_EVENT,
+                    tags=["location:html_descriptor_load_definition"]
+                )
+
                 candidates = cls.backcompat_paths(filepath)
                 # log.debug("candidates = {0}".format(candidates))
                 for candidate in candidates:

common/lib/xmodule/xmodule/modulestore/tests/factories.py

 import pprint
+import threading
+from uuid import uuid4
+from decorator import contextmanager
 import pymongo.message
 
-from factory import Factory, lazy_attribute_sequence, lazy_attribute
+from factory import Factory, Sequence, lazy_attribute_sequence, lazy_attribute
 from factory.containers import CyclicDefinitionError
-from uuid import uuid4
+from mock import Mock, patch
+from nose.tools import assert_less_equal, assert_greater_equal
+import dogstats_wrapper as dog_stats_api
 
-from xmodule.modulestore import prefer_xmodules, ModuleStoreEnum
 from opaque_keys.edx.locations import Location
 from opaque_keys.edx.keys import UsageKey
 from xblock.core import XBlock
 from xmodule.tabs import StaticTab
-from decorator import contextmanager
-from mock import Mock, patch
-from nose.tools import assert_less_equal, assert_greater_equal
-import factory
-import threading
+from xmodule.modulestore import prefer_xmodules, ModuleStoreEnum
 from xmodule.modulestore.django import modulestore
+from xmodule.x_module import DEPRECATION_VSCOMPAT_EVENT
 
 
 class Dummy(object):
@@ -86,9 +87,9 @@ class CourseFactory(XModuleFactory):
     """
     Factory for XModule courses.
     """
-    org = factory.Sequence(lambda n: 'org.%d' % n)
-    number = factory.Sequence(lambda n: 'course_%d' % n)
-    display_name = factory.Sequence(lambda n: 'Run %d' % n)
+    org = Sequence('org.{}'.format)
+    number = Sequence('course_{}'.format)
+    display_name = Sequence('Run {}'.format)
 
     # pylint: disable=unused-argument
     @classmethod
@@ -124,9 +125,9 @@ class LibraryFactory(XModuleFactory):
     """
     Factory for creating a content library
     """
-    org = factory.Sequence('org{}'.format)
-    library = factory.Sequence('lib{}'.format)
-    display_name = factory.Sequence('Test Library {}'.format)
+    org = Sequence('org{}'.format)
+    library = Sequence('lib{}'.format)
+    display_name = Sequence('Test Library {}'.format)
 
     # pylint: disable=unused-argument
     @classmethod
@@ -267,6 +268,14 @@ class ItemFactory(XModuleFactory):
             # if we add one then we need to also add it to the policy information (i.e. metadata)
             # we should remove this once we can break this reference from the course to static tabs
             if category == 'static_tab':
+                dog_stats_api.increment(
+                    DEPRECATION_VSCOMPAT_EVENT,
+                    tags=(
+                        "location:itemfactory_create_static_tab",
+                        u"block:{}".format(location.block_type),
+                    )
+                )
+
                 course = store.get_course(location.course_key)
                 course.tabs.append(
                     StaticTab(

common/lib/xmodule/xmodule/modulestore/xml.py

@@ -18,7 +18,10 @@ from contextlib import contextmanager
 from xmodule.error_module import ErrorDescriptor
 from xmodule.errortracker import make_error_tracker, exc_info_to_str
 from xmodule.mako_module import MakoDescriptorSystem
-from xmodule.x_module import XMLParsingSystem, policy_key, OpaqueKeyReader, AsideKeyGenerator
+from xmodule.x_module import (
+    XMLParsingSystem, policy_key,
+    OpaqueKeyReader, AsideKeyGenerator, DEPRECATION_VSCOMPAT_EVENT
+)
 from xmodule.modulestore.xml_exporter import DEFAULT_CONTENT_FIELDS
 from xmodule.modulestore import ModuleStoreEnum, ModuleStoreReadBase, LIBRARY_ROOT, COURSE_ROOT
 from xmodule.tabs import CourseTabList
@@ -27,12 +30,13 @@ from opaque_keys.edx.locator import CourseLocator, LibraryLocator
 
 from xblock.field_data import DictFieldData
 from xblock.runtime import DictKeyValueStore
+from xblock.fields import ScopeIds
 
+import dogstats_wrapper as dog_stats_api
 
 from .exceptions import ItemNotFoundError
 from .inheritance import compute_inherited_metadata, inheriting_field_data, InheritanceKeyValueStore
 
-from xblock.fields import ScopeIds
 
 edx_xml_parser = etree.XMLParser(dtd_validation=False, load_dtd=False,
                                  remove_comments=True, remove_blank_text=True)
@@ -46,8 +50,14 @@ log = logging.getLogger(__name__)
 # TODO (cpennington): Remove this once all fall 2012 courses have been imported
 # into the cms from xml
 def clean_out_mako_templating(xml_string):
+    orig_xml = xml_string
     xml_string = xml_string.replace('%include', 'include')
     xml_string = re.sub(r"(?m)^\s*%.*$", '', xml_string)
+    if orig_xml != xml_string:
+        dog_stats_api.increment(
+            DEPRECATION_VSCOMPAT_EVENT,
+            tags=["location:xml_clean_out_mako_templating"]
+        )
     return xml_string
 
 
@@ -114,6 +124,14 @@ class ImportSystem(XMLParsingSystem, MakoDescriptorSystem):
                 def fallback_name(orig_name=None):
                     """Return the fallback name for this module.  This is a function instead of a variable
                     because we want it to be lazy."""
+                    dog_stats_api.increment(
+                        DEPRECATION_VSCOMPAT_EVENT,
+                        tags=(
+                            "location:import_system_fallback_name",
+                            u"name:{}".format(orig_name),
+                        )
+                    )
+
                     if looks_like_fallback(orig_name):
                         # We're about to re-hash, in case something changed, so get rid of the tag_ and hash
                         orig_name = orig_name[len(tag) + 1:-12]
@@ -468,12 +486,32 @@ class XMLModuleStore(ModuleStoreReadBase):
 
                 # VS[compat]: remove once courses use the policy dirs.
                 if policy == {}:
+
+                    dog_stats_api.increment(
+                        DEPRECATION_VSCOMPAT_EVENT,
+                        tags=(
+                            "location:xml_load_course_policy_dir",
+                            u"course:{}".format(course),
+                        )
+                    )
+
                     old_policy_path = self.data_dir / course_dir / 'policies' / '{0}.json'.format(url_name)
                     policy = self.load_policy(old_policy_path, tracker)
             else:
                 policy = {}
                 # VS[compat] : 'name' is deprecated, but support it for now...
                 if course_data.get('name'):
+
+                    dog_stats_api.increment(
+                        DEPRECATION_VSCOMPAT_EVENT,
+                        tags=(
+                            "location:xml_load_course_course_data_name",
+                            u"course:{}".format(course_data.get('course')),
+                            u"org:{}".format(course_data.get('org')),
+                            u"name:{}".format(course_data.get('name')),
+                        )
+                    )
+
                     url_name = Location.clean(course_data.get('name'))
                     tracker("'name' is deprecated for module xml.  Please use "
                             "display_name and url_name.")
@@ -660,6 +698,14 @@ class XMLModuleStore(ModuleStoreReadBase):
                         # Hack because we need to pull in the 'display_name' for static tabs (because we need to edit them)
                         # from the course policy
                         if category == "static_tab":
+                            dog_stats_api.increment(
+                                DEPRECATION_VSCOMPAT_EVENT,
+                                tags=(
+                                    "location:xml_load_extra_content_static_tab",
+                                    u"course_dir:{}".format(course_dir),
+                                )
+                            )
+
                             tab = CourseTabList.get_tab_by_slug(tab_list=course_descriptor.tabs, url_slug=slug)
                             if tab:
                                 module.display_name = tab.name

common/lib/xmodule/xmodule/template_module.py

-from xmodule.x_module import XModule
+"""
+Template module
+"""
+from xmodule.x_module import XModule, DEPRECATION_VSCOMPAT_EVENT
 from xmodule.raw_module import RawDescriptor
 from lxml import etree
 from mako.template import Template
+import dogstats_wrapper as dog_stats_api
 
 
 class CustomTagModule(XModule):
@@ -42,6 +46,11 @@ class CustomTagDescriptor(RawDescriptor):
             template_name = xmltree.attrib['impl']
         else:
             # VS[compat]  backwards compatibility with old nested customtag structure
+            dog_stats_api.increment(
+                DEPRECATION_VSCOMPAT_EVENT,
+                tags=["location:customtag_descriptor_render_template"]
+            )
+
             child_impl = xmltree.find('impl')
             if child_impl is not None:
                 template_name = child_impl.text

common/lib/xmodule/xmodule/x_module.py

@@ -38,6 +38,9 @@ log = logging.getLogger(__name__)
 
 XMODULE_METRIC_NAME = 'edxapp.xmodule'
 
+# Stats event sent to DataDog in order to determine if old XML parsing can be deprecated.
+DEPRECATION_VSCOMPAT_EVENT = 'deprecation.vscompat'
+
 # xblock view names
 
 # This is the view that will be rendered to display the XBlock in the LMS.
@@ -860,6 +863,11 @@ class XModuleDescriptor(XModuleMixin, HTMLSnippet, ResourceTemplates, XBlock):
     @classmethod
     def _translate(cls, key):
         'VS[compat]'
+        if key in cls.metadata_translations:
+            dog_stats_api.increment(
+                DEPRECATION_VSCOMPAT_EVENT,
+                tags=["location:xmodule_descriptor_translate"]
+            )
         return cls.metadata_translations.get(key, key)
 
     # ================================= XML PARSING ============================

common/lib/xmodule/xmodule/xml_module.py

@@ -6,10 +6,12 @@ import sys
 from lxml import etree
 
 from xblock.fields import Dict, Scope, ScopeIds
-from xmodule.x_module import XModuleDescriptor
+from xblock.runtime import KvsFieldData
+from xmodule.x_module import XModuleDescriptor, DEPRECATION_VSCOMPAT_EVENT
 from xmodule.modulestore.inheritance import own_metadata, InheritanceKeyValueStore
 from xmodule.modulestore import EdxJSONEncoder
-from xblock.runtime import KvsFieldData
+
+import dogstats_wrapper as dog_stats_api
 
 log = logging.getLogger(__name__)
 
@@ -201,6 +203,11 @@ class XmlDescriptor(XModuleDescriptor):
             definition_xml = copy.deepcopy(xml_object)
             filepath = ''
         else:
+            dog_stats_api.increment(
+                DEPRECATION_VSCOMPAT_EVENT,
+                tags=["location:xmlparser_util_mixin_load_definition_filename"]
+            )
+
             filepath = cls._format_filepath(xml_object.tag, filename)
 
             # VS[compat]
@@ -209,6 +216,11 @@ class XmlDescriptor(XModuleDescriptor):
             # again in the correct format.  This should go away once the CMS is
             # online and has imported all current (fall 2012) courses from xml
             if not system.resources_fs.exists(filepath) and hasattr(cls, 'backcompat_paths'):
+                dog_stats_api.increment(
+                    DEPRECATION_VSCOMPAT_EVENT,
+                    tags=["location:xmlparser_util_mixin_load_definition_backcompat"]
+                )
+
                 candidates = cls.backcompat_paths(filepath)
                 for candidate in candidates:
                     if system.resources_fs.exists(candidate):
@@ -244,6 +256,14 @@ class XmlDescriptor(XModuleDescriptor):
             attr = cls._translate(attr)
 
             if attr in cls.metadata_to_strip:
+                if attr in ('course', 'org', 'url_name', 'filename'):
+                    dog_stats_api.increment(
+                        DEPRECATION_VSCOMPAT_EVENT,
+                        tags=(
+                            "location:xmlparser_util_mixin_load_metadata",
+                            "metadata:{}".format(attr),
+                        )
+                    )
                 # don't load these
                 continue
 
@@ -293,8 +313,12 @@ class XmlDescriptor(XModuleDescriptor):
             definition_xml = cls.load_file(filepath, system.resources_fs, def_id)
             system.parse_asides(definition_xml, def_id, usage_id, id_generator)
         else:
-            definition_xml = xml_object
             filepath = None
+            definition_xml = xml_object
+            dog_stats_api.increment(
+                DEPRECATION_VSCOMPAT_EVENT,
+                tags=["location:xmlparser_util_mixin_parse_xml"]
+            )
 
         definition, children = cls.load_definition(definition_xml, system, def_id, id_generator)  # note this removes metadata
 

lms/djangoapps/courseware/access.py

-"""This file contains (or should), all access control logic for the courseware.
+"""
+This file contains (or should), all access control logic for the courseware.
 Ideally, it will be the only place that needs to know about any special settings
-like DISABLE_START_DATES"""
+like DISABLE_START_DATES
+"""
 import logging
 from datetime import datetime, timedelta
 import pytz
 
 from django.conf import settings
 from django.contrib.auth.models import AnonymousUser
+from django.utils.timezone import UTC
+
+from opaque_keys.edx.keys import CourseKey, UsageKey
+from xblock.core import XBlock
 
 from xmodule.course_module import (
     CourseDescriptor, CATALOG_VISIBILITY_CATALOG_AND_ABOUT,
     CATALOG_VISIBILITY_ABOUT)
 from xmodule.error_module import ErrorDescriptor
-from xmodule.x_module import XModule
+from xmodule.x_module import XModule, DEPRECATION_VSCOMPAT_EVENT
 from xmodule.split_test_module import get_split_user_partitions
-
-from xblock.core import XBlock
 from xmodule.partitions.partitions import NoSuchUserPartitionError, NoSuchUserPartitionGroupError
 
 from external_auth.models import ExternalAuthMap
 from courseware.masquerade import get_masquerade_role, is_masquerading_as_student
-from django.utils.timezone import UTC
 from student import auth
 from student.roles import (
     GlobalStaff, CourseStaffRole, CourseInstructorRole,
     OrgStaffRole, OrgInstructorRole, CourseBetaTesterRole
 )
 from student.models import CourseEnrollment, CourseEnrollmentAllowed
-from opaque_keys.edx.keys import CourseKey, UsageKey
 from util.milestones_helpers import get_pre_requisite_courses_not_completed
+import dogstats_wrapper as dog_stats_api
+
 DEBUG_ACCESS = False
 
 log = logging.getLogger(__name__)
@@ -232,6 +236,14 @@ def _has_access_course_desc(user, action, course):
         # properly configured enrollment_start times (if course should be
         # staff-only, set enrollment_start far in the future.)
         if settings.FEATURES.get('ACCESS_REQUIRE_STAFF_FOR_COURSE'):
+            dog_stats_api.increment(
+                DEPRECATION_VSCOMPAT_EVENT,
+                tags=(
+                    "location:has_access_course_desc_see_exists",
+                    u"course:{}".format(course),
+                )
+            )
+
             # if this feature is on, only allow courses that have ispublic set to be
             # seen by non-staff
             if course.ispublic: