Initial integration tests!

* new env file to run integration tests with mongo backend
  - needs askbot to get db in the right state
  - needs wiki to load pages successfully
  - not using migrations to make it run faster
* import test courses, check that each page loads without erroring.
* fix NonePointerException in index view to make tests pass
* misc formatting cleanups

common/lib/xmodule/xmodule/modulestore/mongo.py

@@ -342,7 +342,7 @@ class MongoModuleStore(ModuleStore):
             while len(queue) > 0:
                 (loc, path) = queue.pop()  # Takes from the end
                 loc = Location(loc)
-                print 'Processing loc={0}, path={1}'.format(loc, path)
+                # print 'Processing loc={0}, path={1}'.format(loc, path)
                 if loc.category == "course":
                     if course_name is None or course_name == loc.name:
                         # Found it!

common/lib/xmodule/xmodule/modulestore/xml_importer.py

@@ -5,19 +5,20 @@ from .xml import XMLModuleStore
 log = logging.getLogger(__name__)
 
 
-def import_from_xml(store, data_dir, course_dirs=None):
+def import_from_xml(store, data_dir, course_dirs=None, eager=True,
+                    default_class='xmodule.raw_module.RawDescriptor'):
     """
     Import the specified xml data_dir into the "store" modulestore,
     using org and course as the location org and course.
 
     course_dirs: If specified, the list of course_dirs to load. Otherwise, load
     all course dirs
-    
+
     """
     module_store = XMLModuleStore(
         data_dir,
-        default_class='xmodule.raw_module.RawDescriptor',
-        eager=True,
+        default_class=default_class,
+        eager=eager,
         course_dirs=course_dirs
     )
     for module in module_store.modules.itervalues():

lms/djangoapps/courseware/tests/__init__.py

+

lms/djangoapps/courseware/tests/tests.py

+import json
+from django.test import TestCase
+from django.test.client import Client
+from mock import patch, Mock
+from override_settings import override_settings
+from django.conf import settings
+from django.core.urlresolvers import reverse
+from path import path
+
+from student.models import Registration
+from django.contrib.auth.models import User
+
+from xmodule.modulestore.django import modulestore
+import xmodule.modulestore.django
+from xmodule.modulestore import Location
+from xmodule.modulestore.xml_importer import import_from_xml
+import copy
+
+
+def parse_json(response):
+    """Parse response, which is assumed to be json"""
+    return json.loads(response.content)
+
+
+def user(email):
+    '''look up a user by email'''
+    return User.objects.get(email=email)
+
+
+def registration(email):
+    '''look up registration object by email'''
+    return Registration.objects.get(user__email=email)
+
+
+TEST_DATA_MODULESTORE = copy.deepcopy(settings.MODULESTORE)
+# TODO (vshnayder): test the real courses
+TEST_DATA_DIR = 'common/test/data'
+TEST_DATA_MODULESTORE['default']['OPTIONS']['fs_root'] = path(TEST_DATA_DIR)
+
+@override_settings(MODULESTORE=TEST_DATA_MODULESTORE)
+class IntegrationTestCase(TestCase):
+    '''Check that all objects in all accessible courses will load properly'''
+
+    def setUp(self):
+        email = 'view@test.com'
+        password = 'foo'
+        self.create_account('viewtest', email, password)
+        self.activate_user(email)
+        self.login(email, password)
+        xmodule.modulestore.django._MODULESTORES = {}
+        xmodule.modulestore.django.modulestore().collection.drop()
+
+
+    # ============ User creation and login ==============
+
+    def _login(self, email, pw):
+        '''Login.  View should always return 200.  The success/fail is in the
+        returned json'''
+        resp = self.client.post(reverse('login'),
+                                {'email': email, 'password': pw})
+        self.assertEqual(resp.status_code, 200)
+        return resp
+
+    def login(self, email, pw):
+        '''Login, check that it worked.'''
+        resp = self._login(email, pw)
+        data = parse_json(resp)
+        self.assertTrue(data['success'])
+        return resp
+
+    def _create_account(self, username, email, pw):
+        '''Try to create an account.  No error checking'''
+        resp = self.client.post('/create_account', {
+            'username': username,
+            'email': email,
+            'password': pw,
+            'name': 'Fred Weasley',
+            'terms_of_service': 'true',
+            'honor_code': 'true',
+        })
+        return resp
+
+    def create_account(self, username, email, pw):
+        '''Create the account and check that it worked'''
+        resp = self._create_account(username, email, pw)
+        self.assertEqual(resp.status_code, 200)
+        data = parse_json(resp)
+        self.assertEqual(data['success'], True)
+
+        # Check both that the user is created, and inactive
+        self.assertFalse(user(email).is_active)
+
+        return resp
+
+    def _activate_user(self, email):
+        '''Look up the activation key for the user, then hit the activate view.
+        No error checking'''
+        activation_key = registration(email).activation_key
+
+        # and now we try to activate
+        resp = self.client.get(reverse('activate', kwargs={'key': activation_key}))
+        return resp
+
+    def activate_user(self, email):
+        resp = self._activate_user(email)
+        self.assertEqual(resp.status_code, 200)
+        # Now make sure that the user is now actually activated
+        self.assertTrue(user(email).is_active)
+
+    # ============ Page loading ==============
+
+    def check_pages_load(self, test_course_name):
+        import_from_xml(modulestore(), TEST_DATA_DIR, [test_course_name])
+
+        n = 0
+        num_bad = 0
+        all_ok = True
+        for descriptor in modulestore().get_items(
+                Location(None, None, None, None, None)):
+            n += 1
+            print "Checking ", descriptor.location.url()
+            #print descriptor.__class__, descriptor.location
+            resp = self.client.get(reverse('jump_to',
+                                   kwargs={'location': descriptor.location.url()}))
+            msg = str(resp.status_code)
+
+            if resp.status_code != 200:
+                msg = "ERROR " + msg
+                all_ok = False
+                num_bad += 1
+            print msg
+            self.assertTrue(all_ok)
+
+        print "{0}/{1} good".format(n - num_bad, n)
+
+
+    def test_toy_course_loads(self):
+        self.check_pages_load('toy')
+
+    def test_full_course_loads(self):
+        self.check_pages_load('full')
+
+
+    # ========= TODO: check ajax interaction here too?

lms/djangoapps/courseware/views.py

@@ -200,10 +200,19 @@ def index(request, course_id, chapter=None, section=None,
     if look_for_module:
         # TODO (cpennington): Pass the right course in here
 
-        section = get_section(course, chapter, section)
-        student_module_cache = StudentModuleCache(request.user, section)
-        module, _, _, _ = get_module(request.user, request, section.location, student_module_cache)
-        context['content'] = module.get_html()
+        section_descriptor = get_section(course, chapter, section)
+        if section_descriptor is not None:
+            student_module_cache = StudentModuleCache(request.user,
+                                                      section_descriptor)
+            module, _, _, _ = get_module(request.user, request,
+                                         section_descriptor.location,
+                                         student_module_cache)
+            context['content'] = module.get_html()
+        else:
+            log.warning("Couldn't find a section descriptor for course_id '{0}',"
+                        "chapter '{1}', section '{2}'".format(
+                        course_id, chapter, section))
+
 
     result = render_to_response('courseware.html', context)
     return result

lms/envs/test.py

@@ -22,7 +22,8 @@ INSTALLED_APPS = [
 # Nose Test Runner
 INSTALLED_APPS += ['django_nose']
 NOSE_ARGS = ['--cover-erase', '--with-xunit', '--with-xcoverage', '--cover-html',
-             '--cover-inclusive', '--cover-html-dir', os.environ.get('NOSE_COVER_HTML_DIR', 'cover_html')]
+             '--cover-inclusive', '--cover-html-dir',
+             os.environ.get('NOSE_COVER_HTML_DIR', 'cover_html')]
 for app in os.listdir(PROJECT_ROOT / 'djangoapps'):
     NOSE_ARGS += ['--cover-package', app]
 TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'
@@ -30,25 +31,26 @@ TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'
 # Local Directories
 TEST_ROOT = path("test_root")
 # Want static files in the same dir for running on jenkins.
-STATIC_ROOT = TEST_ROOT / "staticfiles" 
+STATIC_ROOT = TEST_ROOT / "staticfiles"
 
 COURSES_ROOT = TEST_ROOT / "data"
 DATA_DIR = COURSES_ROOT
 MAKO_TEMPLATES['course'] = [DATA_DIR]
 MAKO_TEMPLATES['sections'] = [DATA_DIR / 'sections']
 MAKO_TEMPLATES['custom_tags'] = [DATA_DIR / 'custom_tags']
-MAKO_TEMPLATES['main'] = [PROJECT_ROOT / 'templates', 
+MAKO_TEMPLATES['main'] = [PROJECT_ROOT / 'templates',
                           DATA_DIR / 'info',
                           DATA_DIR / 'problems']
 
-LOGGING = get_logger_config(TEST_ROOT / "log", 
+LOGGING = get_logger_config(TEST_ROOT / "log",
                             logging_env="dev",
                             tracking_filename="tracking.log",
                             debug=True)
 
 COMMON_TEST_DATA_ROOT = COMMON_ROOT / "test" / "data"
 
-# TODO (cpennington): We need to figure out how envs/test.py can inject things into common.py so that we don't have to repeat this sort of thing
+# TODO (cpennington): We need to figure out how envs/test.py can inject things
+# into common.py so that we don't have to repeat this sort of thing
 STATICFILES_DIRS = [
     COMMON_ROOT / "static",
     PROJECT_ROOT / "static",
@@ -67,7 +69,7 @@ DATABASES = {
 }
 
 CACHES = {
-    # This is the cache used for most things. Askbot will not work without a 
+    # This is the cache used for most things. Askbot will not work without a
     # functioning cache -- it relies on caching to load its settings in places.
     # In staging/prod envs, the sessions also live here.
     'default': {

lms/envs/test_mongo.py

+"""
+This config file runs the test environment, but with mongo as the datastore
+"""
+from .common import *
+
+from .logsettings import get_logger_config
+import os
+from path import path
+
+# can't testing start dates with this True, but on the other hand,
+# can test everything else :)
+MITX_FEATURES['DISABLE_START_DATES'] = True
+
+WIKI_ENABLED = True
+
+GITHUB_REPO_ROOT = ENV_ROOT / "data"
+
+MODULESTORE = {
+    'default': {
+        'ENGINE': 'xmodule.modulestore.mongo.MongoModuleStore',
+        'OPTIONS': {
+            'default_class': 'xmodule.raw_module.RawDescriptor',
+            'host': 'localhost',
+            'db': 'xmodule',
+            'collection': 'modulestore',
+            'fs_root': GITHUB_REPO_ROOT,
+        }
+    }
+}
+
+
+# Nose Test Runner
+INSTALLED_APPS += ('django_nose',)
+NOSE_ARGS = ['--cover-erase', '--with-xunit', '--with-xcoverage', '--cover-html',
+             '--cover-inclusive', '--cover-html-dir',
+             os.environ.get('NOSE_COVER_HTML_DIR', 'cover_html')]
+for app in os.listdir(PROJECT_ROOT / 'djangoapps'):
+    NOSE_ARGS += ['--cover-package', app]
+TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'
+
+
+TEST_ROOT = path("test_root")
+# Want static files in the same dir for running on jenkins.
+STATIC_ROOT = TEST_ROOT / "staticfiles"
+
+
+
+LOGGING = get_logger_config(TEST_ROOT / "log",
+                            logging_env="dev",
+                            tracking_filename="tracking.log",
+                            debug=True)
+
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.sqlite3',
+        'NAME': PROJECT_ROOT / "db" / "mitx.db",
+    }
+}
+
+CACHES = {
+    # This is the cache used for most things. Askbot will not work without a
+    # functioning cache -- it relies on caching to load its settings in places.
+    # In staging/prod envs, the sessions also live here.
+    'default': {
+        'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
+        'LOCATION': 'mitx_loc_mem_cache',
+        'KEY_FUNCTION': 'util.memcache.safe_key',
+    },
+
+    # The general cache is what you get if you use our util.cache. It's used for
+    # things like caching the course.xml file for different A/B test groups.
+    # We set it to be a DummyCache to force reloading of course.xml in dev.
+    # In staging environments, we would grab VERSION from data uploaded by the
+    # push process.
+    'general': {
+        'BACKEND': 'django.core.cache.backends.dummy.DummyCache',
+        'KEY_PREFIX': 'general',
+        'VERSION': 4,
+        'KEY_FUNCTION': 'util.memcache.safe_key',
+    }
+}
+
+# Dummy secret key for dev
+SECRET_KEY = '85920908f28904ed733fe576320db18cabd7b6cd'
+
+# Makes the tests run much faster...
+SOUTH_TESTS_MIGRATE = False # To disable migrations and use syncdb instead
+
+############################ FILE UPLOADS (ASKBOT) #############################
+DEFAULT_FILE_STORAGE = 'django.core.files.storage.FileSystemStorage'
+MEDIA_ROOT = TEST_ROOT / "uploads"
+MEDIA_URL = "/static/uploads/"
+STATICFILES_DIRS.append(("uploads", MEDIA_ROOT))
+
+new_staticfiles_dirs = []
+# Strip out any static files that aren't in the repository root
+# so that the tests can run with only the mitx directory checked out
+for static_dir in STATICFILES_DIRS:
+    # Handle both tuples and non-tuple directory definitions
+    try:
+        _, data_dir = static_dir
+    except ValueError:
+        data_dir = static_dir
+
+    if data_dir.startswith(REPO_ROOT):
+        new_staticfiles_dirs.append(static_dir)
+STATICFILES_DIRS = new_staticfiles_dirs
+
+FILE_UPLOAD_TEMP_DIR = PROJECT_ROOT / "uploads"
+FILE_UPLOAD_HANDLERS = (
+    'django.core.files.uploadhandler.MemoryFileUploadHandler',
+    'django.core.files.uploadhandler.TemporaryFileUploadHandler',
+)

lms/urls.py

@@ -13,23 +13,23 @@ if settings.DEBUG:
 urlpatterns = ('',
     url(r'^$', 'student.views.index', name="root"), # Main marketing page, or redirect to courseware
     url(r'^dashboard$', 'student.views.dashboard', name="dashboard"),
-    
+
     url(r'^change_email$', 'student.views.change_email_request'),
     url(r'^email_confirm/(?P<key>[^/]*)$', 'student.views.confirm_email_change'),
     url(r'^change_name$', 'student.views.change_name_request'),
     url(r'^accept_name_change$', 'student.views.accept_name_change'),
     url(r'^reject_name_change$', 'student.views.reject_name_change'),
     url(r'^pending_name_changes$', 'student.views.pending_name_changes'),
-    
+
     url(r'^event$', 'track.views.user_track'),
     url(r'^t/(?P<template>[^/]*)$', 'static_template_view.views.index'), # TODO: Is this used anymore? What is STATIC_GRAB?
-    
-    url(r'^login$', 'student.views.login_user'),
+
+    url(r'^login$', 'student.views.login_user', name="login"),
     url(r'^login/(?P<error>[^/]*)$', 'student.views.login_user'),
     url(r'^logout$', 'student.views.logout_user', name='logout'),
     url(r'^create_account$', 'student.views.create_account'),
-    url(r'^activate/(?P<key>[^/]*)$', 'student.views.activate_account'),
-    
+    url(r'^activate/(?P<key>[^/]*)$', 'student.views.activate_account', name="activate"),
+
     url(r'^password_reset/$', 'student.views.password_reset', name='password_reset'),
     ## Obsolete Django views for password resets
     ## TODO: Replace with Mako-ized views
@@ -44,48 +44,48 @@ urlpatterns = ('',
         name='auth_password_reset_complete'),
     url(r'^password_reset_done/$', django.contrib.auth.views.password_reset_done,
         name='auth_password_reset_done'),
-    
+
     url(r'^heartbeat$', include('heartbeat.urls')),
-    
+
     url(r'^university_profile/(?P<org_id>[^/]+)$', 'courseware.views.university_profile', name="university_profile"),
-    
+
     #Semi-static views (these need to be rendered and have the login bar, but don't change)
-    url(r'^404$', 'static_template_view.views.render', 
+    url(r'^404$', 'static_template_view.views.render',
         {'template': '404.html'}, name="404"),
-    url(r'^about$', 'static_template_view.views.render', 
+    url(r'^about$', 'static_template_view.views.render',
         {'template': 'about.html'}, name="about_edx"),
-    url(r'^jobs$', 'static_template_view.views.render', 
+    url(r'^jobs$', 'static_template_view.views.render',
         {'template': 'jobs.html'}, name="jobs"),
-    url(r'^contact$', 'static_template_view.views.render', 
+    url(r'^contact$', 'static_template_view.views.render',
         {'template': 'contact.html'}, name="contact"),
     url(r'^press$', 'student.views.press', name="press"),
-    url(r'^faq$', 'static_template_view.views.render', 
+    url(r'^faq$', 'static_template_view.views.render',
         {'template': 'faq.html'}, name="faq_edx"),
-    url(r'^help$', 'static_template_view.views.render', 
+    url(r'^help$', 'static_template_view.views.render',
         {'template': 'help.html'}, name="help_edx"),
 
-    url(r'^tos$', 'static_template_view.views.render', 
+    url(r'^tos$', 'static_template_view.views.render',
         {'template': 'tos.html'}, name="tos"),
-    url(r'^privacy$', 'static_template_view.views.render', 
+    url(r'^privacy$', 'static_template_view.views.render',
         {'template': 'privacy.html'}, name="privacy_edx"),
     # TODO: (bridger) The copyright has been removed until it is updated for edX
-    # url(r'^copyright$', 'static_template_view.views.render', 
+    # url(r'^copyright$', 'static_template_view.views.render',
     #     {'template': 'copyright.html'}, name="copyright"),
-    url(r'^honor$', 'static_template_view.views.render', 
+    url(r'^honor$', 'static_template_view.views.render',
         {'template': 'honor.html'}, name="honor"),
-        
-    #Press releases    
-    url(r'^press/mit-and-harvard-announce-edx$', 'static_template_view.views.render', 
+
+    #Press releases
+    url(r'^press/mit-and-harvard-announce-edx$', 'static_template_view.views.render',
         {'template': 'press_releases/MIT_and_Harvard_announce_edX.html'}, name="press/mit-and-harvard-announce-edx"),
-    url(r'^press/uc-berkeley-joins-edx$', 'static_template_view.views.render', 
+    url(r'^press/uc-berkeley-joins-edx$', 'static_template_view.views.render',
         {'template': 'press_releases/UC_Berkeley_joins_edX.html'}, name="press/uc-berkeley-joins-edx"),
     # Should this always update to point to the latest press release?
-    (r'^pressrelease$', 'django.views.generic.simple.redirect_to', {'url': '/press/uc-berkeley-joins-edx'}), 
-    
-    
-    
+    (r'^pressrelease$', 'django.views.generic.simple.redirect_to', {'url': '/press/uc-berkeley-joins-edx'}),
+
+
+
     (r'^favicon\.ico$', 'django.views.generic.simple.redirect_to', {'url': '/static/images/favicon.ico'}),
-        
+
     # TODO: These urls no longer work. They need to be updated before they are re-enabled
     # url(r'^send_feedback$', 'util.views.send_feedback'),
     # url(r'^reactivate/(?P<key>[^/]*)$', 'student.views.reactivation_email'),
@@ -97,46 +97,46 @@ if settings.PERFSTATS:
 if settings.COURSEWARE_ENABLED:
     urlpatterns += (
         url(r'^masquerade/', include('masquerade.urls')),
-        url(r'^jumpto/(?P<location>.*)$', 'courseware.views.jump_to'),
-        
+        url(r'^jump_to/(?P<location>.*)$', 'courseware.views.jump_to', name="jump_to"),
+
         url(r'^modx/(?P<id>.*?)/(?P<dispatch>[^/]*)$', 'courseware.module_render.modx_dispatch'), #reset_problem'),
         url(r'^xqueue/(?P<userid>[^/]*)/(?P<id>.*?)/(?P<dispatch>[^/]*)$', 'courseware.module_render.xqueue_callback'),
         url(r'^change_setting$', 'student.views.change_setting'),
-        
+
         # TODO: These views need to be updated before they work
         # url(r'^calculate$', 'util.views.calculate'),
         # url(r'^gradebook$', 'courseware.views.gradebook'),
         # TODO: We should probably remove the circuit package. I believe it was only used in the old way of saving wiki circuits for the wiki
         # url(r'^edit_circuit/(?P<circuit>[^/]*)$', 'circuit.views.edit_circuit'),
         # url(r'^save_circuit/(?P<circuit>[^/]*)$', 'circuit.views.save_circuit'),
-                
-        url(r'^courses/?$', 'courseware.views.courses', name="courses"),     
-        url(r'^change_enrollment$', 
+
+        url(r'^courses/?$', 'courseware.views.courses', name="courses"),
+        url(r'^change_enrollment$',
             'student.views.change_enrollment_view', name="change_enrollment"),
-           
+
         #About the course
-        url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/about$', 
+        url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/about$',
             'courseware.views.course_about', name="about_course"),
-        
+
         #Inside the course
-        url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/info$', 
+        url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/info$',
             'courseware.views.course_info', name="info"),
-        url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/book$', 
+        url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/book$',
             'staticbook.views.index', name="book"),
-        url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/book/(?P<page>[^/]*)$', 
+        url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/book/(?P<page>[^/]*)$',
             'staticbook.views.index'),
-        url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/book-shifted/(?P<page>[^/]*)$', 
+        url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/book-shifted/(?P<page>[^/]*)$',
             'staticbook.views.index_shifted'),
-        url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/courseware/?$', 
+        url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/courseware/?$',
             'courseware.views.index', name="courseware"),
-        url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/courseware/(?P<chapter>[^/]*)/(?P<section>[^/]*)/$', 
+        url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/courseware/(?P<chapter>[^/]*)/(?P<section>[^/]*)/$',
             'courseware.views.index', name="courseware_section"),
-        url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/profile$', 
+        url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/profile$',
             'courseware.views.profile', name="profile"),
-        url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/profile/(?P<student_id>[^/]*)/$', 
+        url(r'^courses/(?P<course_id>[^/]+/[^/]+/[^/]+)/profile/(?P<student_id>[^/]*)/$',
             'courseware.views.profile'),
     )
-    
+
     # Multicourse wiki
 if settings.WIKI_ENABLED:
     urlpatterns += (
@@ -164,9 +164,9 @@ urlpatterns = patterns(*urlpatterns)
 
 if settings.DEBUG:
     urlpatterns += static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)
-    
-    
-#Custom error pages 
+
+
+#Custom error pages
 handler404 = 'static_template_view.views.render_404'
 handler500 = 'static_template_view.views.render_500'