add tests of static_asset_path and importing with no static;

fix (again) static_replace to work with static_asset_path

cms/djangoapps/contentstore/tests/test_import_nostatic.py

+#pylint: disable=E1101
+
+import json
+import shutil
+import sys
+import mock
+from django.test.client import Client
+from django.test.utils import override_settings
+from django.conf import settings
+from django.core.urlresolvers import reverse
+from path import path
+from tempdir import mkdtemp_clean
+from fs.osfs import OSFS
+import copy
+from json import loads
+from datetime import timedelta
+
+from django.contrib.auth.models import User
+from django.dispatch import Signal
+from contentstore.utils import get_modulestore
+from contentstore.tests.utils import parse_json
+
+from auth.authz import add_user_to_creator_group
+
+from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
+from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory
+
+from xmodule.modulestore import Location, mongo
+from xmodule.modulestore.store_utilities import clone_course
+from xmodule.modulestore.store_utilities import delete_course
+from xmodule.modulestore.django import modulestore
+from xmodule.contentstore.django import contentstore, _CONTENTSTORE
+from xmodule.modulestore.xml_exporter import export_to_xml
+from xmodule.modulestore.xml_importer import import_from_xml, perform_xlint
+from xmodule.modulestore.inheritance import own_metadata
+from xmodule.contentstore.content import StaticContent
+from xmodule.contentstore.utils import restore_asset_from_trashcan, empty_asset_trashcan
+
+from xmodule.capa_module import CapaDescriptor
+from xmodule.course_module import CourseDescriptor
+from xmodule.seq_module import SequenceDescriptor
+from xmodule.modulestore.exceptions import ItemNotFoundError
+
+from contentstore.views.component import ADVANCED_COMPONENT_TYPES
+from xmodule.exceptions import NotFoundError
+
+from django_comment_common.utils import are_permissions_roles_seeded
+from xmodule.exceptions import InvalidVersionError
+import datetime
+from pytz import UTC
+from uuid import uuid4
+from pymongo import MongoClient
+from student.models import CourseEnrollment
+
+TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE)
+TEST_DATA_CONTENTSTORE['OPTIONS']['db'] = 'test_xcontent_%s' % uuid4().hex
+
+
+class MongoCollectionFindWrapper(object):
+    def __init__(self, original):
+        self.original = original
+        self.counter = 0
+
+    def find(self, query, *args, **kwargs):
+        self.counter = self.counter + 1
+        return self.original(query, *args, **kwargs)
+
+
+@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE)
+class ContentStoreImportNoStaticTest(ModuleStoreTestCase):
+    """
+    Tests that rely on the toy and test_import_course courses.
+    TODO: refactor using CourseFactory so they do not.
+    """
+    def setUp(self):
+
+        settings.MODULESTORE['default']['OPTIONS']['fs_root'] = path('common/test/data')
+        settings.MODULESTORE['direct']['OPTIONS']['fs_root'] = path('common/test/data')
+        uname = 'testuser'
+        email = 'test+courses@edx.org'
+        password = 'foo'
+
+        # Create the use so we can log them in.
+        self.user = User.objects.create_user(uname, email, password)
+
+        # Note that we do not actually need to do anything
+        # for registration if we directly mark them active.
+        self.user.is_active = True
+        # Staff has access to view all courses
+        self.user.is_staff = True
+
+        # Save the data that we've just changed to the db.
+        self.user.save()
+
+        self.client = Client()
+        self.client.login(username=uname, password=password)
+
+    def load_test_import_course(self):
+        '''
+        Load the standard course used to test imports (for do_import_static=False behavior).  
+        '''
+        content_store = contentstore()
+        module_store = modulestore('direct')
+        import_from_xml(module_store, 'common/test/data/', ['test_import_course'], static_content_store=content_store, do_import_static=False, verbose=True)
+        course_location = CourseDescriptor.id_to_location('edX/test_import_course/2012_Fall')
+        course = module_store.get_item(course_location)
+        self.assertIsNotNone(course)
+
+        return module_store, content_store, course, course_location
+
+
+    def test_static_import(self):
+        '''
+        Stuff in static_import should always be imported into contentstore
+        '''
+        module_store, content_store, course, course_location = self.load_test_import_course()
+
+        # make sure we have ONE asset in our contentstore ("should_be_imported.html")
+        all_assets = content_store.get_all_content_for_course(course_location)
+        print "len(all_assets)=%d" % len(all_assets)
+        self.assertEqual(len(all_assets), 1)
+
+        content = None
+        try:
+            location = StaticContent.get_location_from_path('/c4x/edX/test_import_course/asset/should_be_imported.html')
+            content = content_store.find(location)
+        except NotFoundError:
+            pass
+
+        self.assertIsNotNone(content)
+        
+        # make sure course.lms.static_asset_path is correct
+        print "static_asset_path = {0}".format(course.lms.static_asset_path)
+        self.assertEqual(course.lms.static_asset_path, 'test_import_course')
+
+
+    def test_asset_import_nostatic(self):
+        '''
+        This test validates that an image asset is NOT imported when do_import_static=False
+        '''
+        content_store = contentstore()
+
+        module_store = modulestore('direct')
+        import_from_xml(module_store, 'common/test/data/', ['toy'], static_content_store=content_store, do_import_static=False, verbose=True)
+
+        course_location = CourseDescriptor.id_to_location('edX/toy/2012_Fall')
+        course = module_store.get_item(course_location)
+
+        # make sure we have NO assets in our contentstore
+        all_assets = content_store.get_all_content_for_course(course_location)
+        print "len(all_assets)=%d" % len(all_assets)
+        self.assertEqual(len(all_assets), 0)
+
+
+    def test_no_static_link_rewrites_on_import(self):
+        module_store = modulestore('direct')
+        import_from_xml(module_store, 'common/test/data/', ['toy'], do_import_static=False, verbose=True)
+
+        handouts = module_store.get_item(Location(['i4x', 'edX', 'toy', 'course_info', 'handouts', None]))
+        self.assertIn('/static/', handouts.data)
+
+        handouts = module_store.get_item(Location(['i4x', 'edX', 'toy', 'html', 'toyhtml', None]))
+        self.assertIn('/static/', handouts.data)
+

common/djangoapps/static_replace/__init__.py

@@ -117,7 +117,7 @@ def replace_static_urls(text, data_directory, course_id=None, static_asset_path=
         if settings.DEBUG and finders.find(rest, True):
             return original
         # if we're running with a MongoBacked store course_namespace is not None, then use studio style urls
-        elif course_id and modulestore().get_modulestore_type(course_id) != XML_MODULESTORE_TYPE:
+        elif (not static_asset_path) and course_id and modulestore().get_modulestore_type(course_id) != XML_MODULESTORE_TYPE:
             # first look in the static file pipeline and see if we are trying to reference
             # a piece of static content which is in the mitx repo (e.g. JS associated with an xmodule)
             if staticfiles_storage.exists(rest):

common/test/data/test_import_course/about/end_date.html

+TBD
\ No newline at end of file

common/test/data/test_import_course/chapter/vertical_container.xml

+<sequential>
+    <sequential filename='vertical_sequential' slug='vertical_sequential' />
+</sequential>
\ No newline at end of file

common/test/data/test_import_course/course.xml

+<course org="edX" course="test_import_course" url_name="2012_Fall"/>
\ No newline at end of file

common/test/data/test_import_course/course/2012_Fall.xml

+<course>
+    <textbook title="Textbook" book_url="https://s3.amazonaws.com/edx-textbooks/guttag_computation_v3/"/>
+  <chapter url_name="Overview">
+    <videosequence url_name="Toy_Videos">
+      <html url_name="secret:toylab"/>
+      <html url_name="toyjumpto"/>
+      <html url_name="toyhtml"/>
+      <html url_name="nonportable"/>
+      <html url_name="nonportable_link"/>
+      <video url_name="Video_Resources" youtube_id_1_0="1bK-WdDi6Qw" display_name="Video Resources"/>
+    </videosequence>
+    <video url_name="Welcome" youtube_id_1_0="p2Q6BrNhdh8" display_name="Welcome"/>
+    <video url_name="video_123456789012" youtube_id_1_0="p2Q6BrNhdh8" display_name='Test Video'/>
+    <video url_name="video_4f66f493ac8f" youtube_id_1_0="p2Q6BrNhdh8"/>
+  </chapter>
+  <chapter url_name="secret:magic"/>
+  <chapter url_name="poll_test"/>
+  <chapter url_name="vertical_container"/>
+  <chapter url_name="handout_container"/>
+</course>

common/test/data/test_import_course/info/handouts.html

+<a href='/static/handouts/sample_handout.txt'>Sample</a>
\ No newline at end of file

common/test/data/test_import_course/policies/2012_Fall.json

+{
+    "course/2012_Fall": {
+        "graceperiod": "2 days 5 hours 59 minutes 59 seconds",
+        "start": "2015-07-17T12:00",
+        "display_name": "Toy Course",
+        "graded": "true",
+	"tabs": [
+          {"type": "courseware"},
+          {"type": "course_info", "name": "Course Info"},
+          {"type": "static_tab", "url_slug": "syllabus", "name": "Syllabus"},
+          {"type": "static_tab", "url_slug": "resources", "name": "Resources"},
+          {"type": "discussion", "name": "Discussion"},
+          {"type": "wiki", "name": "Wiki"},
+          {"type": "progress", "name": "Progress"}
+        ]
+    },
+    "chapter/Overview": {
+        "display_name": "Overview"
+    },
+    "videosequence/Toy_Videos": {
+        "display_name": "Toy Videos",
+        "format": "Lecture Sequence"
+    },
+    "html/secret:toylab": {
+        "display_name": "Toy lab"
+    },
+    "video/Video_Resources": {
+        "display_name": "Video Resources"
+    },
+    "video/Welcome": {
+        "display_name": "Welcome"
+    }
+}

common/test/data/test_import_course/sequential/vertical_sequential.xml

+<sequential>
+    <vertical filename="vertical_test" slug="vertical_test" />
+    <html slug="unicode">…</html>
+</sequential>
\ No newline at end of file

common/test/data/test_import_course/static_import/should_be_imported.html

+<p>this file should be in the contentstore</p>

common/test/data/test_import_course/vertical/vertical_test.xml

+<sequential>
+        <video display_name="default" youtube_id_0_75="JMD_ifUUfsU" youtube_id_1_0="OEoXaMPEzfM" youtube_id_1_25="AKqURZnYqpk" youtube_id_1_5="DYpADpL7jAY" name="sample_video"/>
+        <video url_name="separate_file_video"/>
+        <poll_question name="T1_changemind_poll_foo_2" display_name="Change your answer" reset="false">
+        <p>Have you changed your mind?</p>
+        <answer id="yes">Yes</answer>
+        <answer id="no">No</answer>
+    </poll_question>
+</sequential>

common/test/data/test_import_course/video/separate_file_video.xml

+<video display_name="default" youtube_id_0_75="JMD_ifUUfsU" youtube_id_1_0="OEoXaMPEzfM" youtube_id_1_25="AKqURZnYqpk" youtube_id_1_5="DYpADpL7jAY" name="sample_video"/>

docs/internal/testing.md

@@ -136,6 +136,25 @@ To run a single nose test:
 
     nosetests common/lib/xmodule/xmodule/tests/test_stringify.py:test_stringify
 
+To run a single test and get stdout, with proper env config:
+
+    python manage.py cms --settings test test contentstore.tests.test_import_nostatic -s
+
+To run a single test and get stdout and get coverage:
+
+    python -m coverage run --rcfile=./common/lib/xmodule/.coveragerc which ./manage.py cms --settings test test --traceback --logging-clear-handlers --liveserver=localhost:8000-9000 contentstore.tests.test_import_nostatic -s # cms example 
+    python -m coverage run --rcfile=./lms/.coveragerc which ./manage.py lms --settings test test --traceback --logging-clear-handlers --liveserver=localhost:8000-9000  courseware.tests.test_module_render -s # lms example
+
+generate coverage report:
+
+    coverage report --rcfile=./common/lib/xmodule/.coveragerc 
+
+or to get html report:
+
+    coverage html --rcfile=./common/lib/xmodule/.coveragerc
+    
+then browse reports/common/lib/xmodule/cover/index.html    
+
 
 Very handy: if you uncomment the `pdb=1` line in `setup.cfg`, it will drop you into pdb on error.  This lets you go up and down the stack and see what the values of the variables are.  Check out [the pdb documentation](http://docs.python.org/library/pdb.html)
 

lms/djangoapps/courseware/courses.py

@@ -81,7 +81,7 @@ def get_opt_course_with_access(user, course_id, action):
 def course_image_url(course):
     """Try to look up the image url for the course.  If it's not found,
     log an error and return the dead link"""
-    if modulestore().get_modulestore_type(course.location.course_id) == XML_MODULESTORE_TYPE:
+    if course.lms.static_asset_path or modulestore().get_modulestore_type(course.location.course_id) == XML_MODULESTORE_TYPE:
         return '/static/' + (course.lms.static_asset_path or getattr(course, 'data_dir', '')) + "/images/course_image.jpg"
     else:
         loc = course.location._replace(tag='c4x', category='asset', name='images_course_image.jpg')

lms/djangoapps/courseware/tests/test_module_render.py

@@ -19,7 +19,7 @@ from courseware.tests.tests import LoginEnrollmentTestCase, TEST_DATA_MONGO_MODU
 from courseware.model_data import ModelDataCache
 from modulestore_config import TEST_DATA_XML_MODULESTORE
 
-from courseware.courses import get_course_with_access
+from courseware.courses import get_course_with_access, course_image_url, get_course_info_section
 
 from .factories import UserFactory
 
@@ -139,6 +139,7 @@ class ModuleRenderTestCase(LoginEnrollmentTestCase):
             self.course_id
         )
 
+    
     def test_xqueue_callback_success(self):
         """
         Test for happy-path xqueue_callback
@@ -336,6 +337,40 @@ class TestHtmlModifiers(ModuleStoreTestCase):
             result_fragment.content
         )
 
+    def test_static_asset_path_use(self):
+        '''
+        when a course is loaded with do_import_static=False (see xml_importer.py), then 
+        static_asset_path is set as an lms kv in course.  That should make static paths
+        not be mangled (ie not changed to c4x://).
+        '''
+        module = render.get_module(
+            self.user,
+            self.request,
+            self.location,
+            self.model_data_cache,
+            self.course.id,
+            static_asset_path="toy_course_dir",
+        )
+        result_fragment = module.runtime.render(module, None, 'student_view')
+        self.assertIn('href="/static/toy_course_dir', result_fragment.content)
+
+
+    def test_course_image(self):
+        url = course_image_url(self.course)
+        self.assertTrue(url.startswith('/c4x/'))
+
+        self.course.lms.static_asset_path = "toy_course_dir"
+        url = course_image_url(self.course)
+        self.assertTrue(url.startswith('/static/toy_course_dir/'))
+        self.course.lms.static_asset_path = ""
+
+
+    def test_get_course_info_section(self):
+        self.course.lms.static_asset_path = "toy_course_dir"
+        handouts = get_course_info_section(self.request, self.course, "handouts")
+        # TODO: check handouts output...right now test course seems to have no such content
+        # at least this makes sure get_course_info_section returns without exception
+
     def test_course_link_rewrite(self):
         module = render.get_module(
             self.user,