Skip to content

E
edx-platform
Commit 2348f71c by Bridger Maxwell
Options

Merge remote-tracking branch 'origin/master' into MITx/feature/bridger/fast_course_grading 

Conflicts:
	lms/djangoapps/courseware/grades.py
	lms/djangoapps/courseware/views.py
parents a9c122d7 9b322f68
Showing with 1840 additions and 1155 deletions
Gemfile
source :rubygems
ruby "1.9.3"
gem 'rake'
gem 'sass', '3.1.15'
gem 'bourbon', '~> 1.3.6'
cms/djangoapps/contentstore/management/commands/import.py
......@@ -23,4 +23,7 @@ class Command(BaseCommand):
course_dirs = args[1:]
else:
course_dirs = None
print "Importing. Data_dir={data}, course_dirs={courses}".format(
data=data_dir,
courses=course_dirs)
import_from_xml(modulestore(), data_dir, course_dirs)
cms/djangoapps/contentstore/views.py
......@@ -108,7 +108,7 @@ def edit_item(request):
'contents': item.get_html(),
'js_module': item.js_module_name,
'category': item.category,
'name': item.name,
'url_name': item.url_name,
'previews': get_module_previews(request, item),
})
......@@ -176,7 +176,7 @@ def load_preview_state(request, preview_id, location):
def save_preview_state(request, preview_id, location, instance_state, shared_state):
"""
Load the state of a preview module to the request
Save the state of a preview module to the request
preview_id (str): An identifier specifying which preview this module is used for
location: The Location of the module to dispatch to
......@@ -214,7 +214,10 @@ def preview_module_system(request, preview_id, descriptor):
get_module=partial(get_preview_module, request, preview_id),
render_template=render_from_lms,
debug=True,
replace_urls=replace_urls
replace_urls=replace_urls,
# TODO (vshnayder): All CMS users get staff view by default
# is that what we want?
is_staff=True,
)
......
cms/djangoapps/github_sync/tests/test_views.py
......@@ -11,33 +11,33 @@ class PostReceiveTestCase(TestCase):
def setUp(self):
self.client = Client()
@patch('github_sync.views.sync_with_github')
def test_non_branch(self, sync_with_github):
@patch('github_sync.views.import_from_github')
def test_non_branch(self, import_from_github):
self.client.post('/github_service_hook', {'payload': json.dumps({
'ref': 'refs/tags/foo'})
})
self.assertFalse(sync_with_github.called)
self.assertFalse(import_from_github.called)
@patch('github_sync.views.sync_with_github')
def test_non_watched_repo(self, sync_with_github):
@patch('github_sync.views.import_from_github')
def test_non_watched_repo(self, import_from_github):
self.client.post('/github_service_hook', {'payload': json.dumps({
'ref': 'refs/heads/branch',
'repository': {'name': 'bad_repo'}})
})
self.assertFalse(sync_with_github.called)
self.assertFalse(import_from_github.called)
@patch('github_sync.views.sync_with_github')
def test_non_tracked_branch(self, sync_with_github):
@patch('github_sync.views.import_from_github')
def test_non_tracked_branch(self, import_from_github):
self.client.post('/github_service_hook', {'payload': json.dumps({
'ref': 'refs/heads/non_branch',
'repository': {'name': 'repo'}})
})
self.assertFalse(sync_with_github.called)
self.assertFalse(import_from_github.called)
@patch('github_sync.views.sync_with_github')
def test_tracked_branch(self, sync_with_github):
@patch('github_sync.views.import_from_github')
def test_tracked_branch(self, import_from_github):
self.client.post('/github_service_hook', {'payload': json.dumps({
'ref': 'refs/heads/branch',
'repository': {'name': 'repo'}})
})
sync_with_github.assert_called_with(load_repo_settings('repo'))
import_from_github.assert_called_with(load_repo_settings('repo'))
cms/djangoapps/github_sync/views.py
......@@ -5,7 +5,7 @@ from django.http import HttpResponse
from django.conf import settings
from django_future.csrf import csrf_exempt
from . import sync_with_github, load_repo_settings
from . import import_from_github, load_repo_settings
log = logging.getLogger()
......@@ -46,6 +46,6 @@ def github_post_receive(request):
log.info('Ignoring changes to non-tracked branch %s in repo %s' % (branch_name, repo_name))
return HttpResponse('Ignoring non-tracked branch')
sync_with_github(repo)
import_from_github(repo)
return HttpResponse('Push received')
cms/envs/dev.py
......@@ -2,13 +2,18 @@
This config file runs the simplest dev environment"""
from .common import *
from .logsettings import get_logger_config
import logging
import sys
logging.basicConfig(stream=sys.stdout, )
DEBUG = True
TEMPLATE_DEBUG = DEBUG
LOGGING = get_logger_config(ENV_ROOT / "log",
logging_env="dev",
tracking_filename="tracking.log",
debug=True)
MODULESTORE = {
'default': {
......@@ -37,7 +42,8 @@ REPOS = {
},
'content-mit-6002x': {
'branch': 'master',
'origin': 'git@github.com:MITx/6002x-fall-2012.git',
#'origin': 'git@github.com:MITx/6002x-fall-2012.git',
'origin': 'git@github.com:MITx/content-mit-6002x.git',
},
'6.00x': {
'branch': 'master',
......@@ -75,3 +81,6 @@ CACHES = {
'KEY_FUNCTION': 'util.memcache.safe_key',
}
}
# Make the keyedcache startup warnings go away
CACHE_TIMEOUT = 0
cms/envs/logsettings.py
......@@ -3,19 +3,19 @@ import os.path
import platform
import sys
def get_logger_config(log_dir,
logging_env="no_env",
def get_logger_config(log_dir,
logging_env="no_env",
tracking_filename=None,
syslog_addr=None,
debug=False):
"""Return the appropriate logging config dictionary. You should assign the
result of this to the LOGGING var in your settings. The reason it's done
result of this to the LOGGING var in your settings. The reason it's done
this way instead of registering directly is because I didn't want to worry
about resetting the logging state if this is called multiple times when
about resetting the logging state if this is called multiple times when
settings are extended."""
# If we're given an explicit place to put tracking logs, we do that (say for
# debugging). However, logging is not safe for multiple processes hitting
# debugging). However, logging is not safe for multiple processes hitting
# the same file. So if it's left blank, we dynamically create the filename
# based on the PID of this worker process.
if tracking_filename:
......@@ -33,6 +33,7 @@ def get_logger_config(log_dir,
return {
'version': 1,
'disable_existing_loggers': False,
'formatters' : {
'standard' : {
'format' : '%(asctime)s %(levelname)s %(process)d [%(name)s] %(filename)s:%(lineno)d - %(message)s',
......
cms/static/sass/_base.scss
......@@ -2,6 +2,7 @@ $fg-column: 70px;
$fg-gutter: 26px;
$fg-max-columns: 12;
$body-font-family: "Open Sans", "Lucida Grande", "Lucida Sans Unicode", "Lucida Sans", Geneva, Verdana, sans-serif;
$sans-serif: "Open Sans", "Lucida Grande", "Lucida Sans Unicode", "Lucida Sans", Geneva, Verdana, sans-serif;
$body-font-size: 14px;
$body-line-height: 20px;
......@@ -12,6 +13,7 @@ $orange: #f96e5b;
$yellow: #fff8af;
$cream: #F6EFD4;
$mit-red: #933;
$border-color: #ddd;
@mixin hide-text {
background-color: transparent;
......
cms/static/sass/_content-types.scss
......@@ -56,10 +56,10 @@
.module a:first-child {
@extend .content-type;
background-image: url('/static/img/content-types/module.png');
background-image: url('../img/content-types/module.png');
}
.module a:first-child {
@extend .content-type;
background-image: url('/static/img/content-types/module.png');
background-image: url('../img/content-types/module.png');
}
cms/templates/unit.html
<section id="unit-wrapper">
<header>
<section>
<h1 class="editable">${name}</h1>
<h1 class="editable">${url_name}</h1>
<p class="${category}"><a href="#">${category}</a></p>
</section>
......
cms/templates/widgets/navigation.html
......@@ -41,7 +41,7 @@
% for week in weeks:
<li class="week" data-id="${week.location.url()}">
<header>
<h1><a href="#" class="week-edit">${week.name}</a></h1>
<h1><a href="#" class="week-edit">${week.url_name}</a></h1>
<ul>
% if 'goals' in week.metadata:
% for goal in week.metadata['goals']:
......@@ -60,7 +60,7 @@
data-type="${module.js_module_name}"
data-preview-type="${module.module_class.js_module_name}">
<a href="#" class="module-edit">${module.name}</a>
<a href="#" class="module-edit">${module.url_name}</a>
<a href="#" class="draggable">handle</a>
</li>
% endfor
......
cms/templates/widgets/sequence-edit.html
......@@ -39,7 +39,7 @@
<a href="#" class="module-edit"
data-id="${child.location.url()}"
data-type="${child.js_module_name}"
data-preview-type="${child.module_class.js_module_name}">${child.name}</a>
data-preview-type="${child.module_class.js_module_name}">${child.url_name}</a>
<a href="#" class="draggable">handle</a>
</li>
%endfor
......
common/djangoapps/external_auth/admin.py 0 → 100644
'''
django admin pages for courseware model
'''
from external_auth.models import *
from django.contrib import admin
admin.site.register(ExternalAuthMap)
common/djangoapps/external_auth/models.py 0 → 100644
"""
WE'RE USING MIGRATIONS!
If you make changes to this model, be sure to create an appropriate migration
file and check it in at the same time as your model changes. To do that,
1. Go to the mitx dir
2. django-admin.py schemamigration student --auto --settings=lms.envs.dev --pythonpath=. description_of_your_change
3. Add the migration file created in mitx/common/djangoapps/external_auth/migrations/
"""
from django.db import models
from django.contrib.auth.models import User
class ExternalAuthMap(models.Model):
class Meta:
unique_together = (('external_id', 'external_domain'), )
external_id = models.CharField(max_length=255, db_index=True)
external_domain = models.CharField(max_length=255, db_index=True)
external_credentials = models.TextField(blank=True) # JSON dictionary
external_email = models.CharField(max_length=255, db_index=True)
external_name = models.CharField(blank=True,max_length=255, db_index=True)
user = models.OneToOneField(User, unique=True, db_index=True, null=True)
internal_password = models.CharField(blank=True, max_length=31) # randomly generated
dtcreated = models.DateTimeField('creation date',auto_now_add=True)
dtsignup = models.DateTimeField('signup date',null=True) # set after signup
def __unicode__(self):
s = "[%s] = (%s / %s)" % (self.external_id, self.external_name, self.external_email)
return s
common/djangoapps/external_auth/views.py 0 → 100644
import json
import logging
import random
import re
import string
from external_auth.models import ExternalAuthMap
from django.conf import settings
from django.contrib.auth import REDIRECT_FIELD_NAME, authenticate, login
from django.contrib.auth.models import Group
from django.contrib.auth.models import User
from django.core.urlresolvers import reverse
from django.http import HttpResponse, HttpResponseRedirect
from django.shortcuts import render_to_response
from django.shortcuts import redirect
from django.template import RequestContext
from mitxmako.shortcuts import render_to_response, render_to_string
try:
from django.views.decorators.csrf import csrf_exempt
except ImportError:
from django.contrib.csrf.middleware import csrf_exempt
from django_future.csrf import ensure_csrf_cookie
from util.cache import cache_if_anonymous
from django_openid_auth import auth as openid_auth
from openid.consumer.consumer import (Consumer, SUCCESS, CANCEL, FAILURE)
import django_openid_auth.views as openid_views
import student.views as student_views
log = logging.getLogger("mitx.external_auth")
@csrf_exempt
def default_render_failure(request, message, status=403, template_name='extauth_failure.html', exception=None):
"""Render an Openid error page to the user."""
message = "In openid_failure " + message
log.debug(message)
data = render_to_string( template_name, dict(message=message, exception=exception))
return HttpResponse(data, status=status)
#-----------------------------------------------------------------------------
# Openid
def edXauth_generate_password(length=12, chars=string.letters + string.digits):
"""Generate internal password for externally authenticated user"""
return ''.join([random.choice(chars) for i in range(length)])
@csrf_exempt
def edXauth_openid_login_complete(request, redirect_field_name=REDIRECT_FIELD_NAME, render_failure=None):
"""Complete the openid login process"""
redirect_to = request.REQUEST.get(redirect_field_name, '')
render_failure = render_failure or \
getattr(settings, 'OPENID_RENDER_FAILURE', None) or \
default_render_failure
openid_response = openid_views.parse_openid_response(request)
if not openid_response:
return render_failure(request, 'This is an OpenID relying party endpoint.')
if openid_response.status == SUCCESS:
external_id = openid_response.identity_url
oid_backend = openid_auth.OpenIDBackend()
details = oid_backend._extract_user_details(openid_response)
log.debug('openid success, details=%s' % details)
return edXauth_external_login_or_signup(request,
external_id,
"openid:%s" % settings.OPENID_SSO_SERVER_URL,
details,
details.get('email',''),
'%s %s' % (details.get('first_name',''),details.get('last_name',''))
)
return render_failure(request, 'Openid failure')
#-----------------------------------------------------------------------------
# generic external auth login or signup
def edXauth_external_login_or_signup(request, external_id, external_domain, credentials, email, fullname,
retfun=None):
# see if we have a map from this external_id to an edX username
try:
eamap = ExternalAuthMap.objects.get(external_id = external_id,
external_domain = external_domain,
)
log.debug('Found eamap=%s' % eamap)
except ExternalAuthMap.DoesNotExist:
# go render form for creating edX user
eamap = ExternalAuthMap(external_id = external_id,
external_domain = external_domain,
external_credentials = json.dumps(credentials),
)
eamap.external_email = email
eamap.external_name = fullname
eamap.internal_password = edXauth_generate_password()
log.debug('created eamap=%s' % eamap)
eamap.save()
internal_user = eamap.user
if internal_user is None:
log.debug('ExtAuth: no user for %s yet, doing signup' % eamap.external_email)
return edXauth_signup(request, eamap)
uname = internal_user.username
user = authenticate(username=uname, password=eamap.internal_password)
if user is None:
log.warning("External Auth Login failed for %s / %s" % (uname,eamap.internal_password))
return edXauth_signup(request, eamap)
if not user.is_active:
log.warning("External Auth: user %s is not active" % (uname))
# TODO: improve error page
return render_failure(request, 'Account not yet activated: please look for link in your email')
login(request, user)
request.session.set_expiry(0)
student_views.try_change_enrollment(request)
log.info("Login success - {0} ({1})".format(user.username, user.email))
if retfun is None:
return redirect('/')
return retfun()
#-----------------------------------------------------------------------------
# generic external auth signup
@ensure_csrf_cookie
@cache_if_anonymous
def edXauth_signup(request, eamap=None):
"""
Present form to complete for signup via external authentication.
Even though the user has external credentials, he/she still needs
to create an account on the edX system, and fill in the user
registration form.
eamap is an ExteralAuthMap object, specifying the external user
for which to complete the signup.
"""
if eamap is None:
pass
request.session['ExternalAuthMap'] = eamap # save this for use by student.views.create_account
context = {'has_extauth_info': True,
'show_signup_immediately' : True,
'extauth_email': eamap.external_email,
'extauth_username' : eamap.external_name.split(' ')[0],
'extauth_name': eamap.external_name,
}
log.debug('ExtAuth: doing signup for %s' % eamap.external_email)
return student_views.main_index(extra_context=context)
#-----------------------------------------------------------------------------
# MIT SSL
def ssl_dn_extract_info(dn):
'''
Extract username, email address (may be anyuser@anydomain.com) and full name
from the SSL DN string. Return (user,email,fullname) if successful, and None
otherwise.
'''
ss = re.search('/emailAddress=(.*)@([^/]+)', dn)
if ss:
user = ss.group(1)
email = "%s@%s" % (user, ss.group(2))
else:
return None
ss = re.search('/CN=([^/]+)/', dn)
if ss:
fullname = ss.group(1)
else:
return None
return (user, email, fullname)
@csrf_exempt
def edXauth_ssl_login(request):
"""
This is called by student.views.index when MITX_FEATURES['AUTH_USE_MIT_CERTIFICATES'] = True
Used for MIT user authentication. This presumes the web server (nginx) has been configured
to require specific client certificates.
If the incoming protocol is HTTPS (SSL) then authenticate via client certificate.
The certificate provides user email and fullname; this populates the ExternalAuthMap.
The user is nevertheless still asked to complete the edX signup.
Else continues on with student.views.main_index, and no authentication.
"""
certkey = "SSL_CLIENT_S_DN" # specify the request.META field to use
cert = request.META.get(certkey,'')
if not cert:
cert = request.META.get('HTTP_'+certkey,'')
if not cert:
try:
cert = request._req.subprocess_env.get(certkey,'') # try the direct apache2 SSL key
except Exception as err:
pass
if not cert:
# no certificate information - go onward to main index
return student_views.main_index()
(user, email, fullname) = ssl_dn_extract_info(cert)
return edXauth_external_login_or_signup(request,
external_id=email,
external_domain="ssl:MIT",
credentials=cert,
email=email,
fullname=fullname,
retfun = student_views.main_index)
common/djangoapps/student/views.py
......@@ -23,7 +23,7 @@ from django.http import HttpResponse, Http404
from django.shortcuts import redirect
from mitxmako.shortcuts import render_to_response, render_to_string
from django.core.urlresolvers import reverse
from BeautifulSoup import BeautifulSoup
from bs4 import BeautifulSoup
from django.core.cache import cache
from django_future.csrf import ensure_csrf_cookie
......@@ -60,6 +60,19 @@ def index(request):
if settings.COURSEWARE_ENABLED and request.user.is_authenticated():
return redirect(reverse('dashboard'))
if settings.MITX_FEATURES.get('AUTH_USE_MIT_CERTIFICATES'):
from external_auth.views import edXauth_ssl_login
return edXauth_ssl_login(request)
return main_index()
def main_index(extra_context = {}):
'''
Render the edX main page.
extra_context is used to allow immediate display of certain modal windows, eg signup,
as used by external_auth.
'''
feed_data = cache.get("students_index_rss_feed_data")
if feed_data == None:
if hasattr(settings, 'RSS_URL'):
......@@ -80,8 +93,9 @@ def index(request):
for course in courses:
universities[course.org].append(course)
return render_to_response('index.html', {'universities': universities, 'entries': entries})
context = {'universities': universities, 'entries': entries}
context.update(extra_context)
return render_to_response('index.html', context)
def course_from_id(id):
course_loc = CourseDescriptor.id_to_location(id)
......@@ -256,11 +270,26 @@ def change_setting(request):
@ensure_csrf_cookie
def create_account(request, post_override=None):
''' JSON call to enroll in the course. '''
'''
JSON call to create new edX account.
Used by form in signup_modal.html, which is included into navigation.html
'''
js = {'success': False}
post_vars = post_override if post_override else request.POST
# if doing signup for an external authorization, then get email, password, name from the eamap
# don't use the ones from the form, since the user could have hacked those
DoExternalAuth = 'ExternalAuthMap' in request.session
if DoExternalAuth:
eamap = request.session['ExternalAuthMap']
email = eamap.external_email
name = eamap.external_name
password = eamap.internal_password
post_vars = dict(post_vars.items())
post_vars.update(dict(email=email, name=name, password=password))
log.debug('extauth test: post_vars = %s' % post_vars)
# Confirm we have a properly formed request
for a in ['username', 'email', 'password', 'name']:
if a not in post_vars:
......@@ -355,8 +384,9 @@ def create_account(request, post_override=None):
'key': r.activation_key,
}
# composes activation email
subject = render_to_string('emails/activation_email_subject.txt', d)
# Email subject *must not* contain newlines
# Email subject *must not* contain newlines
subject = ''.join(subject.splitlines())
message = render_to_string('emails/activation_email.txt', d)
......@@ -381,6 +411,17 @@ def create_account(request, post_override=None):
try_change_enrollment(request)
if DoExternalAuth:
eamap.user = login_user
eamap.dtsignup = datetime.datetime.now()
eamap.save()
log.debug('Updated ExternalAuthMap for %s to be %s' % (post_vars['username'],eamap))
if settings.MITX_FEATURES.get('BYPASS_ACTIVATION_EMAIL_FOR_EXTAUTH'):
log.debug('bypassing activation email')
login_user.is_active = True
login_user.save()
js = {'success': True}
return HttpResponse(json.dumps(js), mimetype="application/json")
......
common/djangoapps/xmodule_modifiers.py
......@@ -76,8 +76,9 @@ def add_histogram(get_html, module):
histogram = grade_histogram(module_id)
render_histogram = len(histogram) > 0
# TODO: fixme - no filename in module.xml in general (this code block for edx4edx)
# the following if block is for summer 2012 edX course development; it will change when the CMS comes online
# TODO: fixme - no filename in module.xml in general (this code block
# for edx4edx) the following if block is for summer 2012 edX course
# development; it will change when the CMS comes online
if settings.MITX_FEATURES.get('DISPLAY_EDIT_LINK') and settings.DEBUG and module_xml.get('filename') is not None:
coursename = multicourse_settings.get_coursename_from_request(request)
github_url = multicourse_settings.get_course_github_url(coursename)
......@@ -88,10 +89,8 @@ def add_histogram(get_html, module):
else:
edit_link = False
# Cast module.definition and module.metadata to dicts so that json can dump them
# even though they are lazily loaded
staff_context = {'definition': json.dumps(dict(module.definition), indent=4),
'metadata': json.dumps(dict(module.metadata), indent=4),
staff_context = {'definition': json.dumps(module.definition, indent=4),
'metadata': json.dumps(module.metadata, indent=4),
'element_id': module.location.html_id(),
'edit_link': edit_link,
'histogram': json.dumps(histogram),
......
common/lib/capa/capa/capa_problem.py
......@@ -288,20 +288,30 @@ class LoncapaProblem(object):
try:
ifp = self.system.filestore.open(file) # open using ModuleSystem OSFS filestore
except Exception as err:
log.error('Error %s in problem xml include: %s' % (err, etree.tostring(inc, pretty_print=True)))
log.error('Cannot find file %s in %s' % (file, self.system.filestore))
if not self.system.get('DEBUG'): # if debugging, don't fail - just log error
log.error('Error %s in problem xml include: %s' % (
err, etree.tostring(inc, pretty_print=True)))
log.error('Cannot find file %s in %s' % (
file, self.system.filestore))
# if debugging, don't fail - just log error
# TODO (vshnayder): need real error handling, display to users
if not self.system.get('DEBUG'):
raise
else: continue
else:
continue
try:
incxml = etree.XML(ifp.read()) # read in and convert to XML
except Exception as err:
log.error('Error %s in problem xml include: %s' % (err, etree.tostring(inc, pretty_print=True)))
log.error('Error %s in problem xml include: %s' % (
err, etree.tostring(inc, pretty_print=True)))
log.error('Cannot parse XML in %s' % (file))
if not self.system.get('DEBUG'): # if debugging, don't fail - just log error
# if debugging, don't fail - just log error
# TODO (vshnayder): same as above
if not self.system.get('DEBUG'):
raise
else: continue
parent = inc.getparent() # insert new XML into tree in place of inlcude
else:
continue
# insert new XML into tree in place of inlcude
parent = inc.getparent()
parent.insert(parent.index(inc), incxml)
parent.remove(inc)
log.debug('Included %s into %s' % (file, self.problem_id))
......@@ -329,7 +339,7 @@ class LoncapaProblem(object):
# path is an absolute path or a path relative to the data dir
dir = os.path.join(self.system.filestore.root_path, dir)
abs_dir = os.path.normpath(dir)
log.debug("appending to path: %s" % abs_dir)
#log.debug("appending to path: %s" % abs_dir)
path.append(abs_dir)
return path
......
common/lib/capa/capa/inputtypes.py
......@@ -313,14 +313,20 @@ def textbox(element, value, status, render_template, msg=''):
size = element.get('size')
rows = element.get('rows') or '30'
cols = element.get('cols') or '80'
mode = element.get('mode') or 'python' # mode for CodeMirror, eg "python" or "xml"
hidden = element.get('hidden', '') # if specified, then textline is hidden and id is stored in div of name given by hidden
linenumbers = element.get('linenumbers','true') # for CodeMirror
if not value: value = element.text # if no student input yet, then use the default input given by the problem
# For CodeMirror
mode = element.get('mode') or 'python' # mode, eg "python" or "xml"
linenumbers = element.get('linenumbers','true') # for CodeMirror
tabsize = element.get('tabsize','4')
tabsize = int(tabsize)
context = {'id': eid, 'value': value, 'state': status, 'count': count, 'size': size, 'msg': msg,
'mode': mode, 'linenumbers': linenumbers,
'rows': rows, 'cols': cols,
'hidden': hidden,
'hidden': hidden, 'tabsize': tabsize,
}
html = render_template("textbox.html", context)
try:
......
common/lib/capa/capa/responsetypes.py
......@@ -811,7 +811,7 @@ class CodeResponse(LoncapaResponse):
def setup_response(self):
xml = self.xml
self.url = xml.get('url', "http://107.20.215.194/xqueue/submit/") # FIXME -- hardcoded url
self.queue_name = xml.get('queuename', 'python') # TODO: Default queue_name should be course-specific
self.queue_name = xml.get('queuename', self.system.xqueue_default_queuename)
answer = xml.find('answer')
if answer is not None:
......@@ -905,7 +905,7 @@ class CodeResponse(LoncapaResponse):
def _send_to_queue(self, extra_payload):
# Prepare payload
xmlstr = etree.tostring(self.xml, pretty_print=True)
header = {'return_url': self.system.xqueue_callback_url,
header = {'lms_callback_url': self.system.xqueue_callback_url,
'queue_name': self.queue_name,
}
......@@ -914,7 +914,7 @@ class CodeResponse(LoncapaResponse):
h.update(str(self.system.seed))
h.update(str(time.time()))
queuekey = int(h.hexdigest(), 16)
header.update({'queuekey': queuekey})
header.update({'lms_key': queuekey})
body = {'xml': xmlstr,
'edX_cmd': 'get_score',
......
common/lib/capa/capa/templates/textbox.html
......@@ -35,15 +35,20 @@
lineNumbers: true,
% endif
mode: "${mode}",
tabsize: 4,
matchBrackets: true,
lineWrapping: true,
indentUnit: "${tabsize}",
tabSize: "${tabsize}",
smartIndent: false
});
});
</script>
<style type="text/css">
.CodeMirror {
border: 2px solid black;
border: 1px solid black;
font-size: 14px;
line-height: 18px;
resize: vertical;
}
</style>
</section>
common/lib/supertrace.py 0 → 100644
"""
A handy util to print a django-debug-screen-like stack trace with
values of local variables.
"""
import sys, traceback
from django.utils.encoding import smart_unicode
def supertrace(max_len=160):
"""
Print the usual traceback information, followed by a listing of all the
local variables in each frame. Should be called from an exception handler.
if max_len is not None, will print up to max_len chars for each local variable.
(cite: modified from somewhere on stackoverflow)
"""
tb = sys.exc_info()[2]
while True:
if not tb.tb_next:
break
tb = tb.tb_next
stack = []
frame = tb.tb_frame
while frame:
stack.append(f)
frame = frame.f_back
stack.reverse()
# First print the regular traceback
traceback.print_exc()
print "Locals by frame, innermost last"
for frame in stack:
print
print "Frame %s in %s at line %s" % (frame.f_code.co_name,
frame.f_code.co_filename,
frame.f_lineno)
for key, value in frame.f_locals.items():
print ("\t%20s = " % smart_unicode(key, errors='ignore')),
# We have to be careful not to cause a new error in our error
# printer! Calling str() on an unknown object could cause an
# error.
try:
s = smart_unicode(value, errors='ignore')
if max_len is not None:
s = s[:max_len]
print s
except:
print "<ERROR WHILE PRINTING VALUE>"
common/lib/xmodule/progress.py deleted 100644 → 0
'''
Progress class for modules. Represents where a student is in a module.
Useful things to know:
- Use Progress.to_js_status_str() to convert a progress into a simple
status string to pass to js.
- Use Progress.to_js_detail_str() to convert a progress into a more detailed
string to pass to js.
In particular, these functions have a canonical handing of None.
For most subclassing needs, you should only need to reimplement
frac() and __str__().
'''
from collections import namedtuple
import numbers
class Progress(object):
'''Represents a progress of a/b (a out of b done)
a and b must be numeric, but not necessarily integer, with
0 <= a <= b and b > 0.
Progress can only represent Progress for modules where that makes sense. Other
modules (e.g. html) should return None from get_progress().
TODO: add tag for module type? Would allow for smarter merging.
'''
def __init__(self, a, b):
'''Construct a Progress object. a and b must be numbers, and must have
0 <= a <= b and b > 0
'''
# Want to do all checking at construction time, so explicitly check types
if not (isinstance(a, numbers.Number) and
isinstance(b, numbers.Number)):
raise TypeError('a and b must be numbers. Passed {0}/{1}'.format(a, b))
if not (0 <= a <= b and b > 0):
raise ValueError(
'fraction a/b = {0}/{1} must have 0 <= a <= b and b > 0'.format(a, b))
self._a = a
self._b = b
def frac(self):
''' Return tuple (a,b) representing progress of a/b'''
return (self._a, self._b)
def percent(self):
''' Returns a percentage progress as a float between 0 and 100.
subclassing note: implemented in terms of frac(), assumes sanity
checking is done at construction time.
'''
(a, b) = self.frac()
return 100.0 * a / b
def started(self):
''' Returns True if fractional progress is greater than 0.
subclassing note: implemented in terms of frac(), assumes sanity
checking is done at construction time.
'''
return self.frac()[0] > 0
def inprogress(self):
''' Returns True if fractional progress is strictly between 0 and 1.
subclassing note: implemented in terms of frac(), assumes sanity
checking is done at construction time.
'''
(a, b) = self.frac()
return a > 0 and a < b
def done(self):
''' Return True if this represents done.
subclassing note: implemented in terms of frac(), assumes sanity
checking is done at construction time.
'''
(a, b) = self.frac()
return a == b
def ternary_str(self):
''' Return a string version of this progress: either
"none", "in_progress", or "done".
subclassing note: implemented in terms of frac()
'''
(a, b) = self.frac()
if a == 0:
return "none"
if a < b:
return "in_progress"
return "done"
def __eq__(self, other):
''' Two Progress objects are equal if they have identical values.
Implemented in terms of frac()'''
if not isinstance(other, Progress):
return False
(a, b) = self.frac()
(a2, b2) = other.frac()
return a == a2 and b == b2
def __ne__(self, other):
''' The opposite of equal'''
return not self.__eq__(other)
def __str__(self):
''' Return a string representation of this string.
subclassing note: implemented in terms of frac().
'''
(a, b) = self.frac()
return "{0}/{1}".format(a, b)
@staticmethod
def add_counts(a, b):
'''Add two progress indicators, assuming that each represents items done:
(a / b) + (c / d) = (a + c) / (b + d).
If either is None, returns the other.
'''
if a is None:
return b
if b is None:
return a
# get numerators + denominators
(n, d) = a.frac()
(n2, d2) = b.frac()
return Progress(n + n2, d + d2)
@staticmethod
def to_js_status_str(progress):
'''
Return the "status string" version of the passed Progress
object that should be passed to js. Use this function when
sending Progress objects to js to limit dependencies.
'''
if progress is None:
return "NA"
return progress.ternary_str()
@staticmethod
def to_js_detail_str(progress):
'''
Return the "detail string" version of the passed Progress
object that should be passed to js. Use this function when
passing Progress objects to js to limit dependencies.
'''
if progress is None:
return "NA"
return str(progress)
common/lib/xmodule/setup.py
......@@ -25,6 +25,7 @@ setup(
"discuss = xmodule.backcompat_module:TranslateCustomTagDescriptor",
"html = xmodule.html_module:HtmlDescriptor",
"image = xmodule.backcompat_module:TranslateCustomTagDescriptor",
"error = xmodule.error_module:ErrorDescriptor",
"problem = xmodule.capa_module:CapaDescriptor",
"problemset = xmodule.vertical_module:VerticalDescriptor",
"section = xmodule.backcompat_module:SemanticSectionDescriptor",
......
common/lib/xmodule/xmodule/backcompat_module.py
......@@ -32,21 +32,25 @@ def process_includes(fn):
# read in and convert to XML
incxml = etree.XML(ifp.read())
# insert new XML into tree in place of inlcude
# insert new XML into tree in place of include
parent.insert(parent.index(next_include), incxml)
except Exception:
msg = "Error in problem xml include: %s" % (etree.tostring(next_include, pretty_print=True))
log.exception(msg)
parent = next_include.getparent()
# Log error
msg = "Error in problem xml include: %s" % (
etree.tostring(next_include, pretty_print=True))
# tell the tracker
system.error_tracker(msg)
# work around
parent = next_include.getparent()
errorxml = etree.Element('error')
messagexml = etree.SubElement(errorxml, 'message')
messagexml.text = msg
stackxml = etree.SubElement(errorxml, 'stacktrace')
stackxml.text = traceback.format_exc()
# insert error XML in place of include
parent.insert(parent.index(next_include), errorxml)
parent.remove(next_include)
next_include = xml_object.find('include')
......
common/lib/xmodule/xmodule/capa_module.py
......@@ -5,6 +5,7 @@ import json
import logging
import traceback
import re
import sys
from datetime import timedelta
from lxml import etree
......@@ -91,7 +92,8 @@ class CapaModule(XModule):
display_due_date_string = self.metadata.get('due', None)
if display_due_date_string is not None:
self.display_due_date = dateutil.parser.parse(display_due_date_string)
#log.debug("Parsed " + display_due_date_string + " to " + str(self.display_due_date))
#log.debug("Parsed " + display_due_date_string +
# " to " + str(self.display_due_date))
else:
self.display_due_date = None
......@@ -99,7 +101,8 @@ class CapaModule(XModule):
if grace_period_string is not None and self.display_due_date:
self.grace_period = parse_timedelta(grace_period_string)
self.close_date = self.display_due_date + self.grace_period
#log.debug("Then parsed " + grace_period_string + " to closing date" + str(self.close_date))
#log.debug("Then parsed " + grace_period_string +
# " to closing date" + str(self.close_date))
else:
self.grace_period = None
self.close_date = self.display_due_date
......@@ -138,10 +141,16 @@ class CapaModule(XModule):
try:
self.lcp = LoncapaProblem(self.definition['data'], self.location.html_id(),
instance_state, seed=seed, system=self.system)
except Exception:
msg = 'cannot create LoncapaProblem %s' % self.location.url()
log.exception(msg)
except Exception as err:
msg = 'cannot create LoncapaProblem {loc}: {err}'.format(
loc=self.location.url(), err=err)
# TODO (vshnayder): do modules need error handlers too?
# We shouldn't be switching on DEBUG.
if self.system.DEBUG:
log.error(msg)
# TODO (vshnayder): This logic should be general, not here--and may
# want to preserve the data instead of replacing it.
# e.g. in the CMS
msg = '<p>%s</p>' % msg.replace('<', '&lt;')
msg += '<p><pre>%s</pre></p>' % traceback.format_exc().replace('<', '&lt;')
# create a dummy problem with error message instead of failing
......@@ -152,7 +161,8 @@ class CapaModule(XModule):
problem_text, self.location.html_id(),
instance_state, seed=seed, system=self.system)
else:
raise
# add extra info and raise
raise Exception(msg), None, sys.exc_info()[2]
@property
def rerandomize(self):
......@@ -191,6 +201,7 @@ class CapaModule(XModule):
try:
return Progress(score, total)
except Exception as err:
# TODO (vshnayder): why is this still here? still needed?
if self.system.DEBUG:
return None
raise
......@@ -210,6 +221,7 @@ class CapaModule(XModule):
try:
html = self.lcp.get_html()
except Exception, err:
# TODO (vshnayder): another switch on DEBUG.
if self.system.DEBUG:
log.exception(err)
msg = (
......@@ -561,6 +573,7 @@ class CapaDescriptor(RawDescriptor):
'problems/' + path[8:],
path[8:],
]
@classmethod
def split_to_file(cls, xml_object):
'''Problems always written in their own files'''
......
common/lib/xmodule/xmodule/course_module.py
......@@ -20,15 +20,22 @@ class CourseDescriptor(SequenceDescriptor):
self._grader = None
self._grade_cutoffs = None
msg = None
try:
self.start = time.strptime(self.metadata["start"], "%Y-%m-%dT%H:%M")
except KeyError:
self.start = time.gmtime(0) #The epoch
log.critical("Course loaded without a start date. %s", self.id)
msg = "Course loaded without a start date. id = %s" % self.id
log.critical(msg)
except ValueError as e:
self.start = time.gmtime(0) #The epoch
log.critical("Course loaded with a bad start date. %s '%s'",
self.id, e)
msg = "Course loaded with a bad start date. %s '%s'" % (self.id, e)
log.critical(msg)
# Don't call the tracker from the exception handler.
if msg is not None:
system.error_tracker(msg)
def has_started(self):
return time.gmtime() > self.start
......@@ -104,3 +111,4 @@ class CourseDescriptor(SequenceDescriptor):
@property
def org(self):
return self.location.org
common/lib/xmodule/xmodule/css/sequence/display.scss
......@@ -2,10 +2,8 @@ nav.sequence-nav {
// TODO (cpennington): This doesn't work anymore. XModules aren't able to
// import from external sources.
@extend .topbar;
border-bottom: 1px solid #ddd;
border-bottom: 1px solid $border-color;
margin: (-(lh())) (-(lh())) lh() (-(lh()));
background: #eee;
position: relative;
@include border-top-right-radius(4px);
......@@ -14,7 +12,7 @@ nav.sequence-nav {
display: table;
height: 100%;
margin: 0;
padding-left: 0;
padding-left: 3px;
padding-right: flex-grid(1, 9);
width: 100%;
......@@ -23,133 +21,104 @@ nav.sequence-nav {
}
li {
border-left: 1px solid #eee;
display: table-cell;
min-width: 20px;
&:first-child {
border-left: none;
}
.inactive {
a {
background-position: center;
background-repeat: no-repeat;
border: 1px solid transparent;
border-bottom: none;
@include border-radius(3px 3px 0 0);
cursor: pointer;
display: block;
height: 10px;
padding: 15px 0 14px;
position: relative;
@include transition();
width: 100%;
&:hover {
background-color: #eee;
background-repeat: no-repeat;
background-position: center;
background-color: #F6F6F6;
}
}
.visited {
background-color: #ddd;
background-repeat: no-repeat;
&.visited {
background-color: #F6F6F6;
&:hover {
background-position: center center;
&:hover {
background-position: center center;
}
}
}
.active {
background-color: #fff;
background-repeat: no-repeat;
@include box-shadow(0 1px 0 #fff);
&:hover {
&.active {
border-color: $border-color;
@include box-shadow(0 2px 0 #fff);
background-color: #fff;
background-position: center;
}
}
z-index: 9;
a {
background-position: center center;
border: none;
cursor: pointer;
display: block;
height: 17px;
padding: 15px 0 14px;
position: relative;
@include transition(all, .4s, $ease-in-out-quad);
width: 100%;
&.progress {
border-bottom-style: solid;
border-bottom-width: 4px;
&:hover {
background-position: center;
background-color: #fff;
}
}
&.progress-none {
@extend .progress;
border-bottom-color: red;
background-color: lighten(red, 50%);
}
&.progress-some {
@extend .progress;
border-bottom-color: yellow;
background-color: yellow;
}
&.progress-done {
@extend .progress;
border-bottom-color: green;
background-color: green;
}
//video
&.seq_video {
&.inactive {
@extend .inactive;
background-image: url('../images/sequence-nav/video-icon-normal.png');
background-position: center;
}
&.visited {
@extend .visited;
background-image: url('../images/sequence-nav/video-icon-visited.png');
background-position: center;
}
&.active {
@extend .active;
background-image: url('../images/sequence-nav/video-icon-current.png');
background-position: center;
}
}
//other
&.seq_other {
&.inactive {
@extend .inactive;
background-image: url('../images/sequence-nav/document-icon-normal.png');
background-position: center;
}
&.visited {
@extend .visited;
background-image: url('../images/sequence-nav/document-icon-visited.png');
background-position: center;
}
&.active {
@extend .active;
background-image: url('../images/sequence-nav/document-icon-current.png');
background-position: center;
}
}
//vertical & problems
&.seq_vertical, &.seq_problem {
&.inactive {
@extend .inactive;
background-image: url('../images/sequence-nav/list-icon-normal.png');
background-position: center;
}
&.visited {
@extend .visited;
background-image: url('../images/sequence-nav/list-icon-visited.png');
background-position: center;
}
&.active {
@extend .active;
background-image: url('../images/sequence-nav/list-icon-current.png');
background-position: center;
}
}
......@@ -157,6 +126,7 @@ nav.sequence-nav {
background: #333;
color: #fff;
display: none;
font-family: $sans-serif;
line-height: lh();
left: 0px;
opacity: 0;
......@@ -207,27 +177,29 @@ nav.sequence-nav {
right: 0;
top: 0;
width: flex-grid(1, 9);
border: 1px solid $border-color;
border-bottom: 0;
@include border-radius(3px 3px 0 0);
li {
float: left;
margin-bottom: 0;
width: 50%;
&.prev, &.next {
a {
// background-color: darken($cream, 5%);
background-position: center center;
background-position: center;
background-repeat: no-repeat;
border-left: 1px solid darken(#f6efd4, 20%);
@include box-shadow(inset 1px 0 0 lighten(#f6efd4, 5%));
@include box-sizing(border-box);
cursor: pointer;
display: block;
height: 10px;
padding: 15px 0 14px;
text-indent: -9999px;
@include transition(all, .2s, $ease-in-out-quad);
&:hover {
opacity: .5;
background-color: #f4f4f4;
}
&.disabled {
......@@ -240,20 +212,13 @@ nav.sequence-nav {
&.prev {
a {
background-image: url('../images/sequence-nav/previous-icon.png');
&:hover {
// background-color: $cream;
}
}
}
&.next {
a {
border-left: 1px solid lighten($border-color, 10%);
background-image: url('../images/sequence-nav/next-icon.png');
&:hover {
// background-color: $cream;
}
}
}
}
......@@ -274,10 +239,8 @@ nav.sequence-bottom {
ul {
@extend .clearfix;
background-color: #eee;
border: 1px solid #ddd;
border: 1px solid $border-color;
@include border-radius(3px);
@include box-shadow(inset 0 0 0 1px lighten(#f6efd4, 5%));
@include inline-block();
li {
......@@ -312,7 +275,7 @@ nav.sequence-bottom {
&.prev {
a {
background-image: url('../images/sequence-nav/previous-icon.png');
border-right: 1px solid darken(#f6efd4, 20%);
border-right: 1px solid lighten($border-color, 10%);
&:hover {
background-color: none;
......
common/lib/xmodule/xmodule/editing_module.py 0 → 100644
from pkg_resources import resource_string
from lxml import etree
from xmodule.mako_module import MakoModuleDescriptor
import logging
log = logging.getLogger(__name__)
class EditingDescriptor(MakoModuleDescriptor):
"""
Module that provides a raw editing view of its data and children. It does not
perform any validation on its definition---just passes it along to the browser.
This class is intended to be used as a mixin.
"""
mako_template = "widgets/raw-edit.html"
js = {'coffee': [resource_string(__name__, 'js/src/raw/edit.coffee')]}
js_module_name = "RawDescriptor"
def get_context(self):
return {
'module': self,
'data': self.definition.get('data', ''),
# TODO (vshnayder): allow children and metadata to be edited.
#'children' : self.definition.get('children, ''),
# TODO: show both own metadata and inherited?
#'metadata' : self.own_metadata,
}
common/lib/xmodule/xmodule/error_module.py 0 → 100644
import sys
import logging
from pkg_resources import resource_string
from lxml import etree
from xmodule.x_module import XModule
from xmodule.mako_module import MakoModuleDescriptor
from xmodule.xml_module import XmlDescriptor
from xmodule.editing_module import EditingDescriptor
from xmodule.errortracker import exc_info_to_str
log = logging.getLogger(__name__)
class ErrorModule(XModule):
def get_html(self):
'''Show an error.
TODO (vshnayder): proper style, divs, etc.
'''
# staff get to see all the details
return self.system.render_template('module-error.html', {
'data' : self.definition['data']['contents'],
'error' : self.definition['data']['error_msg'],
'is_staff' : self.system.is_staff,
})
class ErrorDescriptor(EditingDescriptor):
"""
Module that provides a raw editing view of broken xml.
"""
module_class = ErrorModule
@classmethod
def from_xml(cls, xml_data, system, org=None, course=None,
error_msg='Error not available'):
'''Create an instance of this descriptor from the supplied data.
Does not try to parse the data--just stores it.
Takes an extra, optional, parameter--the error that caused an
issue. (should be a string, or convert usefully into one).
'''
# Use a nested inner dictionary because 'data' is hardcoded
inner = {}
definition = {'data': inner}
inner['error_msg'] = str(error_msg)
try:
# If this is already an error tag, don't want to re-wrap it.
xml_obj = etree.fromstring(xml_data)
if xml_obj.tag == 'error':
xml_data = xml_obj.text
error_node = xml_obj.find('error_msg')
if error_node is not None:
inner['error_msg'] = error_node.text
else:
inner['error_msg'] = 'Error not available'
except etree.XMLSyntaxError:
# Save the error to display later--overrides other problems
inner['error_msg'] = exc_info_to_str(sys.exc_info())
inner['contents'] = xml_data
# TODO (vshnayder): Do we need a unique slug here? Just pick a random
# 64-bit num?
location = ['i4x', org, course, 'error', 'slug']
metadata = {} # stays in the xml_data
return cls(system, definition, location=location, metadata=metadata)
def export_to_xml(self, resource_fs):
'''
If the definition data is invalid xml, export it wrapped in an "error"
tag. If it is valid, export without the wrapper.
NOTE: There may still be problems with the valid xml--it could be
missing required attributes, could have the wrong tags, refer to missing
files, etc. That would just get re-wrapped on import.
'''
try:
xml = etree.fromstring(self.definition['data']['contents'])
return etree.tostring(xml)
except etree.XMLSyntaxError:
# still not valid.
root = etree.Element('error')
root.text = self.definition['data']['contents']
err_node = etree.SubElement(root, 'error_msg')
err_node.text = self.definition['data']['error_msg']
return etree.tostring(root)
common/lib/xmodule/xmodule/errorhandlers.py deleted 100644 → 0
import logging
import sys
log = logging.getLogger(__name__)
def in_exception_handler():
'''Is there an active exception?'''
return sys.exc_info() != (None, None, None)
def strict_error_handler(msg, exc_info=None):
'''
Do not let errors pass. If exc_info is not None, ignore msg, and just
re-raise. Otherwise, check if we are in an exception-handling context.
If so, re-raise. Otherwise, raise Exception(msg).
Meant for use in validation, where any errors should trap.
'''
if exc_info is not None:
raise exc_info[0], exc_info[1], exc_info[2]
if in_exception_handler():
raise
raise Exception(msg)
def logging_error_handler(msg, exc_info=None):
'''Log all errors, but otherwise let them pass, relying on the caller to
workaround.'''
if exc_info is not None:
log.exception(msg, exc_info=exc_info)
return
if in_exception_handler():
log.exception(msg)
return
log.error(msg)
def ignore_errors_handler(msg, exc_info=None):
'''Ignore all errors, relying on the caller to workaround.
Meant for use in the LMS, where an error in one part of the course
shouldn't bring down the whole system'''
pass
common/lib/xmodule/xmodule/errortracker.py 0 → 100644
import logging
import sys
import traceback
from collections import namedtuple
log = logging.getLogger(__name__)
ErrorLog = namedtuple('ErrorLog', 'tracker errors')
def exc_info_to_str(exc_info):
"""Given some exception info, convert it into a string using
the traceback.format_exception() function.
"""
return ''.join(traceback.format_exception(*exc_info))
def in_exception_handler():
'''Is there an active exception?'''
return sys.exc_info() != (None, None, None)
def make_error_tracker():
'''Return an ErrorLog (named tuple), with fields (tracker, errors), where
the logger appends a tuple (message, exception_str) to the errors on every
call. exception_str is in the format returned by traceback.format_exception.
error_list is a simple list. If the caller modifies it, info
will be lost.
'''
errors = []
def error_tracker(msg):
'''Log errors'''
exc_str = ''
if in_exception_handler():
exc_str = exc_info_to_str(sys.exc_info())
errors.append((msg, exc_str))
return ErrorLog(error_tracker, errors)
def null_error_tracker(msg):
'''A dummy error tracker that just ignores the messages'''
pass
common/lib/xmodule/xmodule/html_checker.py 0 → 100644
from lxml import etree
def check_html(html):
'''
Check whether the passed in html string can be parsed by lxml.
Return bool success.
'''
parser = etree.HTMLParser()
try:
etree.fromstring(html, parser)
return True
except Exception as err:
pass
return False
common/lib/xmodule/xmodule/html_module.py
import copy
from fs.errors import ResourceNotFoundError
import logging
import os
import sys
from lxml import etree
from xmodule.x_module import XModule
from xmodule.raw_module import RawDescriptor
from .x_module import XModule
from .xml_module import XmlDescriptor
from .editing_module import EditingDescriptor
from .stringify import stringify_children
from .html_checker import check_html
log = logging.getLogger("mitx.courseware")
class HtmlModule(XModule):
def get_html(self):
return self.html
......@@ -19,33 +24,110 @@ class HtmlModule(XModule):
self.html = self.definition['data']
class HtmlDescriptor(RawDescriptor):
class HtmlDescriptor(XmlDescriptor, EditingDescriptor):
"""
Module for putting raw html in a course
"""
mako_template = "widgets/html-edit.html"
module_class = HtmlModule
filename_extension = "html"
filename_extension = "xml"
# TODO (cpennington): Delete this method once all fall 2012 course are being
# edited in the cms
# VS[compat] TODO (cpennington): Delete this method once all fall 2012 course
# are being edited in the cms
@classmethod
def backcompat_paths(cls, path):
if path.endswith('.html.html'):
path = path[:-5]
origpath = path
if path.endswith('.html.xml'):
path = path[:-9] + '.html' #backcompat--look for html instead of xml
candidates = []
while os.sep in path:
candidates.append(path)
_, _, path = path.partition(os.sep)
# also look for .html versions instead of .xml
if origpath.endswith('.xml'):
candidates.append(origpath[:-4] + '.html')
return candidates
# NOTE: html descriptors are special. We do not want to parse and
# export them ourselves, because that can break things (e.g. lxml
# adds body tags when it exports, but they should just be html
# snippets that will be included in the middle of pages.
@classmethod
def file_to_xml(cls, file_object):
parser = etree.HTMLParser()
return etree.parse(file_object, parser).getroot()
def load_definition(cls, xml_object, system, location):
'''Load a descriptor from the specified xml_object:
If there is a filename attribute, load it as a string, and
log a warning if it is not parseable by etree.HTMLParser.
If there is not a filename attribute, the definition is the body
of the xml_object, without the root tag (do not want <html> in the
middle of a page)
'''
filename = xml_object.get('filename')
if filename is None:
definition_xml = copy.deepcopy(xml_object)
cls.clean_metadata_from_xml(definition_xml)
return {'data' : stringify_children(definition_xml)}
else:
filepath = cls._format_filepath(xml_object.tag, filename)
# VS[compat]
# TODO (cpennington): If the file doesn't exist at the right path,
# give the class a chance to fix it up. The file will be written out
# again in the correct format. This should go away once the CMS is
# online and has imported all current (fall 2012) courses from xml
if not system.resources_fs.exists(filepath):
candidates = cls.backcompat_paths(filepath)
#log.debug("candidates = {0}".format(candidates))
for candidate in candidates:
if system.resources_fs.exists(candidate):
filepath = candidate
break
try:
with system.resources_fs.open(filepath) as file:
html = file.read()
# Log a warning if we can't parse the file, but don't error
if not check_html(html):
msg = "Couldn't parse html in {0}.".format(filepath)
log.warning(msg)
system.error_tracker("Warning: " + msg)
return {'data' : html}
except (ResourceNotFoundError) as err:
msg = 'Unable to load file contents at path {0}: {1} '.format(
filepath, err)
# add more info and re-raise
raise Exception(msg), None, sys.exc_info()[2]
@classmethod
def split_to_file(cls, xml_object):
# never include inline html
'''Never include inline html'''
return True
# TODO (vshnayder): make export put things in the right places.
def definition_to_xml(self, resource_fs):
'''If the contents are valid xml, write them to filename.xml. Otherwise,
write just the <html filename=""> tag to filename.xml, and the html
string to filename.html.
'''
try:
return etree.fromstring(self.definition['data'])
except etree.XMLSyntaxError:
pass
# Not proper format. Write html to file, return an empty tag
filepath = u'{category}/{name}.html'.format(category=self.category,
name=self.url_name)
resource_fs.makedir(os.path.dirname(filepath), allow_recreate=True)
with resource_fs.open(filepath, 'w') as file:
file.write(self.definition['data'])
elt = etree.Element('html')
elt.set("filename", self.url_name)
return elt
common/lib/xmodule/xmodule/mako_module.py
......@@ -2,10 +2,10 @@ from x_module import XModuleDescriptor, DescriptorSystem
class MakoDescriptorSystem(DescriptorSystem):
def __init__(self, load_item, resources_fs, error_handler,
render_template):
def __init__(self, load_item, resources_fs, error_tracker,
render_template, **kwargs):
super(MakoDescriptorSystem, self).__init__(
load_item, resources_fs, error_handler)
load_item, resources_fs, error_tracker, **kwargs)
self.render_template = render_template
......
common/lib/xmodule/xmodule/modulestore/__init__.py
......@@ -3,10 +3,13 @@ This module provides an abstraction for working with XModuleDescriptors
that are stored in a database an accessible using their Location as an identifier
"""
import logging
import re
from collections import namedtuple
from .exceptions import InvalidLocationError
import logging
from .exceptions import InvalidLocationError, InsufficientSpecificationError
from xmodule.errortracker import ErrorLog, make_error_tracker
log = logging.getLogger('mitx.' + 'modulestore')
......@@ -38,15 +41,15 @@ class Location(_LocationBase):
'''
__slots__ = ()
@classmethod
def clean(cls, value):
@staticmethod
def clean(value):
"""
Return value, made into a form legal for locations
"""
return re.sub('_+', '_', INVALID_CHARS.sub('_', value))
@classmethod
def is_valid(cls, value):
@staticmethod
def is_valid(value):
'''
Check if the value is a valid location, in any acceptable format.
'''
......@@ -56,6 +59,21 @@ class Location(_LocationBase):
return False
return True
@staticmethod
def ensure_fully_specified(location):
'''Make sure location is valid, and fully specified. Raises
InvalidLocationError or InsufficientSpecificationError if not.
returns a Location object corresponding to location.
'''
loc = Location(location)
for key, val in loc.dict().iteritems():
if key != 'revision' and val is None:
raise InsufficientSpecificationError(location)
return loc
def __new__(_cls, loc_or_tag=None, org=None, course=None, category=None,
name=None, revision=None):
"""
......@@ -198,6 +216,18 @@ class ModuleStore(object):
"""
raise NotImplementedError
def get_item_errors(self, location):
"""
Return a list of (msg, exception-or-None) errors that the modulestore
encountered when loading the item at location.
location : something that can be passed to Location
Raises the same exceptions as get_item if the location isn't found or
isn't fully specified.
"""
raise NotImplementedError
def get_items(self, location, depth=0):
"""
Returns a list of XModuleDescriptor instances for the items
......@@ -254,25 +284,47 @@ class ModuleStore(object):
'''
raise NotImplementedError
def path_to_location(self, location, course=None, chapter=None, section=None):
def get_parent_locations(self, location):
'''Find all locations that are the parents of this location. Needed
for path_to_location().
returns an iterable of things that can be passed to Location.
'''
Try to find a course/chapter/section[/position] path to this location.
raise NotImplementedError
raise ItemNotFoundError if the location doesn't exist.
If course, chapter, section are not None, restrict search to paths with those
components as specified.
raise NoPathToItem if the location exists, but isn't accessible via
a path that matches the course/chapter/section restrictions.
class ModuleStoreBase(ModuleStore):
'''
Implement interface functionality that can be shared.
'''
def __init__(self):
'''
Set up the error-tracking logic.
'''
self._location_errors = {} # location -> ErrorLog
In general, a location may be accessible via many paths. This method may
return any valid path.
def _get_errorlog(self, location):
"""
If we already have an errorlog for this location, return it. Otherwise,
create one.
"""
location = Location(location)
if location not in self._location_errors:
self._location_errors[location] = make_error_tracker()
return self._location_errors[location]
Return a tuple (course, chapter, section, position).
def get_item_errors(self, location):
"""
Return list of errors for this location, if any. Raise the same
errors as get_item if location isn't present.
If the section a sequence, position should be the position of this location
in that sequence. Otherwise, position should be None.
'''
raise NotImplementedError
NOTE: For now, the only items that track errors are CourseDescriptors in
the xml datastore. This will return an empty list for all other items
and datastores.
"""
# check that item is present and raise the promised exceptions if needed
self.get_item(location)
errorlog = self._get_errorlog(location)
return errorlog.errors
common/lib/xmodule/xmodule/modulestore/mongo.py
......@@ -6,14 +6,13 @@ from itertools import repeat
from path import path
from importlib import import_module
from xmodule.errorhandlers import strict_error_handler
from xmodule.errortracker import null_error_tracker
from xmodule.x_module import XModuleDescriptor
from xmodule.mako_module import MakoDescriptorSystem
from xmodule.course_module import CourseDescriptor
from mitxmako.shortcuts import render_to_string
from . import ModuleStore, Location
from .exceptions import (ItemNotFoundError, InsufficientSpecificationError,
from . import ModuleStoreBase, Location
from .exceptions import (ItemNotFoundError,
NoPathToItem, DuplicateItemError)
# TODO (cpennington): This code currently operates under the assumption that
......@@ -27,7 +26,7 @@ class CachingDescriptorSystem(MakoDescriptorSystem):
from, with a backup of calling to the underlying modulestore for more data
"""
def __init__(self, modulestore, module_data, default_class, resources_fs,
error_handler, render_template):
error_tracker, render_template):
"""
modulestore: the module store that can be used to retrieve additional modules
......@@ -39,13 +38,13 @@ class CachingDescriptorSystem(MakoDescriptorSystem):
resources_fs: a filesystem, as per MakoDescriptorSystem
error_handler:
error_tracker: a function that logs errors for later display to users
render_template: a function for rendering templates, as per
MakoDescriptorSystem
"""
super(CachingDescriptorSystem, self).__init__(
self.load_item, resources_fs, error_handler, render_template)
self.load_item, resources_fs, error_tracker, render_template)
self.modulestore = modulestore
self.module_data = module_data
self.default_class = default_class
......@@ -74,13 +73,17 @@ def location_to_query(location):
return query
class MongoModuleStore(ModuleStore):
class MongoModuleStore(ModuleStoreBase):
"""
A Mongodb backed ModuleStore
"""
# TODO (cpennington): Enable non-filesystem filestores
def __init__(self, host, db, collection, fs_root, port=27017, default_class=None):
def __init__(self, host, db, collection, fs_root, port=27017, default_class=None,
error_tracker=null_error_tracker):
ModuleStoreBase.__init__(self)
self.collection = pymongo.connection.Connection(
host=host,
port=port
......@@ -91,13 +94,17 @@ class MongoModuleStore(ModuleStore):
# Force mongo to maintain an index over _id.* that is in the same order
# that is used when querying by a location
self.collection.ensure_index(zip(('_id.' + field for field in Location._fields), repeat(1)))
self.collection.ensure_index(
zip(('_id.' + field for field in Location._fields), repeat(1)))
# TODO (vshnayder): default arg default_class=None will make this error
module_path, _, class_name = default_class.rpartition('.')
class_ = getattr(import_module(module_path), class_name)
self.default_class = class_
if default_class is not None:
module_path, _, class_name = default_class.rpartition('.')
class_ = getattr(import_module(module_path), class_name)
self.default_class = class_
else:
self.default_class = None
self.fs_root = path(fs_root)
self.error_tracker = error_tracker
def _clean_item_data(self, item):
"""
......@@ -149,7 +156,7 @@ class MongoModuleStore(ModuleStore):
data_cache,
self.default_class,
resource_fs,
strict_error_handler,
self.error_tracker,
render_to_string,
)
return system.load_item(item['location'])
......@@ -172,12 +179,17 @@ class MongoModuleStore(ModuleStore):
return self.get_items(course_filter)
def _find_one(self, location):
'''Look for a given location in the collection.
If revision isn't specified, returns the latest.'''
return self.collection.find_one(
'''Look for a given location in the collection. If revision is not
specified, returns the latest. If the item is not present, raise
ItemNotFoundError.
'''
item = self.collection.find_one(
location_to_query(location),
sort=[('revision', pymongo.ASCENDING)],
)
if item is None:
raise ItemNotFoundError(location)
return item
def get_item(self, location, depth=0):
"""
......@@ -197,14 +209,8 @@ class MongoModuleStore(ModuleStore):
calls to get_children() to cache. None indicates to cache all descendents.
"""
for key, val in Location(location).dict().iteritems():
if key != 'revision' and val is None:
raise InsufficientSpecificationError(location)
location = Location.ensure_fully_specified(location)
item = self._find_one(location)
if item is None:
raise ItemNotFoundError(location)
return self._load_items([item], depth)[0]
def get_items(self, location, depth=0):
......@@ -282,96 +288,20 @@ class MongoModuleStore(ModuleStore):
)
def get_parent_locations(self, location):
'''Find all locations that are the parents of this location.
Mostly intended for use in path_to_location, but exposed for testing
and possible other usefulness.
'''Find all locations that are the parents of this location. Needed
for path_to_location().
If there is no data at location in this modulestore, raise
ItemNotFoundError.
returns an iterable of things that can be passed to Location.
returns an iterable of things that can be passed to Location. This may
be empty if there are no parents.
'''
location = Location(location)
items = self.collection.find({'definition.children': str(location)},
location = Location.ensure_fully_specified(location)
# Check that it's actually in this modulestore.
item = self._find_one(location)
# now get the parents
items = self.collection.find({'definition.children': location.url()},
{'_id': True})
return [i['_id'] for i in items]
def path_to_location(self, location, course_name=None):
'''
Try to find a course_id/chapter/section[/position] path to this location.
The courseware insists that the first level in the course is chapter,
but any kind of module can be a "section".
location: something that can be passed to Location
course_name: [optional]. If not None, restrict search to paths
in that course.
raise ItemNotFoundError if the location doesn't exist.
raise NoPathToItem if the location exists, but isn't accessible via
a chapter/section path in the course(s) being searched.
Return a tuple (course_id, chapter, section, position) suitable for the
courseware index view.
A location may be accessible via many paths. This method may
return any valid path.
If the section is a sequence, position will be the position
of this location in that sequence. Otherwise, position will
be None. TODO (vshnayder): Not true yet.
'''
# Check that location is present at all
if self._find_one(location) is None:
raise ItemNotFoundError(location)
def flatten(xs):
'''Convert lisp-style (a, (b, (c, ()))) lists into a python list.
Not a general flatten function. '''
p = []
while xs != ():
p.append(xs[0])
xs = xs[1]
return p
def find_path_to_course(location, course_name=None):
'''Find a path up the location graph to a node with the
specified category. If no path exists, return None. If a
path exists, return it as a list with target location
first, and the starting location last.
'''
# Standard DFS
# To keep track of where we came from, the work queue has
# tuples (location, path-so-far). To avoid lots of
# copying, the path-so-far is stored as a lisp-style
# list--nested hd::tl tuples, and flattened at the end.
queue = [(location, ())]
while len(queue) > 0:
(loc, path) = queue.pop() # Takes from the end
loc = Location(loc)
# print 'Processing loc={0}, path={1}'.format(loc, path)
if loc.category == "course":
if course_name is None or course_name == loc.name:
# Found it!
path = (loc, path)
return flatten(path)
# otherwise, add parent locations at the end
newpath = (loc, path)
parents = self.get_parent_locations(loc)
queue.extend(zip(parents, repeat(newpath)))
# If we're here, there is no path
return None
path = find_path_to_course(location, course_name)
if path is None:
raise(NoPathToItem(location))
n = len(path)
course_id = CourseDescriptor.location_to_id(path[0])
chapter = path[1].name if n > 1 else None
section = path[2].name if n > 2 else None
# TODO (vshnayder): not handling position at all yet...
position = None
return (course_id, chapter, section, position)
common/lib/xmodule/xmodule/modulestore/search.py 0 → 100644
from itertools import repeat
from xmodule.course_module import CourseDescriptor
from .exceptions import (ItemNotFoundError, NoPathToItem)
from . import ModuleStore, Location
def path_to_location(modulestore, location, course_name=None):
'''
Try to find a course_id/chapter/section[/position] path to location in
modulestore. The courseware insists that the first level in the course is
chapter, but any kind of module can be a "section".
location: something that can be passed to Location
course_name: [optional]. If not None, restrict search to paths
in that course.
raise ItemNotFoundError if the location doesn't exist.
raise NoPathToItem if the location exists, but isn't accessible via
a chapter/section path in the course(s) being searched.
Return a tuple (course_id, chapter, section, position) suitable for the
courseware index view.
A location may be accessible via many paths. This method may
return any valid path.
If the section is a sequence, position will be the position
of this location in that sequence. Otherwise, position will
be None. TODO (vshnayder): Not true yet.
'''
def flatten(xs):
'''Convert lisp-style (a, (b, (c, ()))) list into a python list.
Not a general flatten function. '''
p = []
while xs != ():
p.append(xs[0])
xs = xs[1]
return p
def find_path_to_course(location, course_name=None):
'''Find a path up the location graph to a node with the
specified category.
If no path exists, return None.
If a path exists, return it as a list with target location first, and
the starting location last.
'''
# Standard DFS
# To keep track of where we came from, the work queue has
# tuples (location, path-so-far). To avoid lots of
# copying, the path-so-far is stored as a lisp-style
# list--nested hd::tl tuples, and flattened at the end.
queue = [(location, ())]
while len(queue) > 0:
(loc, path) = queue.pop() # Takes from the end
loc = Location(loc)
# get_parent_locations should raise ItemNotFoundError if location
# isn't found so we don't have to do it explicitly. Call this
# first to make sure the location is there (even if it's a course, and
# we would otherwise immediately exit).
parents = modulestore.get_parent_locations(loc)
# print 'Processing loc={0}, path={1}'.format(loc, path)
if loc.category == "course":
if course_name is None or course_name == loc.name:
# Found it!
path = (loc, path)
return flatten(path)
# otherwise, add parent locations at the end
newpath = (loc, path)
queue.extend(zip(parents, repeat(newpath)))
# If we're here, there is no path
return None
path = find_path_to_course(location, course_name)
if path is None:
raise(NoPathToItem(location))
n = len(path)
course_id = CourseDescriptor.location_to_id(path[0])
# pull out the location names
chapter = path[1].name if n > 1 else None
section = path[2].name if n > 2 else None
# TODO (vshnayder): not handling position at all yet...
position = None
return (course_id, chapter, section, position)
common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py
......@@ -8,6 +8,7 @@ from xmodule.modulestore import Location
from xmodule.modulestore.exceptions import InvalidLocationError, ItemNotFoundError, NoPathToItem
from xmodule.modulestore.mongo import MongoModuleStore
from xmodule.modulestore.xml_importer import import_from_xml
from xmodule.modulestore.search import path_to_location
# from ~/mitx_all/mitx/common/lib/xmodule/xmodule/modulestore/tests/
# to ~/mitx_all/mitx/common/test
......@@ -28,7 +29,7 @@ DEFAULT_CLASS = 'xmodule.raw_module.RawDescriptor'
class TestMongoModuleStore(object):
'''Tests!'''
@classmethod
def setupClass(cls):
cls.connection = pymongo.connection.Connection(HOST, PORT)
......@@ -67,7 +68,7 @@ class TestMongoModuleStore(object):
def test_init(self):
'''Make sure the db loads, and print all the locations in the db.
Call this directly from failing tests to see what's loaded'''
Call this directly from failing tests to see what is loaded'''
ids = list(self.connection[DB][COLLECTION].find({}, {'_id': True}))
pprint([Location(i['_id']).url() for i in ids])
......@@ -93,8 +94,6 @@ class TestMongoModuleStore(object):
self.store.get_item("i4x://edX/toy/video/Welcome"),
None)
def test_find_one(self):
assert_not_equals(
self.store._find_one(Location("i4x://edX/toy/course/2012_Fall")),
......@@ -117,13 +116,13 @@ class TestMongoModuleStore(object):
("edX/toy/2012_Fall", "Overview", "Toy_Videos", None)),
)
for location, expected in should_work:
assert_equals(self.store.path_to_location(location), expected)
assert_equals(path_to_location(self.store, location), expected)
not_found = (
"i4x://edX/toy/video/WelcomeX",
"i4x://edX/toy/video/WelcomeX", "i4x://edX/toy/course/NotHome"
)
for location in not_found:
assert_raises(ItemNotFoundError, self.store.path_to_location, location)
assert_raises(ItemNotFoundError, path_to_location, self.store, location)
# Since our test files are valid, there shouldn't be any
# elements with no path to them. But we can look for them in
......@@ -132,5 +131,5 @@ class TestMongoModuleStore(object):
"i4x://edX/simple/video/Lost_Video",
)
for location in no_path:
assert_raises(NoPathToItem, self.store.path_to_location, location, "toy")
assert_raises(NoPathToItem, path_to_location, self.store, location, "toy")
common/lib/xmodule/xmodule/modulestore/xml.py
import logging
import os
import re
from fs.osfs import OSFS
from importlib import import_module
from lxml import etree
from path import path
from xmodule.errorhandlers import logging_error_handler
from xmodule.errortracker import ErrorLog, make_error_tracker
from xmodule.x_module import XModuleDescriptor, XMLParsingSystem
from xmodule.course_module import CourseDescriptor
from xmodule.mako_module import MakoDescriptorSystem
from cStringIO import StringIO
import os
import re
from . import ModuleStore, Location
from . import ModuleStoreBase, Location
from .exceptions import ItemNotFoundError
etree.set_default_parser(
......@@ -19,7 +21,6 @@ etree.set_default_parser(
log = logging.getLogger('mitx.' + __name__)
# VS[compat]
# TODO (cpennington): Remove this once all fall 2012 courses have been imported
# into the cms from xml
......@@ -29,7 +30,7 @@ def clean_out_mako_templating(xml_string):
return xml_string
class ImportSystem(XMLParsingSystem, MakoDescriptorSystem):
def __init__(self, xmlstore, org, course, course_dir, error_handler):
def __init__(self, xmlstore, org, course, course_dir, error_tracker, **kwargs):
"""
A class that handles loading from xml. Does some munging to ensure that
all elements have unique slugs.
......@@ -40,6 +41,9 @@ class ImportSystem(XMLParsingSystem, MakoDescriptorSystem):
self.used_slugs = set()
def process_xml(xml):
"""Takes an xml string, and returns a XModuleDescriptor created from
that xml.
"""
try:
# VS[compat]
# TODO (cpennington): Remove this once all fall 2012 courses
......@@ -70,37 +74,36 @@ class ImportSystem(XMLParsingSystem, MakoDescriptorSystem):
# log.debug('-> slug=%s' % slug)
xml_data.set('url_name', slug)
module = XModuleDescriptor.load_from_xml(
descriptor = XModuleDescriptor.load_from_xml(
etree.tostring(xml_data), self, org,
course, xmlstore.default_class)
#log.debug('==> importing module location %s' % repr(module.location))
module.metadata['data_dir'] = course_dir
#log.debug('==> importing descriptor location %s' %
# repr(descriptor.location))
descriptor.metadata['data_dir'] = course_dir
xmlstore.modules[module.location] = module
xmlstore.modules[descriptor.location] = descriptor
if xmlstore.eager:
module.get_children()
return module
descriptor.get_children()
return descriptor
render_template = lambda: ''
load_item = xmlstore.get_item
resources_fs = OSFS(xmlstore.data_dir / course_dir)
MakoDescriptorSystem.__init__(self, load_item, resources_fs,
error_handler, render_template)
error_tracker, render_template, **kwargs)
XMLParsingSystem.__init__(self, load_item, resources_fs,
error_handler, process_xml)
error_tracker, process_xml, **kwargs)
class XMLModuleStore(ModuleStore):
class XMLModuleStore(ModuleStoreBase):
"""
An XML backed ModuleStore
"""
def __init__(self, data_dir, default_class=None, eager=False,
course_dirs=None,
error_handler=logging_error_handler):
course_dirs=None):
"""
Initialize an XMLModuleStore from data_dir
......@@ -114,17 +117,13 @@ class XMLModuleStore(ModuleStore):
course_dirs: If specified, the list of course_dirs to load. Otherwise,
load all course dirs
error_handler: The error handler used here and in the underlying
DescriptorSystem. By default, raise exceptions for all errors.
See the comments in x_module.py:DescriptorSystem
"""
ModuleStoreBase.__init__(self)
self.eager = eager
self.data_dir = path(data_dir)
self.modules = {} # location -> XModuleDescriptor
self.courses = {} # course_dir -> XModuleDescriptor for the course
self.error_handler = error_handler
if default_class is None:
self.default_class = None
......@@ -148,15 +147,20 @@ class XMLModuleStore(ModuleStore):
for course_dir in course_dirs:
try:
course_descriptor = self.load_course(course_dir)
# Special-case code here, since we don't have a location for the
# course before it loads.
# So, make a tracker to track load-time errors, then put in the right
# place after the course loads and we have its location
errorlog = make_error_tracker()
course_descriptor = self.load_course(course_dir, errorlog.tracker)
self.courses[course_dir] = course_descriptor
self._location_errors[course_descriptor.location] = errorlog
except:
msg = "Failed to load course '%s'" % course_dir
log.exception(msg)
error_handler(msg)
def load_course(self, course_dir):
def load_course(self, course_dir, tracker):
"""
Load a course into this module store
course_path: Course directory name
......@@ -190,13 +194,13 @@ class XMLModuleStore(ModuleStore):
))
course = course_dir
system = ImportSystem(self, org, course, course_dir,
self.error_handler)
system = ImportSystem(self, org, course, course_dir, tracker)
course_descriptor = system.process_xml(etree.tostring(course_data))
log.debug('========> Done with course import from {0}'.format(course_dir))
return course_descriptor
def get_item(self, location, depth=0):
"""
Returns an XModuleDescriptor instance for the item at location.
......@@ -217,15 +221,19 @@ class XMLModuleStore(ModuleStore):
except KeyError:
raise ItemNotFoundError(location)
def get_courses(self, depth=0):
"""
Returns a list of course descriptors
Returns a list of course descriptors. If there were errors on loading,
some of these may be ErrorDescriptors instead.
"""
return self.courses.values()
def create_item(self, location):
raise NotImplementedError("XMLModuleStores are read-only")
def update_item(self, location, data):
"""
Set the data in the item specified by the location to
......@@ -236,6 +244,7 @@ class XMLModuleStore(ModuleStore):
"""
raise NotImplementedError("XMLModuleStores are read-only")
def update_children(self, location, children):
"""
Set the children for the item specified by the location to
......@@ -246,6 +255,7 @@ class XMLModuleStore(ModuleStore):
"""
raise NotImplementedError("XMLModuleStores are read-only")
def update_metadata(self, location, metadata):
"""
Set the metadata for the item specified by the location to
......
common/lib/xmodule/xmodule/modulestore/xml_importer.py
......@@ -35,6 +35,8 @@ def import_from_xml(store, data_dir, course_dirs=None, eager=True,
store.update_item(module.location, module.definition['data'])
if 'children' in module.definition:
store.update_children(module.location, module.definition['children'])
store.update_metadata(module.location, dict(module.metadata))
# NOTE: It's important to use own_metadata here to avoid writing
# inherited metadata everywhere.
store.update_metadata(module.location, dict(module.own_metadata))
return module_store
common/lib/xmodule/xmodule/raw_module.py
from pkg_resources import resource_string
from lxml import etree
from xmodule.mako_module import MakoModuleDescriptor
from xmodule.editing_module import EditingDescriptor
from xmodule.xml_module import XmlDescriptor
import logging
import sys
log = logging.getLogger(__name__)
class RawDescriptor(MakoModuleDescriptor, XmlDescriptor):
class RawDescriptor(XmlDescriptor, EditingDescriptor):
"""
Module that provides a raw editing view of its data and children
Module that provides a raw editing view of its data and children. It
requires that the definition xml is valid.
"""
mako_template = "widgets/raw-edit.html"
js = {'coffee': [resource_string(__name__, 'js/src/raw/edit.coffee')]}
js_module_name = "RawDescriptor"
def get_context(self):
return {
'module': self,
'data': self.definition['data'],
}
@classmethod
def definition_from_xml(cls, xml_object, system):
return {'data': etree.tostring(xml_object)}
......@@ -30,13 +19,12 @@ class RawDescriptor(MakoModuleDescriptor, XmlDescriptor):
try:
return etree.fromstring(self.definition['data'])
except etree.XMLSyntaxError as err:
# Can't recover here, so just add some info and
# re-raise
lines = self.definition['data'].split('\n')
line, offset = err.position
msg = ("Unable to create xml for problem {loc}. "
"Context: '{context}'".format(
context=lines[line - 1][offset - 40:offset + 40],
loc=self.location))
log.exception(msg)
self.system.error_handler(msg)
# no workaround possible, so just re-raise
raise
raise Exception, msg, sys.exc_info()[2]
common/lib/xmodule/xmodule/stringify.py 0 → 100644
from itertools import chain
from lxml import etree
def stringify_children(node):
'''
Return all contents of an xml tree, without the outside tags.
e.g. if node is parse of
"<html a="b" foo="bar">Hi <div>there <span>Bruce</span><b>!</b></div><html>"
should return
"Hi <div>there <span>Bruce</span><b>!</b></div>"
fixed from
http://stackoverflow.com/questions/4624062/get-all-text-inside-a-tag-in-lxml
'''
parts = ([node.text] +
list(chain(*([etree.tostring(c), c.tail]
for c in node.getchildren())
)))
# filter removes possible Nones in texts and tails
return ''.join(filter(None, parts))
common/lib/xmodule/xmodule/template_module.py
......@@ -7,16 +7,14 @@ from mako.template import Template
class CustomTagModule(XModule):
"""
This module supports tags of the form
<customtag option="val" option2="val2">
<impl>$tagname</impl>
</customtag>
<customtag option="val" option2="val2" impl="tagname"/>
In this case, $tagname should refer to a file in data/custom_tags, which contains
a mako template that uses ${option} and ${option2} for the content.
For instance:
data/custom_tags/book::
data/mycourse/custom_tags/book::
More information given in <a href="/book/${page}">the text</a>
course.xml::
......@@ -34,7 +32,18 @@ class CustomTagModule(XModule):
instance_state, shared_state, **kwargs)
xmltree = etree.fromstring(self.definition['data'])
template_name = xmltree.attrib['impl']
if 'impl' in xmltree.attrib:
template_name = xmltree.attrib['impl']
else:
# VS[compat] backwards compatibility with old nested customtag structure
child_impl = xmltree.find('impl')
if child_impl is not None:
template_name = child_impl.text
else:
# TODO (vshnayder): better exception type
raise Exception("Could not find impl attribute in customtag {0}"
.format(location))
params = dict(xmltree.items())
with self.system.filestore.open(
'custom_tags/{name}'.format(name=template_name)) as template:
......
common/lib/xmodule/tests/__init__.py → common/lib/xmodule/xmodule/tests/__init__.py
......@@ -31,7 +31,8 @@ i4xs = ModuleSystem(
user=Mock(),
filestore=fs.osfs.OSFS(os.path.dirname(os.path.realpath(__file__))),
debug=True,
xqueue_callback_url='/'
xqueue_callback_url='/',
is_staff=False
)
......
common/lib/xmodule/tests/test_export.py → common/lib/xmodule/xmodule/tests/test_export.py
from xmodule.modulestore.xml import XMLModuleStore
from nose.tools import assert_equals
from nose import SkipTest
from tempfile import mkdtemp
from fs.osfs import OSFS
......@@ -26,3 +27,10 @@ def check_export_roundtrip(data_dir):
for location in initial_import.modules.keys():
print "Checking", location
assert_equals(initial_import.modules[location], second_import.modules[location])
def test_toy_roundtrip():
dir = ""
# TODO: add paths and make this run.
raise SkipTest()
check_export_roundtrip(dir)
common/lib/xmodule/xmodule/tests/test_import.py 0 → 100644
from path import path
import unittest
from fs.memoryfs import MemoryFS
from lxml import etree
from xmodule.x_module import XMLParsingSystem, XModuleDescriptor
from xmodule.errortracker import make_error_tracker
from xmodule.modulestore import Location
from xmodule.modulestore.exceptions import ItemNotFoundError
ORG = 'test_org'
COURSE = 'test_course'
class DummySystem(XMLParsingSystem):
def __init__(self):
self.modules = {}
self.resources_fs = MemoryFS()
self.errorlog = make_error_tracker()
def load_item(loc):
loc = Location(loc)
if loc in self.modules:
return self.modules[loc]
print "modules: "
print self.modules
raise ItemNotFoundError("Can't find item at loc: {0}".format(loc))
def process_xml(xml):
print "loading {0}".format(xml)
descriptor = XModuleDescriptor.load_from_xml(xml, self, ORG, COURSE, None)
# Need to save module so we can find it later
self.modules[descriptor.location] = descriptor
# always eager
descriptor.get_children()
return descriptor
XMLParsingSystem.__init__(self, load_item, self.resources_fs,
self.errorlog.tracker, process_xml)
def render_template(self, template, context):
raise Exception("Shouldn't be called")
class ImportTestCase(unittest.TestCase):
'''Make sure module imports work properly, including for malformed inputs'''
@staticmethod
def get_system():
'''Get a dummy system'''
return DummySystem()
def test_fallback(self):
'''Make sure that malformed xml loads as an ErrorDescriptor.'''
bad_xml = '''<sequential display_name="oops"><video url="hi"></sequential>'''
system = self.get_system()
descriptor = XModuleDescriptor.load_from_xml(bad_xml, system, 'org', 'course',
None)
self.assertEqual(descriptor.__class__.__name__,
'ErrorDescriptor')
def test_reimport(self):
'''Make sure an already-exported error xml tag loads properly'''
self.maxDiff = None
bad_xml = '''<sequential display_name="oops"><video url="hi"></sequential>'''
system = self.get_system()
descriptor = XModuleDescriptor.load_from_xml(bad_xml, system, 'org', 'course',
None)
resource_fs = None
tag_xml = descriptor.export_to_xml(resource_fs)
re_import_descriptor = XModuleDescriptor.load_from_xml(tag_xml, system,
'org', 'course',
None)
self.assertEqual(re_import_descriptor.__class__.__name__,
'ErrorDescriptor')
self.assertEqual(descriptor.definition['data'],
re_import_descriptor.definition['data'])
def test_fixed_xml_tag(self):
"""Make sure a tag that's been fixed exports as the original tag type"""
# create a error tag with valid xml contents
root = etree.Element('error')
good_xml = '''<sequential display_name="fixed"><video url="hi"/></sequential>'''
root.text = good_xml
xml_str_in = etree.tostring(root)
# load it
system = self.get_system()
descriptor = XModuleDescriptor.load_from_xml(xml_str_in, system, 'org', 'course',
None)
# export it
resource_fs = None
xml_str_out = descriptor.export_to_xml(resource_fs)
# Now make sure the exported xml is a sequential
xml_out = etree.fromstring(xml_str_out)
self.assertEqual(xml_out.tag, 'sequential')
def test_metadata_inherit(self):
"""Make sure metadata inherits properly"""
system = self.get_system()
v = "1 hour"
start_xml = '''<course graceperiod="{grace}" url_name="test1" display_name="myseq">
<chapter url="hi" url_name="ch" display_name="CH">
<html url_name="h" display_name="H">Two houses, ...</html></chapter>
</course>'''.format(grace=v)
descriptor = XModuleDescriptor.load_from_xml(start_xml, system,
'org', 'course')
print "Errors: {0}".format(system.errorlog.errors)
print descriptor, descriptor.metadata
self.assertEqual(descriptor.metadata['graceperiod'], v)
# Check that the child inherits correctly
child = descriptor.get_children()[0]
self.assertEqual(child.metadata['graceperiod'], v)
# Now export and see if the chapter tag has a graceperiod attribute
resource_fs = MemoryFS()
exported_xml = descriptor.export_to_xml(resource_fs)
print "Exported xml:", exported_xml
root = etree.fromstring(exported_xml)
chapter_tag = root[0]
self.assertEqual(chapter_tag.tag, 'chapter')
self.assertFalse('graceperiod' in chapter_tag.attrib)
common/lib/xmodule/xmodule/tests/test_stringify.py 0 → 100644
from nose.tools import assert_equals
from lxml import etree
from xmodule.stringify import stringify_children
def test_stringify():
text = 'Hi <div x="foo">there <span>Bruce</span><b>!</b></div>'
html = '''<html a="b" foo="bar">{0}</html>'''.format(text)
xml = etree.fromstring(html)
out = stringify_children(xml)
assert_equals(out, text)
common/lib/xmodule/xmodule/video_module.py
......@@ -23,11 +23,12 @@ class VideoModule(XModule):
css = {'scss': [resource_string(__name__, 'css/video/display.scss')]}
js_module_name = "Video"
def __init__(self, system, location, definition, instance_state=None, shared_state=None, **kwargs):
XModule.__init__(self, system, location, definition, instance_state, shared_state, **kwargs)
def __init__(self, system, location, definition,
instance_state=None, shared_state=None, **kwargs):
XModule.__init__(self, system, location, definition,
instance_state, shared_state, **kwargs)
xmltree = etree.fromstring(self.definition['data'])
self.youtube = xmltree.get('youtube')
self.name = xmltree.get('name')
self.position = 0
if instance_state is not None:
......@@ -71,7 +72,7 @@ class VideoModule(XModule):
'streams': self.video_list(),
'id': self.location.html_id(),
'position': self.position,
'name': self.name,
'display_name': self.display_name,
# TODO (cpennington): This won't work when we move to data that isn't on the filesystem
'data_dir': self.metadata['data_dir'],
})
......
common/lib/xmodule/xmodule/x_module.py
from lxml import etree
import pkg_resources
import logging
import pkg_resources
import sys
from xmodule.modulestore import Location
from fs.errors import ResourceNotFoundError
from functools import partial
from lxml import etree
from lxml.etree import XMLSyntaxError
from xmodule.modulestore import Location
from xmodule.errortracker import exc_info_to_str
log = logging.getLogger('mitx.' + __name__)
......@@ -187,13 +191,19 @@ class XModule(HTMLSnippet):
self.instance_state = instance_state
self.shared_state = shared_state
self.id = self.location.url()
self.name = self.location.name
self.url_name = self.location.name
self.category = self.location.category
self.metadata = kwargs.get('metadata', {})
self._loaded_children = None
def get_name(self):
return self.name
@property
def display_name(self):
'''
Return a display name for the module: use display_name if defined in
metadata, otherwise convert the url name.
'''
return self.metadata.get('display_name',
self.url_name.replace('_', ' '))
def get_children(self):
'''
......@@ -338,6 +348,8 @@ class XModuleDescriptor(Plugin, HTMLSnippet):
module
display_name: The name to use for displaying this module to the
user
url_name: The name to use for this module in urls and other places
where a unique name is needed.
format: The format of this module ('Homework', 'Lab', etc)
graded (bool): Whether this module is should be graded or not
start (string): The date for which this module will be available
......@@ -352,13 +364,30 @@ class XModuleDescriptor(Plugin, HTMLSnippet):
self.metadata = kwargs.get('metadata', {})
self.definition = definition if definition is not None else {}
self.location = Location(kwargs.get('location'))
self.name = self.location.name
self.url_name = self.location.name
self.category = self.location.category
self.shared_state_key = kwargs.get('shared_state_key')
self._child_instances = None
self._inherited_metadata = set()
@property
def display_name(self):
'''
Return a display name for the module: use display_name if defined in
metadata, otherwise convert the url name.
'''
return self.metadata.get('display_name',
self.url_name.replace('_', ' '))
@property
def own_metadata(self):
"""
Return the metadata that is not inherited, but was defined on this module.
"""
return dict((k,v) for k,v in self.metadata.items()
if k not in self._inherited_metadata)
def inherit_metadata(self, metadata):
"""
Updates this module with metadata inherited from a containing module.
......@@ -443,16 +472,32 @@ class XModuleDescriptor(Plugin, HTMLSnippet):
system is an XMLParsingSystem
org and course are optional strings that will be used in the generated
modules url identifiers
module's url identifiers
"""
class_ = XModuleDescriptor.load_class(
etree.fromstring(xml_data).tag,
default_class
)
# leave next line, commented out - useful for low-level debugging
# log.debug('[XModuleDescriptor.load_from_xml] tag=%s, class_=%s' % (
# etree.fromstring(xml_data).tag,class_))
return class_.from_xml(xml_data, system, org, course)
try:
class_ = XModuleDescriptor.load_class(
etree.fromstring(xml_data).tag,
default_class
)
# leave next line, commented out - useful for low-level debugging
# log.debug('[XModuleDescriptor.load_from_xml] tag=%s, class_=%s' % (
# etree.fromstring(xml_data).tag,class_))
descriptor = class_.from_xml(xml_data, system, org, course)
except Exception as err:
# Didn't load properly. Fall back on loading as an error
# descriptor. This should never error due to formatting.
# Put import here to avoid circular import errors
from xmodule.error_module import ErrorDescriptor
msg = "Error loading from xml."
log.exception(msg)
system.error_tracker(msg)
err_msg = msg + "\n" + exc_info_to_str(sys.exc_info())
descriptor = ErrorDescriptor.from_xml(xml_data, system, org, course,
err_msg)
return descriptor
@classmethod
def from_xml(cls, xml_data, system, org=None, course=None):
......@@ -521,16 +566,19 @@ class XModuleDescriptor(Plugin, HTMLSnippet):
class DescriptorSystem(object):
def __init__(self, load_item, resources_fs, error_handler):
def __init__(self, load_item, resources_fs, error_tracker, **kwargs):
"""
load_item: Takes a Location and returns an XModuleDescriptor
resources_fs: A Filesystem object that contains all of the
resources needed for the course
error_handler: A hook for handling errors in loading the descriptor.
Must be a function of (error_msg, exc_info=None).
See errorhandlers.py for some simple ones.
error_tracker: A hook for tracking errors in loading the descriptor.
Used for example to get a list of all non-fatal problems on course
load, and display them to the user.
A function of (error_msg). errortracker.py provides a
handy make_error_tracker() function.
Patterns for using the error handler:
try:
......@@ -539,10 +587,8 @@ class DescriptorSystem(object):
except SomeProblem:
msg = 'Grommet {0} is broken'.format(x)
log.exception(msg) # don't rely on handler to log
self.system.error_handler(msg)
# if we get here, work around if possible
raise # if no way to work around
OR
self.system.error_tracker(msg)
# work around
return 'Oops, couldn't load grommet'
OR, if not in an exception context:
......@@ -550,25 +596,27 @@ class DescriptorSystem(object):
if not check_something(thingy):
msg = "thingy {0} is broken".format(thingy)
log.critical(msg)
error_handler(msg)
# if we get here, work around
pass # e.g. if no workaround needed
self.system.error_tracker(msg)
NOTE: To avoid duplication, do not call the tracker on errors
that you're about to re-raise---let the caller track them.
"""
self.load_item = load_item
self.resources_fs = resources_fs
self.error_handler = error_handler
self.error_tracker = error_tracker
class XMLParsingSystem(DescriptorSystem):
def __init__(self, load_item, resources_fs, error_handler, process_xml):
def __init__(self, load_item, resources_fs, error_tracker, process_xml, **kwargs):
"""
load_item, resources_fs, error_handler: see DescriptorSystem
load_item, resources_fs, error_tracker: see DescriptorSystem
process_xml: Takes an xml string, and returns a XModuleDescriptor
created from that xml
"""
DescriptorSystem.__init__(self, load_item, resources_fs, error_handler)
DescriptorSystem.__init__(self, load_item, resources_fs, error_tracker,
**kwargs)
self.process_xml = process_xml
......@@ -584,10 +632,18 @@ class ModuleSystem(object):
Note that these functions can be closures over e.g. a django request
and user, or other environment-specific info.
'''
def __init__(self, ajax_url, track_function,
get_module, render_template, replace_urls,
user=None, filestore=None, debug=False,
xqueue_callback_url=None):
def __init__(self,
ajax_url,
track_function,
get_module,
render_template,
replace_urls,
user=None,
filestore=None,
debug=False,
xqueue_callback_url=None,
xqueue_default_queuename="null",
is_staff=False):
'''
Create a closure around the system environment.
......@@ -613,9 +669,13 @@ class ModuleSystem(object):
replace_urls - TEMPORARY - A function like static_replace.replace_urls
that capa_module can use to fix up the static urls in
ajax results.
is_staff - Is the user making the request a staff user?
TODO (vshnayder): this will need to change once we have real user roles.
'''
self.ajax_url = ajax_url
self.xqueue_callback_url = xqueue_callback_url
self.xqueue_default_queuename = xqueue_default_queuename
self.track_function = track_function
self.filestore = filestore
self.get_module = get_module
......@@ -623,6 +683,7 @@ class ModuleSystem(object):
self.DEBUG = self.debug = debug
self.seed = user.id if user is not None else 0
self.replace_urls = replace_urls
self.is_staff = is_staff
def get(self, attr):
''' provide uniform access to attributes (like etree).'''
......
common/lib/xmodule/xmodule/xml_module.py
from collections import MutableMapping
from xmodule.x_module import XModuleDescriptor
from xmodule.modulestore import Location
from lxml import etree
......@@ -8,74 +7,12 @@ import traceback
from collections import namedtuple
from fs.errors import ResourceNotFoundError
import os
import sys
log = logging.getLogger(__name__)
# TODO (cpennington): This was implemented in an attempt to improve performance,
# but the actual improvement wasn't measured (and it was implemented late at night).
# We should check if it hurts, and whether there's a better way of doing lazy loading
class LazyLoadingDict(MutableMapping):
"""
A dictionary object that lazily loads its contents from a provided
function on reads (of members that haven't already been set).
"""
def __init__(self, loader):
'''
On the first read from this dictionary, it will call loader() to
populate its contents. loader() must return something dict-like. Any
elements set before the first read will be preserved.
'''
self._contents = {}
self._loaded = False
self._loader = loader
self._deleted = set()
def __getitem__(self, name):
if not (self._loaded or name in self._contents or name in self._deleted):
self.load()
return self._contents[name]
def __setitem__(self, name, value):
self._contents[name] = value
self._deleted.discard(name)
def __delitem__(self, name):
del self._contents[name]
self._deleted.add(name)
def __contains__(self, name):
self.load()
return name in self._contents
def __len__(self):
self.load()
return len(self._contents)
def __iter__(self):
self.load()
return iter(self._contents)
def __repr__(self):
self.load()
return repr(self._contents)
def load(self):
if self._loaded:
return
loaded_contents = self._loader()
loaded_contents.update(self._contents)
self._contents = loaded_contents
self._loaded = True
_AttrMapBase = namedtuple('_AttrMap', 'metadata_key to_metadata from_metadata')
class AttrMap(_AttrMapBase):
"""
A class that specifies a metadata_key, and two functions:
......@@ -164,6 +101,45 @@ class XmlDescriptor(XModuleDescriptor):
return etree.parse(file_object).getroot()
@classmethod
def load_definition(cls, xml_object, system, location):
'''Load a descriptor definition from the specified xml_object.
Subclasses should not need to override this except in special
cases (e.g. html module)'''
filename = xml_object.get('filename')
if filename is None:
definition_xml = copy.deepcopy(xml_object)
else:
filepath = cls._format_filepath(xml_object.tag, filename)
# VS[compat]
# TODO (cpennington): If the file doesn't exist at the right path,
# give the class a chance to fix it up. The file will be written out
# again in the correct format. This should go away once the CMS is
# online and has imported all current (fall 2012) courses from xml
if not system.resources_fs.exists(filepath) and hasattr(
cls,
'backcompat_paths'):
candidates = cls.backcompat_paths(filepath)
for candidate in candidates:
if system.resources_fs.exists(candidate):
filepath = candidate
break
try:
with system.resources_fs.open(filepath) as file:
definition_xml = cls.file_to_xml(file)
except Exception:
msg = 'Unable to load file contents at path %s for item %s' % (
filepath, location.url())
# Add info about where we are, but keep the traceback
raise Exception, msg, sys.exc_info()[2]
cls.clean_metadata_from_xml(definition_xml)
return cls.definition_from_xml(definition_xml, system)
@classmethod
def from_xml(cls, xml_data, system, org=None, course=None):
"""
Creates an instance of this descriptor from the supplied xml_data.
......@@ -180,7 +156,7 @@ class XmlDescriptor(XModuleDescriptor):
slug = xml_object.get('url_name', xml_object.get('slug'))
location = Location('i4x', org, course, xml_object.tag, slug)
def metadata_loader():
def load_metadata():
metadata = {}
for attr in cls.metadata_attributes:
val = xml_object.get(attr)
......@@ -192,49 +168,15 @@ class XmlDescriptor(XModuleDescriptor):
metadata[attr_map.metadata_key] = attr_map.to_metadata(val)
return metadata
def definition_loader():
filename = xml_object.get('filename')
if filename is None:
definition_xml = copy.deepcopy(xml_object)
else:
filepath = cls._format_filepath(xml_object.tag, filename)
# VS[compat]
# TODO (cpennington): If the file doesn't exist at the right path,
# give the class a chance to fix it up. The file will be written out again
# in the correct format.
# This should go away once the CMS is online and has imported all current (fall 2012)
# courses from xml
if not system.resources_fs.exists(filepath) and hasattr(cls, 'backcompat_paths'):
candidates = cls.backcompat_paths(filepath)
for candidate in candidates:
if system.resources_fs.exists(candidate):
filepath = candidate
break
try:
with system.resources_fs.open(filepath) as file:
definition_xml = cls.file_to_xml(file)
except (ResourceNotFoundError, etree.XMLSyntaxError):
msg = 'Unable to load file contents at path %s for item %s' % (filepath, location.url())
log.exception(msg)
system.error_handler(msg)
# if error_handler didn't reraise, work around problem.
error_elem = etree.Element('error')
message_elem = etree.SubElement(error_elem, 'error_message')
message_elem.text = msg
stack_elem = etree.SubElement(error_elem, 'stack_trace')
stack_elem.text = traceback.format_exc()
return {'data': etree.tostring(error_elem)}
cls.clean_metadata_from_xml(definition_xml)
return cls.definition_from_xml(definition_xml, system)
definition = cls.load_definition(xml_object, system, location)
metadata = load_metadata()
# VS[compat] -- just have the url_name lookup once translation is done
slug = xml_object.get('url_name', xml_object.get('slug'))
return cls(
system,
LazyLoadingDict(definition_loader),
definition,
location=location,
metadata=LazyLoadingDict(metadata_loader),
metadata=metadata,
)
@classmethod
......@@ -282,8 +224,8 @@ class XmlDescriptor(XModuleDescriptor):
# Write it to a file if necessary
if self.split_to_file(xml_object):
# Put this object in it's own file
filepath = self.__class__._format_filepath(self.category, self.name)
# Put this object in its own file
filepath = self.__class__._format_filepath(self.category, self.url_name)
resource_fs.makedir(os.path.dirname(filepath), allow_recreate=True)
with resource_fs.open(filepath, 'w') as file:
file.write(etree.tostring(xml_object, pretty_print=True))
......@@ -296,10 +238,10 @@ class XmlDescriptor(XModuleDescriptor):
xml_object.tail = ''
xml_object.set('filename', self.name)
xml_object.set('filename', self.url_name)
# Add the metadata
xml_object.set('url_name', self.name)
xml_object.set('url_name', self.url_name)
for attr in self.metadata_attributes:
attr_map = self.xml_attribute_map.get(attr, AttrMap(attr))
metadata_key = attr_map.metadata_key
......
create-dev-env.sh
......@@ -17,8 +17,10 @@ ouch() {
!! ERROR !!
The last command did not complete successfully,
see $LOG for more details or trying running the
For more details or trying running the
script again with the -v flag.
Output of the script is recorded in $LOG
EOL
printf '\E[0m'
......@@ -36,7 +38,7 @@ usage() {
Usage: $PROG [-c] [-v] [-h]
-c compile scipy and numpy
-s do _not_ set --no-site-packages for virtualenv
-s give access to global site-packages for virtualenv
-v set -x + spew
-h this
......@@ -61,28 +63,21 @@ clone_repos() {
if [[ -d "$BASE/mitx/.git" ]]; then
output "Pulling mitx"
cd "$BASE/mitx"
git pull >>$LOG
git pull
else
output "Cloning mitx"
if [[ -d "$BASE/mitx" ]]; then
mv "$BASE/mitx" "${BASE}/mitx.bak.$$"
fi
git clone git@github.com:MITx/mitx.git >>$LOG
git clone git@github.com:MITx/mitx.git
fi
cd "$BASE"
if [[ -d "$BASE/askbot-devel/.git" ]]; then
output "Pulling askbot-devel"
cd "$BASE/askbot-devel"
git pull >>$LOG
else
output "Cloning askbot-devel"
if [[ -d "$BASE/askbot-devel" ]]; then
mv "$BASE/askbot-devel" "${BASE}/askbot-devel.bak.$$"
fi
git clone git@github.com:MITx/askbot-devel >>$LOG
if [[ ! -d "$BASE/mitx/askbot/.git" ]]; then
output "Cloning askbot as a submodule of mitx"
cd "$BASE/mitx"
git submodule update --init
fi
# By default, dev environments start with a copy of 6.002x
cd "$BASE"
mkdir -p "$BASE/data"
......@@ -90,14 +85,14 @@ clone_repos() {
if [[ -d "$BASE/data/$REPO/.git" ]]; then
output "Pulling $REPO"
cd "$BASE/data/$REPO"
git pull >>$LOG
git pull
else
output "Cloning $REPO"
if [[ -d "$BASE/data/$REPO" ]]; then
mv "$BASE/data/$REPO" "${BASE}/data/$REPO.bak.$$"
fi
cd "$BASE/data"
git clone git@github.com:MITx/$REPO >>$LOG
git clone git@github.com:MITx/$REPO
fi
}
......@@ -109,8 +104,8 @@ RUBY_VER="1.9.3"
NUMPY_VER="1.6.2"
SCIPY_VER="0.10.1"
BREW_FILE="$BASE/mitx/brew-formulas.txt"
LOG="/var/tmp/install.log"
APT_PKGS="curl git mercurial python-virtualenv build-essential python-dev gfortran liblapack-dev libfreetype6-dev libpng12-dev libxml2-dev libxslt-dev yui-compressor coffeescript"
LOG="/var/tmp/install-$(date +%Y%m%d-%H%M%S).log"
APT_PKGS="curl git python-virtualenv build-essential python-dev gfortran liblapack-dev libfreetype6-dev libpng12-dev libxml2-dev libxslt-dev yui-compressor coffeescript"
if [[ $EUID -eq 0 ]]; then
error "This script should not be run using sudo or as the root user"
......@@ -163,23 +158,30 @@ cat<<EO
To compile scipy and numpy from source use the -c option
Most of STDOUT is redirected to /var/tmp/install.log, run
$ tail -f /var/tmp/install.log
to monitor progress
!!! Do not run this script from an existing virtualenv !!!
If you are in a ruby/python virtualenv please start a new
shell.
EO
info
output "Press return to begin or control-C to abort"
read dummy
if [[ -f $HOME/.rvmrc ]]; then
output "$HOME/.rvmrc alredy exists, not adding $RUBY_DIR"
else
output "Creating $HOME/.rmrc so rvm uses $RUBY_DIR"
# log all stdout and stderr
exec > >(tee $LOG)
exec 2>&1
if ! grep -q "export rvm_path=$RUBY_DIR" ~/.rvmrc; then
if [[ -f $HOME/.rvmrc ]]; then
output "Copying existing .rvmrc to .rvmrc.bak"
cp $HOME/.rvmrc $HOME/.rvmrc.bak
fi
output "Creating $HOME/.rvmrc so rvm uses $RUBY_DIR"
echo "export rvm_path=$RUBY_DIR" > $HOME/.rvmrc
fi
mkdir -p $BASE
rm -f $LOG
case `uname -s` in
[Ll]inux)
command -v lsb_release &>/dev/null || {
......@@ -201,17 +203,31 @@ case `uname -s` in
esac
;;
Darwin)
if [[ ! -w /usr/local ]]; then
cat<<EO
You need to be able to write to /usr/local for
the installation of brew and brew packages.
Either make sure the group you are in (most likely 'staff')
can write to that directory or simply execute the following
and re-run the script:
$ sudo chown -R $USER /usr/local
EO
exit 1
fi
command -v brew &>/dev/null || {
output "Installing brew"
/usr/bin/ruby -e "$(curl -fsSL https://raw.github.com/mxcl/homebrew/master/Library/Contributions/install_homebrew.rb)"
}
command -v git &>/dev/null || {
output "Installing git"
brew install git >> $LOG
}
command -v hg &>/dev/null || {
output "Installaing mercurial"
brew install mercurial >> $LOG
brew install git
}
clone_repos
......@@ -225,20 +241,22 @@ case `uname -s` in
for pkg in $(cat $BREW_FILE); do
grep $pkg <(brew list) &>/dev/null || {
output "Installing $pkg"
brew install $pkg >>$LOG
brew install $pkg
}
done
command -v pip &>/dev/null || {
output "Installing pip"
sudo easy_install pip >>$LOG
}
command -v virtualenv &>/dev/null || {
output "Installing virtualenv"
sudo pip install virtualenv virtualenvwrapper >> $LOG
sudo easy_install pip
}
if ! grep -Eq ^1.7 <(virtualenv --version 2>/dev/null); then
output "Installing virtualenv >1.7"
sudo pip install 'virtualenv>1.7' virtualenvwrapper
fi
command -v coffee &>/dev/null || {
output "Installing coffee script"
curl http://npmjs.org/install.sh | sh
curl --insecure https://npmjs.org/install.sh | sh
npm install -g coffee-script
}
;;
......@@ -253,10 +271,12 @@ curl -sL get.rvm.io | bash -s stable
source $RUBY_DIR/scripts/rvm
# skip the intro
LESS="-E" rvm install $RUBY_VER
if [[ -n $systempkgs ]]; then
virtualenv "$PYTHON_DIR"
if [[ $systempkgs ]]; then
virtualenv --system-site-packages "$PYTHON_DIR"
else
virtualenv --no-site-packages "$PYTHON_DIR"
# default behavior for virtualenv>1.7 is
# --no-site-packages
virtualenv "$PYTHON_DIR"
fi
source $PYTHON_DIR/bin/activate
output "Installing gem bundler"
......@@ -277,24 +297,24 @@ if [[ -n $compile ]]; then
rm -f numpy.tar.gz scipy.tar.gz
output "Compiling numpy"
cd "$BASE/numpy-${NUMPY_VER}"
python setup.py install >>$LOG 2>&1
python setup.py install
output "Compiling scipy"
cd "$BASE/scipy-${SCIPY_VER}"
python setup.py install >>$LOG 2>&1
python setup.py install
cd "$BASE"
rm -rf numpy-${NUMPY_VER} scipy-${SCIPY_VER}
fi
output "Installing askbot requirements"
pip install -r askbot-devel/askbot_requirements.txt >>$LOG
output "Installing askbot-dev requirements"
pip install -r askbot-devel/askbot_requirements_dev.txt >>$LOG
output "Installing MITx pre-requirements"
pip install -r mitx/pre-requirements.txt >> $LOG
pip install -r mitx/pre-requirements.txt
# Need to be in the mitx dir to get the paths to local modules right
output "Installing MITx requirements"
cd mitx
pip install -r requirements.txt >>$LOG
pip install -r requirements.txt
output "Installing askbot requirements"
pip install -r askbot/askbot_requirements.txt
pip install -r askbot/askbot_requirements_dev.txt
mkdir "$BASE/log" || true
mkdir "$BASE/db" || true
......
lms/askbot/skins/mitx/templates/base.html
......@@ -20,7 +20,7 @@
{% include "widgets/header.html" %} {# Logo, user tool navigation and meta navitation #}
{# include "widgets/secondary_header.html" #} {# Scope selector, search input and ask button #}
<section class="main-content">
<section class="container">
{% block body %}
{% endblock %}
</section>
......
lms/askbot/skins/mitx/templates/meta/html_head_stylesheets.html
{% load extra_filters_jinja %}
<!--<link href="{{"/style/style.css"|media }}" rel="stylesheet" type="text/css" />-->
{{ 'application' | compressed_css }}
{{ 'course' | compressed_css }}
lms/askbot/skins/mitx/templates/navigation.jinja.html
<header class="app" aria-label="Global Navigation">
<header class="global" aria-label="Global Navigation">
<nav>
<a href="{{ MITX_ROOT_URL }}" class="logo">
<img src="/static/images/logo.png" />
</a>
{%if request.user.is_authenticated(): %}
<h1>Circuits and Electronics</h1>
<ol class="user">
<li>
<a href="/dashboard" class="user-dashboard">
<span class="avatar"><img src="/static/images/profile.jpg" /></span>
{{ request.user.username }}
</a>
<h1 class="logo"><a href="${reverse('root')}"></a></h1>
<ol class="left">
<li class="primary">
<a href="${reverse('courses')}">Find Courses</a>
</li>
<li>
<a href="#" class="options">&#9662</a>
<ol class="user-options">
<li><a href="#">Account Settings</a></li>
<li><a href="/logout">Log Out</a></li>
</ol>
</li>
</ol>
{%else:%}
<ol>
<li><a href="/courses">Courses</a></li>
<li><a href="#">How It Works</a></li>
</ol>
<ol class="user">
<li><a href="/dashboard">Log In</a></li>
<li><a href="#">Sign Up</a></li>
</ol>
<ol class="secondary">
<li>
<a href="#">About</a>
</li>
<li>
<a href="#">Jobs</a>
<li class="primary">
<a href="${reverse('dashboard')}" class="user-link">
<span class="avatar"></span>
${user.username}
</a>
</li>
<li>
<a href="#">faq</a>
<li class="primary">
<a href="#" class="dropdown">&#9662</a>
<ul class="dropdown-menu">
## <li><a href="#">Account Settings</a></li>
<li><a href="${reverse('help_edx')}">Help</a></li>
<li><a href="${reverse('logout')}">Log Out</a></li>
</ul>
</li>
</ol>
{%endif %}
</nav>
</header>
lms/askbot/skins/mitx/templates/widgets/footer.html
<!-- template footer.html -->
<footer>
<!-- Template based on a design from http://www.dotemplate.com/ - Donated $10 (pmitros) so we don't need to include credit. -->
<p> Copyright &copy; 2012. MIT. <a href="/t/copyright.html">Some rights reserved.</a>
</p>
<nav>
<ul class="social">
<li class="linkedin">
<a href="http://www.linkedin.com/groups/Friends-Alumni-MITx-4316538">Linked In</a>
</li>
<li class="twitter">
<a href="https://twitter.com/#!/MyMITx">Twitter</a>
</li>
<li class="facebook">
<a href="http://www.facebook.com/pages/MITx/378592442151504">Facebook</a>
</li>
</ul>
<ul>
<footer>
<nav>
<section class="top">
<section class="primary">
<a href="${reverse('root')}" class="logo"></a>
<a href="${reverse('courses')}">Find Courses</a>
<a href="${reverse('about_edx')}">About</a>
<a href="http://edxonline.tumblr.com/">Blog</a>
<a href="${reverse('jobs')}">Jobs</a>
<a href="${reverse('contact')}">Contact</a>
</section>
<li><a href="/s/help.html">Help</a></li>
<li><a href="/logout">Log out</a></li>
</ul>
</nav>
</footer>
<section class="social">
<a href="http://youtube.com/user/edxonline"><img src="${static.url('images/social/youtube-sharing.png')}" /></a>
<a href="https://plus.google.com/108235383044095082735"><img src="${static.url('images/social/google-plus-sharing.png')}" /></a>
<a href="http://www.facebook.com/EdxOnline"><img src="${static.url('images/social/facebook-sharing.png')}" /></a>
<a href="https://twitter.com/edXOnline"><img src="${static.url('images/social/twitter-sharing.png')}" /></a>
</section>
</section>
<section class="bottom">
<section class="copyright">
<p style="float:left;">&copy; 2012 edX, some rights reserved.</p>
</section>
<section class="secondary">
<a href="${reverse('tos')}">Terms of Service</a>
<a href="${reverse('privacy_edx')}">Privacy Policy</a>
<a href="${reverse('honor')}">Honor Code</a>
<a href="${reverse('help_edx')}">Help</a>
</section>
</section>
</nav>
</footer>
<!-- end template footer.html -->
lms/djangoapps/courseware/courses.py
......@@ -33,6 +33,7 @@ def check_course(course_id, course_must_be_open=True, course_required=True):
try:
course_loc = CourseDescriptor.id_to_location(course_id)
course = modulestore().get_item(course_loc)
except (KeyError, ItemNotFoundError):
raise Http404("Course not found.")
......@@ -82,7 +83,7 @@ def get_course_about_section(course, section_key):
log.warning("Missing about section {key} in course {url}".format(key=section_key, url=course.location.url()))
return None
elif section_key == "title":
return course.metadata.get('display_name', course.name)
return course.metadata.get('display_name', course.url_name)
elif section_key == "university":
return course.location.org
elif section_key == "number":
......
lms/djangoapps/courseware/grades.py
......@@ -115,14 +115,17 @@ def grade_sheet(student, course, grader, student_module_cache):
This pulls a summary of all problems in the course.
Returns
- courseware_summary is a summary of all sections with problems in the course. It is organized as an array of chapters,
each containing an array of sections, each containing an array of scores. This contains information for graded and ungraded
problems, and is good for displaying a course summary with due dates, etc.
- courseware_summary is a summary of all sections with problems in the course.
It is organized as an array of chapters, each containing an array of sections,
each containing an array of scores. This contains information for graded and
ungraded problems, and is good for displaying a course summary with due dates,
etc.
Arguments:
student: A User object for the student to grade
course: An XModule containing the course to grade
student_module_cache: A StudentModuleCache initialized with all instance_modules for the student
student_module_cache: A StudentModuleCache initialized with all
instance_modules for the student
"""
chapters = []
for c in course.get_children():
......@@ -135,12 +138,16 @@ def grade_sheet(student, course, grader, student_module_cache):
if correct is None and total is None:
continue
scores.append(Score(correct, total, graded, module.metadata.get('display_name')))
scores.append(Score(correct, total, graded,
module.metadata.get('display_name')))
section_total, graded_total = graders.aggregate_scores(
scores, s.metadata.get('display_name'))
section_total, graded_total = graders.aggregate_scores(scores, s.metadata.get('display_name'))
format = s.metadata.get('format', "")
sections.append({
'section': s.metadata.get('display_name'),
'display_name': s.display_name,
'url_name': s.url_name,
'scores': scores,
'section_total': section_total,
'format': format,
......@@ -148,8 +155,9 @@ def grade_sheet(student, course, grader, student_module_cache):
'graded': graded,
})
chapters.append({'course': course.metadata.get('display_name'),
'chapter': c.metadata.get('display_name'),
chapters.append({'course': course.display_name,
'display_name': c.display_name,
'url_name': c.url_name,
'sections': sections})
return chapters
......
lms/djangoapps/courseware/management/commands/clean_xml.py
......@@ -10,37 +10,17 @@ from lxml import etree
from django.core.management.base import BaseCommand
from xmodule.modulestore.xml import XMLModuleStore
from xmodule.errortracker import make_error_tracker
def traverse_tree(course):
'''Load every descriptor in course. Return bool success value.'''
queue = [course]
while len(queue) > 0:
node = queue.pop()
# print '{0}:'.format(node.location)
# if 'data' in node.definition:
# print '{0}'.format(node.definition['data'])
queue.extend(node.get_children())
return True
def make_logging_error_handler():
'''Return a tuple (handler, error_list), where
the handler appends the message and any exc_info
to the error_list on every call.
'''
errors = []
def error_handler(msg, exc_info=None):
'''Log errors'''
if exc_info is None:
if sys.exc_info() != (None, None, None):
exc_info = sys.exc_info()
errors.append((msg, exc_info))
return (error_handler, errors)
def export(course, export_dir):
"""Export the specified course to course_dir. Creates dir if it doesn't exist.
......@@ -73,32 +53,18 @@ def import_with_checks(course_dir, verbose=True):
data_dir = course_dir.dirname()
course_dirs = [course_dir.basename()]
(error_handler, errors) = make_logging_error_handler()
# No default class--want to complain if it doesn't find plugins for any
# module.
modulestore = XMLModuleStore(data_dir,
default_class=None,
eager=True,
course_dirs=course_dirs,
error_handler=error_handler)
course_dirs=course_dirs)
def str_of_err(tpl):
(msg, exc_info) = tpl
if exc_info is None:
return msg
exc_str = '\n'.join(traceback.format_exception(*exc_info))
(msg, exc_str) = tpl
return '{msg}\n{exc}'.format(msg=msg, exc=exc_str)
courses = modulestore.get_courses()
if len(errors) != 0:
all_ok = False
print '\n'
print "=" * 40
print 'ERRORs during import:'
print '\n'.join(map(str_of_err,errors))
print "=" * 40
print '\n'
n = len(courses)
if n != 1:
......@@ -107,6 +73,16 @@ def import_with_checks(course_dir, verbose=True):
return (False, None)
course = courses[0]
errors = modulestore.get_item_errors(course.location)
if len(errors) != 0:
all_ok = False
print '\n'
print "=" * 40
print 'ERRORs during import:'
print '\n'.join(map(str_of_err, errors))
print "=" * 40
print '\n'
#print course
validators = (
......@@ -143,6 +119,7 @@ def check_roundtrip(course_dir):
# dircmp doesn't do recursive diffs.
# diff = dircmp(course_dir, export_dir, ignore=[], hide=[])
print "======== Roundtrip diff: ========="
sys.stdout.flush() # needed to make diff appear in the right place
os.system("diff -r {0} {1}".format(course_dir, export_dir))
print "======== ideally there is no diff above this ======="
......
lms/djangoapps/courseware/module_render.py
......@@ -35,10 +35,12 @@ def toc_for_course(user, request, course, active_chapter, active_section):
Create a table of contents from the module store
Return format:
[ {'name': name, 'sections': SECTIONS, 'active': bool}, ... ]
[ {'display_name': name, 'url_name': url_name,
'sections': SECTIONS, 'active': bool}, ... ]
where SECTIONS is a list
[ {'name': name, 'format': format, 'due': due, 'active' : bool}, ...]
[ {'display_name': name, 'url_name': url_name,
'format': format, 'due': due, 'active' : bool}, ...]
active is set for the section and chapter corresponding to the passed
parameters. Everything else comes from the xml, or defaults to "".
......@@ -54,19 +56,21 @@ def toc_for_course(user, request, course, active_chapter, active_section):
sections = list()
for section in chapter.get_display_items():
active = (chapter.metadata.get('display_name') == active_chapter and
section.metadata.get('display_name') == active_section)
active = (chapter.display_name == active_chapter and
section.display_name == active_section)
hide_from_toc = section.metadata.get('hide_from_toc', 'false').lower() == 'true'
if not hide_from_toc:
sections.append({'name': section.metadata.get('display_name'),
sections.append({'display_name': section.display_name,
'url_name': section.url_name,
'format': section.metadata.get('format', ''),
'due': section.metadata.get('due', ''),
'active': active})
chapters.append({'name': chapter.metadata.get('display_name'),
chapters.append({'display_name': chapter.display_name,
'url_name': chapter.url_name,
'sections': sections,
'active': chapter.metadata.get('display_name') == active_chapter})
'active': chapter.display_name == active_chapter})
return chapters
......@@ -76,8 +80,8 @@ def get_section(course_module, chapter, section):
or None if this doesn't specify a valid section
course: Course url
chapter: Chapter name
section: Section name
chapter: Chapter url_name
section: Section url_name
"""
if course_module is None:
......@@ -85,7 +89,7 @@ def get_section(course_module, chapter, section):
chapter_module = None
for _chapter in course_module.get_children():
if _chapter.metadata.get('display_name') == chapter:
if _chapter.url_name == chapter:
chapter_module = _chapter
break
......@@ -94,7 +98,7 @@ def get_section(course_module, chapter, section):
section_module = None
for _section in chapter_module.get_children():
if _section.metadata.get('display_name') == section:
if _section.url_name == section:
section_module = _section
break
......@@ -141,8 +145,16 @@ def get_module(user, request, location, student_module_cache, position=None):
# TODO (vshnayder): fix hardcoded urls (use reverse)
# Setup system context for module instance
ajax_url = settings.MITX_ROOT_URL + '/modx/' + descriptor.location.url() + '/'
xqueue_callback_url = (settings.MITX_ROOT_URL + '/xqueue/' +
str(user.id) + '/' + descriptor.location.url() + '/')
# Fully qualified callback URL for external queueing system
xqueue_callback_url = (request.build_absolute_uri('/') + settings.MITX_ROOT_URL +
'xqueue/' + str(user.id) + '/' + descriptor.location.url() + '/' +
'score_update')
# Default queuename is course-specific and is derived from the course that
# contains the current module.
# TODO: Queuename should be derived from 'course_settings.json' of each course
xqueue_default_queuename = descriptor.location.org + '-' + descriptor.location.course
def _get_module(location):
return get_module(user, request, location,
......@@ -155,6 +167,7 @@ def get_module(user, request, location, student_module_cache, position=None):
render_template=render_to_string,
ajax_url=ajax_url,
xqueue_callback_url=xqueue_callback_url,
xqueue_default_queuename=xqueue_default_queuename.replace(' ','_'),
# TODO (cpennington): Figure out how to share info between systems
filestore=descriptor.system.resources_fs,
get_module=_get_module,
......@@ -163,6 +176,7 @@ def get_module(user, request, location, student_module_cache, position=None):
# a module is coming through get_html and is therefore covered
# by the replace_static_urls code below
replace_urls=replace_urls,
is_staff=user.is_staff,
)
# pass position specified in URL to module through ModuleSystem
system.set('position', position)
......@@ -175,7 +189,7 @@ def get_module(user, request, location, student_module_cache, position=None):
)
if settings.MITX_FEATURES.get('DISPLAY_HISTOGRAMS_TO_STAFF') and user.is_staff:
module.get_html = add_histogram(module.get_html)
module.get_html = add_histogram(module.get_html, module)
return module
......@@ -232,13 +246,12 @@ def get_shared_instance_module(user, module, student_module_cache):
return None
# TODO: TEMPORARY BYPASS OF AUTH!
@csrf_exempt
def xqueue_callback(request, userid, id, dispatch):
# Parse xqueue response
get = request.POST.copy()
try:
header = json.loads(get.pop('xqueue_header')[0]) # 'dict'
header = json.loads(get['xqueue_header'])
except Exception as err:
msg = "Error in xqueue_callback %s: Invalid return format" % err
raise Exception(msg)
......@@ -261,7 +274,7 @@ def xqueue_callback(request, userid, id, dispatch):
# Transfer 'queuekey' from xqueue response header to 'get'. This is required to
# use the interface defined by 'handle_ajax'
get.update({'queuekey': header['queuekey']})
get.update({'queuekey': header['lms_key']})
# We go through the "AJAX" path
# So far, the only dispatch from xqueue will be 'score_update'
......
lms/djangoapps/courseware/tests/tests.py
import copy
import json
from path import path
import os
from pprint import pprint
from nose import SkipTest
from django.test import TestCase
from django.test.client import Client
from mock import patch, Mock
from override_settings import override_settings
from django.conf import settings
from django.core.urlresolvers import reverse
from path import path
from mock import patch, Mock
from override_settings import override_settings
from student.models import Registration
from django.contrib.auth.models import User
from student.models import Registration
from xmodule.modulestore.django import modulestore
import xmodule.modulestore.django
......@@ -189,11 +190,12 @@ class RealCoursesLoadTestCase(PageLoader):
xmodule.modulestore.django._MODULESTORES = {}
xmodule.modulestore.django.modulestore().collection.drop()
# TODO: Disabled test for now.. Fix once things are cleaned up.
def Xtest_real_courses_loads(self):
def test_real_courses_loads(self):
'''See if any real courses are available at the REAL_DATA_DIR.
If they are, check them.'''
# TODO: Disabled test for now.. Fix once things are cleaned up.
raise SkipTest
# TODO: adjust staticfiles_dirs
if not os.path.isdir(REAL_DATA_DIR):
# No data present. Just pass.
......
lms/djangoapps/courseware/views.py
......@@ -20,6 +20,7 @@ from module_render import toc_for_course, get_module, get_section
from models import StudentModuleCache
from student.models import UserProfile
from xmodule.modulestore import Location
from xmodule.modulestore.search import path_to_location
from xmodule.modulestore.exceptions import InvalidLocationError, ItemNotFoundError, NoPathToItem
from xmodule.modulestore.django import modulestore
from xmodule.course_module import CourseDescriptor
......@@ -54,15 +55,16 @@ def user_groups(user):
return group_names
def format_url_params(params):
return [urllib.quote(string.replace(' ', '_')) for string in params]
@ensure_csrf_cookie
@cache_if_anonymous
def courses(request):
# TODO: Clean up how 'error' is done.
courses = sorted(modulestore().get_courses(), key=lambda course: course.number)
# filter out any courses that errored.
courses = [c for c in modulestore().get_courses()
if isinstance(c, CourseDescriptor)]
courses = sorted(courses, key=lambda course: course.number)
universities = defaultdict(list)
for course in courses:
universities[course.org].append(course)
......@@ -140,9 +142,9 @@ def render_accordion(request, course, chapter, section):
If chapter and section are '' or None, renders a default accordion.
Returns (initialization_javascript, content)'''
Returns the html string'''
# TODO (cpennington): do the right thing with courses
# grab the table of contents
toc = toc_for_course(request.user, request, course, chapter, section)
active_chapter = 1
......@@ -154,8 +156,6 @@ def render_accordion(request, course, chapter, section):
('toc', toc),
('course_name', course.title),
('course_id', course.id),
#TODO: Do we need format_url_params anymore? What is a better way to create the reversed links?
('format_url_params', format_url_params),
('csrf', csrf(request)['csrf_token'])] + template_imports.items())
return render_to_string('accordion.html', context)
......@@ -172,9 +172,9 @@ def index(request, course_id, chapter=None, section=None,
Arguments:
- request : HTTP request
- course : coursename (str)
- chapter : chapter name (str)
- section : section name (str)
- course_id : course id (str: ORG/course/URL_NAME)
- chapter : chapter url_name (str)
- section : section url_name (str)
- position : position in module, eg of <sequential> module (str)
Returns:
......@@ -182,47 +182,58 @@ def index(request, course_id, chapter=None, section=None,
- HTTPresponse
'''
course = check_course(course_id)
def clean(s):
''' Fixes URLs -- we convert spaces to _ in URLs to prevent
funny encoding characters and keep the URLs readable. This undoes
that transformation.
'''
return s.replace('_', ' ') if s is not None else None
chapter = clean(chapter)
section = clean(section)
context = {
'csrf': csrf(request)['csrf_token'],
'accordion': render_accordion(request, course, chapter, section),
'COURSE_TITLE': course.title,
'course': course,
'init': '',
'content': ''
}
look_for_module = chapter is not None and section is not None
if look_for_module:
# TODO (cpennington): Pass the right course in here
section_descriptor = get_section(course, chapter, section)
if section_descriptor is not None:
student_module_cache = StudentModuleCache(request.user,
section_descriptor)
module = get_module(request.user, request,
section_descriptor.location,
student_module_cache)
context['content'] = module.get_html()
try:
context = {
'csrf': csrf(request)['csrf_token'],
'accordion': render_accordion(request, course, chapter, section),
'COURSE_TITLE': course.title,
'course': course,
'init': '',
'content': ''
}
look_for_module = chapter is not None and section is not None
if look_for_module:
section_descriptor = get_section(course, chapter, section)
if section_descriptor is not None:
student_module_cache = StudentModuleCache(request.user,
section_descriptor)
module, _, _, _ = get_module(request.user, request,
section_descriptor.location,
student_module_cache)
context['content'] = module.get_html()
else:
log.warning("Couldn't find a section descriptor for course_id '{0}',"
"chapter '{1}', section '{2}'".format(
course_id, chapter, section))
else:
log.warning("Couldn't find a section descriptor for course_id '{0}',"
"chapter '{1}', section '{2}'".format(
course_id, chapter, section))
if request.user.is_staff:
# Add a list of all the errors...
context['course_errors'] = modulestore().get_item_errors(course.location)
result = render_to_response('courseware.html', context)
except:
# In production, don't want to let a 500 out for any reason
if settings.DEBUG:
raise
else:
log.exception("Error in index view: user={user}, course={course},"
" chapter={chapter} section={section}"
"position={position}".format(
user=request.user,
course=course,
chapter=chapter,
section=section,
position=position
))
try:
result = render_to_response('courseware-error.html', {})
except:
result = HttpResponse("There was an unrecoverable error")
result = render_to_response('courseware.html', context)
return result
@ensure_csrf_cookie
def jump_to(request, location):
'''
......@@ -243,13 +254,13 @@ def jump_to(request, location):
# Complain if there's not data for this location
try:
(course_id, chapter, section, position) = modulestore().path_to_location(location)
(course_id, chapter, section, position) = path_to_location(modulestore(), location)
except ItemNotFoundError:
raise Http404("No data at this location: {0}".format(location))
except NoPathToItem:
raise Http404("This location is not in any class: {0}".format(location))
# Rely on index to do all error handling
return index(request, course_id, chapter, section, position)
@ensure_csrf_cookie
......
lms/djangoapps/dashboard/models.py 0 → 100644
from django.db import models
# Create your models here.
lms/djangoapps/dashboard/tests.py 0 → 100644
"""
This file demonstrates writing tests using the unittest module. These will pass
when you run "manage.py test".
Replace this with more appropriate tests for your application.
"""
from django.test import TestCase
class SimpleTest(TestCase):
def test_basic_addition(self):
"""
Tests that 1 + 1 always equals 2.
"""
self.assertEqual(1 + 1, 2)
lms/djangoapps/dashboard/views.py 0 → 100644
# Create your views here.
import json
from datetime import datetime
from django.http import HttpResponse, Http404
def dictfetchall(cursor):
'''Returns all rows from a cursor as a dict.
Borrowed from Django documentation'''
desc = cursor.description
return [
dict(zip([col[0] for col in desc], row))
for row in cursor.fetchall()
]
def dashboard(request):
"""
Quick hack to show staff enrollment numbers. This should be
replaced with a real dashboard later. This version is a short-term
bandaid for the next couple weeks.
"""
if not request.user.is_staff:
raise Http404
query = "select count(user_id) as students, course_id from student_courseenrollment group by course_id order by students desc"
from django.db import connection
cursor = connection.cursor()
cursor.execute(query)
results = dictfetchall(cursor)
return HttpResponse(json.dumps(results, indent=4))
lms/envs/common.py
......@@ -304,7 +304,7 @@ PIPELINE_CSS = {
'output_filename': 'css/lms-application.css',
},
'course': {
'source_filenames': ['sass/course.scss', 'js/vendor/CodeMirror/codemirror.css', 'css/vendor/jquery.treeview.css', 'css/vendor/ui-lightness/jquery-ui-1.8.22.custom.css', 'css/vendor/jquery.qtip.min.css'],
'source_filenames': ['js/vendor/CodeMirror/codemirror.css', 'css/vendor/jquery.treeview.css', 'css/vendor/ui-lightness/jquery-ui-1.8.22.custom.css', 'css/vendor/jquery.qtip.min.css', 'sass/course.scss'],
'output_filename': 'css/lms-course.css',
},
'ie-fixes': {
......
lms/envs/dev.py
......@@ -17,7 +17,7 @@ MITX_FEATURES['DISABLE_START_DATES'] = True
WIKI_ENABLED = True
LOGGING = get_logger_config(ENV_ROOT / "log",
LOGGING = get_logger_config(ENV_ROOT / "log",
logging_env="dev",
tracking_filename="tracking.log",
debug=True)
......@@ -30,7 +30,7 @@ DATABASES = {
}
CACHES = {
# This is the cache used for most things. Askbot will not work without a
# This is the cache used for most things. Askbot will not work without a
# functioning cache -- it relies on caching to load its settings in places.
# In staging/prod envs, the sessions also live here.
'default': {
......@@ -52,11 +52,29 @@ CACHES = {
}
}
# Make the keyedcache startup warnings go away
CACHE_TIMEOUT = 0
# Dummy secret key for dev
SECRET_KEY = '85920908f28904ed733fe576320db18cabd7b6cd'
################################ OpenID Auth #################################
MITX_FEATURES['AUTH_USE_OPENID'] = True
MITX_FEATURES['BYPASS_ACTIVATION_EMAIL_FOR_EXTAUTH'] = True
INSTALLED_APPS += ('external_auth',)
INSTALLED_APPS += ('django_openid_auth',)
OPENID_CREATE_USERS = False
OPENID_UPDATE_DETAILS_FROM_SREG = True
OPENID_SSO_SERVER_URL = 'https://www.google.com/accounts/o8/id' # TODO: accept more endpoints
OPENID_USE_AS_ADMIN_LOGIN = False
################################ MIT Certificates SSL Auth #################################
MITX_FEATURES['AUTH_USE_MIT_CERTIFICATES'] = True
################################ DEBUG TOOLBAR #################################
INSTALLED_APPS += ('debug_toolbar',)
INSTALLED_APPS += ('debug_toolbar',)
MIDDLEWARE_CLASSES += ('debug_toolbar.middleware.DebugToolbarMiddleware',)
INTERNAL_IPS = ('127.0.0.1',)
......@@ -71,8 +89,8 @@ DEBUG_TOOLBAR_PANELS = (
'debug_toolbar.panels.logger.LoggingPanel',
# Enabling the profiler has a weird bug as of django-debug-toolbar==0.9.4 and
# Django=1.3.1/1.4 where requests to views get duplicated (your method gets
# hit twice). So you can uncomment when you need to diagnose performance
# Django=1.3.1/1.4 where requests to views get duplicated (your method gets
# hit twice). So you can uncomment when you need to diagnose performance
# problems, but you shouldn't leave it on.
# 'debug_toolbar.panels.profiling.ProfilingDebugPanel',
)
......
lms/static/sass/application.scss
......@@ -2,9 +2,9 @@
@import 'base/reset';
@import 'base/font_face';
@import 'base/mixins';
@import 'base/variables';
@import 'base/base';
@import 'base/mixins';
@import 'base/extends';
@import 'base/animations';
......
lms/static/sass/base/_mixins.scss
@function em($pxval, $base: 16) {
@return #{$pxval / $base}em;
}
// Line-height
@function lh($amount: 1) {
@return $body-line-height * $amount;
......
lms/static/sass/base/_variables.scss
......@@ -4,10 +4,15 @@ $gw-gutter: 20px;
$fg-column: $gw-column;
$fg-gutter: $gw-gutter;
$fg-max-columns: 12;
$fg-max-width: 1400px;
$fg-min-width: 810px;
$sans-serif: 'Open Sans', $verdana;
$body-font-family: $sans-serif;
$serif: $georgia;
$body-font-size: em(14);
$body-line-height: golden-ratio(.875em, 1);
$base-font-color: rgb(60,60,60);
$lighter-base-font-color: rgb(160,160,160);
......@@ -15,18 +20,11 @@ $blue: rgb(29,157,217);
$pink: rgb(182,37,104);
$yellow: rgb(255, 252, 221);
$error-red: rgb(253, 87, 87);
$border-color: #C8C8C8;
// old variables
$body-font-family: "Open Sans", "Lucida Grande", "Lucida Sans Unicode", "Lucida Sans", Geneva, Verdana, sans-serif;
$body-font-size: 14px;
$body-line-height: golden-ratio($body-font-size, 1);
$fg-max-width: 1400px;
$fg-min-width: 810px;
$light-gray: #ddd;
$dark-gray: #333;
$mit-red: #993333;
$cream: #F6EFD4;
$text-color: $dark-gray;
$border-color: $light-gray;
lms/static/sass/course.scss
......@@ -2,9 +2,9 @@
@import 'base/reset';
@import 'base/font_face';
@import 'base/mixins';
@import 'base/variables';
@import 'base/base';
@import 'base/mixins';
@import 'base/extends';
@import 'base/animations';
......
lms/static/sass/course/_help.scss deleted 100644 → 0
section.help.main-content {
padding: lh();
h1 {
border-bottom: 1px solid #ddd;
margin-bottom: lh();
margin-top: 0;
padding-bottom: lh();
}
p {
max-width: 700px;
}
h2 {
margin-top: 0;
}
section.self-help {
float: left;
margin-bottom: lh();
margin-right: flex-gutter();
width: flex-grid(6);
ul {
margin-left: flex-gutter(6);
li {
margin-bottom: lh(.5);
}
}
}
section.help-email {
float: left;
width: flex-grid(6);
dl {
display: block;
margin-bottom: lh();
dd {
margin-bottom: lh();
}
dt {
clear: left;
float: left;
font-weight: bold;
width: flex-grid(2, 6);
}
}
}
}
lms/static/sass/course/_info.scss
......@@ -3,6 +3,7 @@ div.info-wrapper {
section.updates {
@extend .content;
line-height: lh();
> h1 {
@extend .top-header;
......@@ -15,30 +16,35 @@ div.info-wrapper {
> ol {
list-style: none;
padding-left: 0;
margin-bottom: lh();
> li {
@extend .clearfix;
border-bottom: 1px solid #e3e3e3;
margin-bottom: lh(.5);
margin-bottom: lh();
padding-bottom: lh(.5);
list-style-type: disk;
&:first-child {
background: $cream;
border-bottom: 1px solid darken($cream, 10%);
margin: 0 (-(lh(.5))) lh();
padding: lh(.5);
}
ol, ul {
margin: lh() 0 0 lh();
list-style-type: circle;
margin: 0;
list-style-type: disk;
ol,ul {
list-style-type: circle;
}
}
h2 {
float: left;
margin: 0 flex-gutter() 0 0;
width: flex-grid(2, 9);
font-size: $body-font-size;
font-weight: bold;
}
section.update-description {
......@@ -64,16 +70,20 @@ div.info-wrapper {
@extend .sidebar;
border-left: 1px solid #d3d3d3;
@include border-radius(0 4px 4px 0);
@include box-shadow(none);
border-right: 0;
header {
h1 {
@extend .bottom-border;
padding: lh(.5) lh(.75);
padding: lh(.5) lh(.5);
}
h1 {
font-size: 18px;
margin: 0 ;
}
header {
// h1 {
// font-weight: 100;
// font-style: italic;
// }
p {
color: #666;
......@@ -94,7 +104,7 @@ div.info-wrapper {
border-bottom: 1px solid #d3d3d3;
@include box-shadow(0 1px 0 #eee);
@include box-sizing(border-box);
padding: 7px lh(.75);
padding: em(7) lh(.75);
position: relative;
&.expandable,
......@@ -108,13 +118,13 @@ div.info-wrapper {
ul {
background: none;
margin: 7px (-(lh(.75))) 0;
margin: em(7) (-(lh(.75))) 0;
li {
border-bottom: 0;
border-top: 1px solid #d3d3d3;
@include box-shadow(inset 0 1px 0 #eee);
padding-left: 18px + lh(.75);
padding-left: lh(1.5);
}
}
......@@ -150,7 +160,7 @@ div.info-wrapper {
border-bottom: 0;
@include box-shadow(none);
color: #999;
font-size: 12px;
font-size: $body-font-size;
font-weight: bold;
text-transform: uppercase;
}
......
lms/static/sass/course/_textbook.scss
......@@ -62,7 +62,6 @@ div.book-wrapper {
@extend .clearfix;
li {
background-color: darken($cream, 4%);
&.last {
display: block;
......
lms/static/sass/course/base/_base.scss
......@@ -5,3 +5,17 @@ body {
h1, h2, h3, h4, h5, h6 {
font-family: $sans-serif;
}
table {
table-layout: fixed;
}
.container {
padding: lh(2);
> div {
display: table;
width: 100%;
table-layout: fixed;
}
}
lms/static/sass/course/base/_extends.scss
.wrapper {
margin: 0 auto;
max-width: $fg-max-width;
min-width: $fg-min-width;
text-align: left;
width: flex-grid(12);
div.table-wrapper {
display: table;
width: flex-grid(12);
overflow: hidden;
}
}
h1.top-header {
background: #f3f3f3;
border-bottom: 1px solid #e3e3e3;
margin: (-(lh())) (-(lh())) lh();
padding: lh();
text-align: left;
}
.button {
border: 1px solid darken(#888, 10%);
@include border-radius(3px);
@include box-shadow(inset 0 1px 0 lighten(#888, 10%), 0 0 3px #ccc);
color: #fff;
cursor: pointer;
font: bold $body-font-size $body-font-family;
@include linear-gradient(lighten(#888, 5%), darken(#888, 5%));
padding: 4px 8px;
text-decoration: none;
text-shadow: none;
-webkit-font-smoothing: antialiased;
&:hover, &:focus {
border: 1px solid darken(#888, 20%);
@include box-shadow(inset 0 1px 0 lighten(#888, 20%), 0 0 3px #ccc);
@include linear-gradient(lighten(#888, 10%), darken(#888, 5%));
}
font-size: 24px;
font-weight: 100;
padding-bottom: lh();
}
.light-button, a.light-button {
......@@ -84,7 +50,8 @@ h1.top-header {
}
.sidebar {
border-right: 1px solid #d3d3d3;
border-right: 1px solid #C8C8C8;
@include box-shadow(inset -1px 0 0 #e6e6e6);
@include box-sizing(border-box);
display: table-cell;
font-family: $sans-serif;
......@@ -93,11 +60,13 @@ h1.top-header {
width: flex-grid(3);
h1, h2 {
font-size: 18px;
font-weight: bold;
font-size: em(18);
font-weight: 100;
letter-spacing: 0;
text-transform: none;
font-family: $sans-serif;
text-align: left;
font-style: italic;
}
a {
......@@ -146,27 +115,20 @@ h1.top-header {
}
header#open_close_accordion {
border-bottom: 1px solid #d3d3d3;
@include box-shadow(0 1px 0 #eee);
padding: lh(.5) lh();
position: relative;
h2 {
margin: 0;
padding-right: 20px;
}
a {
background: #eee url('../images/slide-left-icon.png') center center no-repeat;
background: #f6f6f6 url('../images/slide-left-icon.png') center center no-repeat;
border: 1px solid #D3D3D3;
@include border-radius(3px 0 0 3px);
height: 16px;
padding: 8px;
padding: 6px;
position: absolute;
right: -1px;
text-indent: -9999px;
top: 6px;
width: 16px;
z-index: 99;
&:hover {
background-color: white;
......@@ -181,33 +143,17 @@ h1.top-header {
.topbar {
@extend .clearfix;
background: $cream;
border-bottom: 1px solid darken($cream, 10%);
border-top: 1px solid #fff;
font-size: 12px;
line-height: 46px;
text-shadow: 0 1px 0 #fff;
border-bottom: 1px solid $border-color;
font-size: 14px;
@media print {
display: none;
}
a {
line-height: 46px;
border-bottom: 0;
color: darken($cream, 80%);
&:hover {
color: darken($cream, 60%);
text-decoration: none;
}
&.block-link {
// background: darken($cream, 5%);
border-left: 1px solid darken($cream, 20%);
@include box-shadow(inset 1px 0 0 lighten($cream, 5%));
border-left: 1px solid lighten($border-color, 10%);
display: block;
text-transform: uppercase;
&:hover {
background: none;
......@@ -219,12 +165,3 @@ h1.top-header {
.tran {
@include transition( all, .2s, $ease-in-out-quad);
}
p.ie-warning {
background: yellow;
display: block !important;
line-height: 1.3em;
margin-bottom: 0;
padding: lh();
text-align: left;
}
lms/static/sass/course/courseware/_courseware.scss
......@@ -3,22 +3,6 @@ html {
max-height: 100%;
}
body.courseware {
height: 100%;
max-height: 100%;
.container {
padding-bottom: 40px;
margin-top: 20px;
}
footer {
&.fixed-bottom {
Position: static;
}
}
}
div.course-wrapper {
@extend .table-wrapper;
......@@ -197,17 +181,9 @@ div.course-wrapper {
overflow: hidden;
header#open_close_accordion {
padding: 0;
min-height: 47px;
a {
background-image: url('../images/slide-right-icon.png');
}
h2 {
visibility: hidden;
width: 10px;
}
}
div#accordion {
......
lms/static/sass/course/courseware/_sidebar.scss
......@@ -13,44 +13,51 @@ section.course-index {
div#accordion {
h3 {
@include box-shadow(inset 0 1px 0 0 #eee);
border-top: 1px solid #d3d3d3;
overflow: hidden;
@include border-radius(0);
border-top: 1px solid #e3e3e3;
margin: 0;
overflow: hidden;
&:first-child {
border: none;
}
&:hover {
@include background-image(linear-gradient(-90deg, rgb(245,245,245), rgb(225,225,225)));
background: #f6f6f6;
text-decoration: none;
}
&.ui-accordion-header {
color: #000;
a {
font-size: $body-font-size;
@include border-radius(0);
@include box-shadow(none);
color: lighten($text-color, 10%);
font-size: $body-font-size;
}
&.ui-state-active {
@include background-image(linear-gradient(-90deg, rgb(245,245,245), rgb(225,225,225)));
@extend .active;
border-bottom: 1px solid #d3d3d3;
border-bottom: none;
&:hover {
background: none;
}
}
}
}
ul.ui-accordion-content {
@include border-radius(0);
@include box-shadow(inset -1px 0 0 #e6e6e6);
background: transparent;
border: none;
font-size: 12px;
margin: 0;
padding: 1em 1.5em;
li {
@include border-radius(0);
margin-bottom: lh(.5);
a {
......@@ -98,7 +105,7 @@ section.course-index {
&:after {
opacity: 1;
right: 15px;
@include transition(all, 0.2s, linear);
@include transition();
}
> a p {
......@@ -120,8 +127,6 @@ section.course-index {
font-weight: bold;
> a {
background: rgb(240,240,240);
@include background-image(linear-gradient(-90deg, rgb(245,245,245), rgb(230,230,230)));
border-color: rgb(200,200,200);
&:after {
......
lms/static/sass/course/discussion/_discussion.scss
// Generic layout styles for the discussion forums
body.askbot {
section.main-content {
section.container {
div.discussion-wrapper {
@extend .table-wrapper;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment