Skip to content

E
edx-ora2
Commit 5cae4ead by Eric Fischer Committed by Andy Armstrong
Options

Enable Staff Grading 

Able to enable full staff grading from studio, save without errors, and view
the staff grading step as expected from LMS.

Includes:
    -template additions, staff grading disabled by default
    -fetching real staff requirements
    -javascript to hook into the python backend
    -changing some default assumptions about assessment types
    -modifications to validation
    -small bugfix for scroll behaviorwith validation alert.
    -updates to existing unit tests
    -additional validation, serialization, and javascript tests
    -fix for staff requirements bug (no more duble-unpacking)
    -updates the bundled studio js file
    -moving StaffWorkflow initilization to api's on_init() method
    -acceptance tests (lms)
    -addition of "Staff Assessment Section" to test course for bokchoy tests
    -renaming `requirements` to `<type>_requirements` in assessment apis
parent eb094edf
Showing with 568 additions and 112 deletions
openassessment/assessment/api/ai.py
......@@ -24,14 +24,14 @@ from openassessment.assessment.worker import grading as grading_tasks
logger = logging.getLogger(__name__)
def submitter_is_finished(submission_uuid, requirements):
def submitter_is_finished(submission_uuid, ai_requirements):
"""
Determine if the submitter has finished their requirements for Example
Based Assessment. Always returns True.
Args:
submission_uuid (str): Not used.
requirements (dict): Not used.
ai_requirements (dict): Not used.
Returns:
True
......@@ -40,14 +40,14 @@ def submitter_is_finished(submission_uuid, requirements):
return True
def assessment_is_finished(submission_uuid, requirements):
def assessment_is_finished(submission_uuid, ai_requirements):
"""
Determine if the assessment of the given submission is completed. This
checks to see if the AI has completed the assessment.
Args:
submission_uuid (str): The UUID of the submission being graded.
requirements (dict): Not used.
ai_requirements (dict): Not used.
Returns:
True if the assessment has been completed for this submission.
......@@ -56,7 +56,7 @@ def assessment_is_finished(submission_uuid, requirements):
return bool(get_latest_assessment(submission_uuid))
def get_score(submission_uuid, requirements):
def get_score(submission_uuid, ai_requirements):
"""
Generate a score based on a completed assessment for the given submission.
If no assessment has been completed for this submission, this will return
......@@ -64,7 +64,7 @@ def get_score(submission_uuid, requirements):
Args:
submission_uuid (str): The UUID for the submission to get a score for.
requirements (dict): Not used.
ai_requirements (dict): Not used.
Returns:
A dictionary with the points earned, points possible, and
......
openassessment/assessment/api/peer.py
......@@ -28,7 +28,7 @@ logger = logging.getLogger("openassessment.assessment.api.peer")
PEER_TYPE = "PE"
def submitter_is_finished(submission_uuid, requirements):
def submitter_is_finished(submission_uuid, peer_requirements):
"""
Check whether the submitter has made the required number of assessments.
......@@ -38,21 +38,21 @@ def submitter_is_finished(submission_uuid, requirements):
Args:
submission_uuid (str): The UUID of the submission being tracked.
requirements (dict): Dictionary with the key "must_grade" indicating
peer_requirements (dict): Dictionary with the key "must_grade" indicating
the required number of submissions the student must grade.
Returns:
bool
"""
if requirements is None:
if peer_requirements is None:
return False
try:
workflow = PeerWorkflow.objects.get(submission_uuid=submission_uuid)
if workflow.completed_at is not None:
return True
elif workflow.num_peers_graded() >= requirements["must_grade"]:
elif workflow.num_peers_graded() >= peer_requirements["must_grade"]:
workflow.completed_at = timezone.now()
workflow.save()
return True
......@@ -63,7 +63,7 @@ def submitter_is_finished(submission_uuid, requirements):
raise PeerAssessmentRequestError(u'Requirements dict must contain "must_grade" key')
def assessment_is_finished(submission_uuid, requirements):
def assessment_is_finished(submission_uuid, peer_requirements):
"""
Check whether the submitter has received enough assessments
to get a score.
......@@ -74,7 +74,7 @@ def assessment_is_finished(submission_uuid, requirements):
Args:
submission_uuid (str): The UUID of the submission being tracked.
requirements (dict): Dictionary with the key "must_be_graded_by"
peer_requirements (dict): Dictionary with the key "must_be_graded_by"
indicating the required number of assessments the student
must receive to get a score.
......@@ -82,7 +82,7 @@ def assessment_is_finished(submission_uuid, requirements):
bool
"""
if not requirements:
if not peer_requirements:
return False
workflow = PeerWorkflow.get_by_submission_uuid(submission_uuid)
......@@ -93,7 +93,7 @@ def assessment_is_finished(submission_uuid, requirements):
assessment__submission_uuid=submission_uuid,
assessment__score_type=PEER_TYPE
)
return scored_items.count() >= requirements["must_be_graded_by"]
return scored_items.count() >= peer_requirements["must_be_graded_by"]
def on_start(submission_uuid):
......@@ -137,7 +137,7 @@ def on_start(submission_uuid):
raise PeerAssessmentInternalError(error_message)
def get_score(submission_uuid, requirements):
def get_score(submission_uuid, peer_requirements):
"""
Retrieve a score for a submission if requirements have been satisfied.
......@@ -152,11 +152,11 @@ def get_score(submission_uuid, requirements):
contributing_assessments information, along with a None staff_id.
"""
if requirements is None:
if peer_requirements is None:
return None
# User hasn't completed their own submission yet
if not submitter_is_finished(submission_uuid, requirements):
if not submitter_is_finished(submission_uuid, peer_requirements):
return None
workflow = PeerWorkflow.get_by_submission_uuid(submission_uuid)
......@@ -171,7 +171,7 @@ def get_score(submission_uuid, requirements):
assessment__score_type=PEER_TYPE
).order_by('-assessment')
submission_finished = items.count() >= requirements["must_be_graded_by"]
submission_finished = items.count() >= peer_requirements["must_be_graded_by"]
if not submission_finished:
return None
......@@ -183,7 +183,7 @@ def get_score(submission_uuid, requirements):
# which is not supported by some versions of MySQL.
# Although this approach generates more database queries, the number is likely to
# be relatively small (at least 1 and very likely less than 5).
for scored_item in items[:requirements["must_be_graded_by"]]:
for scored_item in items[:peer_requirements["must_be_graded_by"]]:
scored_item.scored = True
scored_item.save()
assessments = [item.assessment for item in items]
......
openassessment/assessment/api/self.py
......@@ -23,13 +23,13 @@ SELF_TYPE = "SE"
logger = logging.getLogger("openassessment.assessment.api.self")
def submitter_is_finished(submission_uuid, requirements):
def submitter_is_finished(submission_uuid, self_requirements):
"""
Check whether a self-assessment has been completed for a submission.
Args:
submission_uuid (str): The unique identifier of the submission.
requirements (dict): Any attributes of the assessment module required
self_requirements (dict): Any attributes of the assessment module required
to determine if this assessment is complete. There are currently
no requirements for a self-assessment.
Returns:
......@@ -43,14 +43,14 @@ def submitter_is_finished(submission_uuid, requirements):
).exists()
def assessment_is_finished(submission_uuid, requirements):
def assessment_is_finished(submission_uuid, self_requirements):
"""
Check whether a self-assessment has been completed. For self-assessment,
this function is synonymous with submitter_is_finished.
Args:
submission_uuid (str): The unique identifier of the submission.
requirements (dict): Any attributes of the assessment module required
self_requirements (dict): Any attributes of the assessment module required
to determine if this assessment is complete. There are currently
no requirements for a self-assessment.
Returns:
......@@ -59,16 +59,16 @@ def assessment_is_finished(submission_uuid, requirements):
>>> assessment_is_finished('222bdf3d-a88e-11e3-859e-040ccee02800', {})
True
"""
return submitter_is_finished(submission_uuid, requirements)
return submitter_is_finished(submission_uuid, self_requirements)
def get_score(submission_uuid, requirements):
def get_score(submission_uuid, self_requirements):
"""
Get the score for this particular assessment.
Args:
submission_uuid (str): The unique identifier for the submission
requirements (dict): Not used.
self_requirements (dict): Not used.
Returns:
A dictionary with the points earned, points possible, and
contributing_assessments information, along with a None staff_id.
......
openassessment/assessment/api/staff.py
......@@ -26,14 +26,14 @@ logger = logging.getLogger("openassessment.assessment.api.staff")
STAFF_TYPE = "ST"
def submitter_is_finished(submission_uuid, requirements):
def submitter_is_finished(submission_uuid, staff_requirements):
"""
Determine if the submitter has finished their requirements for staff
assessment. Always returns True.
Args:
submission_uuid (str): Not used.
requirements (dict): Not used.
staff_requirements (dict): Not used.
Returns:
True
......@@ -42,30 +42,39 @@ def submitter_is_finished(submission_uuid, requirements):
return True
def assessment_is_finished(submission_uuid, requirements):
def assessment_is_finished(submission_uuid, staff_requirements):
"""
Determine if the staff assessment step of the given submission is completed.
This checks to see if staff have completed the assessment.
Args:
submission_uuid (str): The UUID of the submission being graded.
requirements (dict): Any variables that may effect this state.
staff_requirements (dict): Any variables that may effect this state.
Returns:
True if a staff assessment has been completed for this submission or if not required.
"""
if requirements and requirements.get('staff', {}).get('required', False):
# Requirements of None means we can't make any assumptions about the done-ness of this step
if staff_requirements is None:
return False
if staff_requirements.get('required', False):
return bool(get_latest_staff_assessment(submission_uuid))
return True
def on_start(submission_uuid):
def on_init(submission_uuid):
"""
Create a new staff workflow for a student item and submission.
Creates a unique staff workflow for a student item, associated with a
submission.
Note that the staff workflow begins things in on_init() instead of
on_start(), because staff shoud be able to access the submission
regardless of which state the workflow is currently in.
Args:
submission_uuid (str): The submission associated with this workflow.
......@@ -125,7 +134,7 @@ def on_cancel(submission_uuid):
raise StaffAssessmentInternalError(error_message)
def get_score(submission_uuid, requirements):
def get_score(submission_uuid, staff_requirements):
"""
Generate a score based on a completed assessment for the given submission.
If no assessment has been completed for this submission, this will return
......@@ -133,7 +142,7 @@ def get_score(submission_uuid, requirements):
Args:
submission_uuid (str): The UUID for the submission to get a score for.
requirements (dict): Not used.
staff_requirements (dict): Not used.
Returns:
A dictionary with the points earned, points possible,
......
openassessment/assessment/api/student_training.py
......@@ -24,14 +24,14 @@ from openassessment.assessment.errors import (
logger = logging.getLogger(__name__)
def submitter_is_finished(submission_uuid, requirements): # pylint:disable=W0613
def submitter_is_finished(submission_uuid, training_requirements): # pylint:disable=W0613
"""
Check whether the student has correctly assessed
all the training example responses.
Args:
submission_uuid (str): The UUID of the student's submission.
requirements (dict): Must contain "num_required" indicating
training_requirements (dict): Must contain "num_required" indicating
the number of examples the student must assess.
Returns:
......@@ -41,11 +41,11 @@ def submitter_is_finished(submission_uuid, requirements): # pylint:disable=W06
StudentTrainingRequestError
"""
if requirements is None:
if training_requirements is None:
return False
try:
num_required = int(requirements['num_required'])
num_required = int(training_requirements['num_required'])
except KeyError:
raise StudentTrainingRequestError(u'Requirements dict must contain "num_required" key')
except ValueError:
......
openassessment/assessment/test/test_staff.py
......@@ -33,7 +33,7 @@ class TestStaffAssessment(CacheResetTest):
"""
STEP_REQUIREMENTS = {}
STEP_REQUIREMENTS_WITH_STAFF = {'staff': {'required': True}}
STEP_REQUIREMENTS_WITH_STAFF = {'required': True}
# This is due to ddt not playing nicely with list comprehensions
ASSESSMENT_SCORES_DDT = [key for key in OPTIONS_SELECTED_DICT]
......
openassessment/templates/openassessmentblock/edit/oa_edit_staff_assessment.html 0 → 100644
{% load i18n %}
{% spaceless %}
<li class="openassessment_assessment_module_settings_editor" id="oa_staff_assessment_editor">
<div class="drag-handle action"></div>
<div class="openassessment_inclusion_wrapper">
<input type="checkbox" id="include_staff_assessment"
{% if assessments.staff_assessment.required %} checked="true" {% endif %}>
<label for="include_staff_assessment">{% trans "Step: Staff Assessment" %}</label>
</div>
<div class="openassessment_assessment_module_editor">
<p class="openassessment_description">
{% trans "Staff members assess learners' responses using the rubric for the assignment." %}
</p>
</div>
</li>
{% endspaceless %}
openassessment/workflow/models.py
......@@ -134,7 +134,9 @@ class AssessmentWorkflow(TimeStampedModel, StatusModel):
"""
submission_dict = sub_api.get_submission_and_student(submission_uuid)
staff_auto_added = False
if 'staff' not in step_names:
staff_auto_added = True
new_list = ['staff']
new_list.extend(step_names)
step_names = new_list
......@@ -167,6 +169,11 @@ class AssessmentWorkflow(TimeStampedModel, StatusModel):
on_init_func = getattr(api, 'on_init', lambda submission_uuid, **params: None)
on_init_func(submission_uuid, **on_init_params.get(step.name, {}))
# If we auto-added a staff step, it is optional and should be marked complete immediately
if step.name == "staff" and staff_auto_added:
step.assessment_completed_at=now()
step.save()
# For the first valid step, update the workflow status
# and notify the assessment module that it's being started
if not has_started_first_step:
......@@ -240,12 +247,12 @@ class AssessmentWorkflow(TimeStampedModel, StatusModel):
get_score_func = getattr(assessment_step.api(), 'get_score', None)
if get_score_func is not None:
if assessment_requirements is None:
requirements = None
step_requirements = None
else:
requirements = assessment_requirements.get(assessment_step_name, {})
score = get_score_func(self.submission_uuid, requirements)
step_requirements = assessment_requirements.get(assessment_step_name, {})
score = get_score_func(self.submission_uuid, step_requirements)
if assessment_step_name == self.STATUS.staff and score == None:
if requirements and requirements.get(assessment_step_name, {}).get('required', False):
if step_requirements and step_requirements.get('required', False):
break # A staff score was not found, and one is required. Return None
continue # A staff score was not found, but it is not required, so try the next type of score
break
......
openassessment/xblock/defaults.py
......@@ -156,4 +156,5 @@ DEFAULT_EDITOR_ASSESSMENTS_ORDER = [
"student-training",
"peer-assessment",
"self-assessment",
"staff-assessment",
]
openassessment/xblock/schema.py
......@@ -60,7 +60,8 @@ VALID_ASSESSMENT_TYPES = [
u'peer-assessment',
u'self-assessment',
u'example-based-assessment',
u'student-training'
u'student-training',
u'staff-assessment',
]
VALID_UPLOAD_FILE_TYPES = [
......@@ -95,6 +96,7 @@ EDITOR_UPDATE_SCHEMA = Schema({
Required('name'): All(utf8_validator, In(VALID_ASSESSMENT_TYPES)),
Required('start', default=None): Any(datetime_validator, None),
Required('due', default=None): Any(datetime_validator, None),
'required': bool,
'must_grade': All(int, Range(min=0)),
'must_be_graded_by': All(int, Range(min=0)),
'examples': [
......
openassessment/xblock/static/js/fixtures/templates.json
......@@ -489,7 +489,8 @@
"student_training",
"peer_assessment",
"self_assessment",
"example_based_assessment"
"example_based_assessment",
"staff_assessment"
]
},
"output": "oa_edit.html"
......@@ -647,7 +648,8 @@
"student_training",
"peer_assessment",
"self_assessment",
"example_based_assessment"
"example_based_assessment",
"staff_assessment"
]
},
"output": "oa_edit_student_training.html"
......
openassessment/xblock/static/js/openassessment-studio.min.js
This source diff could not be displayed because it is too large. You can view the blob instead.
openassessment/xblock/static/js/spec/studio/oa_edit.js
......@@ -117,7 +117,8 @@ describe("OpenAssessment.StudioView", function() {
"student-training",
"peer-assessment",
"self-assessment",
"example-based-assessment"
"example-based-assessment",
"staff-assessment",
]
};
......
openassessment/xblock/static/js/spec/studio/oa_edit_assessments.js
......@@ -250,4 +250,24 @@ describe("OpenAssessment edit assessment views", function() {
it("Loads a description", function() { testLoadXMLExamples(view); });
it("shows an alert when disabled", function() { testAlertOnDisable(view); });
});
describe("OpenAssessment.EditStaffAssessmentView", function() {
var view = null;
beforeEach(function() {
view = new OpenAssessment.EditStaffAssessmentView();
view.isEnabled(true);
});
it("enables and disables", function() { testEnableAndDisable(view); });
it("loads a description", function() {
expect(view.description()).toEqual({
required: true
});
});
it("shows an alert when disabled", function() { testAlertOnDisable(view); });
});
});
openassessment/xblock/static/js/spec/studio/oa_edit_settings.js
......@@ -51,6 +51,7 @@ describe("OpenAssessment.EditSettingsView", function() {
var SELF = "oa_self_assessment_editor";
var AI = "oa_ai_assessment_editor";
var TRAINING = "oa_student_training_editor";
var STAFF = "oa_staff_assessment_editor";
beforeEach(function() {
// Load the DOM fixture
......@@ -62,6 +63,7 @@ describe("OpenAssessment.EditSettingsView", function() {
assessmentViews[PEER] = new StubView("peer-assessment", "Peer assessment description");
assessmentViews[AI] = new StubView("ai-assessment", "Example Based assessment description");
assessmentViews[TRAINING] = new StubView("student-training", "Student Training description");
assessmentViews[STAFF] = new StubView("staff-assessment", "Staff assessment description");
// mock data from backend
data = {
......@@ -126,6 +128,7 @@ describe("OpenAssessment.EditSettingsView", function() {
assessmentViews[SELF].isEnabled(false);
assessmentViews[AI].isEnabled(false);
assessmentViews[TRAINING].isEnabled(false);
assessmentViews[STAFF].isEnabled(false);
expect(view.assessmentsDescription()).toEqual([]);
// Enable the first assessment only
......
openassessment/xblock/static/js/src/studio/oa_edit.js
......@@ -38,6 +38,9 @@ OpenAssessment.StudioView = function(runtime, element, server, data) {
);
// Initialize the settings tab view
var staffAssessmentView = new OpenAssessment.EditStaffAssessmentView(
$("#oa_staff_assessment_editor", this.element).get(0)
);
var studentTrainingView = new OpenAssessment.EditStudentTrainingView(
$("#oa_student_training_editor", this.element).get(0)
);
......@@ -51,6 +54,7 @@ OpenAssessment.StudioView = function(runtime, element, server, data) {
$("#oa_ai_assessment_editor", this.element).get(0)
);
var assessmentLookupDictionary = {};
assessmentLookupDictionary[staffAssessmentView.getID()] = staffAssessmentView;
assessmentLookupDictionary[studentTrainingView.getID()] = studentTrainingView;
assessmentLookupDictionary[peerAssessmentView.getID()] = peerAssessmentView;
assessmentLookupDictionary[selfAssessmentView.getID()] = selfAssessmentView;
......
openassessment/xblock/static/js/src/studio/oa_edit_assessment.js
......@@ -372,7 +372,7 @@ OpenAssessment.EditSelfAssessmentView.prototype = {
};
/**
Interface for editing self assessment settings.
Interface for editing student training assessment settings.
Args:
element (DOM element): The DOM element representing this view.
......@@ -621,3 +621,94 @@ OpenAssessment.EditExampleBasedAssessmentView.prototype = {
validationErrors: function() { return []; },
clearValidationErrors: function() {}
};
/**
* Interface for editing staff assessment settings.
*
* @param {Object} element - The DOM element representing this view.
* @constructor
*
*/
OpenAssessment.EditStaffAssessmentView = function(element) {
this.element = element;
this.name = "staff-assessment";
// Configure the toggle checkbox to enable/disable this assessment
new OpenAssessment.ToggleControl(
$("#include_staff_assessment", this.element),
$("#staff_assessment_description", this.element),
$("#staff_assessment_description", this.element), //open and closed selectors are the same!
new OpenAssessment.Notifier([
new OpenAssessment.AssessmentToggleListener()
])
).install();
};
OpenAssessment.EditStaffAssessmentView.prototype = {
/**
* Return a description of the assessment.
*
* @returns {Object} Representation of the view.
*/
description: function() {
return {
required: this.isEnabled(),
};
},
/**
* Get or set whether the assessment is enabled.
*
* @param {Boolean} isEnabled - If provided, set the enabled state of the assessment.
* @returns {Boolean}
*/
isEnabled: function(isEnabled) {
var sel = $("#include_staff_assessment", this.element);
return OpenAssessment.Fields.booleanField(sel, isEnabled);
},
/**
* Toggle whether the assessment is enabled or disabled.
* This triggers the actual click event and is mainly useful for testing.
*/
toggleEnabled: function() {
$("#include_staff_assessment", this.element).click();
},
/**
* Gets the ID of the assessment
*
* @returns {String} CSS class of the Element object
*/
getID: function() {
return $(this.element).attr('id');
},
/**
* Mark validation errors.
*
* @returns {Boolean} Whether the view is valid.
*
*/
validate: function() {
return true; //Nothing to validate, the only input is a boolean and either state is valid
},
/**
* Return a list of validation errors visible in the UI.
* Mainly useful for testing.
*
* @returns {Array} - always empty, function called but not actually used.
*/
validationErrors: function() {
return [];
},
/**
* Clear all validation errors from the UI.
*/
clearValidationErrors: function() {
//do nothing
},
};
openassessment/xblock/static/js/src/studio/oa_edit_fields.js
......@@ -152,8 +152,8 @@ OpenAssessment.ToggleControl.prototype = {
},
show: function() {
this.shownSection.removeClass('is--hidden');
this.hiddenSection.addClass('is--hidden');
this.shownSection.removeClass('is--hidden');
},
hide: function() {
......
openassessment/xblock/static/js/src/studio/oa_edit_validation_alert.js
......@@ -86,6 +86,7 @@ OpenAssessment.ValidationAlert.prototype = {
$('.oa_editor_content_wrapper', this.editorElement).each(function() {
$(this).css(styles);
$(this).scrollTop($(this).scrollTop() + alertHeight); //keep our relative scroll position the same
});
}
......
openassessment/xblock/studio_mixin.py
......@@ -170,9 +170,19 @@ class StudioMixin(object):
return {'success': False, 'msg': self._('Error updating XBlock configuration')}
# Check that the editor assessment order contains all the assessments. We are more flexible on example-based.
if set(DEFAULT_EDITOR_ASSESSMENTS_ORDER) != (set(data['editor_assessments_order']) - {'example-based-assessment'}):
logger.exception('editor_assessments_order does not contain all expected assessment types')
return {'success': False, 'msg': self._('Error updating XBlock configuration')}
given_without_example_based = set(data['editor_assessments_order']) - {'example-based-assessment'}
if set(DEFAULT_EDITOR_ASSESSMENTS_ORDER) != given_without_example_based:
# Backwards compatibility: "staff-assessment" may not be present.
# If that is the only problem with this data, just add it manually and continue.
if set(DEFAULT_EDITOR_ASSESSMENTS_ORDER) == (
# Check the given set, minus example-based, plus staff
given_without_example_based | {'staff-assessment'}
):
data['editor_assessments_order'].append('staff-assessment')
logger.info('Backwards compatibility: editor_assessments_order now contains staff-assessment')
else:
logger.exception('editor_assessments_order does not contain all expected assessment types')
return {'success': False, 'msg': self._('Error updating XBlock configuration')}
# Backwards compatibility: We used to treat "name" as both a user-facing label
# and a unique identifier for criteria and options.
......
openassessment/xblock/test/data/assessment_combo.json
......@@ -36,6 +36,33 @@
"current_assessments": null,
"is_released": false
},
"staff_only_required": {
"valid": true,
"assessments": [
{
"name": "staff-assessment",
"required": true
}
],
"current_assessments": null,
"is_released": false
},
"staff_optional_after_peer": {
"valid": true,
"assessments": [
{
"name": "peer-assessment",
"must_grade": 5,
"must_be_graded_by": 3
},
{
"name": "staff-assessment",
"required": false
}
],
"current_assessments": null,
"is_released": false
},
"self_before_peer": {
"valid": true,
"assessments": [
......@@ -51,7 +78,7 @@
"current_assessments": null,
"is_released": false
},
"student_training_self_then_peer": {
"student_training_self_peer_staff": {
"valid": true,
"assessments": [
{
......@@ -72,6 +99,10 @@
"name": "peer-assessment",
"must_grade": 5,
"must_be_graded_by": 3
},
{
"name": "staff-assessment",
"required": true
}
],
"current_assessments": null,
......
openassessment/xblock/test/data/invalid_assessments.json
......@@ -118,6 +118,23 @@
"current_assessments": null,
"is_released": false
},
"remove_staff_mid_flight": {
"assessments": [
{
"name": "self-assessment"
},
{
"name": "staff-assessment",
"required": true
}
],
"current_assessments": [
{
"name": "self-assessment"
}
],
"is_released": true
},
"remove_peer_mid_flight": {
"assessments": [
{
......@@ -160,6 +177,42 @@
"is_released": true
},
"only_optional_staff": {
"assessments": [
{
"name": "staff-assessment",
"required": false
}
],
"current_assessments": null,
"is_released": false
},
"staff_must_grade": {
"assessments": [
{
"name": "staff-assessment",
"must_grade": 2
}
],
"current_assessments": null,
"is_released": false
},
"staff_not_last": {
"assessments": [
{
"name": "staff-assessment",
"required": true
},
{
"name": "self-assessment"
}
],
"current_assessments": null,
"is_released": false
},
"example_based_algorithm_id_is_not_ease": {
"assessments": [
{
......
openassessment/xblock/test/data/invalid_rubric.json
......@@ -88,6 +88,6 @@
}
],
"editor_assessments_order": ["student-training", "peer-assessment", "self-assessment"]
"editor_assessments_order": ["student-training", "peer-assessment", "self-assessment", "staff-assessment"]
}
}
openassessment/xblock/test/data/parse_assessments_xml.json
......@@ -4,6 +4,7 @@
"<assessments>",
"<assessment name=\"peer-assessment\" start=\"2014-02-27T09:46:28\" due=\"2014-03-01T00:00:00\" must_grade=\"5\" must_be_graded_by=\"3\" />",
"<assessment name=\"self-assessment\" start=\"2014-04-01T00:00:00\" due=\"2014-06-01T00:00:00\" />",
"<assessment name=\"staff-assessment\" required=\"False\" />",
"</assessments>"
],
"assessments": [
......@@ -18,6 +19,12 @@
"name": "self-assessment",
"start": "2014-04-01T00:00:00",
"due": "2014-06-01T00:00:00"
},
{
"name": "staff-assessment",
"start": null,
"due": null,
"required": false
}
]
},
......
openassessment/xblock/test/data/serialize.json
......@@ -45,6 +45,10 @@
"name": "self-assessment",
"start": "2014-04-01T00:00:00",
"due": "2014-06-01T00:00:00"
},
{
"name": "staff-assessment",
"required": false
}
],
"expected_xml": [
......@@ -53,6 +57,7 @@
"<assessments>",
"<assessment name=\"peer-assessment\" start=\"2014-02-27T09:46:28\" due=\"2014-03-01T00:00:00\" must_grade=\"5\" must_be_graded_by=\"3\" />",
"<assessment name=\"self-assessment\" start=\"2014-04-01T00:00:00\" due=\"2014-06-01T00:00:00\" />",
"<assessment name=\"staff-assessment\" required=\"False\" />",
"</assessments>",
"<prompts>",
"<prompt><description>Test prompt</description></prompt>",
......@@ -323,6 +328,10 @@
"due": "2014-03-01T00:00:00",
"must_grade": 5,
"must_be_graded_by": 3
},
{
"name": "staff-assessment",
"required": false
}
],
"expected_xml": [
......@@ -330,6 +339,7 @@
"<title>ƒσσ</title>",
"<assessments>",
"<assessment name=\"peer-assessment\" start=\"2014-02-27T09:46:28\" due=\"2014-03-01T00:00:00\" must_grade=\"5\" must_be_graded_by=\"3\" />",
"<assessment name=\"staff-assessment\" required=\"False\" />",
"</assessments>",
"<prompts>",
"<prompt><description>Ṫëṡẗ ṗṛöṁṗẗ</description></prompt>",
......@@ -1235,7 +1245,7 @@
]
},
"ai_peer_self": {
"ai_peer_self_staff": {
"title": "Foo",
"prompt": "Test prompt",
"rubric_feedback_prompt": "Test Feedback Prompt",
......@@ -1302,6 +1312,10 @@
"name": "self-assessment",
"start": "2014-04-01T00:00:00",
"due": "2014-06-01T00:00:00"
},
{
"name": "staff-assessment",
"required": true
}
],
"expected_xml": [
......@@ -1324,6 +1338,7 @@
"</assessment>",
"<assessment name=\"peer-assessment\" start=\"2014-02-27T09:46:28\" due=\"2014-03-01T00:00:00\" must_grade=\"5\" must_be_graded_by=\"3\" />",
"<assessment name=\"self-assessment\" start=\"2014-04-01T00:00:00\" due=\"2014-06-01T00:00:00\" />",
"<assessment name=\"staff-assessment\" required=\"True\" />",
"</assessments>",
"<prompts>",
"<prompt><description>Test prompt</description></prompt>",
......
openassessment/xblock/test/data/update_from_xml.json
......@@ -6,6 +6,7 @@
"<assessments>",
"<assessment name=\"peer-assessment\" start=\"2014-02-27T09:46:28\" due=\"2014-03-01T00:00:00\" must_grade=\"5\" must_be_graded_by=\"3\" />",
"<assessment name=\"self-assessment\" start=\"2014-04-01T00:00:00\" due=\"2014-06-01T00:00:00\" />",
"<assessment name=\"staff-assessment\" required=\"False\" />",
"</assessments>",
"<rubric>",
"<prompt>Test prompt</prompt>",
......@@ -61,6 +62,10 @@
"name": "self-assessment",
"start": "2014-04-01T00:00:00",
"due": "2014-06-01T00:00:00"
},
{
"name": "staff-assessment",
"required": false
}
]
},
......
openassessment/xblock/test/data/update_from_xml_error.json
......@@ -321,6 +321,26 @@
]
},
"optional_self_assessment": {
"xml": [
"<openassessment>",
"<title>Foo</title>",
"<assessments>",
"<assessment name=\"self-assessment\" start=\"2014-04-01T00:00:00\" due=\"2014-06-01T00:00:00\" \"required\": \"False\" />",
"</assessments>",
"<rubric>",
"<prompt>Test prompt</prompt>",
"<criterion>",
"<name>Test criterion</name>",
"<prompt>Test criterion prompt</prompt>",
"<option points=\"0\"><name>No</name><explanation>No explanation</explanation></option>",
"<option points=\"2\"><name>Yes</name><explanation>Yes explanation</explanation></option>",
"</criterion>",
"</rubric>",
"</openassessment>"
]
},
"invalid_criterion_feedback_value": {
"xml": [
"<openassessment>",
......
openassessment/xblock/test/data/update_xblock.json
......@@ -47,6 +47,63 @@
"name": "self-assessment",
"start": null,
"due": null
},
{
"name": "staff-assessment",
"required": false
}
],
"editor_assessments_order": ["student-training", "peer-assessment", "self-assessment", "staff-assessment"]
},
"no_staff_backwards_compatible": {
"criteria": [
{
"order_num": 0,
"name": "0",
"label": "Test criterion",
"prompt": "Test criterion prompt",
"options": [
{
"order_num": 0,
"points": 0,
"name": "0",
"label": "No",
"explanation": "No explanation"
},
{
"order_num": 1,
"points": 2,
"name": "1",
"label": "Yes",
"explanation": "Yes explanation"
}
],
"feedback": "required"
}
],
"prompts": [{"description": "My new prompt 1."}, {"description": "My new prompt 2."}],
"feedback_prompt": "Feedback prompt",
"feedback_default_text": "Feedback default text",
"submission_due": "4014-02-27T09:46",
"submission_start": "4014-02-10T09:46",
"title": "My new title.",
"file_upload_type": null,
"white_listed_file_types": null,
"allow_latex": false,
"leaderboard_show": 0,
"assessments": [
{
"name": "peer-assessment",
"must_grade": 5,
"must_be_graded_by": 3,
"start": null,
"due": "4014-03-10T00:00"
},
{
"name": "self-assessment",
"start": null,
"due": null
}
],
"editor_assessments_order": ["student-training", "peer-assessment", "self-assessment"]
......@@ -102,7 +159,7 @@
"due": null
}
],
"editor_assessments_order": ["student-training", "peer-assessment", "self-assessment"]
"editor_assessments_order": ["student-training", "peer-assessment", "self-assessment", "staff-assessment"]
},
"student_training": {
......@@ -167,7 +224,7 @@
"due": "4014-03-10T00:00"
}
],
"editor_assessments_order": ["student-training", "peer-assessment", "self-assessment"]
"editor_assessments_order": ["student-training", "peer-assessment", "self-assessment", "staff-assessment"]
},
"already_has_criteria_and_options_names_assigned": {
......@@ -220,7 +277,7 @@
"due": null
}
],
"editor_assessments_order": ["student-training", "peer-assessment", "self-assessment"]
"editor_assessments_order": ["student-training", "peer-assessment", "self-assessment", "staff-assessment"]
},
"file_upload_type_image_treated_as_image_file_upload_type": {
......@@ -273,7 +330,7 @@
"due": null
}
],
"editor_assessments_order": ["student-training", "peer-assessment", "self-assessment"]
"editor_assessments_order": ["student-training", "peer-assessment", "self-assessment", "staff-assessment"]
},
"file_upload_type_file_treated_as_restricted_file_upload": {
......@@ -326,7 +383,7 @@
"due": null
}
],
"editor_assessments_order": ["student-training", "peer-assessment", "self-assessment"]
"editor_assessments_order": ["student-training", "peer-assessment", "self-assessment", "staff-assessment"]
},
"file_upload_type_custom_treated_as_restrictive_file_upload": {
......@@ -379,6 +436,6 @@
"due": null
}
],
"editor_assessments_order": ["student-training", "peer-assessment", "self-assessment"]
"editor_assessments_order": ["student-training", "peer-assessment", "self-assessment", "staff-assessment"]
}
}
openassessment/xblock/test/data/valid_assessments.json
......@@ -14,6 +14,25 @@
"is_released": false
},
"self_then_peer_then_staff": {
"assessments": [
{
"name": "self-assessment"
},
{
"name": "peer-assessment",
"must_grade": 5,
"must_be_graded_by": 3
},
{
"name": "staff-assessment",
"required": true
}
],
"current_assessments": null,
"is_released": false
},
"self_only": {
"assessments": [
{
......
openassessment/xblock/test/test_studio.py
......@@ -145,6 +145,7 @@ class StudioViewTest(XBlockHandlerTestCase):
"student-training",
"peer-assessment",
"self-assessment",
"staff-assessment",
]
xblock.runtime.modulestore = MagicMock()
xblock.runtime.modulestore.has_published_version.return_value = False
......@@ -165,6 +166,7 @@ class StudioViewTest(XBlockHandlerTestCase):
"student-training",
"peer-assessment",
"self-assessment",
"staff-assessment",
]
xblock.runtime.modulestore = MagicMock()
xblock.runtime.modulestore.has_published_version.return_value = False
......
openassessment/xblock/test/test_xml.py
......@@ -100,7 +100,11 @@ class TestSerializeContent(TestCase):
"name": "self-assessment",
"start": '2014-04-01T00:00:00.000000',
"due": "2014-06-01T00:00:00.92926",
}
},
{
"name": "staff-assessment",
"required": False,
},
]
def setUp(self):
......
openassessment/xblock/validation.py
......@@ -47,9 +47,13 @@ def _duplicates(items):
def _is_valid_assessment_sequence(assessments):
"""
Check whether the sequence of assessments is valid.
For example, we currently allow self-assessment after peer-assessment,
but do not allow peer-assessment before self-assessment.
Check whether the sequence of assessments is valid. The rules enforced are:
-must have one of staff-, peer-, self-, or example-based-assessment listed
-in addition to those, only student-training is a valid entry
-no duplicate entries
-if staff-assessment is present, it must come last
-if example-based-assessment is present, it must come first
-if student-training is present, it must be followed at some point by peer-assessment
Args:
assessments (list of dict): List of assessment dictionaries.
......@@ -58,27 +62,37 @@ def _is_valid_assessment_sequence(assessments):
bool
"""
valid_sequences = [
['self-assessment'],
['peer-assessment'],
['peer-assessment', 'self-assessment'],
['self-assessment', 'peer-assessment'],
['student-training', 'peer-assessment'],
['student-training', 'peer-assessment', 'self-assessment'],
['student-training', 'self-assessment', 'peer-assessment'],
['example-based-assessment'],
['example-based-assessment', 'self-assessment'],
['example-based-assessment', 'peer-assessment'],
['example-based-assessment', 'peer-assessment', 'self-assessment'],
['example-based-assessment', 'self-assessment', 'peer-assessment'],
['example-based-assessment', 'student-training', 'peer-assessment'],
['example-based-assessment', 'student-training', 'peer-assessment', 'self-assessment'],
['example-based-assessment', 'student-training', 'self-assessment', 'peer-assessment'],
]
sequence = [asmnt.get('name') for asmnt in assessments]
return sequence in valid_sequences
required = ['example-based-assessment', 'staff-assessment', 'peer-assessment', 'self-assessment']
optional = ['student-training']
# at least one of required?
if not any(name in required for name in sequence):
return False
# nothing except what appears in required or optional
if any(name not in required + optional for name in sequence):
return False
# no duplicates
if any(sequence.count(name) > 1 for name in sequence):
return False
# if using staff-assessment, it must come last
if 'staff-assessment' in sequence and 'staff-assessment' != sequence[-1]:
return False
# if using example-based, it must be first
if 'example-based-assessment' in sequence and 'example-based-assessment' != sequence[0]:
return False
# if using training, must be followed by peer at some point
if 'student-training' in sequence:
train_index = sequence.index('student-training')
if 'peer-assessment' not in sequence[train_index:]:
return False
return True
def validate_assessments(assessments, current_assessments, is_released, _):
"""
......@@ -145,6 +159,12 @@ def validate_assessments(assessments, current_assessments, is_released, _):
if assessment_dict.get('algorithm_id') not in ['ease', 'fake']:
return (False, _('The "algorithm_id" value must be set to "ease" or "fake"'))
# Staff grading must be required if it is the only step
if assessment_dict.get('name') == 'staff-assessment' and len(assessments) == 1:
required = assessment_dict.get('required')
if not required: # Captures both None and explicit False cases, both are invalid
return (False, _('The "required" value must be true if staff assessment is the only step.'))
if is_released:
if len(assessments) != len(current_assessments):
return (False, _("The number of assessments cannot be changed after the problem has been released."))
......
openassessment/xblock/workflow_mixin.py
......@@ -87,6 +87,12 @@ class WorkflowMixin(object):
"num_required": len(training_module["examples"])
}
staff_assessment_module = self.get_assessment_module('staff-assessment')
if staff_assessment_module:
requirements["staff"] = {
"required": staff_assessment_module["required"]
}
return requirements
def update_workflow_status(self, submission_uuid=None):
......
openassessment/xblock/xml.py
......@@ -569,6 +569,14 @@ def parse_assessments_xml(assessments_root):
except ValueError:
raise UpdateFromXmlError('The "must_be_graded_by" value must be a positive integer.')
# Assessment required
if 'required' in assessment.attrib:
# Staff assessment is the only type to use an explicit required marker
if assessment_dict['name'] != 'staff-assessment':
raise UpdateFromXmlError('The "required" field is only allowed for staff assessment.')
assessment_dict['required'] = _parse_boolean(unicode(assessment.get('required')))
# Training examples
examples = assessment.findall('example')
......@@ -653,6 +661,9 @@ def serialize_assessments(assessments_root, oa_block):
if assessment_dict.get('algorithm_id') is not None:
assessment.set('algorithm_id', unicode(assessment_dict['algorithm_id']))
if assessment_dict.get('required') is not None:
assessment.set('required', unicode(assessment_dict['required']))
# Training examples
examples = assessment_dict.get('examples', [])
if not isinstance(examples, list):
......
test/acceptance/tests.py
......@@ -51,6 +51,9 @@ class OpenAssessmentTest(WebAppTest):
TEST_COURSE_ID = "course-v1:edx+ORA203+course"
PROBLEM_LOCATIONS = {
'staff_only':
u'courses/{test_course_id}/courseware/'
u'61944efb38a349edb140c762c7419b50/415c3ee1b7d04b58a1887a6fe82b31d6/'.format(test_course_id=TEST_COURSE_ID),
'self_only':
u'courses/{test_course_id}/courseware/'
u'a4dfec19cf9b4a6fb5b18be6ccd9cecc/338a4affb58a45459629e0566291381e/'.format(test_course_id=TEST_COURSE_ID),
......@@ -246,6 +249,42 @@ class SelfAssessmentTest(OpenAssessmentTest):
self.submission_page.preview_latex()
class StaffAssessmentTest(OpenAssessmentTest):
"""
Test the staff-assessment flow.
"""
def setUp(self):
super(StaffAssessmentTest, self).setUp('staff_only', staff=True)
@retry()
@attr('acceptance')
def test_staff_assessment(self):
# Set up user and navigate to submission page
self.auto_auth_page.visit()
username = self.auto_auth_page.get_username()
self.submission_page.visit()
# Verify that staff grade step is shown initially
self._verify_staff_grade_section("NOT AVAILABLE", None)
# User submits a response
self.submission_page.submit_response(self.SUBMISSION)
self.assertTrue(self.submission_page.has_submitted)
# Verify staff grade section appears as expected
self._verify_staff_grade_section("NOT AVAILABLE", "WAITING FOR A STAFF GRADE")
# TODO: as part of @cahren's work on TNL-3493, change this section to do a proper full staff grade
# The override is a temporary hack for acceptance test advancement.
self.staff_area_page = StaffAreaPage(self.browser, self.problem_loc)
self.do_staff_override(username)
# Switch back to original user, verify staff grade section appears as expected
self.staff_asmnt_page.visit()
self.staff_asmnt_page.verify_status_value("COMPLETE")
class PeerAssessmentTest(OpenAssessmentTest):
"""
Test the peer-assessment flow.
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment