Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
E
edx-platform
Overview
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
edx
edx-platform
Commits
b290b305
Commit
b290b305
authored
Apr 29, 2015
by
John Eskew
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Reorganize code, change method name, & fix comments.
Change xml file reading code when importing a course.
parent
1c6ab4cc
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
92 additions
and
109 deletions
+92
-109
common/lib/xmodule/xmodule/modulestore/split_mongo/split_draft.py
+5
-2
common/lib/xmodule/xmodule/modulestore/tests/test_xml_importer.py
+4
-4
common/lib/xmodule/xmodule/modulestore/xml_importer.py
+83
-103
No files found.
common/lib/xmodule/xmodule/modulestore/split_mongo/split_draft.py
View file @
b290b305
...
@@ -494,11 +494,14 @@ class DraftVersioningModuleStore(SplitMongoModuleStore, ModuleStoreDraftAndPubli
...
@@ -494,11 +494,14 @@ class DraftVersioningModuleStore(SplitMongoModuleStore, ModuleStoreDraftAndPubli
block_id
=
self
.
DEFAULT_ROOT_LIBRARY_BLOCK_ID
block_id
=
self
.
DEFAULT_ROOT_LIBRARY_BLOCK_ID
new_usage_key
=
course_key
.
make_usage_key
(
block_type
,
block_id
)
new_usage_key
=
course_key
.
make_usage_key
(
block_type
,
block_id
)
# Only the course import process calls import_xblock(). If the branch setting is published_only,
# then the non-draft blocks are being imported.
if
self
.
get_branch_setting
()
==
ModuleStoreEnum
.
Branch
.
published_only
:
if
self
.
get_branch_setting
()
==
ModuleStoreEnum
.
Branch
.
published_only
:
#
override existing draft (PLAT-297, PLAT-299). NOTE: this has the effect of removing
#
Override any existing drafts (PLAT-297, PLAT-299). This import/publish step removes
# any local changes
w/ th
e import.
# any local changes
during the cours
e import.
draft_course
=
course_key
.
for_branch
(
ModuleStoreEnum
.
BranchName
.
draft
)
draft_course
=
course_key
.
for_branch
(
ModuleStoreEnum
.
BranchName
.
draft
)
with
self
.
branch_setting
(
ModuleStoreEnum
.
Branch
.
draft_preferred
,
draft_course
):
with
self
.
branch_setting
(
ModuleStoreEnum
.
Branch
.
draft_preferred
,
draft_course
):
# Importing the block and publishing the block links the draft & published blocks' version history.
draft_block
=
self
.
import_xblock
(
user_id
,
draft_course
,
block_type
,
block_id
,
fields
,
runtime
)
draft_block
=
self
.
import_xblock
(
user_id
,
draft_course
,
block_type
,
block_id
,
fields
,
runtime
)
return
self
.
publish
(
draft_block
.
location
.
version_agnostic
(),
user_id
,
blacklist
=
EXCLUDE_ALL
,
**
kwargs
)
return
self
.
publish
(
draft_block
.
location
.
version_agnostic
(),
user_id
,
blacklist
=
EXCLUDE_ALL
,
**
kwargs
)
...
...
common/lib/xmodule/xmodule/modulestore/tests/test_xml_importer.py
View file @
b290b305
...
@@ -9,7 +9,7 @@ from xmodule.x_module import XModuleMixin
...
@@ -9,7 +9,7 @@ from xmodule.x_module import XModuleMixin
from
opaque_keys.edx.locations
import
Location
from
opaque_keys.edx.locations
import
Location
from
xmodule.modulestore
import
ModuleStoreEnum
from
xmodule.modulestore
import
ModuleStoreEnum
from
xmodule.modulestore.inheritance
import
InheritanceMixin
from
xmodule.modulestore.inheritance
import
InheritanceMixin
from
xmodule.modulestore.xml_importer
import
_
import_module_and_update_references
from
xmodule.modulestore.xml_importer
import
_
update_and_import_module
from
xmodule.modulestore.tests.mongo_connection
import
MONGO_PORT_NUM
,
MONGO_HOST
from
xmodule.modulestore.tests.mongo_connection
import
MONGO_PORT_NUM
,
MONGO_HOST
from
opaque_keys.edx.locations
import
SlashSeparatedCourseKey
from
opaque_keys.edx.locations
import
SlashSeparatedCourseKey
from
xmodule.tests
import
DATA_DIR
from
xmodule.tests
import
DATA_DIR
...
@@ -144,7 +144,7 @@ class RemapNamespaceTest(ModuleStoreNoSettings):
...
@@ -144,7 +144,7 @@ class RemapNamespaceTest(ModuleStoreNoSettings):
# Move to different runtime w/ different course id
# Move to different runtime w/ different course id
target_location_namespace
=
SlashSeparatedCourseKey
(
"org"
,
"course"
,
"run"
)
target_location_namespace
=
SlashSeparatedCourseKey
(
"org"
,
"course"
,
"run"
)
new_version
=
_
import_module_and_update_references
(
new_version
=
_
update_and_import_module
(
self
.
xblock
,
self
.
xblock
,
modulestore
(),
modulestore
(),
999
,
999
,
...
@@ -181,7 +181,7 @@ class RemapNamespaceTest(ModuleStoreNoSettings):
...
@@ -181,7 +181,7 @@ class RemapNamespaceTest(ModuleStoreNoSettings):
# Remap the namespace
# Remap the namespace
target_location_namespace
=
Location
(
"org"
,
"course"
,
"run"
,
"category"
,
"stubxblock"
)
target_location_namespace
=
Location
(
"org"
,
"course"
,
"run"
,
"category"
,
"stubxblock"
)
new_version
=
_
import_module_and_update_references
(
new_version
=
_
update_and_import_module
(
self
.
xblock
,
self
.
xblock
,
modulestore
(),
modulestore
(),
999
,
999
,
...
@@ -213,7 +213,7 @@ class RemapNamespaceTest(ModuleStoreNoSettings):
...
@@ -213,7 +213,7 @@ class RemapNamespaceTest(ModuleStoreNoSettings):
# Remap the namespace
# Remap the namespace
target_location_namespace
=
Location
(
"org"
,
"course"
,
"run"
,
"category"
,
"stubxblock"
)
target_location_namespace
=
Location
(
"org"
,
"course"
,
"run"
,
"category"
,
"stubxblock"
)
new_version
=
_
import_module_and_update_references
(
new_version
=
_
update_and_import_module
(
self
.
xblock
,
self
.
xblock
,
modulestore
(),
modulestore
(),
999
,
999
,
...
...
common/lib/xmodule/xmodule/modulestore/xml_importer.py
View file @
b290b305
...
@@ -316,7 +316,7 @@ class ImportManager(object):
...
@@ -316,7 +316,7 @@ class ImportManager(object):
log
.
debug
(
'course data_dir=
%
s'
,
source_courselike
.
data_dir
)
log
.
debug
(
'course data_dir=
%
s'
,
source_courselike
.
data_dir
)
with
self
.
store
.
branch_setting
(
ModuleStoreEnum
.
Branch
.
draft_preferred
,
dest_id
):
with
self
.
store
.
branch_setting
(
ModuleStoreEnum
.
Branch
.
draft_preferred
,
dest_id
):
course
=
_
import_module_and_update_references
(
course
=
_
update_and_import_module
(
source_courselike
,
self
.
store
,
self
.
user_id
,
source_courselike
,
self
.
store
,
self
.
user_id
,
courselike_key
,
courselike_key
,
dest_id
,
dest_id
,
...
@@ -381,7 +381,7 @@ class ImportManager(object):
...
@@ -381,7 +381,7 @@ class ImportManager(object):
if
self
.
verbose
:
if
self
.
verbose
:
log
.
debug
(
'importing module location
%
s'
,
child
.
location
)
log
.
debug
(
'importing module location
%
s'
,
child
.
location
)
_
import_module_and_update_references
(
_
update_and_import_module
(
child
,
child
,
self
.
store
,
self
.
store
,
self
.
user_id
,
self
.
user_id
,
...
@@ -399,7 +399,7 @@ class ImportManager(object):
...
@@ -399,7 +399,7 @@ class ImportManager(object):
if
self
.
verbose
:
if
self
.
verbose
:
log
.
debug
(
'importing module location
%
s'
,
leftover
)
log
.
debug
(
'importing module location
%
s'
,
leftover
)
_
import_module_and_update_references
(
_
update_and_import_module
(
self
.
xml_module_store
.
get_item
(
leftover
),
self
.
xml_module_store
.
get_item
(
leftover
),
self
.
store
,
self
.
store
,
self
.
user_id
,
self
.
user_id
,
...
@@ -620,13 +620,68 @@ def import_library_from_xml(*args, **kwargs):
...
@@ -620,13 +620,68 @@ def import_library_from_xml(*args, **kwargs):
return
list
(
manager
.
run_imports
())
return
list
(
manager
.
run_imports
())
def
_
import_module_and_update_references
(
def
_
update_and_import_module
(
module
,
store
,
user_id
,
module
,
store
,
user_id
,
source_course_id
,
dest_course_id
,
source_course_id
,
dest_course_id
,
do_import_static
=
True
,
runtime
=
None
):
do_import_static
=
True
,
runtime
=
None
):
"""
Update all the module reference fields to the destination course id,
then import the module into the destination course.
"""
logging
.
debug
(
u'processing import of module
%
s...'
,
unicode
(
module
.
location
))
logging
.
debug
(
u'processing import of module
%
s...'
,
unicode
(
module
.
location
))
def
_update_module_references
(
module
,
source_course_id
,
dest_course_id
):
"""
Move the module to a new course.
"""
def
_convert_ref_fields_to_new_namespace
(
reference
):
# pylint: disable=invalid-name
"""
Convert a reference to the new namespace, but only
if the original namespace matched the original course.
Otherwise, returns the input value.
"""
assert
isinstance
(
reference
,
UsageKey
)
if
source_course_id
==
reference
.
course_key
:
return
reference
.
map_into_course
(
dest_course_id
)
else
:
return
reference
fields
=
{}
for
field_name
,
field
in
module
.
fields
.
iteritems
():
if
field
.
scope
!=
Scope
.
parent
and
field
.
is_set_on
(
module
):
if
isinstance
(
field
,
Reference
):
value
=
field
.
read_from
(
module
)
if
value
is
None
:
fields
[
field_name
]
=
None
else
:
fields
[
field_name
]
=
_convert_ref_fields_to_new_namespace
(
field
.
read_from
(
module
))
elif
isinstance
(
field
,
ReferenceList
):
references
=
field
.
read_from
(
module
)
fields
[
field_name
]
=
[
_convert_ref_fields_to_new_namespace
(
reference
)
for
reference
in
references
]
elif
isinstance
(
field
,
ReferenceValueDict
):
reference_dict
=
field
.
read_from
(
module
)
fields
[
field_name
]
=
{
key
:
_convert_ref_fields_to_new_namespace
(
reference
)
for
key
,
reference
in
reference_dict
.
iteritems
()
}
elif
field_name
==
'xml_attributes'
:
value
=
field
.
read_from
(
module
)
# remove any export/import only xml_attributes
# which are used to wire together draft imports
if
'parent_url'
in
value
:
del
value
[
'parent_url'
]
if
'parent_sequential_url'
in
value
:
del
value
[
'parent_sequential_url'
]
if
'index_in_children_list'
in
value
:
del
value
[
'index_in_children_list'
]
fields
[
field_name
]
=
value
else
:
fields
[
field_name
]
=
field
.
read_from
(
module
)
return
fields
if
do_import_static
and
'data'
in
module
.
fields
and
isinstance
(
module
.
fields
[
'data'
],
xblock
.
fields
.
String
):
if
do_import_static
and
'data'
in
module
.
fields
and
isinstance
(
module
.
fields
[
'data'
],
xblock
.
fields
.
String
):
# we want to convert all 'non-portable' links in the module_data
# we want to convert all 'non-portable' links in the module_data
# (if it is a string) to portable strings (e.g. /static/)
# (if it is a string) to portable strings (e.g. /static/)
...
@@ -636,53 +691,7 @@ def _import_module_and_update_references(
...
@@ -636,53 +691,7 @@ def _import_module_and_update_references(
module
.
data
module
.
data
)
)
# Move the module to a new course
fields
=
_update_module_references
(
module
,
source_course_id
,
dest_course_id
)
def
_convert_reference_fields_to_new_namespace
(
reference
):
"""
Convert a reference to the new namespace, but only
if the original namespace matched the original course.
Otherwise, returns the input value.
"""
assert
isinstance
(
reference
,
UsageKey
)
if
source_course_id
==
reference
.
course_key
:
return
reference
.
map_into_course
(
dest_course_id
)
else
:
return
reference
fields
=
{}
for
field_name
,
field
in
module
.
fields
.
iteritems
():
if
field
.
scope
!=
Scope
.
parent
and
field
.
is_set_on
(
module
):
if
isinstance
(
field
,
Reference
):
value
=
field
.
read_from
(
module
)
if
value
is
None
:
fields
[
field_name
]
=
None
else
:
fields
[
field_name
]
=
_convert_reference_fields_to_new_namespace
(
field
.
read_from
(
module
))
elif
isinstance
(
field
,
ReferenceList
):
references
=
field
.
read_from
(
module
)
fields
[
field_name
]
=
[
_convert_reference_fields_to_new_namespace
(
reference
)
for
reference
in
references
]
elif
isinstance
(
field
,
ReferenceValueDict
):
reference_dict
=
field
.
read_from
(
module
)
fields
[
field_name
]
=
{
key
:
_convert_reference_fields_to_new_namespace
(
reference
)
for
key
,
reference
in
reference_dict
.
iteritems
()
}
elif
field_name
==
'xml_attributes'
:
value
=
field
.
read_from
(
module
)
# remove any export/import only xml_attributes
# which are used to wire together draft imports
if
'parent_url'
in
value
:
del
value
[
'parent_url'
]
if
'parent_sequential_url'
in
value
:
del
value
[
'parent_sequential_url'
]
if
'index_in_children_list'
in
value
:
del
value
[
'index_in_children_list'
]
fields
[
field_name
]
=
value
else
:
fields
[
field_name
]
=
field
.
read_from
(
module
)
return
store
.
import_xblock
(
return
store
.
import_xblock
(
user_id
,
dest_course_id
,
module
.
location
.
category
,
user_id
,
dest_course_id
,
module
.
location
.
category
,
...
@@ -699,14 +708,13 @@ def _import_course_draft(
...
@@ -699,14 +708,13 @@ def _import_course_draft(
target_id
,
target_id
,
mongo_runtime
mongo_runtime
):
):
'''
"""
This will import all the content inside of the 'drafts' folder, if it exists
This method will import all the content inside of the 'drafts' folder, if content exists.
NOTE: This is not a full course import, basically in our current
NOTE: This is not a full course import! In our current application, only verticals
application only verticals (and downwards) can be in draft.
(and blocks beneath) can be in draft. Therefore, different call points into the import
Therefore, we need to use slightly different call points into
process_xml are used as the XMLModuleStore() constructor cannot simply be called
the import process_xml as we can't simply call XMLModuleStore() constructor
(as is done for importing public content).
(like we do for importing public content)
"""
'''
draft_dir
=
course_data_path
+
"/drafts"
draft_dir
=
course_data_path
+
"/drafts"
if
not
os
.
path
.
exists
(
draft_dir
):
if
not
os
.
path
.
exists
(
draft_dir
):
return
return
...
@@ -720,7 +728,9 @@ def _import_course_draft(
...
@@ -720,7 +728,9 @@ def _import_course_draft(
# Whether or not data_dir ends with a "/" differs in production vs. test.
# Whether or not data_dir ends with a "/" differs in production vs. test.
if
not
data_dir
.
endswith
(
"/"
):
if
not
data_dir
.
endswith
(
"/"
):
data_dir
+=
"/"
data_dir
+=
"/"
# Remove absolute path, leaving relative <course_name>/drafts.
draft_course_dir
=
draft_dir
.
replace
(
data_dir
,
''
,
1
)
draft_course_dir
=
draft_dir
.
replace
(
data_dir
,
''
,
1
)
system
=
ImportSystem
(
system
=
ImportSystem
(
xmlstore
=
xml_module_store
,
xmlstore
=
xml_module_store
,
course_id
=
source_course_id
,
course_id
=
source_course_id
,
...
@@ -761,7 +771,7 @@ def _import_course_draft(
...
@@ -761,7 +771,7 @@ def _import_course_draft(
parent
.
children
.
insert
(
index
,
non_draft_location
)
parent
.
children
.
insert
(
index
,
non_draft_location
)
store
.
update_item
(
parent
,
user_id
)
store
.
update_item
(
parent
,
user_id
)
_
import_module_and_update_references
(
_
update_and_import_module
(
module
,
store
,
user_id
,
module
,
store
,
user_id
,
source_course_id
,
source_course_id
,
target_id
,
target_id
,
...
@@ -770,52 +780,23 @@ def _import_course_draft(
...
@@ -770,52 +780,23 @@ def _import_course_draft(
for
child
in
module
.
get_children
():
for
child
in
module
.
get_children
():
_import_module
(
child
)
_import_module
(
child
)
# Now walk the /
vertical
directory.
# Now walk the /
drafts
directory.
# Each file in the directory will be a draft copy of the vertical.
# Each file in the directory will be a draft copy of the vertical.
# First it is necessary to order the draft items by their desired index in the child list,
# First it is necessary to order the draft items by their desired index in the child list,
# since the order in which os.walk() returns the files is not guaranteed.
# since the order in which os.walk() returns the files is not guaranteed.
drafts
=
[]
drafts
=
[]
for
dirname
,
_dirnames
,
filenames
in
os
.
walk
(
draft_dir
):
for
rootdir
,
__
,
filenames
in
os
.
walk
(
draft_dir
):
for
filename
in
filenames
:
for
filename
in
filenames
:
module_path
=
os
.
path
.
join
(
dirname
,
filename
)
if
filename
.
startswith
(
'._'
):
# Skip any OSX quarantine files, prefixed with a '._'.
continue
module_path
=
os
.
path
.
join
(
rootdir
,
filename
)
with
open
(
module_path
,
'r'
)
as
f
:
with
open
(
module_path
,
'r'
)
as
f
:
try
:
try
:
# note, on local dev it seems like OSX will put
xml
=
f
.
read
()
.
decode
(
'utf-8'
)
# some extra files in the directory with "quarantine"
# information. These files are binary files and will
# throw exceptions when we try to parse the file
# as an XML string. Let's make sure we're
# dealing with a string before ingesting
data
=
f
.
read
()
try
:
# The process_xml() call below recursively processes all descendants. If
xml
=
data
.
decode
(
'utf-8'
)
except
UnicodeDecodeError
,
err
:
# seems like on OSX localdev, the OS is making
# quarantine files in the unzip directory
# when importing courses so if we blindly try to
# enumerate through the directory, we'll try
# to process a bunch of binary quarantine files
# (which are prefixed with a '._' character which
# will dump a bunch of exceptions to the output,
# although they are harmless.
#
# Reading online docs there doesn't seem to be
# a good means to detect a 'hidden' file that works
# well across all OS environments. So for now, I'm using
# OSX's utilization of a leading '.' in the filename
# to indicate a system hidden file.
#
# Better yet would be a way to figure out if this is
# a binary file, but I haven't found a good way
# to do this yet.
if
filename
.
startswith
(
'._'
):
continue
# Not a 'hidden file', then re-raise exception
raise
err
# process_xml call below recursively processes all descendants. If
# we call this on all verticals in a course with verticals nested below
# we call this on all verticals in a course with verticals nested below
# the unit level, we try to import the same content twice, causing naming conflicts.
# the unit level, we try to import the same content twice, causing naming conflicts.
# Therefore only process verticals at the unit level, assuming that any other
# Therefore only process verticals at the unit level, assuming that any other
...
@@ -838,13 +819,12 @@ def _import_course_draft(
...
@@ -838,13 +819,12 @@ def _import_course_draft(
draft
=
draft_node_constructor
(
draft
=
draft_node_constructor
(
module
=
descriptor
,
url
=
draft_url
,
parent_url
=
parent_url
,
index
=
index
module
=
descriptor
,
url
=
draft_url
,
parent_url
=
parent_url
,
index
=
index
)
)
drafts
.
append
(
draft
)
drafts
.
append
(
draft
)
except
Exception
:
# pylint: disable=broad-except
except
Exception
:
# pylint: disable=broad-except
logging
.
exception
(
'Error while parsing course xml.'
)
logging
.
exception
(
'Error while parsing course
drafts
xml.'
)
#
sort drafts by `index_in_children_list` attribute
#
Sort drafts by `index_in_children_list` attribute.
drafts
.
sort
(
key
=
lambda
x
:
x
.
index
)
drafts
.
sort
(
key
=
lambda
x
:
x
.
index
)
for
draft
in
get_draft_subtree_roots
(
drafts
):
for
draft
in
get_draft_subtree_roots
(
drafts
):
...
@@ -864,11 +844,11 @@ def allowed_metadata_by_category(category):
...
@@ -864,11 +844,11 @@ def allowed_metadata_by_category(category):
def
check_module_metadata_editability
(
module
):
def
check_module_metadata_editability
(
module
):
'''
"""
Assert that there is no metadata within a particular module that
Assert that there is no metadata within a particular module that
we can't support editing. However we always allow 'display_name'
we can't support editing. However we always allow 'display_name'
and 'xml_attributes'
and 'xml_attributes'
'''
"""
allowed
=
allowed_metadata_by_category
(
module
.
location
.
category
)
allowed
=
allowed_metadata_by_category
(
module
.
location
.
category
)
if
'*'
in
allowed
:
if
'*'
in
allowed
:
# everything is allowed
# everything is allowed
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment