Skip to content

E
edx-platform
Switch branch/tag
Find file
Normal viewHistoryPermalink
migrate.py 8.72 KB
Newer Older
migration views - see modulestore contents, and force reload of course
10f53d62
 
ichuang committed
1 2 3 4 
#
# migration tools for content team to go from stable-edx4edx to LMS+CMS
#
add gitreload to lms migration, so it doesn't have to run separately
7dd7de95
 
ichuang committed
5 
import json
migration views - see modulestore contents, and force reload of course
10f53d62
 
ichuang committed
6 
import logging
add gitreload to lms migration, so it doesn't have to run separately
7dd7de95
 
ichuang committed
7 
import os
migration views - see modulestore contents, and force reload of course
10f53d62
 
ichuang committed
8 9 10 11 12 
import xmodule.modulestore.django as xmodule_django
from xmodule.modulestore.django import modulestore

from django.http import HttpResponse
from django.conf import settings
add gitreload to lms migration, so it doesn't have to run separately
7dd7de95
 
ichuang committed
13 14 15 16 17 18 
import track.views

try:
    from django.views.decorators.csrf import csrf_exempt
except ImportError:
    from django.contrib.csrf.middleware import csrf_exempt
migration views - see modulestore contents, and force reload of course
10f53d62
 
ichuang committed
19
Update logging.getLogger() calls to use edx instead of mitx
e26fc08d
 
David Baumgold committed
20 
log = logging.getLogger("edx.lms_migrate")
migration views - see modulestore contents, and force reload of course
10f53d62
 
ichuang committed
21 22 23 
LOCAL_DEBUG = True
ALLOWED_IPS = settings.LMS_MIGRATION_ALLOWED_IPS
Pep8 autofixes
cfae1cdf
 
Calen Pennington committed
24
migration views - see modulestore contents, and force reload of course
10f53d62
 
ichuang committed
25 26 
def escape(s):
    """escape HTML special characters in string"""
Pep8 autofixes
cfae1cdf
 
Calen Pennington committed
27 28 
    return str(s).replace('<', '&lt;').replace('>', '&gt;')
migration views - see modulestore contents, and force reload of course
10f53d62
 
ichuang committed
29
add gitreload to lms migration, so it doesn't have to run separately
7dd7de95
 
ichuang committed
30 31 32 33 
def getip(request):
    '''
    Extract IP address of requester from header, even if behind proxy
    '''
Pep8 autofixes
cfae1cdf
 
Calen Pennington committed
34 
    ip = request.META.get('HTTP_X_REAL_IP', '')  	# nginx reverse proxy
add gitreload to lms migration, so it doesn't have to run separately
7dd7de95
 
ichuang committed
35 
    if not ip:
Pep8 autofixes
cfae1cdf
 
Calen Pennington committed
36 
        ip = request.META.get('REMOTE_ADDR', 'None')
add gitreload to lms migration, so it doesn't have to run separately
7dd7de95
 
ichuang committed
37 38 
    return ip
commit_id may be specified for modulestore reload - for multi-worker configs
ccbbb5a9
 
ichuang committed
39 40 

def get_commit_id(course):
fix lms migration to be compatible with xblock
14899b90
 
ichuang committed
41 42 
    #return course.metadata.get('GIT_COMMIT_ID', 'No commit id')
    return getattr(course, 'GIT_COMMIT_ID', 'No commit id')
commit_id may be specified for modulestore reload - for multi-worker configs
ccbbb5a9
 
ichuang committed
43 44 45 
    # getattr(def_ms.courses[reload_dir], 'GIT_COMMIT_ID','No commit id')
Pep8 autofixes
cfae1cdf
 
Calen Pennington committed
46 
def set_commit_id(course, commit_id):
fix lms migration to be compatible with xblock
14899b90
 
ichuang committed
47 
    #course.metadata['GIT_COMMIT_ID'] = commit_id
fix-setattr-and-getattr
bbf508ca
 
Ned Batchelder committed
48 49 
    course.GIT_COMMIT_ID = commit_id
    # def_ms.courses[reload_dir].GIT_COMMIT_ID = new_commit_id
commit_id may be specified for modulestore reload - for multi-worker configs
ccbbb5a9
 
ichuang committed
50
Pep8 autofixes
cfae1cdf
 
Calen Pennington committed
51
commit_id may be specified for modulestore reload - for multi-worker configs
ccbbb5a9
 
ichuang committed
52 
def manage_modulestores(request, reload_dir=None, commit_id=None):
migration views - see modulestore contents, and force reload of course
10f53d62
 
ichuang committed
53 54 55 56 57 58 59 60 61 62 63 64 65 
    '''
    Manage the static in-memory modulestores.

    If reload_dir is not None, then instruct the xml loader to reload that course directory.
    '''
    html = "<html><body>"

    def_ms = modulestore()
    courses = def_ms.get_courses()

    #----------------------------------------
    # check on IP address of requester
add gitreload to lms migration, so it doesn't have to run separately
7dd7de95
 
ichuang committed
66 
    ip = getip(request)
Use .display_name instead of metadata['display_name']
691744e3
 
Victor Shnayder committed
67
migration views - see modulestore contents, and force reload of course
10f53d62
 
ichuang committed
68 
    if LOCAL_DEBUG:
commit_id may be specified for modulestore reload - for multi-worker configs
ccbbb5a9
 
ichuang committed
69 70 71 
        html += '<h3>IP address: %s <h3>' % ip
        html += '<h3>User: %s </h3>' % request.user
        html += '<h3>My pid: %s</h3>' % os.getpid()
Unicode fixes for log messages and other things
4e733c6e
 
Omar Al-Ithawi committed
72 
        log.debug(u'request from ip=%s, user=%s', ip, request.user)
migration views - see modulestore contents, and force reload of course
10f53d62
 
ichuang committed
73 74 

    if not (ip in ALLOWED_IPS or 'any' in ALLOWED_IPS):
limit course reload to localhost or user.is_staff
9de6e281
 
ichuang committed
75 
        if request.user and request.user.is_staff:
Unicode fixes for log messages and other things
4e733c6e
 
Omar Al-Ithawi committed
76 
            log.debug(u'request allowed because user=%s is staff', request.user)
limit course reload to localhost or user.is_staff
9de6e281
 
ichuang committed
77 78 79 80 
        else:
            html += 'Permission denied'
            html += "</body></html>"
            log.debug('request denied, ALLOWED_IPS=%s' % ALLOWED_IPS)
github IPs in lms.envs.common; made create_user, create_groups,
e3ecd22f
 
ichuang committed
81 
            return HttpResponse(html, status=403)
migration views - see modulestore contents, and force reload of course
10f53d62
 
ichuang committed
82 83 

    #----------------------------------------
commit_id may be specified for modulestore reload - for multi-worker configs
ccbbb5a9
 
ichuang committed
84 
    # reload course if specified; handle optional commit_id
migration views - see modulestore contents, and force reload of course
10f53d62
 
ichuang committed
85 86 87 

    if reload_dir is not None:
        if reload_dir not in def_ms.courses:
Removed a bunch of font tags
8a5e872c
 
Kyle Fiedler committed
88 
            html += '<h2 class="inline-error">Error: "%s" is not a valid course directory</h2>' % reload_dir
migration views - see modulestore contents, and force reload of course
10f53d62
 
ichuang committed
89 
        else:
commit_id may be specified for modulestore reload - for multi-worker configs
ccbbb5a9
 
ichuang committed
90 91 92 93 94 95 
            # reloading based on commit_id is needed when running mutiple worker threads,
            # so that a given thread doesn't reload the same commit multiple times
            current_commit_id = get_commit_id(def_ms.courses[reload_dir])
            log.debug('commit_id="%s"' % commit_id)
            log.debug('current_commit_id="%s"' % current_commit_id)
Pep8 autofixes
cfae1cdf
 
Calen Pennington committed
96 
            if (commit_id is not None) and (commit_id == current_commit_id):
commit_id may be specified for modulestore reload - for multi-worker configs
ccbbb5a9
 
ichuang committed
97 
                html += "<h2>Already at commit id %s for %s</h2>" % (commit_id, reload_dir)
add tracking log entries for modulestore reload
2f2e71e0
 
ichuang committed
98 99 100 101 102 103 
                track.views.server_track(request,
                                         'reload %s skipped already at %s (pid=%s)' % (reload_dir,
                                                                                       commit_id,
                                                                                       os.getpid(),
                                                                                       ),
                                         {}, page='migrate')
commit_id may be specified for modulestore reload - for multi-worker configs
ccbbb5a9
 
ichuang committed
104 105 106 
            else:
                html += '<h2>Reloaded course directory "%s"</h2>' % reload_dir
                def_ms.try_load_course(reload_dir)
Pep8 autofixes
cfae1cdf
 
Calen Pennington committed
107 
                gdir = settings.DATA_DIR / reload_dir
commit_id may be specified for modulestore reload - for multi-worker configs
ccbbb5a9
 
ichuang committed
108 109 110 
                new_commit_id = os.popen('cd %s; git log -n 1 | head -1' % gdir).read().strip().split(' ')[1]
                set_commit_id(def_ms.courses[reload_dir], new_commit_id)
                html += '<p>commit_id=%s</p>' % new_commit_id
add tracking log entries for modulestore reload
2f2e71e0
 
ichuang committed
111 112 113 
                track.views.server_track(request, 'reloaded %s now at %s (pid=%s)' % (reload_dir,
                                                                                      new_commit_id,
                                                                                      os.getpid()), {}, page='migrate')
migration views - see modulestore contents, and force reload of course
10f53d62
 
ichuang committed
114 115 116 117 118 119 

    #----------------------------------------

    html += '<h2>Courses loaded in the modulestore</h2>'
    html += '<ol>'
    for cdir, course in def_ms.courses.items():
Clean up pep8 E128 issues
308fe26b
 
Calen Pennington committed
120 
        html += '<li><a href="%s/migrate/reload/%s">%s</a> (%s)</li>' % (
MITX_ROOT_URL => EDX_ROOT_URL
b32f2bc1
 
David Baumgold committed
121 
            settings.EDX_ROOT_URL,
Clean up pep8 E128 issues
308fe26b
 
Calen Pennington committed
122 123 
            escape(cdir),
            escape(cdir),
Make course ids and usage ids opaque to LMS and Studio [partial commit]
cd746bf8
 
Calen Pennington committed
124 
            course.location.to_deprecated_string()
Clean up pep8 E128 issues
308fe26b
 
Calen Pennington committed
125 
        )
migration views - see modulestore contents, and force reload of course
10f53d62
 
ichuang committed
126 127 128 129 
    html += '</ol>'

    #----------------------------------------
fix lms migration to be compatible with xblock
14899b90
 
ichuang committed
130 131 
    #dumpfields = ['definition', 'location', 'metadata']
    dumpfields = ['location', 'metadata']
Use .display_name instead of metadata['display_name']
691744e3
 
Victor Shnayder committed
132
migration views - see modulestore contents, and force reload of course
10f53d62
 
ichuang committed
133 134 
    for cdir, course in def_ms.courses.items():
        html += '<hr width="100%"/>'
Make display_name defaulting explicit
93db5acc
 
Calen Pennington committed
135 
        html += '<h2>Course: %s (%s)</h2>' % (course.display_name_with_default, cdir)
migration views - see modulestore contents, and force reload of course
10f53d62
 
ichuang committed
136
commit_id may be specified for modulestore reload - for multi-worker configs
ccbbb5a9
 
ichuang committed
137 138 
        html += '<p>commit_id=%s</p>' % get_commit_id(course)
migration views - see modulestore contents, and force reload of course
10f53d62
 
ichuang committed
139 
        for field in dumpfields:
fix lms migration to be compatible with xblock
14899b90
 
ichuang committed
140 
            data = getattr(course, field, None)
migration views - see modulestore contents, and force reload of course
10f53d62
 
ichuang committed
141 
            html += '<h3>%s</h3>' % field
Pep8 autofixes
cfae1cdf
 
Calen Pennington committed
142 
            if type(data) == dict:
migration views - see modulestore contents, and force reload of course
10f53d62
 
ichuang committed
143 
                html += '<ul>'
Pep8 autofixes
cfae1cdf
 
Calen Pennington committed
144 145 
                for k, v in data.items():
                    html += '<li>%s:%s</li>' % (escape(k), escape(v))
migration views - see modulestore contents, and force reload of course
10f53d62
 
ichuang committed
146 147 148 
                html += '</ul>'
            else:
                html += '<ul><li>%s</li></ul>' % escape(data)
Use .display_name instead of metadata['display_name']
691744e3
 
Victor Shnayder committed
149
migration views - see modulestore contents, and force reload of course
10f53d62
 
ichuang committed
150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 
    #----------------------------------------

    html += '<hr width="100%"/>'
    html += "courses: <pre>%s</pre>" % escape(courses)

    ms = xmodule_django._MODULESTORES
    html += "modules: <pre>%s</pre>" % escape(ms)
    html += "default modulestore: <pre>%s</pre>" % escape(unicode(def_ms))

    #----------------------------------------

    log.debug('_MODULESTORES=%s' % ms)
    log.debug('courses=%s' % courses)
    log.debug('def_ms=%s' % unicode(def_ms))

    html += "</body></html>"
Use .display_name instead of metadata['display_name']
691744e3
 
Victor Shnayder committed
166 
    return HttpResponse(html)
add gitreload to lms migration, so it doesn't have to run separately
7dd7de95
 
ichuang committed
167
Pep8 autofixes
cfae1cdf
 
Calen Pennington committed
168
add gitreload to lms migration, so it doesn't have to run separately
7dd7de95
 
ichuang committed
169 170 171 172 173 174 175 176 177 178 179 180 181 
@csrf_exempt
def gitreload(request, reload_dir=None):
    '''
    This can be used as a github WebHook Service Hook, for reloading of the content repo used by the LMS.

    If reload_dir is not None, then instruct the xml loader to reload that course directory.
    '''
    html = "<html><body>"
    ip = getip(request)

    html += '<h3>IP address: %s ' % ip
    html += '<h3>User: %s ' % request.user
Pep8 autofixes
cfae1cdf
 
Calen Pennington committed
182 183 
    ALLOWED_IPS = []  	# allow none by default
    if hasattr(settings, 'ALLOWED_GITRELOAD_IPS'):  	# allow override in settings
missing "settings." in line in lms_migration
cb07b060
 
ichuang committed
184 
        ALLOWED_IPS = settings.ALLOWED_GITRELOAD_IPS
add gitreload to lms migration, so it doesn't have to run separately
7dd7de95
 
ichuang committed
185 186 187 

    if not (ip in ALLOWED_IPS or 'any' in ALLOWED_IPS):
        if request.user and request.user.is_staff:
Unicode fixes for log messages and other things
4e733c6e
 
Omar Al-Ithawi committed
188 
            log.debug(u'request allowed because user=%s is staff', request.user)
add gitreload to lms migration, so it doesn't have to run separately
7dd7de95
 
ichuang committed
189 190 191 
        else:
            html += 'Permission denied'
            html += "</body></html>"
Pep8 autofixes
cfae1cdf
 
Calen Pennington committed
192 193 194 
            log.debug('request denied from %s, ALLOWED_IPS=%s' % (ip, ALLOWED_IPS))
            return HttpResponse(html)
add gitreload to lms migration, so it doesn't have to run separately
7dd7de95
 
ichuang committed
195 196 197 198 199 200 201 202 203 204 
    #----------------------------------------
    # see if request is from github (POST with JSON)

    if reload_dir is None and 'payload' in request.POST:
        payload = request.POST['payload']
        log.debug("payload=%s" % payload)
        gitargs = json.loads(payload)
        log.debug("gitargs=%s" % gitargs)
        reload_dir = gitargs['repository']['name']
        log.debug("github reload_dir=%s" % reload_dir)
Pep8 autofixes
cfae1cdf
 
Calen Pennington committed
205 
        gdir = settings.DATA_DIR / reload_dir
add gitreload to lms migration, so it doesn't have to run separately
7dd7de95
 
ichuang committed
206 207 208 209 210 
        if not os.path.exists(gdir):
            log.debug("====> ERROR in gitreload - no such directory %s" % reload_dir)
            return HttpResponse('Error')
        cmd = "cd %s; git reset --hard HEAD; git clean -f -d; git pull origin; chmod g+w course.xml" % gdir
        log.debug(os.popen(cmd).read())
Pep8 autofixes
cfae1cdf
 
Calen Pennington committed
211 
        if hasattr(settings, 'GITRELOAD_HOOK'):  	# hit this hook after reload, if set
add gitreload to lms migration, so it doesn't have to run separately
7dd7de95
 
ichuang committed
212 213 
            gh = settings.GITRELOAD_HOOK
            if gh:
Pep8 autofixes
cfae1cdf
 
Calen Pennington committed
214 
                ghurl = '%s/%s' % (gh, reload_dir)
add gitreload to lms migration, so it doesn't have to run separately
7dd7de95
 
ichuang committed
215 216 
                r = requests.get(ghurl)
                log.debug("GITRELOAD_HOOK to %s: %s" % (ghurl, r.text))
Pep8 autofixes
cfae1cdf
 
Calen Pennington committed
217
add gitreload to lms migration, so it doesn't have to run separately
7dd7de95
 
ichuang committed
218 219 220 221 222 223 
    #----------------------------------------
    # reload course if specified

    if reload_dir is not None:
        def_ms = modulestore()
        if reload_dir not in def_ms.courses:
Removed a bunch of font tags
8a5e872c
 
Kyle Fiedler committed
224 
            html += '<h2 class="inline-error">Error: "%s" is not a valid course directory</font></h2>' % reload_dir
add gitreload to lms migration, so it doesn't have to run separately
7dd7de95
 
ichuang committed
225 
        else:
Removed a bunch of font tags
8a5e872c
 
Kyle Fiedler committed
226 
            html += "<h2>Reloaded course directory '%s'</h2>" % reload_dir
add gitreload to lms migration, so it doesn't have to run separately
7dd7de95
 
ichuang committed
227 228 229 
            def_ms.try_load_course(reload_dir)
            track.views.server_track(request, 'reloaded %s' % reload_dir, {}, page='migrate')
Pep8 autofixes
cfae1cdf
 
Calen Pennington committed
230 
    return HttpResponse(html)