Skip to content

E
edx-platform
Switch branch/tag
Find file
Normal viewHistoryPermalink
migrate.py 6.63 KB
Newer Older
migration views - see modulestore contents, and force reload of course
10f53d62
 
ichuang committed
1 2 3 4 
#
# migration tools for content team to go from stable-edx4edx to LMS+CMS
#
add gitreload to lms migration, so it doesn't have to run separately
7dd7de95
 
ichuang committed
5 
import json
migration views - see modulestore contents, and force reload of course
10f53d62
 
ichuang committed
6 
import logging
add gitreload to lms migration, so it doesn't have to run separately
7dd7de95
 
ichuang committed
7 
import os
migration views - see modulestore contents, and force reload of course
10f53d62
 
ichuang committed
8 9 10 11 12 13 
from pprint import pprint
import xmodule.modulestore.django as xmodule_django
from xmodule.modulestore.django import modulestore

from django.http import HttpResponse
from django.conf import settings
add gitreload to lms migration, so it doesn't have to run separately
7dd7de95
 
ichuang committed
14 15 16 17 18 19 
import track.views

try:
    from django.views.decorators.csrf import csrf_exempt
except ImportError:
    from django.contrib.csrf.middleware import csrf_exempt
migration views - see modulestore contents, and force reload of course
10f53d62
 
ichuang committed
20 21 22 23 24 25 26 27 28 

log = logging.getLogger("mitx.lms_migrate")
LOCAL_DEBUG = True
ALLOWED_IPS = settings.LMS_MIGRATION_ALLOWED_IPS

def escape(s):
    """escape HTML special characters in string"""
    return str(s).replace('<','&lt;').replace('>','&gt;')
add gitreload to lms migration, so it doesn't have to run separately
7dd7de95
 
ichuang committed
29 30 31 32 33 34 35 36 37 
def getip(request):
    '''
    Extract IP address of requester from header, even if behind proxy
    '''
    ip = request.META.get('HTTP_X_REAL_IP','')	# nginx reverse proxy
    if not ip:
        ip = request.META.get('REMOTE_ADDR','None')
    return ip
migration views - see modulestore contents, and force reload of course
10f53d62
 
ichuang committed
38 39 40 41 42 43 44 45 46 47 48 49 50 51 
def manage_modulestores(request,reload_dir=None):
    '''
    Manage the static in-memory modulestores.

    If reload_dir is not None, then instruct the xml loader to reload that course directory.
    '''
    html = "<html><body>"

    def_ms = modulestore()
    courses = def_ms.get_courses()

    #----------------------------------------
    # check on IP address of requester
add gitreload to lms migration, so it doesn't have to run separately
7dd7de95
 
ichuang committed
52 
    ip = getip(request)
Use .display_name instead of metadata['display_name']
691744e3
 
Victor Shnayder committed
53
migration views - see modulestore contents, and force reload of course
10f53d62
 
ichuang committed
54 55 
    if LOCAL_DEBUG:
        html += '<h3>IP address: %s ' % ip
minor - have migrate also show user when debugging
fb7b48e1
 
ichuang committed
56 57 
        html += '<h3>User: %s ' % request.user
        log.debug('request from ip=%s, user=%s' % (ip,request.user))
migration views - see modulestore contents, and force reload of course
10f53d62
 
ichuang committed
58 59 

    if not (ip in ALLOWED_IPS or 'any' in ALLOWED_IPS):
limit course reload to localhost or user.is_staff
9de6e281
 
ichuang committed
60 61 62 63 64 65 
        if request.user and request.user.is_staff:
            log.debug('request allowed because user=%s is staff' % request.user)
        else:
            html += 'Permission denied'
            html += "</body></html>"
            log.debug('request denied, ALLOWED_IPS=%s' % ALLOWED_IPS)
github IPs in lms.envs.common; made create_user, create_groups,
e3ecd22f
 
ichuang committed
66 
            return HttpResponse(html, status=403)
migration views - see modulestore contents, and force reload of course
10f53d62
 
ichuang committed
67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 

    #----------------------------------------
    # reload course if specified

    if reload_dir is not None:
        if reload_dir not in def_ms.courses:
            html += "<h2><font color='red'>Error: '%s' is not a valid course directory</font></h2>" % reload_dir
        else:
            html += "<h2><font color='blue'>Reloaded course directory '%s'</font></h2>" % reload_dir
            def_ms.try_load_course(reload_dir)

    #----------------------------------------

    html += '<h2>Courses loaded in the modulestore</h2>'
    html += '<ol>'
    for cdir, course in def_ms.courses.items():
        html += '<li><a href="%s/migrate/reload/%s">%s</a> (%s)</li>' % (settings.MITX_ROOT_URL,
                                                            escape(cdir),
                                                            escape(cdir),
                                                            course.location.url())
    html += '</ol>'

    #----------------------------------------

    dumpfields = ['definition','location','metadata']
Use .display_name instead of metadata['display_name']
691744e3
 
Victor Shnayder committed
92
migration views - see modulestore contents, and force reload of course
10f53d62
 
ichuang committed
93 94 
    for cdir, course in def_ms.courses.items():
        html += '<hr width="100%"/>'
Use .display_name instead of metadata['display_name']
691744e3
 
Victor Shnayder committed
95 
        html += '<h2>Course: %s (%s)</h2>' % (course.display_name,cdir)
migration views - see modulestore contents, and force reload of course
10f53d62
 
ichuang committed
96 97 98 99 100 101 102 103 104 105 106 

        for field in dumpfields:
            data = getattr(course,field)
            html += '<h3>%s</h3>' % field
            if type(data)==dict:
                html += '<ul>'
                for k,v in data.items():
                    html += '<li>%s:%s</li>' % (escape(k),escape(v))
                html += '</ul>'
            else:
                html += '<ul><li>%s</li></ul>' % escape(data)
Use .display_name instead of metadata['display_name']
691744e3
 
Victor Shnayder committed
107
migration views - see modulestore contents, and force reload of course
10f53d62
 
ichuang committed
108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 

    #----------------------------------------

    html += '<hr width="100%"/>'
    html += "courses: <pre>%s</pre>" % escape(courses)

    ms = xmodule_django._MODULESTORES
    html += "modules: <pre>%s</pre>" % escape(ms)
    html += "default modulestore: <pre>%s</pre>" % escape(unicode(def_ms))

    #----------------------------------------

    log.debug('_MODULESTORES=%s' % ms)
    log.debug('courses=%s' % courses)
    log.debug('def_ms=%s' % unicode(def_ms))

    html += "</body></html>"
Use .display_name instead of metadata['display_name']
691744e3
 
Victor Shnayder committed
125 
    return HttpResponse(html)
add gitreload to lms migration, so it doesn't have to run separately
7dd7de95
 
ichuang committed
126 127 128 129 130 131 132 133 134 135 136 137 138 139 

@csrf_exempt
def gitreload(request, reload_dir=None):
    '''
    This can be used as a github WebHook Service Hook, for reloading of the content repo used by the LMS.

    If reload_dir is not None, then instruct the xml loader to reload that course directory.
    '''
    html = "<html><body>"
    ip = getip(request)

    html += '<h3>IP address: %s ' % ip
    html += '<h3>User: %s ' % request.user
github IPs in lms.envs.common; made create_user, create_groups,
e3ecd22f
 
ichuang committed
140 
    ALLOWED_IPS = []	# allow none by default
add gitreload to lms migration, so it doesn't have to run separately
7dd7de95
 
ichuang committed
141 
    if hasattr(settings,'ALLOWED_GITRELOAD_IPS'):	# allow override in settings
missing "settings." in line in lms_migration
cb07b060
 
ichuang committed
142 
        ALLOWED_IPS = settings.ALLOWED_GITRELOAD_IPS
add gitreload to lms migration, so it doesn't have to run separately
7dd7de95
 
ichuang committed
143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 

    if not (ip in ALLOWED_IPS or 'any' in ALLOWED_IPS):
        if request.user and request.user.is_staff:
            log.debug('request allowed because user=%s is staff' % request.user)
        else:
            html += 'Permission denied'
            html += "</body></html>"
            log.debug('request denied from %s, ALLOWED_IPS=%s' % (ip,ALLOWED_IPS))
            return HttpResponse(html)    
        
    #----------------------------------------
    # see if request is from github (POST with JSON)

    if reload_dir is None and 'payload' in request.POST:
        payload = request.POST['payload']
        log.debug("payload=%s" % payload)
        gitargs = json.loads(payload)
        log.debug("gitargs=%s" % gitargs)
        reload_dir = gitargs['repository']['name']
        log.debug("github reload_dir=%s" % reload_dir)
        gdir = settings.DATA_DIR / reload_dir 
        if not os.path.exists(gdir):
            log.debug("====> ERROR in gitreload - no such directory %s" % reload_dir)
            return HttpResponse('Error')
        cmd = "cd %s; git reset --hard HEAD; git clean -f -d; git pull origin; chmod g+w course.xml" % gdir
        log.debug(os.popen(cmd).read())
        if hasattr(settings,'GITRELOAD_HOOK'):	# hit this hook after reload, if set
            gh = settings.GITRELOAD_HOOK
            if gh:
                ghurl = '%s/%s' % (gh,reload_dir)
                r = requests.get(ghurl)
                log.debug("GITRELOAD_HOOK to %s: %s" % (ghurl, r.text))
        
    #----------------------------------------
    # reload course if specified

    if reload_dir is not None:
        def_ms = modulestore()
        if reload_dir not in def_ms.courses:
            html += "<h2><font color='red'>Error: '%s' is not a valid course directory</font></h2>" % reload_dir
        else:
            html += "<h2><font color='blue'>Reloaded course directory '%s'</font></h2>" % reload_dir
            def_ms.try_load_course(reload_dir)
            track.views.server_track(request, 'reloaded %s' % reload_dir, {}, page='migrate')

    return HttpResponse(html)