Convert import script to ansible tasks.

Require versions for the course repo checkouts so we can match the versions in
production.

Update import script to spit out yaml configuration and XML mappings for mixed
module store.

playbooks/roles/edxapp/defaults/main.yml

@@ -207,6 +207,11 @@ EDXAPP_MICROSITE_CONFIGRATION: {}
 EDXAPP_COURSES_WITH_UNSAFE_CODE: []
 EDXAPP_SESSION_COOKIE_DOMAIN: ""
 
+# XML Course related flags
+EDXAPP_XML_FROM_GIT: false
+EDXAPP_XML_S3_BUCKET: !!null
+EDXAPP_XML_S3_KEY: !!null
+EDXAPP_XML_URL: !!null
 #-------- Everything below this line is internal to the role ------------
 
 #Use YAML references (& and *) and hash merge <<: to factor out shared settings
@@ -227,6 +232,7 @@ edxapp_gem_bin: "{{ edxapp_gem_root }}/bin"
 edxapp_user: edxapp
 edxapp_deploy_path: "{{ edxapp_venv_bin }}:{{ edxapp_code_dir }}/bin:{{ edxapp_rbenv_bin }}:{{ edxapp_rbenv_shims }}:{{ edxapp_gem_bin }}:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
 edxapp_staticfile_dir: "{{ edxapp_data_dir }}/staticfiles"
+edxapp_course_static_dir: "{{ edxapp_data_dir }}/course_static"
 edxapp_course_data_dir: "{{ edxapp_data_dir }}/data"
 edxapp_upload_dir: "{{ edxapp_data_dir }}/uploads"
 edxapp_theme_dir: "{{ edxapp_data_dir }}/themes"

playbooks/roles/edxapp/tasks/deploy.yml

@@ -232,13 +232,13 @@
     object="{{ EDXAPP_XML_S3_KEY }}"
     mode="get"
     dest="/var/tmp/xml_coursedata.tar.gz"
-  when: EDXAPP_XML_S3_BUCKET is defined and EDXAPP_XML_S3_BUCKET != "" and EDXAPP_XML_S3_KEY is defined and EDXAPP_XML_S3_KEY != ""
+  when: not EDXAPP_XML_FROM_GIT and EDXAPP_XML_S3_BUCKET and EDXAPP_XML_S3_KEY
   register: download_xml_s3
 
 # Install the xml courses from a url.
 - name: download xml coursedata
   get_url: url="{{ EDXAPP_XML_URL }}" dest="/var/tmp/xml_coursedata.tar.gz"
-  when: EDXAPP_XML_URL is defined and EDXAPP_XML_URL != ""
+  when: not EDXAPP_XML_FROM_GIT and EDXAPP_XML_URL
   register: download_xml_url
 
 - name: unzip the data to the data dir
@@ -247,6 +247,9 @@
     chdir="{{ edxapp_course_data_dir }}"
   when: download_xml_s3.changed or download_xml_url.changed
 
+- include: xml.yml
+  when: EDXAPP_XML_FROM_GIT
+
 # If using CAS and you have a function for mapping attributes, install
 # the module here.  The next few tasks set up the python code sandbox
 - name: install CAS attribute module

playbooks/roles/edxapp/tasks/main.yml

@@ -25,6 +25,7 @@
     - "{{ edxapp_venvs_dir }}"
     - "{{ edxapp_theme_dir }}"
     - "{{ edxapp_staticfile_dir }}"
+    - "{{ edxapp_course_static_dir }}"
 
 - name: create edxapp log dir
   file: >

playbooks/roles/edxapp/tasks/xml.yml

+- name: clone the xml course repo
+  git: >
+    repo="{{ item.repo_url }}"
+    dest="{{ edxapp_course_data_dir }}/{{ item.repo_name }}"
+    version="{{ item.version }}"
+    accept_hostkey=True
+    depth=1
+  sudo_user: "{{ edxapp_user }}"
+  environment:
+    GIT_SSH: "{{ edxapp_git_ssh }}"
+  with_items: EDXAPP_XML_COURSES
+
+- name: update course.xml
+  template: >
+    src=course.xml.j2
+    dest="{{ edxapp_course_data_dir }}/{{ item.repo_name }}/course.xml"
+  sudo_user: "{{ edxapp_user }}"
+  with_items: EDXAPP_XML_COURSES
+
+- name: make symlinks for the static data
+  shell: >
+    executable=/bin/bash
+    if [[ -d {{ edxapp_course_data_dir }}/{{ item.repo_name }}/static ]]; then
+      ln -sf {{ edxapp_course_data_dir }}/{{ item.repo_name }}/static {{ edxapp_course_static_dir }}/{{ item.repo_name }}
+    else
+      ln -sf {{ edxapp_course_data_dir }}/{{ item.repo_name }} {{ edxapp_course_static_dir }}/{{ item.repo_name }}
+    fi
+  with_items: EDXAPP_XML_COURSES
+  when: item.disposition == "on disk" or item.disposition == "no static import"
+
+- name: import courses with nostatic flag
+  shell: >
+    {{ edxapp_venv_bin }}/python manage.py cms --settings=aws import --nostatic {{ edxapp_course_data_dir }} {{ item.repo_name }}
+    chdir="{{ edxapp_code_dir }}"
+  sudo_user: "{{ edxapp_user }}"
+  with_items: EDXAPP_XML_COURSES
+  when: item.disposition == "no static import"
+  
+- name: import courses including static data
+  shell: >
+    {{ edxapp_venv_bin }}/python manage.py cms --settings=aws import {{ edxapp_course_data_dir }} {{ item.repo_name }}
+    chdir="{{ edxapp_code_dir }}"
+  sudo_user: "{{ edxapp_user }}"
+  with_items: EDXAPP_XML_COURSES
+  when: item.disposition == "import"
+
+- name: delete courses that were fully imported
+  file: path="{{ edxapp_course_data_dir }}/{{ item.repo_name }}" state=absent
+  with_items: EDXAPP_XML_COURSES
+  when: item.disposition == "import"
+
+- name: create an archive of course data and course static dirs
+  shell: tar czf /tmp/static_course_content.tar.gz -C {{ edxapp_data_dir }} {{ edxapp_course_data_dir|basename }} {{ edxapp_course_static_dir|basename }}
+  when: EDXAPP_XML_S3_BUCKET and EDXAPP_XML_S3_KEY
+
+- name: upload archive to s3
+  s3: >
+    bucket="{{ EDXAPP_XML_S3_BUCKET }}"
+    object="{{ EDXAPP_XML_S3_KEY }}"
+    mode=put
+    overwrite=True
+    src="/tmp/static_course_content.tar.gz"
+  when: EDXAPP_XML_S3_BUCKET and EDXAPP_XML_S3_KEY
+
+- name: remove archive from disk
+  file: path="/tmp/static_course_content.tar.gz" state=absent
+  when: EDXAPP_XML_S3_BUCKET and EDXAPP_XML_S3_KEY

playbooks/roles/edxapp/templates/course.xml.j2

+<course org="{{ item.org }}" course="{{ item.course }}" url_name="{{ item.run }}" />

util/import_xml_courses.py

@@ -10,116 +10,53 @@
 #    course:
 #    run:
 #    disposition:
+#    version:
 #}
 
 import argparse
-import logging
-import subprocess
-from os.path import basename, exists, join
-import os
-import contextlib
-from getpass import getuser
-
-logging.basicConfig(level=logging.DEBUG)
-
-@contextlib.contextmanager
-def chdir(dirname=None):
-  curdir = os.getcwd()
-  try:
-    if dirname is not None:
-      os.chdir(dirname)
-    yield
-  finally:
-    os.chdir(curdir)
+from os.path import basename
+import yaml
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(
         description="Import XML courses from git repos.")
     parser.add_argument("-c", "--courses-csv", required=True,
         help="A CSV of xml courses to import.")
-    parser.add_argument("-d", "--var-dir", default="/edx/var/edxapp",
-        help="The location of the edxapp var directory that hold the data and courstatic dirs.")
     args = parser.parse_args()
 
     courses = open(args.courses_csv, 'r')
-    data_dir = join(args.var_dir, 'data')
-    static_dir = join(args.var_dir, 'course_static')
-
-
-    cwd = getcwd()
-    platform_dir = "/edx/app/edxapp/edx-platform"
 
+    all_course_data = []
+    all_xml_mappings = {}
     for line in courses:
         cols = line.strip().split(',')
         slug = cols[0]
         author_format = cols[1]
         disposition = cols[2]
         repo_url = cols[4]
+        version = cols[5]
 
         if author_format.lower() != 'xml' \
           or disposition.lower() == "don't import":
             continue
 
         # Checkout w/tilde
-        logging.debug("{}: {}".format(slug, repo_url))
         org, course, run = slug.split("/")
         repo_name = "{}~{}".format(basename(repo_url).rstrip('.git'), run)
 
-        # Clone course into data dir.
-        repo_location = join(args.data_dir, repo_name)
-        if not exists(repo_location):
-            cmd = "git clone {} {}".format(repo_url, repo_location)
-            subprocess.check_call(cmd,shell=True)
-            chown_cmd = "chown -R www-data:edxapp {}".format(repo_location)
-            subprocess.check_call(chown_cmd, shell=True)
-
-        # Update course.xml
-        course_xml_path = join(repo_location, "course.xml")
-        xml_content = '<course org="{}" course="{}" url_name="{}"/>'
-        f = open(course_xml_path, 'w')
-        f.write(xml_content.format(org, course, run))
-        f.close()
-
+        course_info = {
+            "repo_url": repo_url,
+            "repo_name": repo_name,
+            "org": org,
+            "course": course,
+            "run": run,
+            "disposition": disposition.lower(),
+            "version": version,
+        }
+        all_course_data.append(course_info)
 
         if disposition.lower() == "on disk":
-            with chdir(args.static_dir):
-                # Link static files to course_static dir
-                static_location = join(repo_location, 'static')
-                cmd = "ln -sf {} {}".format(static_location, repo_name)
-                logging.debug("Running cmd: {}".format(cmd))
-                subprocess.check_call(cmd, shell=True)
-        elif disposition.lower() == 'no static import':
-            # Go to the platform dir.
-            # Need this for dealer to work.
-            with chdir(platform_dir):
-                # Import w/nostatic flag
-                cmd = "sudo -E -u edxapp /edx/bin/python.edxapp /edx/app/edxapp/edx-platform/manage.py cms --settings=aws import --nostatic {} {}".format(args.data_dir, repo_name)
-                logging.debug("Running cmd: {}".format(cmd))
-                subprocess.check_call(cmd, shell=True)
-
-            with chdir(args.static_dir):
-                # Link static files to course_static dir.
-                static_location = join(repo_location, 'static')
-                cmd = "ln -sf {} {}".format(static_location, repo_name)
-                logging.debug("Running cmd: {}".format(cmd))
-                subprocess.check_call(cmd, shell=True)
-
-        elif disposition.lower() == 'import':
-            # Go to the platform dir.
-            # Need this for dealer to work.
-            with chdir(platform_dir):
-                # Import
-                cmd = "sudo -E -u edxapp /edx/bin/python.edxapp /edx/app/edxapp/edx-platform/manage.py cms --settings=aws import {} {}".format(args.data_dir, repo_name)
-                logging.debug("Running cmd: {}".format(cmd))
-                subprocess.check_call(cmd, shell=True)
-
-            # Remove from disk.
-            cmd = "sudo rm -rf {}".format(repo_location)
-            logging.debug("Running cmd: {}".format(cmd))
-            subprocess.check_call(cmd, shell=True)
-
-    # Create the tar file of all xml courses
-    cmd = "sudo tar cvzf static_course_content.tar.gz -C {} data course_static".format(args.var_dir)
-    logging.debug("Running cmd: {}".format(cmd))
-    subprocess.check_call(cmd, shell=True)
+            all_xml_mappings[slug] = 'xml'
 
+    edxapp_xml_courses = { "EDXAPP_XML_COURSES": all_course_data, "EDXAPP_XML_MAPPINGS": all_xml_mappings }
+    print yaml.safe_dump(edxapp_xml_courses, default_flow_style=False)