Merge pull request #1073 from edx/feanil/import_xml_courses

Feanil/import xml courses

playbooks/roles/aws/defaults/main.yml

@@ -56,6 +56,7 @@ aws_debian_pkgs:
 aws_pip_pkgs:
   - https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz
   - awscli
+  - boto==2.20.1
 
 aws_redhat_pkgs: []
 aws_s3cmd_version: s3cmd-1.5.0-beta1

playbooks/roles/edxapp/defaults/main.yml

@@ -207,6 +207,10 @@ EDXAPP_MICROSITE_CONFIGRATION: {}
 EDXAPP_COURSES_WITH_UNSAFE_CODE: []
 EDXAPP_SESSION_COOKIE_DOMAIN: ""
 
+# XML Course related flags
+EDXAPP_XML_FROM_GIT: false
+EDXAPP_XML_S3_BUCKET: !!null
+EDXAPP_XML_S3_KEY: !!null
 #-------- Everything below this line is internal to the role ------------
 
 #Use YAML references (& and *) and hash merge <<: to factor out shared settings
@@ -227,6 +231,7 @@ edxapp_gem_bin: "{{ edxapp_gem_root }}/bin"
 edxapp_user: edxapp
 edxapp_deploy_path: "{{ edxapp_venv_bin }}:{{ edxapp_code_dir }}/bin:{{ edxapp_rbenv_bin }}:{{ edxapp_rbenv_shims }}:{{ edxapp_gem_bin }}:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
 edxapp_staticfile_dir: "{{ edxapp_data_dir }}/staticfiles"
+edxapp_course_static_dir: "{{ edxapp_data_dir }}/course_static"
 edxapp_course_data_dir: "{{ edxapp_data_dir }}/data"
 edxapp_upload_dir: "{{ edxapp_data_dir }}/uploads"
 edxapp_theme_dir: "{{ edxapp_data_dir }}/themes"

playbooks/roles/edxapp/tasks/deploy.yml

@@ -219,7 +219,6 @@
   - "restart edxapp"
   - "restart edxapp_workers"
 
-
 # If using CAS and you have a function for mapping attributes, install
 # the module here.  The next few tasks set up the python code sandbox
 - name: install CAS attribute module
@@ -250,6 +249,32 @@
   - "restart edxapp"
   - "restart edxapp_workers"
 
+# The next few tasks install xml courses.
+
+# Install the xml courses from an s3 bucket
+- name: get s3 one time url
+  s3: >
+    bucket="{{ EDXAPP_XML_S3_BUCKET }}"
+    object="{{ EDXAPP_XML_S3_KEY }}"
+    mode="geturl"
+    expiration=300
+  when: not EDXAPP_XML_FROM_GIT and EDXAPP_XML_S3_BUCKET and EDXAPP_XML_S3_KEY
+  register: s3_one_time_url
+
+- name: download from one time url
+  get_url: url="{{ s3_one_time_url.s3_url }}" dest="/tmp/{{ EDXAPP_XML_S3_KEY|basename }}"
+  when: not EDXAPP_XML_FROM_GIT and EDXAPP_XML_S3_BUCKET and EDXAPP_XML_S3_KEY
+  register: download_xml_s3
+
+- name: unzip the data to the data dir
+  shell: >
+    tar xzf /tmp/{{ EDXAPP_XML_S3_KEY|basename }}
+    chdir="{{ edxapp_course_data_dir }}"
+  when: download_xml_s3.changed
+
+- include: xml.yml
+  when: EDXAPP_XML_FROM_GIT
+
 # The next few tasks set up the python code sandbox
 
 # need to disable this profile, otherwise the pip inside the sandbox venv has no permissions

playbooks/roles/edxapp/tasks/main.yml

@@ -25,6 +25,7 @@
     - "{{ edxapp_venvs_dir }}"
     - "{{ edxapp_theme_dir }}"
     - "{{ edxapp_staticfile_dir }}"
+    - "{{ edxapp_course_static_dir }}"
 
 - name: create edxapp log dir
   file: >

playbooks/roles/edxapp/tasks/xml.yml

+- name: clone the xml course repo
+  git: >
+    repo="{{ item.repo_url }}"
+    dest="{{ edxapp_course_data_dir }}/{{ item.repo_name }}"
+    version="{{ item.version }}"
+    accept_hostkey=True
+  sudo_user: "{{ edxapp_user }}"
+  environment:
+    GIT_SSH: "{{ edxapp_git_ssh }}"
+  with_items: EDXAPP_XML_COURSES
+
+- name: update course.xml
+  template: >
+    src=course.xml.j2
+    dest="{{ edxapp_course_data_dir }}/{{ item.repo_name }}/course.xml"
+  sudo_user: "{{ edxapp_user }}"
+  with_items: EDXAPP_XML_COURSES
+
+- name: make symlinks for the static data
+  shell: >
+    executable=/bin/bash
+    if [[ -d {{ edxapp_course_data_dir }}/{{ item.repo_name }}/static ]]; then
+      ln -sf {{ edxapp_course_data_dir }}/{{ item.repo_name }}/static {{ edxapp_course_static_dir }}/{{ item.repo_name }}
+    else
+      ln -sf {{ edxapp_course_data_dir }}/{{ item.repo_name }} {{ edxapp_course_static_dir }}/{{ item.repo_name }}
+    fi
+  with_items: EDXAPP_XML_COURSES
+  when: item.disposition == "on disk" or item.disposition == "no static import"
+
+- name: import courses with nostatic flag
+  shell: >
+    {{ edxapp_venv_bin }}/python manage.py cms --settings=aws import --nostatic {{ edxapp_course_data_dir }} {{ item.repo_name }}
+    chdir="{{ edxapp_code_dir }}"
+  sudo_user: "{{ edxapp_user }}"
+  with_items: EDXAPP_XML_COURSES
+  when: item.disposition == "no static import"
+  
+- name: import courses including static data
+  shell: >
+    {{ edxapp_venv_bin }}/python manage.py cms --settings=aws import {{ edxapp_course_data_dir }} {{ item.repo_name }}
+    chdir="{{ edxapp_code_dir }}"
+  sudo_user: "{{ edxapp_user }}"
+  with_items: EDXAPP_XML_COURSES
+  when: item.disposition == "import"
+
+- name: delete courses that were fully imported
+  file: path="{{ edxapp_course_data_dir }}/{{ item.repo_name }}" state=absent
+  with_items: EDXAPP_XML_COURSES
+  when: item.disposition == "import"
+
+- name: delete .git repos
+  file: path="{{ edxapp_course_data_dir }}/{{ item.repo_name }}/.git" state=absent
+  with_items: EDXAPP_XML_COURSES
+  when: item.disposition == "on disk" or item.disposition == "no static import"
+
+- name: create an archive of course data and course static dirs
+  shell: tar cjf /tmp/static_course_content.tar.bz2 -C {{ edxapp_data_dir }} {{ edxapp_course_data_dir|basename }} {{ edxapp_course_static_dir|basename }}
+
+- name: upload archive to s3
+  s3: >
+    bucket="{{ EDXAPP_XML_S3_BUCKET }}"
+    object="{{ EDXAPP_XML_S3_KEY }}"
+    mode=put
+    overwrite=True
+    src="/tmp/static_course_content.tar.bz2"
+  when: EDXAPP_XML_S3_BUCKET and EDXAPP_XML_S3_KEY
+
+- name: remove archive from disk
+  file: path="/tmp/static_course_content.tar.bz2" state=absent
+  when: EDXAPP_XML_S3_BUCKET and EDXAPP_XML_S3_KEY

playbooks/roles/edxapp/templates/course.xml.j2

+<course org="{{ item.org }}" course="{{ item.course }}" url_name="{{ item.run }}" />

util/import_xml_courses.py

+# Import XML Courses from git repos into the CMS.
+# Run with sudo and make sure the user can clone
+# the course repos.
+
+# Output Has per course
+#{
+#    repo_url:
+#    repo_name:
+#    org:
+#    course:
+#    run:
+#    disposition:
+#    version:
+#}
+
+import argparse
+from os.path import basename
+import yaml
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        description="Import XML courses from git repos.")
+    parser.add_argument("-c", "--courses-csv", required=True,
+        help="A CSV of xml courses to import.")
+    args = parser.parse_args()
+
+    courses = open(args.courses_csv, 'r')
+
+    all_course_data = []
+    all_xml_mappings = {}
+    for line in courses:
+        cols = line.strip().split(',')
+        slug = cols[0]
+        author_format = cols[1]
+        disposition = cols[2]
+        repo_url = cols[4]
+        version = cols[5]
+
+        if author_format.lower() != 'xml' \
+          or disposition.lower() == "don't import":
+            continue
+
+        # Checkout w/tilde
+        org, course, run = slug.split("/")
+        repo_name = "{}~{}".format(basename(repo_url).rstrip('.git'), run)
+
+        course_info = {
+            "repo_url": repo_url,
+            "repo_name": repo_name,
+            "org": org,
+            "course": course,
+            "run": run,
+            "disposition": disposition.lower(),
+            "version": version,
+        }
+        all_course_data.append(course_info)
+
+        if disposition.lower() == "on disk":
+            all_xml_mappings[slug] = 'xml'
+
+    edxapp_xml_courses = { "EDXAPP_XML_COURSES": all_course_data, "EDXAPP_XML_MAPPINGS": all_xml_mappings }
+    print yaml.safe_dump(edxapp_xml_courses, default_flow_style=False)