Addition of gitreload role for github Web hook course imports

Also support updating repositories such as python graders
from xqwatcher/xserver

playbooks/roles/gitreload/defaults/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+##
+# Defaults for role gitreload
+# 
+
+GITRELOAD_COURSE_CHECKOUT: false
+GITRELOAD_GIT_IDENTITY: !!null
+GITRELOAD_REPOS:
+  - name: "edx-demo-course"
+    url: "https://github.com/edx/edx-demo-course.git"
+    commit: "master"
+GITRELOAD_REPODIR: "{{ edxapp_data_dir }}/course_repos"
+GITRELOAD_LOG_LEVEL: "debug"
+GITRELOAD_HOSTS:
+  - github.com
+GITRELOAD_NUM_THREADS: 1
+GITRELOAD_NGINX_PORT: "18095"
+
+gitreload_version: "master"
+gitreload_dir: "{{ COMMON_APP_DIR }}/gitreload"
+gitreload_user: "gitreload"
+gitreload_repo: "https://github.com/mitodl/gitreload.git"
+gitreload_gunicorn_port: "8095"
+gitreload_venv: "{{ gitreload_dir }}/venvs/gitreload"
+gitreload_venv_bin: "{{ gitreload_venv }}/bin"
+gitreload_gunicorn_workers: 1
+gitreload_gunicorn_host: "127.0.0.1"
+
+gitreload_edxapp_django_settings: "aws"
+
+gitreload_env:
+  REPODIR: "{{ GITRELOAD_REPODIR }}"
+  LOG_LEVEL: "{{ GITRELOAD_LOG_LEVEL }}"
+  NUM_THREADS: "{{ GITRELOAD_NUM_THREADS }}"
+  VIRTUAL_ENV: "{{ edxapp_venv_dir }}"
+  EDX_PLATFORM: "{{ edxapp_code_dir }}"
+  DJANGO_SETTINGS: "{{ gitreload_edxapp_django_settings }}"

playbooks/roles/gitreload/handlers/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+#
+#
+# Handlers for role gitreload
+# 
+# Overview:
+# 
+#
+- name: restart gitreload
+  supervisorctl_local: >
+    name=gitreload
+    supervisorctl_path={{ supervisor_ctl }}
+    config={{ supervisor_cfg }}
+    state=restarted
+  when: not disable_edx_services

playbooks/roles/gitreload/meta/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+##
+# Role includes for role gitreload
+# 
+dependencies:
+  - common
+  - supervisor

playbooks/roles/gitreload/tasks/course_pull.yml

+# Tasks to run if cloning repos to edx-platform.
+
+- name: clone all course repos
+  git: dest={{ GITRELOAD_REPODIR }}/{{ item.name }} repo={{ item.url }} version={{ item.commit }}
+  sudo_user: "{{ common_web_user }}"
+  with_items: GITRELOAD_REPOS
+
+- name: do import of courses
+  shell: >
+    executable=/bin/bash
+    chdir="{{ edxapp_code_dir }}"
+    SERVICE_VARIANT=lms {{ edxapp_venv_bin }}/python manage.py lms --settings=aws git_add_course {{ item.url }} {{ GITRELOAD_REPODIR }}/{{ item.name }}
+  sudo_user: "{{ common_web_user }}"
+  with_items: GITRELOAD_REPOS
+
+- name: change ownership on repos for access by edxapp and www-data
+  file: >
+    path={{ GITRELOAD_REPODIR }}
+    state=directory
+    owner={{ common_web_user }}
+    owner={{ common_web_group }}
+    recurse=yes
+
+- name: change group on repos if using devstack
+  file: >
+    path={{ GITRELOAD_REPODIR }}
+    state=directory
+    group={{ edxapp_user }}
+    recurse=yes
+  when: devstack
+
+- name: change mode on repos with using devstack
+  command: chmod -R o=rwX,g=srwX,o=rX {{ GITRELOAD_REPODIR }}
+  when: devstack

playbooks/roles/gitreload/tasks/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+#
+#
+# Tasks for role gitreload
+# 
+# Overview: Install gitreload for doing course reload via github Web hooks
+# 
+#
+# Dependencies: supervisor, common
+#
+# 
+# Example playbook:
+#
+# - hosts: all
+#   sudo: True
+#   gather_facts: True
+#   vars:
+#     COMMON_ENABLE_BASIC_AUTH: True
+#   roles:
+#     - gitreload
+#     - role: nginx
+#       nginx_sites:
+#         - gitreload
+#
+
+# grab edxapp vars without being dependent on its tasks
+- include_vars: ../../edxapp/defaults/main.yml
+
+- name: create gitreload user
+  user: >
+    name="{{ gitreload_user }}"
+    home="{{ gitreload_dir }}"
+    createhome=no
+    shell=/bin/false
+
+- name: ensure home folder exists
+  file: >
+    path={{ gitreload_dir }}
+    state=directory
+    owner={{ gitreload_user }}
+    group={{ gitreload_user }}
+
+- name: ensure repo dir exists
+  file: >
+    path={{ GITRELOAD_REPODIR }}
+    state=directory
+    owner={{ common_web_user }}
+    group={{ common_web_group }}
+
+- name: create ssh dir for the content repos key
+  file: path=~/.ssh state=directory mode=0700
+  sudo_user: "{{ common_web_user }}"
+  tags:
+    - deploy
+
+- name: install ssh key for the content repos
+  copy: content="{{ GITRELOAD_GIT_IDENTITY }}" dest=~/.ssh/id_rsa mode=0600
+  sudo_user: "{{ common_web_user }}"
+  tags:
+    - deploy
+
+- name: grab ssh host keys
+  shell: ssh-keyscan {{ item }}
+  sudo_user: "{{ common_web_user }}"
+  with_items: GITRELOAD_HOSTS
+  register: gitreload_repo_host_keys
+
+- name: add host keys if needed to known_hosts
+  lineinfile: >
+    create=yes
+    dest=~/.ssh/known_hosts
+    line="{{ item.stdout }}"
+  sudo_user: "{{ common_web_user }}"
+  with_items: gitreload_repo_host_keys.results
+
+- name: install gitreload
+  pip: >
+    name=git+{{ gitreload_repo }}@{{ gitreload_version }}#egg=gitreload
+    virtualenv={{ gitreload_venv }}
+    extra_args="--exists-action w"
+  sudo_user: "{{ gitreload_user }}"
+  notify: restart gitreload
+  tags:
+    - deploy
+
+- name: copy configuration
+  template: src=edx/app/gitreload/gr.env.json.j2 dest={{ gitreload_dir }}/gr.env.json
+  sudo_user: "{{ gitreload_user }}"
+  notify: restart gitreload
+  tags:
+    - deploy
+
+- include: course_pull.yml
+  when: GITRELOAD_COURSE_CHECKOUT|bool
+  tags: course_pull
+
+- name: "writing supervisor script"
+  template: >
+    src=edx/app/supervisor/conf.available.d/gitreload.conf.j2 dest={{ supervisor_available_dir }}/gitreload.conf
+    owner={{ supervisor_user }} group={{ common_web_user }} mode=0644
+
+- name: "enable supervisor script"
+  file: >
+    src={{ supervisor_available_dir }}/gitreload.conf
+    dest={{ supervisor_cfg_dir }}/gitreload.conf
+    owner={{ supervisor_user }} group={{ common_web_user }} mode=0644
+    state=link force=yes
+  when: not disable_edx_services
+
+  # call supervisorctl update. this reloads
+  # the supervisorctl config and restarts
+  # the services if any of the configurations
+  # have changed.
+  #
+- name: update supervisor configuration
+  shell:  "{{ supervisor_ctl }} -c {{ supervisor_cfg }} update"
+  register: supervisor_update
+  changed_when: supervisor_update.stdout is defined and supervisor_update.stdout != ""
+  when: not disable_edx_services
+
+- name: ensure gitreload is started
+  supervisorctl_local: >
+    name=gitreload
+    supervisorctl_path={{ supervisor_ctl }}
+    config={{ supervisor_cfg }}
+    state=started
+  when: not disable_edx_services
+  tags:
+    - deploy
+
+- name: create a symlink for venv python
+  file: >
+    src="{{ gitreload_venv_bin }}/{{ item }}"
+    dest={{ COMMON_BIN_DIR }}/{{ item }}.gitreload
+    state=link
+  with_items:
+  - python
+  - pip

playbooks/roles/gitreload/templates/edx/app/gitreload/gr.env.json.j2

+{{ gitreload_env | to_nice_json }} 

playbooks/roles/gitreload/templates/edx/app/supervisor/conf.available.d/gitreload.conf.j2

+[program:gitreload]
+
+user={{ common_web_user }}
+directory={{ gitreload_dir }}
+umask=002
+
+command={{ gitreload_venv }}/bin/gunicorn --preload -b {{ gitreload_gunicorn_host }}:{{ gitreload_gunicorn_port }} -w {{ gitreload_gunicorn_workers }} --timeout=10 gitreload.web:app
+
+environment=PID=/var/tmp/gitreload.pid
+stdout_logfile={{ supervisor_log_dir }}/%(program_name)-stdout.log
+stderr_logfile={{ supervisor_log_dir }}/%(program_name)-stderr.log
+killasgroup=true
+stopasgroup=true

playbooks/roles/nginx/defaults/main.yml

@@ -89,6 +89,8 @@ nginx_analytics_api_gunicorn_hosts:
   - 127.0.0.1
 nginx_insights_gunicorn_hosts:
   - 127.0.0.1
+nginx_gitreload_gunicorn_hosts:
+  - 127.0.0.1
 
 nginx_cfg:
   #   - link - turn on

playbooks/roles/nginx/templates/edx/app/nginx/sites-available/gitreload.j2

+upstream gitreload_app_server {
+    {% for host in nginx_gitreload_gunicorn_hosts %}
+        server {{ host }}:{{ gitreload_gunicorn_port }} fail_timeout=0;
+    {% endfor %}
+}
+
+server {
+  listen {{ GITRELOAD_NGINX_PORT }} default_server;
+
+  location / {
+     auth_basic            "Restricted";
+     auth_basic_user_file  {{ nginx_htpasswd_file }};
+
+     try_files $uri @proxy_to_app;
+  }
+
+  # No basic auth security on the queue status url, so that it can
+  # checked easily
+  location /queue{
+    try_files $uri @proxy_to_app;
+  }
+
+  {% include "robots.j2" %}
+
+location @proxy_to_app {
+    proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto;
+    proxy_set_header X-Forwarded-Port $http_x_forwarded_port;
+    proxy_set_header X-Forwarded-For $http_x_forwarded_for;
+    proxy_set_header Host $http_host;
+
+    proxy_redirect off;
+    proxy_pass http://gitreload_app_server;
+  }
+}
+