Merge remote-tracking branch 'origin/master' into vik/ml-api

Conflicts:
	playbooks/group_vars/all

AUTHORS

+John Jarvis <jarv@edx.org>
+Sef Kloninger <sef@kloninger.com>
+Joe Blaylock <jrbl@jrbl.org>
+Vik Paruchuri <vik@edx.org>
+Jason Bau <jbau@stanford.edu>
+Ed Zarecor <ed@edx.org>
+John Kern <kern3020@gmail.com>

cloudformation_templates/edx-reference-architecture.json

@@ -1672,8 +1672,8 @@
         "LaunchConfigurationName":{
           "Ref":"EdxappServer"
         },
-        "MinSize":"1",
-        "MaxSize":"6",
+        "MinSize":"2",
+        "MaxSize":"2",
         "DesiredCapacity":{
           "Ref":"EdxappDesiredCapacity"
         },
@@ -2048,7 +2048,7 @@
         "LaunchConfigurationName":{
           "Ref":"XqueueServer"
         },
-        "MinSize":"1",
+        "MinSize":"2",
         "MaxSize":"2",
         "DesiredCapacity":{
           "Ref":"XqueueDesiredCapacity"
@@ -2421,7 +2421,7 @@
         "LaunchConfigurationName":{
           "Ref":"RabbitMQServer"
         },
-        "MinSize":"1",
+        "MinSize":"2",
         "MaxSize":"2",
         "DesiredCapacity":{
           "Ref":"RabbitMQDesiredCapacity"
@@ -2806,7 +2806,7 @@
         "LaunchConfigurationName":{
           "Ref":"XServer"
         },
-        "MinSize":"1",
+        "MinSize":"2",
         "MaxSize":"2",
         "DesiredCapacity":{
           "Ref":"XServerDesiredCapacity"

playbooks/group_vars/all

@@ -2,10 +2,13 @@
 app_base_dir: /opt/wwc
 log_base_dir: /mnt/logs
 venv_dir: /opt/edx
+platform_code_dir: $app_base_dir/edx-platform
+os_name: ubuntu
 
 # these pathes are relative to the playbook dir
 # directory for secret settings (keys, etc)
 secure_dir: '../../edx-secret'
+
 # this indicates the path to site-specific (with precedence)
 # things like nginx template files
 local_dir:  '../../ansible_local'

playbooks/roles/common/tasks/software_update.yml

 ---
 - name: edx-update.sh, manual lms/cms update script
-  copy: src=roles/common/files/edx-update.sh dest=/usr/local/bin/edx-update.sh owner=ubuntu group=adm mode=0775
+  template: src=edx-update.sh.j2 dest=/usr/local/bin/edx-update.sh owner=ubuntu group=adm mode=0775
   tags:
   - update
   - release

playbooks/roles/common/files/edx-update.sh → playbooks/roles/common/templates/edx-update.sh.j2

 #!/bin/bash
+# Placed by ansible; some hardcoded values originate from variables substituted
+# by jinja
 
 function usage() {
     echo "update.sh [cms|lms|all|none]"
@@ -41,10 +43,10 @@ function run() {
 }
 
 source /etc/profile
-source /opt/edx/bin/activate
+source {{venv_dir}}/bin/activate
 export PATH=$PATH:/opt/www/.gem/bin
 
-cd /opt/wwc/mitx
+cd {{platform_code_dir}}
 
 BRANCH="origin/feature/edx-west/stanford-theme"
 

playbooks/roles/gunicorn/templates/cms.conf.j2

 # gunicorn
+# Templated and placed by ansible from jinja2 source
  
 description "gunicorn server"
 author "Calen Pennington <cpennington@mitx.mit.edu>"
@@ -17,8 +18,8 @@ env PORT=8010
 env LANG=en_US.UTF-8
 env DJANGO_SETTINGS_MODULE=cms.envs.aws
 env SERVICE_VARIANT="cms"
-  
-chdir ${app_base_dir}/mitx
+ 
+chdir {{platform_code_dir}}
 setuid www-data
  
-exec ${venv_dir}/bin/gunicorn_django -b 127.0.0.1:$PORT -w $WORKERS --timeout=300 --pythonpath=${app_base_dir}/mitx --settings=cms.envs.aws
+exec {{venv_dir}}/bin/gunicorn_django -b 127.0.0.1:$PORT -w $WORKERS --timeout=300 --pythonpath={{platform_code_dir}} --settings=cms.envs.aws

playbooks/roles/gunicorn/templates/lms-preview.conf.j2

@@ -18,10 +18,10 @@ env LANG=en_US.UTF-8
 env DJANGO_SETTINGS_MODULE=lms.envs.aws
 env SERVICE_VARIANT="lms-preview"
   
-chdir ${app_base_dir}/mitx
+chdir {{platform_code_dir}}
 setuid www-data
 
-exec ${venv_dir}/bin/gunicorn --preload -b 127.0.0.1:$PORT -w $WORKERS --timeout=300 --pythonpath=${app_base_dir}/mitx lms.wsgi
+exec ${venv_dir}/bin/gunicorn --preload -b 127.0.0.1:$PORT -w $WORKERS --timeout=300 --pythonpath={{platform_code_dir}} lms.wsgi
 
 post-start script
   while true

playbooks/roles/gunicorn/templates/lms.conf.j2

@@ -17,11 +17,11 @@ env PORT=8000
 env LANG=en_US.UTF-8
 env DJANGO_SETTINGS_MODULE=lms.envs.aws
 env SERVICE_VARIANT="lms"
-  
-chdir ${app_base_dir}/mitx
+
+chdir {{platform_code_dir}}
 setuid www-data
 
-exec ${venv_dir}/bin/gunicorn --preload -b 127.0.0.1:$PORT -w $WORKERS --timeout=300 --pythonpath=${app_base_dir}/mitx lms.wsgi
+exec ${venv_dir}/bin/gunicorn --preload -b 127.0.0.1:$PORT -w $WORKERS --timeout=300 --pythonpath={{platform_code_dir}} lms.wsgi
 
 post-start script
   while true

playbooks/roles/lms/tasks/main.yml

@@ -39,12 +39,12 @@
 # If we set up LMS, we have to set up edx logging
 - include: ../../common/tasks/edx_logging_base.yml
 
-# Install ssh keys for ubuntu account to be able to check out from mitx
+# Install ssh keys for ubuntu account to be able to check out from edx-platform
 # Temprory behavior, not needed after June 1. Perhaps still useful as a recipe.
 # {{ secure_dir }} is relative to the top-level playbooks dir so there is some
 # ugly relative pathing here
 
-- name: install read-only ssh key for mitx repo (private)
+- name: install read-only ssh key for edx-platform repo (private)
   copy: src=../../../{{ secure_dir }}/files/git-identity dest=/etc/git-identity force=yes owner=ubuntu group=adm mode=600
   tags:
   - lms
@@ -58,7 +58,7 @@
   - cms
   - update
 
-# Check out mitx repo to $app_base_dir
+# Check out edx-platform repo to $app_base_dir
 - name: install git and its recommends
   apt: pkg=git state=present install_recommends=yes 
   tags:
@@ -76,7 +76,7 @@
 
 ## Install the debian package requirements system-wide
 - name: store remote apt_repos list for ansible use
-  command: cat {{app_base_dir}}/mitx/apt-repos.txt
+  command: cat {{ apt_sources_file }}
   register: apt_repos_list
   tags:
   - lms
@@ -103,7 +103,7 @@
   - install
 
 - name: store remote apt_packages list for ansible use
-  command: cat {{app_base_dir}}/mitx/apt-packages.txt
+  command: cat {{ apt_packages_file }}
   register: apt_packages_list
   tags:
   - lms
@@ -118,25 +118,42 @@
   - cms
   - install
 
-# Install the python requirements into $venv_dir 
-- name : install python pre-requirements
-  pip: requirements="{{app_base_dir}}/mitx/pre-requirements.txt" virtualenv="{{venv_dir}}" state=present
+
+# Install the python modules into {{ venv_dir }}
+- name : install base python packages using the shell
+  # Need to use shell rather than pip so that we can maintain the context of our current working directory; some 
+  # requirements are pathed relative to the edx-platform repo. Using the pip from inside the virtual environment implicitly
+  # installs everything into that virtual environment.
+  shell: cd {{ platform_code_dir }} && {{ venv_dir }}/bin/pip install --use-mirrors -r {{ base_requirements_file }}
+  tags:
+  - lms
+  - cms
+  - install
+  
+  
+# Install the python post requirements into {{ venv_dir }}
+- name : install python post-requirements
+  pip: requirements="{{ post_requirements_file }}" virtualenv="{{ venv_dir }}" state=present
   tags:
   - lms
   - cms
   - install
 
-# Install the python modules into $venv_dir 
-- name : install python packages using the shell
-  #pip: requirements="{{app_base_dir}}/mitx/requirements.txt" virtualenv="{{venv_dir}}" 
+# Install the final python modules into {{ venv_dir }}
+- name : install python post-post requirements using the shell
   # Need to use shell rather than pip so that we can maintain the context of our current working directory; some 
-  # requirements are pathed relative to the mitx repo. Using the pip from inside the virtual environment implicitly
+  # requirements are pathed relative to the edx-platform repo. Using the pip from inside the virtual environment implicitly
   # installs everything into that virtual environment.
-  shell: cd {{app_base_dir}}/mitx && {{venv_dir}}/bin/pip install --use-mirrors -r {{app_base_dir}}/mitx/requirements.txt
+  shell: cd {{ platform_code_dir }} && {{ venv_dir }}/bin/pip install --use-mirrors -r {{ item }}
+  with_items:
+  - "{{ repo_requirements_file }}"
+  - "{{ github_requirements_file }}"
+  - "{{ local_requirements_file }}"
   tags:
   - lms
   - cms
   - install
   
+
 # Creates LMS upstart file
 - include: ../../gunicorn/tasks/upstart.yml service_variant=lms

playbooks/roles/lms/vars/main.yml

@@ -4,6 +4,16 @@
 lms_auth_config: {}
 lms_env_config: {}
 lms_source_repo: git@github.com:edx/edx-platform.git
+
+apt_sources_file:  "{{ platform_code_dir }}/requirements/system/{{os_name}}/apt-repos.txt"
+apt_packages_file: "{{ platform_code_dir }}/requirements/system/{{os_name}}/apt-packages.txt"
+
+local_requirements_file:  "{{ platform_code_dir }}/requirements/local.txt"
+post_requirements_file:   "{{ platform_code_dir }}/requirements/post.txt"
+base_requirements_file:   "{{ platform_code_dir }}/requirements/base.txt"
+github_requirements_file: "{{ platform_code_dir }}/requirements/github.txt"
+repo_requirements_file:   "{{ platform_code_dir }}/requirements/repo.txt"
+
 lms_debian_pkgs:
   - apparmor-utils
   - aspell

playbooks/roles/npm/tasks/main.yml

@@ -8,8 +8,8 @@
   - npm
   - install
   
-- name: Install mitx npm dependencies
-  shell: npm install chdir=${app_base_dir}/mitx
+- name: Install edx-platform npm dependencies
+  shell: npm install chdir=${platform_code_dir}
   tags:
   - npm
   - update

playbooks/roles/ruby/tasks/main.yml

@@ -113,13 +113,13 @@
   - install
 
 - name: gem | gem install bundler
-  shell: RBENV_ROOT=${rbenv_root} GEM_HOME=${gem_home} ${rbenv_root}/shims/gem install bundle chdir=${app_base_dir}/mitx
+  shell: RBENV_ROOT=${rbenv_root} GEM_HOME=${gem_home} ${rbenv_root}/shims/gem install bundle chdir=${platform_code_dir}
   tags:
   - ruby
   - install
 
 - name: bundle | bundle install
-  shell: RBENV_ROOT=${rbenv_root} GEM_HOME=${gem_home} ${gem_home}/bin/bundle install --binstubs chdir=${app_base_dir}/mitx
+  shell: RBENV_ROOT=${rbenv_root} GEM_HOME=${gem_home} ${gem_home}/bin/bundle install --binstubs chdir=${platform_code_dir}
   tags:
   - ruby
   - install