resolving merge conflicts

README.md

@@ -210,10 +210,47 @@ If that works fine, then you can add an export of PYTHONPATH to
 * Creates base directories
 * Creates the lms json configuration files
 
+Because the reference architecture makes use of an Amazon VPC, you will not be able
+to address the hosts in the private subnets directly.  However, you can easily set 
+up a transparent "jumpbox" so that for all hosts in your vpc, connections are 
+tunneled
+
+Add something like the following to your `~/.ssh/config` file.
+
+```
+Host vpc-00000000-jumpbox 
+  HostName 54.236.224.226
+  IdentityFile /path/to/aws/key.pem
+  ForwardAgent yes
+  User ubuntu
+
+Host 10.0.10.1
+  ProxyCommand ssh -W %h:%p vpc-00000000-jumpbox
+  ForwardAgent yes
+  HostName 10.0.10.242
+  IdentityFile /path/to/aws/key.pem
+  User ubuntu
+  
+  ...
+
+```
+You will need a host entry for each server that ansible will configure.
+
+Test this by typing `ssh 10.0.10.1`, use a hostname the exists in
+your environment.   If things are configured correctly you will ssh 
+to 10.0.10.1, jumping transparently via your basion host.
+
+Assuming that the edxapp_stage.yml playbook targets hosts in your vpc
+for which there are entiries in your `.ssh/config`, do the 
+following to run your playbook.
+
 ```
   cd playbooks
-  ansible-playbook -v --user=ubuntu edxapp_stage.yml -i ./ec2.py --private-key=/path/to/aws/key.pem
+  ansible-playbook -v --user=ubuntu edxapp_stage.yml -i ./ec2.py -c ssh
 ```
+We are in the process of writing utility scripts to generate this config,
+but for the moment it is a manual process.
+
 
 *Note: this assumes the group used for the edx stack was "edxapp_stage"*
 

playbooks/group_vars/all

@@ -2,3 +2,4 @@
 app_base_dir: /opt/wwc
 # this path is relative to the playbook dir
 secure_dir: 'secure_example'
+venv_dir: /opt/edx

playbooks/roles/common/tasks/create_users.yml

 ---
-- name: Create 'www' user (replicating historical environment)
-  user: name=www state=present
-  sudo: True
 - name: Create 'edx' users group
   group: name=edx state=present
   sudo: True

playbooks/roles/common/tasks/create_venv.yml

+---
+# create the 'edx' virtual environment in /opt so that roles can populate it
+- name: easy_install pip
+  easy_install: name=pip
+  sudo: True
+  tags:
+  - venv_base
+- name: pip install virtualenv
+  pip: name=virtualenv state=latest
+  sudo: True
+  tags:
+  - venv_base
+- name: pip install virtualenvwrapper
+  pip: name=virtualenvwrapper state=latest
+  sudo: True
+  tags:
+  - venv_base
+- name: create edx virtualenv directory
+  file: path=$venv_dir owner=ubuntu group=edx mode=2775 state=directory
+  sudo: True
+  tags:
+  - venv_base
+- name: create the edx virtualenv directory initial contents
+  command: /usr/local/bin/virtualenv $venv_dir creates=$venv_dir/bin/activate
+  tags:
+  - venv_base

playbooks/roles/common/tasks/main.yml

 ---
 - include: create_users.yml
 - name: Create application root
-  sudo: True
   file: path=$app_base_dir state=directory owner=root group=root
-- name: Create log directory
   sudo: True
+- name: Create log directory
   file: path=/mnt/logs state=directory
-- name: Update apt cache
   sudo: True
+- name: Update apt cache
   apt: update_cache=yes
+  sudo: True
+
+- include: create_venv.yml
+
+- name: Install role-independent useful system packages
+  apt: pkg={{item}} install_recommends=yes state=present
+  with_items:
+  - mosh
+  - tmux
+  sudo: True
+

playbooks/roles/lms/tasks/main.yml

 # requires:
+#  - group_vars/all
 #  - common/tasks/main.yml
 #  - nginx/tasks/main.yml
 ---
 - name: create lms application config
   template: src=env.json.j2 dest=$app_base_dir/lms.env.json
   sudo: True
+  tags:
+  - lms
 
 - name: create lms auth file
   template: src=auth.json.j2 dest=$app_base_dir/lms.auth.json
   sudo: True
+  tags:
+  - lms
 
 - include: ../../nginx/tasks/nginx_site.yml state=link site_name=lms
 - include: ../../nginx/tasks/nginx_site.yml state=link site_name=lms-backend
@@ -21,20 +26,88 @@
 - name: install read-only ssh key for mitx repo (private)
   copy: src=../../../{{ secure_dir }}/files/git-identity dest=/etc/git-identity force=yes owner=root group=root mode=644
   sudo: True
+  tags:
+  - lms
+  - cms
 
 - name: upload ssh script
   copy: src=git_ssh.sh dest=/tmp/git_ssh.sh force=yes owner=root group=root mode=755
   sudo: True
+  tags:
+  - lms
+  - cms
 
 # Check out mitx repo to $app_base_dir
 - name: set permissions on $app_base_dir sgid for edx
   file: path=$app_base_dir owner=root group=edx mode=2775 state=directory
-  file: path=$app_base_dir owner=ubuntu group=edx mode=2775 state=directory
   sudo: True
+  tags:
+  - lms
+  - cms
 - name: install git and its recommends
-  apt: pkg=git state=installed install_recommends=yes 
+  apt: pkg=git state=present install_recommends=yes 
   sudo: True
+  tags:
+  - lms
+  - cms
 - name: git checkout mitx repo into $app_base_dir
-  git: dest=$app_base_dir/mitx repo=git@github.com:MITx/mitx.git
+  git: dest={{app_base_dir}}/mitx repo={{lms_source_repo}}
   environment:
     GIT_SSH: /tmp/git_ssh.sh
+  tags:
+  - lms
+  - cms
+
+## Install the debian package requirements system-wide
+- name: store remote apt_repos list for ansible use
+  command: cat {{app_base_dir}}/mitx/apt-repos.txt
+  register: apt_repos_list
+  tags:
+  - lms
+  - cms
+- name: add apt_repos to the remote hosts
+  apt_repository: repo="$item"
+  with_items: "{{apt_repos_list.stdout.split()}}"
+  register: apt_repos_list_repo_adds
+  sudo: True
+  tags:
+  - lms
+  - cms
+- name: update apt cache (if necessary)
+  apt: update_cache=yes
+  sudo: True
+  only_if: "{{apt_repos_list_repo_adds.changed}}"
+  tags:
+  - lms
+  - cms
+- name: store remote apt_packages list for ansible use
+  command: cat {{app_base_dir}}/mitx/apt-packages.txt
+  register: apt_packages_list
+  tags:
+  - lms
+  - cms
+- name: install a bunch of system packages on which LMS and CMS rely
+  apt: pkg={{item}} state=present
+  with_items: lms_debian_pkgs
+  sudo: True
+  tags:
+  - lms
+  - cms
+
+# Install the python requirements into $venv_dir 
+- name : install python pre-requirements
+  pip: requirements="{{app_base_dir}}/mitx/pre-requirements.txt" virtualenv="{{venv_dir}}" state=present
+  tags:
+  - lms
+  - cms
+
+# Install the python modules into $venv_dir 
+- name : install python pre-requirements
+  #pip: requirements="{{app_base_dir}}/mitx/requirements.txt" virtualenv="{{venv_dir}}" 
+  # Need to use shell rather than pip so that we can maintain the context of our current working directory; some 
+  # requirements are pathed relative to the mitx repo. Using the pip from inside the virtual environment implicitly
+  # installs everything into that virtual environment.
+  shell: cd {{app_base_dir}}/mitx && {{venv_dir}}/bin/pip install --use-mirrors -r {{app_base_dir}}/mitx/requirements.txt
+  tags:
+  - lms
+  - cms

playbooks/roles/lms/vars/main.yml

@@ -43,3 +43,83 @@ env_config:
     'stage-num':  'MITx',  'stage-sjsu':  'MITx'}
   'VIRTUAL_UNIVERSITIES': []
   'WIKI_ENABLED': true
+
+lms_source_repo: git@github.com:MITx/mitx.git
+lms_debian_pkgs:
+  - apparmor-utils
+  - aspell
+  - build-essential
+  - curl
+  - dvipng
+  - fabric
+  - facter
+  - g++
+  - gcc
+  - gfortran
+  - ghostscript
+  - git
+  - github-cli
+  - graphviz
+  - graphviz-dev
+  - gunicorn
+  - inoticoming
+  - ipython
+  - libcrypt-ssleay-perl
+  - libcurl4-openssl-dev
+  - libdigest-sha-perl
+  - libfreetype6-dev
+  - libgeos-dev
+  - libgraphviz-dev
+  - libjpeg8-dev
+  - liblapack-dev
+  - liblwp-protocol-https-perl
+  - libmysqlclient-dev
+  - libnet-amazon-ec2-perl
+  - libpng12-dev
+  - libreadline-dev
+  - libreadline6-dev
+  - libssl-dev
+  - libswitch-perl
+  - libwww-perl
+  - libxml++2.6-dev
+  - libxml2-dev
+  - libxml2-utils
+  - libxslt1-dev
+  - maven2
+  - mongodb
+  - mongodb-clients
+  - mysql-client
+  - nodejs
+  - ntp
+  - openjdk-7-jdk
+  - openjdk-7-jre
+  - pep8
+  - perl
+  - pkg-config
+  - postfix
+  - puppet
+  - puppet-common
+  - puppet-lint
+  - puppetmaster
+  - puppetmaster-common
+  - pylint
+  - python-boto
+  - python-coverage-test-runner
+  - python-django-nose
+  - python-jenkins
+  - python-nose
+  - python-nosexcover
+  - python-numpy
+  - python-pip
+  - python-scipy
+  - rake
+  - reprepro
+  - rsyslog
+  - rubygems
+  - sqlite3
+  - super
+  - vagrant
+  - vim-puppet
+  - yui-compressor
+  - zip
+  - zlib1g-dev

playbooks/roles/nginx/tasks/nginx_site.yml

@@ -6,6 +6,7 @@
   notify: restart nginx
   tags:
   - nginx
+  - lms
 
 - name: Creating nginx config link {{ site_name }}
   sudo: True
@@ -13,3 +14,4 @@
   notify: restart nginx
   tags:
   - nginx
+  - lms

playbooks/roles/ruby/tasks/main.yml

 #
 #cribbed from https://github.com/mmoya/ansible-playbooks/blob/master/rbenv/main.yml
+- name: Create 'www' user (replicating historical environment)
+  user: name=www state=present
+  sudo: True
 - name: Create ruby base
   sudo: True
   file: path=$ruby_base state=directory owner=www group=www
@@ -73,9 +76,9 @@
   sudo: true
   when_failed: $ruby_installed
 
-- name: gem | gem install bundle
+- name: gem | gem install bundler
   shell: RBENV_ROOT=${rbenv_root} GEM_HOME=${gem_home} ${rbenv_root}/shims/gem install bundle chdir=${app_base_dir}/mitx
   sudo: true
 
 - name: bundle | bundle install
-  shell: RBENV_ROOT=${rbenv_root} GEM_HOME=${gem_home} ${gem_home}/bin/bundle install chdir=${app_base_dir}/mitx
+  shell: RBENV_ROOT=${rbenv_root} GEM_HOME=${gem_home} ${gem_home}/bin/bundle install --deployment --binstubs chdir=${app_base_dir}/mitx

playbooks/roles/ruby/templates/rbenv.sh.j2

 export RBENV_ROOT="{{ rbenv_root }}"
+export GEM_HOME="{{ gem_home }}"
 eval "$(rbenv init -)"