Merge branch 'master' of github.com:edx/configuration into feature/sef/refarch-vpc-dns-ssh

README.md

@@ -170,10 +170,47 @@ If that works fine, then you can add an export of PYTHONPATH to
 * Creates base directories
 * Creates the lms json configuration files
 
+Because the reference architecture makes use of an Amazon VPC, you will not be able
+to address the hosts in the private subnets directly.  However, you can easily set 
+up a transparent "jumpbox" so that for all hosts in your vpc, connections are 
+tunneled
+
+Add something like the following to your `~/.ssh/config` file.
+
+```
+Host vpc-00000000-jumpbox 
+  HostName 54.236.224.226
+  IdentityFile /path/to/aws/key.pem
+  ForwardAgent yes
+  User ubuntu
+
+Host 10.0.10.1
+  ProxyCommand ssh -W %h:%p vpc-00000000-jumpbox
+  ForwardAgent yes
+  HostName 10.0.10.242
+  IdentityFile /path/to/aws/key.pem
+  User ubuntu
+  
+  ...
+
+```
+You will need a host entry for each server that ansible will configure.
+
+Test this by typing `ssh 10.0.10.1`, use a hostname the exists in
+your environment.   If things are configured correctly you will ssh 
+to 10.0.10.1, jumping transparently via your basion host.
+
+Assuming that the edxapp_stage.yml playbook targets hosts in your vpc
+for which there are entiries in your `.ssh/config`, do the 
+following to run your playbook.
+
 ```
   cd playbooks
-  ansible-playbook -v --user=ubuntu edxapp_stage.yml -i ./ec2.py --private-key=/path/to/aws/key.pem
+  ansible-playbook -v --user=ubuntu edxapp_stage.yml -i ./ec2.py -c ssh
 ```
+We are in the process of writing utility scripts to generate this config,
+but for the moment it is a manual process.
+
 
 *Note: this assumes the group used for the edx stack was "edxapp_stage"*
 

playbooks/edxapp_prod.yml

@@ -7,3 +7,4 @@
     - common
     - nginx
     - lms
+    - ruby

playbooks/edxapp_rolling_example.yml

@@ -21,6 +21,7 @@
     - common
     - nginx 
     - lms
+    - ruby
   post_tasks:
     - local_action: command util/elb_reg.py -e {{ ",".join(elbs[ansible_ec2_instance_id]) }} -i {{ ansible_ec2_instance_id }} register
 # Register will pass in the same elb list and the same instance id 

playbooks/edxapp_stage.yml

@@ -7,3 +7,4 @@
     - common
     - nginx
     - lms
+    - ruby

playbooks/group_vars/all

 ---
 app_base_dir: /opt/wwc
+venv_dir: /opt/edx
 
 #where are the secure files on the deploying machine?
 secure_file_dir: ../../edx-secret/ansible/vars/

playbooks/roles/common/tasks/create_venv.yml

+---
+# create the 'edx' virtual environment in /opt so that roles can populate it
+- name: easy_install pip
+  easy_install: name=pip
+  sudo: True
+  tags:
+  - venv_base
+- name: pip install virtualenv
+  pip: name=virtualenv state=latest
+  sudo: True
+  tags:
+  - venv_base
+- name: pip install virtualenvwrapper
+  pip: name=virtualenvwrapper state=latest
+  sudo: True
+  tags:
+  - venv_base
+- name: create edx virtualenv directory
+  file: path=$venv_dir owner=ubuntu group=edx mode=2775 state=directory
+  sudo: True
+  tags:
+  - venv_base
+- name: create the edx virtualenv directory initial contents
+  command: /usr/local/bin/virtualenv $venv_dir creates=$venv_dir/bin/activate
+  tags:
+  - venv_base

playbooks/roles/common/tasks/main.yml

 ---
 - include: create_users.yml
 - name: Create application root
-  sudo: True
   file: path=$app_base_dir state=directory owner=root group=root
-- name: Create log directory
   sudo: True
+- name: Create log directory
   file: path=/mnt/logs state=directory
-- name: Update apt cache
   sudo: True
+- name: Update apt cache
   apt: update_cache=yes
+  sudo: True
+
+- include: create_venv.yml
+
+- name: Install role-independent useful system packages
+  apt: pkg={{item}} install_recommends=yes state=present
+  with_items:
+  - mosh
+  - tmux
+  sudo: True
+

playbooks/roles/lms/tasks/main.yml

 # requires:
+#  - group_vars/all
 #  - common/tasks/main.yml
 #  - nginx/tasks/main.yml
 ---
 - name: create lms application config
   template: src=env.json.j2 dest=$app_base_dir/lms.env.json
   sudo: True
+  tags:
+  - lms
 
 - name: create lms auth file
   template: src=auth.json.j2 dest=$app_base_dir/lms.auth.json
   sudo: True
+  tags:
+  - lms
 
 - include: ../../nginx/tasks/nginx_site.yml state=link site_name=lms
 - include: ../../nginx/tasks/nginx_site.yml state=link site_name=lms-backend
@@ -17,18 +22,90 @@
 # Temprory behavior, not needed after June 1. Perhaps still useful as a recipe.
 - name: install read-only ssh key for mitx repo (private)
   copy: src={{ secure_file_dir }}/ssh_deploy_private dest=/home/ubuntu/.ssh/id_rsa force=yes owner=ubuntu group=ubuntu mode=600
+  tags:
+  - lms
+  - cms
 - name: install read-only ssh key for mitx repo (public)
   copy: src={{ secure_file_dir }}/ssh_deploy_public dest=/home/ubuntu/.ssh/id_rsa.pub force=yes owner=ubuntu group=ubuntu mode=644
+  tags:
+  - lms
+  - cms
 - name: install read-only ssh key for mitx repo (host github known)
   copy: src={{ secure_file_dir }}/ssh_deploy_known_hosts dest=/home/ubuntu/.ssh/known_hosts force=yes owner=ubuntu group=ubuntu mode=600
+  tags:
+  - lms
+  - cms
 
 # Check out mitx repo to $app_base_dir
 - name: set permissions on $app_base_dir sgid for edx
   file: path=$app_base_dir owner=root group=edx mode=2775 state=directory
   file: path=$app_base_dir owner=ubuntu group=edx mode=2775 state=directory
   sudo: True
+  tags:
+  - lms
+  - cms
 - name: install git and its recommends
-  apt: pkg=git state=installed install_recommends=yes 
+  apt: pkg=git state=present install_recommends=yes 
   sudo: True
+  tags:
+  - lms
+  - cms
 - name: git checkout mitx repo into $app_base_dir
-  git: dest=$app_base_dir/mitx repo=git@github.com:MITx/mitx.git
+  git: dest={{app_base_dir}}/mitx repo={{lms_source_repo}}
+  tags:
+  - lms
+  - cms
+
+## Install the debian package requirements system-wide
+- name: store remote apt_repos list for ansible use
+  command: cat {{app_base_dir}}/mitx/apt-repos.txt
+  register: apt_repos_list
+  tags:
+  - lms
+  - cms
+- name: add apt_repos to the remote hosts
+  apt_repository: repo="$item"
+  with_items: "{{apt_repos_list.stdout.split()}}"
+  register: apt_repos_list_repo_adds
+  sudo: True
+  tags:
+  - lms
+  - cms
+- name: update apt cache (if necessary)
+  apt: update_cache=yes
+  sudo: True
+  only_if: "{{apt_repos_list_repo_adds.changed}}"
+  tags:
+  - lms
+  - cms
+- name: store remote apt_packages list for ansible use
+  command: cat {{app_base_dir}}/mitx/apt-packages.txt
+  register: apt_packages_list
+  tags:
+  - lms
+  - cms
+- name: install a bunch of system packages on which LMS and CMS rely
+  apt: pkg={{item}} state=present
+  with_items: lms_debian_pkgs
+  sudo: True
+  tags:
+  - lms
+  - cms
+
+# Install the python requirements into $venv_dir 
+- name : install python pre-requirements
+  pip: requirements="{{app_base_dir}}/mitx/pre-requirements.txt" virtualenv="{{venv_dir}}" state=present
+  tags:
+  - lms
+  - cms
+
+# Install the python modules into $venv_dir 
+- name : install python pre-requirements
+  #pip: requirements="{{app_base_dir}}/mitx/requirements.txt" virtualenv="{{venv_dir}}" 
+  # Need to use shell rather than pip so that we can maintain the context of our current working directory; some 
+  # requirements are pathed relative to the mitx repo. Using the pip from inside the virtual environment implicitly
+  # installs everything into that virtual environment.
+  shell: cd {{app_base_dir}}/mitx && {{venv_dir}}/bin/pip install --use-mirrors -r {{app_base_dir}}/mitx/requirements.txt
+  tags:
+  - lms
+  - cms

playbooks/roles/lms/vars/main.yml

@@ -43,3 +43,83 @@ env_config:
     'stage-num':  'MITx',  'stage-sjsu':  'MITx'}
   'VIRTUAL_UNIVERSITIES': []
   'WIKI_ENABLED': true
+
+lms_source_repo: git@github.com:MITx/mitx.git
+lms_debian_pkgs:
+  - apparmor-utils
+  - aspell
+  - build-essential
+  - curl
+  - dvipng
+  - fabric
+  - facter
+  - g++
+  - gcc
+  - gfortran
+  - ghostscript
+  - git
+  - github-cli
+  - graphviz
+  - graphviz-dev
+  - gunicorn
+  - inoticoming
+  - ipython
+  - libcrypt-ssleay-perl
+  - libcurl4-openssl-dev
+  - libdigest-sha-perl
+  - libfreetype6-dev
+  - libgeos-dev
+  - libgraphviz-dev
+  - libjpeg8-dev
+  - liblapack-dev
+  - liblwp-protocol-https-perl
+  - libmysqlclient-dev
+  - libnet-amazon-ec2-perl
+  - libpng12-dev
+  - libreadline-dev
+  - libreadline6-dev
+  - libssl-dev
+  - libswitch-perl
+  - libwww-perl
+  - libxml++2.6-dev
+  - libxml2-dev
+  - libxml2-utils
+  - libxslt1-dev
+  - maven2
+  - mongodb
+  - mongodb-clients
+  - mysql-client
+  - nodejs
+  - ntp
+  - openjdk-7-jdk
+  - openjdk-7-jre
+  - pep8
+  - perl
+  - pkg-config
+  - postfix
+  - puppet
+  - puppet-common
+  - puppet-lint
+  - puppetmaster
+  - puppetmaster-common
+  - pylint
+  - python-boto
+  - python-coverage-test-runner
+  - python-django-nose
+  - python-jenkins
+  - python-nose
+  - python-nosexcover
+  - python-numpy
+  - python-pip
+  - python-scipy
+  - rake
+  - reprepro
+  - rsyslog
+  - rubygems
+  - sqlite3
+  - super
+  - vagrant
+  - vim-puppet
+  - yui-compressor
+  - zip
+  - zlib1g-dev

playbooks/roles/nginx/tasks/nginx_site.yml

@@ -6,6 +6,7 @@
   notify: restart nginx
   tags:
   - nginx
+  - lms
 
 - name: Creating nginx config link {{ site_name }}
   sudo: True
@@ -13,3 +14,4 @@
   notify: restart nginx
   tags:
   - nginx
+  - lms

playbooks/roles/ruby/tasks/main.yml

+#
+#cribbed from https://github.com/mmoya/ansible-playbooks/blob/master/rbenv/main.yml
+- name: Create 'www' user (replicating historical environment)
+  user: name=www state=present
+  sudo: True
+- name: Create ruby base
+  sudo: True
+  file: path=$ruby_base state=directory owner=www group=www
+- name: rbenv | install build depends
+  sudo: true
+  apt: pkg=$item state=present install_recommends=no
+  with_items:
+    - build-essential
+    - git
+    - libcurl4-openssl-dev
+    - libmysqlclient-dev
+    - libreadline-dev
+    - libssl-dev
+    - libxml2-dev
+    - libxslt1-dev
+    - zlib1g-dev
+
+- name: rbenv | update rbenv repo
+  sudo: true
+  git: repo=git://github.com/sstephenson/rbenv.git dest=$rbenv_root version=v0.4.0
+
+- name: rbenv | add rbenv to path
+  sudo: true
+  file: path=/usr/local/bin/rbenv src=${rbenv_root}/bin/rbenv state=link
+
+- name: rbenv | add rbenv initialization to profile
+  sudo: true
+  template: src=rbenv.sh.j2 dest=/etc/profile.d/rbenv.sh owner=root group=root mode=0755
+
+- name: rbenv | check ruby-build installed
+  sudo: true
+  command: test -x /usr/local/bin/ruby-build
+  register: rbuild_present
+  ignore_errors: yes
+
+- name: rbenv | create temporary directory
+  command: mktemp -d
+  register: tempdir
+  when_failed: $rbuild_present
+
+- name: rbenv | clone ruby-build repo
+  git: repo=git://github.com/sstephenson/ruby-build.git dest=${tempdir.stdout}/ruby-build
+  when_failed: $rbuild_present
+
+- name: rbenv | install ruby-build
+  sudo: true
+  command: ./install.sh chdir=${tempdir.stdout}/ruby-build
+  when_failed: $rbuild_present
+
+- name: rbenv | remove temporary directory
+  file: path=${tempdir.stdout} state=absent
+  when_failed: $rbuild_present
+
+- name: rbenv | check ruby $ruby_version installed
+  shell: RBENV_ROOT=${rbenv_root} rbenv versions | grep $ruby_version
+  register: ruby_installed
+  ignore_errors: yes
+
+- name: rbenv | install ruby $ruby_version
+  shell: RBENV_ROOT=${rbenv_root} rbenv install $ruby_version
+  sudo: true
+  when_failed: $ruby_installed
+
+- name: rbenv | set global ruby $ruby_version
+  shell: RBENV_ROOT=${rbenv_root} rbenv global $ruby_version
+  sudo: true
+  when_failed: $ruby_installed
+
+- name: rbenv | rehash
+  shell: RBENV_ROOT=${rbenv_root} rbenv rehash
+  sudo: true
+  when_failed: $ruby_installed
+
+- name: gem | gem install bundler
+  shell: RBENV_ROOT=${rbenv_root} GEM_HOME=${gem_home} ${rbenv_root}/shims/gem install bundle chdir=${app_base_dir}/mitx
+  sudo: true
+
+- name: bundle | bundle install
+  shell: RBENV_ROOT=${rbenv_root} GEM_HOME=${gem_home} ${gem_home}/bin/bundle install --deployment --binstubs chdir=${app_base_dir}/mitx

playbooks/roles/ruby/templates/rbenv.sh.j2

+export RBENV_ROOT="{{ rbenv_root }}"
+export GEM_HOME="{{ gem_home }}"
+eval "$(rbenv init -)"

playbooks/roles/ruby/vars/main.yml

+---
+ruby_base: /opt/www
+rbenv_root: "{{ ruby_base }}/.rbenv"
+ruby_version: "1.9.3-p374"
+gem_home: "{{ ruby_base }}/.gem"