Merge pull request #26 from edx/feature/jrbl/apt_pip_installs

Install debian and python dependencies for LMS/CMS

playbooks/group_vars/all

 ---
 app_base_dir: /opt/wwc
+venv_dir: /opt/edx
 
 #where are the secure files on the deploying machine?
 secure_file_dir: ../../edx-secret/ansible/vars/

playbooks/roles/common/tasks/create_venv.yml

+---
+# create the 'edx' virtual environment in /opt so that roles can populate it
+- name: easy_install pip
+  easy_install: name=pip
+  sudo: True
+  tags:
+  - venv_base
+- name: pip install virtualenv
+  pip: name=virtualenv state=latest
+  sudo: True
+  tags:
+  - venv_base
+- name: pip install virtualenvwrapper
+  pip: name=virtualenvwrapper state=latest
+  sudo: True
+  tags:
+  - venv_base
+- name: create edx virtualenv directory
+  file: path=$venv_dir owner=ubuntu group=edx mode=2775 state=directory
+  sudo: True
+  tags:
+  - venv_base
+- name: create the edx virtualenv directory initial contents
+  command: /usr/local/bin/virtualenv $venv_dir creates=$venv_dir/bin/activate
+  tags:
+  - venv_base

playbooks/roles/common/tasks/main.yml

 ---
 - include: create_users.yml
 - name: Create application root
-  sudo: True
   file: path=$app_base_dir state=directory owner=root group=root
-- name: Create log directory
   sudo: True
+- name: Create log directory
   file: path=/mnt/logs state=directory
-- name: Update apt cache
   sudo: True
+- name: Update apt cache
   apt: update_cache=yes
+  sudo: True
+
+- include: create_venv.yml
+
+- name: Install role-independent useful system packages
+  apt: pkg={{item}} install_recommends=yes state=present
+  with_items:
+  - mosh
+  - tmux
+  sudo: True
+

playbooks/roles/lms/tasks/main.yml

 # requires:
+#  - group_vars/all
 #  - common/tasks/main.yml
 #  - nginx/tasks/main.yml
 ---
 - name: create lms application config
   template: src=env.json.j2 dest=$app_base_dir/lms.env.json
   sudo: True
+  tags:
+  - lms
 
 - name: create lms auth file
   template: src=auth.json.j2 dest=$app_base_dir/lms.auth.json
   sudo: True
+  tags:
+  - lms
 
 - include: ../../nginx/tasks/nginx_site.yml state=link site_name=lms
 - include: ../../nginx/tasks/nginx_site.yml state=link site_name=lms-backend
@@ -17,18 +22,90 @@
 # Temprory behavior, not needed after June 1. Perhaps still useful as a recipe.
 - name: install read-only ssh key for mitx repo (private)
   copy: src={{ secure_file_dir }}/ssh_deploy_private dest=/home/ubuntu/.ssh/id_rsa force=yes owner=ubuntu group=ubuntu mode=600
+  tags:
+  - lms
+  - cms
 - name: install read-only ssh key for mitx repo (public)
   copy: src={{ secure_file_dir }}/ssh_deploy_public dest=/home/ubuntu/.ssh/id_rsa.pub force=yes owner=ubuntu group=ubuntu mode=644
+  tags:
+  - lms
+  - cms
 - name: install read-only ssh key for mitx repo (host github known)
   copy: src={{ secure_file_dir }}/ssh_deploy_known_hosts dest=/home/ubuntu/.ssh/known_hosts force=yes owner=ubuntu group=ubuntu mode=600
+  tags:
+  - lms
+  - cms
 
 # Check out mitx repo to $app_base_dir
 - name: set permissions on $app_base_dir sgid for edx
   file: path=$app_base_dir owner=root group=edx mode=2775 state=directory
   file: path=$app_base_dir owner=ubuntu group=edx mode=2775 state=directory
   sudo: True
+  tags:
+  - lms
+  - cms
 - name: install git and its recommends
-  apt: pkg=git state=installed install_recommends=yes 
+  apt: pkg=git state=present install_recommends=yes 
   sudo: True
+  tags:
+  - lms
+  - cms
 - name: git checkout mitx repo into $app_base_dir
-  git: dest=$app_base_dir/mitx repo=git@github.com:MITx/mitx.git
+  git: dest={{app_base_dir}}/mitx repo={{lms_source_repo}}
+  tags:
+  - lms
+  - cms
+
+## Install the debian package requirements system-wide
+- name: store remote apt_repos list for ansible use
+  command: cat {{app_base_dir}}/mitx/apt-repos.txt
+  register: apt_repos_list
+  tags:
+  - lms
+  - cms
+- name: add apt_repos to the remote hosts
+  apt_repository: repo="$item"
+  with_items: "{{apt_repos_list.stdout.split()}}"
+  register: apt_repos_list_repo_adds
+  sudo: True
+  tags:
+  - lms
+  - cms
+- name: update apt cache (if necessary)
+  apt: update_cache=yes
+  sudo: True
+  only_if: "{{apt_repos_list_repo_adds.changed}}"
+  tags:
+  - lms
+  - cms
+- name: store remote apt_packages list for ansible use
+  command: cat {{app_base_dir}}/mitx/apt-packages.txt
+  register: apt_packages_list
+  tags:
+  - lms
+  - cms
+- name: install a bunch of system packages on which LMS and CMS rely
+  apt: pkg={{item}} state=present
+  with_items: lms_debian_pkgs
+  sudo: True
+  tags:
+  - lms
+  - cms
+
+# Install the python requirements into $venv_dir 
+- name : install python pre-requirements
+  pip: requirements="{{app_base_dir}}/mitx/pre-requirements.txt" virtualenv="{{venv_dir}}" state=present
+  tags:
+  - lms
+  - cms
+
+# Install the python modules into $venv_dir 
+- name : install python pre-requirements
+  #pip: requirements="{{app_base_dir}}/mitx/requirements.txt" virtualenv="{{venv_dir}}" 
+  # Need to use shell rather than pip so that we can maintain the context of our current working directory; some 
+  # requirements are pathed relative to the mitx repo. Using the pip from inside the virtual environment implicitly
+  # installs everything into that virtual environment.
+  shell: cd {{app_base_dir}}/mitx && {{venv_dir}}/bin/pip install --use-mirrors -r {{app_base_dir}}/mitx/requirements.txt
+  tags:
+  - lms
+  - cms

playbooks/roles/lms/vars/main.yml

@@ -43,3 +43,83 @@ env_config:
     'stage-num':  'MITx',  'stage-sjsu':  'MITx'}
   'VIRTUAL_UNIVERSITIES': []
   'WIKI_ENABLED': true
+
+lms_source_repo: git@github.com:MITx/mitx.git
+lms_debian_pkgs:
+  - apparmor-utils
+  - aspell
+  - build-essential
+  - curl
+  - dvipng
+  - fabric
+  - facter
+  - g++
+  - gcc
+  - gfortran
+  - ghostscript
+  - git
+  - github-cli
+  - graphviz
+  - graphviz-dev
+  - gunicorn
+  - inoticoming
+  - ipython
+  - libcrypt-ssleay-perl
+  - libcurl4-openssl-dev
+  - libdigest-sha-perl
+  - libfreetype6-dev
+  - libgeos-dev
+  - libgraphviz-dev
+  - libjpeg8-dev
+  - liblapack-dev
+  - liblwp-protocol-https-perl
+  - libmysqlclient-dev
+  - libnet-amazon-ec2-perl
+  - libpng12-dev
+  - libreadline-dev
+  - libreadline6-dev
+  - libssl-dev
+  - libswitch-perl
+  - libwww-perl
+  - libxml++2.6-dev
+  - libxml2-dev
+  - libxml2-utils
+  - libxslt1-dev
+  - maven2
+  - mongodb
+  - mongodb-clients
+  - mysql-client
+  - nodejs
+  - ntp
+  - openjdk-7-jdk
+  - openjdk-7-jre
+  - pep8
+  - perl
+  - pkg-config
+  - postfix
+  - puppet
+  - puppet-common
+  - puppet-lint
+  - puppetmaster
+  - puppetmaster-common
+  - pylint
+  - python-boto
+  - python-coverage-test-runner
+  - python-django-nose
+  - python-jenkins
+  - python-nose
+  - python-nosexcover
+  - python-numpy
+  - python-pip
+  - python-scipy
+  - rake
+  - reprepro
+  - rsyslog
+  - rubygems
+  - sqlite3
+  - super
+  - vagrant
+  - vim-puppet
+  - yui-compressor
+  - zip
+  - zlib1g-dev

playbooks/roles/nginx/tasks/nginx_site.yml

@@ -6,6 +6,7 @@
   notify: restart nginx
   tags:
   - nginx
+  - lms
 
 - name: Creating nginx config link {{ site_name }}
   sudo: True
@@ -13,3 +14,4 @@
   notify: restart nginx
   tags:
   - nginx
+  - lms