Merge pull request #427 from edx/will/vagrant-devstack

Will/vagrant devstack

playbooks/jenkins_worker.yml

@@ -10,4 +10,5 @@
     - common
     - edxlocal
     - mongo
+    - browsers
     - jenkins_worker

playbooks/roles/browsers/defaults/main.yml

+browser_deb_pkgs:
+    - xvfb
+    - firefox
+    - dbus-x11
+    - libgconf2-4
+    - libxss1
+    - libnss3-1d
+    - libcurl3
+    - xdg-utils
+    - google-chrome-stable
+
+# Chrome and ChromeDriver
+chrome_apt_key: "https://dl-ssl.google.com/linux/linux_signing_key.pub"
+chrome_apt_repo: "http://dl.google.com/linux/chrome/deb/"
+chromedriver_version: 2.3
+chromedriver_url: "http://chromedriver.storage.googleapis.com/{{ chromedriver_version }}/chromedriver_linux64.zip"
+
+browser_xvfb_display: ":1"

playbooks/roles/browsers/tasks/main.yml

+# Install browsers required to run the JavaScript
+# and acceptance test suite locally without a display
+---
+- name: browsers | Google Chrome apt key
+  apt_key: url={{ chrome_apt_key }} state=present
+
+- name: browsers | Google Chrome apt repo
+  apt_repository: repo='deb {{ chrome_apt_repo }} stable main'
+
+- name: browsers | install system packages
+  apt: pkg={{','.join(browser_deb_pkgs)}}
+       state=present update_cache=yes
+
+- name: browsers | Install ChromeDriver
+  get_url:
+    url={{ chromedriver_url }}
+    dest=/var/tmp/chromedriver_{{ chromedriver_version }}.zip
+
+- name: browsers | Install ChromeDriver 2
+  shell: unzip /var/tmp/chromedriver_{{ chromedriver_version }}.zip
+         chdir=/var/tmp
+
+- name: browsers | Install ChromeDriver 3
+  shell: mv /var/tmp/chromedriver /usr/local/bin/chromedriver
+
+- name: browsers | Install Chromedriver 4
+  file: path=/usr/local/bin/chromedriver mode=0755
+
+- name: browsers | create xvfb upstart script
+  template: src=xvfb.conf.j2 dest=/etc/init/xvfb.conf owner=root group=root
+
+- name: browsers | start xvfb
+  shell: start xvfb
+  ignore_errors: yes
+

playbooks/roles/jenkins_worker/templates/xvfb.conf.j2 → playbooks/roles/browsers/templates/xvfb.conf.j2

 description     "Xvfb X Server"
-start on (net-device-up
-          and local-filesystems
-          and runlevel [2345])
+start on (net-device-up and local-filesystems and runlevel [2345])
 stop on runlevel [016]
-exec /usr/bin/Xvfb :1 -screen 0 1024x768x24
+exec /usr/bin/Xvfb {{ browser_xvfb_display }} -screen 0 1024x768x24
 respawn
-respawn limit 15 5
\ No newline at end of file
+respawn limit 15 5

playbooks/roles/edxapp/defaults/main.yml

@@ -385,7 +385,6 @@ service_variants_enabled:
 edxapp_lms_env: 'lms.envs.aws'
 edxapp_cms_env: 'cms.envs.aws'
 
-
 #Number of gunicorn worker processes to spawn, as a multiplier to number of virtual cores
 worker_core_mult:
   lms: 4
@@ -457,3 +456,7 @@ edxapp_cms_variant: cms
 
 # Worker Settings
 worker_django_settings_module: 'aws'
+
+# Skip supervisor tasks
+# Useful when supervisor is not installed (local dev)
+edxapp_skip_supervisor: False

playbooks/roles/edxapp/handlers/main.yml

@@ -5,6 +5,7 @@
     supervisorctl_path={{ supervisor_ctl }}
     config={{ supervisor_cfg }}
     name="edxapp:{{ item }}"
+  when: not edxapp_skip_supervisor
   sudo_user: "{{ supervisor_service_user }}"
   with_items: service_variants_enabled
   tags: deploy
@@ -15,7 +16,7 @@
     supervisorctl_path={{ supervisor_ctl }}
     config={{ supervisor_cfg }}
     state=restarted
-  when: celery_worker is defined
+  when: celery_worker is defined and not edxapp_skip_supervisor
   with_items: edxapp_workers
   sudo_user: "{{ common_web_user }}"
   tags: deploy

playbooks/roles/edxapp/tasks/deploy.yml

@@ -227,6 +227,7 @@
   register: supervisor_update
   sudo_user: "{{ supervisor_service_user }}"
   changed_when: supervisor_update.stdout != ""
+  when: not edxapp_skip_supervisor
   tags: deploy
 
 - name: edxapp | ensure edxapp has started
@@ -236,6 +237,7 @@
     config={{ supervisor_cfg }}
     name="edxapp:{{ item }}"
   sudo_user: "{{ supervisor_service_user }}"
+  when: not edxapp_skip_supervisor
   with_items: service_variants_enabled
   tags: deploy
 
@@ -245,7 +247,7 @@
     supervisorctl_path={{ supervisor_ctl }}
     config={{ supervisor_cfg }}
     state=started
-  when: celery_worker is defined
+  when: celery_worker is defined and not edxapp_skip_supervisor
   with_items: edxapp_workers
   sudo_user: "{{ supervisor_service_user }}"
   tags: deploy

playbooks/roles/edxapp/tasks/service_variant_config.yml

@@ -29,7 +29,7 @@
     src={{ item }}.conf.j2 dest={{ supervisor_cfg_dir }}/{{ item }}.conf
     owner={{ supervisor_user }}
   with_items: service_variants_enabled
-  when: celery_worker is not defined
+  when: celery_worker is not defined and not edxapp_skip_supervisor
   sudo_user: "{{ supervisor_user }}"
   tags:
   - deploy
@@ -38,7 +38,7 @@
   template: >
     src=edxapp.conf.j2 dest={{ supervisor_cfg_dir }}/edxapp.conf
     owner={{ supervisor_user }}
-  when: celery_worker is not defined
+  when: celery_worker is not defined and not edxapp_skip_supervisor
   sudo_user: "{{ supervisor_user }}"
   tags:
   - deploy
@@ -49,7 +49,7 @@
   template: >
     src=workers.conf.j2 dest={{ supervisor_cfg_dir }}/workers.conf
     owner={{ supervisor_user }}
-  when: celery_worker is defined
+  when: celery_worker is defined and not edxapp_skip_supervisor
   sudo_user: "{{ supervisor_user }}"
   tags:
   - deploy

playbooks/roles/forum/defaults/main.yml

@@ -57,3 +57,7 @@ forum_services:
   - {service: "mongo", host: "{{ FORUM_MONGO_HOST }}", port: "28017"}
   - {service: "elasticsearch", host: "{{ forum_elasticsearch_host }}", port: "9200"}
   - {service: "elasticsearch", host: "{{ forum_elasticsearch_host }}", port: "9300"}
+
+# Skip supervisor tasks
+# Used in local dev where supervisor isn't installed
+forum_skip_supervisor: False

playbooks/roles/forum/handlers/main.yml

@@ -5,4 +5,5 @@
     supervisorctl_path={{ supervisor_ctl }}
     config={{ supervisor_cfg }}
     state=restarted
+  when: not forum_skip_supervisor
   tags: deploy

playbooks/roles/forum/tasks/deploy.yml

@@ -6,6 +6,7 @@
     owner={{ supervisor_user }}
     mode=0644
   sudo_user: "{{ supervisor_user }}"
+  when: not forum_skip_supervisor
   register: forum_supervisor
   tags: deploy
 
@@ -15,6 +16,7 @@
     dest={{ forum_supervisor_wrapper }}
     mode=0755
   sudo_user: "{{ forum_user }}"
+  when: not forum_skip_supervisor
   notify: forum | restart the forum service
   tags: deploy
 
@@ -43,6 +45,7 @@
   shell:  "{{ supervisor_ctl }} -c {{ supervisor_cfg }} update"
   register: supervisor_update
   changed_when: supervisor_update.stdout != ""
+  when: not forum_skip_supervisor
   tags: deploy
 
 - name: forum | ensure forum is started
@@ -51,4 +54,5 @@
     supervisorctl_path={{ supervisor_ctl }}
     config={{ supervisor_cfg }}
     state=started
+  when: not forum_skip_supervisor
   tags: deploy

playbooks/roles/forum/tasks/test.yml

@@ -3,6 +3,7 @@
 - name: forum | test that the required service are listening
   wait_for: port={{ item.port }} host={{ item.host }} timeout=30
   with_items: "{{ forum_services }}"
+  when: not forum_skip_supervisor
   tags:
   - forum
   - test

playbooks/roles/jenkins_worker/defaults/main.yml

@@ -18,12 +18,6 @@ jenkins_debian_pkgs:
     - npm
     - pkg-config
 
-# Chrome and ChromeDriver
-chrome_apt_key: "https://dl-ssl.google.com/linux/linux_signing_key.pub"
-chrome_apt_repo: "http://dl.google.com/linux/chrome/deb/"
-chromedriver_version: 2.3
-chromedriver_url: "http://chromedriver.storage.googleapis.com/{{ chromedriver_version }}/chromedriver_linux64.zip"
-
 # Ruby Specific Vars
 jenkins_rbenv_root: "{{ jenkins_home }}/.rbenv"
 jenkins_ruby_version: "1.9.3-p374"

playbooks/roles/jenkins_worker/tasks/browsers.yml

----
-- name: jenkins_worker | install xvfb
-  apt: pkg=xvfb state=present
-
-- name: jenkins_worker | create xvfb upstart script
-  template: src=xvfb.conf.j2 dest=/etc/init/xvfb.conf owner=root group=root
-
-- name: jenkins_worker | start xvfb
-  shell: start xvfb
-  ignore_errors: yes
-
-- name: jenkins_worker | Install Firefox
-  apt: pkg=firefox
-
-- name: jenkins_worker | Install dbus-x11 (FF requirement)
-  apt: pkg=dbus-x11
-
-- name: jenkins_worker | Install Chrome dependencies
-  apt: pkg=libgconf2-4,libxss1,libnss3-1d,libcurl3,xdg-utils
-
-- name: jenkins_worker | Google Chrome apt key
-  apt_key: url={{ chrome_apt_key }} state=present
-
-- name: jenkins_worker | Google Chrome apt repo
-  apt_repository: repo='deb {{ chrome_apt_repo }} stable main'
-
-- name: jenkins_worker | Install Google Chrome
-  apt: pkg=google-chrome-stable state=present update_cache=yes
-
-- name: jenkins_worker | Install ChromeDriver
-  get_url: url={{ chromedriver_url }}
-           dest=/var/tmp/chromedriver_{{ chromedriver_version }}.zip
-
-- name: jenkins_worker | Install ChromeDriver 2
-  shell: unzip /var/tmp/chromedriver_{{ chromedriver_version }}.zip
-         chdir=/var/tmp
-
-- name: jenkins_worker | Install ChromeDriver 3
-  shell: mv /var/tmp/chromedriver /usr/local/bin/chromedriver
-
-- name: jenkins_worker | Install Chromedriver 4
-  file: path=/usr/local/bin/chromedriver mode=0755

playbooks/roles/local_dev/defaults/main.yml

+---
+
+localdev_accounts:
+    - { user: "{{ edxapp_user}}", home: "{{ edxapp_app_dir }}" }
+    - { user: "{{ forum_user }}", home: "{{ forum_app_dir }}" }
+
+localdev_env:
+    DISPLAY: "{{ browser_xvfb_display }}"
+
+# Helpful system packages for local dev
+local_dev_pkgs:
+    - vim
+    - emacs

playbooks/roles/local_dev/files/gitconfig

+[branch]
+    autosetupmerge = true
+    autosetuprebase = always
+[push]
+    default = current

playbooks/roles/local_dev/tasks/main.yml

+---
+- name: local_dev | install useful system packages
+  apt:
+    pkg={{','.join(local_dev_pkgs)}} install_recommends=yes
+    state=present update_cache=yes
+
+- name: local_dev | set login shell for app accounts
+  user: name={{ item.user }} shell="/bin/bash"
+  with_items: "{{ localdev_accounts }}"
+
+# Ensure forum user has permissions to access .gem and .rbenv
+# This is a little twisty: the forum role sets the owner and group to www-data
+# So we add the forum user to the www-data group and give group write permissions
+- name: local_dev | add forum user to www-data group
+  user: name={{ forum_user }} groups={{ common_web_group }} append=yes
+
+- name: local_dev | set forum rbenv and gem permissions
+  file:
+    path={{ item }} state=directory mode=770
+  with_items:
+    - "{{ forum_app_dir }}/.gem"
+    - "{{ forum_app_dir }}/.rbenv"
+
+# Create scripts to configure environment
+- name: local_dev | create login scripts
+  template:
+    src={{ item.user }}_bashrc.j2 dest={{ item.home }}/.bashrc
+    owner={{ item.user }} mode=755
+  with_items: "{{ localdev_accounts }}"
+
+# Default to the correct git config
+# No more accidentally force pushing to master! :)
+- name: local_dev | configure git
+  copy:
+    src=gitconfig dest={{ item.home }}/.gitconfig
+    owner={{ item.user }} mode=700
+  with_items: "{{ localdev_accounts }}"

playbooks/roles/local_dev/templates/edxapp_bashrc.j2

+#! /usr/bin/env bash
+
+# {{ ansible_managed }}
+
+source "{{ edxapp_app_dir }}/edxapp_env"
+export DISPLAY="{{ browser_xvfb_display }}"
+
+cd $HOME/edx-platform

playbooks/roles/local_dev/templates/forum_bashrc.j2

+#! /usr/bin/env bash
+
+# {{ ansible_managed }}
+
+source "{{ forum_app_dir }}/forum_env"
+export DISPLAY="{{ browser_xvfb_display }}"
+
+cd $HOME/cs_comments_service

playbooks/vagrant-devstack.yml

+- name: Configure instance(s)
+  hosts: vagrant
+  sudo: True
+  gather_facts: True
+  vars:
+    migrate_db: "yes"
+    openid_workaround: True
+    edxapp_skip_supervisor: True
+    forum_skip_supervisor: True
+    edx_platform_commit: 'master'
+  vars_files:
+    - "group_vars/all"
+  roles:
+    - common
+    - edxlocal
+    - mongo
+    - edxapp
+    - oraclejdk
+    - elasticsearch
+    - forum
+    - browsers
+    - local_dev

vagrant/devstack/README.rst

+devstack
+========
+
+Vagrant instance for local development.
+
+
+Overview
+--------
+
+``devstack`` is a Vagrant instance designed for local development.  The instance:
+
+- Uses the same system requirements as production.  This allows developers to discover and fix system configuration issues early in development.
+- Simplifies certain production settings to make development more convenient.  For example, it disables ``nginx`` and ``gunicorn`` in favor of ``runserver`` for Django development.
+
+The ``devstack`` instance is designed to run code and tests, but you can do most development in the host environment:
+
+- Git repositories are shared with the host system, so you can use your preferred text editor/IDE.
+- You can load pages served by the running Vagrant instance.
+
+
+Prerequisites
+-------------
+
+This guide assumes:
+
+- You know how to open a terminal and use basic Linux operating system commands.
+- You understand basic virtualization concepts and are familiar with `Vagrant`__
+- You are familiar with `Django`__ development (for LMS and Studio)
+- You are familiar with `Sinatra`__ development (for Forums)
+
+__ http://docs.vagrantup.com/v2/
+__ http://www.djangoproject.com/
+__ http://www.sinatrarb.com/
+
+
+Installation
+------------
+
+1. `Install Vagrant`__ (version 1.3.4 or higher)
+2. Install the ``vagrant-hostsupdater`` plugin:
+
+.. code:: bash
+
+    vagrant plugin install vagrant-hostsupdater
+
+3. `Install Ansible`__ (version 1.3.2 or higher)
+4. Clone the edX configuration repo:
+
+.. code:: bash
+
+    git clone https://github.com/edx/configuration
+
+5. From the ``configuration/vagrant/devstack`` directory, create the Vagrant instance.  If you have never used Vagrant before, take a moment to read the `Why Vagrant?`__ page, which explains the Vagrant commands in detail.
+
+.. code:: bash
+
+    vagrant up
+
+A new Vagrant instance will be created, with requirements for these repositories installed:
+
+- `edx-platform`__ (LMS and Studio)
+- `cs_comments_service`__
+
+The first time you create the instance, Vagrant will download the base box, which is about 2GB.
+After downloading the base box, Vagrant will automatically provision your new virtual server using the Ansible scripts from this repo.
+If you destroy and recreate the VM, Vagrant will re-use the box it downloaded.
+
+__ http://docs.vagrantup.com/v2/installation/index.html
+__ http://www.ansibleworks.com/docs/intro_installation.html
+__ http://docs.vagrantup.com/v2/why-vagrant/index.html
+__ https://github.com/edx/edx-platform
+__ https://github.com/edx/cs_comments_service
+
+
+Troubleshooting
+---------------
+
+Please refer to the `edx-platform wiki`__ for solutions to common issues.
+
+__ https://github.com/edx/edx-platform/wiki/Simplified-install-with-vagrant#troubleshooting
+
+
+Getting Started
+---------------
+
+In the ``configuration/vagrant/devstack`` directory, `ssh into the Vagrant instance`__:
+
+.. code:: bash
+
+    vagrant ssh
+
+This will log you in as the ``vagrant`` user.  Within the Vagrant instance, you will be able to start services (LMS, Studio, and Forums) and run tests.
+
+- The ``edx-platform`` repo (for LMS and Studio) is cloned to ``/edx/app/edxapp/edx-platform`` and synced with ``devstack/edx-platform`` on the host system.
+- The ``cs_comments_service`` repo (for Forums) is cloned to ``/edx/app/forum/cs_comments_service`` and synced with ``devstack/cs_comments_service`` on the host system.
+
+To start the services and run tests, you will need to log in as either the ``edxapp`` or ``forum`` user.  See below for instructions.
+
+__ http://docs.vagrantup.com/v2/getting-started/up.html
+
+
+LMS Workflow
+------------
+
+1. Within the Vagrant instance, switch to the ``edxapp`` account:
+
+.. code:: bash
+
+    sudo su edxapp
+
+2. Compile Sass and CoffeeScript:
+
+.. code:: bash
+
+    rake assets[lms,devstack]
+
+3. Update Python requirements:
+
+.. code:: bash
+
+    pip install -r requirements/edx/base.txt
+
+4. Update the Ruby requirements:
+
+.. code:: bash
+
+    bundle install
+
+5. Start the LMS using `runserver`__:
+
+.. code:: bash
+
+    ./manage.py lms runserver --settings=devstack 0.0.0.0:8000
+
+6. Open a browser on your host machine and navigate to ``localhost:8000`` to load the LMS.  (Vagrant will forward port 8000 to the LMS server running in the VM.)
+
+__ https://docs.djangoproject.com/en/dev/ref/django-admin/#runserver-port-or-address-port
+
+
+Studio Workflow
+---------------
+
+1. Within the Vagrant instance, switch to the ``edxapp`` account:
+
+.. code:: bash
+
+    sudo su edxapp
+
+2. Compile Sass and CoffeeScript:
+
+.. code:: bash
+
+    rake assets[cms,devstack]
+
+3. Update Python requirements:
+
+.. code:: bash
+
+    pip install -r requirements/edx/base.txt
+
+4. Update the Ruby requirements:
+
+.. code:: bash
+
+    bundle install
+
+5. Start Studio using `runserver`__:
+
+.. code:: bash
+
+    ./manage.py cms runserver --settings=devstack 0.0.0.0:8001
+
+6. Open a browser on your host machine and navigate to ``localhost:8001`` to load Studio.  (Vagrant will forward port 8001 to the Studio server running in the VM.)
+
+
+__ https://docs.djangoproject.com/en/dev/ref/django-admin/#runserver-port-or-address-port
+
+
+Forum Workflow
+--------------
+
+1. Within the Vagrant instance, switch to the ``forum`` account:
+
+.. code:: bash
+
+    sudo su forum
+
+2. Update Ruby requirements:
+
+.. code:: bash
+
+    bundle install
+
+3. Start the server:
+
+.. code:: bash
+
+    ruby app.rb
+
+4. Access the API at ``localhost:4567`` (Vagrant will forward port 4567 to the Forum server running in the VM.)
+
+
+
+Running LMS/Studio Tests
+------------------------
+
+
+1. Within the Vagrant instance, switch to the ``edxapp`` account:
+
+.. code:: bash
+
+    sudo su edxapp
+
+2. Run the Python unit tests:
+
+.. code:: bash
+
+    rake test:python
+
+3. Run the JavaScript unit tests:
+
+.. code:: bash
+
+    rake test:js
+
+4. Run the LMS and Studio acceptance tests:
+
+.. code:: bash
+
+    rake test:acceptance
+
+See `edx-platform testing documentation`__ for detailed information about writing and running tests.
+
+__ https://github.com/edx/edx-platform/blob/master/docs/internal/testing.md
+
+
+
+Enabling Asset Pipeline for LMS/Studio
+--------------------------------------
+
+By default, Django's ``runserver`` serves static assets (CSS and JavaScript) directly from the source files in the ``edx-platform`` repo.  This allows you to make changes and quickly see the results.
+
+In production, however, we process static assets using `django-pipeline`__  To verify that your changes will work in production, you can run LMS and Studio with the pipeline enabled:
+
+1. Switch to the ``edxapp`` account:
+
+.. code:: bash
+
+    sudo su edxapp
+
+2. Gather static assets:
+
+.. code:: bash
+
+    rake lms:gather_assets:devstack
+    rake cms:gather_assets:devstack
+
+3. Start the server with ``--nostatic``:
+
+.. code:: bash
+
+    # To start the LMS
+    ./manage.py lms runserver --settings=devstack --nostatic 0.0.0.0:8000
+
+    # To start Studio
+    ./manage.py cms runserver --settings=devstack --nostatic 0.0.0.0:8001
+
+
+__ http://django-pipeline.readthedocs.org/en/latest/
+
+
+Updating the Environment
+------------------------
+
+If system requirements change, you will need to update the Vagrant instance:
+
+1. Checkout the release branch of the configuration repo:
+
+.. code:: bash
+
+    git checkout release
+    git pull
+
+2. From the ``configuration/vagrant/devstack`` directory, provision the Vagrant instance:
+
+.. code:: bash
+
+    vagrant provision
+
+
+This process will perform a ``git clean`` of the ``edx-platform`` and ``cs_comments_service`` repositories, so make sure that any changes you had are checked in or stashed.
+
+
+Recreating the Environment
+--------------------------
+
+To destroy and recreate the environment:
+
+.. code:: bash
+
+    vagrant destroy
+    vagrant up
+
+This will perform a ``git clean`` of the ``edx-platform`` and ``cs_comments_service`` repositories.  You will also lose any work stored in the Vagrant instance as that instance will be destroyed and a new one created from scratch.

vagrant/devstack/Vagrantfile

+MEMORY = 2048
+CPU_COUNT = 2
+
+Vagrant.configure("2") do |config|
+
+# For efficiency, we create custom base box from an Ubuntu 12.04 image
+# Once the custom base box is created, we use that as a starting point.
+#  config.vm.box     = "precise64"
+#  config.vm.box_url = "http://files.vagrantup.com/precise64.box"
+  config.vm.box    = "edx-devstack"
+  config.vm.box_url = "https://s3.amazonaws.com/vagrant.testeng.edx.org/edx-devstack.box"
+
+  config.vm.network :private_network, ip: "192.168.33.10"
+  config.vm.network :forwarded_port, guest: 8000, host: 8000
+  config.vm.network :forwarded_port, guest: 8001, host: 8001
+  config.vm.network :forwarded_port, guest: 4567, host: 4567
+
+  config.vm.synced_folder "edx-platform", "/edx/app/edxapp/edx-platform", :create => true, nfs: true
+  config.vm.synced_folder "cs_comments_service", "/edx/app/forum/cs_comments_service", :create => true, nfs: true
+
+  config.hostsupdater.aliases = ["preview.localhost"]
+
+  config.vm.provider :virtualbox do |vb|
+    vb.customize ["modifyvm", :id, "--memory", MEMORY.to_s]
+    vb.customize ["modifyvm", :id, "--cpus", CPU_COUNT.to_s]
+  end
+
+  config.vm.provision :ansible do |ansible|
+    ansible.playbook = "../../playbooks/vagrant-devstack.yml"
+    ansible.inventory_path = "../../playbooks/vagrant/inventory.ini"
+    ansible.verbose = "extra"
+  end
+end

vagrant/devstack/ansible.cfg

+../../playbooks/ansible.cfg
\ No newline at end of file