Added Vagrantfile for devstack with folder syncing and port forwarding

Moved headless browser config into a separate role
Added localdev role
Configured login shell for edxapp and forum
Added host update for Studio preview (preview.localhost)
Use custom base box for devstack
Added troubleshooting section to README

playbooks/jenkins_worker.yml

@@ -10,4 +10,5 @@
     - common
     - edxlocal
     - mongo
+    - browsers
     - jenkins_worker

playbooks/roles/browsers/defaults/main.yml

+browser_deb_pkgs:
+    - xvfb
+    - firefox
+    - dbus-x11
+    - libgconf2-4
+    - libxss1
+    - libnss3-1d
+    - libcurl3
+    - xdg-utils
+    - google-chrome-stable
+
+# Chrome and ChromeDriver
+chrome_apt_key: "https://dl-ssl.google.com/linux/linux_signing_key.pub"
+chrome_apt_repo: "http://dl.google.com/linux/chrome/deb/"
+chromedriver_version: 2.3
+chromedriver_url: "http://chromedriver.storage.googleapis.com/{{ chromedriver_version }}/chromedriver_linux64.zip"
+
+browser_xvfb_display: ":1"

playbooks/roles/browsers/tasks/main.yml

+# Install browsers required to run the JavaScript
+# and acceptance test suite locally without a display
+---
+- name: browsers | Google Chrome apt key
+  apt_key: url={{ chrome_apt_key }} state=present
+
+- name: browsers | Google Chrome apt repo
+  apt_repository: repo='deb {{ chrome_apt_repo }} stable main'
+
+- name: browsers | install system packages
+  apt: pkg={{','.join(browser_deb_pkgs)}}
+       state=present update_cache=yes
+
+- name: browsers | Install ChromeDriver
+  get_url:
+    url={{ chromedriver_url }}
+    dest=/var/tmp/chromedriver_{{ chromedriver_version }}.zip
+
+- name: browsers | Install ChromeDriver 2
+  shell: unzip /var/tmp/chromedriver_{{ chromedriver_version }}.zip
+         chdir=/var/tmp
+
+- name: browsers | Install ChromeDriver 3
+  shell: mv /var/tmp/chromedriver /usr/local/bin/chromedriver
+
+- name: browsers | Install Chromedriver 4
+  file: path=/usr/local/bin/chromedriver mode=0755
+
+- name: browsers | create xvfb upstart script
+  template: src=xvfb.conf.j2 dest=/etc/init/xvfb.conf owner=root group=root
+
+- name: browsers | start xvfb
+  shell: start xvfb
+  ignore_errors: yes
+

playbooks/roles/jenkins_worker/templates/xvfb.conf.j2 → playbooks/roles/browsers/templates/xvfb.conf.j2

 description     "Xvfb X Server"
-start on (net-device-up
-          and local-filesystems
-          and runlevel [2345])
+start on (net-device-up and local-filesystems and runlevel [2345])
 stop on runlevel [016]
-exec /usr/bin/Xvfb :1 -screen 0 1024x768x24
+exec /usr/bin/Xvfb {{ browser_xvfb_display }} -screen 0 1024x768x24
 respawn
-respawn limit 15 5
\ No newline at end of file
+respawn limit 15 5

playbooks/roles/edxapp/defaults/main.yml

@@ -385,7 +385,6 @@ service_variants_enabled:
 edxapp_lms_env: 'lms.envs.aws'
 edxapp_cms_env: 'cms.envs.aws'
 
-
 #Number of gunicorn worker processes to spawn, as a multiplier to number of virtual cores
 worker_core_mult:
   lms: 4
@@ -457,3 +456,7 @@ edxapp_cms_variant: cms
 
 # Worker Settings
 worker_django_settings_module: 'aws'
+
+# Skip supervisor tasks
+# Useful when supervisor is not installed (local dev)
+edxapp_skip_supervisor: False

playbooks/roles/edxapp/handlers/main.yml

@@ -5,6 +5,7 @@
     supervisorctl_path={{ supervisor_ctl }}
     config={{ supervisor_cfg }}
     name="edxapp:{{ item }}"
+  when: not edxapp_skip_supervisor
   sudo_user: "{{ supervisor_service_user }}"
   with_items: service_variants_enabled
   tags: deploy
@@ -15,7 +16,7 @@
     supervisorctl_path={{ supervisor_ctl }}
     config={{ supervisor_cfg }}
     state=restarted
-  when: celery_worker is defined
+  when: celery_worker is defined and not edxapp_skip_supervisor
   with_items: edxapp_workers
   sudo_user: "{{ common_web_user }}"
   tags: deploy

playbooks/roles/edxapp/tasks/deploy.yml

@@ -227,6 +227,7 @@
   register: supervisor_update
   sudo_user: "{{ supervisor_service_user }}"
   changed_when: supervisor_update.stdout != ""
+  when: not edxapp_skip_supervisor
   tags: deploy
 
 - name: edxapp | ensure edxapp has started
@@ -236,6 +237,7 @@
     config={{ supervisor_cfg }}
     name="edxapp:{{ item }}"
   sudo_user: "{{ supervisor_service_user }}"
+  when: not edxapp_skip_supervisor
   with_items: service_variants_enabled
   tags: deploy
 
@@ -245,7 +247,7 @@
     supervisorctl_path={{ supervisor_ctl }}
     config={{ supervisor_cfg }}
     state=started
-  when: celery_worker is defined
+  when: celery_worker is defined and not edxapp_skip_supervisor
   with_items: edxapp_workers
   sudo_user: "{{ supervisor_service_user }}"
   tags: deploy

playbooks/roles/edxapp/tasks/service_variant_config.yml

@@ -29,7 +29,7 @@
     src={{ item }}.conf.j2 dest={{ supervisor_cfg_dir }}/{{ item }}.conf
     owner={{ supervisor_user }}
   with_items: service_variants_enabled
-  when: celery_worker is not defined
+  when: celery_worker is not defined and not edxapp_skip_supervisor
   sudo_user: "{{ supervisor_user }}"
   tags:
   - deploy
@@ -38,7 +38,7 @@
   template: >
     src=edxapp.conf.j2 dest={{ supervisor_cfg_dir }}/edxapp.conf
     owner={{ supervisor_user }}
-  when: celery_worker is not defined
+  when: celery_worker is not defined and not edxapp_skip_supervisor
   sudo_user: "{{ supervisor_user }}"
   tags:
   - deploy
@@ -49,7 +49,7 @@
   template: >
     src=workers.conf.j2 dest={{ supervisor_cfg_dir }}/workers.conf
     owner={{ supervisor_user }}
-  when: celery_worker is defined
+  when: celery_worker is defined and not edxapp_skip_supervisor
   sudo_user: "{{ supervisor_user }}"
   tags:
   - deploy

playbooks/roles/forum/defaults/main.yml

@@ -57,3 +57,7 @@ forum_services:
   - {service: "mongo", host: "{{ FORUM_MONGO_HOST }}", port: "28017"}
   - {service: "elasticsearch", host: "{{ forum_elasticsearch_host }}", port: "9200"}
   - {service: "elasticsearch", host: "{{ forum_elasticsearch_host }}", port: "9300"}
+
+# Skip supervisor tasks
+# Used in local dev where supervisor isn't installed
+forum_skip_supervisor: False

playbooks/roles/forum/handlers/main.yml

@@ -5,4 +5,5 @@
     supervisorctl_path={{ supervisor_ctl }}
     config={{ supervisor_cfg }}
     state=restarted
+  when: not forum_skip_supervisor
   tags: deploy

playbooks/roles/forum/tasks/deploy.yml

@@ -6,6 +6,7 @@
     owner={{ supervisor_user }}
     mode=0644
   sudo_user: "{{ supervisor_user }}"
+  when: not forum_skip_supervisor
   register: forum_supervisor
   tags: deploy
 
@@ -15,6 +16,7 @@
     dest={{ forum_supervisor_wrapper }}
     mode=0755
   sudo_user: "{{ forum_user }}"
+  when: not forum_skip_supervisor
   notify: forum | restart the forum service
   tags: deploy
 
@@ -43,6 +45,7 @@
   shell:  "{{ supervisor_ctl }} -c {{ supervisor_cfg }} update"
   register: supervisor_update
   changed_when: supervisor_update.stdout != ""
+  when: not forum_skip_supervisor
   tags: deploy
 
 - name: forum | ensure forum is started
@@ -51,4 +54,5 @@
     supervisorctl_path={{ supervisor_ctl }}
     config={{ supervisor_cfg }}
     state=started
+  when: not forum_skip_supervisor
   tags: deploy

playbooks/roles/forum/tasks/test.yml

@@ -3,6 +3,7 @@
 - name: forum | test that the required service are listening
   wait_for: port={{ item.port }} host={{ item.host }} timeout=30
   with_items: "{{ forum_services }}"
+  when: not forum_skip_supervisor
   tags:
   - forum
   - test

playbooks/roles/jenkins_worker/defaults/main.yml

@@ -18,12 +18,6 @@ jenkins_debian_pkgs:
     - npm
     - pkg-config
 
-# Chrome and ChromeDriver
-chrome_apt_key: "https://dl-ssl.google.com/linux/linux_signing_key.pub"
-chrome_apt_repo: "http://dl.google.com/linux/chrome/deb/"
-chromedriver_version: 2.3
-chromedriver_url: "http://chromedriver.storage.googleapis.com/{{ chromedriver_version }}/chromedriver_linux64.zip"
-
 # Ruby Specific Vars
 jenkins_rbenv_root: "{{ jenkins_home }}/.rbenv"
 jenkins_ruby_version: "1.9.3-p374"

playbooks/roles/jenkins_worker/tasks/browsers.yml

----
-- name: jenkins_worker | install xvfb
-  apt: pkg=xvfb state=present
-
-- name: jenkins_worker | create xvfb upstart script
-  template: src=xvfb.conf.j2 dest=/etc/init/xvfb.conf owner=root group=root
-
-- name: jenkins_worker | start xvfb
-  shell: start xvfb
-  ignore_errors: yes
-
-- name: jenkins_worker | Install Firefox
-  apt: pkg=firefox
-
-- name: jenkins_worker | Install dbus-x11 (FF requirement)
-  apt: pkg=dbus-x11
-
-- name: jenkins_worker | Install Chrome dependencies
-  apt: pkg=libgconf2-4,libxss1,libnss3-1d,libcurl3,xdg-utils
-
-- name: jenkins_worker | Google Chrome apt key
-  apt_key: url={{ chrome_apt_key }} state=present
-
-- name: jenkins_worker | Google Chrome apt repo
-  apt_repository: repo='deb {{ chrome_apt_repo }} stable main'
-
-- name: jenkins_worker | Install Google Chrome
-  apt: pkg=google-chrome-stable state=present update_cache=yes
-
-- name: jenkins_worker | Install ChromeDriver
-  get_url: url={{ chromedriver_url }}
-           dest=/var/tmp/chromedriver_{{ chromedriver_version }}.zip
-
-- name: jenkins_worker | Install ChromeDriver 2
-  shell: unzip /var/tmp/chromedriver_{{ chromedriver_version }}.zip
-         chdir=/var/tmp
-
-- name: jenkins_worker | Install ChromeDriver 3
-  shell: mv /var/tmp/chromedriver /usr/local/bin/chromedriver
-
-- name: jenkins_worker | Install Chromedriver 4
-  file: path=/usr/local/bin/chromedriver mode=0755

playbooks/roles/local_dev/defaults/main.yml

+---
+
+localdev_accounts:
+    - { user: "{{ edxapp_user}}", home: "{{ edxapp_app_dir }}" }
+    - { user: "{{ forum_user }}", home: "{{ forum_app_dir }}" }
+
+localdev_env:
+    DISPLAY: "{{ browser_xvfb_display }}"
+
+# Helpful system packages for local dev
+local_dev_pkgs:
+    - vim
+    - emacs

playbooks/roles/local_dev/files/gitconfig

+[branch]
+    autosetupmerge = true
+    autosetuprebase = always
+[push]
+    default = current

playbooks/roles/local_dev/tasks/main.yml

+---
+- name: local_dev | install useful system packages
+  apt:
+    pkg={{','.join(local_dev_pkgs)}} install_recommends=yes
+    state=present update_cache=yes
+
+- name: local_dev | set login shell for app accounts
+  user: name={{ item.user }} shell="/bin/bash"
+  with_items: "{{ localdev_accounts }}"
+
+# Ensure forum user has permissions to access .gem and .rbenv
+# This is a little twisty: the forum role sets the owner and group to www-data
+# So we add the forum user to the www-data group and give group write permissions
+- name: local_dev | add forum user to www-data group
+  user: name={{ forum_user }} groups={{ common_web_group }} append=yes
+
+- name: local_dev | set forum rbenv and gem permissions
+  file:
+    path={{ item }} state=directory mode=770
+  with_items:
+    - "{{ forum_app_dir }}/.gem"
+    - "{{ forum_app_dir }}/.rbenv"
+
+# Create scripts to configure environment
+- name: local_dev | create login scripts
+  template:
+    src={{ item.user }}_bashrc.j2 dest={{ item.home }}/.bashrc
+    owner={{ item.user }} mode=755
+  with_items: "{{ localdev_accounts }}"
+
+# Default to the correct git config
+# No more accidentally force pushing to master! :)
+- name: local_dev | configure git
+  copy:
+    src=gitconfig dest={{ item.home }}/.gitconfig
+    owner={{ item.user }} mode=700
+  with_items: "{{ localdev_accounts }}"

playbooks/roles/local_dev/templates/edxapp_bashrc.j2

+#! /usr/bin/env bash
+
+# {{ ansible_managed }}
+
+source "{{ edxapp_app_dir }}/edxapp_env"
+export DISPLAY="{{ browser_xvfb_display }}"
+
+cd $HOME/edx-platform

playbooks/roles/local_dev/templates/forum_bashrc.j2

+#! /usr/bin/env bash
+
+# {{ ansible_managed }}
+
+source "{{ forum_app_dir }}/forum_env"
+export DISPLAY="{{ browser_xvfb_display }}"
+
+cd $HOME/cs_comments_service

playbooks/vagrant-devstack.yml

+- name: Configure instance(s)
+  hosts: vagrant
+  sudo: True
+  gather_facts: True
+  vars:
+    migrate_db: "yes"
+    openid_workaround: True
+    edxapp_skip_supervisor: True
+    forum_skip_supervisor: True
+    edx_platform_commit: 'master'
+  vars_files:
+    - "group_vars/all"
+  roles:
+    - common
+    - edxlocal
+    - mongo
+    - edxapp
+    - oraclejdk
+    - elasticsearch
+    - forum
+    - browsers
+    - local_dev

vagrant/devstack/README.rst

+devstack
+========
+
+Vagrant instance for local development.
+
+
+Overview
+--------
+
+``devstack`` is a Vagrant instance designed for local development.  The instance:
+
+- Uses the same system requirements as production.  This allows developers to discover and fix system configuration issues early in development.
+- Simplifies certain production settings to make development more convenient.  For example, it disables ``nginx`` and ``gunicorn`` in favor of ``runserver`` for Django development.
+
+The ``devstack`` instance is designed to run code and tests, but you can do most development in the host environment:
+
+- Git repositories are shared with the host system, so you can use your preferred text editor/IDE.
+- You can load pages served by the running Vagrant instance.
+
+
+Prerequisites
+-------------
+
+This guide assumes:
+
+- You know how to open a terminal and use basic Linux operating system commands.
+- You understand basic virtualization concepts and are familiar with `Vagrant`__
+- You are familiar with `Django`__ development (for LMS and Studio)
+- You are familiar with `Sinatra`__ development (for Forums)
+
+__ http://docs.vagrantup.com/v2/
+__ http://www.djangoproject.com/
+__ http://www.sinatrarb.com/
+
+
+Installation
+------------
+
+1. `Install Vagrant`__ (version 1.3.4 or higher)
+2. Install the ``vagrant-hostsupdater`` plugin:
+
+.. code:: bash
+
+    vagrant plugin install vagrant-hostsupdater
+
+3. `Install Ansible`__ (version 1.3.2 or higher)
+4. Clone the edX configuration repo:
+
+.. code:: bash
+
+    git clone https://github.com/edx/configuration
+
+5. From the ``configuration/vagrant/devstack`` directory, create the Vagrant instance.  If you have never used Vagrant before, take a moment to read the `Why Vagrant?`__ page, which explains the Vagrant commands in detail.
+
+.. code:: bash
+
+    vagrant up
+
+A new Vagrant instance will be created, with requirements for these repositories installed:
+
+- `edx-platform`__ (LMS and Studio)
+- `cs_comments_service`__
+
+The first time you create the instance, Vagrant will download the base box, which is about 2GB.
+After downloading the base box, Vagrant will automatically provision your new virtual server using the Ansible scripts from this repo.
+If you destroy and recreate the VM, Vagrant will re-use the box it downloaded.
+
+__ http://docs.vagrantup.com/v2/installation/index.html
+__ http://www.ansibleworks.com/docs/intro_installation.html
+__ http://docs.vagrantup.com/v2/why-vagrant/index.html
+__ https://github.com/edx/edx-platform
+__ https://github.com/edx/cs_comments_service
+
+
+Troubleshooting
+---------------
+
+Please refer to the `edx-platform wiki`__ for solutions to common issues.
+
+__ https://github.com/edx/edx-platform/wiki/Simplified-install-with-vagrant#troubleshooting
+
+
+Getting Started
+---------------
+
+In the ``configuration/vagrant/devstack`` directory, `ssh into the Vagrant instance`__:
+
+.. code:: bash
+
+    vagrant ssh
+
+This will log you in as the ``vagrant`` user.  Within the Vagrant instance, you will be able to start services (LMS, Studio, and Forums) and run tests.
+
+- The ``edx-platform`` repo (for LMS and Studio) is cloned to ``/edx/app/edxapp/edx-platform`` and synced with ``devstack/edx-platform`` on the host system.
+- The ``cs_comments_service`` repo (for Forums) is cloned to ``/edx/app/forum/cs_comments_service`` and synced with ``devstack/cs_comments_service`` on the host system.
+
+To start the services and run tests, you will need to log in as either the ``edxapp`` or ``forum`` user.  See below for instructions.
+
+__ http://docs.vagrantup.com/v2/getting-started/up.html
+
+
+LMS Workflow
+------------
+
+1. Within the Vagrant instance, switch to the ``edxapp`` account:
+
+.. code:: bash
+
+    sudo su edxapp
+
+2. Compile Sass and CoffeeScript:
+
+.. code:: bash
+
+    rake assets[lms,devstack]
+
+3. Update Python requirements:
+
+.. code:: bash
+
+    pip install -r requirements/edx/base.txt
+
+4. Update the Ruby requirements:
+
+.. code:: bash
+
+    bundle install
+
+5. Start the LMS using `runserver`__:
+
+.. code:: bash
+
+    ./manage.py lms runserver --settings=devstack 0.0.0.0:8000
+
+6. Open a browser on your host machine and navigate to ``localhost:8000`` to load the LMS.  (Vagrant will forward port 8000 to the LMS server running in the VM.)
+
+__ https://docs.djangoproject.com/en/dev/ref/django-admin/#runserver-port-or-address-port
+
+
+Studio Workflow
+---------------
+
+1. Within the Vagrant instance, switch to the ``edxapp`` account:
+
+.. code:: bash
+
+    sudo su edxapp
+
+2. Compile Sass and CoffeeScript:
+
+.. code:: bash
+
+    rake assets[cms,devstack]
+
+3. Update Python requirements:
+
+.. code:: bash
+
+    pip install -r requirements/edx/base.txt
+
+4. Update the Ruby requirements:
+
+.. code:: bash
+
+    bundle install
+
+5. Start Studio using `runserver`__:
+
+.. code:: bash
+
+    ./manage.py cms runserver --settings=devstack 0.0.0.0:8001
+
+6. Open a browser on your host machine and navigate to ``localhost:8001`` to load Studio.  (Vagrant will forward port 8001 to the Studio server running in the VM.)
+
+
+__ https://docs.djangoproject.com/en/dev/ref/django-admin/#runserver-port-or-address-port
+
+
+Forum Workflow
+--------------
+
+1. Within the Vagrant instance, switch to the ``forum`` account:
+
+.. code:: bash
+
+    sudo su forum
+
+2. Update Ruby requirements:
+
+.. code:: bash
+
+    bundle install
+
+3. Start the server:
+
+.. code:: bash
+
+    ruby app.rb
+
+4. Access the API at ``localhost:4567`` (Vagrant will forward port 4567 to the Forum server running in the VM.)
+
+
+
+Running LMS/Studio Tests
+------------------------
+
+
+1. Within the Vagrant instance, switch to the ``edxapp`` account:
+
+.. code:: bash
+
+    sudo su edxapp
+
+2. Run the Python unit tests:
+
+.. code:: bash
+
+    rake test:python
+
+3. Run the JavaScript unit tests:
+
+.. code:: bash
+
+    rake test:js
+
+4. Run the LMS and Studio acceptance tests:
+
+.. code:: bash
+
+    rake test:acceptance
+
+See `edx-platform testing documentation`__ for detailed information about writing and running tests.
+
+__ https://github.com/edx/edx-platform/blob/master/docs/internal/testing.md
+
+
+
+Enabling Asset Pipeline for LMS/Studio
+--------------------------------------
+
+By default, Django's ``runserver`` serves static assets (CSS and JavaScript) directly from the source files in the ``edx-platform`` repo.  This allows you to make changes and quickly see the results.
+
+In production, however, we process static assets using `django-pipeline`__  To verify that your changes will work in production, you can run LMS and Studio with the pipeline enabled:
+
+1. Switch to the ``edxapp`` account:
+
+.. code:: bash
+
+    sudo su edxapp
+
+2. Gather static assets:
+
+.. code:: bash
+
+    rake lms:gather_assets:devstack
+    rake cms:gather_assets:devstack
+
+3. Start the server with ``--nostatic``:
+
+.. code:: bash
+
+    # To start the LMS
+    ./manage.py lms runserver --settings=devstack --nostatic 0.0.0.0:8000
+
+    # To start Studio
+    ./manage.py cms runserver --settings=devstack --nostatic 0.0.0.0:8001
+
+
+__ http://django-pipeline.readthedocs.org/en/latest/
+
+
+Updating the Environment
+------------------------
+
+If system requirements change, you will need to update the Vagrant instance:
+
+1. Checkout the release branch of the configuration repo:
+
+.. code:: bash
+
+    git checkout release
+    git pull
+
+2. From the ``configuration/vagrant/devstack`` directory, provision the Vagrant instance:
+
+.. code:: bash
+
+    vagrant provision
+
+
+This process will perform a ``git clean`` of the ``edx-platform`` and ``cs_comments_service`` repositories, so make sure that any changes you had are checked in or stashed.
+
+
+Recreating the Environment
+--------------------------
+
+To destroy and recreate the environment:
+
+.. code:: bash
+
+    vagrant destroy
+    vagrant up
+
+This will perform a ``git clean`` of the ``edx-platform`` and ``cs_comments_service`` repositories.  You will also lose any work stored in the Vagrant instance as that instance will be destroyed and a new one created from scratch.

vagrant/devstack/Vagrantfile

+MEMORY = 2048
+CPU_COUNT = 2
+
+Vagrant.configure("2") do |config|
+
+# For efficiency, we create custom base box from an Ubuntu 12.04 image
+# Once the custom base box is created, we use that as a starting point.
+#  config.vm.box     = "precise64"
+#  config.vm.box_url = "http://files.vagrantup.com/precise64.box"
+  config.vm.box    = "edx-devstack"
+  config.vm.box_url = "https://s3.amazonaws.com/vagrant.testeng.edx.org/edx-devstack.box"
+
+  config.vm.network :private_network, ip: "192.168.33.10"
+  config.vm.network :forwarded_port, guest: 8000, host: 8000
+  config.vm.network :forwarded_port, guest: 8001, host: 8001
+  config.vm.network :forwarded_port, guest: 4567, host: 4567
+
+  config.vm.synced_folder "edx-platform", "/edx/app/edxapp/edx-platform", :create => true, nfs: true
+  config.vm.synced_folder "cs_comments_service", "/edx/app/forum/cs_comments_service", :create => true, nfs: true
+
+  config.hostsupdater.aliases = ["preview.localhost"]
+
+  config.vm.provider :virtualbox do |vb|
+    vb.customize ["modifyvm", :id, "--memory", MEMORY.to_s]
+    vb.customize ["modifyvm", :id, "--cpus", CPU_COUNT.to_s]
+  end
+
+  config.vm.provision :ansible do |ansible|
+    ansible.playbook = "../../playbooks/vagrant-devstack.yml"
+    ansible.inventory_path = "../../playbooks/vagrant/inventory.ini"
+    ansible.verbose = "extra"
+  end
+end

vagrant/devstack/ansible.cfg

+../../playbooks/ansible.cfg
\ No newline at end of file