Merge remote-tracking branch 'origin/master' into rc/empanada

.gitignore

@@ -5,7 +5,7 @@
 \#*\#
 *~
 .#*
-vagrant/devstack/cs_comments_service
-vagrant/devstack/edx-platform
-vagrant/release/*/devstack/cs_comments_service
-vagrant/release/*/devstack/edx-platform
+
+vagrant/*/devstack/edx-platform
+vagrant/*/devstack/cs_comments_service
+vagrant/*/devstack/ora

cloudformation_templates/edx-reference-architecture.json

@@ -16,7 +16,7 @@
       "Description":"Name of an existing EC2 KeyPair to enable SSH access to the web server",
       "Default":"deployment"
     },
-    "InstanceType":{
+    "EdxappInstanceType":{
       "Description":"WebServer EC2 instance type",
       "Type":"String",
       "Default":"m1.small",
@@ -131,6 +131,52 @@
       ],
       "ConstraintDescription":"must be a valid EC2 instance type."
     },
+    "XserverInstanceType":{
+      "Description":"RabbitMQ server EC2 instance type",
+      "Type":"String",
+      "Default":"m1.small",
+      "AllowedValues":[
+        "t1.micro",
+        "m1.small",
+        "m1.medium",
+        "m1.large",
+        "m1.xlarge",
+        "m2.xlarge",
+        "m2.2xlarge",
+        "m2.4xlarge",
+        "m3.xlarge",
+        "m3.2xlarge",
+        "c1.medium",
+        "c1.xlarge",
+        "cc1.4xlarge",
+        "cc2.8xlarge",
+        "cg1.4xlarge"
+      ],
+      "ConstraintDescription":"must be a valid EC2 instance type."
+    },
+    "XqueueInstanceType":{
+      "Description":"RabbitMQ server EC2 instance type",
+      "Type":"String",
+      "Default":"m1.small",
+      "AllowedValues":[
+        "t1.micro",
+        "m1.small",
+        "m1.medium",
+        "m1.large",
+        "m1.xlarge",
+        "m2.xlarge",
+        "m2.2xlarge",
+        "m2.4xlarge",
+        "m3.xlarge",
+        "m3.2xlarge",
+        "c1.medium",
+        "c1.xlarge",
+        "cc1.4xlarge",
+        "cc2.8xlarge",
+        "cg1.4xlarge"
+      ],
+      "ConstraintDescription":"must be a valid EC2 instance type."
+    },
     "SSHLocation":{
       "Description":"The IP address range that can be used to SSH to the EC2 instances",
       "Type":"String",
@@ -2475,7 +2521,7 @@
               "Fn::FindInMap":[
                 "AWSInstanceType2Arch",
                 {
-                  "Ref":"InstanceType"
+                  "Ref":"EdxappInstanceType"
                 },
                 "Arch"
               ]
@@ -2522,7 +2568,7 @@
           "Ref":"KeyName"
         },
         "InstanceType":{
-          "Ref":"InstanceType"
+          "Ref":"EdxappInstanceType"
         },
         "BlockDeviceMappings":[
           {
@@ -2830,7 +2876,7 @@
               "Fn::FindInMap":[
                 "AWSInstanceType2Arch",
                 {
-                  "Ref":"InstanceType"
+                  "Ref":"XqueueInstanceType"
                 },
                 "Arch"
               ]
@@ -2873,7 +2919,7 @@
           "Ref":"KeyName"
         },
         "InstanceType":{
-          "Ref":"InstanceType"
+          "Ref":"XqueueInstanceType"
         },
         "BlockDeviceMappings":[
           {
@@ -3207,7 +3253,7 @@
           "Ref":"KeyName"
         },
         "InstanceType":{
-          "Ref":"InstanceType"
+          "Ref":"RabbitInstanceType"
         },
         "BlockDeviceMappings":[
           {
@@ -3534,7 +3580,7 @@
               "Fn::FindInMap":[
                 "AWSInstanceType2Arch",
                 {
-                  "Ref":"InstanceType"
+                  "Ref":"XserverInstanceType"
                 },
                 "Arch"
               ]
@@ -3577,7 +3623,7 @@
           "Ref":"KeyName"
         },
         "InstanceType":{
-          "Ref":"InstanceType"
+          "Ref":"XserverInstanceType"
         },
         "BlockDeviceMappings":[
           {
@@ -4040,7 +4086,7 @@
           "Ref":"KeyName"
         },
         "InstanceType":{
-          "Ref":"InstanceType"
+          "Ref":"WorkerInstanceType"
         },
         "BlockDeviceMappings":[
           {
@@ -4289,7 +4335,7 @@
           "Ref":"KeyName"
         },
         "InstanceType":{
-          "Ref":"InstanceType"
+          "Ref":"ForumInstanceType"
         },
         "BlockDeviceMappings":[
           {
@@ -4664,7 +4710,7 @@
           "Ref":"KeyName"
         },
         "InstanceType":{
-          "Ref":"InstanceType"
+          "Ref":"MongoInstanceType"
         },
         "BlockDeviceMappings":[
           {

playbooks/edx-east/legacy_ora.yml

+# ansible-playbook -i ec2.py --limit="tag_group_grader:&tag_environment_stage" legacy_ora.yml -e "COMMON_ENV_TYPE=stage secure_dir=/path/to/secure/dir"
+- name: Deploy legacy_ora
+  hosts: all
+  sudo: True
+  gather_facts: True
+  vars:
+    ora_app_dir: '/opt/wwc'
+    ora_user: 'www-data'
+  serial: 1
+  roles:
+    - legacy_ora

playbooks/roles/common/defaults/main.yml

@@ -22,6 +22,7 @@ COMMON_GIT_MIRROR: 'github.com'
 COMMON_HOSTNAME: !!null
 
 common_debian_pkgs:
+  - ntp
   - ack-grep
   - lynx-cur
   - logrotate

playbooks/roles/common/tasks/main.yml

@@ -15,11 +15,6 @@
     - "{{ COMMON_BIN_DIR }}"
     - "{{ COMMON_CFG_DIR }}"
 
-- name: common | Create common log directory
-  file: >
-    path={{ COMMON_LOG_DIR }} state=directory owner=syslog
-    group=syslog mode=0755
-
 # Need to install python-pycurl to use Ansible's apt_repository module
 - name: common | Install python-pycurl
   apt: pkg=python-pycurl state=present update_cache=yes
@@ -36,6 +31,11 @@
     pkg={{','.join(common_debian_pkgs)}} install_recommends=yes
     state=present update_cache=yes
 
+- name: common | Create common log directory
+  file: >
+    path={{ COMMON_LOG_DIR }} state=directory owner=syslog
+    group=syslog mode=0755
+
 - name: common | upload sudo config for key forwarding as root
   copy: >
     src=ssh_key_forward dest=/etc/sudoers.d/ssh_key_forward

playbooks/roles/edxapp/defaults/main.yml

@@ -45,8 +45,6 @@ EDXAPP_COMMENTS_SERVICE_KEY:  'password'
 
 EDXAPP_EDXAPP_SECRET_KEY: ''
 
-EDXAPP_PEARSON_TEST_PASWORD: ''
-
 EDXAPP_OEE_URL: 'http://localhost:18060/'
 EDXAPP_OEE_USER: 'lms'
 EDXAPP_OEE_PASSWORD: 'password'
@@ -64,7 +62,6 @@ EDXAPP_FEATURES:
   CERTIFICATES_ENABLED: true
   ENABLE_DISCUSSION_SERVICE: true
   ENABLE_INSTRUCTOR_ANALYTICS: true
-  ENABLE_PEARSON_HACK_TEST: false
   SUBDOMAIN_BRANDING: false
   SUBDOMAIN_COURSE_LISTINGS: false
   PREVIEW_LMS_BASE: $EDXAPP_PREVIEW_LMS_BASE
@@ -97,7 +94,7 @@ EDXAPP_CONTACT_EMAIL: 'info@example.com'
 EDXAPP_BUGS_EMAIL: 'bugs@example.com'
 EDXAPP_DEFAULT_FROM_EMAIL: 'registration@example.com'
 EDXAPP_DEFAULT_FEEDBACK_EMAIL: 'feedback@example.com'
-EDXAPP_SERVER_EMAIL: 'devops@example.com'
+EDXAPP_DEFAULT_SERVER_EMAIL: 'devops@example.com'
 EDXAPP_BULK_EMAIL_DEFAULT_FROM_EMAIL: 'no-reply@example.com'
 
 EDXAPP_ENV_EXTRA: {}
@@ -253,7 +250,6 @@ edxapp_generic_auth_config:  &edxapp_generic_auth
       PASSWORD: $EDXAPP_MYSQL_PASSWORD
       HOST: $EDXAPP_MYSQL_HOST
       PORT: $EDXAPP_MYSQL_PORT
-  PEARSON_TEST_PASSWORD:  $EDXAPP_PEARSON_TEST_PASSWORD
   OPEN_ENDED_GRADING_INTERFACE:
     url: $EDXAPP_OEE_URL
     password: $EDXAPP_OEE_PASSWORD

playbooks/roles/legacy_ora/tasks/main.yml

+#
+# Update config for a legacy ora installation.
+#
+# This role requires that ora_app_dir and ora_user both be defined.
+# There is no default for them.
+#
+- fail: msg="ora_app_dir not defined. eg. /edx/app/ora, /opt/wwc"
+  when: ora_app_dir is not defined
+
+- fail: msg="ora_user not defined. eg. ora, www-data"
+  when: ora_user is not defined
+
+- fail: msg="COMMON_ENV_TYPE not defined. eg. stage, prod"
+  when: COMMON_ENV_TYPE is not defined
+
+- fail: msg="secure_dir not defined. This is a path to the secure ora config file."
+  when: secure_dir is not defined
+
+- name: legacy_ora | create ora application config
+  copy:
+    src={{secure_dir}}/files/{{COMMON_ENV_TYPE}}/legacy_ora/ora.env.json 
+    dest={{ora_app_dir}}/env.json
+  sudo_user: "{{ ora_user }}"
+  register: env_state
+  tags:
+  - deploy
+
+- name: legacy_ora | create ora auth file
+  copy:
+    src={{secure_dir}}/files/{{COMMON_ENV_TYPE}}/legacy_ora/ora.auth.json 
+    dest={{ora_app_dir}}/auth.json
+  sudo_user: "{{ ora_user }}"
+  register: auth_state
+  tags:
+  - deploy
+
+# Restart ORA Services
+- name: legacy_ora | restart edx-ora
+  service:
+    name=edx-ora
+    state=restarted
+  when: env_state.changed or auth_state.changed
+
+- name: legacy_ora | restart edx-ora-celery
+  service:
+    name=edx-ora-celery
+    state=restarted
+  when: env_state.changed or auth_state.changed

playbooks/roles/local_dev/app_bashrc.j2

+#! /usr/bin/env bash
+
+# {{ ansible_managed }}
+
+source "{{ item.home }}/{{ item.env }}"
+
+# If X11 forwarding is enabled, then use the DISPLAY value
+# already set and use the X11 session cookie
+if [ -n "$DISPLAY" ]; then
+    export XAUTHORITY="{{ localdev_xauthority }}"
+
+# Otherwise, configure the display to use the virtual frame buffer
+else
+    export DISPLAY="{{ localdev_xvfb_display }}"
+fi
+
+# Default to the code repository
+cd "{{ item.home }}/{{ item.repo }}"

playbooks/roles/local_dev/defaults/main.yml

 ---
 
+localdev_user: "vagrant"
+localdev_home: "/home/vagrant"
+localdev_xauthority: "{{ localdev_home }}/.Xauthority"
+localdev_xvfb_display: ":1"
+
 localdev_accounts:
-    - { user: "{{ edxapp_user}}", home: "{{ edxapp_app_dir }}" }
-    - { user: "{{ forum_user }}", home: "{{ forum_app_dir }}" }
-    - { user: "{{ ora_user }}", home: "{{ ora_app_dir }}" }
+    - { user: "{{ edxapp_user}}", home: "{{ edxapp_app_dir }}",
+        env: "edxapp_env", repo: "edx-platform" }
+
+    - { user: "{{ forum_user }}",home: "{{ forum_app_dir }}",
+        env: "forum_env", repo: "cs_comments_service" }
+
+    - { user: "{{ ora_user }}", home: "{{ ora_app_dir }}",
+        env: "ora_env", repo: "ora" }
 
-localdev_env:
-    DISPLAY: "{{ browser_xvfb_display }}"
 
 # Helpful system packages for local dev
 local_dev_pkgs:
     - vim
     - emacs
+    - xorg
+    - openbox

playbooks/roles/local_dev/files/x11_display

+Defaults        env_keep+=DISPLAY

playbooks/roles/local_dev/tasks/main.yml

@@ -27,7 +27,7 @@
 # Create scripts to configure environment
 - name: local_dev | create login scripts
   template:
-    src={{ item.user }}_bashrc.j2 dest={{ item.home }}/.bashrc
+    src=app_bashrc.j2 dest={{ item.home }}/.bashrc
     owner={{ item.user }} mode=755
   with_items: "{{ localdev_accounts }}"
 
@@ -38,3 +38,21 @@
     src=gitconfig dest={{ item.home }}/.gitconfig
     owner={{ item.user }} mode=700
   with_items: "{{ localdev_accounts }}"
+
+# Configure X11 for application users
+- name: local_dev | preserve DISPLAY for sudo
+  copy:
+    src=x11_display dest=/etc/sudoers.d/x11_display
+    owner=root group=root mode=0440
+
+- name: local_dev | login share X11 auth to app users
+  template:
+    src=share_x11.j2 dest={{ localdev_home }}/share_x11
+    owner={{ localdev_user }} mode=0700
+
+- name: local_dev | update bashrc with X11 share script
+  lineinfile:
+    dest={{ localdev_home }}/.bashrc
+    regexp=". {{ localdev_home }}/share_x11"
+    line=". {{ localdev_home }}/share_x11"
+    state=present

playbooks/roles/local_dev/templates/app_bashrc.j2

+#! /usr/bin/env bash
+
+# {{ ansible_managed }}
+
+source "{{ item.home }}/{{ item.env }}"
+
+# If X11 forwarding is enabled, then use the DISPLAY value
+# already set and use the X11 session cookie
+if [ -n "$DISPLAY" ]; then
+    export XAUTHORITY="{{ localdev_xauthority }}"
+
+# Otherwise, configure the display to use the virtual frame buffer
+else
+    export DISPLAY="{{ localdev_xvfb_display }}"
+fi
+
+cd "{{ item.home }}/{{ item.repo }}"

playbooks/roles/local_dev/templates/edxapp_bashrc.j2

-#! /usr/bin/env bash
-
-# {{ ansible_managed }}
-
-source "{{ edxapp_app_dir }}/edxapp_env"
-export DISPLAY="{{ browser_xvfb_display }}"
-
-cd $HOME/edx-platform

playbooks/roles/local_dev/templates/forum_bashrc.j2

-#! /usr/bin/env bash
-
-# {{ ansible_managed }}
-
-source "{{ forum_app_dir }}/forum_env"
-export DISPLAY="{{ browser_xvfb_display }}"
-
-cd $HOME/cs_comments_service

playbooks/roles/local_dev/templates/ora_bashrc.j2

-#! /usr/bin/env bash
-
-# {{ ansible_managed }}
-
-source "{{ ora_app_dir }}/ora_env"
-cd {{ ora_code_dir }}

playbooks/roles/local_dev/templates/share_x11.j2

+#!/usr/bin/env bash
+
+# Change permissions on the X11 session cookie
+# so application users can use the same X11 session.
+# This is very insecure and should *only* be used for local VMs.
+if [ -f {{ localdev_xauthority }} ]; then
+    chmod og+r {{ localdev_xauthority }}
+fi

playbooks/roles/rbenv/defaults/main.yml

@@ -10,6 +10,7 @@ rbenv_bin: "{{ rbenv_dir }}/.rbenv/bin"
 rbenv_shims: "{{ rbenv_root }}/shims"
 rbenv_path: "{{ rbenv_bin }}:{{ rbenv_shims }}:{{ rbenv_gem_bin }}:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
 rbenv_debian_pkgs:
+  - curl
   - build-essential
   - libcurl4-openssl-dev
   - libreadline-dev

util/jenkins/create-var-file.sh

@@ -8,8 +8,8 @@ EDXAPP_LMS_BASE: ${deploy_host}
 EDXAPP_LMS_NGINX_PORT: 80
 EDXAPP_LMS_PREVIEW_NGINX_PORT: 80
 EDXAPP_CMS_NGINX_PORT: 80
+EDXAPP_SITE_NAME: ${deploy_host}
 COMMON_PYPI_MIRROR_URL: 'https://pypi.edx.org/root/pypi/+simple/'
-COMMON_GIT_MIRROR: 'git.edx.org'
 XSERVER_GRADER_DIR: "{{ xserver_data_dir }}/data/content-mit-600x~2012_Fall"
 XSERVER_GRADER_SOURCE: "git@github.com:/MITx/6.00x.git"
 XSERVER_LOCAL_GIT_IDENTITY: /var/lib/jenkins/git-identity-edx-pull

vagrant/base/devstack/Vagrantfile

@@ -16,7 +16,7 @@ end
 
 Vagrant.configure("2") do |config|
 
-# Creates a devstack from a base Ubuntu 12.04 image
+  # Creates a devstack from a base Ubuntu 12.04 image
   config.vm.box     = "precise64"
   config.vm.box_url = "http://files.vagrantup.com/precise64.box"
 
@@ -31,6 +31,11 @@ Vagrant.configure("2") do |config|
 
   config.hostsupdater.aliases = ["preview.localhost"]
 
+  # Enable X11 forwarding so we can interact with GUI applications
+  if ENV['VAGRANT_X11']
+      config.ssh.forward_x11 = true
+  end
+
   config.vm.provider :virtualbox do |vb|
     vb.customize ["modifyvm", :id, "--memory", MEMORY.to_s]
     vb.customize ["modifyvm", :id, "--cpus", CPU_COUNT.to_s]

vagrant/release/devstack/Vagrantfile

@@ -2,8 +2,20 @@ MEMORY = 2048
 CPU_COUNT = 2
 
 $script = <<SCRIPT
+if [ ! -d /edx/app/edx_ansible ]; then
+    echo "Error: Base box is missing provisioning scripts." 1>&2
+    exit 1
+fi
+export PYTHONUNBUFFERED=1
 source /edx/app/edx_ansible/venvs/edx_ansible/bin/activate
 cd /edx/app/edx_ansible/edx_ansible/playbooks
+
+# Need to ensure that the configuration repo is updated
+# The vagrant-devstack.yml playbook will also do this, but only
+# after loading the playbooks into memory.  If these are out of date,
+# this can cause problems (e.g. looking for templates that no longer exist).
+/edx/bin/update configuration master
+
 ansible-playbook -i localhost, -c local vagrant-devstack.yml
 SCRIPT