Merge pull request #3630 from open-release/ficus.master

Ficus release changes

# Conflicts:
#	playbooks/roles/discovery/tasks/main.yml

playbooks/edx-east/edx_jenkins_tests.yml

@@ -18,7 +18,6 @@
       nginx_sites:
       - lms
       - cms
-      - lms-preview
       - xqueue
       - xserver
       nginx_default_sites:

playbooks/edx_sandbox.yml

@@ -53,6 +53,7 @@
       when: SANDBOX_ENABLE_ECOMMERCE
     - role: ecomworker
       when: SANDBOX_ENABLE_ECOMMERCE
+    - programs
     - analytics_api
     - insights
     # not ready yet: - edx_notes_api

playbooks/roles/ansible_debug/tasks/main.yml

@@ -21,19 +21,6 @@
     - dumpall
     - debug
 
-- name: Dump lms-preview auth|env file
-  template:
-    src: "../../edxapp/templates/lms-preview.{{ item }}.json.j2"
-    dest: "/tmp/lms-preview.{{ item }}.json"
-    mode: 0600
-  with_items:
-    - env
-    - auth
-  when: "'lms-preview' in service_variants_enabled"
-  tags:
-    - dumpall
-    - debug
-
 - name: Dump cms auth|env file
   template:
     src: "../../edxapp/templates/cms.{{ item }}.json.j2"

playbooks/roles/browsers/defaults/main.yml

 browser_deb_pkgs:
-    - xvfb
     - dbus-x11
+    - gdebi
+    - libcurl3
     - libgconf2-4
-    - libxss1
     - libnss3-1d
-    - libcurl3
+    - libxss1
     - xdg-utils
-    - gdebi
+    - xvfb
+
+# Firefox for Xenial
+firefox_version: version 45.*
 
-# Debian packages we host in S3 to ensure correct browser version
-# Both Chrome and FireFox update their apt repos with the latest version,
-# which often causes spurious acceptance test failures.
+# Packages we host in S3 to ensure correct browser version Both Chrome and
+# FireFox update their apt repos with the latest version, which often causes
+# spurious acceptance test failures.
 browser_s3_deb_pkgs:
-    - { name: "firefox-mozilla-build_42.0-0ubuntu1_amd64.deb", url: "https://s3.amazonaws.com/vagrant.testeng.edx.org/firefox-mozilla-build_42.0-0ubuntu1_amd64.deb" }
-    - { name: "google-chrome-stable_55.0.2883.87-1_amd64.deb", url: "https://s3.amazonaws.com/vagrant.testeng.edx.org/google-chrome-stable_55.0.2883.87-1_amd64.deb" }
+  - name: firefox_45.0.2+build1-0ubuntu1_amd64
+    url: https://s3.amazonaws.com/vagrant.testeng.edx.org/firefox_45.0.2%2Bbuild1-0ubuntu1_amd64.deb
+  - name: google-chrome-stable_55.0.2883.87-1_amd64.deb
+    url: https://s3.amazonaws.com/vagrant.testeng.edx.org/google-chrome-stable_55.0.2883.87-1_amd64.deb
 
 # ChromeDriver
 chromedriver_version: 2.27

playbooks/roles/browsers/tasks/main.yml

 # Install browsers required to run the JavaScript
 # and acceptance test suite locally without a display
 ---
+- name: lock Firefox version
+  copy:
+    dest: /etc/apt/preferences.d/firefox-pinned-version
+    content: |
+      Package: firefox
+      Pin: {{ firefox_version }}
+      Pin-Priority: 1001
+  tags:
+    - install
+    - install:system-requirements
+
 - name: install system packages
-  apt: pkg={{','.join(browser_deb_pkgs)}}
-       state=present update_cache=yes
+  apt:
+    name: "{{ item }}"
+    update_cache: yes
+  with_items: "{{ browser_deb_pkgs }}"
+  tags:
+    - install
+    - install:system-requirements
 
-- name: download browser debian packages from S3
-  get_url: dest="/tmp/{{ item.name }}" url="{{ item.url }}"
+- name: download browser packages from S3
+  get_url:
+    dest: /tmp/{{ item.name }}
+    url: "{{ item.url }}"
   register: download_deb
   with_items: "{{ browser_s3_deb_pkgs }}"
+  tags:
+    - install
+    - install:system-requirements
 
-- name: install browser debian packages
+- name: install browser packages
   shell: gdebi -nq /tmp/{{ item.name }}
   when: download_deb.changed
   with_items: "{{ browser_s3_deb_pkgs }}"
-
-# Because the source location has been deprecated, we need to
-# ensure it does not interfere with subsequent apt commands
-- name: remove google chrome debian source list
-  file:
-    path: /etc/apt/sources.list.d/google-chrome.list
-    state: absent
+  tags:
+    - install
+    - install:system-requirements
 
 - name: download ChromeDriver
   get_url:
-    url={{ chromedriver_url }}
-    dest=/var/tmp/chromedriver_{{ chromedriver_version }}.zip
+    url: "{{ chromedriver_url }}"
+    dest: /var/tmp/chromedriver_{{ chromedriver_version }}.zip
   register: download_chromedriver
+  tags:
+    - install
+    - install:system-requirements
 
 - name: unzip ChromeDriver tarfile
   shell: unzip /var/tmp/chromedriver_{{ chromedriver_version }}.zip
-         chdir=/var/tmp
+  args:
+    chdir: /var/tmp
   when: download_chromedriver.changed
+  tags:
+    - install
+    - install:system-requirements
 
 - name: move ChromeDriver binary to /usr/local
   shell: mv /var/tmp/chromedriver /usr/local/bin/chromedriver
   when: download_chromedriver.changed
+  tags:
+    - install
+    - install:system-requirements
 
 - name: make ChromeDriver binary executable
-  file: path=/usr/local/bin/chromedriver mode=0755
+  file:
+    path: /usr/local/bin/chromedriver
+    mode: 0755
   when: download_chromedriver.changed
+  tags:
+    - install
+    - install:system-requirements
 
 - name: verify ChromeDriver location and mode
-  stat: path=/usr/local/bin/chromedriver
+  stat:
+    path: /usr/local/bin/chromedriver
   register: chromedriver
+  tags:
+    - install
+    - install:system-requirements
+
 - assert:
     that:
       - "chromedriver.stat.exists"
       - "chromedriver.stat.mode == '0755'"
+  tags:
+    - install
+    - install:system-requirements
 
 - name: download PhantomJS
   get_url:
     url: "{{ phantomjs_url }}"
     dest: "/var/tmp/{{ phantomjs_tarfile }}"
   register: download_phantom_js
+  tags:
+    - install
+    - install:system-requirements
 
 - name: unpack the PhantomJS tarfile
   shell: "tar -xjf /var/tmp/{{ phantomjs_tarfile }}"
   args:
     chdir: "/var/tmp"
   when: download_phantom_js.changed
+  tags:
+    - install
+    - install:system-requirements
 
 - name: move PhantomJS binary to /usr/local
   shell: mv /var/tmp/{{ phantomjs_version }}/bin/phantomjs /usr/local/bin/phantomjs
   when: download_phantom_js.changed
+  tags:
+    - install
+    - install:system-requirements
 
 - name: verify phantomjs location
   stat: path=/usr/local/bin/phantomjs
   register: phantomjs
-- assert:
-    that: "phantomjs.stat.exists"
-
-- name: create xvfb upstart script for Precise and Trusty (12.04 and 14.04)
-  template:
-    src: xvfb.conf.j2
-    dest: /etc/init/xvfb.conf
-    owner: root
-    group: root
-  when: ansible_distribution_release == 'precise' or ansible_distribution_release == 'trusty'
   tags:
     - install
-    - install:configuration
+    - install:system-requirements
 
-- name: start xvfb upstart script for Precise and Trusty (12.04 and 14.04)
-  shell: start xvfb
-  ignore_errors: yes
-  when: ansible_distribution_release == 'precise' or ansible_distribution_release == 'trusty'
+- assert:
+    that: "phantomjs.stat.exists"
   tags:
     - install
-    - install:configuration
+    - install:system-requirements
 
-- name: create xvfb systemd service for Xenial (16.04)
+- name: create xvfb systemd service
   template:
     src: xvfb.service.j2
     dest: /etc/systemd/system/xvfb.service
     owner: root
     group: root
-  when: ansible_distribution_release == 'xenial'
   tags:
     - install
     - install:configuration
 
-- name: enable and start xvfb systemd service for Xenial (16.04)
+- name: enable and start xvfb systemd service
   systemd:
     name: xvfb
     enabled: yes
     state: started
-  when: ansible_distribution_release == 'xenial'
   tags:
     - install
     - install:configuration

playbooks/roles/browsers/templates/xvfb.conf.j2

-description     "Xvfb X Server"
-start on (net-device-up and local-filesystems and runlevel [2345])
-stop on runlevel [016]
-exec /usr/bin/Xvfb {{ browser_xvfb_display }} -screen 0 1024x768x24
-respawn
-respawn limit 15 5

playbooks/roles/browsers/templates/xvfb.service.j2

@@ -2,7 +2,7 @@
 Description=Xvfb X Server
 After=network.target
 
-[Service] 
+[Service]
 ExecStart=/usr/bin/Xvfb {{ browser_xvfb_display }} -screen 0 1024x768x24
 
 [Install]

playbooks/roles/common_vars/defaults/main.yml

@@ -45,7 +45,7 @@ common_directories:
   - path: "/etc/logrotate.d/hourly"
   - path: "/etc/rsyslog.d/50-default.conf"
     state: absent
-    
+
 COMMON_ENVIRONMENT: 'default_env'
 COMMON_DEPLOYMENT: 'default_deployment'
 COMMON_PIP_VERBOSITY: ''
@@ -98,6 +98,7 @@ COMMON_ENABLE_NEWRELIC_APP: False
 COMMON_ENABLE_MINOS: False
 COMMON_TAG_EC2_INSTANCE: False
 common_boto_version: '2.34.0'
+common_node_version: '6.9.4'
 common_redhat_pkgs:
   - ntp
   - lynx

playbooks/roles/credentials/defaults/main.yml

@@ -9,7 +9,7 @@
 #
 ##
 # Defaults for role credentials
-# 
+#
 CREDENTIALS_GIT_IDENTITY: !!null
 
 # depends upon Newrelic being enabled via COMMON_ENABLE_NEWRELIC
@@ -219,6 +219,7 @@ credentials_nodeenv_dir: "{{ credentials_home }}/nodeenvs/{{ credentials_service
 credentials_nodeenv_bin: "{{ credentials_nodeenv_dir }}/bin"
 credentials_node_modules_dir: "{{ credentials_code_dir }}/node_modules"
 credentials_node_bin: "{{ credentials_node_modules_dir }}/.bin"
+credentials_node_version: "{{ common_node_version }}"
 
 credentials_environment:
   DJANGO_SETTINGS_MODULE: "{{ CREDENTIALS_DJANGO_SETTINGS_MODULE }}"

playbooks/roles/credentials/tasks/main.yml

@@ -42,14 +42,23 @@
     - install
     - install:app-requirements
 
+# See https://github.com/ekalinin/nodeenv/issues/182
+# This can be removed once https://github.com/ekalinin/nodeenv/pull/183 is merged
+- name: Remove nodeenv src
+  file:
+    path: "{{ credentials_nodeenv_dir }}/src"
+    state: absent
+  become_user: "{{ credentials_user }}"
+  tags:
+    - install
+    - install:system-requirements
+
 - name: create nodeenv
-  shell: "{{ credentials_venv_dir }}/bin/nodeenv {{ credentials_nodeenv_dir }} --prebuilt"
-  args:
-    creates: "{{ credentials_nodeenv_dir }}"
+  shell: "{{ credentials_venv_dir }}/bin/nodeenv {{ credentials_nodeenv_dir }} --node={{ credentials_node_version }} --prebuilt --force"
   become_user: "{{ credentials_user }}"
   tags:
     - install
-    - install:app-requirements
+    - install:system-requirements
 
 - name: install development requirements
   command: make requirements

playbooks/roles/discovery/defaults/main.yml

@@ -99,7 +99,7 @@ DISCOVERY_MEDIA_STORAGE_BACKEND:
   DEFAULT_FILE_STORAGE: 'django.core.files.storage.FileSystemStorage'
   MEDIA_ROOT: '{{ DISCOVERY_MEDIA_ROOT }}'
   MEDIA_URL: '{{ DISCOVERY_MEDIA_URL }}'
-  
+
 DISCOVERY_STATICFILES_STORAGE: 'django.contrib.staticfiles.storage.StaticFilesStorage'
 
 # You can set different email backends with django:
@@ -241,7 +241,7 @@ discovery_nodeenv_dir: "{{ discovery_home }}/nodeenvs/{{ discovery_service_name
 discovery_nodeenv_bin: "{{ discovery_nodeenv_dir }}/bin"
 discovery_node_modules_dir: "{{ discovery_code_dir }}/node_modules"
 discovery_node_bin: "{{ discovery_node_modules_dir }}/.bin"
-discovery_node_version: "6.9.4"
+discovery_node_version: "{{ common_node_version }}"
 
 discovery_gunicorn_host: "127.0.0.1"
 discovery_gunicorn_port: 8381

playbooks/roles/discovery/tasks/main.yml

@@ -67,10 +67,19 @@
     - install
     - install:system-requirements
 
+# See https://github.com/ekalinin/nodeenv/issues/182
+# This can be removed once https://github.com/ekalinin/nodeenv/pull/183 is merged
+- name: remove nodeenv src
+  file:
+    path: "{{ discovery_nodeenv_dir }}/src"
+    state: absent
+  become_user: "{{ discovery_user }}"
+  tags:
+    - install
+    - install:system-requirements
+
 - name: create nodeenv
-  shell: "{{ discovery_venv_dir }}/bin/nodeenv {{ discovery_nodeenv_dir }} --node={{ discovery_node_version }} --prebuilt"
-  args:
-    creates: "{{ discovery_nodeenv_dir }}"
+  shell: "{{ discovery_venv_dir }}/bin/nodeenv {{ discovery_nodeenv_dir }} --node={{ discovery_node_version }} --prebuilt --force"
   become_user: "{{ discovery_user }}"
   tags:
     - install
@@ -238,4 +247,3 @@
   tags:
     - install
     - install:vhosts
-

playbooks/roles/ecommerce/defaults/main.yml

@@ -214,6 +214,7 @@ ecommerce_nodeenv_dir: "{{ ecommerce_home }}/nodeenvs/{{ ecommerce_service_name
 ecommerce_nodeenv_bin: "{{ ecommerce_nodeenv_dir }}/bin"
 ecommerce_node_modules_dir: "{{ ecommerce_code_dir }}/node_modules"
 ecommerce_node_bin: "{{ ecommerce_node_modules_dir }}/.bin"
+ecommerce_node_version: "{{ common_node_version }}"
 
 ecommerce_gunicorn_host: "127.0.0.1"
 ecommerce_gunicorn_port: "8130"

playbooks/roles/ecommerce/tasks/main.yml

@@ -40,20 +40,30 @@
     - install
     - install:app-requirements
 
+# See https://github.com/ekalinin/nodeenv/issues/182
+# This can be removed once https://github.com/ekalinin/nodeenv/pull/183 is merged
+- name: Remove nodeenv src
+  file:
+    path: "{{ ecommerce_nodeenv_dir }}/src"
+    state: absent
+  become_user: "{{ ecommerce_user }}"
+  tags:
+    - install
+    - install:system-requirements
+
 - name: Create nodeenv
-  shell: "{{ ecommerce_venv_dir }}/bin/nodeenv {{ ecommerce_nodeenv_dir }} --prebuilt"
-  args:
-    creates: "{{ ecommerce_nodeenv_dir }}"
+  shell: "{{ ecommerce_venv_dir }}/bin/nodeenv {{ ecommerce_nodeenv_dir }} --node={{ ecommerce_node_version }} --prebuilt --force"
   become_user: "{{ ecommerce_user }}"
   tags:
     - install
-    - install:app-requirements
+    - install:system-requirements
 
 - name: Install node dependencies
   npm:
     executable: "{{ ecommerce_nodeenv_bin }}/npm"
     path: "{{ ecommerce_code_dir }}"
     production: yes
+    state: latest
   become_user: "{{ ecommerce_user }}"
   environment: "{{ ecommerce_environment }}"
   tags:

playbooks/roles/edx_ansible/templates/pre-box.j2

@@ -13,8 +13,9 @@ apt-get autoclean -y
 # Clean out pip caches.
 find / -type d -path '*/.cache/pip' -print -exec rm -rf '{}/*' \;
 
-# Remove all the .pyc files.
-find /edx/app -name '*.pyc' -delete
+# We used to remove all .pyc files. This caused problems in sandboxes,
+# where code couldn't write .pyc files, and everything took too long.
+#   find /edx/app -name '*.pyc' -delete
 
 # Last thing: fill the disk with zeros so they'll compress well.
 # The dd command fails because the disk fills, which is the point, so silence

playbooks/roles/edxapp/defaults/main.yml

@@ -24,7 +24,7 @@ EDXAPP_DJFS:
   url_root : '/static/django-pyfs'
 
 EDXAPP_LMS_BASE: "{{ EDXAPP_LMS_SITE_NAME }}:{{ EDXAPP_LMS_NGINX_PORT }}"
-EDXAPP_PREVIEW_LMS_BASE: "preview.{{ EDXAPP_LMS_SITE_NAME }}:{{ EDXAPP_LMS_PREVIEW_NGINX_PORT }}"
+EDXAPP_PREVIEW_LMS_BASE: "preview.{{ EDXAPP_LMS_SITE_NAME }}:{{ EDXAPP_LMS_NGINX_PORT }}"
 EDXAPP_CMS_BASE: "{{ EDXAPP_CMS_SITE_NAME }}:{{ EDXAPP_CMS_NGINX_PORT }}"
 
 EDXAPP_LMS_GUNICORN_EXTRA: ""
@@ -267,7 +267,6 @@ EDXAPP_RABBIT_HOSTNAME: 'localhost'
 EDXAPP_LMS_NGINX_PORT: 18000
 EDXAPP_LMS_SSL_NGINX_PORT: 48000
 
-EDXAPP_LMS_PREVIEW_NGINX_PORT: 18020
 EDXAPP_CMS_NGINX_PORT: 18010
 EDXAPP_CMS_SSL_NGINX_PORT: 48010
 
@@ -643,7 +642,7 @@ edxapp_venv_bin: "{{ edxapp_venv_dir }}/bin"
 edxapp_nodeenv_dir: "{{ edxapp_app_dir }}/nodeenvs/edxapp"
 edxapp_nodeenv_bin: "{{ edxapp_nodeenv_dir }}/bin"
 edxapp_settings: '{{ EDXAPP_SETTINGS }}'
-edxapp_node_version: "6.9.2"
+edxapp_node_version: "{{ common_node_version }}"
 # This is where node installs modules, not node itself
 edxapp_node_bin: "{{ edxapp_code_dir }}/node_modules/.bin"
 edxapp_user: edxapp

playbooks/roles/edxapp/tasks/deploy.yml

 ---
 - name: create edxapp configuration dir
   file:
-    path: "{{ EDXAPP_CFG_DIR }}" 
+    path: "{{ EDXAPP_CFG_DIR }}"
     state: directory
     owner: "{{ edxapp_user }}"
     group: "{{ common_web_group }}"
@@ -13,7 +13,7 @@
   template:
     src: "{{ item.src }}"
     dest: "{{ item.dest }}"
-    owner: "{{ item.owner }}"  
+    owner: "{{ item.owner }}"
     group: "{{ item.group }}"
     mode: "{{ item.mode }}"
   with_items:
@@ -210,13 +210,21 @@
     - install
     - install:app-requirements
 
+# See https://github.com/ekalinin/nodeenv/issues/182
+# This can be removed once https://github.com/ekalinin/nodeenv/pull/183 is merged
+- name: Remove nodeenv src
+  file:
+    path: "{{ edxapp_nodeenv_dir }}/src"
+    state: absent
+  tags:
+    - install
+    - install:system-requirements
+
 - name: create nodeenv
-  shell: "{{ edxapp_venv_dir }}/bin/nodeenv {{ edxapp_nodeenv_dir }} --node={{ edxapp_node_version }} --prebuilt"
-  args:
-    creates: "{{ edxapp_nodeenv_dir }}"
+  shell: "{{ edxapp_venv_dir }}/bin/nodeenv {{ edxapp_nodeenv_dir }} --node={{ edxapp_node_version }} --prebuilt --force"
   tags:
     - install
-    - install:app-requirements
+    - install:system-requirements
 
 # Set the npm registry
 # This needs to be done as root since npm is weird about
@@ -245,6 +253,7 @@
     executable: "{{ edxapp_nodeenv_bin }}/npm"
     path: "{{ edxapp_code_dir }}"
     production: yes
+    state: latest
   environment: "{{ edxapp_environment }}"
   become_user: "{{ edxapp_user }}"
   tags:

playbooks/roles/insights/defaults/main.yml

@@ -9,7 +9,7 @@
 #
 #
 # Defaults for role insights
-# 
+#
 
 INSIGHTS_LMS_BASE: 'http://127.0.0.1:8000'
 INSIGHTS_CMS_BASE: 'http://127.0.0.1:8010'
@@ -169,7 +169,7 @@ insights_environment:
   ANALYTICS_DASHBOARD_CFG: "{{ COMMON_CFG_DIR  }}/{{ insights_service_name }}.yml"
   PATH: "{{ insights_nodeenv_bin }}:{{ insights_venv_dir }}/bin:{{ ansible_env.PATH }}"
 
-  
+
 insights_service_name: insights
 insights_venv_dir: "{{ insights_home }}/venvs/{{ insights_service_name }}"
 insights_user: "{{ insights_service_name }}"
@@ -185,6 +185,7 @@ insights_nodeenv_dir: "{{ insights_home }}/nodeenvs/{{ insights_service_name }}"
 insights_nodeenv_bin: "{{ insights_nodeenv_dir }}/bin"
 insights_node_modules_dir: "{{ insights_code_dir }}/node_modules"
 insights_node_bin: "{{ insights_node_modules_dir }}/.bin"
+insights_node_version: "{{ common_node_version }}"
 
 insights_gunicorn_host: "127.0.0.1"
 insights_gunicorn_port: "8110"

playbooks/roles/insights/tasks/main.yml

@@ -44,17 +44,30 @@
     - install
     - install:app-requirements
 
+# See https://github.com/ekalinin/nodeenv/issues/182
+# This can be removed once https://github.com/ekalinin/nodeenv/pull/183 is merged
+- name: Remove nodeenv src
+  file:
+    path: "{{ insights_nodeenv_dir }}/src"
+    state: absent
+  become_user: "{{ insights_user }}"
+  tags:
+    - install
+    - install:system-requirements
+
 - name: create nodeenv
-  shell: "{{ insights_venv_dir }}/bin/nodeenv {{ insights_nodeenv_dir }}  --prebuilt"
-  args:
-    creates: "{{ insights_nodeenv_dir }}"
+  shell: "{{ insights_venv_dir }}/bin/nodeenv {{ insights_nodeenv_dir }}  --node={{ insights_node_version }} --prebuilt --force"
   become_user: "{{ insights_user }}"
   tags:
     - install
-    - install:base
+    - install:system-requirements
 
 - name: install node dependencies
-  npm: executable={{ insights_nodeenv_bin }}/npm path={{ insights_code_dir }} production=yes
+  npm:
+    executable: "{{ insights_nodeenv_bin }}/npm"
+    path: "{{ insights_code_dir }}"
+    production: yes
+    state: latest
   become_user: "{{ insights_user }}"
   tags:
     - install

playbooks/roles/nginx/templates/edx/app/nginx/sites-available/lms-preview.j2

-upstream lms-preview-backend {
-    {% for host in nginx_lms_preview_gunicorn_hosts %}
-        server {{ host }}:{{ edxapp_lms_preview_gunicorn_port }} fail_timeout=0;
-    {% endfor %}
-}
-
-server {
-  # LMS-preview configuration file for nginx, templated by ansible
-
-  listen {{ EDXAPP_LMS_PREVIEW_NGINX_PORT }};
-
-  server_name preview.*;
-
-  # CS184 requires uploads of up to 4MB for submitting screenshots.
-  # CMS requires larger value for course assest, values provided
-  # via hiera.
-  client_max_body_size 4M;
-  # request the browser to use SSL for all connections
-  add_header Strict-Transport-Security "max-age=31536000; includeSubDomains";
-
-  rewrite ^(.*)/favicon.ico$ /static/images/favicon.ico last;
-
-  {% include "python_lib.zip.j2" %}
-  {% include "common-settings.j2" %}
-
-  location @proxy_to_lms-preview_app {
-    {% if NGINX_SET_X_FORWARDED_HEADERS %}
-    proxy_set_header X-Forwarded-Proto $scheme;
-    proxy_set_header X-Forwarded-Port $server_port;
-    proxy_set_header X-Forwarded-For $remote_addr;
-    {% else %}
-    proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto;
-    proxy_set_header X-Forwarded-Port $http_x_forwarded_port;
-    proxy_set_header X-Forwarded-For $http_x_forwarded_for;
-    {% endif %}
-
-    # newrelic-specific header records the time when nginx handles a request.
-    proxy_set_header X-Queue-Start "t=${msec}";
-
-    proxy_set_header Host $http_host;
-
-    proxy_redirect off;
-    proxy_pass http://lms-preview-backend;
-  }
-
-  location / {
-    {% if EDXAPP_LMS_PREVIEW_ENABLE_BASIC_AUTH|bool %}
-      {% include "basic-auth.j2" %}
-    {% endif %}
-    try_files $uri @proxy_to_lms-preview_app;
-  }
-
-  # No basic auth security on the github_service_hook url, so that github can use it for cms
-  location /github_service_hook {
-    try_files $uri @proxy_to_lms-preview_app;
-  }
-
-  # No basic auth security on the heartbeat url, so that ELB can use it
-  location /heartbeat {
-    try_files $uri @proxy_to_lms-preview_app;
-  }
-
-  {% include "robots.j2" %}
-
-  # Check security on this
-  location ~ ^/static/(?P<file>.*) {
-    root {{ edxapp_data_dir}};
-    try_files /staticfiles/$file /course_static/$file =404;
-
-    # return a 403 for static files that shouldn't be
-    # in the staticfiles directory
-    location ~ ^/static/(?:.*)(?:\.xml|\.json|README.TXT) {
-        return 403;
-    }
-    # Set django-pipelined files to maximum cache time
-    location ~ "/static/(?P<collected>.*\.[0-9a-f]{12}\..*)" {
-        expires max;
-        # Without this try_files, files that have been run through
-        # django-pipeline return 404s
-        try_files /staticfiles/$collected /course_static/$collected =404;
-    }
-
-    # Expire other static files immediately (there should be very few / none of these)
-    expires epoch;
-  }
-
-  # Nginx does not support nested condition or or conditions so
-  # there is an unfortunate mix of conditonals here.
-  {% if NGINX_REDIRECT_TO_HTTPS %}
-     {% if NGINX_HTTPS_REDIRECT_STRATEGY == "scheme" %}
-  # Redirect http to https over single instance
-  if ($scheme != "https") 
-  { 
-   set $do_redirect_to_https "true";
-  }
-
-     {% elif NGINX_HTTPS_REDIRECT_STRATEGY == "forward_for_proto" %}
-
-  # Forward to HTTPS if we're an HTTP request... and the server is behind ELB 
-  if ($http_x_forwarded_proto = "http") 
-  {
-   set $do_redirect_to_https "true";
-  }
-  {% endif %}
-
-  # Execute the actual redirect
-  if ($do_redirect_to_https = "true")
-  {
-  return 301 https://$host$request_uri;
-  }
-  {% endif %}
-
-}

playbooks/sample_vars/server_vars.yml

@@ -69,7 +69,6 @@
 #EDXAPP_LMS_SSL_NGINX_PORT: 443
 #EDXAPP_CMS_SSL_NGINX_PORT: 443
 #EDXAPP_LMS_NGINX_PORT: 80
-#EDXAPP_LMS_PREVIEW_NGINX_PORT: 80
 #EDXAPP_CMS_NGINX_PORT: 80
 #EDXAPP_WORKERS:
 #  lms: 2

util/install/install_stack.sh

@@ -136,6 +136,11 @@ exec > >(tee $log_file) 2>&1
 echo "Capturing output to $log_file"
 echo "Installation started at $(date '+%Y-%m-%d %H:%M:%S')"
 
+function finish {
+    echo "Installation finished at $(date '+%Y-%m-%d %H:%M:%S')"
+}
+trap finish EXIT
+
 export OPENEDX_RELEASE=$release
 echo "Installing release '$OPENEDX_RELEASE'"
 
@@ -188,6 +193,5 @@ else
     sudo bash -c "echo '192.168.33.10  preview.localhost' >> /etc/hosts"
 fi
 
-echo "Installation finished at $(date '+%Y-%m-%d %H:%M:%S')"
 echo -e "${SUCCESS}Finished installing! You may now log in using 'vagrant ssh'"
 echo -e "Refer to the edX wiki ($wiki_link) for more information on using $stack.${NC}"

util/install/sandbox.sh

@@ -11,8 +11,8 @@
 ##
 ## Sanity check
 ##
-if [[ `lsb_release -rs` != "12.04" ]]; then
-   echo "This script is only known to work on Ubuntu 12.04, exiting...";
+if [[ `lsb_release -rs` != "16.04" ]]; then
+   echo "This script is only known to work on Ubuntu 16.04, exiting...";
    exit;
 fi
 
@@ -31,18 +31,12 @@ sudo apt-get upgrade -y
 ##
 ## Install system pre-requisites
 ##
-sudo apt-get install -y build-essential software-properties-common curl git-core libxml2-dev libxslt1-dev python-pip libmysqlclient-dev python-apt python-dev libxmlsec1-dev libfreetype6-dev swig gcc-4.8 g++-4.8
+sudo apt-get install -y build-essential software-properties-common curl git-core libxml2-dev libxslt1-dev python-pip libmysqlclient-dev python-apt python-dev libxmlsec1-dev libfreetype6-dev swig gcc g++
 sudo pip install --upgrade pip==8.1.2
 sudo pip install --upgrade setuptools==24.0.3
 sudo -H pip install --upgrade virtualenv==15.0.2
 
 ##
-## Update alternatives so that gcc/g++ 4.8 is the default compiler
-##
-sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-4.8 50
-sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-4.8 50
-
-##
 ## Overridable version variables in the playbooks. Each can be overridden
 ## individually, or with $OPENEDX_RELEASE.
 ##
@@ -56,9 +50,12 @@ VERSION_VARS=(
   NOTIFIER_VERSION
   INSIGHTS_VERSION
   ANALYTICS_API_VERSION
+  ECOMMERCE_VERSION
+  ECOMMERCE_WORKER_VERSION
+  PROGRAMS_VERSION
 )
 
-EXTRA_VARS=""
+EXTRA_VARS="-e SANDBOX_ENABLE_ECOMMERCE=True $EXTRA_VARS"
 for var in ${VERSION_VARS[@]}; do
   # Each variable can be overridden by a similarly-named environment variable,
   # or OPENEDX_RELEASE, if provided.
@@ -78,6 +75,7 @@ cd /var/tmp
 git clone https://github.com/edx/configuration
 cd configuration
 git checkout $CONFIGURATION_VERSION
+git pull
 
 ##
 ## Install the ansible requirements
@@ -88,4 +86,4 @@ sudo -H pip install -r requirements.txt
 ##
 ## Run the edx_sandbox.yml playbook in the configuration/playbooks directory
 ##
-cd /var/tmp/configuration/playbooks && sudo ansible-playbook -c local ./edx_sandbox.yml -i "localhost," $EXTRA_VARS
+cd /var/tmp/configuration/playbooks && sudo -E ansible-playbook -c local ./edx_sandbox.yml -i "localhost," $EXTRA_VARS

util/jenkins/ansible-provision.sh

@@ -187,7 +187,6 @@ EDXAPP_COMPREHENSIVE_THEME_DIRS: $edxapp_comprehensive_theme_dirs
 
 EDXAPP_STATIC_URL_BASE: $static_url_base
 EDXAPP_LMS_NGINX_PORT: 80
-EDXAPP_LMS_PREVIEW_NGINX_PORT: 80
 EDXAPP_CMS_NGINX_PORT: 80
 
 ECOMMERCE_NGINX_PORT: 80

vagrant/base/devstack/Vagrantfile

-Vagrant.require_version ">= 1.5.3"
+Vagrant.require_version ">= 1.8.7"
 unless Vagrant.has_plugin?("vagrant-vbguest")
   raise "Please install the vagrant-vbguest plugin by running `vagrant plugin install vagrant-vbguest`"
 end
@@ -31,9 +31,8 @@ VERSION_VARS = [
 
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
 
-  # Creates a devstack from a base Ubuntu 12.04 image for virtualbox
-  config.vm.box     = "precise64"
-  config.vm.box_url = "http://files.vagrantup.com/precise64.box"
+  # Creates a devstack from a base Ubuntu 16.04 image for virtualbox
+  config.vm.box = "boxcutter/ubuntu1604"
 
   config.vm.network :private_network, ip: vm_guest_ip, nic_type: "virtio"
 
@@ -72,18 +71,20 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
     vb.customize ["modifyvm", :id, "--natdnshostresolver1", "on"]
   end
 
-
   # Make LC_ALL default to en_US.UTF-8 instead of en_US.
   # See: https://github.com/mitchellh/vagrant/issues/1188
   config.vm.provision "shell", inline: 'echo \'LC_ALL="en_US.UTF-8"\' > /etc/default/locale'
 
+  # Get ready for ansible on this box.
+  config.vm.provision "shell", path: '../../../util/install/ansible-bootstrap.sh'
+
   # Use vagrant-vbguest plugin to make sure Guest Additions are in sync
   config.vbguest.auto_reboot = true
   config.vbguest.auto_update = true
 
   config.vm.provision :ansible do |ansible|
     ansible.playbook = "../../../playbooks/vagrant-devstack.yml"
-    ansible.verbose = "vvvv"
+    ansible.verbose = "vv"
 
     ansible.extra_vars = {}
     VERSION_VARS.each do |var|

vagrant/base/fullstack/Vagrantfile

-Vagrant.require_version ">= 1.5.3"
+Vagrant.require_version ">= 1.8.7"
 
 VAGRANTFILE_API_VERSION = "2"
 
 MEMORY = 4096
 CPU_COUNT = 2
 
+vm_guest_ip = "192.168.33.10"
+if ENV["VAGRANT_GUEST_IP"]
+  vm_guest_ip = ENV["VAGRANT_GUEST_IP"]
+end
+
 # These are versioning variables in the roles. Each can be overridden, first
 # with OPENEDX_RELEASE, and then with a specific environment variable of the
 # same name but upper-cased.
@@ -15,19 +20,24 @@ VERSION_VARS = [
     'forum_version',
     'xqueue_version',
     'demo_version',
+    'NOTIFIER_VERSION',
+    'ECOMMERCE_VERSION',
+    'ECOMMERCE_WORKER_VERSION',
+    'PROGRAMS_VERSION',
 ]
 
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
-  config.vm.box     = "precise64"
-  config.vm.box_url = "http://files.vagrantup.com/precise64.box"
+
+  # Creates a machine from a base Ubuntu 16.04 image for virtualbox
+  config.vm.box = "boxcutter/ubuntu1604"
+
+  config.vm.network :private_network, ip: vm_guest_ip, nic_type: "virtio"
+
   config.ssh.insert_key = true
-  config.vm.synced_folder  ".", "/vagrant", disabled: true
-  config.vm.network :private_network, ip: "192.168.33.10"
+  config.vm.synced_folder ".", "/vagrant", disabled: true
 
   config.vm.provider :virtualbox do |vb|
     vb.customize ["modifyvm", :id, "--memory", MEMORY.to_s]
-
-    # You can adjust this to the amount of CPUs your system has available
     vb.customize ["modifyvm", :id, "--cpus", CPU_COUNT.to_s]
 
     # Allow DNS to work for Ubuntu 12.10 host
@@ -41,8 +51,8 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
 
   config.vm.provision :ansible do |ansible|
     # point Vagrant at the location of your playbook you want to run
-    ansible.playbook = "../../../playbooks/vagrant-fullstack.yml"
-    ansible.verbose = "vvv"
+    ansible.playbook = "../../../playbooks/edx_sandbox.yml"
+    ansible.verbose = "vv"
 
     # Set extra-vars here instead of in the vagrant play so that
     # they are written out to /edx/etc/server-vars.yml which can

vagrant/release/devstack/Vagrantfile

-Vagrant.require_version ">= 1.6.5"
+Vagrant.require_version ">= 1.8.7"
 unless Vagrant.has_plugin?("vagrant-vbguest")
   raise "Please install the vagrant-vbguest plugin by running `vagrant plugin install vagrant-vbguest`"
 end
@@ -44,10 +44,14 @@ end
 # to a name and a file path, which are used for retrieving
 # a Vagrant box from the internet.
 openedx_releases = {
+  "open-release/ficus.master" => "ficus-devstack-2017-01-11",
+  "open-release/ficus.1rc1" => "ficus-devstack-2017-01-11",
+
   "open-release/eucalyptus.master" => "eucalyptus-devstack-2016-09-01",
   "open-release/eucalyptus.1rc2" => "eucalyptus-devstack-2016-08-19",
   "open-release/eucalyptus.1" => "eucalyptus-devstack-2016-08-19",
   "open-release/eucalyptus.2" => "eucalyptus-devstack-2016-09-01",
+
   "named-release/dogwood.rc" => "dogwood-devstack-2016-03-09",
   "named-release/dogwood.1" => "dogwood-devstack-2016-03-09",
   "named-release/dogwood.2" => "dogwood-devstack-2016-03-09",
@@ -56,17 +60,12 @@ openedx_releases = {
     :name => "dogwood-devstack-rc2", :file => "20151221-dogwood-devstack-rc2.box",
   },
   # Cypress is deprecated and unsupported
-  # "named-release/cypress" => {
-  #   :name => "cypress-devstack",
-  # },
   # Birch is deprecated and unsupported
-  # "named-release/birch.2" => {
-  #   :name => "birch-devstack-2", :file => "birch-2-devstack.box",
-  # },
 }
 openedx_releases.default = "eucalyptus-devstack-2016-09-01"
 
 openedx_release = ENV['OPENEDX_RELEASE']
+boxname = ENV['OPENEDX_BOXNAME']
 
 # Build -e override lines for each overridable variable.
 extra_vars_lines = ""
@@ -96,18 +95,24 @@ SCRIPT
 
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
 
-  reldata = openedx_releases[openedx_release]
-  if Hash == reldata.class
-    boxname = openedx_releases[openedx_release][:name]
-    boxfile = openedx_releases[openedx_release].fetch(:file, "#{boxname}.box")
-  else
-    boxname = reldata
+  boxfile = ""
+  if not boxname
+    reldata = openedx_releases[openedx_release]
+    if Hash == reldata.class
+      boxname = openedx_releases[openedx_release][:name]
+      boxfile = openedx_releases[openedx_release].fetch(:file, "")
+    else
+      boxname = reldata
+    end
+  end
+  if boxfile == ""
     boxfile = "#{boxname}.box"
   end
 
   # Creates an edX devstack VM from an official release
   config.vm.box     = boxname
   config.vm.box_url = "http://files.edx.org/vagrant-images/#{boxfile}"
+  config.vm.box_check_update = false
 
   config.vm.network :private_network, ip: "192.168.33.10"
 

vagrant/release/fullstack/Vagrantfile

-Vagrant.require_version ">= 1.5.3"
+Vagrant.require_version ">= 1.8.7"
 
 VAGRANTFILE_API_VERSION = "2"
 
@@ -41,26 +41,33 @@ openedx_releases = {
 openedx_releases.default = "eucalyptus-fullstack-2016-09-01"
 
 openedx_release = ENV['OPENEDX_RELEASE']
+boxname = ENV['OPENEDX_BOXNAME']
 
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
 
-  reldata = openedx_releases[openedx_release]
-  if Hash == reldata.class
-    boxname = openedx_releases[openedx_release][:name]
-    boxfile = openedx_releases[openedx_release].fetch(:file, "#{boxname}.box")
-  else
-    boxname = reldata
+  boxfile = ""
+  if not boxname
+    reldata = openedx_releases[openedx_release]
+    if Hash == reldata.class
+      boxname = openedx_releases[openedx_release][:name]
+      boxfile = openedx_releases[openedx_release].fetch(:file, "")
+    else
+      boxname = reldata
+    end
+  end
+  if boxfile == ""
     boxfile = "#{boxname}.box"
   end
 
   # Creates an edX fullstack VM from an official release
   config.vm.box     = boxname
   config.vm.box_url = "http://files.edx.org/vagrant-images/#{boxfile}"
+  config.vm.box_check_update = false
 
+  config.vm.network :private_network, ip: "192.168.33.10"
   config.vm.synced_folder  ".", "/vagrant", disabled: true
   config.ssh.insert_key = true
 
-  config.vm.network :private_network, ip: "192.168.33.10"
   config.hostsupdater.aliases = ["preview.localhost"]
 
   config.vm.provider :virtualbox do |vb|
@@ -70,5 +77,11 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
     # Allow DNS to work for Ubuntu 12.10 host
     # http://askubuntu.com/questions/238040/how-do-i-fix-name-service-for-vagrant-client
     vb.customize ["modifyvm", :id, "--natdnshostresolver1", "on"]
+
+    # Virtio is faster, but the box needs to have support for it.  We didn't
+    # have support in the boxes before Ficus.
+    if !(boxname.include?("dogwood") || boxname.include?("eucalyptus"))
+      vb.customize ['modifyvm', :id, '--nictype1', 'virtio']
+    end
   end
 end