Merge branch 'master' into arbab/devops-2784

AUTHORS

@@ -46,3 +46,4 @@ Kevin Falcone <kevin@edx.org>
 Max Rothman <max@edx.org>
 Andy Armstrong <andya@edx.org>
 Xiang Junfu <xiangjf.fnst@cn.fujitsu.com>
+Sarina Canelake <sarina@edx.org>
\ No newline at end of file

playbooks/roles/ecommerce/tasks/main.yml

@@ -38,7 +38,7 @@
 - name: create nodeenv
   shell: >
     creates={{ ecommerce_nodeenv_dir }}
-    {{ ecommerce_home }}/venvs/{{ ecommerce_service_name }}/bin/nodeenv {{ ecommerce_nodeenv_dir }}
+    {{ ecommerce_home }}/venvs/{{ ecommerce_service_name }}/bin/nodeenv {{ ecommerce_nodeenv_dir }} --prebuilt
   sudo_user: "{{ ecommerce_user }}"
 
 - name: install node dependencies
@@ -61,6 +61,23 @@
   environment: "{{ ecommerce_environment }}"
   when: migrate_db is defined and migrate_db|lower == "yes"
 
+- name: populate countries
+  shell: >
+    chdir={{ ecommerce_code_dir }}
+    DB_MIGRATION_USER={{ COMMON_MYSQL_MIGRATE_USER }}
+    DB_MIGRATION_PASS={{ COMMON_MYSQL_MIGRATE_PASS }}
+    {{ ecommerce_home }}/venvs/{{ ecommerce_service_name }}/bin/python ./manage.py oscar_populate_countries
+  sudo_user: "{{ ecommerce_user }}"
+  environment: "{{ ecommerce_environment }}"
+  when: migrate_db is defined and migrate_db|lower == "yes"
+  # the `register` and `failed_when` directives below are here to emulate idempotency for this oscar command.
+  # if and when https://github.com/django-oscar/django-oscar/pull/1841 is merged, the directives can be removed
+  # in favor of the (proposed) --initial-only command option.
+  register: command_result
+  failed_when:
+    - "'You already have countries in your database' not in command_result.stderr"
+    - "command_result.rc != 0"
+
 - name: run r.js optimizer
   shell: >
     chdir={{ ecommerce_code_dir }}

playbooks/roles/edx_ansible/templates/update.j2

@@ -39,7 +39,7 @@ if [[ -f {{ edx_ansible_var_file }} ]]; then
 fi
 
 {% if devstack %}
-extra_args="$extra_args -e 'disable_edx_services=true'"
+extra_args="$extra_args -e 'disable_edx_services=true' -e devstack=true"
 {% endif %}
 
 declare -A repos_to_cmd

playbooks/roles/edxapp/defaults/main.yml

@@ -71,6 +71,13 @@ EDXAPP_MYSQL_REPLICA_PORT: "{{ EDXAPP_MYSQL_PORT }}"
 EDXAPP_MYSQL_HOST: 'localhost'
 EDXAPP_MYSQL_PORT: '3306'
 
+# list of dictionaries of the format
+# { 'host': 'hostname', 'port': 'portnumber', 'otherconfigsuchas use_ssl': 'True' }
+# http://elasticsearch-py.readthedocs.org/en/master/api.html#elasticsearch
+EDXAPP_ELASTIC_SEARCH_CONFIG:
+    - host: "localhost"
+      port: 9200
+
 EDXAPP_LMS_ENV: 'lms.envs.aws'
 EDXAPP_CMS_ENV: 'cms.envs.aws'
 
@@ -96,6 +103,8 @@ EDXAPP_OEE_PASSWORD: 'password'
 
 EDXAPP_ANALYTICS_API_KEY:  ""
 EDXAPP_LTI_USER_EMAIL_DOMAIN: "lti.example.com"
+# 900s, or 15 mins
+EDXAPP_LTI_AGGREGATE_SCORE_PASSBACK_DELAY: 900
 EDXAPP_PAYMENT_SUPPORT_EMAIL: "billing@example.com"
 EDXAPP_YOUTUBE_API_KEY: "PUT_YOUR_API_KEY_HERE"
 EDXAPP_ZENDESK_USER: ""
@@ -241,6 +250,11 @@ EDXAPP_JWT_EXPIRATION: 30  # Number of seconds until expiration
 
 EDXAPP_PLATFORM_TWITTER_ACCOUNT: '@YourPlatformTwitterAccount'
 EDXAPP_PLATFORM_FACEBOOK_ACCOUNT: 'http://www.facebook.com/YourPlatformFacebookAccount'
+EDXAPP_FACEBOOK_APP_ID: "FACEBOOK_APP_ID"
+EDXAPP_FACEBOOK_APP_SECRET: "FACEBOOK_APP_SECRET"
+EDXAPP_FACEBOOK_API_VERSION: "v2.1"
+
+
 
 EDXAPP_SOCIAL_MEDIA_FOOTER_URLS: {}
 EDXAPP_MOBILE_STORE_URLS: {}
@@ -525,6 +539,10 @@ EDXAPP_COMP_THEME_DIR: ""
 EDXAPP_SOCIAL_AUTH_SAML_SP_PRIVATE_KEY: ''
 EDXAPP_SOCIAL_AUTH_SAML_SP_PUBLIC_CERT: ''
 
+# Session cookie setting
+# Only set this to true for client side profiling, never for production
+EDXAPP_SESSION_SAVE_EVERY_REQUEST: false
+
 #-------- Everything below this line is internal to the role ------------
 
 #Use YAML references (& and *) and hash merge <<: to factor out shared settings
@@ -686,6 +704,9 @@ edxapp_generic_auth_config:  &edxapp_generic_auth
   CREDIT_PROVIDER_SECRET_KEYS: "{{ EDXAPP_CREDIT_PROVIDER_SECRET_KEYS }}"
   SOCIAL_AUTH_SAML_SP_PRIVATE_KEY: "{{ EDXAPP_SOCIAL_AUTH_SAML_SP_PRIVATE_KEY }}"
   SOCIAL_AUTH_SAML_SP_PUBLIC_CERT: "{{ EDXAPP_SOCIAL_AUTH_SAML_SP_PUBLIC_CERT }}"
+  FACEBOOK_APP_ID: "{{ EDXAPP_FACEBOOK_APP_ID }}"
+  FACEBOOK_APP_SECRET: "{{ EDXAPP_FACEBOOK_APP_SECRET }}"
+  FACEBOOK_API_VERSION: "{{ EDXAPP_FACEBOOK_API_VERSION }}"
 
 generic_cache_config: &default_generic_cache
   BACKEND:  'django.core.cache.backends.memcached.MemcachedCache'
@@ -821,6 +842,7 @@ generic_env_config:  &edxapp_generic_env
   XBLOCK_SETTINGS: "{{ EDXAPP_XBLOCK_SETTINGS }}"
   EDXMKTG_USER_INFO_COOKIE_NAME: "{{ EDXAPP_EDXMKTG_USER_INFO_COOKIE_NAME }}"
   COMP_THEME_DIR: "{{ EDXAPP_COMP_THEME_DIR }}"
+  SESSION_SAVE_EVERY_REQUEST: "{{ EDXAPP_SESSION_SAVE_EVERY_REQUEST }}"
 
 lms_auth_config:
   <<: *edxapp_generic_auth
@@ -887,8 +909,10 @@ lms_env_config:
   EDXNOTES_PUBLIC_API: "{{ EDXAPP_EDXNOTES_PUBLIC_API }}"
   EDXNOTES_INTERNAL_API: "{{ EDXAPP_EDXNOTES_INTERNAL_API }}"
   LTI_USER_EMAIL_DOMAIN: "{{ EDXAPP_LTI_USER_EMAIL_DOMAIN }}"
+  LTI_AGGREGATE_SCORE_PASSBACK_DELAY: "{{ EDXAPP_LTI_AGGREGATE_SCORE_PASSBACK_DELAY }}"
   PROCTORING_SETTINGS: "{{ EDXAPP_PROCTORING_SETTINGS }}"
   CREDIT_HELP_LINK_URL: "{{ EDXAPP_CREDIT_HELP_LINK_URL }}"
+  ELASTIC_SEARCH_CONFIG: "{{ EDXAPP_ELASTIC_SEARCH_CONFIG }}"
 
 cms_auth_config:
   <<: *edxapp_generic_auth

playbooks/roles/flower/defaults/main.yml

@@ -23,7 +23,7 @@ flower_venv_dir: "{{ flower_app_dir }}/venvs/flower"
 flower_venv_bin: "{{ flower_venv_dir }}/bin"
 
 flower_python_reqs:
-  - "flower==0.8.1"
+  - "flower==0.8.3"
 
 flower_deploy_path: "{{ flower_venv_bin }}:/usr/local/sbin:/usr/local/bin:/usr/bin:/sbin:/bin"
 

playbooks/roles/insights/tasks/main.yml

@@ -40,7 +40,7 @@
 - name: create nodeenv
   shell: >
     creates={{ insights_nodeenv_dir }}
-    {{ insights_home }}/venvs/{{ insights_service_name }}/bin/nodeenv {{ insights_nodeenv_dir }}
+    {{ insights_home }}/venvs/{{ insights_service_name }}/bin/nodeenv {{ insights_nodeenv_dir }}  --prebuilt
   sudo_user: "{{ insights_user }}"
 
 - name: install node dependencies

playbooks/roles/jenkins_admin/defaults/main.yml

@@ -94,9 +94,7 @@ jenkins_admin_plugins:
   - { name: "matrix-auth", version: "1.2" }
   - { name: "matrix-project", version: "1.3" }
   - { name: "mailer", version: "1.9" }
-  - { name: "ldap", version: "1.10.2" }
   - { name: "pam-auth", version: "1.1" }
-  - { name: "ant", version: "1.2" }
   - { name: "build-user-vars-plugin", version: "1.3" }
   - { name: "credentials", version: "1.15" }
   - { name: "ssh-credentials", version: "1.7.1" }
@@ -105,7 +103,6 @@ jenkins_admin_plugins:
   - { name: "parameterized-trigger", version: "2.25" }
   - { name: "multiple-scms", version: "0.3" }
   - { name: "git", version: "2.2.2" }
-  - { name: "thinBackup", version: "1.7.4" }
   - { name: "maven-plugin", version: "2.5" }
   - { name: "copy-project-link", version: "1.2" }
   - { name: "scriptler", version: "2.6.1" }

playbooks/roles/mongo_3_0/defaults/main.yml

@@ -3,7 +3,7 @@ mongo_logappend: true
 #This way, when mongod receives a SIGUSR1, it'll close and reopen its log file handle
 mongo_logrotate: reopen
 
-mongo_version: 3.0.4
+mongo_version: 3.0.6
 mongo_port: "27017"
 mongo_extra_conf: ''
 mongo_key_file: '/etc/mongodb_key'
@@ -48,6 +48,10 @@ MONGO_BIND_IP: 127.0.0.1
 MONGO_STORAGE_ENGINE: "mmapv1"
 ##
 
+# WiredTiger takes a number of optional configuration settings
+# which can be defined as a yaml structure in your secure configuration.
+MONGO_STORAGE_ENGINE_OPTIONS: !!null
+
 mongo_logpath: "{{ mongo_log_dir }}/mongodb.log"
 mongo_dbpath: "{{ mongo_data_dir }}/mongodb"
 

playbooks/roles/mongo_3_0/tasks/main.yml

@@ -157,7 +157,7 @@
     login_user={{ MONGO_ADMIN_USER }}
     login_password={{ MONGO_ADMIN_PASSWORD }}
     name={{ item.user }}
-    password={{ item.password }}
+    password="{{ item.password }}"
     roles={{ item.roles }}
     state=present
   with_items: MONGO_USERS
@@ -169,7 +169,7 @@
     login_user={{ MONGO_ADMIN_USER }}
     login_password={{ MONGO_ADMIN_PASSWORD }}
     name={{ item.user }}
-    password={{ item.password }}
+    password="{{ item.password }}"
     roles={{ item.roles }}
     state=present
     replica_set={{ mongo_repl_set }}

playbooks/roles/mongo_3_0/templates/mongodb-clustered.conf.j2

@@ -14,6 +14,9 @@ storage:
 {% else %}
     enabled: false
 {% endif %}
+{% if MONGO_STORAGE_ENGINE_OPTIONS %}
+  {{ MONGO_STORAGE_ENGINE_OPTIONS | to_nice_yaml }}
+{% endif %}
 
 systemLog:
   #where to log

playbooks/roles/mongo_3_0/templates/mongodb-standalone.conf.j2

@@ -14,6 +14,9 @@ storage:
 {% else %}
     enabled: false
 {% endif %}
+{% if MONGO_STORAGE_ENGINE_OPTIONS %}
+  {{ MONGO_STORAGE_ENGINE_OPTIONS | to_nice_yaml }}
+{% endif %}
 
 systemLog:
   #where to log

playbooks/roles/nginx/templates/edx/app/nginx/sites-available/lms.j2

@@ -139,11 +139,17 @@ error_page {{ k }} {{ v }};
     try_files $uri @proxy_to_lms_app;
   }
 
-  # No basic auth security on oath2 endpoint
+  # No basic auth security on oauth2 endpoint
   location /oauth2 {
     try_files $uri @proxy_to_lms_app;
   }
 
+  # No basic auth security on third party auth endpoints
+  location /auth {
+    try_files $uri @proxy_to_lms_app;
+  }
+
+
   # No basic auth security on the heartbeat url, so that ELB can use it
   location /heartbeat {
     try_files $uri @proxy_to_lms_app;

playbooks/roles/notifier/defaults/main.yml

@@ -51,7 +51,7 @@ NOTIFIER_USER_SERVICE_API_KEY: "PUT_YOUR_API_KEY_HERE"
 NOTIFIER_USER_SERVICE_HTTP_AUTH_USER: ""
 NOTIFIER_USER_SERVICE_HTTP_AUTH_PASS: ""
 NOTIFIER_CELERY_BROKER_URL: "django://"
-NOTIFIER_LOGO_IMAGE_URL: "{{ NOTIFIER_LMS_URL_BASE }}/static/images/header-logo.png"
+NOTIFIER_LOGO_IMAGE_URL: "{{ NOTIFIER_LMS_URL_BASE }}/static/images/default-theme/logo.png"
 NOTIFIER_SUPERVISOR_LOG_DEST: "{{ COMMON_DATA_DIR }}/log/supervisor"
 
 NOTIFER_REQUESTS_CA_BUNDLE: "/etc/ssl/certs/ca-certificates.crt"

playbooks/roles/user/defaults/main.yml

@@ -35,4 +35,6 @@ user_rbash_links:
 # will take precedence over the paramter
 user_info: []
 
-
+user_debian_pkgs:
+  # Need this to get keys.
+  - curl

playbooks/roles/user/tasks/main.yml

@@ -74,6 +74,15 @@
 - name: create the edxadmin group
   group: name=edxadmin state=present
 
+- name: Install role-independent useful system packages
+  # do this before log dir setup; rsyslog package guarantees syslog user present
+  apt:
+    pkg: "{{ item }}"
+    install_recommends: yes
+    state: present
+    update_cache: yes
+  with_items: user_debian_pkgs
+
 # some AMIs (such as EMR master nodes) don't read the config files out of /etc/sudoers.d by default
 - name: ensure sudoers.d is read
   lineinfile: >

util/jenkins/ansible-provision.sh

@@ -119,6 +119,10 @@ if [[ -z $enable_datadog ]]; then
   enable_datadog="false"
 fi
 
+if [[ -z $enable_client_profiling ]]; then
+  enable_client_profiling="false"
+fi
+
 # Lowercase the dns name to deal with an ansible bug
 dns_name="${dns_name,,}"
 
@@ -173,6 +177,12 @@ EOF_AUTH
 
 fi
 
+if [[ $enable_client_profiling == "true" ]]; then
+    cat << EOF_PROFILING >> $extra_vars_file
+EDXAPP_SESSION_SAVE_EVERY_REQUEST: True
+EOF_PROFILING
+fi
+
 if [[ $edx_internal == "true" ]]; then
     # if this isn't a public server add the github
     # user and set edx_internal to True so that

vagrant/release/analyticstack/Vagrantfile

@@ -8,6 +8,19 @@ VAGRANTFILE_API_VERSION = "2"
 MEMORY = 4096
 CPU_COUNT = 2
 
+$script = <<SCRIPT
+if [ ! -d /edx/app/edx_ansible ]; then
+    echo "Error: Base box is missing provisioning scripts." 1>&2
+    exit 1
+fi
+export PYTHONUNBUFFERED=1
+source /edx/app/edx_ansible/venvs/edx_ansible/bin/activate
+cd /edx/app/edx_ansible/edx_ansible/playbooks
+
+ansible-playbook -i localhost, -c local vagrant-analytics.yml -e configuration_version=master
+
+SCRIPT
+
 edx_platform_mount_dir = "edx-platform"
 themes_mount_dir = "themes"
 forum_mount_dir = "cs_comments_service"
@@ -29,7 +42,8 @@ end
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
 
   # Creates an edX devstack VM from an official release
-  config.vm.box     = "analyticstack-beta"
+  config.vm.box     = "analyticstack"
+  config.vm.box_url = "http://files.edx.org/vagrant-images/analyticstack.box"
 
   config.vm.network :private_network, ip: "192.168.33.10"
   config.vm.network :forwarded_port, guest: 8000, host: 8000
@@ -90,4 +104,8 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   # Use vagrant-vbguest plugin to make sure Guest Additions are in sync
   config.vbguest.auto_reboot = true
   config.vbguest.auto_update = true
+
+  # Assume that the base box has the edx_ansible role installed
+  # We can then tell the Vagrant instance to update itself.
+  config.vm.provision "shell", inline: $script
 end

vagrant/release/devstack/Vagrantfile

-Vagrant.require_version ">= 1.5.3"
+Vagrant.require_version ">= 1.6.5"
 unless Vagrant.has_plugin?("vagrant-vbguest")
   raise "Please install the vagrant-vbguest plugin by running `vagrant plugin install vagrant-vbguest`"
 end
@@ -43,6 +43,7 @@ edx_platform_mount_dir = "edx-platform"
 themes_mount_dir = "themes"
 forum_mount_dir = "cs_comments_service"
 ora_mount_dir = "ora"
+ecommerce_mount_dir = "ecommerce"
 insights_mount_dir = "insights"
 analytics_api_mount_dir = "analytics_api"
 
@@ -52,6 +53,7 @@ if ENV['VAGRANT_MOUNT_BASE']
   themes_mount_dir = ENV['VAGRANT_MOUNT_BASE'] + "/" + themes_mount_dir
   forum_mount_dir = ENV['VAGRANT_MOUNT_BASE'] + "/" + forum_mount_dir
   ora_mount_dir = ENV['VAGRANT_MOUNT_BASE'] + "/" + ora_mount_dir
+  ecommerce_mount_dir = ENV['VAGRANT_MOUNT_BASE'] + "/" + ecommerce_mount_dir
   insights_mount_dir = ENV['VAGRANT_MOUNT_BASE'] + "/" + insights_mount_dir
   analytics_api_mount_dir = ENV['VAGRANT_MOUNT_BASE'] + "/" + analytics_api_mount_dir
 
@@ -124,11 +126,15 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   config.vm.box_url = "http://files.edx.org/vagrant-images/#{openedx_releases[rel][:file]}"
 
   config.vm.network :private_network, ip: "192.168.33.10"
-  config.vm.network :forwarded_port, guest: 8000, host: 8000
-  config.vm.network :forwarded_port, guest: 8001, host: 8001
-  config.vm.network :forwarded_port, guest: 18080, host: 18080
+  config.vm.network :forwarded_port, guest: 8000, host: 8000  # LMS
+  config.vm.network :forwarded_port, guest: 8001, host: 8001  # Studio
+  config.vm.network :forwarded_port, guest: 8002, host: 8002  # Ecommerce
+  config.vm.network :forwarded_port, guest: 8003, host: 8003  # LMS for Bok Choy
+  config.vm.network :forwarded_port, guest: 8031, host: 8031  # Studio for Bok Choy
+  config.vm.network :forwarded_port, guest: 8120, host: 8120  # edX Notes Service
   config.vm.network :forwarded_port, guest: 8765, host: 8765
-  config.vm.network :forwarded_port, guest: 9200, host: 9200
+  config.vm.network :forwarded_port, guest: 9200, host: 9200  # Elasticsearch
+  config.vm.network :forwarded_port, guest: 18080, host: 18080  # Forums
   config.vm.network :forwarded_port, guest: 8100, host: 8100  # Analytics Data API
   config.vm.network :forwarded_port, guest: 8110, host: 8110  # Insights
   config.vm.network :forwarded_port, guest: 50070, host: 50070  # HDFS Admin UI
@@ -149,6 +155,8 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
       create: true, owner: "edxapp", group: "www-data"
     config.vm.synced_folder "#{forum_mount_dir}", "/edx/app/forum/cs_comments_service",
       create: true, owner: "forum", group: "www-data"
+    config.vm.synced_folder "#{ecommerce_mount_dir}", "/edx/app/ecommerce/ecommerce",
+      create: true, owner: "ecommerce", group: "www-data"
     config.vm.synced_folder "#{ora_mount_dir}", "/edx/app/ora/ora",
       create: true, owner: "ora", group: "www-data"
     config.vm.synced_folder "#{insights_mount_dir}", "/edx/app/insights/edx_analytics_dashboard",
@@ -162,6 +170,8 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
       create: true, nfs: true
     config.vm.synced_folder "#{forum_mount_dir}", "/edx/app/forum/cs_comments_service",
       create: true, nfs: true
+    config.vm.synced_folder "#{ecommerce_mount_dir}", "/edx/app/ecommerce/ecommerce",
+      create: true, nfs: true
     config.vm.synced_folder "#{ora_mount_dir}", "/edx/app/ora/ora",
       create: true, nfs: true
     config.vm.synced_folder "#{insights_mount_dir}", "/edx/app/insights/edx_analytics_dashboard",