Merge branch 'master' into proversity/course-visibility-in-catalog

CHANGELOG.md

+- Role: edxapp
+  - Added `EDXAPP_LMS_INTERNAL_ROOT_URL` setting (defaults to `EDXAPP_LMS_ROOT_URL`).
+
+- Role: edxapp
+  - Added `EDXAPP_CELERY_BROKER_TRANSPORT` and renamed `EDXAPP_RABBIT_HOSTNAME`
+    to `EDXAPP_CELERY_BROKER_HOSTNAME`. This is to support non-amqp brokers,
+    specifically redis. If `EDXAPP_CELERY_BROKER_HOSTNAME` is unset it will use
+    the value of `EDXAPP_RABBIT_HOSTNAME`, however it is recommended to update
+    your configuration to set `EDXAPP_CELERY_BROKER_TRANSPORT` explicitly.
+
+- Role: edxapp
+  - Added `EDXAPP_LMS_SPLIT_DOC_STORE_READ_PREFERENCE` with a default value of
+    SECONDARY_PREFERED to distribute read workload across the replica set.
+  - Changed `EDXAPP_MONGO_HOSTS` to be a comma seperated string, which is
+    required by pymongo.MongoReplicaSetClient for multiple hosts instead of an
+    array.
+  - Added `EDXAPP_MONGO_REPLICA_SET`, which is required to use
+    pymongo.MongoReplicaSetClient in PyMongo 2.9.1, whis is required to use the
+    read_preference setting. This should be set to the name of your replica set.
+
+- Role: nginx
+  - Modified `lms.j2` , `cms.j2` , `credentials.j2` , `edx_notes_api.j2` and `insights.j2` to enable HTTP Strict Transport Security
+  - Added `NGINX_HSTS_MAX_AGE` to make HSTS header `max_age` value configurable and used in templates
+
+- Role: server_utils
+  - Install "vim", not "vim-tiny".
+
+- Role: edxapp
+  - Added GOOGLE_ANALYTICS_TRACKING_ID setting for inserting GA tracking into emails generated via ACE.
+
+- Role: notifier
+  - Added notifier back to continuous integration.
+
 - Role: ecommerce
   - This role is now dependent on the edx_django_service role. Settings are all the same, but nearly all of the tasks are performed by the edx_django_service role.
 
@@ -25,6 +58,9 @@
   - Added `EDXAPP_PLATFORM_DESCRIPTION` used to describe the specific Open edX platform.
 
 - Role: edxapp
+  - Added `EDXAPP_REINDEX_ALL_COURSES` to rebuild the course index on deploy. Disabled by default.
+
+- Role: edxapp
   - Added `ENTERPRISE_SUPPORT_URL` variable used by the LMS.
 
 - Role: edxapp
@@ -426,3 +462,6 @@
 - Role: edxapp
   - Added `EDXAPP_VIDEO_TRANSCRIPTS_SETTINGS` to configure S3-backed video transcripts.
   - Removed unused `EDXAPP_BOOK_URL` setting
+
+- Role: edxapp
+  - Added `EDXAPP_ZENDESK_OAUTH_ACCESS_TOKEN` for making requests to Zendesk through front-end.

playbooks/edx-east/edx_continuous_integration.yml

@@ -29,9 +29,7 @@
     - oraclejdk
     - elasticsearch
     - forum
-# Removing until Notifier is fully fixed.
-# Can be uncommented once EDUCATOR-1594 has been resolved.
-#    - { role: notifier, NOTIFIER_DIGEST_TASK_INTERVAL: "5" }
+    - { role: notifier, NOTIFIER_DIGEST_TASK_INTERVAL: "5" }
     - { role: "xqueue", update_users: True }
     - role: xserver
       when: XSERVER_GIT_IDENTITY|length > 0

playbooks/edx-east/edxapp_migrate.yml

-- name: Run edxapp migrations
-  hosts: all
-  become: False
-  gather_facts: False
-  vars:
-    db_dry_run: "--list"
-  roles:
-    - edxapp
-  tasks:
-    - name: migrate lms
-      shell: "python manage.py lms migrate --database {{ item }} --noinput {{ db_dry_run }} --settings=aws"
-      args:
-        chdir: "{{ edxapp_code_dir }}"
-      environment:
-        DB_MIGRATION_USER: "{{ COMMON_MYSQL_MIGRATE_USER }}"
-        DB_MIGRATION_PASS: "{{ COMMON_MYSQL_MIGRATE_PASS }}"
-      # Migrate any database in the config, but not the read_replica
-      when: item != 'read_replica'
-      with_items:
-        - "{{ lms_auth_config.DATABASES.keys() }}"
-      tags:
-        - always
-    - name: migrate cms
-      shell: "python manage.py cms migrate --database {{ item }} --noinput {{ db_dry_run }} --settings=aws"
-      args:
-        chdir: "{{ edxapp_code_dir }}"
-      environment:
-        DB_MIGRATION_USER: "{{ COMMON_MYSQL_MIGRATE_USER }}"
-        DB_MIGRATION_PASS: "{{ COMMON_MYSQL_MIGRATE_PASS }}"
-      # Migrate any database in the config, but not the read_replica
-      when: item != 'read_replica'
-      with_items:
-        - "{{ cms_auth_config.DATABASES.keys() }}"
-      tags:
-        - always

playbooks/edx-east/jenkins_build.yml

@@ -18,12 +18,12 @@
     SECURITY_UPGRADE_ON_ANSIBLE: true
 
     SPLUNKFORWARDER_LOG_ITEMS:
-      - source: '/var/lib/jenkins/jobs/*/builds/*/junitResult.xml'
+      - source: '/var/lib/jenkins/jobs/edx-platform-*/builds/*/junitResult.xml'
         recursive: true
         index: 'testeng'
         sourcetype: junit
         followSymlink: false
-        blacklist: '\.gz$'
+        blacklist: coverage|private|subset|specific|custom|special|\.gz$
         crcSalt: '<SOURCE>'
 
       - source: '/var/lib/jenkins/jobs/*/builds/*/build.xml'
@@ -34,7 +34,7 @@
         crcSalt: '<SOURCE>'
         blacklist: '\.gz$'
 
-      - source: '/var/lib/jenkins/jobs/edx-platform-*/builds/*/archive/test_root/log/timing.*.log'
+      - source: '/var/lib/jenkins/jobs/edx-platform-*/builds/*/archive/.../test_root/log/timing.*.log'
         index: 'testeng'
         recursive: true
         sourcetype: 'json_timing_log'

playbooks/edx-east/jenkins_de.yml

+---
+- name: Bootstrap instance(s)
+  hosts: all
+  gather_facts: no
+  become: True
+  roles:
+    - python
+
+- name: Configure instance(s)
+  hosts: all
+  become: True
+  gather_facts: True
+  vars:
+    COMMON_ENABLE_DATADOG: True
+    COMMON_ENABLE_SPLUNKFORWARDER: False
+    COMMON_SECURITY_UPDATES: yes
+    SECURITY_UPGRADE_ON_ANSIBLE: true
+
+  roles:
+    - aws
+    - role: datadog
+      when: COMMON_ENABLE_DATADOG
+    - jenkins_de

playbooks/edx-east/jenkins_worker.yml

@@ -24,7 +24,7 @@
     - mysql
     - edxlocal
     - memcache
-    - mongo
+    - mongo_3_2
     - browsers
     - browsermob-proxy
     - jenkins_worker

playbooks/roles/automated/meta/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://openedx.atlassian.net/wiki/display/OpenOPS
+# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+
+# Allow this role to be duplicated in dependencies.
+allow_duplicates: yes

playbooks/roles/certs/tasks/deploy.yml

@@ -10,6 +10,14 @@
     - { src: 'certs.env.json.j2', dest: 'env.json' }
     - { src: 'certs.auth.json.j2', dest: 'auth.json' }
 
+- name: Copy the boto file
+  template:
+    src: "boto.j2"
+    dest: "{{ certs_app_dir }}/.boto"
+    owner: "{{ certs_user }}"
+    group: "{{ common_web_user }}"
+    mode: 0644
+
 - name: Writing supervisor script for certificates
   template:
     src: certs.conf.j2

playbooks/roles/certs/templates/certs.conf.j2

 [program:certs]
 command={{ certs_venv_bin }}/python {{ certs_code_dir }}/certificate_agent.py
 priority=999
-environment=SERVICE_VARIANT="certs",HOME="/"
+environment=SERVICE_VARIANT="certs",HOME="/",BOTO_CONFIG="{{ certs_app_dir }}/.boto"
 user={{ common_web_user }}
 stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log
 stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log

playbooks/roles/credentials/defaults/main.yml

@@ -123,7 +123,7 @@ CREDENTIALS_CORS_ORIGIN_WHITELIST: "{{ CREDENTIALS_CORS_ORIGIN_WHITELIST_DEFAULT
 
 CREDENTIALS_CERTIFICATE_LANGUAGES:
   'en': 'English'
-  'es_419': 'Español'
+  'es_419': 'Spanish'
 
 CREDENTIALS_VERSION: "master"
 CREDENTIALS_REPOS:

playbooks/roles/ecommerce/defaults/main.yml

@@ -135,11 +135,12 @@ ECOMMERCE_PAYMENT_PROCESSOR_CONFIG:
 ECOMMERCE_PLATFORM_NAME: 'Your Platform Name Here'
 ECOMMERCE_THEME_SCSS: 'sass/themes/default.scss'
 ECOMMERCE_COMPREHENSIVE_THEME_DIRS:
-  - '{{ THEMES_CODE_DIR }}'
+  - '{{ THEMES_CODE_DIR }}/{{ ecommerce_service_name }}'
   - '{{ COMMON_APP_DIR }}/{{ ecommerce_service_name }}/{{ ecommerce_service_name }}/ecommerce/themes'
 
 ECOMMERCE_ENABLE_COMPREHENSIVE_THEMING: false
 ECOMMERCE_DEFAULT_SITE_THEME: !!null
+ECOMMERCE_STATICFILES_STORAGE: 'ecommerce.theming.storage.ThemeStorage'
 
 # Celery
 ECOMMERCE_BROKER_USERNAME: 'celery'

playbooks/roles/ecommerce/meta/main.yml

@@ -35,6 +35,7 @@ dependencies:
     edx_django_service_social_auth_edx_oidc_secret: '{{ ECOMMERCE_SOCIAL_AUTH_EDX_OIDC_SECRET }}'
     edx_django_service_social_auth_redirect_is_https: '{{ ECOMMERCE_SOCIAL_AUTH_REDIRECT_IS_HTTPS }}'
     edx_django_service_session_expire_at_browser_close: '{{ ECOMMERCE_SESSION_EXPIRE_AT_BROWSER_CLOSE }}'
+    edx_django_service_staticfiles_storage: '{{ ECOMMERCE_STATICFILES_STORAGE }}'
     edx_django_service_post_migrate_commands: '{{ ecommerce_post_migrate_commands }}'
     edx_django_service_basic_auth_exempted_paths_extra:
       - payment

playbooks/roles/edx_service/meta/main.yml

@@ -30,3 +30,6 @@ dependencies:
     GIT_REPOS: "{{ edx_service_repos }}"
     git_home: "{{ edx_service_home }}"
     when: edx_service_repos is defined
+
+# Allow this role to be duplicated in dependencies.
+allow_duplicates: yes

playbooks/roles/edxapp/defaults/main.yml

@@ -65,12 +65,16 @@ EDXAPP_XQUEUE_DJANGO_AUTH:
   password: 'password'
 EDXAPP_XQUEUE_URL: 'http://localhost:18040'
 
-EDXAPP_MONGO_HOSTS: ['localhost']
+# EDXAPP_MONGO_HOSTS must be a comma seperated list of hosts/ips for
+# compatibility with pymongo.MongoReplicaSetClient in PyMongo 2.9.1
+EDXAPP_MONGO_HOSTS: 'localhost'
 EDXAPP_MONGO_PASSWORD: 'password'
 EDXAPP_MONGO_PORT: 27017
 EDXAPP_MONGO_USER: 'edxapp'
 EDXAPP_MONGO_DB_NAME: 'edxapp'
 EDXAPP_MONGO_USE_SSL: False
+EDXAPP_MONGO_REPLICA_SET: ''
+EDXAPP_LMS_SPLIT_DOC_STORE_READ_PREFERENCE: 'SECONDARY_PREFERRED'
 
 EDXAPP_MYSQL_DB_NAME: 'edxapp'
 EDXAPP_MYSQL_USER: 'edxapp001'
@@ -145,8 +149,11 @@ EDXAPP_ZENDESK_USER: ""
 EDXAPP_ZENDESK_URL: ""
 EDXAPP_ZENDESK_API_KEY: ""
 EDXAPP_ZENDESK_CUSTOM_FIELDS: {}
+EDXAPP_ZENDESK_OAUTH_ACCESS_TOKEN: ""
 EDXAPP_CELERY_USER: 'celery'
 EDXAPP_CELERY_PASSWORD: 'celery'
+EDXAPP_CELERY_BROKER_HOSTNAME: "{{ EDXAPP_RABBIT_HOSTNAME }}"
+EDXAPP_CELERY_BROKER_TRANSPORT: 'amqp'
 EDXAPP_CELERY_BROKER_VHOST: ""
 EDXAPP_CELERY_BROKER_USE_SSL: false
 EDXAPP_CELERY_EVENT_QUEUE_TTL: "None"
@@ -287,7 +294,6 @@ EDXAPP_LMS_SITE_NAME: "{{ EDXAPP_SITE_NAME }}"
 EDXAPP_CMS_SITE_NAME: 'localhost'
 EDXAPP_MEDIA_URL:  "/media"
 EDXAPP_FEEDBACK_SUBMISSION_EMAIL: ""
-EDXAPP_CELERY_BROKER_HOSTNAME: ""
 EDXAPP_LOGGING_ENV:  'sandbox'
 
 EDXAPP_SYSLOG_SERVER: ""
@@ -329,6 +335,7 @@ EDXAPP_BULK_EMAIL_LOG_SENT_EMAILS: false
 EDXAPP_UNIVERSITY_EMAIL: 'university@example.com'
 EDXAPP_PRESS_EMAIL: 'press@example.com'
 EDXAPP_LMS_ROOT_URL: "{{ EDXAPP_LMS_BASE_SCHEME | default('https') }}://{{ EDXAPP_LMS_BASE }}"
+EDXAPP_LMS_INTERNAL_ROOT_URL: "{{ EDXAPP_LMS_ROOT_URL }}"
 EDXAPP_LMS_ISSUER: "{{ COMMON_JWT_ISSUER }}"
 EDXAPP_JWT_EXPIRATION: 30  # Number of seconds until expiration
 EDXAPP_JWT_AUDIENCE: "{{ COMMON_JWT_AUDIENCE }}"
@@ -454,6 +461,7 @@ EDXAPP_VERIFY_STUDENT:
   DAYS_GOOD_FOR: 365
   EXPIRING_SOON_WINDOW: 28
 EDXAPP_GOOGLE_ANALYTICS_LINKEDIN: ""
+EDXAPP_GOOGLE_ANALYTICS_TRACKING_ID: ""
 EDXAPP_CONTENTSTORE_ADDITIONAL_OPTS: {}
 EDXAPP_BULK_EMAIL_EMAILS_PER_TASK: 500
 # If using microsites this should point to the microsite repo
@@ -465,6 +473,9 @@ EDXAPP_COURSES_WITH_UNSAFE_CODE: []
 EDXAPP_SESSION_COOKIE_DOMAIN: ""
 EDXAPP_SESSION_COOKIE_NAME: "sessionid"
 
+# Whether to run reindex_course on deploy
+EDXAPP_REINDEX_ALL_COURSES: false
+
 # XML Course related flags
 EDXAPP_XML_FROM_GIT: false
 EDXAPP_XML_S3_BUCKET: !!null
@@ -588,6 +599,7 @@ EDXAPP_CREDENTIALS_PUBLIC_SERVICE_URL: "http://localhost:8005"
 EDXAPP_COURSE_CATALOG_VISIBILITY_PERMISSION: 'see_exists'
 EDXAPP_COURSE_ABOUT_VISIBILITY_PERMISSION: 'see_exists'
 EDXAPP_DEFAULT_COURSE_VISIBILITY_IN_CATALOG: 'both'
+EDXAPP_DEFAULT_MOBILE_AVAILABLE: false
 
 # Mailchimp Settings
 EDXAPP_MAILCHIMP_NEW_USER_LIST_ID: null
@@ -728,7 +740,7 @@ EDXAPP_BLOCK_STRUCTURES_SETTINGS:
   TASK_MAX_RETRIES: 5
 
 # Configuration settings needed for the LMS to communicate with the Enterprise service.
-EDXAPP_ENTERPRISE_API_URL: "{{ EDXAPP_LMS_ROOT_URL }}/enterprise/api/v1"
+EDXAPP_ENTERPRISE_API_URL: "{{ EDXAPP_LMS_INTERNAL_ROOT_URL }}/enterprise/api/v1"
 
 EDXAPP_ENTERPRISE_SERVICE_WORKER_EMAIL: "enterprise_worker@example.com"
 EDXAPP_ENTERPRISE_SERVICE_WORKER_USERNAME: "enterprise_worker"
@@ -737,7 +749,7 @@ EDXAPP_ENTERPRISE_COURSE_ENROLLMENT_AUDIT_MODES:
   - audit
   - honor
 
-EDXAPP_ENTERPRISE_ENROLLMENT_API_URL: "{{ EDXAPP_LMS_ROOT_URL }}/api/enrollment/v1/"
+EDXAPP_ENTERPRISE_ENROLLMENT_API_URL: "{{ EDXAPP_LMS_INTERNAL_ROOT_URL }}/api/enrollment/v1/"
 
 # The default value of this needs to be a 16 character string
 EDXAPP_ENTERPRISE_REPORTING_SECRET: '0000000000000000'
@@ -834,6 +846,7 @@ edxapp_environment_default:
   # be updated to /edx/etc/edxapp when the switch to
   # yaml based configs is complete
   CONFIG_ROOT: "{{ edxapp_app_dir }}"
+  BOTO_CONFIG: "{{ edxapp_app_dir }}/.boto"
 
 edxapp_environment_extra: {}
 
@@ -858,6 +871,8 @@ EDXAPP_LMS_DRAFT_DOC_STORE_CONFIG:
 
 EDXAPP_LMS_SPLIT_DOC_STORE_CONFIG:
   <<: *edxapp_generic_default_docstore
+  replicaSet: "{{ EDXAPP_MONGO_REPLICA_SET }}"
+  read_preference: "{{ EDXAPP_LMS_SPLIT_DOC_STORE_READ_PREFERENCE }}"
 
 EDXAPP_CMS_DOC_STORE_CONFIG:
   <<: *edxapp_generic_default_docstore
@@ -941,6 +956,7 @@ edxapp_generic_auth_config:  &edxapp_generic_auth
   YOUTUBE_API_KEY: "{{ EDXAPP_YOUTUBE_API_KEY }}"
   ZENDESK_USER: "{{ EDXAPP_ZENDESK_USER }}"
   ZENDESK_API_KEY: "{{ EDXAPP_ZENDESK_API_KEY }}"
+  ZENDESK_OAUTH_ACCESS_TOKEN: "{{ EDXAPP_ZENDESK_OAUTH_ACCESS_TOKEN }}"
   CELERY_BROKER_USER: "{{ EDXAPP_CELERY_USER }}"
   CELERY_BROKER_PASSWORD: "{{ EDXAPP_CELERY_PASSWORD }}"
   GOOGLE_ANALYTICS_ACCOUNT: "{{ EDXAPP_GOOGLE_ANALYTICS_ACCOUNT }}"
@@ -967,6 +983,7 @@ generic_env_config:  &edxapp_generic_env
   COURSE_CATALOG_VISIBILITY_PERMISSION: "{{ EDXAPP_COURSE_CATALOG_VISIBILITY_PERMISSION }}"
   COURSE_ABOUT_VISIBILITY_PERMISSION: "{{ EDXAPP_COURSE_ABOUT_VISIBILITY_PERMISSION }}"
   DEFAULT_COURSE_VISIBILITY_IN_CATALOG: "{{ EDXAPP_DEFAULT_COURSE_VISIBILITY_IN_CATALOG }}"
+  DEFAULT_MOBILE_AVAILABLE: "{{ EDXAPP_DEFAULT_MOBILE_AVAILABLE }}"
   FINANCIAL_REPORTS: "{{ EDXAPP_FINANCIAL_REPORTS }}"
   ONLOAD_BEACON_SAMPLE_RATE: "{{ EDXAPP_ONLOAD_BEACON_SAMPLE_RATE }}"
   CORS_ORIGIN_WHITELIST: "{{ EDXAPP_CORS_ORIGIN_WHITELIST }}"
@@ -1002,6 +1019,7 @@ generic_env_config:  &edxapp_generic_env
   LMS_BASE: "{{ EDXAPP_LMS_BASE }}"
   CMS_BASE: "{{ EDXAPP_CMS_BASE }}"
   LMS_ROOT_URL: "{{ EDXAPP_LMS_ROOT_URL }}"
+  LMS_INTERNAL_ROOT_URL: "{{ EDXAPP_LMS_INTERNAL_ROOT_URL }}"
   PARTNER_SUPPORT_EMAIL: "{{ EDXAPP_PARTNER_SUPPORT_EMAIL }}"
   PLATFORM_NAME: "{{ EDXAPP_PLATFORM_NAME }}"
   PLATFORM_DESCRIPTION: "{{ EDXAPP_PLATFORM_DESCRIPTION }}"
@@ -1083,8 +1101,8 @@ generic_env_config:  &edxapp_generic_env
       LOCATION: "{{ EDXAPP_CACHE_COURSE_STRUCTURE_MEMCACHE }}"
       # Default to two hours
       TIMEOUT: "7200"
-  CELERY_BROKER_TRANSPORT: 'amqp'
-  CELERY_BROKER_HOSTNAME: "{{ EDXAPP_RABBIT_HOSTNAME }}"
+  CELERY_BROKER_TRANSPORT: "{{ EDXAPP_CELERY_BROKER_TRANSPORT }}"
+  CELERY_BROKER_HOSTNAME: "{{ EDXAPP_CELERY_BROKER_HOSTNAME }}"
   COMMENTS_SERVICE_URL: "{{ EDXAPP_COMMENTS_SERVICE_URL }}"
   LOGGING_ENV: "{{ EDXAPP_LOGGING_ENV }}"
   SESSION_COOKIE_DOMAIN:  "{{ EDXAPP_SESSION_COOKIE_DOMAIN }}"
@@ -1155,6 +1173,7 @@ lms_auth_config:
   EDX_API_KEY: "{{ EDXAPP_EDX_API_KEY }}"
   VERIFY_STUDENT: "{{ EDXAPP_VERIFY_STUDENT }}"
   GOOGLE_ANALYTICS_LINKEDIN: "{{ EDXAPP_GOOGLE_ANALYTICS_LINKEDIN }}"
+  GOOGLE_ANALYTICS_TRACKING_ID: "{{ EDXAPP_GOOGLE_ANALYTICS_TRACKING_ID }}"
   CC_PROCESSOR_NAME: "{{ EDXAPP_CC_PROCESSOR_NAME }}"
   CC_PROCESSOR: "{{ EDXAPP_CC_PROCESSOR }}"
   TRACKING_SEGMENTIO_WEBHOOK_SECRET: "{{ EDXAPP_TRACKING_SEGMENTIO_WEBHOOK_SECRET }}"

playbooks/roles/edxapp/tasks/deploy.yml

@@ -20,6 +20,7 @@
     - { src: 'edxapp_env.j2', dest: '{{ edxapp_app_dir }}/edxapp_env', owner: '{{ edxapp_user }}', group: '{{ common_web_user }}', mode: '0644' }
     - { src: 'newrelic.ini.j2', dest: '{{ edxapp_app_dir }}/newrelic.ini', owner: '{{ edxapp_user }}', group: '{{ common_web_user }}', mode: '0644' }
     - { src: 'git_ssh.sh.j2', dest: '{{ edxapp_git_ssh }}', owner: '{{ edxapp_user }}', group: '{{ edxapp_user }}', mode: '0750' }
+    - { src: 'boto.j2', dest: '{{ edxapp_app_dir }}/.boto', owner: '{{ edxapp_user }}', group: '{{ common_web_user }}', mode: '0644' }
   tags:
     - install
     - install:base
@@ -428,3 +429,13 @@
   tags:
     - manage
     - manage:db
+
+- name: reindex all courses
+  shell: "{{ edxapp_venv_bin }}/python ./manage.py cms reindex_course --setup --settings={{ edxapp_settings }}"
+  args:
+    chdir: "{{ edxapp_code_dir }}"
+  become_user: "{{ common_web_user }}"
+  when: EDXAPP_REINDEX_ALL_COURSES
+  tags:
+    - install
+    - install:base

playbooks/roles/edxapp/templates/cms.conf.j2

@@ -10,7 +10,7 @@ command={{ executable }} -c {{ edxapp_app_dir }}/cms_gunicorn.py {{ EDXAPP_CMS_G
 
 user={{ common_web_user }}
 directory={{ edxapp_code_dir }}
-environment={% if COMMON_ENABLE_NEWRELIC_APP %}NEW_RELIC_APP_NAME={{ EDXAPP_NEWRELIC_CMS_APPNAME }},NEW_RELIC_LICENSE_KEY={{ NEWRELIC_LICENSE_KEY }},{% endif -%}PORT={{ edxapp_cms_gunicorn_port }},ADDRESS={{ edxapp_cms_gunicorn_host }},LANG={{ EDXAPP_LANG }},DJANGO_SETTINGS_MODULE={{ EDXAPP_CMS_ENV }},SERVICE_VARIANT="cms"
+environment={% if COMMON_ENABLE_NEWRELIC_APP %}NEW_RELIC_APP_NAME={{ EDXAPP_NEWRELIC_CMS_APPNAME }},NEW_RELIC_LICENSE_KEY={{ NEWRELIC_LICENSE_KEY }},{% endif -%}PORT={{ edxapp_cms_gunicorn_port }},ADDRESS={{ edxapp_cms_gunicorn_host }},LANG={{ EDXAPP_LANG }},DJANGO_SETTINGS_MODULE={{ EDXAPP_CMS_ENV }},SERVICE_VARIANT="cms",BOTO_CONFIG="{{ edxapp_app_dir }}/.boto"
 stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log
 stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log
 killasgroup=true

playbooks/roles/edxapp/templates/lms.conf.j2

@@ -10,7 +10,7 @@ command={{ executable }} -c {{ edxapp_app_dir }}/lms_gunicorn.py lms.wsgi
 
 user={{ common_web_user }}
 directory={{ edxapp_code_dir }}
-environment={% if COMMON_ENABLE_NEWRELIC_APP %}NEW_RELIC_APP_NAME={{ EDXAPP_NEWRELIC_LMS_APPNAME }},NEW_RELIC_LICENSE_KEY={{ NEWRELIC_LICENSE_KEY }},NEW_RELIC_CONFIG_FILE={{ edxapp_app_dir }}/newrelic.ini,{% endif -%}  PORT={{ edxapp_lms_gunicorn_port }},ADDRESS={{ edxapp_lms_gunicorn_host }},LANG={{ EDXAPP_LANG }},DJANGO_SETTINGS_MODULE={{ EDXAPP_LMS_ENV }},SERVICE_VARIANT="lms",PATH="{{ edxapp_deploy_path }}"
+environment={% if COMMON_ENABLE_NEWRELIC_APP %}NEW_RELIC_APP_NAME={{ EDXAPP_NEWRELIC_LMS_APPNAME }},NEW_RELIC_LICENSE_KEY={{ NEWRELIC_LICENSE_KEY }},NEW_RELIC_CONFIG_FILE={{ edxapp_app_dir }}/newrelic.ini,{% endif -%}  PORT={{ edxapp_lms_gunicorn_port }},ADDRESS={{ edxapp_lms_gunicorn_host }},LANG={{ EDXAPP_LANG }},DJANGO_SETTINGS_MODULE={{ EDXAPP_LMS_ENV }},SERVICE_VARIANT="lms",PATH="{{ edxapp_deploy_path }}",BOTO_CONFIG="{{ edxapp_app_dir }}/.boto"
 stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log
 stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log
 killasgroup=true

playbooks/roles/edxapp/templates/workers.conf.j2

 {% for w in edxapp_workers %}
 [program:{{ w.service_variant }}_{{ w.queue }}_{{ w.concurrency }}]
 
-environment={% if COMMON_ENABLE_NEWRELIC_APP %}NEW_RELIC_APP_NAME={{ EDXAPP_NEWRELIC_WORKERS_APPNAME }},NEW_RELIC_LICENSE_KEY={{ NEWRELIC_LICENSE_KEY }},{% endif -%}CONCURRENCY={{ w.concurrency }},LOGLEVEL=info,DJANGO_SETTINGS_MODULE={{ worker_django_settings_module }},LANG={{ EDXAPP_LANG }},PYTHONPATH={{ edxapp_code_dir }},SERVICE_VARIANT={{ w.service_variant }}
+environment={% if COMMON_ENABLE_NEWRELIC_APP %}NEW_RELIC_APP_NAME={{ EDXAPP_NEWRELIC_WORKERS_APPNAME }},NEW_RELIC_LICENSE_KEY={{ NEWRELIC_LICENSE_KEY }},{% endif -%}CONCURRENCY={{ w.concurrency }},LOGLEVEL=info,DJANGO_SETTINGS_MODULE={{ worker_django_settings_module }},LANG={{ EDXAPP_LANG }},PYTHONPATH={{ edxapp_code_dir }},SERVICE_VARIANT={{ w.service_variant }},BOTO_CONFIG="{{ edxapp_app_dir }}/.boto"
 user={{ common_web_user }}
 directory={{ edxapp_code_dir }}
 stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log

playbooks/roles/jenkins_build/defaults/main.yml

@@ -65,7 +65,7 @@ build_jenkins_plugins_list:
     version: '2.5'
     group: 'org.jenkins-ci.plugins'
   - name: 'junit'
-    version: '1.3'
+    version: '1.21'
     group: 'org.jenkins-ci.plugins'
   - name: 'pam-auth'
     version: '1.2'
@@ -107,7 +107,7 @@ build_jenkins_plugins_list:
     version: '1.5'
     group: 'org.jenkins-ci.plugins'
   - name: 'cobertura'
-    version: '1.9.6'
+    version: '1.11'
     group: 'org.jenkins-ci.plugins'
   - name: 'copyartifact'
     version: '1.32.1'
@@ -136,6 +136,9 @@ build_jenkins_plugins_list:
   - name: 'github-oauth'
     version: '0.24'
     group: 'org.jenkins-ci.plugins'
+  - name: 'github-api'
+    version: '1.90'
+    group: 'org.jenkins-ci.plugins'
   - name: 'gradle'
     version: '1.24'
     group: 'org.jenkins-ci.plugins'

playbooks/roles/jenkins_de/defaults/main.yml

+de_jenkins_user_uid: 1002
+de_jenkins_group_gid: 1004
+de_jenkins_version: jenkins-2.73.2
+de_jenkins_common_war_source: https://edx-analytics-public.s3.amazonaws.com/packages
+de_jenkins_jvm_args: '-Djava.awt.headless=true -Xmx8192m -Djenkins.install.runSetupWizard=false'
+de_jenkins_configuration_scripts:
+  - 1addJarsToClasspath.groovy
+  - 2checkInstalledPlugins.groovy
+  - 3importCredentials.groovy
+  - 3setGlobalProperties.groovy
+  - 3shutdownCLI.groovy
+  - 4configureGit.groovy
+  - 4configureJobConfigHistory.groovy
+  - 4configureMailerPlugin.groovy
+  - 4configureMaskPasswords.groovy
+  - 5createLoggers.groovy
+
+# plugins
+de_jenkins_plugins_list:
+  - name: 'antisamy-markup-formatter'
+    version: '1.3'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'script-security'
+    version: '1.27'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'mailer'
+    version: '1.16'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'cvs'
+    version: '2.12'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'ldap'
+    version: '1.11'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'ant'
+    version: '1.2'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'matrix-auth'
+    version: '1.2'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'matrix-project'
+    version: '1.4.1'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'credentials'
+    version: '1.24'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'ssh-credentials'
+    version: '1.11'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'external-monitor-job'
+    version: '1.4'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'translation'
+    version: '1.12'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'subversion'
+    version: '2.4.5'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'junit'
+    version: '1.3'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'pam-auth'
+    version: '1.2'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'maven-plugin'
+    version: '2.8'
+    group: 'org.jenkins-ci.main'
+  - name: 'ssh-slaves'
+    version: '1.9'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'javadoc'
+    version: '1.3'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'ansicolor'
+    version: '0.4.1'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'buildgraph-view'
+    version: '1.1.1'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'build-name-setter'
+    version: '1.3'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'build-timeout'
+    version: '1.14.1'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'build-user-vars-plugin'
+    version: '1.5'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'cobertura'
+    version: '1.9.6'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'copyartifact'
+    version: '1.32.1'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'credentials-binding'
+    version: '1.7'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'ec2'
+    version: '1.28'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'envinject'
+    version: '1.92.1'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'exclusive-execution'
+    version: '0.8'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'flexible-publish'
+    version: '0.15.2'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'gradle'
+    version: '1.24'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'groovy'
+    version: '1.29'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'groovy-postbuild'
+    version: '2.2'
+    group: 'org.jvnet.hudson.plugins'
+  - name: 'hockeyapp'
+    version: '1.2.1'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'htmlpublisher'
+    version: '1.10'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'jobConfigHistory'
+    version: '2.10'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'job-dsl'
+    version: '1.45'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'mask-passwords'
+    version: '2.8'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'monitoring'
+    version: '1.56.0'
+    group: 'org.jvnet.hudson.plugins'
+  - name: 'multiple-scms'
+    version: '0.6'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'nodelabelparameter'
+    version: '1.7.2'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'parameterized-trigger'
+    version: '2.25'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'PrioritySorter'
+    version: '2.9'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'rebuild'
+    version: '1.25'
+    group: 'com.sonyericsson.hudson.plugins.rebuild'
+  - name: 'run-condition'
+    version: '1.0'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'shiningpanda'
+    version: '0.21'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'ssh-agent'
+    version: '1.5'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'text-finder'
+    version: '1.10'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'timestamper'
+    version: '1.5.15'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'violations'
+    version: '0.7.11'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'xunit'
+    version: '1.93'
+    group: 'org.jenkins-ci.plugins'
+  - name: 'reverse-proxy-auth-plugin'
+    version: '1.5'
+    group: 'org.jenkins-ci.plugins'
+
+# ghprb
+de_jenkins_ghprb_white_list_phrase: '.*[Aa]dd\W+to\W+whitelist.*'
+de_jenkins_ghprb_ok_phrase: '.*ok\W+to\W+test.*'
+de_jenkins_ghprb_retest_phrase: '.*jenkins\W+run\W+all.*'
+de_jenkins_ghprb_skip_phrase: '.*\[[Ss]kip\W+ci\].*'
+de_jenkins_ghprb_cron_schedule: 'H/5 * * * *'
+
+# github
+JENKINS_GITHUB_CONFIG: ''
+
+# hipchat
+de_jenkins_hipchat_room: 'Data Engineering'
+
+# ec2
+de_jenkins_instance_cap: '250'
+
+# seed
+de_jenkins_seed_name: 'manually_seed_one_job'
+
+# logs
+de_jenkins_log_list: {}
+
+# job config history
+de_jenkins_history_max_days: '15'
+de_jenkins_history_exclude_pattern: 'queue|nodeMonitors|UpdateCenter|global-build-stats|GhprbTrigger'

playbooks/roles/jenkins_de/meta/main.yml

+---
+dependencies:
+  - common
+  - role: jenkins_common
+    JENKINS_SERVER_NAME: 'scheduler.analytics.edx.org'
+
+    jenkins_common_version: '{{ de_jenkins_version }}'
+    jenkins_common_war_source: '{{ de_jenkins_common_war_source }}'
+    jenkins_common_user_uid: '{{ de_jenkins_user_uid }}'
+    jenkins_common_group_gid: '{{ de_jenkins_group_gid }}'
+    jenkins_common_jvm_args: '{{ de_jenkins_jvm_args }}'
+    jenkins_common_configuration_scripts: '{{ de_jenkins_configuration_scripts }}'
+    jenkins_common_template_files: '{{ de_jenkins_template_files }}'
+    jenkins_common_plugins_list: '{{ de_jenkins_plugins_list }}'
+    jenkins_common_ghprb_white_list_phrase: '{{ de_jenkins_ghprb_white_list_phrase }}'
+    jenkins_common_ghprb_ok_phrase: '{{ de_jenkins_ghprb_ok_phrase }}'
+    jenkins_common_ghprb_retest_phrase: '{{ de_jenkins_ghprb_retest_phrase }}'
+    jenkins_common_ghprb_skip_phrase: '{{ de_jenkins_ghprb_skip_phrase }}'
+    jenkins_common_ghprb_cron_schedule: '{{ de_jenkins_ghprb_cron_schedule }}'
+    jenkins_common_github_configs: '{{ JENKINS_GITHUB_CONFIG }}'
+    jenkins_common_hipchat_room: '{{ de_jenkins_hipchat_room }}'
+    jenkins_common_instance_cap: '{{ de_jenkins_instance_cap }}'
+    jenkins_common_seed_name: '{{ de_jenkins_seed_name }}'
+    jenkins_common_log_list: '{{ de_jenkins_log_list }}'
+    jenkins_common_history_max_days: '{{ de_jenkins_history_max_days }}'
+    jenkins_common_history_exclude_pattern: '{{ de_jenkins_history_exclude_pattern }}'
+    jenkins_common_server_name: '{{ JENKINS_SERVER_NAME }}'
+    JENKINS_MAIN_GITHUB_OWNER_WHITELIST: ''

playbooks/roles/jenkins_worker/defaults/main.yml

@@ -11,6 +11,9 @@ jenkins_debian_pkgs:
     - libffi-dev
     - python-dev
     - libsqlite3-dev
+    - libfreetype6-dev
 
 # packer direct download URL
 packer_url: "https://releases.hashicorp.com/packer/0.8.6/packer_0.8.6_linux_amd64.zip"
+
+jenkins_worker_key_url: null

playbooks/roles/jenkins_worker/tasks/system.yml

@@ -16,8 +16,12 @@
     owner={{ jenkins_user }} group={{ jenkins_group }}
   ignore_errors: yes
 
-- name: Copy ssh keys for jenkins
-  command: cp /home/ubuntu/.ssh/authorized_keys /home/{{ jenkins_user }}/.ssh/authorized_keys
+- name: Get the authorized key that should be used for this machine.
+  authorized_key:
+    user: "{{ jenkins_user }}"
+    state: present
+    key: "{{ jenkins_worker_key_url }}"
+  when: jenkins_worker_key_url
   ignore_errors: yes
 
 - name: Set key permissions

playbooks/roles/mongo_3_2/defaults/main.yml

@@ -26,6 +26,9 @@ mongodb_debian_pkgs:
   - "mongodb-org-tools={{ mongo_version }}"
 
 
+
+mongo_configure_replica_set: true
+
 # Vars Meant to be overridden
 MONGO_ADMIN_USER: 'admin'
 MONGO_ADMIN_PASSWORD: 'password'

playbooks/roles/mongo_3_2/tasks/main.yml

@@ -280,6 +280,7 @@
     rs_config: "{{ MONGO_RS_CONFIG }}"
   run_once: true
   register: replset_status
+  when: mongo_configure_replica_set
   tags:
     - "manage"
     - "manage:db"
@@ -297,6 +298,7 @@
       password: "{{ MONGO_ADMIN_PASSWORD }}"
   register: status
   until: status.status is defined and 'PRIMARY' in status.status.members|map(attribute='stateStr')|list
+  when: mongo_configure_replica_set
   retries: 5
   delay: 2
   run_once: true
@@ -318,6 +320,7 @@
     replica_set: "{{ MONGO_REPL_SET }}"
   with_items: "{{ MONGO_USERS }}"
   run_once: true
+  when: mongo_configure_replica_set
   tags:
     - "manage"
     - "manage:db"

playbooks/roles/nginx/defaults/main.yml

@@ -18,6 +18,7 @@ NGINX_USERS:
 
 NGINX_ENABLE_SSL: False
 NGINX_REDIRECT_TO_HTTPS: False
+NGINX_HSTS_MAX_AGE: 31536000
 # Set these to real paths on your
 # filesystem, otherwise nginx will
 # use a self-signed snake-oil cert

playbooks/roles/nginx/templates/edx/app/nginx/sites-available/cms.j2

@@ -38,8 +38,11 @@ error_page {{ k }} {{ v }};
 
   ssl_certificate /etc/ssl/certs/{{ NGINX_SSL_CERTIFICATE|basename }};
   ssl_certificate_key /etc/ssl/private/{{ NGINX_SSL_KEY|basename }};
+  {% endif %}
+  
+  {% if NGINX_ENABLE_SSL or NGINX_REDIRECT_TO_HTTPS %}
   # request the browser to use SSL for all connections
-  add_header Strict-Transport-Security "max-age=31536000; includeSubDomains";
+  add_header Strict-Transport-Security "max-age={{ NGINX_HSTS_MAX_AGE }}";
   {% endif %}
 
   # Prevent invalid display courseware in IE 10+ with high privacy settings

playbooks/roles/nginx/templates/edx/app/nginx/sites-available/credentials.j2

@@ -27,12 +27,15 @@ server {
 
   ssl_certificate /etc/ssl/certs/{{ NGINX_SSL_CERTIFICATE|basename }};
   ssl_certificate_key /etc/ssl/private/{{ NGINX_SSL_KEY|basename }};
-  # request the browser to use SSL for all connections
-  add_header Strict-Transport-Security "max-age=31536000; includeSubDomains";
 
   {% else %}
   listen {{ CREDENTIALS_NGINX_PORT }} {{ default_site }};
   {% endif %}
+  
+  {% if NGINX_ENABLE_SSL or NGINX_REDIRECT_TO_HTTPS %}
+  # request the browser to use SSL for all connections
+  add_header Strict-Transport-Security "max-age={{ NGINX_HSTS_MAX_AGE }}";
+  {% endif %}
 
   # Prevent invalid display courseware in IE 10+ with high privacy settings
   add_header P3P '{{ NGINX_P3P_MESSAGE }}';

playbooks/roles/nginx/templates/edx/app/nginx/sites-available/edx_notes_api.j2

@@ -13,8 +13,11 @@ server {
 
   ssl_certificate /etc/ssl/certs/{{ NGINX_SSL_CERTIFICATE|basename }};
   ssl_certificate_key /etc/ssl/private/{{ NGINX_SSL_KEY|basename }};
+  {% endif %}
+
+  {% if NGINX_ENABLE_SSL or NGINX_REDIRECT_TO_HTTPS %}
   # request the browser to use SSL for all connections
-  add_header Strict-Transport-Security "max-age=31536000; includeSubDomains";
+  add_header Strict-Transport-Security "max-age={{ NGINX_HSTS_MAX_AGE }}";
   {% endif %}
 
   {% include "common-settings.j2" %}

playbooks/roles/nginx/templates/edx/app/nginx/sites-available/insights.j2

@@ -27,8 +27,11 @@ server {
 
   ssl_certificate /etc/ssl/certs/{{ NGINX_SSL_CERTIFICATE|basename }};
   ssl_certificate_key /etc/ssl/private/{{ NGINX_SSL_KEY|basename }};
+  {% endif %}
+
+  {% if NGINX_ENABLE_SSL or NGINX_REDIRECT_TO_HTTPS %}
   # request the browser to use SSL for all connections
-  add_header Strict-Transport-Security "max-age=31536000; includeSubDomains";
+  add_header Strict-Transport-Security "max-age={{ NGINX_HSTS_MAX_AGE }}";
   {% endif %}
 
   location ~ ^/static/(?P<file>.*) {

playbooks/roles/nginx/templates/edx/app/nginx/sites-available/lms.j2

@@ -86,8 +86,11 @@ error_page {{ k }} {{ v }};
 
   ssl_certificate /etc/ssl/certs/{{ NGINX_SSL_CERTIFICATE|basename }};
   ssl_certificate_key /etc/ssl/private/{{ NGINX_SSL_KEY|basename }};
+  {% endif %}
+
+  {% if NGINX_ENABLE_SSL or NGINX_REDIRECT_TO_HTTPS %}
   # request the browser to use SSL for all connections
-  add_header Strict-Transport-Security "max-age=31536000; includeSubDomains";
+  add_header Strict-Transport-Security "max-age={{ NGINX_HSTS_MAX_AGE }}";
   {% endif %}
 
   # Prevent invalid display courseware in IE 10+ with high privacy settings

playbooks/roles/notifier/defaults/main.yml

@@ -34,7 +34,7 @@ NOTIFIER_EMAIL_SENDER_POSTAL_ADDRESS: ""
 
 NOTIFIER_ENV_EXTRA: {}
 
-NOTIFIER_LANGUAGE: ""
+NOTIFIER_LANGUAGE: "en"
 
 NOTIFIER_ENV: "Development"
 
@@ -128,3 +128,4 @@ notifier_env_vars:
   FORUM_DIGEST_TASK_BATCH_SIZE: "{{ NOTIFIER_FORUM_DIGEST_TASK_BATCH_SIZE }}"
   FORUM_DIGEST_TASK_RATE_LIMIT: "{{ NOTIFIER_FORUM_DIGEST_TASK_RATE_LIMIT }}"
   DEAD_MANS_SNITCH_URL: "{{ NOTIFIER_DEAD_MANS_SNITCH_URL }}"
+  BOTO_CONFIG: "{{ notifier_app_dir }}/.boto"

playbooks/roles/notifier/tasks/deploy.yml

@@ -97,8 +97,8 @@
   - "install"
   - "install:configuration"
 
-- name: Syncdb
-  shell: "{{ NOTIFIER_VENV_DIR }}/bin/python manage.py syncdb"
+- name: Migrate the notifier db
+  shell: "{{ NOTIFIER_VENV_DIR }}/bin/python manage.py migrate --fake-initial"
   args:
     chdir: "{{ NOTIFIER_CODE_DIR }}"
   become: true

playbooks/roles/notifier/tasks/main.yml

@@ -154,6 +154,14 @@
     - "install"
     - "install:base"
 
+- name: Copy the boto file
+  template:
+    src: "boto.j2"
+    dest: "{{ notifier_app_dir }}/.boto"
+    owner: "{{ notifier_user }}"
+    group: "{{ NOTIFIER_WEB_USER }}"
+    mode: 0644
+
 - name: Write supervisord wrapper for celery workers and scheduler
   template:
     src: "{{ item.src }}"

playbooks/roles/oraclejdk/tasks/main.yml

@@ -17,11 +17,10 @@
   with_items: "{{ oraclejdk_debian_pkgs }}"
 
 - name: Download Oracle Java
-  shell: "curl -b gpw_e24=http%3A%2F%2Fwww.oracle.com -b oraclelicense=accept-securebackup-cookie -O -L {{ oraclejdk_url }}"
-  args:
-    executable: /bin/bash
-    chdir: /var/tmp
-    creates: "/var/tmp/{{ oraclejdk_file }}"
+  get_url:
+    url: "{{ oraclejdk_url }}"
+    headers: 'Cookie:oraclelicense=accept-securebackup-cookie'
+    dest: "/var/tmp/{{ oraclejdk_file }}"
 
 - name: Create jvm dir
   file:
@@ -31,10 +30,10 @@
     group: root
 
 - name: Untar Oracle Java
-  shell: "tar -C /usr/lib/jvm -zxvf /var/tmp/{{ oraclejdk_file }}"
-  args:
-    executable: /bin/bash
-    creates: "/usr/lib/jvm/{{ oraclejdk_base }}"
+  unarchive:
+    src: "/var/tmp/{{ oraclejdk_file }}"
+    dest: "/usr/lib/jvm"
+    copy: no
 
 - name: Create symlink expected by elasticsearch
   file:

playbooks/roles/server_utils/defaults/main.yml

@@ -9,7 +9,7 @@
 #
 ##
 # Defaults for role server_utils
-# 
+#
 
 #
 # vars are namespaced with the module name.
@@ -29,12 +29,11 @@ server_utils_debian_pkgs:
   # Not installed by default on vagrant ubuntu
   # boxes.
   # TODO: move to Vagrant role
-  - curl
   - tree
   - screen
   - tmux
   - curl
-  - vim-tiny
+  - vim
   - dnsutils
   - inetutils-telnet
   - netcat

playbooks/roles/splunk-server/defaults/main.yml

@@ -9,13 +9,12 @@
 #
 ##
 # Defaults for role splunk-server
-# 
+#
 
 #
 # vars are namespaced with the module name.
 #
-SPLUNK_INDEXES: 
-  - "default"
+SPLUNK_INDEXES: []
 
 SPLUNK_ALERTS: []
   # A list of dicts with the following keys:
@@ -83,6 +82,7 @@ SPLUNK_SMTP_USERNAME: username
 SPLUNK_SMTP_PASSWORD: password
 SPLUNK_FROM_ADDRESS: no-reply@example.com
 SPLUNK_EMAIL_FOOTER: Generated by {{ SPLUNK_HOSTNAME }}
+SPLUNK_SSL_HOSTNAME: splunk.example.com:443
 
 # SSL settings. Either all or none of these must be defined.
 # For more details about setting up splunk with SSL, see
@@ -94,7 +94,7 @@ SPLUNK_SSL_ROOT_CA: !!null
 splunk-server_role_name: splunk-server
 
 splunk_user: "splunk"
-splunk_root: "/vol/splunk/storage"
+splunk_root: "/vol/splunk"
 splunk_home: "/opt/splunk"
 
 splunk_hot_dir: "{{ splunk_root }}/hot"

playbooks/roles/splunk-server/tasks/main.yml

@@ -55,7 +55,7 @@
     dest: "{{ splunk_home }}/etc/system/local/inputs.conf"
     owner: splunk
     group: splunk
-    mode: "0644"
+    mode: "0600"
 
 - name: Create bucket directories
   file:
@@ -90,7 +90,7 @@
     dest: "{{ splunk_home }}/etc/apps/search/local/indexes.conf"
     owner: "{{ splunk_user }}"
     group: "{{ splunk_user }}"
-    mode: 0700
+    mode: 0600
   tags:
     - "install"
     - "install:configuration"
@@ -101,7 +101,7 @@
     dest: "{{ splunk_home }}/etc/system/local/alert_actions.conf"
     owner: "{{ splunk_user }}"
     group: "{{ splunk_user }}"
-    mode: 0700
+    mode: 0600
   tags:
     - install
     - install:configuration
@@ -112,7 +112,7 @@
     dest: "{{ splunk_home }}/etc/apps/search/local/savedsearches.conf"
     owner: "{{ splunk_user }}"
     group: "{{ splunk_user }}"
-    mode: 0700
+    mode: 0600
   tags:
     - "install"
     - "install:configuration"

playbooks/roles/splunk-server/templates/opt/splunk/etc/apps/search/local/indexes.conf.j2

-{% for name in SPLUNK_INDEXES %}
-[{{ name }}]
-coldPath = {{ splunk_cold_dir }}/{{ name }}/colddb
-homePath = {{ splunk_hot_dir }}/{{ name }}/db
-thawedPath = {{ splunk_thawed_dir }}/{{ name }}/thaweddb
-coldToFrozenDir = {{ splunk_frozen_dir }}/{{ name }}/frozendb
+{% for index in SPLUNK_INDEXES %}
+[{{ index.name }}]
+{% if index.coldPath is defined %}
+coldPath = {{ index.coldPath }}
+{% else %}
+coldPath = {{ splunk_cold_dir }}/{{ index.name }}/colddb
+{% endif %}
+{% if index.homePath is defined %}
+homePath = {{ index.homePath }}
+{% else %}
+homePath = {{ splunk_hot_dir }}/{{ index.name }}/db
+{% endif %}
+{% if index.maxTotalDataSizeMB is defined %}
+maxTotalDataSizeMB = {{ index.maxTotalDataSizeMB }}
+{% endif %}
+{% if index.thawedPath is defined %}
+thawedPath = {{ index.thawedPath }}
+{% else %}
+thawedPath = {{ splunk_thawed_dir }}/{{ index.name }}/thaweddb
+{% endif %}
+{% if index.coldToFrozenDir is not defined %}
+coldToFrozenDir = {{ splunk_frozen_dir }}/{{ index.name }}/frozendb
+{% endif %}
+{% if index.disabled is defined %}
+disabled = {{ index.disabled }}
+{% endif %}
+{% if index.home is defined %}
+home = {{ index.home }}
+{% endif %}
+{% if index.enableDataIntegrityControl is defined %}
+enableDataIntegrityControl = {{ index.enableDataIntegrityControl }}
+{% endif %}
+{% if index.enableTsidxReduction is defined %}
+enableTsidxReduction = {{ index.enableTsidxReduction }}
+{% endif %}
 
 {% endfor %}
+

playbooks/roles/splunk-server/templates/opt/splunk/etc/apps/search/local/props.conf.j2

@@ -4,6 +4,43 @@
 {% elif 'sourcetype' in extraction %}
 [{{ extraction.sourcetype }}]
 {% endif %}
+{% if extraction.break_before is defined%}
+BREAK_ONLY_BEFORE = {{ extraction.break_before }}
+{% endif %}
+{% if extraction.max_events is defined%}
+MAX_EVENTS = {{ extraction.max_events }}
+{% endif %}
+{% if extraction.datetime_config is defined %}
+DATETIME_CONFIG = {{ extraction.datetime_config }}
+{% endif %}
+{% if extraction.indexed_extractions is defined %}
+INDEXED_EXTRACTIONS = {{ extraction.indexed_extractions }}
+{% endif %}
+{% if extraction.no_binary_check is defined %}
+NO_BINARY_CHECK = {{ extraction.no_binary_check }}
+{% endif %}
+{% if extraction.timestamp_fields is defined %}
+TIMESTAMP_FIELDS = {{ extraction.timestamp_fields }}
+{% endif %}
+{% if extraction.category is defined %}
+category = {{ extraction.category }}
+{% endif %}
+{% if extraction.description is defined %}
+description = {{ extraction.description }}
+{% endif %}
+{% if extraction.disabled is defined %}
+disabled = {{ extraction.disabled }}
+{% endif %}
+{% if extraction.pulldown_type is defined %}
+pulldown_type = {{ extraction.pulldown_type }}
+{% endif %}
+{% if extraction.name is defined %}
 EXTRACT-{{ extraction.name }} = {{ extraction.regex }}
+{% endif %}
+{% if 'sourcetype' in extraction and extraction.sourcetype == "build_log" %}
+EXTRACT-run-results = Setting status of .* and message: 'Build finished. (?P<num_run>\d+) tests run, (?P<num_skipped>\d+) skipped, (?P<num_failed>\d+) failed.
+EXTRACT-error_msg = \n?ERROR: (?P<error_msg>[^\n]*)
+EXTRACT-buildResult = Finished: (?P<buildResult>[A-Z]+)$
+{% endif %}
 
 {% endfor %}

playbooks/roles/splunk-server/templates/opt/splunk/etc/system/local/alert_actions.conf.j2

 [email]
-mailserver = {{ SPLUNK_SMTP_SERVER }}
-pdf.header_left = none
-pdf.header_right = none
 auth_password = {{ SPLUNK_SMTP_PASSWORD }}
 auth_username = {{ SPLUNK_SMTP_USERNAME }}
 footer.text = {{ SPLUNK_EMAIL_FOOTER }}
-hostname = {{ SPLUNK_HOSTNAME }}
+hostname = {{ SPLUNK_SSL_HOSTNAME }}
+mailserver = {{ SPLUNK_SMTP_SERVER }}
+reportServerURL =
+use_tls = 1
+pdf.header_left = none
+pdf.header_right = none
+use_ssl = 0
 from = {{ SPLUNK_FROM_ADDRESS }}
-pdf.footer_enabled = 0
-pdf.header_enabled = 0
-use_tls = 1
\ No newline at end of file

playbooks/roles/splunk-server/templates/opt/splunk/etc/system/local/inputs.conf.j2

@@ -6,8 +6,8 @@ host = {{ SPLUNK_HOSTNAME }}
 compressed = true
 
 [SSL]
-serverCert = $SPLUNK_HOME/{{ splunk_cert_path }}/forwarder.pem
 password = {{ SPLUNK_SSL_PASSWORD }}
 requireClientCert = false
 rootCA = $SPLUNK_HOME/{{ splunk_cert_path }}/cacert.pem
-{% endif %}
\ No newline at end of file
+serverCert = $SPLUNK_HOME/{{ splunk_cert_path }}/forwarder.pem
+{% endif %}

util/jenkins/ansible-provision.sh

@@ -127,9 +127,9 @@ fi
 
 if [[ -z $ami ]]; then
   if [[ $server_type == "full_edx_installation" ]]; then
-    ami="ami-dd9d81a6"
+    ami="ami-8609a6fc"
   elif [[ $server_type == "ubuntu_16.04" || $server_type == "full_edx_installation_from_scratch" ]]; then
-    ami="ami-1d4e7a66"
+    ami="ami-da05a4a0"
   fi
 fi
 

util/packer/jenkins_worker.json

@@ -9,7 +9,8 @@
     "test_platform_version": "{{env `TEST_PLATFORM_VERSION`}}",
     "security_group": "{{env `AWS_SECURITY_GROUP`}}",
     "delete_or_keep": "{{env `DELETE_OR_KEEP_AMI`}}",
-    "remote_branch": "{{env `REMOTE_BRANCH`}}"
+    "remote_branch": "{{env `REMOTE_BRANCH`}}",
+    "jenkins_worker_key_url": "{{env `JENKINS_WORKER_KEY_URL`}}"
   },
   "builders": [{
     "type": "amazon-ebs",
@@ -25,7 +26,13 @@
     "security_group_id": "{{user `security_group`}}",
     "tags": {
       "delete_or_keep": "{{user `delete_or_keep`}}"
-    }
+    },
+    "launch_block_device_mappings": [{
+      "delete_on_termination": true,
+      "device_name": "/dev/sda1",
+      "volume_size": "40",
+      "volume_type": "gp2"
+    }]
   }],
   "provisioners": [{
     "type": "shell",
@@ -52,7 +59,7 @@
     "command": ". {{user `venv_dir`}}/bin/activate && ansible-playbook",
     "inventory_groups": "jenkins_worker",
     "extra_arguments": [
-      "-e \"jenkins_edx_platform_version={{user `test_platform_version`}} NEWRELIC_LICENSE_KEY={{user `new_relic_key`}}\"",
+      "-e \"jenkins_edx_platform_version={{user `test_platform_version`}} NEWRELIC_LICENSE_KEY={{user `new_relic_key`}} initialize_replica_set=false mongo_configure_replica_set=false jenkins_worker_key_url='{{user `jenkins_worker_key_url`}}'\"",
       "-vvv"
       ]
   }, {