Merge branch 'master' into arbab/remove-user-homedir

docker/build/go-agent/Dockerfile

 # Build using: docker build -f Dockerfile.gocd-agent -t gocd-agent .
-FROM gocd/gocd-agent:16.2.1
+FROM gocd/gocd-agent:16.5.0
 
 LABEL version="0.02" \
       description="This custom go-agent docker file installs additional requirements for the edx pipeline"
@@ -8,6 +8,8 @@ LABEL version="0.02" \
 RUN \
   echo oracle-java7-installer shared/accepted-oracle-license-v1-1 select true | debconf-set-selections && \
   add-apt-repository -y ppa:webupd8team/java && \
+  add-apt-repository -y 'deb http://ppa.edx.org trusty main' && \
+  apt-key adv --recv-keys --keyserver keyserver.ubuntu.com 69464050 && \
   apt-get update
 
 # Install Java 7

docker/build/jenkins_analytics/Dockerfile.noci

+FROM edxops/precise-common:latest
+MAINTAINER edxops
+
+USER root
+RUN apt-get update
+
+ADD . /edx/app/edx_ansible/edx_ansible
+WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays
+COPY docker/build/jenkins_analytics/ansible_overrides.yml /
+RUN PYTHONUNBUFFERED=1 /edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook -v jenkins_analytics.yml -i '127.0.0.1,' -c local -e@/ansible_overrides.yml

docker/build/jenkins_analytics/ansible_overrides.yml

+---
+
+JENKINS_ANALYTICS_USER_PASSWORD_PLAIN: aaaaa
+
+JENKINS_ANALYTICS_GITHUB_CREDENTIAL_USER: 'aaa-secure'
+JENKINS_ANALYTICS_GITHUB_CREDENTIAL_KEY: |
+  -----BEGIN RSA PRIVATE KEY-----
+  -----END RSA PRIVATE KEY-----
+
+ANALYTICS_SCHEDULE_COMMON_VARS: "@{{ ANALYTICS_SCHEDULE_SECURE_REPO_DEST }}/job-configs/common.yml"
+ANALYTICS_SCHEDULE_COURSE_ACTIVITY_WEEKLY_EXTRA_VARS: "@{{ ANALYTICS_SCHEDULE_SECURE_REPO_DEST }}/job-configs/course_activity.yml"
+ANALYTICS_SCHEDULE_ANSWER_DISTRIBUTION_EXTRA_VARS: "@{{ ANALYTICS_SCHEDULE_SECURE_REPO_DEST }}/job-configs/answer_distribution.yml"
+ANALYTICS_SCHEDULE_IMPORT_ENROLLMENTS_INTO_MYSQL_EXTRA_VARS: "@{{ ANALYTICS_SCHEDULE_SECURE_REPO_DEST }}/job-configs/enrollments.yml"
+ANALYTICS_SCHEDULE_INSERT_TO_MYSQL_ALL_VIDEO_EXTRA_VARS: "@{{ ANALYTICS_SCHEDULE_SECURE_REPO_DEST }}/job-configs/video.yml"
+ANALYTICS_SCHEDULE_INSERT_TO_MYSQL_COURSE_ENROLL_BY_COUNTRY_EXTRA_VARS: "@{{ ANALYTICS_SCHEDULE_SECURE_REPO_DEST }}/job-configs/geolocation.yml"
+
+
+#ANALYTICS_SCHEDULE_JOBS_DSL_REPO_URL: "we use the default, but might need to override it someday"
+#ANALYTICS_SCHEDULE_JOBS_DSL_REPO_VERSION: "master"
+
+ANALYTICS_SCHEDULE_SECURE_REPO_URL: "git@github.com:something/something-secure.git"
+#ANALYTICS_SCHEDULE_SECURE_REPO_VERSION: "master" # we use the default, but for testing this is helpful to override
+
+ANALYTICS_SCHEDULE_MASTER_SSH_CREDENTIAL_KEY: |
+  -----BEGIN RSA PRIVATE KEY-----
+  -----END RSA PRIVATE KEY-----
+
+JENKINS_ANALYTICS_AUTH_REALM: unix

docker/plays/jenkins_analytics.yml

+- name: Deploy the analytics jenkins
+  hosts: all
+  sudo: True
+  gather_facts: True
+  vars:
+    serial_count: 1
+  serial: "{{ serial_count }}"
+  roles:
+    - common_vars
+    - docker
+    - jenkins_analytics

playbooks/edx-east/analytics_single.yml

@@ -12,6 +12,7 @@
     - aws
     - mysql
     - edxlocal
+    - memcache
     - analytics_api
     - analytics_pipeline
     - role: nginx

playbooks/edx-east/edx_continuous_integration.yml

@@ -24,6 +24,7 @@
     - mysql
     - role: edxlocal
       tags: edxlocal
+    - memcache
     - mongo
     - { role: 'edxapp', celery_worker: True }
     - edxapp

playbooks/edx-east/edx_jenkins_tests.yml

@@ -26,6 +26,7 @@
       - lms
     - mysql
     - edxlocal
+    - memcache
     - mongo
     - edxapp
     - xqueue

playbooks/edx-east/jenkins_worker.yml

@@ -20,6 +20,7 @@
     - aws
     - mysql
     - edxlocal
+    - memcache
     - mongo
     - browsers
     - browsermob-proxy

playbooks/edx_sandbox.yml

@@ -34,9 +34,10 @@
       nginx_sites:
       - ecommerce
       when: SANDBOX_ENABLE_ECOMMERCE
-    - role: mysql
+    - role: edxlocal
       when: EDXAPP_MYSQL_HOST == 'localhost'
-    - edxlocal
+    - role: memcache
+      when: "'localhost' in ' '.join(EDXAPP_MEMCACHE)"
     - role: mongo
       when: "'localhost' in EDXAPP_MONGO_HOSTS"
     - { role: 'rabbitmq', rabbitmq_ip: '127.0.0.1' }

playbooks/roles/add_user/defaults/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://openedx.atlassian.net/wiki/display/OpenOPS
+# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+# Example play:
+#
+# Rather than being included in the play, this role
+# is included as a dependency by other roles in the meta/main.yml
+# file.  The including role should add the following
+# dependency definition.
+#
+# dependencies:
+#   - role: add_user
+#     user_name: edx-themes
+#     user_home: /edx/etc/edx-themes
+#     group_name: edx-themes
+#     dirs:
+#       - {path: /edx/var/edx-themes, owner: 'edx-themes', group: "edx-themes", mode: "0646"}
+#       - {path: /edx/etc/edx-themes, owner: 'edx-themes', group: "edx-themes", mode: "0664"}
+#       - ...
+#
+##
+# Defaults for role add_user
+#

playbooks/roles/add_user/meta/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://openedx.atlassian.net/wiki/display/OpenOPS
+# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+##
+# Role includes for role add_user
+#
+
+# Allow this role to be duplicated in dependencies
+allow_duplicates: yes

playbooks/roles/add_user/tasks/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://openedx.atlassian.net/wiki/display/OpenOPS
+# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+#
+#
+# Tasks for role add_user
+#
+# Overview:
+#
+# This role performs the repetitive tasks that most edX roles
+# require in our default configuration.
+#
+# Generating an ssh key so users can do a git
+# clone over ssh for public repositories without any
+# additional configuration
+- name: create application user
+  user:
+    name: "{{ user_name }}"
+    home: "{{ user_home }}"
+    createhome: yes
+    shell: /bin/false
+    generate_ssh_key: yes
+  tags:
+    - install
+    - install:base
+
+# Assumes that the home directory has been created above.
+# In some cases(vagrant boxes) the home directory gets created
+# but does not have the correct owner and group. In vagrant for
+# example we were seeing it defaulting to `root` for both.
+# Here we ensure that the ownership
+# of the home directory is always correct before proceeding.
+- name: ensure correct ownership of home directory
+  file:
+    path: "{{ user_home }}"
+    state: directory
+    owner: "{{ user_name }}"
+    group: "{{ group_name }}"
+  tags:
+    - install
+    - install:base
+
+- name: create dirs for the user
+  file:
+    path: "{{ item.path }}"
+    state: directory
+    owner: "{{ item.owner }}"
+    group: "{{ item.group }}"
+    mode: "{{ item.mode | default('0755') }}"
+  with_items: dirs
+  when: dirs is defined
+  tags:
+    - install
+    - install:base

playbooks/roles/ecommerce/meta/main.yml

@@ -22,6 +22,6 @@ dependencies:
     edx_service_packages:
       debian: "{{ ecommerce_debian_pkgs }}"
       redhat: "{{ ecommerce_redhat_pkgs }}"
-  - role: edx-themes
+  - role: edx_themes
     when: "{{ ECOMMERCE_ENABLE_COMPREHENSIVE_THEMING }}"
   - oraclejdk

playbooks/roles/edx_service/meta/main.yml

@@ -18,3 +18,15 @@
 #   my_role_var0: "foo"
 #   my_role_var1: "bar"
 # }
+
+dependencies:
+  - role: add_user
+    user_name: "{{ edx_service_name }}"
+    user_home: "{{ edx_service_home }}"
+    group_name: "{{ common_web_group }}"
+  - role: git_clone
+    repo_owner: "{{ edx_service_user }}"
+    repo_group: "{{ edx_service_user }}"
+    GIT_REPOS: "{{ edx_service_repos }}"
+    git_home: "{{ edx_service_home }}"
+    when: edx_service_repos is defined

playbooks/roles/edx_service/tasks/main.yml

@@ -43,25 +43,10 @@
 #       debian: [ pkg1, pkg2, pkg3 ]      
 #       redhat: [ pkg4, pkg5 ]
 #
-
-# Generating an ssh key so service users can do a git
-# clone over ssh for public repositories without any
-# additional configuration
-- name: create application user
-  user: >
-    name="{{ edx_service_name }}"
-    home="{{ edx_service_home }}"
-    createhome=yes
-    shell=/bin/false
-    generate_ssh_key=yes
-  tags:
-    - install
-    - install:base
-
 # Assumes that the home directory has been created above.
 # In some cases(vagrant boxes) the home directory gets created
 # but does not have the corrent owner and group. In vagrant for
-# example we were seeing it defaulting it to `root` for both.
+# example we were seeing it defaulting to `root` for both.
 # The item that is a blank string("") ensures the ownership
 # of the home directory is always correct before proceeding.
 - name: create edx_service app, venv, data, and staticfiles dirs
@@ -135,58 +120,6 @@
     - install
     - install:system-requirements
 
-- name: set git fetch.prune to ignore deleted remote refs
-  shell: git config --global fetch.prune true
-  sudo_user: "{{ edx_service_user }}"
-  when: edx_service_repos is defined
-  tags:
-    - install
-    - install:code
-
-- name: validate git protocol
-  fail: msg='REPOS.PROTOCOL must be "https" or "ssh"'
-  when: (item.PROTOCOL != "https") and (item.PROTOCOL != "ssh") and edx_service_repos is defined
-  with_items: edx_service_repos
-  tags:
-    - install
-    - install:code
-
-- name: install read-only ssh key
-  copy: >
-    dest="{{ edx_service_home }}/.ssh/{{ item.REPO }}"
-    content="{{ item.SSH_KEY }}" owner={{ edx_service_user }}
-    group={{ edx_service_user }} mode=0600
-  when: item.PROTOCOL == "ssh" and edx_service_repos is defined
-  with_items: edx_service_repos
-  tags:
-    - install
-    - install:code
-
-- name: checkout code over ssh
-  git_2_0_1: >
-    repo=git@{{ item.DOMAIN }}:{{ item.PATH }}/{{ item.REPO }}
-    dest={{ item.DESTINATION }}  version={{ item.VERSION }}
-    accept_hostkey=yes key_file={{ edx_service_home }}/.ssh/{{ item.REPO }}
-  sudo_user: "{{ edx_service_user }}"
-  register: code_checkout
-  when: item.PROTOCOL == "ssh" and edx_service_repos is defined
-  with_items: edx_service_repos
-  tags:
-    - install
-    - install:code
-
-- name: checkout code over https
-  git_2_0_1: >
-    repo=https://{{ item.DOMAIN }}/{{ item.PATH }}/{{ item.REPO }}
-    dest={{ item.DESTINATION }}  version={{ item.VERSION }}
-  sudo_user: "{{ edx_service_user }}"
-  register: code_checkout
-  when: item.PROTOCOL == "https" and edx_service_repos is defined
-  with_items: edx_service_repos
-  tags:
-    - install
-    - install:code
-
 - name: get instance information
   action: ec2_facts
   tags:

playbooks/roles/edx_themes/meta/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://openedx.atlassian.net/wiki/display/OpenOPS
+# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+##
+# Role includes for role edx_themes
+# 
+dependencies:
+  - role: add_user
+    user_name: "{{ themes_user }}"
+    user_home: "{{ themes_home }}"
+    group_name: "{{ themes_user }}"
+  - role: git_clone
+    repo_owner: "{{ themes_user }}"
+    repo_group: "{{ themes_user }}"
+    GIT_REPOS: "{{ THEMES_REPOS }}"
+    git_home: "{{ themes_home }}"

playbooks/roles/edx_themes/tasks/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://openedx.atlassian.net/wiki/display/OpenOPS
+# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+#
+#
+# Tasks for role themes
+#
+# Overview:
+#
+# This role performs the repetive tasks that most edX roles
+# require in our default configuration.
+#
+# Example play:
+#
+# Rather than being included in the play, this role
+# is included as a dependency by other roles in the meta/main.yml
+# file.  The including role should add the following
+# dependency definition.
+#
+# dependencies:
+#   - role: edx_themes
+#     when do_setup_themes
+#

playbooks/roles/edxapp/tasks/main.yml

@@ -3,72 +3,71 @@
 #  - common/tasks/main.yml
 ---
 - name: create application user
-  user: >
-    name="{{ edxapp_user }}" home="{{ edxapp_app_dir }}"
-    createhome=no shell=/bin/false
+  user:
+    name: "{{ edxapp_user }}"
+    home: "{{ edxapp_app_dir }}"
+    createhome: no
+    shell: /bin/false
   tags:
     - install
     - install:base
 
 - name: create edxapp user dirs
-  file: >
-    path="{{ item }}" state=directory
-    owner="{{ edxapp_user }}" group="{{ common_web_group }}"
+  file:
+    path: "{{ item.path }}"
+    state: directory
+    owner: "{{ edxapp_user }}"
+    group: "{{ common_web_group }}"
+    mode: "{{ item.mode | default(0755) }}"
   with_items:
-    - "{{ edxapp_app_dir }}"
+    - { path: "{{ edxapp_app_dir }}" }
     # needed for the ansible 1.5 git module
-    - "{{ edxapp_app_dir }}/.ssh"
-    - "{{ edxapp_venvs_dir }}"
-    - "{{ edxapp_theme_dir }}"
-    - "{{ edxapp_staticfile_dir }}"
-    - "{{ edxapp_course_static_dir }}"
-    - "{{ edxapp_course_data_dir }}"
-  tags:
-    - install
-    - install:base
-    
-# var should have more permissive permissions than the rest
-- name: create edxapp var dir
-  file: >
-    path={{ edxapp_data_dir }} state=directory mode=0775
-    owner="{{ edxapp_user }}" group="{{ common_web_group }}"
+    - { path: "{{ edxapp_app_dir }}/.ssh" }
+    - { path: "{{ edxapp_venvs_dir }}" }
+    - { path: "{{ edxapp_theme_dir }}" }
+    - { path: "{{ edxapp_staticfile_dir }}" }
+    - { path: "{{ edxapp_course_static_dir }}" }
+    - { path: "{{ edxapp_course_data_dir }}" }
+    # var should have more permissive permissions than the rest
+    - { path: "{{ edxapp_data_dir }}", mode: "0775" }
+    # directory to import the courses from github
+    - { path: "{{ EDXAPP_GIT_REPO_DIR }}", mode: "0775" }
   tags:
     - install
     - install:base
 
-# directory to import the courses from github
-- name: create directory to import the courses from github
-  file: >
-    path={{ EDXAPP_GIT_REPO_DIR }} state=directory mode=0775
-    owner="{{ edxapp_user }}" group="{{ common_web_group }}"
 # This is a symlink that has to exist because
 # we currently can't override the DATA_DIR var
 # in edx-platform. TODO: This can be removed once
 # VPC-122 is closed
 - name: make the course data dir
   file:
-    src="{{ edxapp_course_data_dir }}"
-    dest="{{ edxapp_legacy_course_data_dir }}"
-    state=link
-    owner="{{ edxapp_user }}"
-    group="{{ common_web_group }}"
+    src: "{{ edxapp_course_data_dir }}"
+    dest: "{{ edxapp_legacy_course_data_dir }}"
+    state: link
+    owner: "{{ edxapp_user }}"
+    group: "{{ common_web_group }}"
   tags:
     - install
     - install:base
     
 - name: create edxapp log dir
-  file: >
-    path="{{ edxapp_log_dir }}" state=directory
-    owner="{{ common_log_user }}" group="{{ common_log_user }}"
+  file:
+    path: "{{ edxapp_log_dir }}"
+    state: directory
+    owner: "{{ common_log_user }}"
+    group: "{{ common_log_user }}"
   tags:
     - install
     - install:base
     
 - name: create web-writable edxapp data dirs
-  file: >
-    path="{{ item }}" state=directory
-    owner="{{ common_web_user }}" group="{{ edxapp_user }}"
-    mode="0775"
+  file:
+    path: "{{ item }}"
+    state: directory
+    owner: "{{ common_web_user }}"
+    group: "{{ edxapp_user }}"
+    mode: "0775"
   with_items:
     - "{{ edxapp_course_data_dir }}"
     - "{{ edxapp_upload_dir }}"
@@ -79,38 +78,51 @@
     
 # adding chris-lea nodejs repo
 - name: add ppas for current versions of nodejs
-  apt_repository: repo="{{ edxapp_chrislea_ppa }}"
+  apt_repository:
+    repo: "{{ edxapp_chrislea_ppa }}"
   tags:
     - install
     - install:base
   
 - name: install system packages on which LMS and CMS rely
-  apt: pkg={{','.join(edxapp_debian_pkgs)}} state=present update_cache=yes
+  apt:
+    name: "{{ item }}"
+    state: present
+    update_cache: yes
+  with_items: "{{ edxapp_debian_pkgs }}"
   tags:
     - install
     - install:base
   
 - name: set up edxapp .npmrc
   template:
-    src=.npmrc.j2 dest={{ edxapp_app_dir }}/.npmrc
-    owner={{ edxapp_user }} group={{ common_web_group }}
-    mode=0600
+    src: .npmrc.j2
+    dest: "{{ edxapp_app_dir }}/.npmrc"
+    owner: "{{ edxapp_user }}"
+    group: "{{ common_web_group }}"
+    mode: 0600
   tags:
     - install
     - install:base
     
 - name: create log directories for service variants
-  file: >
-    path={{ edxapp_log_dir }}/{{ item }} state=directory
-    owner={{ common_log_user }} group={{ common_log_user }}
-    mode=0750
-  with_items: service_variants_enabled
+  file:
+    path: "{{ edxapp_log_dir }}/{{ item }}"
+    state: directory
+    owner: "{{ common_log_user }}"
+    group: "{{ common_log_user }}"
+    mode: 0750
+  with_items: "{{ service_variants_enabled }}"
   tags:
     - install
     - install:base
     
 # Set up the python sandbox execution environment
-- include: python_sandbox_env.yml tags=deploy
+- include: python_sandbox_env.yml
   when: EDXAPP_PYTHON_SANDBOX
+  tags:
+    - deploy
 
-- include: deploy.yml tags=deploy
+- include: deploy.yml
+  tags: 
+    - deploy

playbooks/roles/edxapp/tasks/service_variant_config.yml

-- name: "create {{ item }} application config"
-  template: >
-    src={{ item }}.env.json.j2
-    dest={{ edxapp_app_dir }}/{{ item }}.env.json
+---
+- name: create application and auth config
+  template:
+    src: "{{ item[0] }}.{{ item[1] }}.json.j2"
+    dest: "{{ edxapp_app_dir }}/{{ item[0] }}.{{ item[1] }}.json"
   sudo_user: "{{ edxapp_user }}"
-  with_items: service_variants_enabled
+  with_nested: 
+    - "{{ service_variants_enabled }}"
+    - [ 'env', 'auth' ]
   tags:
     - install
     - install:configuration
     - edxapp_cfg
 
-- name: "create {{ item }} auth file"
-  template: >
-    src={{ item }}.auth.json.j2
-    dest={{ edxapp_app_dir }}/{{ item }}.auth.json
+- name: create auth and application yaml config
+  template:
+    src: "{{ item[0] }}.{{ item[1] }}.yaml.j2"
+    dest: "{{ EDXAPP_CFG_DIR }}/{{ item[0] }}.{{ item[1] }}.yaml"
   sudo_user: "{{ edxapp_user }}"
-  with_items: service_variants_enabled
-  tags:
-    - install
-    - install:configuration
-    - edxapp_cfg
-
-- name: "create {{ item }} yaml application config"
-  template: >
-    src={{ item }}.env.yaml.j2
-    dest={{ EDXAPP_CFG_DIR }}/{{ item }}.env.yaml
-  sudo_user: "{{ edxapp_user }}"
-  with_items: service_variants_enabled
-  tags:
-    - install
-    - install:configuration
-    - edxapp_cfg
-
-- name: "create {{ item }} yaml auth file"
-  template: >
-    src={{ item }}.auth.yaml.j2
-    dest={{ EDXAPP_CFG_DIR }}/{{ item }}.auth.yaml
-  sudo_user: "{{ edxapp_user }}"
-  with_items: service_variants_enabled
+  with_nested:
+    - "{{ service_variants_enabled }}"
+    - [ 'env', 'auth' ]
   tags:
     - install
     - install:configuration
     - edxapp_cfg
 
 # write the supervisor scripts for the service variants
-
 - name: "writing {{ item }} supervisor script"
-  template: >
-    src={{ item }}.conf.j2 dest={{ supervisor_available_dir }}/{{ item }}.conf
-    owner={{ supervisor_user }}
-    group={{ supervisor_user }}
-  with_items: service_variants_enabled
+  template:
+    src: "{{ item }}.conf.j2"
+    dest: "{{ supervisor_available_dir }}/{{ item }}.conf"
+    owner: "{{ supervisor_user }}"
+    group: "{{ supervisor_user }}"
   sudo_user: "{{ supervisor_user }}"
+  with_items: "{{ service_variants_enabled }}"
   tags:
     - install
     - install:configuration
 
-- name: writing edxapp supervisor script
-  template: >
-    src=edxapp.conf.j2 dest={{ supervisor_available_dir }}/edxapp.conf
-    owner={{ supervisor_user }}
-    group={{ supervisor_user }}
+# write the supervisor script for edxapp and celery workers
+- name: writing edxapp and celery supervisor scripts
+  template:
+    src: "{{ item }}.j2"
+    dest: "{{ supervisor_available_dir }}/{{ item }}"
+    owner: "{{ supervisor_user }}"
+    group: "{{ supervisor_user }}"
   sudo_user: "{{ supervisor_user }}"
+  with_items:
+    - edxapp.conf
+    - workers.conf
   tags:
     - install
     - install:configuration
 
-- name: "add gunicorn configuration files"
-  template: >
-    src={{ item }}_gunicorn.py.j2 dest={{ edxapp_app_dir }}/{{ item }}_gunicorn.py
-  with_items: service_variants_enabled
+- name: add gunicorn configuration files
+  template:
+    src: "{{ item }}_gunicorn.py.j2"
+    dest: "{{ edxapp_app_dir }}/{{ item }}_gunicorn.py"
   sudo_user: "{{ edxapp_user }}"
-  tags:
-    - install
-    - install:configuration
-
-# write the supervisor script for celery workers
-
-- name: writing celery worker supervisor script
-  template: >
-    src=workers.conf.j2 dest={{ supervisor_available_dir }}/workers.conf
-    owner={{ supervisor_user }}
-    group={{ supervisor_user }}
-  sudo_user: "{{ supervisor_user }}"
+  with_items: "{{ service_variants_enabled }}"
   tags:
     - install
     - install:configuration
 
 # Enable the supervisor jobs
 - name: "enable {{ item }} supervisor script"
-  file: >
-    src={{ supervisor_available_dir }}/{{ item }}.conf
-    dest={{ supervisor_cfg_dir }}/{{ item }}.conf
-    state=link
-    force=yes
-  with_items: service_variants_enabled
-  when: celery_worker is not defined and not disable_edx_services
+  file:
+    src: "{{ supervisor_available_dir }}/{{ item }}.conf"
+    dest: "{{ supervisor_cfg_dir }}/{{ item }}.conf"
+    state: link
+    force: yes
   sudo_user: "{{ supervisor_user }}"
+  with_items: "{{ service_variants_enabled }}"
+  when: celery_worker is not defined and not disable_edx_services
   tags:
     - install
     - install:configuration
 
-- name: "enable edxapp supervisor script"
-  file: >
-    src={{ supervisor_available_dir }}/edxapp.conf
-    dest={{ supervisor_cfg_dir }}/edxapp.conf
-    state=link
-    force=yes
-  when: celery_worker is not defined and not disable_edx_services
+- name: enable edxapp supervisor script
+  file:
+    src: "{{ supervisor_available_dir }}/edxapp.conf"
+    dest: "{{ supervisor_cfg_dir }}/edxapp.conf"
+    state: link
+    force: yes
   sudo_user: "{{ supervisor_user }}"
+  when: celery_worker is not defined and not disable_edx_services
   tags:
     - install
     - install:configuration
 
-- name: "enable celery worker supervisor script"
-  file: >
-    src={{ supervisor_available_dir }}/workers.conf
-    dest={{ supervisor_cfg_dir }}/workers.conf
-    state=link
-    force=yes
-  when: celery_worker is defined and not disable_edx_services
+- name: enable celery worker supervisor script
+  file:
+    src: "{{ supervisor_available_dir }}/workers.conf"
+    dest: "{{ supervisor_cfg_dir }}/workers.conf"
+    state: link
+    force: yes
   sudo_user: "{{ supervisor_user }}"
+  when: celery_worker is defined and not disable_edx_services
   tags:
     - install
     - install:configuration
 
 - name: create helper scripts for managing edxapp
-  template: >
-    src=edx/bin/{{ item[0] }}-{{ item[1] }}.j2
-    dest={{ COMMON_BIN_DIR }}/{{ item[0] }}-{{ item[1] }}
-    owner={{ edxapp_user }}
-    mode=0755
+  template:
+    src: "edx/bin/{{ item[0] }}-{{ item[1] }}.j2"
+    dest: "{{ COMMON_BIN_DIR }}/{{ item[0] }}-{{ item[1] }}"
+    owner: "{{ edxapp_user }}"
+    mode: 0755
   with_nested:
-    - edxapp_helper_scripts
-    - service_variants_enabled
+    - "{{ edxapp_helper_scripts }}"
+    - "{{ service_variants_enabled }}"
   tags:
     - install
     - install:configuration
@@ -145,16 +122,33 @@
     DB_MIGRATION_USER: "{{ COMMON_MYSQL_MIGRATE_USER }}"
     DB_MIGRATION_PASS: "{{ COMMON_MYSQL_MIGRATE_PASS }}"
     EDX_PLATFORM_SETTINGS_OVERRIDE: "aws_migrate"
-  with_items: service_variants_enabled
+  with_items: "{{ service_variants_enabled }}"
   tags:
     - migrate
 
-# Gather assets using paver if possible
+# There are problems with django collectstatic copying files.  It doesn't retain
+# last modified timestamps, but relies on those same timestamps to know if a new file
+# should be recopied.  While collectstatic --clear exists, it only clears some of the
+# files in edxapp_staticfile_dir, it leaves postprocessed or otherwise hashed files.
+# This ensures we have a totally clean directory.
+- name: Remove and recreate the staticfiles directory so nothing stale can exist
+  file:
+      path: "{{ edxapp_staticfile_dir }}"
+      state: "{{ item }}"
+      owner: "{{ edxapp_user }}"
+      group: "{{ common_web_group }}"
+      mode:  0755
+  when: celery_worker is not defined and not devstack
+  with_items: ['absent', 'directory']
+  tags:
+    - gather_static_assets
+    - assets
 
-- name: gather {{ item }} static assets with paver
+# Gather assets using paver if possible
+- name: "gather {{ item }} static assets with paver"
   command: "{{ COMMON_BIN_DIR }}/edxapp-update-assets-{{ item }}"
   when: celery_worker is not defined and not devstack and item != "lms-preview"
-  with_items: service_variants_enabled
+  with_items: "{{ service_variants_enabled }}"
   tags:
     - gather_static_assets
     - assets

playbooks/roles/edxapp/templates/code.sandbox.j2

@@ -20,6 +20,7 @@
     /usr/lib/python2.7/lib-dynload/datetime.so mr,
     /usr/lib/python2.7/lib-dynload/_elementtree.so mr,
     /usr/lib/python2.7/lib-dynload/pyexpat.so mr,
+    /usr/lib/python2.7/lib-dynload/future_builtins.so mr,
 
     # Matplot lib  needs a place for temp caches
     {{ edxapp_sandbox_venv_dir }}/.config/ wrix,

playbooks/roles/edxlocal/meta/main.yml

 ---
 dependencies:
   - common
+  - mysql

playbooks/roles/edxlocal/tasks/main.yml

@@ -74,7 +74,3 @@
     name={{ COMMON_MYSQL_ADMIN_USER }}
     password={{ COMMON_MYSQL_ADMIN_PASS }}
     priv='*.*:CREATE USER'
-
-
-- name: install memcached
-  apt: pkg=memcached state=present

playbooks/roles/forum/handlers/main.yml

 ---
 - name: restart the forum service
-  supervisorctl: >
-    name=forum
-    supervisorctl_path={{ supervisor_ctl }}
-    config={{ supervisor_cfg }}
-    state=restarted
+  supervisorctl:
+    name: forum
+    supervisorctl_path: "{{ supervisor_ctl }}"
+    config: "{{ supervisor_cfg }}"
+    state: restarted
   when: forum_installed is defined and not disable_edx_services

playbooks/roles/forum/tasks/deploy.yml

 ---
-
 - name: create the supervisor config
-  template: >
-    src=forum.conf.j2 dest={{ supervisor_available_dir }}/forum.conf
-    owner={{ supervisor_user }}
-    group={{ supervisor_user }}
-    mode=0644
+  template:
+    src: forum.conf.j2
+    dest: "{{ supervisor_available_dir }}/forum.conf"
+    owner: "{{ supervisor_user }}"
+    group: "{{ supervisor_user }}"
+    mode: 0644
   sudo_user: "{{ supervisor_user }}"
   register: forum_supervisor
   tags:
@@ -13,13 +13,13 @@
     - install:configuration
 
 - name: enable the supervisor config
-  file: >
-    src={{ supervisor_available_dir }}/forum.conf
-    dest={{ supervisor_cfg_dir }}/forum.conf
-    owner={{ supervisor_user }}
-    state=link
-    force=yes
-    mode=0644
+  file:
+    src: "{{ supervisor_available_dir }}/forum.conf"
+    dest: "{{ supervisor_cfg_dir }}/forum.conf"
+    owner: "{{ supervisor_user }}"
+    state: link
+    force: yes
+    mode: 0644
   sudo_user: "{{ supervisor_user }}"
   when: not disable_edx_services
   register: forum_supervisor
@@ -28,20 +28,22 @@
     - install:configuration
   
 - name: create the supervisor wrapper
-  template: >
-    src={{ forum_supervisor_wrapper|basename }}.j2
-    dest={{ forum_supervisor_wrapper }}
-    mode=0755
+  template:
+    src: "{{ forum_supervisor_wrapper|basename }}.j2"
+    dest: "{{ forum_supervisor_wrapper }}"
+    mode: 0755
   sudo_user: "{{ forum_user }}"
   notify: restart the forum service
   tags:
     - install
     - install:configuration
 
-- name:  git checkout forum repo into {{ forum_code_dir }}
-  git_2_0_1: >
-    dest={{ forum_code_dir }} repo={{ forum_source_repo }} version={{ forum_version }}
-    accept_hostkey=yes
+- name: git checkout forum repo into {{ forum_code_dir }}
+  git_2_0_1:
+    dest: "{{ forum_code_dir }}"
+    repo: "{{ forum_source_repo }}"
+    version: "{{ forum_version }}"
+    accept_hostkey: yes
   sudo_user: "{{ forum_user }}"
   register: forum_checkout
   notify: restart the forum service
@@ -50,7 +52,7 @@
     - install:code
 
 - name: install comments service bundle
-  shell: bundle install --deployment --path {{ forum_gem_root }} chdir={{ forum_code_dir }}
+  shell: "bundle install --deployment --path {{ forum_gem_root }} chdir={{ forum_code_dir }}"
   sudo_user: "{{ forum_user }}"
   environment: "{{ forum_environment }}"
   notify: restart the forum service
@@ -73,18 +75,23 @@
     - manage:update
 
 - name: ensure forum is started
-  supervisorctl: >
-    name=forum
-    supervisorctl_path={{ supervisor_ctl }}
-    config={{ supervisor_cfg }}
-    state=started
+  supervisorctl:
+    name: forum
+    supervisorctl_path: "{{ supervisor_ctl }}"
+    config: "{{ supervisor_cfg }}"
+    state: started
   when: not disable_edx_services
   tags:
     - manage
 
-- include: test.yml tags=deploy
+- include: test.yml
+  tags:
+    - deploy
 
-- include: tag_ec2.yml tags=deploy
+- include: tag_ec2.yml
   when: COMMON_TAG_EC2_INSTANCE
+  tags: 
+    - deploy
 
-- set_fact: forum_installed=true
+- set_fact:
+    forum_installed: true

playbooks/roles/forum/tasks/main.yml

 ---
-
 # forum
 #
 # Dependencies:
@@ -22,12 +21,12 @@
 #   - forum
 
 - name: create application user
-  user: >
-    name="{{ forum_user }}"
-    home="{{ forum_app_dir }}"
-    createhome=yes
-    shell=/bin/false
-    generate_ssh_key=yes
+  user:
+    name: "{{ forum_user }}"
+    home: "{{ forum_app_dir }}"
+    createhome: yes
+    shell: /bin/false
+    generate_ssh_key: yes
   notify: restart the forum service
   tags:
     - install
@@ -35,21 +34,23 @@
 
 # Ensure the directory is accessible to the web service
 - name: set forum app dir permissions
-  file: >
-    path="{{ forum_app_dir }}"
-    state=directory
-    owner="{{ forum_user }}"
-    group="{{ common_web_group }}"
+  file:
+    path: "{{ forum_app_dir }}"
+    state: directory
+    owner: "{{ forum_user }}"
+    group: "{{ common_web_group }}"
   notify: restart the forum service
   tags:
     - install
     - install:base
 
 - name: setup the forum env
-  template: >
-    src=forum_env.j2 dest={{ forum_app_dir }}/forum_env
-    owner={{ forum_user }}  group={{ common_web_user }}
-    mode=0644
+  template:
+    src: forum_env.j2
+    dest: "{{ forum_app_dir }}/forum_env"
+    owner: "{{ forum_user }}" 
+    group: "{{ common_web_user }}"
+    mode: 0644
   notify:
     - restart the forum service
   tags:
@@ -57,12 +58,16 @@
     - install:base
 
 - name: create {{ forum_data_dir }}
-  file: >
-    path={{ forum_data_dir }} state=directory
-    owner="{{ common_web_user }}" group="{{ common_web_group }}"
-    mode=0777
+  file:
+    path: "{{ forum_data_dir }}"
+    state: directory
+    owner: "{{ common_web_user }}"
+    group: "{{ common_web_group }}"
+    mode: 0777
   tags:
     - install
     - install:base
     
-- include: deploy.yml tags=deploy
+- include: deploy.yml
+  tags: 
+    - deploy

playbooks/roles/forum/tasks/tag_ec2.yml

 ---
-
 - name: get instance information
   action: ec2_facts
 
 - name: tag instance
-  ec2_tag: resource={{ ansible_ec2_instance_id }} region={{ ansible_ec2_placement_region }}
-  args:
-    tags:
+  ec2_tag:
+    resource: "{{ ansible_ec2_instance_id }}"
+    region: "{{ ansible_ec2_placement_region }}"
+  tags:
       "version:forum" : "{{ forum_source_repo }} {{ forum_checkout.after|truncate(7,True,'') }}"
   when: forum_checkout.after is defined

playbooks/roles/forum/tasks/test.yml

 ---
-
 - name: test that the required service are listening
-  wait_for: port={{ item.port }} host={{ item.host }} timeout=30
-  with_items: forum_services
+  wait_for:
+    port: "{{ item.port }}"
+    host: "{{ item.host }}"
+    timeout: 30
+  with_items: "{{ forum_services }}"
   when: not disable_edx_services
 
 - name: test that mongo replica set members are listing
-  wait_for: port={{ FORUM_MONGO_PORT }} host={{ item }} timeout=30
-  with_items: FORUM_MONGO_HOSTS
+  wait_for:
+    port: "{{ FORUM_MONGO_PORT }}"
+    host: "{{ item }}"
+    timeout: 30
+  with_items: "{{ FORUM_MONGO_HOSTS }}"
   when: not disable_edx_services

playbooks/roles/git_clone/defaults/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://openedx.atlassian.net/wiki/display/OpenOPS
+# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+# Example play:
+#
+# Rather than being included in the play, this role
+# is included as a dependency by other roles in the meta/main.yml
+# file.  The including role should add the following
+# dependency definition.
+#
+# dependencies:
+#   - role: git_clone
+#    repo_owner: edx-themes
+#    repo_group: edx-themes
+#    git_home: /edx/etc/edx-themes
+#    GIT_REPOS:
+#      - PROTOCOL: ssh
+#        DOMAIN: github.com
+#        PATH: edx
+#        REPO: sample-themes.git
+#        VERSION: release
+#        DESTINATION: /edx/etc/edx-themes/edx-themes
+#        SSH_KEY: "{{ THEMES_GIT_IDENTITY }}"
+#
+##
+# Defaults for role git_clone
+#

playbooks/roles/git_clone/meta/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://openedx.atlassian.net/wiki/display/OpenOPS
+# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+##
+# Role includes for role git_clone
+#
+
+# Allow this role to be duplicated in dependencies
+allow_duplicates: yes

playbooks/roles/edx-themes/tasks/main.yml → playbooks/roles/git_clone/tasks/main.yml

@@ -9,82 +9,39 @@
 #
 #
 #
-# Tasks for role themes
+# Tasks for role git_clone
 #
 # Overview:
 #
-# This role performs the repetive tasks that most edX roles
+# This role performs the repetitive tasks that most edX roles
 # require in our default configuration.
 #
-# Example play:
 #
-# Rather than being included in the play, this role
-# is included as a dependency by other roles in the meta/main.yml
-# file.  The including role should add the following
-# dependency definition.
-#
-# dependencies:
-#   - role: themes
-#     when do_setup_themes
-#
-
-# Generating an ssh key so service users can do a git
-# clone over ssh for public repositories without any
-# additional configuration
-- name: create application user
-  user:
-    name: "{{ themes_user }}"
-    home: "{{ themes_home }}"
-    createhome: yes
-    shell: /bin/false
-    generate_ssh_key: yes
-  tags:
-    - install
-    - install:base
-
-# Assumes that the home directory has been created above.
-# In some cases(vagrant boxes) the home directory gets created
-# but does not have the current owner and group. In vagrant for
-# example we were seeing it defaulting it to `root` for both.
-# The item that is a blank string ("") ensures the ownership
-# of the home directory is always correct before proceeding.
-- name: create themes home dirs
-  file:
-    path: "{{ themes_home }}/{{ item }}"
-    state: directory
-    owner: "{{ themes_user }}"
-    group: "{{ common_web_group }}"
-  with_items:
-    - ""
-  tags:
-    - install
-    - install:base
-
 - name: set git fetch.prune to ignore deleted remote refs
   shell: git config --global fetch.prune true
-  sudo_user: "{{ themes_user }}"
-  when: THEMES_REPOS is defined
+  sudo_user: "{{ repo_owner }}"
+  when: GIT_REPOS is defined
   tags:
     - install
     - install:code
 
 - name: validate git protocol
-  fail: msg='THEMES_REPOS.PROTOCOL must be "https" or "ssh"'
-  when: (item.PROTOCOL != "https") and (item.PROTOCOL != "ssh") and THEMES_REPOS is defined
-  with_items: THEMES_REPOS
+  fail: msg='GIT_REPOS.PROTOCOL must be "https" or "ssh"'
+  when: (item.PROTOCOL != "https") and (item.PROTOCOL != "ssh") and GIT_REPOS is defined
+  with_items: GIT_REPOS
   tags:
     - install
     - install:code
 
 - name: install read-only ssh key
   copy:
-    dest: "{{ themes_home }}/.ssh/{{ item.REPO }}"
+    dest: "{{ git_home }}/.ssh/{{ item.REPO }}"
     content: "{{ item.SSH_KEY }}"
-    owner: "{{ themes_user }}"
-    group: "{{ themes_user }}"
+    owner: "{{ repo_owner }}"
+    group: "{{ repo_group }}"
     mode: 0600
-  when: item.PROTOCOL == "ssh" and THEMES_REPOS is defined
-  with_items: THEMES_REPOS
+  when: item.PROTOCOL == "ssh" and GIT_REPOS is defined
+  with_items: GIT_REPOS
   tags:
     - install
     - install:code
@@ -95,11 +52,11 @@
     dest: "{{ item.DESTINATION }}"
     version: "{{ item.VERSION }}"
     accept_hostkey: yes
-    key_file: "{{ themes_home }}/.ssh/{{ item.REPO }}"
-  sudo_user: "{{ themes_user }}"
+    key_file: "{{ git_home }}/.ssh/{{ item.REPO }}"
+  sudo_user: "{{ repo_owner }}"
   register: code_checkout
-  when: item.PROTOCOL == "ssh" and THEMES_REPOS is defined
-  with_items: THEMES_REPOS
+  when: item.PROTOCOL == "ssh" and GIT_REPOS is defined
+  with_items: GIT_REPOS
   tags:
     - install
     - install:code
@@ -109,20 +66,20 @@
     repo: "https://{{ item.DOMAIN }}/{{ item.PATH }}/{{ item.REPO }}"
     dest: "{{ item.DESTINATION }}"
     version: "{{ item.VERSION }}"
-  sudo_user: "{{ themes_user }}"
+  sudo_user: "{{ repo_owner }}"
   register: code_checkout
-  when: item.PROTOCOL == "https" and THEMES_REPOS is defined
-  with_items: THEMES_REPOS
+  when: item.PROTOCOL == "https" and GIT_REPOS is defined
+  with_items: GIT_REPOS
   tags:
     - install
     - install:code
 
 - name: remove read-only ssh key
   file:
-    dest: "{{ themes_home }}/.ssh/{{ item.REPO }}"
+    dest: "{{ git_home }}/.ssh/{{ item.REPO }}"
     state: absent
-  when: THEMES_REPOS is defined
-  with_items: THEMES_REPOS
+  when: item.PROTOCOL == "ssh" and GIT_REPOS is defined
+  with_items: GIT_REPOS
   tags:
     - install
     - install:code

playbooks/roles/go-server/defaults/main.yml

@@ -13,7 +13,7 @@
 GO_SERVER_SERVICE_NAME: "go-server"
 GO_SERVER_USER: "go"
 GO_SERVER_GROUP: "{{ GO_SERVER_USER }}"
-GO_SERVER_VERSION: "16.4.0-3223"
+GO_SERVER_VERSION: "16.5.0-3305"
 GO_SERVER_HOME: "/var/lib/go-server"
 GO_SERVER_CONF_HOME: "/etc/go"
 GO_SERVER_PLUGIN_DIR: "{{ GO_SERVER_HOME }}/plugins/external/"

playbooks/roles/jenkins_master/defaults/main.yml

@@ -100,6 +100,7 @@ jenkins_debian_pkgs:
     - maven
     - daemon
     - python-pycurl
+    - psmisc
 
 # Extra packages need for a specific jenkins instance.
 JENKINS_EXTRA_PKGS: []

playbooks/roles/memcache/tasks/main.yml

+# Installs memcached
+
+- name: install memcached
+  apt: pkg=memcached state=present update_cache=yes
+  tags:
+    - install
+    - install:memcache

playbooks/roles/splunk-server/defaults/main.yml

@@ -64,6 +64,13 @@ SPLUNK_FIELD_EXTRACTIONS: []
   #   name:
   #   regex:
 
+SPLUNK_DASHBOARDS: []
+  # A list of paths of dashboard xml.j2 templates.
+  # xml templates must be of the form expected by Splunk.
+  # For an example, create a dashboard in the Splunk UI and go to Edit > Edit Source
+  # e.g.
+  # - {{ role_path }}/../../../secure-repo/path/to/templates/template.xml.j2
+
 SPLUNK_ALERT_DEFAULT_SEVERITY: "3"
 
 

playbooks/roles/splunk-server/tasks/main.yml

@@ -100,6 +100,25 @@
     - install:configuration
   when: SPLUNK_FIELD_EXTRACTIONS is defined
 
+- name: Make dashboards directory
+  file:
+    state: directory
+    path: /opt/splunk/etc/apps/search/local/data/ui/views
+    owner: "{{ splunk_user  }}"
+    group: "{{ splunk_user }}"
+
+- name: configure splunk dashboards
+  template:
+    src: "{{ item }}"
+    dest: /opt/splunk/etc/apps/search/local/data/ui/views/{{ item.rstrip('.j2') | basename }}
+    owner: "{{ splunk_user }}"
+    group: "{{ splunk_user }}"
+    mode: 0700
+  with_items: SPLUNK_DASHBOARDS
+  tags:
+    - install
+    - install:configuration
+
 - name: restart splunk
   service: 
     name: splunk

playbooks/vagrant-analytics.yml

@@ -21,6 +21,7 @@
     - edx_ansible
     - mysql
     - edxlocal
+    - memcache
     - mongo
     - edxapp
     - oraclejdk

playbooks/vagrant-devstack.yml

@@ -21,6 +21,7 @@
     - edx_ansible
     - mysql
     - edxlocal
+    - memcache
     - mongo
     - { role: 'rabbitmq', rabbitmq_ip: '127.0.0.1' }
     - edxapp

playbooks/vagrant-fullstack.yml

@@ -31,6 +31,7 @@
       - cms
     - mysql
     - edxlocal
+    - memcache
     - mongo
     - { role: 'rabbitmq', rabbitmq_ip: '127.0.0.1' }
     - edxapp