Merge pull request #2352 from edx/e0d/diverse-virtualization-targets

E0d/diverse virtualization targets

docker/build/edxapp/Dockerfile

-FROM edxops/precise-common:v2
+FROM edxops/precise-common
 MAINTAINER edxops
 
 USER docker
 WORKDIR /edx/app/edx_ansible/edx_ansible
 RUN sudo git fetch --all
-RUN sudo git checkout e0d/docker-latest
-RUN sudo git reset --hard origin/e0d/docker-latest
+RUN sudo git checkout e0d/diverse-virtualization-targets
+RUN sudo git reset --hard e0d/diverse-virtualization-targets
 RUN sudo git pull
 WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays
-RUN sudo ansible-playbook edxapp.yml -c local -e "EDXAPP_PYTHON_SANDBOX=false"
+ADD ansible.cfg /edx/app/edx_ansible/edx_ansible/docker/plays/ansible.cfg 
+RUN sudo ansible-playbook edxapp.yml -c local -e "EDXAPP_PYTHON_SANDBOX=false" -t 'install:base'
+RUN sudo ansible-playbook edxapp.yml -c local -e "EDXAPP_PYTHON_SANDBOX=false" -t 'install:code'
+RUN sudo ansible-playbook edxapp.yml -c local -e "EDXAPP_PYTHON_SANDBOX=false" -t 'install:configuration'
 USER root 
 CMD ["/edx/app/supervisor/venvs/supervisor/bin/supervisord", "-n", "--configuration", "/edx/app/supervisor/supervisord.conf"]
 EXPOSE 18000 48000 18020 18010 48010 8010 8000 

docker/build/edxapp/ansible.cfg

+[defaults]
+
+jinja2_extensions=jinja2.ext.do
+

docker/build/forums/Dockerfile

@@ -2,8 +2,17 @@ FROM edxops/precise-common:v2
 MAINTAINER edxops
 
 USER docker
+
+RUN sudo apt-get update
+
+WORKDIR /edx/app/edx_ansible/edx_ansible
+RUN sudo git fetch --all
+RUN sudo git checkout e0d/diverse-virtualization-targets
+RUN sudo git reset --hard e0d/diverse-virtualization-targets
+RUN sudo git pull
 WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays
-RUN sudo ansible-playbook forum.yml -c local
+
+RUN sudo ansible-playbook forum.yml -c local -t 'install:base,install:configuration,install:code' -e '{"FORUM_MONGO_HOSTS":["forums-mongo.local.edx.org"]}' -e FORUM_ELASTICSEARCH_HOST='forums-elasticsearch.local.edx.org'
 USER root 
 CMD ["/edx/app/supervisor/venvs/supervisor/bin/supervisord", "-n", "--configuration", "/edx/app/supervisor/supervisord.conf"]
 #ENTRYPOINT ["/bin/bash"]

docker/build/precise-common/Dockerfile

 FROM ubuntu:precise 
-MAINTAINER e0d
+MAINTAINER edxops 
 
 RUN apt-get update
 RUN apt-get -y install sudo
@@ -9,7 +9,6 @@ RUN mkdir -p /home/docker && chown -R docker:docker /home/docker
 RUN apt-get install -y python2.7 python2.7-dev python-pip python-apt python-yaml python-jinja2 git
 USER docker
 
-RUN echo "cachebust"
 # bootstrap
 RUN sudo git clone --recursive https://github.com/edx/ansible /tmp/ansible
 WORKDIR /tmp/ansible
@@ -20,11 +19,11 @@ ENV PATH /tmp/ansible/bin:/bin:/sbin:/usr/sbin:/usr/bin
 RUN sudo git clone http://github.com/edx/configuration.git /tmp/configuration
 ADD inventory /etc/ansible/hosts
 WORKDIR /tmp/configuration
-RUN sudo git checkout e0d/sever-aws-deps 
+RUN sudo git checkout e0d/diverse-virtualization-targets
 RUN sudo pip install -r pre-requirements.txt
 RUN sudo pip install -r requirements.txt
 WORKDIR /tmp/configuration/playbooks/edx-east
-RUN sudo /tmp/ansible/bin/ansible-playbook edx_ansible.yml -c local -e "configuration_version=e0d/sever-aws-deps"
+RUN sudo /tmp/ansible/bin/ansible-playbook edx_ansible.yml -c local -e "configuration_version=e0d/diverse-virtualization-targets"
 
 # cleanup
 RUN sudo rm -rf /tmp/ansible

docker/compose/docker-compose.yml

+---
+
 db:
   image: mysql:5.6 
   environment:

docker/plays/edxapp.yml

@@ -8,15 +8,3 @@
   roles:
     - docker
     - edxapp
-    - role: datadog
-      when: COMMON_ENABLE_DATADOG
-    - role: splunkforwarder
-      when: COMMON_ENABLE_SPLUNKFORWARDER
-    - role: newrelic
-      NEWRELIC_LOGWATCH:
-        - logwatch-503.j2
-        - logwatch-cms-errors.j2
-        - logwatch-lms-errors.j2
-      when: COMMON_ENABLE_NEWRELIC
-    - role: minos
-      when: COMMON_ENABLE_MINOS

playbooks/roles/common/tasks/main.yml

@@ -108,14 +108,3 @@
   template: src=etc/dhcp/dhclient.conf.j2 dest=/etc/dhcp/dhclient.conf
   when: COMMON_CUSTOM_DHCLIENT_CONFIG
 
-- name: add ssh-warning banner motd
-  template: >
-    dest=/etc/motd.tail
-    src={{ COMMON_MOTD_TEMPLATE }} mode=0755 owner=root group=root
-
-- name: update ssh config
-  template: >
-    dest=/etc/ssh/sshd_config
-    src=sshd_config.j2 mode=0644 owner=root group=root
-  notify: restart ssh
-  
\ No newline at end of file

playbooks/roles/container/defaults/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+##
+# Defaults for role container
+# 
+
+#
+# vars are namespace with the module name.
+#
+container_role_name: container
+
+#
+# OS packages
+#
+
+container_debian_pkgs: []
+
+container_redhat_pkgs: []

playbooks/roles/container/meta/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+##
+# Role includes for role container
+# 
+# Example:
+#
+# dependencies:
+#   - {
+#   role: my_role 
+#   my_role_var0: "foo"
+#   my_role_var1: "bar"
+# }
+
+dependencies:
+  - common

playbooks/roles/container/tasks/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+#
+#
+# Tasks for role container
+# 
+# Overview:
+# 
+#
+# Dependencies:
+#
+# 
+# Example play:
+#
+#
+
+- name: stub ansible task
+  debug: msg="This is a stub task created by the ansible-role role"
+  notify: notify me

playbooks/roles/docker/defaults/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+##
+# Defaults for role docker
+# 
+
+#
+# vars are namespace with the module name.
+#
+docker_role_name: docker
+
+#
+# OS packages
+#
+
+docker_debian_pkgs: []
+
+docker_redhat_pkgs: []

playbooks/roles/docker/meta/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+##
+# Role includes for role docker
+# 
+# Example:
+#
+# dependencies:
+#   - {
+#   role: my_role 
+#   my_role_var0: "foo"
+#   my_role_var1: "bar"
+# }
+dependencies:
+  - container

playbooks/roles/docker/tasks/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+#
+#
+# Tasks for role docker
+# 
+# Overview:
+# 
+#
+# Dependencies:
+#
+# 
+# Example play:
+#
+#
+
+- name: stub ansible task
+  debug: msg="This is a stub task created by the ansible-role role"
+  notify: notify me

playbooks/roles/edxapp/defaults/main.yml

@@ -1021,6 +1021,7 @@ edxapp_debian_pkgs:
   - ntp
   # matplotlib needs libfreetype6-dev
   - libfreetype6-dev
+  - libmysqlclient-dev
 
 # Ruby Specific Vars
 edxapp_ruby_version: "1.9.3-p374"

playbooks/roles/edxapp/meta/main.yml

@@ -6,8 +6,6 @@ dependencies:
     rbenv_user: "{{ edxapp_user }}"
     rbenv_dir: "{{ edxapp_app_dir }}"
     rbenv_ruby_version: "{{ edxapp_ruby_version }}"
-  - devpi
-  - nltk
   - role: user
     user_info:
       - name: "{{ EDXAPP_AUTOMATOR_NAME }}"

playbooks/roles/edxapp/tasks/deploy.yml

@@ -3,34 +3,52 @@
     src=edxapp_env.j2 dest={{ edxapp_app_dir }}/edxapp_env
     owner={{ edxapp_user }}  group={{ common_web_user }}
     mode=0644
-
+  tags:
+    - install
+    - install:configuration
+    
 - name: create edxapp configuration dir
   file: >
     path="{{ EDXAPP_CFG_DIR }}" state=directory
     owner="{{ edxapp_user }}" group="{{ common_web_group }}"
-
+  tags:
+    - install
+    - install:configuration
+    
 # Optional auth for git
 - name: create ssh script for git (not authenticated)
   template: >
     src=git_ssh_noauth.sh.j2 dest={{ edxapp_git_ssh }}
     owner={{ edxapp_user }} mode=750
   when: not EDXAPP_USE_GIT_IDENTITY
-
+  tags:
+    - install
+    - install:base
+  
 - name: create ssh script for git (authenticated)
   template: >
     src=git_ssh_auth.sh.j2 dest={{ edxapp_git_ssh }}
     owner={{ edxapp_user }} mode=750
   when: EDXAPP_USE_GIT_IDENTITY
+  tags:
+    - install
+    - install:base
 
 - name: install read-only ssh key
   copy: >
     content="{{ EDXAPP_GIT_IDENTITY }}" dest={{ edxapp_git_identity }}
     force=yes owner={{ edxapp_user }} mode=0600
   when: EDXAPP_USE_GIT_IDENTITY
+  tags:
+    - install
+    - install:base
 
 - name: set git fetch.prune to ignore deleted remote refs
   shell: git config --global fetch.prune true
   sudo_user: "{{ edxapp_user }}"
+  tags:
+    - install
+    - install:base
 
 # Do A Checkout
 - name: checkout edx-platform repo into {{ edxapp_code_dir }}
@@ -43,10 +61,16 @@
   environment:
     GIT_SSH: "{{ edxapp_git_ssh }}"
   register: edxapp_platform_checkout
+  tags:
+    - install
+    - install:code
 
 - name: git clean after checking out edx-platform
   shell: cd {{ edxapp_code_dir }} && git clean -xdf
   sudo_user: "{{ edxapp_user }}"
+  tags:
+    - install
+    - install:code
 
 - name: checkout theme
   git: >
@@ -59,11 +83,17 @@
   environment:
     GIT_SSH: "{{ edxapp_git_ssh }}"
   register: edxapp_theme_checkout
+  tags:
+    - install
+    - install:code
 
 - name: Stat each requirements file to ensure it exists
   stat: path="{{ item }}"
   with_items: "{{ edxapp_requirements_with_github_urls }}"
   register: requirement_file_stats
+  tags:
+    - install
+    - install:code
 
 # Substitute github mirror in all requirements files
 # This is run on every single deploy
@@ -73,6 +103,9 @@
   sudo_user: "{{ edxapp_user }}"
   when: item.stat.exists
   with_items: "{{ requirement_file_stats.results }}"
+  tags:
+    - install
+    - install:code
 
 # Ruby plays that need to be run after platform updates.
 - name: gem install bundler
@@ -82,6 +115,9 @@
     executable=/bin/bash
   environment: "{{ edxapp_environment }}"
   sudo_user: "{{ edxapp_user }}"
+  tags:
+    - install
+    - install:code
 
 - name: bundle install
   shell: >
@@ -90,6 +126,9 @@
     executable=/bin/bash
   sudo_user: "{{ edxapp_user }}"
   environment: "{{ edxapp_environment }}"
+  tags:
+    - install
+    - install:code
 
 # Set the npm registry
 # This needs to be done as root since npm is weird about
@@ -99,19 +138,27 @@
     npm config set registry '{{ COMMON_NPM_MIRROR_URL }}'
     creates="{{ edxapp_app_dir }}/.npmrc"
   environment: "{{ edxapp_environment }}"
+  tags:
+    - install
+    - install:code
 
 # Set the npm registry permissions
 - name: Set the npm registry permissions
   file:
     path="{{ edxapp_app_dir }}/.npmrc"
     owner=edxapp group=edxapp
+  tags:
+    - install
+    - install:base
 
 # Node play that need to be run after platform updates.
 - name: Install edx-platform npm dependencies
   shell: npm install chdir={{ edxapp_code_dir }}
   sudo_user: "{{ edxapp_user }}"
   environment: "{{ edxapp_environment }}"
-
+  tags:
+    - install
+    - install:code
 
 # Install the python pre requirements into {{ edxapp_venv_dir }}
 - name : install python pre-requirements
@@ -122,6 +169,9 @@
     extra_args="-i {{ COMMON_PYPI_MIRROR_URL }} --exists-action w"
   sudo_user: "{{ edxapp_user }}"
   environment: "{{ edxapp_environment }}"
+  tags:
+    - install
+    - install:code
 
 # Install the python modules into {{ edxapp_venv_dir }}
 - name : install python base-requirements
@@ -133,11 +183,17 @@
     chdir={{ edxapp_code_dir }}
   environment: "{{ edxapp_environment }}"
   sudo_user: "{{ edxapp_user }}"
-
+  tags:
+    - install
+    - install:code
+    
 - stat: path="{{ post_requirements_file }}"
   register: post_requirements
   sudo_user: "{{ edxapp_user }}"
-
+  tags:
+    - install
+    - install:code
+    
 # Install the python post requirements into {{ edxapp_venv_dir }}
 - name : install python post-requirements
   pip: >
@@ -148,7 +204,10 @@
   sudo_user: "{{ edxapp_user }}"
   environment: "{{ edxapp_environment }}"
   when: post_requirements.stat.exists
-
+  tags:
+    - install
+    - install:code
+    
 # Install the python paver requirements into {{ edxapp_venv_dir }}
 - name : install python paver-requirements
   pip: >
@@ -158,13 +217,19 @@
     extra_args="-i {{ COMMON_PYPI_MIRROR_URL }} --exists-action w"
   sudo_user: "{{ edxapp_user }}"
   environment: "{{ edxapp_environment }}"
-
+  tags:
+    - install
+    - install:code
+    
 # Install the python custom requirements into {{ edxapp_venv_dir }}
 
 - stat: path="{{ custom_requirements_file }}"
   register: custom_requirements
   sudo_user: "{{ edxapp_user }}"
-
+  tags:
+    - install
+    - install:code
+    
 - name : install python custom-requirements
   pip: >
     requirements="{{ custom_requirements_file }}"
@@ -174,7 +239,10 @@
   sudo_user: "{{ edxapp_user }}"
   environment: "{{ edxapp_environment }}"
   when: custom_requirements.stat.exists
-
+  tags:
+    - install
+    - install:code
+    
 # Install the final python modules into {{ edxapp_venv_dir }}
 - name : install python post-post requirements
   # Need to use shell rather than pip so that we can maintain the context of our current working directory; some
@@ -187,7 +255,10 @@
   - "{{ github_requirements_file }}"
   - "{{ local_requirements_file }}"
   sudo_user: "{{ edxapp_user }}"
-
+  tags:
+    - install
+    - install:code
+    
 # Private requriements require a ssh key to install, use the same key as the private key for edx-platform
 # If EDXAPP_INSTALL_PRIVATE_REQUIREMENTS is set to true EDXAPP_USE_GIT_IDENTITY must also be true
 - name : install python private requirements
@@ -203,7 +274,10 @@
   environment:
     GIT_SSH: "{{ edxapp_git_ssh }}"
   when: EDXAPP_INSTALL_PRIVATE_REQUIREMENTS
-
+  tags:
+    - install
+    - install:code
+    
 # Install any custom extra requirements if defined in EDXAPP_EXTRA_REQUIREMENTS.
 - name: install python extra requirements
   pip: >
@@ -214,7 +288,10 @@
     state=present
   with_items: EDXAPP_EXTRA_REQUIREMENTS
   sudo_user: "{{ edxapp_user }}"
-
+  tags:
+    - install
+    - install:code
+    
 # If using CAS and you have a function for mapping attributes, install
 # the module here.  The next few tasks set up the python code sandbox
 - name: install CAS attribute module
@@ -225,7 +302,10 @@
     extra_args="-i {{ COMMON_PYPI_MIRROR_URL }} --exists-action w"
   sudo_user: "{{ edxapp_user }}"
   when: EDXAPP_CAS_ATTRIBUTE_PACKAGE|length > 0
-
+  tags:
+    - install
+    - install:code
+    
 # Install the sandbox python modules into {{ edxapp_venv_dir }}
 - name : install sandbox requirements into regular venv
   # Need to use shell rather than pip so that we can maintain the context of our current working directory; some
@@ -240,7 +320,10 @@
   - "{{ sandbox_post_requirements }}"
   sudo_user: "{{ edxapp_user }}"
   when: not EDXAPP_PYTHON_SANDBOX
-
+  tags:
+    - install
+    - install:code
+    
 # The next few tasks set up the python code sandbox
 
 # need to disable this profile, otherwise the pip inside the sandbox venv has no permissions
@@ -250,6 +333,8 @@
   when: EDXAPP_PYTHON_SANDBOX
   tags:
   - edxapp-sandbox
+  - install
+  - install:code
 
 - name: code sandbox | Install base sandbox requirements and create sandbox virtualenv
   pip: >
@@ -261,6 +346,8 @@
   when: EDXAPP_PYTHON_SANDBOX
   tags:
   - edxapp-sandbox
+  - install
+  - install:code
 
 - name: code sandbox | Install sandbox requirements into sandbox venv
   shell: >
@@ -275,22 +362,32 @@
   changed_when: sandbox_install_output.stdout is defined and 'installed' in sandbox_install_output.stdout
   tags:
   - edxapp-sandbox
+  - install
+  - install:code
 
 - name: code sandbox | put code sandbox into aa-enforce or aa-complain mode, depending on EDXAPP_SANDBOX_ENFORCE
   command: /usr/sbin/{{ edxapp_aa_command }} /etc/apparmor.d/code.sandbox
   when: EDXAPP_PYTHON_SANDBOX
   tags:
   - edxapp-sandbox
+  - install
+  - install:code
 
 - name: compiling all py files in the edx-platform repo
   shell: "{{ edxapp_venv_bin }}/python -m compileall -x .git/.* {{ edxapp_code_dir }}"
   sudo_user: "{{ edxapp_user }}"
-
+  tags:
+    - install
+    - install:code
+  
   # alternative would be to give {{ common_web_user }} read access
   # to the virtualenv but that permission change will require
   # root access.
 - name: give other read permissions to the virtualenv
   command: chmod -R o+r "{{ edxapp_venv_dir }}"
+  tags:
+    - install
+    - install:code
 
 # https://code.launchpad.net/~wligtenberg/django-openid-auth/mysql_fix/+merge/22726
 # This is necessary for when syncdb is run and the django_openid_auth module is installed,
@@ -299,7 +396,10 @@
   shell: sed -i -e 's/claimed_id = models.TextField(max_length=2047, unique=True/claimed_id = models.TextField(max_length=2047/' {{ edxapp_venv_dir }}/lib/python2.7/site-packages/django_openid_auth/models.py
   when: openid_workaround is defined
   sudo_user: "{{ edxapp_user }}"
-
+  tags:
+    - install
+    - install:code
+  
 # The next few tasks install xml courses.
 
 # Install the xml courses from an s3 bucket
@@ -311,7 +411,9 @@
     expiration=30
   when: not EDXAPP_XML_FROM_GIT and EDXAPP_XML_S3_BUCKET and EDXAPP_XML_S3_KEY
   register: s3_one_time_url
-
+  tags:
+    - remove
+  
 - name: download from one time url
   get_url:
     url="{{ s3_one_time_url.url }}"
@@ -319,12 +421,16 @@
     mode=0600
   when: not EDXAPP_XML_FROM_GIT and EDXAPP_XML_S3_BUCKET and EDXAPP_XML_S3_KEY
   register: download_xml_s3
+  tags:
+    - remove
 
 - name: unzip the data to the data dir
   shell: >
     tar xzf {{ edxapp_data_dir }}/{{ EDXAPP_XML_S3_KEY|basename }}
     chdir="{{ edxapp_data_dir }}"
   when: download_xml_s3.changed
+  tags:
+    - remove
 
 # This currently has to be done because
 # the course coffescript is compiled on the fly
@@ -337,6 +443,8 @@
     recurse=yes
     owner="{{ common_web_user }}"
     group="{{ edxapp_user }}"
+  tags:
+    - remove
 
 # creates the supervisor jobs for the
 # service variants configured, runs
@@ -361,6 +469,8 @@
   sudo_user: "{{ supervisor_service_user }}"
   changed_when: supervisor_update.stdout is defined and supervisor_update.stdout != ""
   when: not disable_edx_services
+  tags:
+    - manage
 
 - name: ensure edxapp has started
   supervisorctl: >
@@ -371,6 +481,8 @@
   sudo_user: "{{ supervisor_service_user }}"
   when: celery_worker is not defined and not disable_edx_services
   with_items: service_variants_enabled
+  tags:
+    - manage
 
 - name: ensure edxapp_workers has started
   supervisorctl: >
@@ -381,6 +493,8 @@
   when: celery_worker is defined and not disable_edx_services
   with_items: edxapp_workers
   sudo_user: "{{ supervisor_service_user }}"
+  tags:
+    - manage
 
 - name: create symlinks from the venv bin dir
   file: >
@@ -391,6 +505,9 @@
   - python
   - pip
   - django-admin.py
+  tags:
+    - install
+    - install:configuration
 
 - name: create symlinks from the repo dir
   file: >
@@ -399,13 +516,23 @@
     state=link
   with_items:
   - manage.py
+  tags:
+    - install
+    - install:configuration
 
 - name: remove read-only ssh key
   file: path={{ edxapp_git_identity }} state=absent
   when: EDXAPP_USE_GIT_IDENTITY
-
+  tags:
+    - install
+    - install:configuration
+    - install:code
+  
 - include: tag_ec2.yml tags=deploy
   when: COMMON_TAG_EC2_INSTANCE
+  tags:
+    - remove
+    - aws
 
 - set_fact: edxapp_installed=true
 
@@ -418,6 +545,8 @@
   when: edxapp_installed is defined and celery_worker is not defined and not disable_edx_services
   sudo_user: "{{ supervisor_service_user }}"
   with_items: service_variants_enabled
+  tags:
+    - manage
 
 - name: restart edxapp_workers
   supervisorctl: >
@@ -428,3 +557,5 @@
   when: edxapp_installed is defined and celery_worker is defined and not disable_edx_services
   with_items: edxapp_workers
   sudo_user: "{{ common_web_user }}"
+  tags:
+    - manage

playbooks/roles/edxapp/tasks/main.yml

@@ -6,6 +6,9 @@
   user: >
     name="{{ edxapp_user }}" home="{{ edxapp_app_dir }}"
     createhome=no shell=/bin/false
+  tags:
+    - install
+    - install:base
 
 - name: create edxapp user dirs
   file: >
@@ -20,13 +23,19 @@
     - "{{ edxapp_staticfile_dir }}"
     - "{{ edxapp_course_static_dir }}"
     - "{{ edxapp_course_data_dir }}"
-
+  tags:
+    - install
+    - install:base
+    
 # var should have more permissive permissions than the rest
 - name: create edxapp var dir
   file: >
     path={{ edxapp_data_dir }} state=directory mode=0775
     owner="{{ edxapp_user }}" group="{{ common_web_group }}"
-
+  tags:
+    - install
+    - install:base
+    
 # This is a symlink that has to exist because
 # we currently can't override the DATA_DIR var
 # in edx-platform. TODO: This can be removed once
@@ -38,12 +47,18 @@
     state=link
     owner="{{ edxapp_user }}"
     group="{{ common_web_group }}"
-
+  tags:
+    - install
+    - install:base
+    
 - name: create edxapp log dir
   file: >
     path="{{ edxapp_log_dir }}" state=directory
     owner="{{ common_log_user }}" group="{{ common_log_user }}"
-
+  tags:
+    - install
+    - install:base
+    
 - name: create web-writable edxapp data dirs
   file: >
     path="{{ item }}" state=directory
@@ -53,27 +68,42 @@
     - "{{ edxapp_course_data_dir }}"
     - "{{ edxapp_upload_dir }}"
     - "{{ edxapp_media_dir }}"
-
+  tags:
+    - install
+    - install:base
+    
 # adding chris-lea nodejs repo
 - name: add ppas for current versions of nodejs
   apt_repository: repo="{{ edxapp_chrislea_ppa }}"
-
+  tags:
+    - install
+    - install:base
+  
 - name: install system packages on which LMS and CMS rely
   apt: pkg={{','.join(edxapp_debian_pkgs)}} state=present update_cache=yes
-
+  tags:
+    - install
+    - install:base
+  
 - name: set up edxapp .npmrc
   template:
     src=.npmrc.j2 dest={{ edxapp_app_dir }}/.npmrc
     owner={{ edxapp_user }} group={{ common_web_group }}
     mode=0600
-
+  tags:
+    - install
+    - install:base
+    
 - name: create log directories for service variants
   file: >
     path={{ edxapp_log_dir }}/{{ item }} state=directory
     owner={{ common_log_user }} group={{ common_log_user }}
     mode=0750
   with_items: service_variants_enabled
-
+  tags:
+    - install
+    - install:base
+    
 # Set up the python sandbox execution environment
 - include: python_sandbox_env.yml tags=deploy
   when: EDXAPP_PYTHON_SANDBOX

playbooks/roles/edxapp/tasks/service_variant_config.yml

@@ -3,33 +3,44 @@
     src={{ item }}.env.json.j2
     dest={{ edxapp_app_dir }}/{{ item }}.env.json
   sudo_user: "{{ edxapp_user }}"
-  tags: edxapp_cfg
   with_items: service_variants_enabled
+  tags:
+    - install
+    - install:configration
+    - edxapp_cfg
 
 - name: "create {{ item }} auth file"
   template: >
     src={{ item }}.auth.json.j2
     dest={{ edxapp_app_dir }}/{{ item }}.auth.json
   sudo_user: "{{ edxapp_user }}"
-  tags: edxapp_cfg
   with_items: service_variants_enabled
+  tags:
+    - install
+    - install:configuration
+    - edxapp_cfg
 
 - name: "create {{ item }} yaml application config"
   template: >
     src={{ item }}.env.yaml.j2
     dest={{ EDXAPP_CFG_DIR }}/{{ item }}.env.yaml
   sudo_user: "{{ edxapp_user }}"
-  tags: edxapp_cfg
   with_items: service_variants_enabled
+  tags:
+    - install
+    - install:configuration
+    - edxapp_cfg
 
 - name: "create {{ item }} yaml auth file"
   template: >
     src={{ item }}.auth.yaml.j2
     dest={{ EDXAPP_CFG_DIR }}/{{ item }}.auth.yaml
   sudo_user: "{{ edxapp_user }}"
-  tags: edxapp_cfg
   with_items: service_variants_enabled
-
+  tags:
+    - install
+    - install:configuration
+    - edxapp_cfg
 
 # write the supervisor scripts for the service variants
 
@@ -40,6 +51,9 @@
     group={{ supervisor_user }}
   with_items: service_variants_enabled
   sudo_user: "{{ supervisor_user }}"
+  tags:
+    - install
+    - install:configuration
 
 - name: writing edxapp supervisor script
   template: >
@@ -47,12 +61,18 @@
     owner={{ supervisor_user }}
     group={{ supervisor_user }}
   sudo_user: "{{ supervisor_user }}"
+  tags:
+    - install
+    - install:configuration
 
 - name: "add gunicorn configuration files"
   template: >
     src={{ item }}_gunicorn.py.j2 dest={{ edxapp_app_dir }}/{{ item }}_gunicorn.py
   with_items: service_variants_enabled
   sudo_user: "{{ edxapp_user }}"
+  tags:
+    - install
+    - install:configuration
 
 # write the supervisor script for celery workers
 
@@ -62,6 +82,9 @@
     owner={{ supervisor_user }}
     group={{ supervisor_user }}
   sudo_user: "{{ supervisor_user }}"
+  tags:
+    - install
+    - install:configuration
 
 # Enable the supervisor jobs
 - name: "enable {{ item }} supervisor script"
@@ -73,6 +96,9 @@
   with_items: service_variants_enabled
   when: celery_worker is not defined and not disable_edx_services
   sudo_user: "{{ supervisor_user }}"
+  tags:
+    - install
+    - install:configuration
 
 - name: "enable edxapp supervisor script"
   file: >
@@ -82,6 +108,9 @@
     force=yes
   when: celery_worker is not defined and not disable_edx_services
   sudo_user: "{{ supervisor_user }}"
+  tags:
+    - install
+    - install:configuration
 
 - name: "enable celery worker supervisor script"
   file: >
@@ -91,6 +120,9 @@
     force=yes
   when: celery_worker is defined and not disable_edx_services
   sudo_user: "{{ supervisor_user }}"
+  tags:
+    - install
+    - install:configuration
 
 - name: create helper scripts for managing edxapp
   template: >
@@ -101,6 +133,9 @@
   with_nested:
     - edxapp_helper_scripts
     - service_variants_enabled
+  tags:
+    - install
+    - install:configuration
 
 # Syncdb with migrate when the migrate user is overridden in extra vars
 - name: syncdb and migrate
@@ -111,6 +146,8 @@
     DB_MIGRATION_PASS: "{{ COMMON_MYSQL_MIGRATE_PASS }}"
     EDX_PLATFORM_SETTINGS_OVERRIDE: "aws_migrate"
   with_items: service_variants_enabled
+  tags:
+    - migrate
 
 # Gather assets using paver if possible
 
@@ -119,3 +156,5 @@
   when: celery_worker is not defined and not devstack and item != "lms-preview"
   tags: gather_static_assets
   with_items: service_variants_enabled
+  tags:
+    - assets

playbooks/roles/edxapp_common/tasks/main.yml

@@ -3,3 +3,6 @@
 - name: Install system packages
   apt: pkg={{','.join(edxapp_common_debian_pkgs)}}
        state=present update_cache=yes
+  tags:
+    - install
+    - install:base

playbooks/roles/forum/tasks/deploy.yml

@@ -25,7 +25,7 @@
   register: forum_supervisor
   tags:
     - install
-    - install:confugration
+    - install:configuration
   
 - name: create the supervisor wrapper
   template: >
@@ -72,8 +72,8 @@
   changed_when: supervisor_update.stdout is defined and supervisor_update.stdout != ""
   when: not disable_edx_services
   tags:
-    - install
-    - install:configuration
+    - manage
+    - manage:update
 
 - name: ensure forum is started
   supervisorctl: >

playbooks/roles/forum/tasks/main.yml

-1;2802;0c---
+---
 
 # forum
 #

playbooks/roles/rbenv/tasks/main.yml

@@ -39,15 +39,24 @@
     name={{ rbenv_user }} home={{ rbenv_dir }}
     shell=/bin/false createhome=no
   when: rbenv_user != common_web_user
+  tags:
+    - install
+    - install:base
 
 - name: create rbenv dir if it does not exist
   file: >
     path="{{ rbenv_dir }}" owner="{{ rbenv_user }}"
     state=directory
+  tags:
+    - install
+    - install:base
 
 - name: install build depends
   apt: pkg={{ ",".join(rbenv_debian_pkgs) }} state=present install_recommends=no
   with_items: rbenv_debian_pkgs
+  tags:
+    - install
+    - install:base
 
 - name: update rbenv repo
   git: >
@@ -55,22 +64,34 @@
     dest={{ rbenv_dir }}/.rbenv version={{ rbenv_version }}
     accept_hostkey=yes
   sudo_user: "{{ rbenv_user }}"
+  tags:
+    - install
+    - install:base
 
 - name: ensure ruby_env exists
   template: >
     src=ruby_env.j2 dest={{ rbenv_dir }}/ruby_env
   sudo_user: "{{ rbenv_user }}"
+  tags:
+    - install
+    - install:base
 
 - name: check ruby-build installed
   command: test -x /usr/local/bin/ruby-build
   register: rbuild_present
   ignore_errors: yes
+  tags:
+    - install
+    - install:base
 
 - name: if ruby-build exists, which versions we can install
   command: /usr/local/bin/ruby-build --definitions
   when: rbuild_present|success
   register: installable_ruby_vers
   ignore_errors: yes
+  tags:
+    - install
+    - install:base
 
 ### in this block, we (re)install ruby-build if it doesn't exist or if it can't install the requested version
 - name: create temporary directory
@@ -78,6 +99,9 @@
   register: tempdir
   sudo_user: "{{ rbenv_user }}"
   when: rbuild_present|failed or (installable_ruby_vers is defined and rbenv_ruby_version not in installable_ruby_vers)
+  tags:
+    - install
+    - install:base
 
 - name: clone ruby-build repo
   git: >
@@ -85,14 +109,23 @@
     accept_hostkey=yes
   when: tempdir.stdout is defined and (rbuild_present|failed or (installable_ruby_vers is defined and rbenv_ruby_version not in installable_ruby_vers))
   sudo_user: "{{ rbenv_user }}"
+  tags:
+    - install
+    - install:base
 
 - name: install ruby-build
   command: ./install.sh chdir={{ tempdir.stdout }}/ruby-build
   when: tempdir.stdout is defined and (rbuild_present|failed or (installable_ruby_vers is defined and rbenv_ruby_version not in installable_ruby_vers))
+  tags:
+    - install
+    - install:base
 
 - name: remove temporary directory
   file: path={{ tempdir.stdout }} state=absent
   when: tempdir.stdout is defined and (rbuild_present|failed or (installable_ruby_vers is defined and rbenv_ruby_version not in installable_ruby_vers))
+  tags:
+    - install
+    - install:base
 
 - name: check ruby {{ rbenv_ruby_version }} installed
   shell: "rbenv versions | grep {{ rbenv_ruby_version }}"
@@ -100,34 +133,55 @@
   sudo_user: "{{ rbenv_user }}"
   environment: "{{ rbenv_environment }}"
   ignore_errors: yes
+  tags:
+    - install
+    - install:base
 
 - name: install ruby {{ rbenv_ruby_version }}
   shell: "rbenv install {{ rbenv_ruby_version }} creates={{ rbenv_dir }}/.rbenv/versions/{{ rbenv_ruby_version }}"
   when: ruby_installed|failed
   sudo_user: "{{ rbenv_user }}"
   environment: "{{ rbenv_environment }}"
+  tags:
+    - install
+    - install:base
 
 - name: set global ruby {{ rbenv_ruby_version }}
   shell: "rbenv global {{ rbenv_ruby_version }}"
   sudo_user: "{{ rbenv_user }}"
   environment: "{{ rbenv_environment }}"
+  tags:
+    - install
+    - install:base
 
 - name: install bundler
   shell: "gem install bundler -v {{ rbenv_bundler_version }}"
   sudo_user: "{{ rbenv_user }}"
   environment: "{{ rbenv_environment }}"
+  tags:
+    - install
+    - install:base
 
 - name: remove rbenv version of rake
   file: path="{{ rbenv_dir }}/.rbenv/versions/{{ rbenv_ruby_version }}/bin/rake" state=absent
   when: jenkins_worker is not defined or not jenkins_worker
+  tags:
+    - install
+    - install:base  
 
 - name: install rake gem
   shell: "gem install rake -v {{ rbenv_rake_version }}"
   sudo_user: "{{ rbenv_user }}"
   environment: "{{ rbenv_environment }}"
   when: jenkins_worker is not defined or not jenkins_worker
-
+  tags:
+    - install
+    - install:base
+    
 - name: rehash
   shell: "rbenv rehash"
   sudo_user: "{{ rbenv_user }}"
   environment: "{{ rbenv_environment }}"
+  tags:
+    - install
+    - install:base
\ No newline at end of file

playbooks/roles/supervisor/meta/main.yml

 ---
 dependencies:
-  - common
+

playbooks/roles/supervisor/tasks/main.yml

@@ -55,13 +55,19 @@
     name="{{ supervisor_user }}"
     createhome=no
     shell=/bin/false
+  tags:
+    - install
+    - install:base
 
 - name: create supervisor service user
   user: >
     name="{{ supervisor_service_user }}"
     createhome=no
     shell=/bin/false
-
+  tags:
+    - install
+    - install:base
+    
 - name: create supervisor directories
   file: >
     name={{ item }}
@@ -72,6 +78,9 @@
   with_items:
     - "{{ supervisor_app_dir }}"
     - "{{ supervisor_venv_dir }}"
+  tags:
+    - install
+    - install:base
 
 - name: create service user accessible dirs
   file: >
@@ -83,6 +92,9 @@
   with_items:
     - "{{ supervisor_cfg_dir }}"
     - "{{ supervisor_available_dir }}"
+  tags:
+    - install
+    - install:base
 
 - name: create supervisor directories
   file: >
@@ -93,13 +105,18 @@
   with_items:
     - "{{ supervisor_data_dir }}"
     - "{{ supervisor_log_dir }}"
-
+  tags:
+    - install
+    - install:base
 
 - name: install supervisor in its venv
   pip: >
     name=supervisor virtualenv="{{ supervisor_venv_dir }}" state=present
     extra_args="-i {{ COMMON_PYPI_MIRROR_URL }}"
   sudo_user: "{{ supervisor_user }}"
+  tags:
+    - install
+    - install:base
 
 - name: install supervisor in its venv
   pip: >
@@ -107,11 +124,17 @@
     extra_args="-i {{ COMMON_PYPI_MIRROR_URL }}"
   sudo_user: "{{ supervisor_user }}"
   with_items: supervisor_pip_pkgs
+  tags:
+    - install
+    - install:base
 
 - name: create supervisor upstart job
   template: >
     src=etc/init/supervisor-upstart.conf.j2 dest=/etc/init/{{ supervisor_service }}.conf
     owner=root group=root
+  tags:
+    - install
+    - install:base
 
 # This script is aws specific and looks up instances
 # tags and enables services based on the 'services' tag
@@ -121,6 +144,9 @@
     src=etc/init/pre_supervisor.conf.j2 dest=/etc/init/pre_supervisor.conf
     owner=root group=root
   when: supervisor_service == "supervisor" and disable_edx_services and not devstack
+  tags:
+    - to-remove
+    - aws-specfic
 
 - name: write the pre_suprevisor python script
   copy: >
@@ -130,12 +156,18 @@
     owner={{ supervisor_user }}
     group={{ supervisor_service_user }}
   when: disable_edx_services
+  tags:
+    - to-remove
+    - aws-specfic
 
 - name: create supervisor master config
   template: >
     src=edx/app/supervisor/supervisord.conf.j2 dest={{ supervisor_cfg }}
     owner={{ supervisor_user }} group={{ supervisor_service_user }}
     mode=0644
+  tags:
+    - install
+    - install:configuration
 
 - name: create a symlink for supervisortctl
   # these links are deprecated in favor of the shell wrapper
@@ -144,7 +176,10 @@
     dest={{ COMMON_BIN_DIR }}/{{ supervisor_ctl|basename }}
     state=absent
   when: supervisor_service == "supervisor"
-
+  tags:
+    - install
+    - install:configuration
+  
 - name: create a symlink for supervisor cfg
   # these links are deprecated in favor of the shell wrapper
   file: >
@@ -155,6 +190,9 @@
   with_items:
   - "{{ supervisor_cfg }}"
   - "{{ supervisor_cfg_dir }}"
+  tags:
+    - install
+    - install:configuration
 
 - name: create helper script for running supervisor
   template: >
@@ -163,18 +201,27 @@
     owner={{ supervisor_service_user }}
     mode=0755
   when: supervisor_service == "supervisor"
+  tags:
+    - install
+    - install:configuration
 
 - name: start supervisor
   service: >
     name={{ supervisor_service }}
     state=started
   register: start_supervisor
+  tags:
+    - manage
+    - manager:start
 
   # calling update on supervisor too soon after it
   # starts will result in an errror.
 - name: wait for web port to be available
   wait_for: port={{ supervisor_http_bind_port }} timeout=5
   when: start_supervisor.changed
+  tags:
+    - manage
+    - manage:start
 
   # call supervisorctl update every time, this reloads
   # the supervisorctl config
@@ -186,3 +233,6 @@
   shell:  "{{ supervisor_ctl }} -c {{ supervisor_cfg }} update"
   register: supervisor_update
   changed_when: supervisor_update.stdout is defined and supervisor_update.stdout != ""
+  tags:
+    - manage
+    - manage:start
\ No newline at end of file

playbooks/roles/vhost/defaults/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+##
+# Defaults for role vhost
+# 
+
+#
+# vars are namespace with the module name.
+#
+vhost_role_name: vhost
+
+#
+# OS packages
+#
+
+vhost_debian_pkgs: []
+
+vhost_redhat_pkgs: []

playbooks/roles/vhost/meta/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+##
+# Role includes for role vhost
+# 
+# Example:
+#
+# dependencies:
+#   - {
+#   role: my_role 
+#   my_role_var0: "foo"
+#   my_role_var1: "bar"
+# }

playbooks/roles/vhost/tasks/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+#
+#
+# Tasks for role vhost
+# 
+# Overview:
+# 
+#
+# Dependencies:
+#
+# 
+# Example play:
+#
+#
+
+- name: add ssh-warning banner motd
+  template: >
+    dest=/etc/motd.tail
+    src={{ COMMON_MOTD_TEMPLATE }} mode=0755 owner=root group=root
+
+- name: update ssh config
+  template: >
+    dest=/etc/ssh/sshd_config
+    src=sshd_config.j2 mode=0644 owner=root group=root
+  notify: restart ssh
+