Merge pull request #2509 from edx/e0d/edxapp-tags

E0d/edxapp tags

playbooks/roles/edxapp/tasks/deploy.yml

@@ -3,11 +3,17 @@
     src=edxapp_env.j2 dest={{ edxapp_app_dir }}/edxapp_env
     owner={{ edxapp_user }}  group={{ common_web_user }}
     mode=0644
+  tags:
+    - install
+    - install:configuration
     
 - name: create edxapp configuration dir
   file: >
     path="{{ EDXAPP_CFG_DIR }}" state=directory
     owner="{{ edxapp_user }}" group="{{ common_web_group }}"
+  tags:
+    - install
+    - install:configuration
     
 # Optional auth for git
 - name: create ssh script for git (not authenticated)
@@ -15,22 +21,34 @@
     src=git_ssh_noauth.sh.j2 dest={{ edxapp_git_ssh }}
     owner={{ edxapp_user }} mode=750
   when: not EDXAPP_USE_GIT_IDENTITY
+  tags:
+    - install
+    - install:base
   
 - name: create ssh script for git (authenticated)
   template: >
     src=git_ssh_auth.sh.j2 dest={{ edxapp_git_ssh }}
     owner={{ edxapp_user }} mode=750
   when: EDXAPP_USE_GIT_IDENTITY
+  tags:
+    - install
+    - install:base
 
 - name: install read-only ssh key
   copy: >
     content="{{ EDXAPP_GIT_IDENTITY }}" dest={{ edxapp_git_identity }}
     force=yes owner={{ edxapp_user }} mode=0600
   when: EDXAPP_USE_GIT_IDENTITY
+  tags:
+    - install
+    - install:base
 
 - name: set git fetch.prune to ignore deleted remote refs
   shell: git config --global fetch.prune true
   sudo_user: "{{ edxapp_user }}"
+  tags:
+    - install
+    - install:base
 
 # Do A Checkout
 - name: checkout edx-platform repo into {{ edxapp_code_dir }}
@@ -43,10 +61,16 @@
   environment:
     GIT_SSH: "{{ edxapp_git_ssh }}"
   register: edxapp_platform_checkout
+  tags:
+    - install
+    - install:code
 
 - name: git clean after checking out edx-platform
   shell: cd {{ edxapp_code_dir }} && git clean -xdf
   sudo_user: "{{ edxapp_user }}"
+  tags:
+    - install
+    - install:code
 
 - name: checkout theme
   git: >
@@ -59,11 +83,18 @@
   environment:
     GIT_SSH: "{{ edxapp_git_ssh }}"
   register: edxapp_theme_checkout
+  tags:
+    - install
+    - install:code
 
 - name: Stat each requirements file to ensure it exists
   stat: path="{{ item }}"
   with_items: "{{ edxapp_requirements_with_github_urls }}"
   register: requirement_file_stats
+  tags:
+    - install
+    - install:code
+    - install:app-requirements
 
 # Substitute github mirror in all requirements files
 # This is run on every single deploy
@@ -73,6 +104,10 @@
   sudo_user: "{{ edxapp_user }}"
   when: item.stat.exists
   with_items: "{{ requirement_file_stats.results }}"
+  tags:
+    - install
+    - install:code
+    - install:app-requirements
 
 # Ruby plays that need to be run after platform updates.
 - name: gem install bundler
@@ -82,6 +117,9 @@
     executable=/bin/bash
   environment: "{{ edxapp_environment }}"
   sudo_user: "{{ edxapp_user }}"
+  tags:
+    - install
+    - install:app-requirements
 
 - name: bundle install
   shell: >
@@ -90,6 +128,9 @@
     executable=/bin/bash
   sudo_user: "{{ edxapp_user }}"
   environment: "{{ edxapp_environment }}"
+  tags:
+    - install
+    - install:app-requirements
 
 # Set the npm registry
 # This needs to be done as root since npm is weird about
@@ -99,19 +140,27 @@
     npm config set registry '{{ COMMON_NPM_MIRROR_URL }}'
     creates="{{ edxapp_app_dir }}/.npmrc"
   environment: "{{ edxapp_environment }}"
+  tags:
+    - install
+    - install:app-requirements
 
 # Set the npm registry permissions
 - name: Set the npm registry permissions
   file:
     path="{{ edxapp_app_dir }}/.npmrc"
     owner=edxapp group=edxapp
+  tags:
+    - install
+    - install:app-requirements
 
 # Node play that need to be run after platform updates.
 - name: Install edx-platform npm dependencies
   shell: npm install chdir={{ edxapp_code_dir }}
   sudo_user: "{{ edxapp_user }}"
   environment: "{{ edxapp_environment }}"
-
+  tags:
+    - install
+    - install:app-requirements
 
 # Install the python pre requirements into {{ edxapp_venv_dir }}
 - name : install python pre-requirements
@@ -122,6 +171,9 @@
     extra_args="-i {{ COMMON_PYPI_MIRROR_URL }} --exists-action w"
   sudo_user: "{{ edxapp_user }}"
   environment: "{{ edxapp_environment }}"
+  tags:
+    - install
+    - install:app-requirements
 
 # Install the python modules into {{ edxapp_venv_dir }}
 - name : install python base-requirements
@@ -133,10 +185,16 @@
     chdir={{ edxapp_code_dir }}
   environment: "{{ edxapp_environment }}"
   sudo_user: "{{ edxapp_user }}"
+  tags:
+    - install
+    - install:app-requirements
         
 - stat: path="{{ post_requirements_file }}"
   register: post_requirements
   sudo_user: "{{ edxapp_user }}"
+  tags:
+    - install
+    - install:app-requirements
     
 # Install the python post requirements into {{ edxapp_venv_dir }}
 - name : install python post-requirements
@@ -148,6 +206,9 @@
   sudo_user: "{{ edxapp_user }}"
   environment: "{{ edxapp_environment }}"
   when: post_requirements.stat.exists
+  tags:
+    - install
+    - install:app-requirements
     
 # Install the python paver requirements into {{ edxapp_venv_dir }}
 - name : install python paver-requirements
@@ -158,12 +219,18 @@
     extra_args="-i {{ COMMON_PYPI_MIRROR_URL }} --exists-action w"
   sudo_user: "{{ edxapp_user }}"
   environment: "{{ edxapp_environment }}"
+  tags:
+    - install
+    - install:app-requirements
     
 # Install the python custom requirements into {{ edxapp_venv_dir }}
 
 - stat: path="{{ custom_requirements_file }}"
   register: custom_requirements
   sudo_user: "{{ edxapp_user }}"
+  tags:
+    - install
+    - install:app-requirements
     
 - name : install python custom-requirements
   pip: >
@@ -174,6 +241,9 @@
   sudo_user: "{{ edxapp_user }}"
   environment: "{{ edxapp_environment }}"
   when: custom_requirements.stat.exists
+  tags:
+    - install
+    - install:app-requirements
     
 # Install the final python modules into {{ edxapp_venv_dir }}
 - name : install python post-post requirements
@@ -187,6 +257,9 @@
   - "{{ github_requirements_file }}"
   - "{{ local_requirements_file }}"
   sudo_user: "{{ edxapp_user }}"
+  tags:
+    - install
+    - install:app-requirements
     
 # Private requriements require a ssh key to install, use the same key as the private key for edx-platform
 # If EDXAPP_INSTALL_PRIVATE_REQUIREMENTS is set to true EDXAPP_USE_GIT_IDENTITY must also be true
@@ -203,6 +276,9 @@
   environment:
     GIT_SSH: "{{ edxapp_git_ssh }}"
   when: EDXAPP_INSTALL_PRIVATE_REQUIREMENTS
+  tags:
+    - install
+    - install:app-requirements
     
 # Install any custom extra requirements if defined in EDXAPP_EXTRA_REQUIREMENTS.
 - name: install python extra requirements
@@ -214,6 +290,9 @@
     state=present
   with_items: EDXAPP_EXTRA_REQUIREMENTS
   sudo_user: "{{ edxapp_user }}"
+  tags:
+    - install
+    - install:app-requirements
     
 # If using CAS and you have a function for mapping attributes, install
 # the module here.  The next few tasks set up the python code sandbox
@@ -225,6 +304,9 @@
     extra_args="-i {{ COMMON_PYPI_MIRROR_URL }} --exists-action w"
   sudo_user: "{{ edxapp_user }}"
   when: EDXAPP_CAS_ATTRIBUTE_PACKAGE|length > 0
+  tags:
+    - install
+    - install:app-requirements
     
 # Install the sandbox python modules into {{ edxapp_venv_dir }}
 - name : install sandbox requirements into regular venv
@@ -240,6 +322,9 @@
   - "{{ sandbox_post_requirements }}"
   sudo_user: "{{ edxapp_user }}"
   when: not EDXAPP_PYTHON_SANDBOX
+  tags:
+    - install
+    - install:app-requirements
     
 # The next few tasks set up the python code sandbox
 
@@ -250,6 +335,8 @@
   when: EDXAPP_PYTHON_SANDBOX
   tags:
   - edxapp-sandbox
+  - install
+  - install:app-requirements
 
 - name: code sandbox | Install base sandbox requirements and create sandbox virtualenv
   pip: >
@@ -261,6 +348,8 @@
   when: EDXAPP_PYTHON_SANDBOX
   tags:
   - edxapp-sandbox
+  - install
+  - install:app-requirements
 
 - name: code sandbox | Install sandbox requirements into sandbox venv
   shell: >
@@ -275,22 +364,32 @@
   changed_when: sandbox_install_output.stdout is defined and 'installed' in sandbox_install_output.stdout
   tags:
   - edxapp-sandbox
+  - install
+  - install:app-requirements
 
 - name: code sandbox | put code sandbox into aa-enforce or aa-complain mode, depending on EDXAPP_SANDBOX_ENFORCE
   command: /usr/sbin/{{ edxapp_aa_command }} /etc/apparmor.d/code.sandbox
   when: EDXAPP_PYTHON_SANDBOX
   tags:
   - edxapp-sandbox
+  - install
+  - install:app-requirements
 
 - name: compiling all py files in the edx-platform repo
   shell: "{{ edxapp_venv_bin }}/python -m compileall -x .git/.* {{ edxapp_code_dir }}"
   sudo_user: "{{ edxapp_user }}"
+  tags:
+    - install
+    - install:code
   
   # alternative would be to give {{ common_web_user }} read access
   # to the virtualenv but that permission change will require
   # root access.
 - name: give other read permissions to the virtualenv
   command: chmod -R o+r "{{ edxapp_venv_dir }}"
+  tags:
+    - install
+    - install:code
 
 # https://code.launchpad.net/~wligtenberg/django-openid-auth/mysql_fix/+merge/22726
 # This is necessary for when syncdb is run and the django_openid_auth module is installed,
@@ -299,6 +398,9 @@
   shell: sed -i -e 's/claimed_id = models.TextField(max_length=2047, unique=True/claimed_id = models.TextField(max_length=2047/' {{ edxapp_venv_dir }}/lib/python2.7/site-packages/django_openid_auth/models.py
   when: openid_workaround is defined
   sudo_user: "{{ edxapp_user }}"
+  tags:
+    - install
+    - install:code
   
 # creates the supervisor jobs for the
 # service variants configured, runs
@@ -319,6 +421,8 @@
   sudo_user: "{{ supervisor_service_user }}"
   changed_when: supervisor_update.stdout is defined and supervisor_update.stdout != ""
   when: not disable_edx_services
+  tags:
+    - manage
 
 - name: ensure edxapp has started
   supervisorctl: >
@@ -329,6 +433,8 @@
   sudo_user: "{{ supervisor_service_user }}"
   when: celery_worker is not defined and not disable_edx_services
   with_items: service_variants_enabled
+  tags:
+    - manage
 
 - name: ensure edxapp_workers has started
   supervisorctl: >
@@ -339,6 +445,8 @@
   when: celery_worker is defined and not disable_edx_services
   with_items: edxapp_workers
   sudo_user: "{{ supervisor_service_user }}"
+  tags:
+    - manage
 
 - name: create symlinks from the venv bin dir
   file: >
@@ -349,6 +457,9 @@
   - python
   - pip
   - django-admin.py
+  tags:
+    - install
+    - install:configuration
 
 - name: create symlinks from the repo dir
   file: >
@@ -357,13 +468,23 @@
     state=link
   with_items:
   - manage.py
+  tags:
+    - install
+    - install:configuration
 
 - name: remove read-only ssh key
   file: path={{ edxapp_git_identity }} state=absent
   when: EDXAPP_USE_GIT_IDENTITY
+  tags:
+    - install
+    - install:configuration
+    - install:code
   
 - include: tag_ec2.yml tags=deploy
   when: COMMON_TAG_EC2_INSTANCE
+  tags:
+    - remove
+    - aws
 
 - set_fact: edxapp_installed=true
 
@@ -376,6 +497,8 @@
   when: edxapp_installed is defined and celery_worker is not defined and not disable_edx_services
   sudo_user: "{{ supervisor_service_user }}"
   with_items: service_variants_enabled
+  tags:
+    - manage
 
 - name: restart edxapp_workers
   supervisorctl: >
@@ -386,3 +509,5 @@
   when: edxapp_installed is defined and celery_worker is defined and not disable_edx_services
   with_items: edxapp_workers
   sudo_user: "{{ common_web_user }}"
+  tags:
+    - manage

playbooks/roles/edxapp/tasks/main.yml

@@ -6,6 +6,9 @@
   user: >
     name="{{ edxapp_user }}" home="{{ edxapp_app_dir }}"
     createhome=no shell=/bin/false
+  tags:
+    - install
+    - install:base
 
 - name: create edxapp user dirs
   file: >
@@ -20,12 +23,18 @@
     - "{{ edxapp_staticfile_dir }}"
     - "{{ edxapp_course_static_dir }}"
     - "{{ edxapp_course_data_dir }}"
+  tags:
+    - install
+    - install:base
     
 # var should have more permissive permissions than the rest
 - name: create edxapp var dir
   file: >
     path={{ edxapp_data_dir }} state=directory mode=0775
     owner="{{ edxapp_user }}" group="{{ common_web_group }}"
+  tags:
+    - install
+    - install:base
 
 # directory to import the courses from github
 - name: create directory to import the courses from github
@@ -43,11 +52,17 @@
     state=link
     owner="{{ edxapp_user }}"
     group="{{ common_web_group }}"
+  tags:
+    - install
+    - install:base
     
 - name: create edxapp log dir
   file: >
     path="{{ edxapp_log_dir }}" state=directory
     owner="{{ common_log_user }}" group="{{ common_log_user }}"
+  tags:
+    - install
+    - install:base
     
 - name: create web-writable edxapp data dirs
   file: >
@@ -58,19 +73,31 @@
     - "{{ edxapp_course_data_dir }}"
     - "{{ edxapp_upload_dir }}"
     - "{{ edxapp_media_dir }}"
+  tags:
+    - install
+    - install:base
     
 # adding chris-lea nodejs repo
 - name: add ppas for current versions of nodejs
   apt_repository: repo="{{ edxapp_chrislea_ppa }}"
+  tags:
+    - install
+    - install:base
   
 - name: install system packages on which LMS and CMS rely
   apt: pkg={{','.join(edxapp_debian_pkgs)}} state=present update_cache=yes
+  tags:
+    - install
+    - install:base
   
 - name: set up edxapp .npmrc
   template:
     src=.npmrc.j2 dest={{ edxapp_app_dir }}/.npmrc
     owner={{ edxapp_user }} group={{ common_web_group }}
     mode=0600
+  tags:
+    - install
+    - install:base
     
 - name: create log directories for service variants
   file: >
@@ -78,6 +105,9 @@
     owner={{ common_log_user }} group={{ common_log_user }}
     mode=0750
   with_items: service_variants_enabled
+  tags:
+    - install
+    - install:base
     
 # Set up the python sandbox execution environment
 - include: python_sandbox_env.yml tags=deploy

playbooks/roles/edxapp/tasks/service_variant_config.yml

@@ -3,33 +3,44 @@
     src={{ item }}.env.json.j2
     dest={{ edxapp_app_dir }}/{{ item }}.env.json
   sudo_user: "{{ edxapp_user }}"
-  tags: edxapp_cfg
   with_items: service_variants_enabled
+  tags:
+    - install
+    - install:configuration
+    - edxapp_cfg
 
 - name: "create {{ item }} auth file"
   template: >
     src={{ item }}.auth.json.j2
     dest={{ edxapp_app_dir }}/{{ item }}.auth.json
   sudo_user: "{{ edxapp_user }}"
-  tags: edxapp_cfg
   with_items: service_variants_enabled
+  tags:
+    - install
+    - install:configuration
+    - edxapp_cfg
 
 - name: "create {{ item }} yaml application config"
   template: >
     src={{ item }}.env.yaml.j2
     dest={{ EDXAPP_CFG_DIR }}/{{ item }}.env.yaml
   sudo_user: "{{ edxapp_user }}"
-  tags: edxapp_cfg
   with_items: service_variants_enabled
+  tags:
+    - install
+    - install:configuration
+    - edxapp_cfg
 
 - name: "create {{ item }} yaml auth file"
   template: >
     src={{ item }}.auth.yaml.j2
     dest={{ EDXAPP_CFG_DIR }}/{{ item }}.auth.yaml
   sudo_user: "{{ edxapp_user }}"
-  tags: edxapp_cfg
   with_items: service_variants_enabled
-
+  tags:
+    - install
+    - install:configuration
+    - edxapp_cfg
 
 # write the supervisor scripts for the service variants
 
@@ -40,6 +51,9 @@
     group={{ supervisor_user }}
   with_items: service_variants_enabled
   sudo_user: "{{ supervisor_user }}"
+  tags:
+    - install
+    - install:configuration
 
 - name: writing edxapp supervisor script
   template: >
@@ -47,12 +61,18 @@
     owner={{ supervisor_user }}
     group={{ supervisor_user }}
   sudo_user: "{{ supervisor_user }}"
+  tags:
+    - install
+    - install:configuration
 
 - name: "add gunicorn configuration files"
   template: >
     src={{ item }}_gunicorn.py.j2 dest={{ edxapp_app_dir }}/{{ item }}_gunicorn.py
   with_items: service_variants_enabled
   sudo_user: "{{ edxapp_user }}"
+  tags:
+    - install
+    - install:configuration
 
 # write the supervisor script for celery workers
 
@@ -62,6 +82,9 @@
     owner={{ supervisor_user }}
     group={{ supervisor_user }}
   sudo_user: "{{ supervisor_user }}"
+  tags:
+    - install
+    - install:configuration
 
 # Enable the supervisor jobs
 - name: "enable {{ item }} supervisor script"
@@ -73,6 +96,9 @@
   with_items: service_variants_enabled
   when: celery_worker is not defined and not disable_edx_services
   sudo_user: "{{ supervisor_user }}"
+  tags:
+    - install
+    - install:configuration
 
 - name: "enable edxapp supervisor script"
   file: >
@@ -82,6 +108,9 @@
     force=yes
   when: celery_worker is not defined and not disable_edx_services
   sudo_user: "{{ supervisor_user }}"
+  tags:
+    - install
+    - install:configuration
 
 - name: "enable celery worker supervisor script"
   file: >
@@ -91,6 +120,9 @@
     force=yes
   when: celery_worker is defined and not disable_edx_services
   sudo_user: "{{ supervisor_user }}"
+  tags:
+    - install
+    - install:configuration
 
 - name: create helper scripts for managing edxapp
   template: >
@@ -101,6 +133,9 @@
   with_nested:
     - edxapp_helper_scripts
     - service_variants_enabled
+  tags:
+    - install
+    - install:configuration
 
 # Syncdb with migrate when the migrate user is overridden in extra vars
 - name: migrate
@@ -111,6 +146,8 @@
     DB_MIGRATION_PASS: "{{ COMMON_MYSQL_MIGRATE_PASS }}"
     EDX_PLATFORM_SETTINGS_OVERRIDE: "aws_migrate"
   with_items: service_variants_enabled
+  tags:
+    - migrate
 
 # Gather assets using paver if possible
 
@@ -119,3 +156,5 @@
   when: celery_worker is not defined and not devstack and item != "lms-preview"
   tags: gather_static_assets
   with_items: service_variants_enabled
+  tags:
+    - assets

playbooks/roles/edxapp_common/tasks/main.yml

@@ -3,3 +3,6 @@
 - name: Install system packages
   apt: pkg={{','.join(edxapp_common_debian_pkgs)}}
        state=present update_cache=yes
+  tags:
+    - install
+    - install:base