Merge pull request #2364 from emrob/hack2015/emrob/analytics-tags

Add lifecycle tags for analytics

playbooks/roles/analytics_api/tasks/main.yml

@@ -39,12 +39,18 @@
     owner={{ analytics_api_user }}
     group={{ analytics_api_user }}
     mode=0644
+  tags:
+    - install
+    - install:configuration
 
 - name: "add gunicorn configuration file"
   template: >
     src=edx/app/analytics_api/analytics_api_gunicorn.py.j2
     dest={{ analytics_api_home }}/analytics_api_gunicorn.py
   sudo_user: "{{ analytics_api_user }}"
+  tags:
+    - install
+    - install:configuration
 
 - name: install application requirements
   pip: >
@@ -53,6 +59,9 @@
     state=present
   sudo_user: "{{ analytics_api_user }}"
   with_items: analytics_api_requirements
+  tags:
+    - install
+    - install:app-requirements
 
 - name: migrate
   shell: >
@@ -63,6 +72,9 @@
   sudo_user: "{{ analytics_api_user }}"
   environment: "{{ analytics_api_environment }}"
   when: migrate_db is defined and migrate_db|lower == "yes"
+  tags:
+    - migrate
+    - migrate:db
 
 - name: run collectstatic
   shell: >
@@ -70,6 +82,9 @@
     {{ analytics_api_home }}/venvs/{{ analytics_api_service_name }}/bin/python manage.py collectstatic  --noinput
   sudo_user: "{{ analytics_api_user }}"
   environment: "{{ analytics_api_environment }}"
+  tags:
+    - assets
+    - assets:gather
 
 - name: create api users
   shell: >
@@ -78,18 +93,27 @@
   sudo_user: "{{ analytics_api_user }}"
   environment: "{{ analytics_api_environment }}"
   with_dict: ANALYTICS_API_USERS
+  tags:
+    - manage
+    - manage:app-users
 
 - name: write out the supervisor wrapper
   template: >
     src=edx/app/analytics_api/analytics_api.sh.j2
     dest={{ analytics_api_home }}/{{ analytics_api_service_name }}.sh
     mode=0650 owner={{ supervisor_user }} group={{ common_web_user }}
+  tags:
+    - install
+    - install:configuration
 
 - name: write supervisord config
   template: >
     src=edx/app/supervisor/conf.d.available/analytics_api.conf.j2
     dest="{{ supervisor_available_dir }}/{{ analytics_api_service_name }}.conf"
     owner={{ supervisor_user }} group={{ common_web_user }} mode=0644
+  tags:
+    - install
+    - install:configuration
 
 - name: enable supervisor script
   file: >
@@ -98,10 +122,16 @@
     state=link
     force=yes
   when: not disable_edx_services
+  tags:
+    - install
+    - install:configuration
 
 - name: update supervisor configuration
   shell: "{{ supervisor_ctl }} -c {{ supervisor_cfg }} update"
   when: not disable_edx_services
+  tags:
+    - install
+    - install:configuration
 
 - name: create symlinks from the venv bin dir
   file: >
@@ -112,6 +142,9 @@
   - python
   - pip
   - django-admin.py
+  tags:
+    - install
+    - install:base
 
 - name: create symlinks from the repo dir
   file: >
@@ -120,6 +153,9 @@
     state=link
   with_items:
   - manage.py
+  tags:
+    - install
+    - install:base
 
 - name: restart analytics_api
   supervisorctl: >
@@ -129,3 +165,5 @@
     name={{ analytics_api_service_name }}
   when: not disable_edx_services
   sudo_user: "{{ supervisor_service_user }}"
+  tags:
+    - manage:start

playbooks/roles/analytics_pipeline/tasks/main.yml

@@ -42,45 +42,69 @@
     path="{{ ANALYTICS_PIPELINE_CONFIG_DIR }}"
     mode=0755 owner={{ hadoop_common_user }} group={{ hadoop_common_group }}
     state=directory
+  tags:
+    - install
+    - install:configuration
 
 - name: store output database credentials for analytics pipeline
   copy: >
     content="{{ ANALYTICS_PIPELINE_OUTPUT_DATABASE | to_json }}"
     dest={{ COMMON_CFG_DIR }}/edx-analytics-pipeline/output.json
     mode=0644 owner={{ hadoop_common_user }} group={{ hadoop_common_group }}
+  tags:
+    - install
+    - install:configuration
 
 - name: store input database credentials for analytics pipeline
   copy: >
     content="{{ ANALYTICS_PIPELINE_INPUT_DATABASE | to_json }}"
     dest={{ COMMON_CFG_DIR }}/edx-analytics-pipeline/input.json
     mode=0644 owner={{ hadoop_common_user }} group={{ hadoop_common_group }}
+  tags:
+    - install
+    - install:configuration
 
 - name: luigi configuration directory created
   file: >
     path=/etc/luigi
     state=directory
     mode=755
+  tags:
+    - install
+    - install:configuration
 
 - name: luigi configuration file written
   template: >
     src=client.cfg.j2
     dest=/etc/luigi/client.cfg
     mode=644
+  tags:
+    - install
+    - install:configuration
 
 - name: util library source checked out
   git: >
     dest={{ analytics_pipeline_util_library.path }} repo={{ analytics_pipeline_util_library.repo }}
     version={{ analytics_pipeline_util_library.version }}
+  tags:
+    - install
+    - install:code
 
 - name: lib directory created
   file: >
     path={{ HADOOP_COMMON_USER_HOME }}/lib
     owner={{ hadoop_common_user }} group={{ hadoop_common_group }} state=directory
+  tags:
+    - install
+    - install:app-requirements
 
 - name: check if the util library needs to be built
   stat: >
     path={{ HADOOP_COMMON_USER_HOME }}/lib/edx-analytics-hadoop-util.jar
   register: util_lib_built
+  tags:
+    - install
+    - install:app-requirements
 
 - name: util library built
   shell: >
@@ -89,40 +113,64 @@
     {{ hadoop_common_java_home }}/bin/jar cf {{ HADOOP_COMMON_USER_HOME }}/lib/edx-analytics-hadoop-util.jar org/edx/hadoop/input/ManifestTextInputFormat.class &&
     chown {{ hadoop_common_user }}:{{ hadoop_common_group }} {{ HADOOP_COMMON_USER_HOME }}/lib/edx-analytics-hadoop-util.jar
   when: not util_lib_built.stat.exists
+  tags:
+    - install
+    - install:app-requirements
 
 - name: ensure hdfs services are started
   service: >
     name=hdfs
     state=started
+  tags:
+    - manage
+    - manage:start
 
 - name: ensure map reduce services are started
   service: >
     name=yarn
     state=started
+  tags:
+    - manage
+    - manage:start
 
 - name: ensure package dir exists in HDFS
   shell: >
     . {{ HADOOP_COMMON_CONF_DIR }}/hadoop-env.sh && hdfs dfs -mkdir -p /edx-analytics-pipeline/packages/
   sudo_user: "{{ hadoop_common_user }}"
+  tags:
+    - install
+    - install:app-requirements
 
 - name: ensure util library is in HDFS
   shell: >
     . {{ HADOOP_COMMON_CONF_DIR }}/hadoop-env.sh && hdfs dfs -put -f {{ HADOOP_COMMON_USER_HOME }}/lib/edx-analytics-hadoop-util.jar /edx-analytics-pipeline/packages/
   sudo_user: "{{ hadoop_common_user }}"
+  tags:
+    - install
+    - install:app-requirements
 
 - name: ensure the data directory exists
   shell: >
     . {{ HADOOP_COMMON_CONF_DIR }}/hadoop-env.sh && hdfs dfs -mkdir -p {{ ANALYTICS_PIPELINE_HDFS_DATA_DIR }}
   sudo_user: "{{ hadoop_common_user }}"
+  tags:
+    - install
+    - install:base
 
 - name: ensure tracking log file can be read
   file: >
     path={{ COMMON_LOG_DIR }}/tracking/tracking.log
     mode=0644
   ignore_errors: yes
+  tags:
+    - install
+    - install:configuration
 
 - name: cron job syncs tracking log file to hdfs
   cron: >
     user={{ hadoop_common_user }}
     name="Sync tracking log to HDFS"
     job="{{ HADOOP_COMMON_HOME }}/bin/hdfs dfs -put -f {{ COMMON_LOG_DIR }}/tracking/tracking.log {{ ANALYTICS_PIPELINE_HDFS_DATA_DIR }}/tracking.log"
+  tags:
+    - install
+    - install:configuration