Commit f80a9c83 by Morgan Robertson

Add lifecycle tags for analytics_pipeline role.

parent 7337797c
...@@ -42,45 +42,69 @@ ...@@ -42,45 +42,69 @@
path="{{ ANALYTICS_PIPELINE_CONFIG_DIR }}" path="{{ ANALYTICS_PIPELINE_CONFIG_DIR }}"
mode=0755 owner={{ hadoop_common_user }} group={{ hadoop_common_group }} mode=0755 owner={{ hadoop_common_user }} group={{ hadoop_common_group }}
state=directory state=directory
tags:
- install
- install:configuration
- name: store output database credentials for analytics pipeline - name: store output database credentials for analytics pipeline
copy: > copy: >
content="{{ ANALYTICS_PIPELINE_OUTPUT_DATABASE | to_json }}" content="{{ ANALYTICS_PIPELINE_OUTPUT_DATABASE | to_json }}"
dest={{ COMMON_CFG_DIR }}/edx-analytics-pipeline/output.json dest={{ COMMON_CFG_DIR }}/edx-analytics-pipeline/output.json
mode=0644 owner={{ hadoop_common_user }} group={{ hadoop_common_group }} mode=0644 owner={{ hadoop_common_user }} group={{ hadoop_common_group }}
tags:
- install
- install:configuration
- name: store input database credentials for analytics pipeline - name: store input database credentials for analytics pipeline
copy: > copy: >
content="{{ ANALYTICS_PIPELINE_INPUT_DATABASE | to_json }}" content="{{ ANALYTICS_PIPELINE_INPUT_DATABASE | to_json }}"
dest={{ COMMON_CFG_DIR }}/edx-analytics-pipeline/input.json dest={{ COMMON_CFG_DIR }}/edx-analytics-pipeline/input.json
mode=0644 owner={{ hadoop_common_user }} group={{ hadoop_common_group }} mode=0644 owner={{ hadoop_common_user }} group={{ hadoop_common_group }}
tags:
- install
- install:configuration
- name: luigi configuration directory created - name: luigi configuration directory created
file: > file: >
path=/etc/luigi path=/etc/luigi
state=directory state=directory
mode=755 mode=755
tags:
- install
- install:configuration
- name: luigi configuration file written - name: luigi configuration file written
template: > template: >
src=client.cfg.j2 src=client.cfg.j2
dest=/etc/luigi/client.cfg dest=/etc/luigi/client.cfg
mode=644 mode=644
tags:
- install
- install:configuration
- name: util library source checked out - name: util library source checked out
git: > git: >
dest={{ analytics_pipeline_util_library.path }} repo={{ analytics_pipeline_util_library.repo }} dest={{ analytics_pipeline_util_library.path }} repo={{ analytics_pipeline_util_library.repo }}
version={{ analytics_pipeline_util_library.version }} version={{ analytics_pipeline_util_library.version }}
tags:
- install
- install:code
- name: lib directory created - name: lib directory created
file: > file: >
path={{ HADOOP_COMMON_USER_HOME }}/lib path={{ HADOOP_COMMON_USER_HOME }}/lib
owner={{ hadoop_common_user }} group={{ hadoop_common_group }} state=directory owner={{ hadoop_common_user }} group={{ hadoop_common_group }} state=directory
tags:
- install
- install:app-requirements
- name: check if the util library needs to be built - name: check if the util library needs to be built
stat: > stat: >
path={{ HADOOP_COMMON_USER_HOME }}/lib/edx-analytics-hadoop-util.jar path={{ HADOOP_COMMON_USER_HOME }}/lib/edx-analytics-hadoop-util.jar
register: util_lib_built register: util_lib_built
tags:
- install
- install:app-requirements
- name: util library built - name: util library built
shell: > shell: >
...@@ -89,40 +113,64 @@ ...@@ -89,40 +113,64 @@
{{ hadoop_common_java_home }}/bin/jar cf {{ HADOOP_COMMON_USER_HOME }}/lib/edx-analytics-hadoop-util.jar org/edx/hadoop/input/ManifestTextInputFormat.class && {{ hadoop_common_java_home }}/bin/jar cf {{ HADOOP_COMMON_USER_HOME }}/lib/edx-analytics-hadoop-util.jar org/edx/hadoop/input/ManifestTextInputFormat.class &&
chown {{ hadoop_common_user }}:{{ hadoop_common_group }} {{ HADOOP_COMMON_USER_HOME }}/lib/edx-analytics-hadoop-util.jar chown {{ hadoop_common_user }}:{{ hadoop_common_group }} {{ HADOOP_COMMON_USER_HOME }}/lib/edx-analytics-hadoop-util.jar
when: not util_lib_built.stat.exists when: not util_lib_built.stat.exists
tags:
- install
- install:app-requirements
- name: ensure hdfs services are started - name: ensure hdfs services are started
service: > service: >
name=hdfs name=hdfs
state=started state=started
tags:
- manage
- manage:start
- name: ensure map reduce services are started - name: ensure map reduce services are started
service: > service: >
name=yarn name=yarn
state=started state=started
tags:
- manage
- manage:start
- name: ensure package dir exists in HDFS - name: ensure package dir exists in HDFS
shell: > shell: >
. {{ HADOOP_COMMON_CONF_DIR }}/hadoop-env.sh && hdfs dfs -mkdir -p /edx-analytics-pipeline/packages/ . {{ HADOOP_COMMON_CONF_DIR }}/hadoop-env.sh && hdfs dfs -mkdir -p /edx-analytics-pipeline/packages/
sudo_user: "{{ hadoop_common_user }}" sudo_user: "{{ hadoop_common_user }}"
tags:
- install
- install:app-requirements
- name: ensure util library is in HDFS - name: ensure util library is in HDFS
shell: > shell: >
. {{ HADOOP_COMMON_CONF_DIR }}/hadoop-env.sh && hdfs dfs -put -f {{ HADOOP_COMMON_USER_HOME }}/lib/edx-analytics-hadoop-util.jar /edx-analytics-pipeline/packages/ . {{ HADOOP_COMMON_CONF_DIR }}/hadoop-env.sh && hdfs dfs -put -f {{ HADOOP_COMMON_USER_HOME }}/lib/edx-analytics-hadoop-util.jar /edx-analytics-pipeline/packages/
sudo_user: "{{ hadoop_common_user }}" sudo_user: "{{ hadoop_common_user }}"
tags:
- install
- install:app-requirements
- name: ensure the data directory exists - name: ensure the data directory exists
shell: > shell: >
. {{ HADOOP_COMMON_CONF_DIR }}/hadoop-env.sh && hdfs dfs -mkdir -p {{ ANALYTICS_PIPELINE_HDFS_DATA_DIR }} . {{ HADOOP_COMMON_CONF_DIR }}/hadoop-env.sh && hdfs dfs -mkdir -p {{ ANALYTICS_PIPELINE_HDFS_DATA_DIR }}
sudo_user: "{{ hadoop_common_user }}" sudo_user: "{{ hadoop_common_user }}"
tags:
- install
- install:base
- name: ensure tracking log file can be read - name: ensure tracking log file can be read
file: > file: >
path={{ COMMON_LOG_DIR }}/tracking/tracking.log path={{ COMMON_LOG_DIR }}/tracking/tracking.log
mode=0644 mode=0644
ignore_errors: yes ignore_errors: yes
tags:
- install
- install:configuration
- name: cron job syncs tracking log file to hdfs - name: cron job syncs tracking log file to hdfs
cron: > cron: >
user={{ hadoop_common_user }} user={{ hadoop_common_user }}
name="Sync tracking log to HDFS" name="Sync tracking log to HDFS"
job="{{ HADOOP_COMMON_HOME }}/bin/hdfs dfs -put -f {{ COMMON_LOG_DIR }}/tracking/tracking.log {{ ANALYTICS_PIPELINE_HDFS_DATA_DIR }}/tracking.log" job="{{ HADOOP_COMMON_HOME }}/bin/hdfs dfs -put -f {{ COMMON_LOG_DIR }}/tracking/tracking.log {{ ANALYTICS_PIPELINE_HDFS_DATA_DIR }}/tracking.log"
tags:
- install
- install:configuration
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment