Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
C
configuration
Overview
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
edx
configuration
Commits
f80a9c83
Commit
f80a9c83
authored
Oct 15, 2015
by
Morgan Robertson
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Add lifecycle tags for analytics_pipeline role.
parent
7337797c
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
48 additions
and
0 deletions
+48
-0
playbooks/roles/analytics_pipeline/tasks/main.yml
+48
-0
No files found.
playbooks/roles/analytics_pipeline/tasks/main.yml
View file @
f80a9c83
...
@@ -42,45 +42,69 @@
...
@@ -42,45 +42,69 @@
path="{{ ANALYTICS_PIPELINE_CONFIG_DIR }}"
path="{{ ANALYTICS_PIPELINE_CONFIG_DIR }}"
mode=0755 owner={{ hadoop_common_user }} group={{ hadoop_common_group }}
mode=0755 owner={{ hadoop_common_user }} group={{ hadoop_common_group }}
state=directory
state=directory
tags
:
-
install
-
install:configuration
-
name
:
store output database credentials for analytics pipeline
-
name
:
store output database credentials for analytics pipeline
copy
:
>
copy
:
>
content="{{ ANALYTICS_PIPELINE_OUTPUT_DATABASE | to_json }}"
content="{{ ANALYTICS_PIPELINE_OUTPUT_DATABASE | to_json }}"
dest={{ COMMON_CFG_DIR }}/edx-analytics-pipeline/output.json
dest={{ COMMON_CFG_DIR }}/edx-analytics-pipeline/output.json
mode=0644 owner={{ hadoop_common_user }} group={{ hadoop_common_group }}
mode=0644 owner={{ hadoop_common_user }} group={{ hadoop_common_group }}
tags
:
-
install
-
install:configuration
-
name
:
store input database credentials for analytics pipeline
-
name
:
store input database credentials for analytics pipeline
copy
:
>
copy
:
>
content="{{ ANALYTICS_PIPELINE_INPUT_DATABASE | to_json }}"
content="{{ ANALYTICS_PIPELINE_INPUT_DATABASE | to_json }}"
dest={{ COMMON_CFG_DIR }}/edx-analytics-pipeline/input.json
dest={{ COMMON_CFG_DIR }}/edx-analytics-pipeline/input.json
mode=0644 owner={{ hadoop_common_user }} group={{ hadoop_common_group }}
mode=0644 owner={{ hadoop_common_user }} group={{ hadoop_common_group }}
tags
:
-
install
-
install:configuration
-
name
:
luigi configuration directory created
-
name
:
luigi configuration directory created
file
:
>
file
:
>
path=/etc/luigi
path=/etc/luigi
state=directory
state=directory
mode=755
mode=755
tags
:
-
install
-
install:configuration
-
name
:
luigi configuration file written
-
name
:
luigi configuration file written
template
:
>
template
:
>
src=client.cfg.j2
src=client.cfg.j2
dest=/etc/luigi/client.cfg
dest=/etc/luigi/client.cfg
mode=644
mode=644
tags
:
-
install
-
install:configuration
-
name
:
util library source checked out
-
name
:
util library source checked out
git
:
>
git
:
>
dest={{ analytics_pipeline_util_library.path }} repo={{ analytics_pipeline_util_library.repo }}
dest={{ analytics_pipeline_util_library.path }} repo={{ analytics_pipeline_util_library.repo }}
version={{ analytics_pipeline_util_library.version }}
version={{ analytics_pipeline_util_library.version }}
tags
:
-
install
-
install:code
-
name
:
lib directory created
-
name
:
lib directory created
file
:
>
file
:
>
path={{ HADOOP_COMMON_USER_HOME }}/lib
path={{ HADOOP_COMMON_USER_HOME }}/lib
owner={{ hadoop_common_user }} group={{ hadoop_common_group }} state=directory
owner={{ hadoop_common_user }} group={{ hadoop_common_group }} state=directory
tags
:
-
install
-
install:app-requirements
-
name
:
check if the util library needs to be built
-
name
:
check if the util library needs to be built
stat
:
>
stat
:
>
path={{ HADOOP_COMMON_USER_HOME }}/lib/edx-analytics-hadoop-util.jar
path={{ HADOOP_COMMON_USER_HOME }}/lib/edx-analytics-hadoop-util.jar
register
:
util_lib_built
register
:
util_lib_built
tags
:
-
install
-
install:app-requirements
-
name
:
util library built
-
name
:
util library built
shell
:
>
shell
:
>
...
@@ -89,40 +113,64 @@
...
@@ -89,40 +113,64 @@
{{ hadoop_common_java_home }}/bin/jar cf {{ HADOOP_COMMON_USER_HOME }}/lib/edx-analytics-hadoop-util.jar org/edx/hadoop/input/ManifestTextInputFormat.class &&
{{ hadoop_common_java_home }}/bin/jar cf {{ HADOOP_COMMON_USER_HOME }}/lib/edx-analytics-hadoop-util.jar org/edx/hadoop/input/ManifestTextInputFormat.class &&
chown {{ hadoop_common_user }}:{{ hadoop_common_group }} {{ HADOOP_COMMON_USER_HOME }}/lib/edx-analytics-hadoop-util.jar
chown {{ hadoop_common_user }}:{{ hadoop_common_group }} {{ HADOOP_COMMON_USER_HOME }}/lib/edx-analytics-hadoop-util.jar
when
:
not util_lib_built.stat.exists
when
:
not util_lib_built.stat.exists
tags
:
-
install
-
install:app-requirements
-
name
:
ensure hdfs services are started
-
name
:
ensure hdfs services are started
service
:
>
service
:
>
name=hdfs
name=hdfs
state=started
state=started
tags
:
-
manage
-
manage:start
-
name
:
ensure map reduce services are started
-
name
:
ensure map reduce services are started
service
:
>
service
:
>
name=yarn
name=yarn
state=started
state=started
tags
:
-
manage
-
manage:start
-
name
:
ensure package dir exists in HDFS
-
name
:
ensure package dir exists in HDFS
shell
:
>
shell
:
>
. {{ HADOOP_COMMON_CONF_DIR }}/hadoop-env.sh && hdfs dfs -mkdir -p /edx-analytics-pipeline/packages/
. {{ HADOOP_COMMON_CONF_DIR }}/hadoop-env.sh && hdfs dfs -mkdir -p /edx-analytics-pipeline/packages/
sudo_user
:
"
{{
hadoop_common_user
}}"
sudo_user
:
"
{{
hadoop_common_user
}}"
tags
:
-
install
-
install:app-requirements
-
name
:
ensure util library is in HDFS
-
name
:
ensure util library is in HDFS
shell
:
>
shell
:
>
. {{ HADOOP_COMMON_CONF_DIR }}/hadoop-env.sh && hdfs dfs -put -f {{ HADOOP_COMMON_USER_HOME }}/lib/edx-analytics-hadoop-util.jar /edx-analytics-pipeline/packages/
. {{ HADOOP_COMMON_CONF_DIR }}/hadoop-env.sh && hdfs dfs -put -f {{ HADOOP_COMMON_USER_HOME }}/lib/edx-analytics-hadoop-util.jar /edx-analytics-pipeline/packages/
sudo_user
:
"
{{
hadoop_common_user
}}"
sudo_user
:
"
{{
hadoop_common_user
}}"
tags
:
-
install
-
install:app-requirements
-
name
:
ensure the data directory exists
-
name
:
ensure the data directory exists
shell
:
>
shell
:
>
. {{ HADOOP_COMMON_CONF_DIR }}/hadoop-env.sh && hdfs dfs -mkdir -p {{ ANALYTICS_PIPELINE_HDFS_DATA_DIR }}
. {{ HADOOP_COMMON_CONF_DIR }}/hadoop-env.sh && hdfs dfs -mkdir -p {{ ANALYTICS_PIPELINE_HDFS_DATA_DIR }}
sudo_user
:
"
{{
hadoop_common_user
}}"
sudo_user
:
"
{{
hadoop_common_user
}}"
tags
:
-
install
-
install:base
-
name
:
ensure tracking log file can be read
-
name
:
ensure tracking log file can be read
file
:
>
file
:
>
path={{ COMMON_LOG_DIR }}/tracking/tracking.log
path={{ COMMON_LOG_DIR }}/tracking/tracking.log
mode=0644
mode=0644
ignore_errors
:
yes
ignore_errors
:
yes
tags
:
-
install
-
install:configuration
-
name
:
cron job syncs tracking log file to hdfs
-
name
:
cron job syncs tracking log file to hdfs
cron
:
>
cron
:
>
user={{ hadoop_common_user }}
user={{ hadoop_common_user }}
name="Sync tracking log to HDFS"
name="Sync tracking log to HDFS"
job="{{ HADOOP_COMMON_HOME }}/bin/hdfs dfs -put -f {{ COMMON_LOG_DIR }}/tracking/tracking.log {{ ANALYTICS_PIPELINE_HDFS_DATA_DIR }}/tracking.log"
job="{{ HADOOP_COMMON_HOME }}/bin/hdfs dfs -put -f {{ COMMON_LOG_DIR }}/tracking/tracking.log {{ ANALYTICS_PIPELINE_HDFS_DATA_DIR }}/tracking.log"
tags
:
-
install
-
install:configuration
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment