Skip to content

C
configuration
Commit 1a56e5c1 by Edward Zarecor
Options

Merge pull request #2502 from edx/e0d/cherry-pick-tagging 

E0d/cherry pick tagging
parents 443f13e4 2b967ab2
Showing with 133 additions and 0 deletions
playbooks/roles/analytics_api/tasks/main.yml
......@@ -39,12 +39,18 @@
owner={{ analytics_api_user }}
group={{ analytics_api_user }}
mode=0644
tags:
- install
- install:configuration
- name: "add gunicorn configuration file"
template: >
src=edx/app/analytics_api/analytics_api_gunicorn.py.j2
dest={{ analytics_api_home }}/analytics_api_gunicorn.py
sudo_user: "{{ analytics_api_user }}"
tags:
- install
- install:configuration
- name: install application requirements
pip: >
......@@ -53,6 +59,9 @@
state=present
sudo_user: "{{ analytics_api_user }}"
with_items: analytics_api_requirements
tags:
- install
- install:app-requirements
- name: migrate
shell: >
......@@ -63,6 +72,9 @@
sudo_user: "{{ analytics_api_user }}"
environment: "{{ analytics_api_environment }}"
when: migrate_db is defined and migrate_db|lower == "yes"
tags:
- migrate
- migrate:db
- name: run collectstatic
shell: >
......@@ -70,6 +82,9 @@
{{ analytics_api_home }}/venvs/{{ analytics_api_service_name }}/bin/python manage.py collectstatic --noinput
sudo_user: "{{ analytics_api_user }}"
environment: "{{ analytics_api_environment }}"
tags:
- assets
- assets:gather
- name: create api users
shell: >
......@@ -78,18 +93,27 @@
sudo_user: "{{ analytics_api_user }}"
environment: "{{ analytics_api_environment }}"
with_dict: ANALYTICS_API_USERS
tags:
- manage
- manage:app-users
- name: write out the supervisor wrapper
template: >
src=edx/app/analytics_api/analytics_api.sh.j2
dest={{ analytics_api_home }}/{{ analytics_api_service_name }}.sh
mode=0650 owner={{ supervisor_user }} group={{ common_web_user }}
tags:
- install
- install:configuration
- name: write supervisord config
template: >
src=edx/app/supervisor/conf.d.available/analytics_api.conf.j2
dest="{{ supervisor_available_dir }}/{{ analytics_api_service_name }}.conf"
owner={{ supervisor_user }} group={{ common_web_user }} mode=0644
tags:
- install
- install:configuration
- name: enable supervisor script
file: >
......@@ -98,10 +122,16 @@
state=link
force=yes
when: not disable_edx_services
tags:
- install
- install:configuration
- name: update supervisor configuration
shell: "{{ supervisor_ctl }} -c {{ supervisor_cfg }} update"
when: not disable_edx_services
tags:
- install
- install:configuration
- name: create symlinks from the venv bin dir
file: >
......@@ -112,6 +142,9 @@
- python
- pip
- django-admin.py
tags:
- install
- install:base
- name: create symlinks from the repo dir
file: >
......@@ -120,6 +153,9 @@
state=link
with_items:
- manage.py
tags:
- install
- install:base
- name: restart analytics_api
supervisorctl: >
......@@ -129,3 +165,5 @@
name={{ analytics_api_service_name }}
when: not disable_edx_services
sudo_user: "{{ supervisor_service_user }}"
tags:
- manage:start
playbooks/roles/analytics_pipeline/tasks/main.yml
......@@ -42,45 +42,69 @@
path="{{ ANALYTICS_PIPELINE_CONFIG_DIR }}"
mode=0755 owner={{ hadoop_common_user }} group={{ hadoop_common_group }}
state=directory
tags:
- install
- install:configuration
- name: store output database credentials for analytics pipeline
copy: >
content="{{ ANALYTICS_PIPELINE_OUTPUT_DATABASE | to_json }}"
dest={{ COMMON_CFG_DIR }}/edx-analytics-pipeline/output.json
mode=0644 owner={{ hadoop_common_user }} group={{ hadoop_common_group }}
tags:
- install
- install:configuration
- name: store input database credentials for analytics pipeline
copy: >
content="{{ ANALYTICS_PIPELINE_INPUT_DATABASE | to_json }}"
dest={{ COMMON_CFG_DIR }}/edx-analytics-pipeline/input.json
mode=0644 owner={{ hadoop_common_user }} group={{ hadoop_common_group }}
tags:
- install
- install:configuration
- name: luigi configuration directory created
file: >
path=/etc/luigi
state=directory
mode=755
tags:
- install
- install:configuration
- name: luigi configuration file written
template: >
src=client.cfg.j2
dest=/etc/luigi/client.cfg
mode=644
tags:
- install
- install:configuration
- name: util library source checked out
git: >
dest={{ analytics_pipeline_util_library.path }} repo={{ analytics_pipeline_util_library.repo }}
version={{ analytics_pipeline_util_library.version }}
tags:
- install
- install:code
- name: lib directory created
file: >
path={{ HADOOP_COMMON_USER_HOME }}/lib
owner={{ hadoop_common_user }} group={{ hadoop_common_group }} state=directory
tags:
- install
- install:app-requirements
- name: check if the util library needs to be built
stat: >
path={{ HADOOP_COMMON_USER_HOME }}/lib/edx-analytics-hadoop-util.jar
register: util_lib_built
tags:
- install
- install:app-requirements
- name: util library built
shell: >
......@@ -89,40 +113,64 @@
{{ hadoop_common_java_home }}/bin/jar cf {{ HADOOP_COMMON_USER_HOME }}/lib/edx-analytics-hadoop-util.jar org/edx/hadoop/input/ManifestTextInputFormat.class &&
chown {{ hadoop_common_user }}:{{ hadoop_common_group }} {{ HADOOP_COMMON_USER_HOME }}/lib/edx-analytics-hadoop-util.jar
when: not util_lib_built.stat.exists
tags:
- install
- install:app-requirements
- name: ensure hdfs services are started
service: >
name=hdfs
state=started
tags:
- manage
- manage:start
- name: ensure map reduce services are started
service: >
name=yarn
state=started
tags:
- manage
- manage:start
- name: ensure package dir exists in HDFS
shell: >
. {{ HADOOP_COMMON_CONF_DIR }}/hadoop-env.sh && hdfs dfs -mkdir -p /edx-analytics-pipeline/packages/
sudo_user: "{{ hadoop_common_user }}"
tags:
- install
- install:app-requirements
- name: ensure util library is in HDFS
shell: >
. {{ HADOOP_COMMON_CONF_DIR }}/hadoop-env.sh && hdfs dfs -put -f {{ HADOOP_COMMON_USER_HOME }}/lib/edx-analytics-hadoop-util.jar /edx-analytics-pipeline/packages/
sudo_user: "{{ hadoop_common_user }}"
tags:
- install
- install:app-requirements
- name: ensure the data directory exists
shell: >
. {{ HADOOP_COMMON_CONF_DIR }}/hadoop-env.sh && hdfs dfs -mkdir -p {{ ANALYTICS_PIPELINE_HDFS_DATA_DIR }}
sudo_user: "{{ hadoop_common_user }}"
tags:
- install
- install:base
- name: ensure tracking log file can be read
file: >
path={{ COMMON_LOG_DIR }}/tracking/tracking.log
mode=0644
ignore_errors: yes
tags:
- install
- install:configuration
- name: cron job syncs tracking log file to hdfs
cron: >
user={{ hadoop_common_user }}
name="Sync tracking log to HDFS"
job="{{ HADOOP_COMMON_HOME }}/bin/hdfs dfs -put -f {{ COMMON_LOG_DIR }}/tracking/tracking.log {{ ANALYTICS_PIPELINE_HDFS_DATA_DIR }}/tracking.log"
tags:
- install
- install:configuration
playbooks/roles/insights/tasks/main.yml
......@@ -28,6 +28,9 @@
owner={{ insights_user }}
group={{ insights_user }}
mode=0644
tags:
- install
- install:configuration
- name: install application requirements
pip: >
......@@ -36,22 +39,34 @@
state=present extra_args="--exists-action w"
sudo_user: "{{ insights_user }}"
with_items: insights_requirements
tags:
- install
- install:app-requirements
- name: create nodeenv
shell: >
creates={{ insights_nodeenv_dir }}
{{ insights_home }}/venvs/{{ insights_service_name }}/bin/nodeenv {{ insights_nodeenv_dir }} --prebuilt
sudo_user: "{{ insights_user }}"
tags:
- install
- install:base
- name: install node dependencies
npm: executable={{ insights_nodeenv_bin }}/npm path={{ insights_code_dir }} production=yes
sudo_user: "{{ insights_user }}"
tags:
- install
- install:app-requirements
- name: install bower dependencies
shell: >
chdir={{ insights_code_dir }}
. {{ insights_nodeenv_bin }}/activate && {{ insights_node_bin }}/bower install --production --config.interactive=false
sudo_user: "{{ insights_user }}"
tags:
- install
- install:app-requirements
- name: migrate
shell: >
......@@ -62,12 +77,18 @@
sudo_user: "{{ insights_user }}"
environment: "{{ insights_environment }}"
when: migrate_db is defined and migrate_db|lower == "yes"
tags:
- migrate
- migrate:db
- name: run r.js optimizer
shell: >
chdir={{ insights_code_dir }}
. {{ insights_nodeenv_bin }}/activate && {{ insights_node_bin }}/r.js -o build.js
sudo_user: "{{ insights_user }}"
tags:
- assets
- assets:gather
- name: run collectstatic
shell: >
......@@ -78,24 +99,36 @@
with_items:
- "collectstatic --noinput"
- "compress"
tags:
- assets
- assets:gather
- name: compile translations
shell: >
chdir={{ insights_code_dir }}/analytics_dashboard
. {{ insights_home }}/venvs/{{ insights_service_name }}/bin/activate && i18n_tool generate -v
sudo_user: "{{ insights_user }}"
tags:
- assets
- assets:gather
- name: write out the supervisior wrapper
template: >
src=edx/app/insights/insights.sh.j2
dest={{ insights_app_dir }}/{{ insights_service_name }}.sh
mode=0650 owner={{ supervisor_user }} group={{ common_web_user }}
tags:
- install
- install:configuration
- name: write supervisord config
template: >
src=edx/app/supervisor/conf.d.available/insights.conf.j2
dest="{{ supervisor_available_dir }}/{{ insights_service_name }}.conf"
owner={{ supervisor_user }} group={{ common_web_user }} mode=0644
tags:
- install
- install:configuration
- name: enable supervisor script
file: >
......@@ -104,10 +137,16 @@
state=link
force=yes
when: not disable_edx_services
tags:
- install
- install:configuration
- name: update supervisor configuration
shell: "{{ supervisor_ctl }} -c {{ supervisor_cfg }} update"
when: not disable_edx_services
tags:
- install
- install:configuration
- name: create symlinks from the venv bin dir
file: >
......@@ -118,12 +157,18 @@
- python
- pip
- django-admin.py
tags:
- install
- install:base
- name: create manage.py symlink
file: >
src="{{ insights_manage }}"
dest="{{ COMMON_BIN_DIR }}/manage.{{ insights_service_name }}"
state=link
tags:
- install
- install:base
- name: restart insights
supervisorctl: >
......@@ -133,3 +178,5 @@
name={{ insights_service_name }}
when: not disable_edx_services
sudo_user: "{{ supervisor_service_user }}"
tags:
- manage:start
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment