Add ecommerce_worker role

Allows deployment of the ecommerce worker, a Celery-based IDA for asynchronously handling ecommerce tasks. ECOM-2306.

playbooks/edx-east/ecommerce_worker.yml

+- name: Deploy edX Ecommerce Worker
+  hosts: all
+  sudo: True
+  gather_facts: True
+  vars:
+    ENABLE_DATADOG: False
+    ENABLE_SPLUNKFORWARDER: False
+    ENABLE_NEWRELIC: False
+  roles:
+    - aws
+    - ecommerce_worker
+    - role: datadog
+      when: COMMON_ENABLE_DATADOG
+    - role: splunkforwarder
+      when: COMMON_ENABLE_SPLUNKFORWARDER
+    - role: newrelic
+      when: COMMON_ENABLE_NEWRELIC

playbooks/roles/ecommerce/defaults/main.yml

@@ -99,7 +99,9 @@ ECOMMERCE_THEME_SCSS: 'sass/themes/default.scss'
 # Celery
 ECOMMERCE_BROKER_USERNAME: 'celery'
 ECOMMERCE_BROKER_PASSWORD: 'celery'
-ECOMMERCE_BROKER_HOST: '127.0.0.1'
+# Used as the default RabbitMQ IP.
+ECOMMERCE_BROKER_HOST: '{{ ansible_default_ipv4.address }}'
+# Used as the default RabbitMQ port.
 ECOMMERCE_BROKER_PORT: 5672
 ECOMMERCE_BROKER_URL: 'amqp://{{ ECOMMERCE_BROKER_USERNAME }}:{{ ECOMMERCE_BROKER_PASSWORD }}@{{ ECOMMERCE_BROKER_HOST }}:{{ ECOMMERCE_BROKER_PORT }}'
 

playbooks/roles/ecommerce_worker/defaults/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+# Defaults for role ecommerce_worker.
+#
+
+ECOMMERCE_WORKER_GIT_IDENTITY: !!null
+ECOMMERCE_WORKER_VERSION: 'master'
+
+ECOMMERCE_WORKER_REPOS:
+  - PROTOCOL: '{{ COMMON_GIT_PROTOCOL }}'
+    DOMAIN: '{{ COMMON_GIT_MIRROR }}'
+    PATH: '{{ COMMON_GIT_PATH }}'
+    REPO: ecommerce-worker.git
+    VERSION: '{{ ECOMMERCE_WORKER_VERSION }}'
+    DESTINATION: '{{ ecommerce_worker_code_dir }}'
+    SSH_KEY: '{{ ECOMMERCE_WORKER_GIT_IDENTITY }}'
+
+# Requires that New Relic be enabled via COMMON_ENABLE_NEWRELIC, and that
+# a key be provided via NEWRELIC_LICENSE_KEY.
+ECOMMERCE_WORKER_NEWRELIC_APPNAME: '{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-{{ ecommerce_worker_service_name }}'
+
+# CELERY
+ECOMMERCE_WORKER_BROKER_USERNAME: 'celery'
+ECOMMERCE_WORKER_BROKER_PASSWORD: 'celery'
+# Used as the default RabbitMQ IP.
+ECOMMERCE_WORKER_BROKER_HOST: '{{ ansible_default_ipv4.address }}'
+# Used as the default RabbitMQ port.
+ECOMMERCE_WORKER_BROKER_PORT: 5672
+# Default broker URL. See http://celery.readthedocs.org/en/latest/configuration.html#broker-url.
+ECOMMERCE_WORKER_BROKER_URL: 'amqp://{{ ECOMMERCE_WORKER_BROKER_USERNAME }}:{{ ECOMMERCE_WORKER_BROKER_PASSWORD }}@{{ ECOMMERCE_WORKER_BROKER_HOST }}:{{ ECOMMERCE_WORKER_BROKER_PORT }}'
+ECOMMERCE_WORKER_CONCURRENCY: 4
+# END CELERY
+
+# ORDER FULFILLMENT
+# Absolute URL used to construct API calls against the ecommerce service.
+ECOMMERCE_WORKER_ECOMMERCE_API_ROOT: 'http://127.0.0.1:8002/api/v2/'
+
+# Long-lived access token used by Celery workers to authenticate against the ecommerce service.
+ECOMMERCE_WORKER_WORKER_ACCESS_TOKEN: 'your-secret-here'
+
+# Maximum number of retries before giving up on the fulfillment of an order.
+# For reference, 11 retries with exponential backoff yields a maximum waiting
+# time of 2047 seconds (about 30 minutes). Defaulting this to None could yield
+# unwanted behavior: infinite retries.
+ECOMMERCE_WORKER_MAX_FULFILLMENT_RETRIES: 11
+# END ORDER FULFILLMENT
+
+ECOMMERCE_WORKER_SERVICE_CONFIG:
+  BROKER_URL: '{{ ECOMMERCE_WORKER_BROKER_URL }}'
+  ECOMMERCE_API_ROOT: '{{ ECOMMERCE_WORKER_ECOMMERCE_API_ROOT }}'
+  WORKER_ACCESS_TOKEN: '{{ ECOMMERCE_WORKER_WORKER_ACCESS_TOKEN }}'
+  MAX_FULFILLMENT_RETRIES: '{{ ECOMMERCE_WORKER_MAX_FULFILLMENT_RETRIES }}'
+
+ecommerce_worker_environment:
+  WORKER_CONFIGURATION_MODULE: 'ecommerce_worker.configuration.production'
+  ECOMMERCE_WORKER_CFG: '{{ COMMON_CFG_DIR }}/{{ ecommerce_worker_service_name }}.yml'
+
+ecommerce_worker_service_name: 'ecommerce_worker'
+ecommerce_worker_user: '{{ ecommerce_worker_service_name }}'
+ecommerce_worker_home: '{{ COMMON_APP_DIR }}/{{ ecommerce_worker_service_name }}'
+ecommerce_worker_code_dir: '{{ ecommerce_worker_home }}/{{ ecommerce_worker_service_name }}'
+
+ecommerce_worker_log_dir: '{{ COMMON_LOG_DIR }}/{{ ecommerce_worker_service_name }}'
+
+ecommerce_worker_requirements_base: '{{ ecommerce_worker_code_dir }}/requirements'
+ecommerce_worker_requirements:
+  - production.txt
+  - optional.txt
+
+# OS packages
+ecommerce_worker_debian_pkgs: []
+ecommerce_worker_redhat_pkgs: []

playbooks/roles/ecommerce_worker/meta/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+# Includes for role ecommerce_worker.
+# 
+
+dependencies:
+  - common
+  - role: edx_service
+    edx_service_name: "{{ ecommerce_worker_service_name }}"
+    edx_service_config: "{{ ECOMMERCE_WORKER_SERVICE_CONFIG }}"
+    edx_service_repos: "{{ ECOMMERCE_WORKER_REPOS }}"
+    edx_service_user: "{{ ecommerce_worker_user }}"
+    edx_service_home: "{{ ecommerce_worker_home }}"
+    edx_service_packages:
+      debian: "{{ ecommerce_worker_debian_pkgs }}"
+      redhat: "{{ ecommerce_worker_redhat_pkgs }}"
+  - supervisor

playbooks/roles/ecommerce_worker/tasks/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+# Tasks for role ecommerce_worker.
+# 
+
+- name: install application requirements
+  pip:
+    requirements: '{{ ecommerce_worker_requirements_base }}/{{ item }}'
+    virtualenv: '{{ ecommerce_worker_home }}/venvs/{{ ecommerce_worker_service_name }}'
+    state: present
+  sudo_user: '{{ ecommerce_worker_user }}'
+  with_items: ecommerce_worker_requirements
+
+- name: write out the supervisor wrapper
+  template:
+    src: edx/app/ecommerce_worker/ecommerce_worker.sh.j2
+    dest: '{{ ecommerce_worker_home }}/{{ ecommerce_worker_service_name }}.sh'
+    mode: 0650
+    owner: '{{ supervisor_user }}'
+    group: '{{ common_web_user }}'
+
+- name: write supervisord config
+  template:
+    src: edx/app/supervisor/conf.d.available/ecommerce_worker.conf.j2
+    dest: '{{ supervisor_available_dir }}/{{ ecommerce_worker_service_name }}.conf'
+    owner: '{{ supervisor_user }}'
+    group: '{{ common_web_user }}'
+    mode: 0644
+
+- name: setup the ecommerce_worker env file
+  template:
+    src: './{{ ecommerce_worker_home }}/{{ ecommerce_worker_service_name }}_env.j2'
+    dest: '{{ ecommerce_worker_home }}/ecommerce_worker_env'
+    owner: '{{ ecommerce_worker_user }}'
+    group: '{{ ecommerce_worker_user }}'
+    mode: 0644
+
+- name: enable supervisor script
+  file:
+    src: '{{ supervisor_available_dir }}/{{ ecommerce_worker_service_name }}.conf'
+    dest: '{{ supervisor_cfg_dir }}/{{ ecommerce_worker_service_name }}.conf'
+    state: link
+    force: yes
+  when: not disable_edx_services
+
+- name: update supervisor configuration
+  shell: '{{ supervisor_ctl }} -c {{ supervisor_cfg }} update'
+  when: not disable_edx_services
+
+- name: create symlinks from the venv bin dir
+  file:
+    src: '{{ ecommerce_worker_home }}/venvs/{{ ecommerce_worker_service_name }}/bin/{{ item }}'
+    dest: '{{ COMMON_BIN_DIR }}/{{ item }}.ecommerce_worker'
+    state: link
+  with_items:
+  - python
+  - pip
+
+- name: restart the applicaton
+  supervisorctl:
+    state: restarted
+    supervisorctl_path: '{{ supervisor_ctl }}'
+    config: '{{ supervisor_cfg }}'
+    name: '{{ ecommerce_worker_service_name }}'
+  when: not disable_edx_services
+  sudo_user: '{{ supervisor_service_user }}'

playbooks/roles/ecommerce_worker/templates/edx/app/ecommerce_worker/ecommerce_worker.sh.j2

+#!/usr/bin/env bash
+
+# {{ ansible_managed }}
+
+{% set ecommerce_worker_venv_bin = ecommerce_worker_home + '/venvs/' + ecommerce_worker_service_name + '/bin' %}
+{% if COMMON_ENABLE_NEWRELIC_APP %}
+{% set executable = ecommerce_worker_venv_bin + '/newrelic-admin run-program ' + ecommerce_worker_venv_bin + '/celery' %}
+{% else %}
+{% set executable = ecommerce_worker_venv_bin + '/celery' %}
+{% endif %}
+
+{% if COMMON_ENABLE_NEWRELIC_APP %}
+export NEW_RELIC_APP_NAME='{{ ECOMMERCE_WORKER_NEWRELIC_APPNAME }}'
+export NEW_RELIC_LICENSE_KEY='{{ NEWRELIC_LICENSE_KEY }}'
+{% endif -%}
+
+source {{ ecommerce_worker_home }}/ecommerce_worker_env
+{{ executable }} -A ecommerce_worker worker --app=ecommerce_worker.celery_app:app --concurrency={{ ECOMMERCE_WORKER_CONCURRENCY }} --loglevel=info

playbooks/roles/ecommerce_worker/templates/edx/app/ecommerce_worker/ecommerce_worker_env.j2

+# {{ ansible_managed }}
+
+{% for name,value in ecommerce_worker_environment.items() -%}
+{%- if value -%}
+export {{ name }}="{{ value }}"
+{% endif %}
+{%- endfor %}

playbooks/roles/ecommerce_worker/templates/edx/app/supervisor/conf.d.available/ecommerce_worker.conf.j2

+#
+# {{ ansible_managed }}
+# 
+[program:{{ ecommerce_worker_service_name }}]
+
+command={{ ecommerce_worker_home }}/{{ ecommerce_worker_service_name }}.sh
+user={{ common_web_user }}
+directory={{ ecommerce_worker_code_dir }}
+stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log
+stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log
+killasgroup=true
+stopasgroup=true

vagrant/base/test_playbook/Vagrantfile

 # Vagrantfile for testing throwaway Ansible playbooks. To use, create a playbook
 # under /playbooks (e.g., "foo.yml"), export its name (e.g., "foo") as the value
-# of the environment variable VAGRANT_ANSIBLE_PLAYBOOK, and execute `vagrant provision`
+# of the environment variable VAGRANT_ANSIBLE_PLAYBOOK, and execute `vagrant up`
 # from within this directory.
 MEMORY = 2048
 CPU_COUNT = 2