Move XQueue to use edx_service

The main driving force is going from xqueue.(auth|env).json to
/edx/etc/xqueue.yml like other IDAs

There's a few additional cleanups such as normalization of the repo
configuration

playbooks/roles/xqueue/defaults/main.yml

@@ -66,29 +66,11 @@ XQUEUE_SUBMISSION_PROCESSING_DELAY: 1
 # Number of seconds to delay before querying for available push submissions
 XQUEUE_CONSUMER_DELAY: 10
 
-# Internal vars below this line
-#############################################
-
-xqueue_app_dir: "{{ COMMON_APP_DIR }}/xqueue"
-xqueue_code_dir: "{{ xqueue_app_dir }}/xqueue"
-xqueue_data_dir: "{{ COMMON_DATA_DIR }}/xqueue"
-xqueue_venvs_dir: "{{ xqueue_app_dir }}/venvs"
-xqueue_venv_dir: "{{ xqueue_venvs_dir }}/xqueue"
-xqueue_venv_bin: "{{ xqueue_venv_dir }}/bin"
-xqueue_user: "xqueue"
-
-# Default nginx listen port
-# These should be overrided if you want
-# to serve all content on port 80
-xqueue_gunicorn_port: 8040
-xqueue_gunicorn_host: 127.0.0.1
-
-xqueue_environment:
-  DJANGO_SETTINGS_MODULE: '{{ XQUEUE_SETTINGS_MODULE }}'
-  PATH: '{{ xqueue_venv_bin }}:{{ ansible_env.PATH }}'
-
+INSIGHTS_CSRF_COOKIE_SECURE: false
 
-xqueue_env_config:
+# This block of config is dropped into /edx/etc/xqueue.yml
+# and is read in by XQUEUE_SETTINGS
+XQUEUE_CONFIG:
   XQUEUES: "{{ XQUEUE_QUEUES }}"
   LOGGING_ENV: "{{ XQUEUE_LOGGING_ENV }}"
   SYSLOG_SERVER: "{{ XQUEUE_SYSLOG_SERVER }}"
@@ -101,8 +83,6 @@ xqueue_env_config:
   NEWRELIC_APPNAME: "{{ XQUEUE_NEWRELIC_APPNAME }}"
   SUBMISSION_PROCESSING_DELAY: "{{ XQUEUE_SUBMISSION_PROCESSING_DELAY }}"
   CONSUMER_DELAY: "{{ XQUEUE_CONSUMER_DELAY }}"
-
-xqueue_auth_config:
   AWS_ACCESS_KEY_ID:  "{{ XQUEUE_AWS_ACCESS_KEY_ID }}"
   AWS_SECRET_ACCESS_KEY: "{{ XQUEUE_AWS_SECRET_ACCESS_KEY }}"
   SWIFT_USERNAME: "{{ XQUEUE_SWIFT_USERNAME }}"
@@ -127,8 +107,43 @@ xqueue_auth_config:
       OPTIONS: "{{ XQUEUE_MYSQL_OPTIONS }}"
   NEWRELIC_LICENSE_KEY: "{{ NEWRELIC_LICENSE_KEY | default('') }}"
 
-xqueue_source_repo: "https://github.com/edx/xqueue.git"
-xqueue_version: 'master'
+XQUEUE_VERSION: "master"
+XQUEUE_GIT_IDENTITY: !!null
+
+XQUEUE_REPOS:
+  - PROTOCOL: "{{ COMMON_GIT_PROTOCOL }}"
+    DOMAIN: "{{ COMMON_GIT_MIRROR }}"
+    PATH: "{{ COMMON_GIT_PATH }}"
+    REPO: xqueue.git
+    VERSION: "{{ XQUEUE_VERSION }}"
+    DESTINATION: "{{ xqueue_code_dir }}"
+    SSH_KEY: "{{ XQUEUE_GIT_IDENTITY }}"
+
+# Internal vars below this line
+#############################################
+
+xqueue_service_name: "xqueue"
+
+xqueue_app_dir: "{{ COMMON_APP_DIR }}/xqueue"
+xqueue_code_dir: "{{ xqueue_app_dir }}/xqueue"
+xqueue_data_dir: "{{ COMMON_DATA_DIR }}/xqueue"
+xqueue_venvs_dir: "{{ xqueue_app_dir }}/venvs"
+xqueue_venv_dir: "{{ xqueue_venvs_dir }}/xqueue"
+xqueue_venv_bin: "{{ xqueue_venv_dir }}/bin"
+
+xqueue_user: "xqueue"
+xqueue_home: "{{ COMMON_APP_DIR }}/{{ xqueue_service_name }}"
+
+# Default nginx listen port
+# These should be overrided if you want
+# to serve all content on port 80
+xqueue_gunicorn_port: 8040
+xqueue_gunicorn_host: 127.0.0.1
+
+xqueue_environment:
+  DJANGO_SETTINGS_MODULE: '{{ XQUEUE_SETTINGS_MODULE }}'
+  PATH: '{{ xqueue_venv_bin }}:{{ ansible_env.PATH }}'
+
 xqueue_requirements_file:      "{{ xqueue_code_dir }}/requirements.txt"
 xqueue_openstack_requirements_file: "{{ xqueue_code_dir }}/openstack-requirements.txt"
 

playbooks/roles/xqueue/meta/main.yml

@@ -2,3 +2,12 @@
 dependencies:
   - common
   - supervisor
+  - role: edx_service
+    edx_service_name: "{{ xqueue_service_name }}"
+    edx_service_config: "{{ XQUEUE_CONFIG }}"
+    edx_service_repos: "{{ XQUEUE_REPOS }}"
+    edx_service_user: "{{ xqueue_user }}"
+    edx_service_home: "{{ xqueue_home }}"
+    edx_service_packages:
+      debian: "{{ xqueue_debian_pkgs }}"
+      redhat: []

playbooks/roles/xqueue/tasks/deploy.yml

@@ -30,16 +30,11 @@
     - install
     - install:configuration
 
-- name: Copy the configurations to the desired directory
+- name: add gunicorn configuration file
   template:
-    src: "{{ item.src }}"
-    dest: "{{ xqueue_app_dir }}/{{ item.dest }}"
-    mode: "{{ item.mode }}"
+    src: "xqueue_gunicorn.py.j2"
+    dest: "{{ xqueue_app_dir }}/xqueue_gunicorn.py"
   become_user: "{{ xqueue_user }}"
-  with_items:
-    - { src: 'xqueue_gunicorn.py.j2', dest: 'xqueue_gunicorn.py', mode: '0644' }
-    - { src: 'xqueue.env.json.j2', dest: 'xqueue.env.json', mode: '0644' }
-    - { src: 'xqueue.auth.json.j2', dest: 'xqueue.auth.json', mode: '0644' }
   tags:
     - install
     - install:configuration
@@ -55,19 +50,6 @@
     - install
     - install:configuration
 
-# Do A Checkout
-- name: "Git checkout xqueue repo into {{ xqueue_code_dir }}"
-  git:
-    repo: "{{ xqueue_source_repo }}"
-    dest: "{{ xqueue_code_dir }}"
-    version: "{{ xqueue_version }}"
-    accept_hostkey: yes
-  become_user: "{{ xqueue_user }}"
-  register: xqueue_checkout
-  tags:
-    - install
-    - install:code
-
 # Install the python requirements into {{ xqueue_venv_dir }}
 - name: "Install python requirements"
   pip:

playbooks/roles/xqueue/tasks/main.yml

-# requires:
-#  - group_vars/all
-#  - common/tasks/main.yml
 ---
-# Check out xqueue repo to {{ xqueue_code_dir }}
-#
-#
-- name: Create application user
-  user:
-    name: "{{ xqueue_user }}"
-    home: "{{ xqueue_app_dir }}"
-    createhome: no
-    shell: /bin/false
-  tags:
-    - install
-    - install:base
-
-- name: Create xqueue app and venv dir
-  file:
-    path: "{{ item }}"
-    state: directory
-    owner: "{{ xqueue_user }}"
-    group: "{{ common_web_group }}"
-  with_items:
-    - "{{ xqueue_app_dir }}"
-    - "{{ xqueue_venvs_dir }}"
-  tags:
-    - install
-    - install:base
-
-- name: Install a bunch of system packages on which xqueue relies
-  apt:
-    name: "{{ item }}"
-    state: present
-  with_items: "{{ xqueue_debian_pkgs }}"
-  tags:
-    - install
-    - install:system-requirements
-
 - name: write devstack script
   template:
     src: "devstack.sh.j2"

playbooks/roles/xqueue/templates/xqueue.auth.json.j2

-{{ xqueue_auth_config | to_nice_json }}

playbooks/roles/xqueue/templates/xqueue.env.json.j2

-{{ xqueue_env_config | to_nice_json }}