Merge pull request #3297 from edx/e0d/upgrade-rabbit-rebase

E0d/upgrade rabbit rebase

CHANGELOG.md

+- Role: rabbitmq
+  - Upgraded to 3.6.9
+  - Switched to a PPA rather than a .deb hosted in S3
+  - Note that you generally cannot upgrade RabbitMQ live in place https://www.rabbitmq.com/clustering.html
+    this is particularly true coming from 3.2 to 3.6.  We are using the shovel plugin to move tasks across clusters
+    but their documentation covers different scenarios.
 - Role: edxapp
   - Set preload_app to False in gunicorn config for LMS and Studio.
 - Role: analytics_api

docker/build/rabbitmq/Dockerfile

-FROM edxops/precise-common:latest
+FROM edxops/xenial-common:latest
 MAINTAINER edxops
 
-USER root
 ADD . /edx/app/edx_ansible/edx_ansible
+COPY docker/build/rabbitmq/ansible_overrides.yml /
+COPY docker/build/rabbitmq/run_rabbitmq.sh /
+RUN chmod +x /run_rabbitmq.sh
+
 WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays
 
-# Role is currently untagged
-RUN /edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook rabbitmq.yml -c local \
-   -i '127.0.0.1,'
+RUN /edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook rabbitmq.yml \
+    -i '127.0.0.1,' -c local \
+    -t 'install,manage:app-users' \
+    -e@/ansible_overrides.yml
 
-USER rabbitmq
-# TBD what we want to run rabbit under
+WORKDIR /edx/app
 EXPOSE 15672 5672
+CMD ["/run_rabbitmq.sh"]
+

docker/build/rabbitmq/ansible_overrides.yml

+---
+FLOCK_TLD: "edx"

docker/build/rabbitmq/run_rabbitmq.sh

+#!/bin/bash
+
+ulimit -n 1024
+exec rabbitmq-server $@
\ No newline at end of file

docker/build/rabbitmq/sample.json

+{
+    "test-pull":
+    {
+        "AUTH": [
+            "lms", 
+            "password"
+        ], 
+        "CONNECTIONS": 2, 
+        "HANDLERS": [
+            {
+                "CODEJAIL": {
+                    "name": "demo", 
+                    "python_bin": "/edx/app/xqwatcher/venvs/demo/bin/python", 
+                    "user": "demo"
+                }, 
+                "HANDLER": "xqueue_watcher.jailedgrader.JailedGrader", 
+                "KWARGS": {
+                    "grader_root": "../data/edx-demo-course/graders/"
+                }
+            }
+        ], 
+        "SERVER": "http://xqueue.edx"
+    }
+}

playbooks/roles/common_vars/defaults/main.yml

@@ -108,6 +108,7 @@ common_redhat_pkgs:
   - unzip
   - acl
 common_debian_pkgs:
+  - apt-transport-https
   - ntp
   - acl
   - lynx-cur

playbooks/roles/rabbitmq/defaults/main.yml

-#Variables for rabbitmq
 ---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+#
+# Defaults for role rabbitmq
+# 
+
+#Setting this to false means we will not check your rabbit version.
+#It is not safe to upgrade rabbit in place except for certain minor versions.
+CHECK_RABBIT_VERSION: true
 
 rabbitmq_app_dir: "{{ COMMON_APP_DIR }}/rabbitmq"
 rabbitmq_data_dir: "{{ COMMON_DATA_DIR }}/rabbitmq"
@@ -31,20 +45,18 @@ RABBITMQ_CLUSTERED_HOSTS: []
 # https://www.rabbitmq.com/production-checklist.html
 RABBITMQ_VM_MEMORY_HIGH_WATERMARK: 0.4
 
+RABBITMQ_VERSION: 3.6.9-1
+
 # Internal role variables below this line
 
 # option to force deletion of the mnesia dir
 rabbitmq_refresh: false
 
-# We mirror the deb package for rabbitmq-server because
-# nodes need to be running the same version
-rabbitmq_pkg_url: "http://files.edx.org/rabbitmq_packages/rabbitmq-server_3.2.3-1_all.deb"
-rabbitmq_pkg: "rabbitmq-server"
+rabbitmq_repo: "deb https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu/ {{ ansible_distribution_release }} main"
+rabbitmq_repo_key: "https://packagecloud.io/rabbitmq/rabbitmq-server/gpgkey"
+
 rabbitmq_debian_pkgs:
-  - python-software-properties
-  # for installing the deb package with
-  # dependencies
-  - gdebi
+  - "apt-transport-https"
 
 rabbitmq_config_dir: "/etc/rabbitmq"
 rabbitmq_cookie_dir: "/var/lib/rabbitmq"

playbooks/roles/rabbitmq/tasks/main.yml

@@ -2,37 +2,81 @@
 # It is recommended that this role be played with serial set to 1 because
 # There is a bug with initializing multiple nodes in the HA cluster at once
 # http://rabbitmq.1065348.n5.nabble.com/Rabbitmq-boot-failure-with-quot-tables-not-present-quot-td24494.html
-
-- name: Install python-software-properties if debian
+#
+- name: install packages needed by rabbit
   apt:
     name: "{{ item }}"
     state: present
+    install_recommends: yes
+    force: yes
+    update_cache: yes
   with_items: "{{ rabbitmq_debian_pkgs }}"
+  tags:
+    - install
+    - install:app-requirements
 
-- name: Fetch the rabbitmq server deb
-  get_url:
-    url: "{{ rabbitmq_pkg_url }}"
-    dest: "/var/tmp/{{ rabbitmq_pkg_url | basename }}"
+- name: trust rabbit's packagecloud repository
+  apt_key:
+    url: "{{ rabbitmq_repo_key }}"
+    state: present
+  tags:
+    - "install"
+    - "install:app-requirements"
 
-- name: Check if rabbit is installed
-  shell: 'dpkg -s rabbitmq-server >/dev/null 2>&1 || echo "not installed"'
-  register: is_installed
+- name: add rabbit's packagecloud repository
+  apt_repository:
+    repo: "{{ rabbitmq_repo }}"
+    state: present
+    update_cache: yes
+  tags:
+    - "install"
+    - "install:app-requirements"
 
-- name: Install rabbit package using gdebi
-  shell: "gdebi --n {{ rabbitmq_pkg_url|basename }}"
+# If we don't set pipefail first, `||` will be looking at the exit code of the last command in the pipe
+- name: Check if rabbit is installed
+  shell: |
+    set -o pipefail
+    dpkg -s rabbitmq-server | grep Version | sed -r 's/.*: (.*)/\1/' || echo 'not installed'
   args:
-    chdir: /var/tmp
-  when: is_installed.stdout is defined and is_installed.stdout == "not installed"
+    executable: /bin/bash
+  register: installed_version
+  tags:
+    - "install"
+    - "install:app-requirements"
+
+- name: Fail if wrong rabbit version is installed
+  fail: Expected rabbitmq version {{ RABBITMQ_VERSION }}, found {{ installed_version.stdout }}
+  when: CHECK_RABBIT_VERSION and installed_version.stdout is defined and installed_version.stdout not in [RABBITMQ_VERSION, 'not installed']
+  tags:
+    - "install"
+    - "install:app-requirements"
+
+- name: Install rabbit package
+  apt:
+    name: "rabbitmq-server={{ RABBITMQ_VERSION }}"
+    state: present
+    force: yes
+    update_cache: yes
+  when: installed_version.stdout is defined and installed_version.stdout == "not installed"
+  tags:
+    - "install"
+    - "install:app-requirements"
 
 - name: Stop rabbit cluster
   service:
     name: rabbitmq-server
     state: stopped
+  tags:
+    - "install"
+    - "install:app-configuration"
 
 # In case there are lingering processes, ignore errors
 # silently
 - name: Send sigterm to any running rabbitmq processes
   shell: "pkill -u rabbitmq || true"
+  tags:
+    - "install"
+    - "install:app-configuration"
 
 - name: Create rabbitmq edx directories
   file:
@@ -43,6 +87,9 @@
   with_items:
     - "{{ rabbitmq_app_dir }}"
     - "{{ rabbitmq_log_dir }}"
+  tags:
+    - "install"
+    - "install:app-configuration"
 
 - name: Add queue monitoring script
   template:
@@ -52,6 +99,8 @@
     group: "{{ rabbitmq_group }}"
     mode: "0755"
   tags:
+    - "install"
+    - "install:app-configuration"
     - "monitoring"
 
 - name: Add RabbitMQ memory usage script
@@ -62,6 +111,8 @@
     group: "{{ rabbitmq_group }}"
     mode: "0775"
   tags:
+    - "install"
+    - "install:app-configuration"
     - "monitoring"
 
 - name: Set up a cron job to run queue script
@@ -69,6 +120,8 @@
     name: "log-queue-lenghts"
     job: "{{ rabbitmq_app_dir }}/log-rabbitmq-queues.sh >/dev/null 2>&1"
   tags:
+    - "install"
+    - "install:app-configuration"
     - "monitoring"
 
 - name: Set up a cron job to run the script
@@ -76,6 +129,8 @@
     name: "log-rabbitmq-memory-usage"
     job: "{{ rabbitmq_app_dir }}/log-rabbitmq-memory.sh >/dev/null 2>&1"
   tags:
+    - "install"
+    - "install:app-configuration"
     - "monitoring"
 
 - name: install logrotate configuration
@@ -84,7 +139,7 @@
     dest: /etc/logrotate.d/rabbitmq
   tags:
     - "install"
-    - "install:configuration"
+    - "install:app-configuration"
     - "logrotate"
 
 # Defaulting to /var/lib/rabbitmq
@@ -95,6 +150,9 @@
     owner: rabbitmq
     group: rabbitmq
     mode: "0755"
+  tags:
+    - "install"
+    - "install:app-configuration"
 
 - name: Add rabbitmq erlang cookie
   template:
@@ -104,6 +162,9 @@
     group: rabbitmq
     mode: "0400"
   register: erlang_cookie
+  tags:
+    - "install"
+    - "install:app-configuration"
 
 # Defaulting to /etc/rabbitmq
 - name: Create rabbitmq config directory
@@ -113,6 +174,9 @@
     owner: root
     group: root
     mode: "0755"
+  tags:
+    - "install"
+    - "install:app-configuration"
 
 - name: Add rabbitmq environment configuration
   template:
@@ -121,6 +185,9 @@
     owner: root
     group: root
     mode: "0644"
+  tags:
+    - "install"
+    - "install:app-configuration"
 
 - name: Add rabbitmq cluster configuration
   template:
@@ -130,12 +197,18 @@
     group: root
     mode: "0644"
   register: cluster_configuration
+  tags:
+    - "install"
+    - "install:app-configuration"
 
 - name: Install plugins
   rabbitmq_plugin:
     names: "{{ item }}"
     state: enabled
   with_items: "{{ rabbitmq_plugins }}"
+  tags:
+    - "install"
+    - "install:app-configuration"
 
 # When rabbitmq starts up it creates a folder of metadata at '/var/lib/rabbitmq/mnesia'.
 # This folder should be deleted before clustering is setup because it retains data
@@ -145,16 +218,25 @@
     path: "{{ rabbitmq_mnesia_folder }}"
     state: absent
   when: erlang_cookie.changed or cluster_configuration.changed or rabbitmq_refresh
+  tags:
+    - "install"
+    - "install:app-configuration"
 
 - name: Start rabbit nodes
   service:
     name: rabbitmq-server
     state: started
+  tags:
+    - "install"
+    - "install:app-configuration"
 
 - name: Wait for rabbit to start
   wait_for:
     port: "{{ rabbitmq_management_port }}"
     delay: 2
+  tags:
+    - "install"
+    - "install:app-configuration"
 
 - name: Remove guest user
   rabbitmq_user:
@@ -163,6 +245,8 @@
   tags:
     - users
     - maintenance
+    - "manage"
+    - "manage:app-users"
 
 - name: Add vhosts
   rabbitmq_vhost:
@@ -172,6 +256,8 @@
   tags:
     - vhosts
     - maintenance
+    - "install"
+    - "install:app-configuration"
 
 - name: Add admin users
   rabbitmq_user:
@@ -190,14 +276,25 @@
   tags:
     - users
     - maintenance
+    - "manage"
+    - "manage:app-users"
 
 - name: Make queues mirrored
-  shell: "/usr/sbin/rabbitmqctl -p {{ item }} set_policy HA \"\" '{\"ha-mode\":\"all\",\"ha-sync-mode\":\"automatic\"}'"
-  when: RABBITMQ_CLUSTERED_HOSTS|length > 1
+  rabbitmq_policy:
+    name: HA
+    pattern: .*
+    vhost: "{{ item }}"
+  args:
+    tags:
+      ha-mode: all
+      ha-sync-mode: automatic
   with_items: "{{ RABBITMQ_VHOSTS }}"
+  when: RABBITMQ_CLUSTERED_HOSTS|length > 1
   tags:
     - ha
     - maintenance
+    - "install"
+    - "install:app-configuration"
 
 #
 # Depends upon the management plugin
@@ -206,6 +303,9 @@
   get_url:
     url: "http://localhost:{{ rabbitmq_management_port }}/cli/rabbitmqadmin"
     dest: "/usr/local/bin/rabbitmqadmin"
+  tags:
+    - "install"
+    - "install:app-configuration"
 
 - name: Ensure rabbitmqadmin attributes
   file:
@@ -213,3 +313,6 @@
     owner: root
     group: root
     mode: "0655"
+  tags:
+    - "install"
+    - "install:app-configuration"