Merge pull request #2882 from edx/jeskew/restore_gocd_backup

Add scripting to restore a GoCD server backup.

playbooks/roles/go-server/defaults/main.yml

@@ -15,7 +15,7 @@ GO_SERVER_USER: "go"
 GO_SERVER_GROUP: "{{ GO_SERVER_USER }}"
 GO_SERVER_VERSION: "16.1.0-2855"
 GO_SERVER_HOME: "/var/lib/go-server"
-GO_SERVER_CONF_HOME: "/etc/go/"
+GO_SERVER_CONF_HOME: "/etc/go"
 
 # Java version settings
 GO_SERVER_ORACLEJDK_VERSION: "7u80"
@@ -30,6 +30,7 @@ GO_SERVER_JAVA_HOME: "{{ GO_SERVER_ORACLEJDK_LINK }}"
 GO_SERVER_APT_SOURCE: "deb http://dl.bintray.com/gocd/gocd-deb/ /"
 GO_SERVER_APT_KEY_URL: "https://bintray.com/user/downloadSubjectPublicKey?username=gocd"
 GO_SERVER_APT_NAME: "go-server"
+GO_SERVER_APT_PKGS: ["apache2-utils"]
 
 # gocd-oauth-login
 GO_SERVER_OAUTH_LOGIN_VERSION: "1.2"
@@ -37,10 +38,34 @@ GO_SERVER_OAUTH_LOGIN_MD5: "31ad9ad1fe08452f73c56a44b035ee91"
 GO_SERVER_OAUTH_LOGIN_JAR_URL: "https://github.com/gocd-contrib/gocd-oauth-login/releases/download/v{{ GO_SERVER_OAUTH_LOGIN_VERSION }}/github-oauth-login-{{ GO_SERVER_OAUTH_LOGIN_VERSION }}.jar"
 GO_SERVER_OAUTH_LOGIN_JAR_DESTINATION: "{{ GO_SERVER_HOME }}/plugins/external/"
 
+# go-server credentials
+GO_SERVER_ADMIN_USERNAME: ""
+GO_SERVER_ADMIN_PASSWORD: ""
+GO_SERVER_BACKUP_USERNAME: ""
+GO_SERVER_BACKUP_PASSWORD: ""
+
+# H2 db settings
+GO_SERVER_H2DB_PATH: "db/h2db"
+GO_SERVER_H2DB_NAME: "cruise.h2.db"
+GO_SERVER_H2DB_LOCATION: "{{ GO_SERVER_HOME }}/{{ GO_SERVER_H2DB_PATH }}/{{ GO_SERVER_H2DB_NAME}}"
+GO_SERVER_H2DB_BACKUP_PATH: "artifacts/db_backup/"
+GO_SERVER_H2DB_BACKUP_LOCATION: "{{ GO_SERVER_HOME }}/{{ GO_SERVER_H2DB_BACKUP_PATH }}/{{ GO_SERVER_H2DB_NAME }}"
+
+# For use in backup/restore of a GoCD configuration.
+GO_SERVER_BACKUP_FILENAME: "go-server-latest-backup.tgz"
+GO_SERVER_TEMP_RESTORE_DIRECTORY: "/tmp/go-server-backup"
+GO_SERVER_BACKUP_S3_BUCKET: "edx-tools-gocd-backup"
+GO_SERVER_BACKUP_S3_OBJECT: "{{ GO_SERVER_BACKUP_FILENAME }}"
+GO_SERVER_BACKUP_SNITCH_URL: "https://nosnch.in/4444444444"
+GO_SERVER_BACKUP_APT_PKGS: ["python-pip", "jq"]
+GO_SERVER_BACKUP_PIP_PKGS: ["boto", "awscli"]
+GO_SERVER_BACKUP_API_URL: "http://localhost:8153/go/api/backups"
+GO_SERVER_BACKUP_TMP_LOCATION: "/tmp/{{ GO_SERVER_BACKUP_FILENAME }}"
+GO_SERVER_BACKUP_CRON_SCRIPT_LOCATION: "/root/gocd_backup.sh"
+
 # password file setup
 GO_SERVER_PASSWORD_FILE_NAME: "password.txt"
 GO_SERVER_ADMIN_USERS: ["admin"]
-GO_SERVER_CRUISE_CONTROL_DB_DESTIONATION: "/var/lib/go-server/db/h2db/cruise.h2.db"
 
 # key for go-agents to autoregister with the go-server
 GO_SERVER_AUTO_REGISTER_KEY: "dev-only-override-this-key"

playbooks/roles/go-server/tasks/download_backup.yml

+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+#
+#
+# Task to download go-server backup
+#
+# Overview:
+#
+# Downloads a TGZ file containing a go-server backup.
+#
+# Example play:
+#
+# - name: Configure instance(s)
+#   hosts: go-server
+#   sudo: True
+#   vars_files:
+#     - "{{ secure_dir }}/admin/sandbox.yml"
+#   gather_facts: True
+#   roles:
+#     - common
+#
+
+- name: stop go-server
+  service:
+    name: "{{ GO_SERVER_SERVICE_NAME }}"
+    state: stopped
+
+- name: install system packages needed for downloading backup
+  apt:
+    pkg: "{{ item }}"
+    state: present
+    update_cache: true
+    cache_valid_time: 3600
+  with_items: GO_SERVER_BACKUP_APT_PKGS
+
+- name: install required python packages
+  pip:
+    name: "{{ item }}"
+    state: present
+  with_items: GO_SERVER_BACKUP_PIP_PKGS
+
+- name: create the temp directory
+  file:
+    path: "{{ GO_SERVER_TEMP_RESTORE_DIRECTORY }}"
+    state: directory
+    mode: 0755
+
+- name: get s3 one time url
+  s3: >
+    bucket="{{ GO_SERVER_BACKUP_S3_BUCKET }}"
+    object="{{ GO_SERVER_BACKUP_S3_OBJECT }}"
+    mode="geturl"
+    expiration=30
+  when: GO_SERVER_BACKUP_S3_BUCKET and GO_SERVER_BACKUP_S3_OBJECT
+  register: s3_one_time_url
+
+- name: download from one time url
+  get_url:
+    url="{{ s3_one_time_url.url }}"
+    dest="{{ GO_SERVER_TEMP_RESTORE_DIRECTORY }}/{{ GO_SERVER_BACKUP_FILENAME }}"
+    mode=0600
+  when: GO_SERVER_BACKUP_S3_BUCKET and GO_SERVER_BACKUP_S3_OBJECT
+  register: download_backup_s3
+
+- name: copy go-server backup from backup storage
+  unarchive:
+    src: "{{ GO_SERVER_TEMP_RESTORE_DIRECTORY }}/{{ GO_SERVER_BACKUP_FILENAME }}"
+    dest: "{{ GO_SERVER_TEMP_RESTORE_DIRECTORY }}/"
+    copy: no
+  when: download_backup_s3.changed
+
+- include: restore_backup.yml
+  when: download_backup_s3.changed
+
+- name: remove the temp directory
+  file:
+    path: "{{ GO_SERVER_TEMP_RESTORE_DIRECTORY }}"
+    state: absent

playbooks/roles/go-server/tasks/main.yml

@@ -43,6 +43,14 @@
     name: "{{ GO_SERVER_APT_NAME }}={{ GO_SERVER_VERSION }}"
     update_cache: yes
 
+- name: install other needed system packages
+  apt:
+    pkg: "{{ item }}"
+    state: present
+    update_cache: true
+    cache_valid_time: 3600
+  with_items: GO_SERVER_APT_PKGS
+
 - name: create go-server plugin directory
   file:
     path: "{{ GO_SERVER_OAUTH_LOGIN_JAR_DESTINATION }}"
@@ -59,14 +67,26 @@
     owner: "{{ GO_SERVER_USER }}"
     group: "{{ GO_SERVER_GROUP }}"
 
-- name: setup password file if secret hash exists
+- name: generate line for go-server password file for admin user
+  command: >
+    /usr/bin/htpasswd -nbs "{{ GO_SERVER_ADMIN_USERNAME }}" "{{ GO_SERVER_ADMIN_PASSWORD }}"
+  register: admin_user_password_line
+  when: GO_SERVER_ADMIN_USERNAME and GO_SERVER_ADMIN_PASSWORD
+
+- name: generate line for go-server password file for backup user
+  command: >
+    /usr/bin/htpasswd -nbs "{{ GO_SERVER_BACKUP_USERNAME }}" "{{ GO_SERVER_BACKUP_PASSWORD }}"
+  register: backup_user_password_line
+  when: GO_SERVER_BACKUP_USERNAME and GO_SERVER_BACKUP_PASSWORD
+
+- name: setup password file
   template:
     src: edx/app/go-server/password.txt.j2
     dest: "{{ GO_SERVER_CONF_HOME }}/{{ GO_SERVER_PASSWORD_FILE_NAME }}"
     mode: 0600
     owner: "{{ GO_SERVER_USER }}"
     group: "{{ GO_SERVER_GROUP }}"
-  when: GO_SERVER_ADMIN_PASSWORD_HASH is defined
+  when: GO_SERVER_ADMIN_PASSWORD and GO_SERVER_BACKUP_PASSWORD
 
 - name: install go-server configuration
   template:
@@ -76,8 +96,28 @@
     owner: "{{ GO_SERVER_USER }}"
     group: "{{ GO_SERVER_GROUP }}"
 
+# If a GoCD restore file is specified, attempt to download and restore it.
+- include: download_backup.yml
+  when: GO_SERVER_BACKUP_S3_BUCKET and GO_SERVER_BACKUP_S3_OBJECT
+
+- name: replace the admin line in the password file post-restore
+  lineinfile:
+    dest: "{{ GO_SERVER_CONF_HOME }}/{{ GO_SERVER_PASSWORD_FILE_NAME }}"
+    regexp: "^{{ GO_SERVER_ADMIN_USERNAME }}"
+    line: "{{ admin_user_password_line.stdout }}"
+  when: GO_SERVER_ADMIN_USERNAME and GO_SERVER_ADMIN_PASSWORD
+
+- name: replace the backup line in the password file post-restore
+  lineinfile:
+    dest: "{{ GO_SERVER_CONF_HOME }}/{{ GO_SERVER_PASSWORD_FILE_NAME }}"
+    regexp: "^{{ GO_SERVER_BACKUP_USERNAME }}"
+    line: "{{ backup_user_password_line.stdout }}"
+  when: GO_SERVER_BACKUP_USERNAME and GO_SERVER_BACKUP_PASSWORD
+
 - name: restart go-server
   service:
     name: "{{ GO_SERVER_SERVICE_NAME }}"
     state: restarted
 
+- include: setup_regular_backup.yml
+  when: GO_SERVER_BACKUP_S3_BUCKET and GO_SERVER_BACKUP_S3_OBJECT

playbooks/roles/go-server/tasks/restore_backup.yml

+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+#
+#
+# Task to restore go-server backup
+#
+# Overview:
+#
+# Restores a TGZ file containing a go-server backup to a go-server instance.
+#
+# Example play:
+#
+# - name: Configure instance(s)
+#   hosts: go-server
+#   sudo: True
+#   vars_files:
+#     - "{{ secure_dir }}/admin/sandbox.yml"
+#   gather_facts: True
+#   roles:
+#     - common
+#
+
+- name: check if H2 db exists
+  stat: "path={{ GO_SERVER_H2DB_LOCATION }}"
+  register: h2db_stat
+
+- name: create the db backup directory
+  file:
+    path: "{{ GO_SERVER_H2DB_BACKUP_LOCATION }}"
+    state: directory
+    mode: 0775
+    owner: "{{ GO_SERVER_USER }}"
+    group: "{{ GO_SERVER_GROUP }}"
+
+- name: back up existing H2 db if needed
+  command: "mv {{ GO_SERVER_H2DB_LOCATION }} {{ GO_SERVER_H2DB_BACKUP_LOCATION }}"
+  when: h2db_stat.stat.exists
+
+- name: delete the db directory to clear out
+  file:
+    path: "{{ GO_SERVER_HOME }}/{{ GO_SERVER_H2DB_PATH }}"
+    state: absent
+  when: h2db_stat.stat.exists
+
+- name: re-create the db backup directory
+  file:
+    path: "{{ GO_SERVER_HOME }}/{{ GO_SERVER_H2DB_PATH }}"
+    state: directory
+    mode: 0775
+    owner: "{{ GO_SERVER_USER }}"
+    group: "{{ GO_SERVER_GROUP }}"
+
+- name: move the backup contents to the correct location
+  shell: "cp {{ GO_SERVER_TEMP_RESTORE_DIRECTORY }}/backup*/* {{ GO_SERVER_TEMP_RESTORE_DIRECTORY }}/"
+
+- name: unzip the db backup
+  unarchive:
+    copy: no
+    src: "{{ GO_SERVER_TEMP_RESTORE_DIRECTORY }}/db.zip"
+    dest: "{{ GO_SERVER_TEMP_RESTORE_DIRECTORY }}"
+    mode: 0600
+    owner: "{{ GO_SERVER_USER }}"
+    group: "{{ GO_SERVER_GROUP }}"
+
+- name: copy backup db file into GO_SERVER db directory
+  command: "mv {{ GO_SERVER_TEMP_RESTORE_DIRECTORY }}/{{ GO_SERVER_H2DB_NAME }} {{ GO_SERVER_H2DB_LOCATION }}"
+
+- name: unzip config-dir.zip in backup directory
+  unarchive:
+    copy: no
+    src: "{{ GO_SERVER_TEMP_RESTORE_DIRECTORY }}/config-dir.zip"
+    dest: "{{ GO_SERVER_CONF_HOME }}/"
+    mode: 0600
+    owner: "{{ GO_SERVER_USER }}"
+    group: "{{ GO_SERVER_GROUP }}"
+
+- name: create the config directory
+  file:
+    path: "{{ GO_SERVER_CONF_HOME }}/db/config.git/"
+    state: directory
+    mode: 0775
+    owner: "{{ GO_SERVER_USER }}"
+    group: "{{ GO_SERVER_GROUP }}"
+
+- name: unzip config-repo.zip into backup directory
+  unarchive:
+    copy: no
+    src: "{{ GO_SERVER_TEMP_RESTORE_DIRECTORY }}/config-repo.zip"
+    dest: "{{ GO_SERVER_CONF_HOME }}/db/config.git/"
+    mode: 0600
+    owner: "{{ GO_SERVER_USER }}"
+    group: "{{ GO_SERVER_GROUP }}"
+
+- name: fix the permissions in config directory
+  shell: find /etc/go -type d -exec chmod 775 {} \;

playbooks/roles/go-server/tasks/setup_regular_backup.yml

+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+#
+#
+# Task to setup a regular go-server backup via cron
+#
+# Overview:
+#
+# Uses cron to generate a regular go-server backup and send it to AWS S3.
+# A successful backup and transfer is then reported to DeadMan'sSnitch.
+#
+# Example play:
+#
+# - name: Configure instance(s)
+#   hosts: go-server
+#   sudo: True
+#   vars_files:
+#     - "{{ secure_dir }}/admin/sandbox.yml"
+#   gather_facts: True
+#   roles:
+#     - common
+#
+
+- name: install required pkgs
+  apt:
+    pkg: "{{ item }}"
+    state: present
+    update_cache: true
+    cache_valid_time: 3600
+  with_items: GO_SERVER_BACKUP_APT_PKGS
+
+- name: install required Python modules
+  pip:
+    name: "{{ item }}"
+    state: present
+  with_items: GO_SERVER_BACKUP_PIP_PKGS
+
+- name: create backup shell script
+  template:
+    src: gocd_backup.j2
+    dest: "{{ GO_SERVER_BACKUP_CRON_SCRIPT_LOCATION }}"
+    mode: 0700
+    owner: root
+    group: root
+
+- name: create cron entry
+  cron:
+    name: "gocd backup"
+    minute: 0
+    hour: 2
+    job: "{{ GO_SERVER_BACKUP_CRON_SCRIPT_LOCATION }}"

playbooks/roles/go-server/templates/edx/app/go-server/password.txt.j2

-admin:{{ GO_SERVER_ADMIN_PASSWORD_HASH }}
+{{ admin_user_password_line.stdout }}
\ No newline at end of file
+{{ backup_user_password_line.stdout }}

playbooks/roles/go-server/templates/gocd_backup.j2

+#!/bin/bash
+
+# Variables.
+gocd_backup_api_call="{{ GO_SERVER_BACKUP_API_URL }}"
+gocd_username="{{ GO_SERVER_BACKUP_USERNAME }}"
+gocd_password="{{ GO_SERVER_BACKUP_PASSWORD }}"
+gocd_backup_location="{{ GO_SERVER_BACKUP_TMP_LOCATION }}"
+s3_backup_bucket="{{ GO_SERVER_BACKUP_S3_BUCKET }}"
+snitch_url="{{ GO_SERVER_BACKUP_SNITCH_URL }}"
+
+# Trigger the backup and capture the backup path.
+# The output of the api call is documented here:
+#   https://api.go.cd/current/#create-a-backup
+# Below is an example of the API output:
+# {
+#   "_links": {
+#     "doc": {
+#       "href": "http://api.go.cd/#backups"
+#     }
+#   },
+#   "time": "2015-08-07T10:07:19.868Z",
+#   "path": "/var/lib/go-server/serverBackups/backup_20150807-153719",
+#   "user": {
+#     "_links": {
+#       "doc": {
+#         "href": "http://api.go.cd/#users"
+#       },
+#       "self": {
+#         "href": "https://ci.example.com/go/api/users/username"
+#       },
+#       "find": {
+#         "href": "https://ci.example.com/go/api/users/:login_name"
+#       }
+#     },
+#     "login_name": "username"
+#   }
+# }
+backup_path=\
+$(curl -sS "$gocd_backup_api_call" \
+    -u "$gocd_username:$gocd_password" \
+    -H 'Accept: application/vnd.go.cd.v1+json' \
+    -X POST | \
+jq .path | tr -d '"')
+
+case $backup_path in
+null)
+  echo "GoCD backup failed."
+  exit
+  ;;
+*)
+  echo "GoCD backup succeeded to: $backup_path"
+  ;;
+esac
+
+# Make backup into a tarball.
+tar zcvf "$gocd_backup_location" -C "$(dirname "$backup_path")" "$(basename "$backup_path")"
+
+# Transfer tarball to S3.
+aws_output=$(aws s3 cp "$gocd_backup_location" "s3://$s3_backup_bucket" 2>&1)
+case $aws_output in
+*failed*)
+  echo "Backup transfer failed: $aws_output"
+  exit
+  ;;
+*)
+  echo "Backup transfer to S3 bucket $s3_backup_bucket succeeded."
+  # Report to Dead Man's Snitch.
+  curl $snitch_url
+  ;;
+esac
+
+# Remove the tarball.
+rm -f "$gocd_backup_location"