Merge pull request #375 from edx/jarv/certs-role

Jarv/certs role

playbooks/edx-east/deploy_xqueue.yml

@@ -5,4 +5,5 @@
   roles:
     - common
     - supervisor
-    - xqueue
+    - role: xqueue
+      tags: ['xqueue']

playbooks/roles/certs/defaults/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+##
+# Defaults for role certs
+#
+
+CERTS_QUEUE_URL: "http://localhost:18040"
+CERTS_BUCKET: ""
+# basic auth credentials for connecting
+# to the xqueue server
+CERTS_XQUEUE_AUTH_USER: "edx"
+CERTS_XQUEUE_AUTH_PASS: "edx"
+# credentials for connecting to the xqueue server
+CERTS_QUEUE_USER: "lms"
+CERTS_QUEUE_PASS: "password"
+# AWS credentials for certificate upload
+CERTS_AWS_KEY: ""
+CERTS_AWS_ID: ""
+# GPG key ID, defaults to the dummy key
+CERTS_KEY_ID: "FEF8D954"
+# Path to git identity file for pull access to
+# the edX certificates repo - REQUIRED
+# Example - {{ secure_dir }}/files/git-identity
+CERTS_LOCAL_GIT_IDENTITY: !!null
+# Path to public and private gpg key for signing
+# the edX certificate. Default is a dummy key
+CERTS_LOCAL_PRIVATE_KEY: "example-private-key.txt"
+
+########## Internal role vars below
+
+certs_user: certs
+certs_app_dir: "{{ COMMON_APP_DIR }}/certs"
+certs_code_dir: "{{ certs_app_dir }}/certificates"
+certs_venvs_dir: "{{ certs_app_dir }}/venvs"
+certs_venv_dir: "{{ certs_venvs_dir }}/certs"
+certs_venv_bin: "{{ certs_venv_dir }}/bin"
+certs_git_ssh: /tmp/git_ssh.sh
+certs_git_identity: "{{ certs_app_dir }}/git-identity"
+certs_requirements_file: "{{ certs_code_dir }}/requirements.txt"
+certs_repo: "git@github.com:/edx/certificates"
+certs_version: 'master'
+certs_gpg_dir: "{{ certs_app_dir }}/gnupg"
+certs_env_config:
+  # CERTS_DATA is legacy, not used
+  CERTS_DATA: {}
+  QUEUE_NAME: "certificates"
+  QUEUE_URL: $CERTS_QUEUE_URL
+  CERTS_BUCKET: $CERTS_BUCKET
+  # gnupg signing key
+  CERTS_KEY_ID: $CERTS_KEY_ID
+  LOGGING_ENV: ""
+  CERTS_GPG_DIR: $certs_gpg_dir
+
+certs_auth_config:
+  QUEUE_USER: $CERTS_QUEUE_USER
+  QUEUE_PASS: $CERTS_QUEUE_PASS
+  QUEUE_AUTH_USER: $CERTS_XQUEUE_AUTH_USER
+  QUEUE_AUTH_PASS: $CERTS_XQUEUE_AUTH_PASS
+  CERTS_KEY_ID: $CERTS_KEY_ID
+  CERTS_AWS_ID: $CERTS_AWS_ID
+  CERTS_AWS_KEY: $CERTS_AWS_KEY

playbooks/roles/certs/files/example-private-key.txt

+-----BEGIN PGP PRIVATE KEY BLOCK-----
+Version: GnuPG v1.4.11 (GNU/Linux)
+
+lQOYBFJwVOkBCAC4heT6+P1sGgITAB5C+hKNr4RACS47K1nxgIiEqiFMIycluDmM
+4kdqFInzDK8GHF2W5KijZmYf7LrWIg4+PmnyYAB7cO+eJUDfTE7n7bjGQL3LohJN
+FTlRsXKOKGWoBqlytE3D16lQIIp0JkqB9sHO3Y9yOgEsSy3cMWKtT8U6qx40xV+e
+t0FYmqL7pBE7OFfvCIe7+kthsTqFys/jkRNFvbSo5fjA1m9ubjEJqqfnhuvLaL5O
+YHGe1nKQRLi45gmZ1JYvxfZrWUO2BeulNY/mvAFQnRNRRiWfM3Ic4Ya9Wv62wS3p
+dYY4HEtDQDyKpOkJ2R31+1FhZYIKJTYR89jxABEBAAEAB/wOApyQMbeMLa1ao/eo
+PjSKbXktI4VPGMuLeqbi68f7b+/Y/VPhToz9kPGocp4XaK/ydQoY3f2DDwZgm9VZ
+BIQm0wM2XCzVZR631aNoGLSe2OuQOo4JLENd4ItCH+8YAul6vBXreMRyQQZCK2Yc
+2A9/FXN+yMiuBEdHILjNT/E5swNm0J85YlXpIW6Jm3aR6OjzfFS/j+7AEDSL5MZX
+JotfGjYXuC1MOw5YJZKWkQBz+5IaVceOd9s8TlZFq/eYrN5sqAWh06CBY+Zye3fg
+/WiWFUdTgpG81lbAXGxHrjQ5f22saOzkbv0FjdEfx1M9Wcj6OAIRXI7k8EkZJia1
+IYEBBADYRvRE2zyR5M72MfCX1XUOpx/9OZCrVsYoKqp1BORXjt58Szs7UFdeAXE+
+bPzbpcjENiVYVjoeQKCNTU78gzjD+NzkfTdsF7rrvXObo6NpChTCOdQfg56Ll+3y
+3nUDKIcFXsYP1NIC2SL0APcpUtDLPIWb0XRnlvBQlakmnyb7bQQA2mnn15LVRK0J
+1wYZiSwrRIcE7X+zy6t8iERr+E0jIyQQV4vaOYItCDTP8fzNiiX10Nkt5imRqML3
+NBPs0jInjmYxMmzvjVxyUDv4rGGbiXXeh+W1v/mweMH2pbiItjhyeSVt4U2l6GKI
+Ob/K+khx1ftfOTktLTZVMPg6NzPRfRUD/jyLL92V7eoHshreqFIUcBmUdQnMVyz2
+NBhci7RHGn3J84TTMCXBJL8MLUu9PKxfFcZjw1Wet8avX5oPjgl3OiQpjjx69sO5
+S/UWpGaEOrz87j4VRGPb6zmegkZs44sQEfwhDJk6O1eQ5dYzniRC46nzMpETFfIF
+U8m8bJrrus4HPEy0HmV4YW1wbGUga2V5IDx0ZXN0QGV4YW1wbGUuY29tPokBOAQT
+AQIAIgUCUnBU6QIbAwYLCQgHAwIGFQgCCQoLBBYCAwECHgECF4AACgkQBECJtv74
+2VSWKQf/f0PWmbGxdiBIK5gf2pmaK0aDDM09v7vedysIn/URnj26BMN/YEyDYnZS
+BN+iuU6VartvEYlNeYiRAnaG/6gl7DJh2l2X/iuDn0xKT0GjqDpjh7n6964OKAz7
+RHWADXqsr5BWms1EPFtDVnAJfN2A3cxTeA5vUUl41WvCJQa7L8Bw7SezkS0yn8Rn
+u4icNKyew7TrFofIydws6LTM1DhHpCB32z6b7HHt85OOzpuUm07HP83S59lxBp6x
+x3NH9AH/WPeXiS5QRh1jP6qzUAHoHQpsV2XonmC4JXl+ZFxNyZeJ000ldDFfEHrO
+RLg3d5GkZ1pDVDn3HlZ+SKqYilRXCJ0DmARScFTpAQgA1KGTRGcqwla5/VOuHwxw
+ABpLYdpsetYoOORjJQvHakG8QBchxsJVniBijD09gFmHYpdSJaeHnvqkeHGO1fJa
+E4QxS4AYt/HVoi86RhBLD/Gr0/DWC/0XUV5613PSmWkYCCTgWLaxT9MpPjtGVd4v
+L6Iv/d8Go/Wrq55zCl82PTA7ao4PxSSxlforfZOZqsJ/pzjCRkF6Z7co+LO24KSl
+Lt4iN2vwJ2VhvOrMFuV91WQeEJWdTX+yx035eU/MFu9u243CE0UGNzWHjYLpgBxl
+Pg0W5GFRZM/LYkXAfHAM4/Ic2ex2LQ0RLiH4i0FbzoSwjvz586v2Sagc5nsYMoGu
+gQARAQABAAf/W6W+23taJ0SJSuLACJrsRWcP+b/TBQj8cjUidKvEioyFztwJj2lg
+zNSplUeqFAHCxGBzpE42uvPOYymTBq08XPAb7S5ruREP4yVXCS7po5gnVyUVpToz
+zDscWHQQIFZ3aL84QZSRDVZ3Dt8unEE1dmMCK3rvGkl/8mtLq3tJXgp7/wdsK4G0
+3AuJVQ918XlozNTayGfdCPhWicE7fv5peUlWRWlSuSNmTrHiAbysd2xwXnMq/OdQ
+Q+z7ogQHhUvQQ+31msGlcCJQqqWr250/HBrTATrRJNIVvvzCgpw3/6r99MNwlSWV
+ZhDotwf745fdzZiwdgJ04nhEj9QmKynKXwQA29p7DmMWMZg61qU481YNWgc5RMjL
+ADUS2iC5nr/Y5HuAsGWj9ZkDvRXKSyexkZ+OXi6EonGCGjCNONPaB8JWRO7LssEc
+VG+lPp4mwASE38cjfFy7DdEGpxn3eZPDsNwv7vnWhNyGSh8FXKoYXyZiJ6F3zvkU
+aWwfaTtxVplfn88EAPeXHLkwl+D8zkk8ILYnsJKEKjcqUwiQ6L7JMEhd+GVo5xR+
+WUDdhnmEkH/QZZt8zTpYL3Hl3JsqQYidq0uzy39qg+cVvD9yJkHP4KMAqe9QcRYR
+eQvpopMYt5va7pyaebZbpxfP9M7Y2/5VT59GBO6uHy4CMR1uM4Z50QA0kZCvA/oD
+D/9qEaWzwqLtXjN1iRxOv7ioor8ExvA/8HY8xtCsCLuFuo9P44xtYzSCzLdoOYCE
+4Lrn7DeE2hXEoq/2VEWoRS4+kU1vUBIJEAxfHk6HozA0apFPqm8ODH44s68VRTce
+pGwORxsFhMHw1/m5A1RBZF8UP7VXFxluYuwx6S5NyjSfiQEfBBgBAgAJBQJScFTp
+AhsMAAoJEARAibb++NlUojkIAKHwS1VSeW6fgWv7H2qaTjdMeNG7vXUYKUE7KMpQ
+UmvdHobMfbO9SEgihwG+WdgPy96RlYx5PuVfeWkPVdVsbrU9BuR+9qdYyGGH4FvP
+qAaruT3dFLRFvDj/ta94gDFGCH1LrtGI/t78wjjIEd8QOGIj+8Uo1Z6HKExSsNuG
++8usut6je50a2CsAyoZtrPmybZdkU6eOuM5ZSGDpgfTlFNpeK3sf7CTnYA5NTLPC
+wWbyCxUb7EUrch+StmJWsIzS4mClMd6nB4480FwwhGbdFejSF20z64c6hbxuwgfS
+nyXklWktEX0d5T7wdAi+UOvNsdoigzUMWpBoo07VOlzjMFU=
+=iNqX
+-----END PGP PRIVATE KEY BLOCK-----

playbooks/roles/certs/files/example-public-key.txt

+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: GnuPG v1.4.11 (GNU/Linux)
+
+mQENBFJwVOkBCAC4heT6+P1sGgITAB5C+hKNr4RACS47K1nxgIiEqiFMIycluDmM
+4kdqFInzDK8GHF2W5KijZmYf7LrWIg4+PmnyYAB7cO+eJUDfTE7n7bjGQL3LohJN
+FTlRsXKOKGWoBqlytE3D16lQIIp0JkqB9sHO3Y9yOgEsSy3cMWKtT8U6qx40xV+e
+t0FYmqL7pBE7OFfvCIe7+kthsTqFys/jkRNFvbSo5fjA1m9ubjEJqqfnhuvLaL5O
+YHGe1nKQRLi45gmZ1JYvxfZrWUO2BeulNY/mvAFQnRNRRiWfM3Ic4Ya9Wv62wS3p
+dYY4HEtDQDyKpOkJ2R31+1FhZYIKJTYR89jxABEBAAG0HmV4YW1wbGUga2V5IDx0
+ZXN0QGV4YW1wbGUuY29tPokBOAQTAQIAIgUCUnBU6QIbAwYLCQgHAwIGFQgCCQoL
+BBYCAwECHgECF4AACgkQBECJtv742VSWKQf/f0PWmbGxdiBIK5gf2pmaK0aDDM09
+v7vedysIn/URnj26BMN/YEyDYnZSBN+iuU6VartvEYlNeYiRAnaG/6gl7DJh2l2X
+/iuDn0xKT0GjqDpjh7n6964OKAz7RHWADXqsr5BWms1EPFtDVnAJfN2A3cxTeA5v
+UUl41WvCJQa7L8Bw7SezkS0yn8Rnu4icNKyew7TrFofIydws6LTM1DhHpCB32z6b
+7HHt85OOzpuUm07HP83S59lxBp6xx3NH9AH/WPeXiS5QRh1jP6qzUAHoHQpsV2Xo
+nmC4JXl+ZFxNyZeJ000ldDFfEHrORLg3d5GkZ1pDVDn3HlZ+SKqYilRXCLkBDQRS
+cFTpAQgA1KGTRGcqwla5/VOuHwxwABpLYdpsetYoOORjJQvHakG8QBchxsJVniBi
+jD09gFmHYpdSJaeHnvqkeHGO1fJaE4QxS4AYt/HVoi86RhBLD/Gr0/DWC/0XUV56
+13PSmWkYCCTgWLaxT9MpPjtGVd4vL6Iv/d8Go/Wrq55zCl82PTA7ao4PxSSxlfor
+fZOZqsJ/pzjCRkF6Z7co+LO24KSlLt4iN2vwJ2VhvOrMFuV91WQeEJWdTX+yx035
+eU/MFu9u243CE0UGNzWHjYLpgBxlPg0W5GFRZM/LYkXAfHAM4/Ic2ex2LQ0RLiH4
+i0FbzoSwjvz586v2Sagc5nsYMoGugQARAQABiQEfBBgBAgAJBQJScFTpAhsMAAoJ
+EARAibb++NlUojkIAKHwS1VSeW6fgWv7H2qaTjdMeNG7vXUYKUE7KMpQUmvdHobM
+fbO9SEgihwG+WdgPy96RlYx5PuVfeWkPVdVsbrU9BuR+9qdYyGGH4FvPqAaruT3d
+FLRFvDj/ta94gDFGCH1LrtGI/t78wjjIEd8QOGIj+8Uo1Z6HKExSsNuG+8usut6j
+e50a2CsAyoZtrPmybZdkU6eOuM5ZSGDpgfTlFNpeK3sf7CTnYA5NTLPCwWbyCxUb
+7EUrch+StmJWsIzS4mClMd6nB4480FwwhGbdFejSF20z64c6hbxuwgfSnyXklWkt
+EX0d5T7wdAi+UOvNsdoigzUMWpBoo07VOlzjMFU=
+=WP59
+-----END PGP PUBLIC KEY BLOCK-----

playbooks/roles/certs/handlers/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+#
+#
+# Handlers for role certs
+#
+# Overview:
+#
+
+- name: certs | restart the certs service
+  supervisorctl: >
+    name=certs
+    supervisorctl_path={{ supervisor_ctl }}
+    config={{ supervisor_cfg }}
+    state=restarted
+

playbooks/roles/certs/tasks/deploy.yml

+---
+
+- name: certs | stop the certificates service
+  supervisorctl: >
+    name="certs"
+    supervisorctl_path={{ supervisor_ctl }}
+    config={{ supervisor_cfg }}
+    state=stopped
+  tags: deploy
+
+- name: certs | create certificate application config
+  template: >
+    src=certs.env.json.j2
+    dest={{ certs_app_dir }}/env.json
+  sudo_user: "{{ certs_user }}"
+  tags: deploy
+
+- name: certs | create certificate auth file
+  template: >
+    src=certs.auth.json.j2
+    dest={{ certs_app_dir }}/auth.json
+  sudo_user: "{{ certs_user }}"
+  tags: deploy
+
+- name: certs | writing supervisor script for certificates
+  template: >
+    src=certs.conf.j2 dest={{ supervisor_cfg_dir }}/certs.conf
+    owner={{ supervisor_user }} group={{ common_web_user }} mode=0644
+  notify: supervisor | reload supervisor
+  tags: deploy
+
+- name: certs | create ssh script for git
+  template: >
+    src={{ certs_git_ssh|basename }}.j2 dest={{ certs_git_ssh }}
+    owner={{ certs_user }} mode=750
+  tags: deploy
+
+- name: certs | install read-only ssh key for the certs repo
+  copy: >
+    src={{ CERTS_LOCAL_GIT_IDENTITY }} dest={{ certs_git_identity }}
+    force=yes owner={{ certs_user }} mode=0600
+  tags: deploy
+
+- name: certs | checkout certificates repo into {{ certs_code_dir }}
+  git: dest={{ certs_code_dir }} repo={{ certs_repo }} version={{ certs_version }}
+  sudo_user: "{{ certs_user }}"
+  environment:
+    GIT_SSH: "{{ certs_git_ssh }}"
+  tags: deploy
+
+- name: certs | remove read-only ssh key for the certs repo
+  file: path={{ certs_git_identity }} state=absent
+  tags: deploy
+
+- name : install python requirements
+  pip: requirements="{{ certs_requirements_file }}" virtualenv="{{ certs_venv_dir }}" state=present
+  sudo_user: "{{ certs_user }}"
+  tags: deploy
+
+- name: certs | restart the certificate service
+  supervisorctl: >
+    name=certs
+    supervisorctl_path={{ supervisor_ctl }}
+    config={{ supervisor_cfg }}
+    state=restarted
+  tags: deploy

playbooks/roles/certs/tasks/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+#
+#
+# Tasks for role certs
+#
+# Overview:
+#
+# Installs the edX certificate server.
+#
+# The certificates repo is currently *not* public
+# due to sensitive information in it, it may be made
+# public in the future.
+#
+# Dependencies:
+#   - common
+#   - supervisor
+#
+#
+# Example play:
+#
+#   - roles:
+#     - common
+#     - supervisor
+#     - certs
+#
+- name: Checking to see if git identity is set
+  fail: msg="You must set CERTS_LOCAL_GIT_IDENTITY var for this role!"
+  when: not CERTS_LOCAL_GIT_IDENTITY
+
+- name: certs | create application user
+  user: >
+    name="{{ certs_user }}"
+    home="{{ certs_app_dir }}"
+    createhome=no
+    shell=/bin/false
+
+- name: certs | create certs app and data dirs
+  file: >
+    path="{{ item }}"
+    state=directory
+    owner="{{ certs_user }}"
+    group="{{ common_web_group }}"
+  with_items:
+    - "{{ certs_app_dir }}"
+    - "{{ certs_venvs_dir }}"
+
+- name: certs | create certs gpg dir
+  file: >
+    path="{{ certs_gpg_dir }}" state=directory
+    owner="{{ certs_user }}" group="{{ certs_user }}"
+    mode=0700
+
+- name: certs | copy the private gpg signing key
+  copy: >
+    src={{ CERTS_LOCAL_PRIVATE_KEY }}
+    dest={{ certs_app_dir }}/{{ CERTS_LOCAL_PRIVATE_KEY|basename }}
+    owner={{ certs_user }} mode=0600
+  register: certs_gpg_key
+  tags: deploy
+
+
+- name: certs | load the gpg key
+  shell: >
+    /usr/bin/gpg --homedir {{ certs_gpg_dir }} --import {{ certs_app_dir }}/{{ CERTS_LOCAL_PRIVATE_KEY|basename }}
+  sudo_user: "{{ certs_user }}"
+  when: certs_gpg_key.changed
+
+- include: deploy.yml
+
+- name: certs | create a symlink for venv python
+  file: >
+    src="{{ certs_venv_bin }}/python"
+    dest={{ COMMON_BIN_DIR }}/python.certs
+    state=link

playbooks/roles/certs/templates/certs.auth.json.j2

+{{ certs_auth_config | to_nice_json }}

playbooks/roles/certs/templates/certs.conf.j2

+[program:certs]
+command={{ certs_venv_bin }}/python {{ certs_code_dir }}/certificate_agent.py
+priority=999
+environment=SERVICE_VARIANT="certs",HOME="/"
+user={{ common_web_user }}
+startsecs=10
+stdout_logfile={{ supervisor_log_dir }}/%(program_name)-stdout.log
+stderr_logfile={{ supervisor_log_dir }}/%(program_name)-stderr.log
+killasgroup=true
+stopasgroup=true

playbooks/roles/certs/templates/certs.env.json.j2

+{{ certs_env_config | to_nice_json }}

playbooks/roles/certs/templates/git_ssh.sh.j2

+#!/bin/sh
+exec /usr/bin/ssh -o StrictHostKeyChecking=no -i {{ certs_git_identity }} "$@"