complete and operationalize harstorage role (OPS-1372)

docker/build/harstorage/Dockerfile.out

+FROM edxops/precise-common
+MAINTAINER edxops
+
+USER root
+
+# Fix selinux issue with useradd on 12.04
+RUN curl http://salilab.org/~ben/libselinux1_2.1.0-5.1ubuntu1_amd64.deb -o /tmp/libselinux1_2.1.0-5.1ubuntu1_amd64.deb
+RUN dpkg -i /tmp/libselinux1_2.1.0-5.1ubuntu1_amd64.deb
+
+RUN apt-get update
+ADD . /edx/app/edx_ansible/edx_ansible
+COPY docker/build/xqwatcher/ansible_overrides.yml /
+WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays
+RUN /edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook harstorage.yml \
+    -i '127.0.0.1,' -c local \
+    -t "install:base,install:configuration,install:app-requirements,install:code" \    
+    -e@/ansible_overrides.yml
+WORKDIR /edx/app/harstorage/harstorage
+CMD ["/edx/app/harstorage/venvs/harstorage/bin/paster", "serve", "--daemon", "/edx/app/harstorage/venvs/harstorage/edx/etc/harstorage/production.ini"]
+

docker/build/harstorage/ansible_overrides.yml

+---
+
+FLOCK_TLD: "edx"

docker/plays/harstorage.yml

+- name: Deploy Harstorage
+  hosts: all
+  sudo: True
+  gather_facts: True
+  roles:
+    - docker
+    - mongo
+    - harstorage

playbooks/edx-east/harstorage.yml

+- name: Deploy Harstorage
+  hosts: all
+  sudo: True
+  gather_facts: True
+  vars:
+    nginx_default_sites:
+      - harstorage
+  roles:
+    - aws
+    - mongo
+    - nginx
+    - harstorage

playbooks/roles/harstorage/defaults/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+##
+# Defaults for role harstorage
+# 
+
+#
+# vars are namespaced with the module name.
+#
+harstorage_role_name: harstorage
+
+harstorage_user: '{{ harstorage_role_name }}'
+harstorage_home: '{{ COMMON_APP_DIR }}/{{ harstorage_role_name }}'
+harstorage_code_dir: '{{ harstorage_home }}/{{ harstorage_role_name }}'
+harstorage_venv_dir: '{{ harstorage_home }}/venvs/{{ harstorage_role_name }}'
+harstorage_bin_dir: '{{ harstorage_home }}/bin'
+harstorage_etc: '/edx/etc/harstorage'
+# Source Code
+
+HARSTORAGE_REPOS:
+ - PROTOCOL: https
+   DOMAIN: github.com
+   PATH: edx
+   REPO: harstorage
+   VERSION: e0d/update-requirements
+   DESTINATION: '{{ harstorage_code_dir }}'
+
+#
+# OS packages
+#
+
+harstorage_debian_pkgs:
+  -  lib32stdc++6
+
+harstorage_pagespeed_binary: "https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/harstorage/pagespeed_bin"
+
+harstorage_python_pkgs:
+  - { name: "pylons", version: "1.0.2"}
+  - { name: "webob", version: "1.5.1"}
+  - { name: "pymongo", version: "3.2.1"}
+  - { name: "PasteScript", version: "1.7.5"}
+
+harstorage_redhat_pkgs: []
+harstorage_port: "5000"
+harstorage_host: "0.0.0.0"
+harstorage_version: "1.0"
+
+# mongo packages
+mongo_port: "27017"
+mongo_repl_set: "repl1"
+mongo_admin_user: "admin"
+mongo_admin_password: "admin"
+
+harstorage_gunicorn_hosts:
+  - 127.0.0.1
+harstorage_gunicorn_port: '{{ harstorage_port }}'
+HARSTORAGE_HOSTNAME: '~^((stage|prod)-)?harstorage.*'
+HARSTORAGE_NGINX_PORT: 18170
+HARSTORAGE_SSL_NGINX_PORT: 48170

playbooks/roles/harstorage/meta/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+##
+# Role includes for role harstorage
+# 
+# Example:
+#
+# dependencies:
+#   - {
+#   role: my_role 
+#   my_role_var0: "foo"
+#   my_role_var1: "bar"
+# }
+
+dependencies:
+  - common
+  - supervisor
+  - role: edx_service
+    edx_service_name: "{{ harstorage_role_name }}"
+    edx_service_repos: "{{ HARSTORAGE_REPOS }}"
+    edx_service_user: "{{ harstorage_user }}"
+    edx_service_home: "{{ harstorage_home }}"
+    edx_service_packages:
+      debian: "{{ harstorage_debian_pkgs }}"
+      redhat: "{{ harstorage_redhat_pkgs }}"

playbooks/roles/harstorage/tasks/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+#
+#
+# Tasks for role harstorage
+# 
+# Overview:
+# 
+#
+# Dependencies:
+#
+# 
+# Example play:
+#
+#
+
+- name: install python packages
+  pip:
+    name: "{{ item.name }}"
+    version: "{{ item.version }}"
+    virtualenv: "{{ harstorage_venv_dir }}"
+    virtualenv_command: virtualenv
+  tags:
+    - install
+    - install:app-requirements
+  sudo_user: "{{ harstorage_user }}"
+  with_items: harstorage_python_pkgs
+
+- name: create directories
+  file:
+    path: "{{ item }}"
+    owner: "{{ harstorage_user }}"
+    group: "{{ harstorage_user }}"
+    state: directory
+    mode: 0755
+  tags:
+    - install
+    - install:configuration
+  with_items:
+    - "{{ harstorage_etc }}"
+    - "{{ harstorage_bin_dir }}"
+
+- name: ensure common web user can write to /edx/var/harstorage
+  file:
+    path: "{{ COMMON_DATA_DIR }}/{{ harstorage_user }}"
+    state: directory
+    mode: 0775
+  tags:
+    - install
+    - install:configuration
+
+- name: download pagespeed
+  get_url:
+    url: "{{ harstorage_pagespeed_binary }}"
+    dest: "{{ harstorage_bin_dir }}"
+    mode: "0755"
+    owner: "{{ harstorage_user }}"
+
+- name: setup the harstorage production.ini file
+  template:
+    src: '.{{ harstorage_etc }}/production.ini.j2'
+    dest: '{{ harstorage_etc }}/production.ini'
+    owner: '{{ harstorage_user }}'
+    group: '{{ harstorage_user }}'
+    mode: 0644
+  tags:
+    - install
+    - install:configuration
+
+- name: install harstorage
+  command: >
+    {{ harstorage_venv_dir }}/bin/python ./setup.py install
+  args:
+    chdir: "{{ harstorage_code_dir }}"
+  tags:
+    - install
+    - install:code
+
+- name: apply config
+  command: "{{ harstorage_venv_dir }}/bin/paster setup-app {{ harstorage_etc }}/production.ini"
+  args:
+    chdir: "{{ harstorage_code_dir }}"
+  tags:
+    - install
+    - install:configuration
+
+- name: write supervisor wrapper script
+  template:
+    src: edx/app/harstorage/harstorage.sh.j2
+    dest: "{{ harstorage_home }}/{{ harstorage_role_name }}.sh"
+    mode: 0650
+    owner: "{{ supervisor_user }}"
+    group: "{{ common_web_user }}"
+
+- name: write supervisord config
+  template:
+    src: edx/app/supervisor/conf.d.available/harstorage.conf.j2
+    dest: "{{ supervisor_available_dir }}/{{ harstorage_role_name }}.conf"
+    owner: "{{ supervisor_user }}"
+    group: "{{ common_web_user }}"
+    mode: 0644
+
+- name: enable supervisor script
+  file:
+    src: "{{ supervisor_available_dir }}/{{ harstorage_role_name }}.conf"
+    dest: "{{ supervisor_cfg_dir }}/{{ harstorage_role_name }}.conf"
+    state: link
+    force: yes
+  when: not disable_edx_services
+
+- name: update supervisor configuration
+  shell: "{{ supervisor_ctl }} -c {{ supervisor_cfg }} update"
+  when: not disable_edx_services
+
+- name: Copying nginx configs for harstorage
+  template:
+    src: "edx/app/nginx/sites-available/harstorage.j2"
+    dest: "{{ nginx_sites_available_dir }}/harstorage"
+    owner: root
+    group: "{{ common_web_user }}"
+    mode: 0640
+  notify: reload nginx
+  tags:
+    - install
+    - install:vhosts
+
+- name: Creating nginx config links for discovery
+  file:
+    src: "{{ nginx_sites_available_dir }}/harstorage"
+    dest: "{{ nginx_sites_enabled_dir }}/harstorage"
+    state: link
+    owner: root
+    group: root
+  notify: reload nginx
+  tags:
+    - install
+    - install:vhosts

playbooks/roles/harstorage/templates/edx/app/harstorage/harstorage.sh.j2

+#!/usr/bin/env bash
+
+# {{ ansible_managed }}
+
+{{ harstorage_venv_dir }}/bin/paster serve {{ harstorage_etc }}/production.ini

playbooks/roles/harstorage/templates/edx/app/nginx/sites-available/harstorage.j2

+#
+# {{ ansible_managed }}
+#
+
+
+{% if nginx_default_sites is defined and "harstorage" in nginx_default_sites %}
+  {% set default_site = "default" %}
+{% else %}
+  {% set default_site = "" %}
+{% endif %}
+
+upstream harstorage_app_server {
+{% for host in harstorage_gunicorn_hosts %}
+    server {{ host }}:{{ harstorage_gunicorn_port }} fail_timeout=0;
+{% endfor %}
+}
+
+server {
+  server_name {{ HARSTORAGE_HOSTNAME }};
+
+  {% if NGINX_ENABLE_SSL %}
+
+  listen {{ HARSTORAGE_NGINX_PORT }} {{ default_site }};
+  listen {{ HARSTORAGE_SSL_NGINX_PORT }} ssl;
+
+  ssl_certificate /etc/ssl/certs/{{ NGINX_SSL_CERTIFICATE|basename }};
+  ssl_certificate_key /etc/ssl/private/{{ NGINX_SSL_KEY|basename }};
+  # request the browser to use SSL for all connections
+  add_header Strict-Transport-Security "max-age=31536000; includeSubDomains";
+
+  {% else %}
+  listen {{ HARSTORAGE_NGINX_PORT }} {{ default_site }};
+  {% endif %}
+
+  location ~ ^/static/(?P<file>.*) {
+    root {{ COMMON_DATA_DIR }}/{{ harstorage_role_name }};
+    try_files /staticfiles/$file =404;
+  }
+
+  location / {
+    try_files $uri @proxy_to_app;
+  }
+
+  {% if NGINX_ROBOT_RULES|length > 0 %}
+  location /robots.txt {
+      root {{ nginx_app_dir }};
+      try_files $uri /robots.txt =404;
+  }
+  {% endif %}
+
+  location @proxy_to_app {
+    {% if NGINX_SET_X_FORWARDED_HEADERS %}
+    proxy_set_header X-Forwarded-Proto $scheme;
+    proxy_set_header X-Forwarded-Port $server_port;
+    proxy_set_header X-Forwarded-For $remote_addr;
+    {% else %}
+    proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto;
+    proxy_set_header X-Forwarded-Port $http_x_forwarded_port;
+    proxy_set_header X-Forwarded-For $http_x_forwarded_for;
+    {% endif %}
+    proxy_set_header Host $http_host;
+
+    proxy_redirect off;
+    proxy_pass http://harstorage_app_server;
+  }
+
+  # Forward to HTTPS if we're an HTTP request...
+  if ($http_x_forwarded_proto = "http") {
+    set $do_redirect "true";
+  }
+
+  # Run our actual redirect...
+  if ($do_redirect = "true") {
+    rewrite ^ https://$host$request_uri? permanent;
+  }
+}

playbooks/roles/harstorage/templates/edx/app/supervisor/conf.d.available/harstorage.conf.j2

+#
+# {{ ansible_managed }}
+#
+[program:{{ harstorage_role_name }}]
+
+command={{ harstorage_home }}/{{ harstorage_role_name }}.sh
+user={{ common_web_user }}
+directory={{ harstorage_code_dir }}
+stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log
+stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log
+killasgroup=true
+stopasgroup=true

playbooks/roles/harstorage/templates/edx/etc/harstorage/production.ini.j2

+#
+# harstorage - Pylons development environment configuration
+#
+# The %(here)s variable will be replaced with the parent directory of this file
+#
+[DEFAULT]
+debug = false
+
+[server:main]
+use = egg:Paste#http
+host = {{ harstorage_host }}
+port = {{ harstorage_port }}
+
+[app:main]
+use = egg:harstorage
+full_stack = true
+static_files = true
+temp_store = {{ COMMON_DATA_DIR }}/{{ harstorage_user }}
+bin_store  = {{ harstorage_bin_dir }}
+ps_enabled = true
+static_version = {{ harstorage_version }}
+
+mongo_replicate = false
+mongo_replset = {{ mongo_repl_set }}
+mongo_host = localhost
+mongo_port = {{ mongo_port }}
+mongo_db   = {{ harstorage_role_name }}
+mongo_auth = false
+mongo_user = {{ mongo_admin_user }}
+mongo_pswd = {{ mongo_admin_password }}
+cache_dir = {{ COMMON_DATA_DIR }}/{{ harstorage_user }}
+beaker.session.key = harstorage
+beaker.session.secret = somesecret
+
+# Logging configuration
+[loggers]
+keys = root
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = INFO
+handlers = console
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(asctime)s,%(msecs)03d %(levelname)-5.5s [%(name)s] [%(threadName)s] %(message)s
+datefmt = %H:%M:%S

playbooks/roles/nginx/templates/edx/app/nginx/sites-available/harstorage.j2

+#
+# {{ ansible_managed }}
+#
+
+
+{% if nginx_default_sites is defined and "harstorage" in nginx_default_sites %}
+  {% set default_site = "default" %}
+{% else %}
+  {% set default_site = "" %}
+{% endif %}
+
+upstream harstorage_app_server {
+{% for host in harstorage_gunicorn_hosts %}
+    server {{ host }}:{{ harstorage_gunicorn_port }} fail_timeout=0;
+{% endfor %}
+}
+
+server {
+  server_name {{ HARSTORAGE_HOSTNAME }};
+
+  {% if NGINX_ENABLE_SSL %}
+
+  listen {{ HARSTORAGE_NGINX_PORT }} {{ default_site }};
+  listen {{ HARSTORAGE_SSL_NGINX_PORT }} ssl;
+
+  ssl_certificate /etc/ssl/certs/{{ NGINX_SSL_CERTIFICATE|basename }};
+  ssl_certificate_key /etc/ssl/private/{{ NGINX_SSL_KEY|basename }};
+  # request the browser to use SSL for all connections
+  add_header Strict-Transport-Security "max-age=31536000; includeSubDomains";
+
+  {% else %}
+  listen {{ HARSTORAGE_NGINX_PORT }} {{ default_site }};
+  {% endif %}
+
+  location ~ ^/static/(?P<file>.*) {
+    root {{ COMMON_DATA_DIR }}/{{ harstorage_role_name }};
+    try_files /staticfiles/$file =404;
+  }
+
+  location / {
+    try_files $uri @proxy_to_app;
+  }
+
+  {% if NGINX_ROBOT_RULES|length > 0 %}
+  location /robots.txt {
+      root {{ nginx_app_dir }};
+      try_files $uri /robots.txt =404;
+  }
+  {% endif %}
+
+  location @proxy_to_app {
+    {% if NGINX_SET_X_FORWARDED_HEADERS %}
+    proxy_set_header X-Forwarded-Proto $scheme;
+    proxy_set_header X-Forwarded-Port $server_port;
+    proxy_set_header X-Forwarded-For $remote_addr;
+    {% else %}
+    proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto;
+    proxy_set_header X-Forwarded-Port $http_x_forwarded_port;
+    proxy_set_header X-Forwarded-For $http_x_forwarded_for;
+    {% endif %}
+    proxy_set_header Host $http_host;
+
+    proxy_redirect off;
+    proxy_pass http://harstorage_app_server;
+  }
+
+  # Forward to HTTPS if we're an HTTP request...
+  if ($http_x_forwarded_proto = "http") {
+    set $do_redirect "true";
+  }
+
+  # Run our actual redirect...
+  if ($do_redirect = "true") {
+    rewrite ^ https://$host$request_uri? permanent;
+  }
+}