Merge pull request #309 from edx/e0d/analytics-server

E0d/analytics server

playbooks/roles/analytics-server/defaults/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+#
+# Vars for role analytics-server
+# vars are namespace with the module name.
+#
+
+AS_DB_ANALYTICS_PASSWORD: 'CHANGEME!'
+AS_DB_ANALYTICS_USER: 'analytics001'
+AS_DB_ANALYTICS_HOST: 'localhost'
+AS_SERVER_PORT: '9000'
+AS_ENV_LANG: 'en_US.UTF-8'
+AS_LOG_LEVEL: 'INFO'
+AS_WORKERS: '4'
+
+DATABASES:
+  default: &databases_default
+    ENGINE: 'django.db.backends.mysql'
+    NAME: 'wwc'
+    USER: 'analytics001'
+    PASSWORD: 'CHANGEME!'
+    HOST: 'CHANGEME'
+    PORT: 3306
+
+analytics_auth_config:
+  DATABASES:
+    analytics:
+      <<: *databases_default
+      USER: $AS_DB_ANALYTICS_USER
+      PASSWORD: $AS_DB_ANALYTICS_PASSWORD
+      HOST: $AS_DB_ANALYTICS_HOST
+  ANALYTICS_API_KEY: $AS_API_KEY
+  ANALYTICS_RESULTS_DB:
+    MONGO_URI: $AS_DB_RESULTS_URL
+    MONGO_DB: $AS_DB_RESULTS_DB
+    MONGO_STORED_QUERIES_COLLECTION: $AS_DB_RESULTS_COLLECTION
+
+as_role_name: "analytics-server"
+as_user: "analytics-server" 
+as_home: "/opt/wwc/analytics-server"
+as_venv_dir: "{{ as_home }}/virtualenvs/analytics-server"
+as_source_repo: "git@github.com:edx/analytics-server.git"
+as_code_dir: "{{ as_home }}/src"
+as_version: "master"
+as_git_identity_path: "{{ secure_dir }}/files/git-identity"
+as_git_identity_dest: "/etc/{{ as_role_name }}.git-identity"
+as_git_ssh: "/tmp/{{ as_role_name }}.git_ssh.sh"
+as_requirements_file: "{{ as_code_dir }}/requirements.txt"
+as_rsyslog_enabled: "yes"
+as_web_user: "www-data"
+as_env: "analytics-server_env"
+as_service_variant: 'analytics'
+as_django_settings: 'anserv.settings'
+
+as_env_vars:
+  ANALYTICS_SERVER_LOG_LEVEL: "{{ AS_LOG_LEVEL }}"
+
+#
+# Used by the included role, automated.
+# See meta/main.yml
+#
+as_automated_rbash_links:
+  - /usr/bin/sudo
+  - /usr/bin/scp
+
+#
+# OS packages
+#
+
+as_debian_pkgs:
+  - mongodb-clients
+  - zip
+  - libmysqlclient-dev
+
+as_redhat_pkgs:
+  - zip
+  - community-mysql-libs
+
+#
+# Installed via pip to get the IAM role feature.
+#
+as_pip_pkgs:
+  - git+https://github.com/s3tools/s3cmd.git#egg=s3cmd
\ No newline at end of file

playbooks/roles/analytics-server/files/etc/sudoers.d/99-automator-analytics-server

+automator  ALL=(www-data) NOPASSWD:SETENV:/opt/wwc/analytics-server/virtualenvs/analytics-server/bin/django-admin.py	run_all_queries	*

playbooks/roles/analytics-server/files/git_ssh.sh

+#!/bin/sh
+exec /usr/bin/ssh -o StrictHostKeyChecking=no -i /etc/git-identity "$@"

playbooks/roles/analytics-server/handlers/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+#
+#
+# Handlers for role analytics-server
+# 
+# Overview:
+# 
+#
+
+- name: analytics-server | stop the analytics service
+  service: name=analytics state=stopped
+
+- name: analytics-server | start the analytics service
+  service: name=analytics state=started

playbooks/roles/analytics-server/meta/main.yml

+---
+
+dependencies:
+  - {
+  role: automated,
+  automated_rbash_links: $as_automated_rbash_links,
+  autmoated_sudoers_dest: '99-automator-analytics-server',
+  automated_sudoers_template: 'roles/analytics-server/templates/etc/sudoers.d/99-automator-analytics-server.j2'
+  }
+
+  
\ No newline at end of file

playbooks/roles/analytics-server/tasks/deploy.yml

+#
+# TODO: Needed while this repo is private
+#
+- name: analytics-server | upload ssh script
+  template: 
+    src=tmp/{{ as_role_name }}.git_ssh.sh.j2 dest={{ as_git_ssh }}
+    force=yes owner=root group=adm mode=750
+  tags:
+  - analytics-server
+  - deploy
+  - install
+  - update
+
+#
+# TODO: Needed while this repo is private
+#
+- name: analytics-server | install read-only ssh key required for checkout
+  copy: 
+    src={{ as_git_identity_path }} dest={{ as_git_identity_dest }}
+    force=yes owner=ubuntu group=adm mode=0600
+  tags:
+  - analytics-server
+  - deploy
+  - install
+  - update
+
+- name: analytics-server | checkout code
+  git: 
+    dest={{ as_code_dir }} repo={{ as_source_repo }} 
+    version={{ as_version }} force=true
+  environment:
+    GIT_SSH: $as_git_ssh
+    notify: analytics-server | restart the analytics service
+  notify: analytics-server | start the analytics service
+  tags:
+    - analytics-server
+    - deploy
+    - install
+    - update
+
+#
+# TODO: Needed while this repo is private
+#
+- name: analytics-server | update src permissions
+  file: 
+    path={{ as_code_dir }} state=directory owner={{ as_user }} 
+    group={{ as_web_user }} mode=2750 recurse=yes
+  tags:
+  - analytics-server
+  - deploy
+  - install
+  - update
+
+#
+# TODO: Needed while this repo is private
+#
+- name: analytics-server | remove read-only ssh key for the content repo
+  file: path={{ as_git_identity_dest }} state=absent
+  tags:
+  - analytics-server
+  - deploy
+  - install
+  - update
+
+#
+# TODO: Needed while this repo is private
+#
+- name: analytics-server | remove ssh script
+  file: path={{ as_git_ssh }} state=absent
+  tags:
+  - analytics-server
+  - deploy
+  - install
+  - update
+
+- name: analytics-server | install application requirements
+  pip: 
+    requirements={{ as_requirements_file }}
+    virtualenv={{ as_venv_dir }} state=present
+  sudo: true  
+  sudo_user: "{{ as_user }}"
+  notify: analytics-server | start the analytics service  
+  tags:
+  - analytics-server
+  - deploy
+  - install
+  - update

playbooks/roles/analytics-server/tasks/main.yml

+---
+
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+#
+#
+# Tasks for role analytics-server
+# 
+# Overview:
+#
+# Installs the edX analytics-server Django application which provides
+# basic analytics to the LMS instructor dashboard via service calls.
+#
+# Dependencies:
+#
+# common role
+#
+# Depends upon the automated role
+# 
+# Example play:
+#
+# - name: Configure analytics instance(s)
+#   hosts: analytics-servers
+#   sudo: True
+#   vars_files:
+#     - "{{ secure_dir }}/vars/common/common.yml"
+#     - "{{ secure_dir }}/vars/stage/analytics-server.yml"
+#     - "{{ secure_dir }}/vars/users.yml"
+#   gather_facts: True
+#   roles:
+#     - common
+#     - analytics-server
+#
+- name: analytics-server | install system packages
+  apt: pkg={{','.join(as_debian_pkgs)}} state=present
+  tags:
+  - analytics-server
+  - install
+  - update
+
+- name: analytics-server | create analytics-server user {{ as_user }}
+  user: 
+    name={{ as_user }} state=present shell=/bin/bash 
+    home={{ as_home }} createhome=yes
+  tags:
+  - analytics-server
+  - install
+  - update
+
+- name: analytics-server | setup the analytics-server env
+  template: 
+    src=opt/wwc/analytics-server/{{ as_env }}.j2
+    dest={{ as_home }}/{{ as_env }} 
+    owner="{{ as_user }}" group="{{ as_user }}"
+  tags:
+  - analytics-server
+  - install
+  - update
+
+- name: analytics-server | drop a bash_profile
+  copy: >
+    src=../../common/files/bash_profile 
+    dest={{ as_home }}/.bash_profile 
+    owner={{ as_user }} 
+    group={{ as_user }}
+
+# Awaiting next ansible release.
+#- name: analytics-server | ensure .bashrc exists
+# file: path={{ as_home }}/.bashrc state=touch
+# sudo: true  
+# sudo_user: "{{ as_user }}"
+# tags:
+# - analytics-server
+# - install
+# - update
+
+- name: analytics-server | ensure .bashrc exists
+  shell: touch {{ as_home }}/.bashrc
+  sudo: true  
+  sudo_user: "{{ as_user }}"
+  tags:
+  - analytics-server
+  - install
+  - update
+
+- name: analytics-server | add source of analytics-server_env to .bashrc
+  lineinfile:
+    dest={{ as_home }}/.bashrc
+    regexp='. {{ as_home }}/analytics-server_env'    
+    line='. {{ as_home }}/analytics_server_env'
+  tags:
+  - analytics-server
+  - install
+  - update
+
+- name: analytics-server | add source venv to .bashrc
+  lineinfile:
+    dest={{ as_home }}/.bashrc
+    regexp='. {{ as_venv_dir }}/bin/activate'    
+    line='. {{ as_venv_dir }}/bin/activate'
+  tags:
+  - analytics-server
+  - install
+  - update
+
+- name: analytics-server | install global python requirements
+  pip: name={{ item }}
+  with_items: as_pip_pkgs
+  tags:
+  - analytics-server
+  - install
+  - update
+
+- name: analytics-server | create config
+  template: 
+    src=opt/wwc/analytics.auth.json.j2
+    dest=/opt/wwc/analytics.auth.json
+    mode=0600
+    owner="{{ as_web_user }}" group="{{ as_web_user }}"
+  tags:
+  - analytics-server
+  - install
+  - update
+  
+- name: analytics-server | install service
+  template:
+    src=etc/init/analytics.conf.j2 dest=/etc/init/analytics.conf
+    owner=root group=root
+
+- include: deploy.yml    
\ No newline at end of file

playbooks/roles/analytics-server/templates/etc/init/analytics.conf.j2

+# {{ ansible_managed }}
+
+description "Analytics server under gunicorn"
+
+start on runlevel [2345]
+stop on runlevel [!2345]
+
+respawn
+respawn limit 3 30
+
+env SERVICE_VARIANT={{ as_service_variant }}
+env PID=/var/tmp/analytics.pid
+env WORKERS={{ AS_WORKERS }}
+env PORT={{ AS_SERVER_PORT }}
+env LANG={{ AS_ENV_LANG }}
+env DJANGO_SETTINGS_MODULE={{ as_django_settings }}
+
+chdir {{ as_code_dir }}
+setuid {{ as_web_user }}
+
+exec {{ as_venv_dir }}/bin/gunicorn -b 0.0.0.0:$PORT -w $WORKERS --pythonpath={{ as_code_dir }}/anserv anserv.wsgi

playbooks/roles/analytics-server/templates/opt/wwc/analytics-server/analytics-server_env.j2

+# {{ ansible_managed }}
+
+{% for name,value in as_env_vars.items() %}
+{% if value %}
+export {{ name }}="{{ value }}"
+{% endif %}		 
+{% endfor %}
\ No newline at end of file

playbooks/roles/analytics-server/templates/opt/wwc/analytics.auth.json.j2

+{{ analytics_auth_config | to_nice_json }}

playbooks/roles/analytics-server/templates/tmp/analytics-server.git_ssh.sh.j2

+#!/bin/sh
+exec /usr/bin/ssh -o StrictHostKeyChecking=no -i {{ as_git_identity_dest }} "$@"

playbooks/roles/analytics/tasks/main.yml

+---
+
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+#
+#
+# Tasks for role analytics
+# 
+# Overview:
+#
+# Installs the edX analytics Django application which provides
+# basic analytics to the LMS instructor dashboard via service calls.
+#
+# Dependencies:
+#
+# common role
+#
+# Depends upon the automated role
+# 
+# Example play:
+#
+# - name: Configure analytics instance(s)
+#   hosts: analyticss
+#   sudo: True
+#   vars_files:
+#     - "{{ secure_dir }}/vars/common/common.yml"
+#     - "{{ secure_dir }}/vars/stage/analytics.yml"
+#     - "{{ secure_dir }}/vars/users.yml"
+#   gather_facts: True
+#   roles:
+#     - common
+#     - analytics
+#
+- name: analytics | install system packages
+  apt: pkg={{','.join(analytics_debian_pkgs)}} state=present
+  tags:
+  - analytics
+  - install
+  - update
+
+- name: analytics | create analytics user {{ analytics_user }}
+  user: 
+    name={{ analytics_user }} state=present shell=/bin/bash 
+    home={{ analytics_home }} createhome=yes
+  tags:
+  - analytics
+  - install
+  - update
+
+- name: analytics | setup the analytics env
+  template: 
+    src=opt/wwc/analytics/{{ analytics_env }}.j2
+    dest={{ analytics_home }}/{{ analytics_env }} 
+    owner="{{ analytics_user }}" group="{{ analytics_user }}"
+  tags:
+  - analytics
+  - install
+  - update
+
+- name: analytics | drop a bash_profile
+  copy: >
+    src=../../common/files/bash_profile 
+    dest={{ analytics_home }}/.bash_profile 
+    owner={{ analytics_user }} 
+    group={{ analytics_user }}
+
+# Awaiting next ansible release.
+#- name: analytics | ensure .bashrc exists
+# file: path={{ analytics_home }}/.bashrc state=touch
+# sudo: true  
+# sudo_user: "{{ analytics_user }}"
+# tags:
+# - analytics
+# - install
+# - update
+
+- name: analytics | ensure .bashrc exists
+  shell: touch {{ analytics_home }}/.bashrc
+  sudo: true  
+  sudo_user: "{{ analytics_user }}"
+  tags:
+  - analytics
+  - install
+  - update
+
+- name: analytics | add source of analytics_env to .bashrc
+  lineinfile:
+    dest={{ analytics_home }}/.bashrc
+    regexp='. {{ analytics_home }}/analytics_env'    
+    line='. {{ analytics_home }}/analytics_env'
+  tags:
+  - analytics
+  - install
+  - update
+
+- name: analytics | add source venv to .bashrc
+  lineinfile:
+    dest={{ analytics_home }}/.bashrc
+    regexp='. {{ analytics_venv_dir }}/bin/activate'    
+    line='. {{ analytics_venv_dir }}/bin/activate'
+  tags:
+  - analytics
+  - install
+  - update
+
+- name: analytics | install global python requirements
+  pip: name={{ item }}
+  with_items: analytics_pip_pkgs
+  tags:
+  - analytics
+  - install
+  - update
+
+- name: analytics | create config
+  template: 
+    src=opt/wwc/analytics.auth.json.j2
+    dest=/opt/wwc/analytics.auth.json
+    mode=0600
+    owner="{{ analytics_web_user }}" group="{{ analytics_web_user }}"
+  tags:
+  - analytics
+  - install
+  - update
+  
+- name: analytics | install service
+  template:
+    src=etc/init/analytics.conf.j2 dest=/etc/init/analytics.conf
+    owner=root group=root
+
+- include: deploy.yml    
\ No newline at end of file

playbooks/roles/analytics/templates/etc/sudoers.d/99-automator-analytics-server.j2

+automator  ALL=({{ analytics_web_user }}) NOPASSWD:SETENV:{{ analytics_venv_dir }}/bin/django-admin.py	run_all_queries	*

playbooks/roles/s3fs/defaults/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+##
+# Vars for role s3fs
+#
+
+s3fs_version: 'master'
+s3fs_download_src: 'https://github.com/tongwang/s3fs-c/archive'
+s3fs_archive: 'master.zip'
+s3fs_unzip_dest: 's3fs-c-master'
+s3fs_download_url: '{{ s3fs_download_src }}/{{ s3fs_archive }}'
+s3fs_temp_dir: '/var/tmp'
+
+#
+# vars are namespace with the module name.
+#
+s3fs_role_name: s3fs
+
+#
+# OS packages
+#
+
+s3fs_debian_pkgs:
+  - make
+  - g++
+  - libcurl4-openssl-dev
+  - libssl-dev
+  - libxml2-dev
+  - libfuse-dev
+
+s3fs_redhat_pkgs:
+  - gcc
+  - libstdc++-devel
+  - gcc-c++
+  - fuse
+  - fuse-devel
+  - curl-devel
+  - libxml2-devel
+  - openssl-devel
+  - mailcap
\ No newline at end of file

playbooks/roles/s3fs/tasks/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+# Tasks for role s3fs
+# 
+# Overview:
+#
+# Installs a forked version of s3fs from
+# https://github.com/tongwang/s3fs-c/archive that supports
+# using IAM roles for authentication.
+#
+# Dependencies:
+#
+# The role depending upon s3fs is responsible for mounting the
+# buckets.
+#
+# Assuming the following config
+#
+# my_role_s3fs_mounts:
+#   - { bucket: "my_bucket", mount_point: "/mnt/s3/my_bucket", owner: "root", group: "adm", mode: "0755" }
+#
+# The role would need to include tasks like the following
+#
+# - name: my_role | create s3fs mount points
+#   file:
+#     path={{ item.mount_point }} owner={{ item.owner }}
+#     group={{ item.group }} mode={{ item.mode }} state="directory"
+#   with_items: "{{ my_role_s3fs_mounts }}"
+#
+# - name: my_role | mount s3 buckets
+#   mount:
+#     name={{ item.mount_point }} src={{ item.bucket }} fstype=fuse.s3fs
+#     opts=use_cache=/tmp,iam_role={{ task_iam_role }},allow_other state=mounted
+#   with_items: "{{ myrole_s3fs_mounts }}"
+# 
+# Example play:
+#
+# Required sudo for the installation phase.
+#
+# - name: Configure instance(s)
+#   hosts: s3fs_hosts
+#   sudo: True
+#   vars_files:
+#     - "{{ secure_dir }}/vars/common/common.yml"
+#     - "{{ secure_dir }}/vars/users.yml"
+#   gather_facts: True
+#   roles:
+#     - common
+#     - s3fs
+#
+
+- name: s3fs | install system packages
+  apt: pkg={{','.join(s3fs_debian_pkgs)}} state=present
+  tags:
+  - s3fs
+  - install
+  - update
+
+- name: s3fs | fetch package
+  get_url:
+    url={{ s3fs_download_url }}
+    dest={{ s3fs_temp_dir }}
+    
+- name: s3fs | unzip package
+  shell:
+    /usr/bin/unzip {{ s3fs_archive }}
+    chdir={{ s3fs_temp_dir }}
+    creates={{ s3fs_temp_dir }}/{{ s3fs_unzip_dest }}/configure
+    
+- name: s3fs | configure
+  shell:
+    ./configure
+    chdir={{ s3fs_temp_dir }}/{{ s3fs_unzip_dest }}
+    creates={{ s3fs_temp_dir }}/{{ s3fs_unzip_dest }}/config.status
+    
+- name: s3fs | make
+  shell:
+    /usr/bin/make
+    chdir={{ s3fs_temp_dir }}/{{ s3fs_unzip_dest }}
+    creates={{ s3fs_temp_dir }}/{{ s3fs_unzip_dest }}/src/s3cmd
+
+- name: s3fs | make install
+  shell:
+    /usr/bin/make install
+    chdir={{ s3fs_temp_dir }}/{{ s3fs_unzip_dest }}
+    
\ No newline at end of file