Merge pull request #1227 from edx/jarv/analytics-api

Jarv/analytics api

cloud_migrations/vpc-migrate-analytics_api.yml

+#
+# Overview:
+# This play needs to be run per environment-deployment and you will need to
+# provide the boto environment and vpc_id as arguments
+#
+# ansible-playbook -i 'localhost,' ./vpc-migrate-analytics_api-edge-stage.yml \
+# -e 'profile=edge vpc_id=vpc-416f9b24'
+#
+# Caveats
+#
+# - This requires ansible 1.6
+# - Required the following branch of Ansible /e0d/add-instance-profile from
+#   https://github.com/e0d/ansible.git
+# - This play isn't full idempotent because of and ec2 module update issue
+#   with ASGs.  This can be worked around by deleting the ASG and re-running
+#   the play
+# - The instance_profile_name will need to be created in advance as there
+#   isn't a way to do so from ansible.
+#
+# Prequisities:
+# Create a iam ec2 role
+#
+
+- name: Add resources for the Analytics API
+  hosts: localhost
+  connection: local
+  gather_facts: False
+  tasks:
+      # Fail intermittantly with the following error:
+      # The specified rule does not exist in this security group
+    - name: Create instance security group
+      ec2_group:
+        profile: "{{ profile }}"
+        description: "Open up SSH access"
+        name: "{{ security_group }}"
+        vpc_id: "{{ vpc_id }}"
+        region: "{{ ec2_region }}"
+        rules:
+          - proto: tcp
+            from_port: "{{ sec_group_ingress_from_port }}"
+            to_port: "{{ sec_group_ingress_to_port }}"
+            cidr_ip: "{{ item }}"
+      with_items: sec_group_ingress_cidrs
+      register: created_sec_group
+      ignore_errors: True
+
+    - name: debug
+      debug:
+        msg: "Registered created_sec_group: {{ created_sec_group }}"
+
+    # Needs ansible 1.7 for vpc support of elbs
+    # - name: Create elb security group
+    #   ec2_group:
+    #     profile: "{{ profile }}"
+    #     description: "ELB security group"
+    #     name: "ELB-{{ security_group }}"
+    #     vpc_id: "{{ vpc_id }}"
+    #     region: "{{ ec2_region }}"
+    #     rules:
+    #       - proto: tcp
+    #         from_port: "443"
+    #         to_port: "443"
+    #         cidr_ip: "0.0.0.0/0"
+    #   register: created_elb_sec_group
+    #   ignore_errors: True
+      
+    # Needs 1.7 for VPC support
+    # - name: "Create ELB"
+    #   ec2_elb_lb:
+    #     profile: "{{ profile }}"
+    #     region: "{{ ec2_region }}"
+    #     zones:
+    #       - us-east-1b
+    #       - us-east-1c
+    #     name: "{{ edp }}"
+    #     state: present
+    #     security_group_ids: "{{ created_elb_sec_group.group_id }}"
+    #     listeners:
+    #       - protocol: https
+    #         load_balancer_port: 443
+    #         instance_protocol: http # optional, defaults to value of protocol setting
+    #         instance_port: 80
+    #         # ssl certificate required for https or ssl
+    #         ssl_certificate_id: "{{ ssl_cert }}"
+
+      # instance_profile_name was added by me in my fork
+    - name: Create the launch configuration
+      ec2_lc:
+        profile: "{{ profile }}"
+        region: "{{ ec2_region }}"
+        name: "{{ lc_name }}"
+        image_id: "{{ lc_ami }}"
+        key_name: "{{ key_name }}"
+        security_groups: "{{ created_sec_group.results[0].group_id }}"
+        instance_type: "{{ instance_type }}"
+        instance_profile_name: "{{ instance_profile_name }}"
+        volumes:
+          - device_name: "/dev/sda1"
+            volume_size: "{{ instance_volume_size }}"
+
+    - name: Create ASG
+      ec2_asg:
+        profile: "{{ profile }}"
+        region: "{{ ec2_region }}"
+        name: "{{ asg_name }}"
+        launch_config_name: "{{ lc_name }}"
+        load_balancers: "{{ elb_name }}"
+        availability_zones:
+          - us-east-1b
+          - us-east-1c
+        min_size: 0
+        max_size: 2
+        desired_capacity: 1
+        vpc_zone_identifier: "{{ subnets|join(',') }}"
+        instance_tags:
+          Name: "{{ env }}-{{ deployment }}-{{ play }}"
+          autostack: "true"
+          environment: "{{ env }}"
+          deployment: "{{ deployment }}"
+          play: "{{ play }}"
+          services: "{{ play }}"
+      register: asg
+
+    - name: debug
+      debug:
+        msg: "DEBUG: {{ asg }}"
+
+    - name: Create scale up policy
+      ec2_scaling_policy:
+        state: present
+        profile: "{{ profile }}"
+        region: "{{ ec2_region }}"
+        name: "{{ edp }}-ScaleUpPolicy"
+        adjustment_type: "ChangeInCapacity"
+        asg_name: "{{ asg_name }}"
+        scaling_adjustment: 1
+        min_adjustment_step: 1
+        cooldown: 60
+      register: scale_up_policy
+
+    - name: debug
+      debug:
+        msg: "Registered scale_up_policy: {{ scale_up_policy }}"
+
+    - name: Create scale down policy
+      ec2_scaling_policy:
+        state: present
+        profile: "{{ profile }}"
+        region: "{{ ec2_region }}"
+        name: "{{ edp }}-ScaleDownPolicy"
+        adjustment_type: "ChangeInCapacity"
+        asg_name: "{{ asg_name }}"
+        scaling_adjustment: -1
+        min_adjustment_step: 1
+        cooldown: 60
+      register: scale_down_policy
+      
+    - name: debug
+      debug:
+        msg: "Registered scale_down_policy: {{ scale_down_policy }}"
+
+#
+# Sometimes the scaling policy reports itself changed, but
+# does not return data about the policy. It's bad enough
+# that consistent data isn't returned when things
+# have and have not changed; this make writing idempotent
+# tasks difficult.
+    - name: create high-cpu alarm
+      ec2_metric_alarm:
+        state: present
+        region: "{{ ec2_region }}"
+        name: "cpu-high"
+        metric: "CPUUtilization"
+        namespace: "AWS/EC2"
+        statistic: Average
+        comparison: ">="
+        threshold: 90.0
+        period: 300
+        evaluation_periods: 2
+        unit: "Percent"
+        description: "Scale-up if CPU > 90% for 10 minutes"
+        dimensions: {"AutoScalingGroupName":"{{ asg_name }}"}
+        alarm_actions: ["{{ scale_up_policy.arn }}"]
+      when: scale_up_policy.arn is defined
+
+    - name: create low-cpu alarm
+      ec2_metric_alarm:
+        state: present
+        region: "{{ ec2_region }}"
+        name: "cpu-low"
+        metric: "CPUUtilization"
+        namespace: "AWS/EC2"
+        statistic: Average
+        comparison: "<="
+        threshold: 50.0
+        period: 300
+        evaluation_periods: 2
+        unit: "Percent"
+        description: "Scale-down if CPU < 50% for 10 minutes"
+        dimensions: {"AutoScalingGroupName":"{{ asg_name }}"}
+        alarm_actions: ["{{ scale_down_policy.arn }}"]
+      when: scale_down_policy.arn is defined
\ No newline at end of file

playbooks/edx-east/analytics_api.yml

+- name: Deploy Analytics API
+  hosts: all
+  sudo: True
+  gather_facts: True
+  vars:
+    ENABLE_DATADOG: False
+    ENABLE_SPLUNKFORWARDER: False
+    ENABLE_NEWRELIC: False
+  roles:
+    - aws
+    - analytics_api 
+    - role: datadog
+      when: COMMON_ENABLE_DATADOG
+    - role: splunkforwarder
+      when: COMMON_ENABLE_SPLUNKFORWARDER
+    - role: newrelic
+      when: COMMON_ENABLE_NEWRELIC

playbooks/roles/analytics_api/defaults/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+##
+# Defaults for role analytics_api
+#
+
+ANALYTICS_API_GIT_IDENTITY: !!null
+
+# depends upon Newrelic being enabled via COMMON_ENABLE_NEWRELIC
+# and a key being provided via NEWRELIC_LICENSE_KEY
+ANALYTICS_API_NEWRELIC_APPNAME: "your Newrelic appname"
+ANALYTICS_API_PIP_EXTRA_ARGS: "-i {{ COMMON_PYPI_MIRROR_URL }}"
+
+#
+# vars are namespace with the module name.
+#
+analytics_api_role_name: "analytics_api"
+analytics_api_service_name: "analytics_api"
+analytics_api_user: "analytics_api"
+
+analytics_api_app_dir: "{{ COMMON_APP_DIR }}/{{ analytics_api_service_name }}"
+analytics_api_home: "{{ COMMON_APP_DIR }}/{{ analytics_api_service_name }}"
+analytics_api_venv_base: "{{ analytics_api_home }}/venvs"
+analytics_api_venv_dir: "{{ analytics_api_venv_base }}/{{ analytics_api_service_name }}"
+analytics_api_venv_bin: "{{ analytics_api_venv_dir }}/bin"
+analytics_api_code_dir: "{{ analytics_api_app_dir }}/src/edx-analytics-data-api"
+analytics_api_conf_dir: "{{ analytics_api_home }}"
+analytics_api_gunicorn_host: "0.0.0.0"
+analytics_api_gunicorn_port: "8080"
+analytics_api_gunicorn_workers: "8"
+analytics_api_gunicorn_timeout: "300"
+
+analytics_api_django_settings: "production"
+analytics_api_source_repo: "git@{{ COMMON_GIT_MIRROR }}:edx/edx-analytics-data-api"
+analytics_api_git_ssh_opts: "-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i {{ analytics_api_git_identity_file }}"
+ANALYTICS_API_VERSION: "master"
+analytics_api_git_identity_file: "{{ analytics_api_app_dir }}/git-identity"
+
+analytics_api_log_dir: "{{ COMMON_LOG_DIR }}/{{ analytics_api_service_name }}"
+
+analytics_api_requirements_base: "{{ analytics_api_code_dir }}/requirements"
+analytics_api_requirements:
+  - base.txt
+  - production.txt
+  - optional.txt
+
+ANALYTICS_API_CONFIG:
+  SECRET_KEY: 'Your secret key here'
+  TIME_ZONE: 'America/New_York'
+  LANGUAGE_CODE: 'en-us'
+  # email config
+  EMAIL_HOST: 'smtp.example.com'
+  EMAIL_HOST_PASSWORD: ''
+  EMAIL_HOST_USER: ''
+  EMAIL_PORT: 587
+  API_AUTH_TOKEN: 'put-your-api-token-here'
+  # db config
+  DATABASES:
+    default:
+      ENGINE: 'django.db.backends.mysql'
+      NAME: 'name'
+      USER: 'user'
+      PASSWORD: 'password'
+      HOST: 'localhost'
+      PORT: '3306'
+
+#
+# OS packages
+#
+
+analytics_api_debian_pkgs:
+  - 'libmysqlclient-dev'
+
+analytics_api_redhat_pkgs: []

playbooks/roles/analytics_api/meta/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+#
+# Role includes for role analytics_api
+#
+# Example:
+#
+# dependencies:
+#   - {
+#   role: my_role
+#   my_role_var0: "foo"
+#   my_role_var1: "bar"
+# }
+
+dependencies:
+  - role: edx_service
+    edx_role_name: "{{ analytics_api_role_name }}"
+    edx_service_name: "{{ analytics_api_service_name }}"
+  - supervisor
\ No newline at end of file

playbooks/roles/analytics_api/tasks/deploy.yml

+---
+
+- name: install read-only ssh key
+  copy: >
+    content="{{ ANALYTICS_API_GIT_IDENTITY }}" dest={{ analytics_api_git_identity_file }}
+    owner={{ analytics_api_user }} group={{ analytics_api_user }} mode=0600
+
+- name: checkout code
+  git: >
+    dest={{ analytics_api_code_dir }} repo={{ analytics_api_source_repo }} version={{ ANALYTICS_API_VERSION }}
+    accept_hostkey=yes
+    ssh_opts="{{ analytics_api_git_ssh_opts }}"
+
+- name: install application requirements
+  pip: >
+    requirements="{{ analytics_api_requirements_base }}/{{ item }}"
+    virtualenv="{{ analytics_api_venv_dir }}" state=present
+  sudo_user: "{{ analytics_api_user }}"
+  with_items: analytics_api_requirements
+
+- name: write out app config file
+  template: >
+    src=edx/app/analytics_api/analytics_api.yaml.j2
+    dest={{ analytics_api_app_dir }}/{{ analytics_api_service_name }}.yaml
+    mode=0644 owner={{ analytics_api_user }} group={{ analytics_api_user }}
+
+- name: write out the supervisior wrapper
+  template: >
+    src=edx/app/analytics_api/analytics_api.sh.j2
+    dest={{ analytics_api_app_dir }}/{{ analytics_api_service_name }}.sh
+    mode=0650 owner={{ supervisor_user }} group={{ common_web_user }}
+
+- name: write supervisord config
+  template: >
+    src=edx/app/supervisor/conf.d.available/analytics_api.conf.j2
+    dest="{{ supervisor_available_dir }}/{{ analytics_api_service_name }}.conf"
+    owner={{ supervisor_user }} group={{ common_web_user }} mode=0644
+
+- name: enable supervisor script
+  file: >
+    src={{ supervisor_available_dir }}/{{ analytics_api_service_name }}.conf
+    dest={{ supervisor_cfg_dir }}/{{ analytics_api_service_name }}.conf
+    state=link
+    force=yes
+  when: not disable_edx_services
+
+- name: update supervisor configuration
+  shell: "{{ supervisor_ctl }} -c {{ supervisor_cfg }} update"
+  when: not disable_edx_services
+
+- name: remove read-only ssh key for the content repo
+  file: path={{ analytics_api_git_identity_file }} state=absent

playbooks/roles/analytics_api/tasks/main.yml

+---
+#
+# edX Configuration
+#
+# github:     https://github.com/edx/configuration
+# wiki:       https://github.com/edx/configuration/wiki
+# code style: https://github.com/edx/configuration/wiki/Ansible-Coding-Conventions
+# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
+#
+# Tasks for role analytics_api
+#
+# Overview:
+#
+# Install the Analytics Data API server, a python
+# django application that runs under gunicorn
+#
+# Dependencies:
+#
+# Example play:
+# - name: Deploy Analytics API
+#   hosts: all
+#   sudo: True
+#   gather_facts: True
+#   vars:
+#     ENABLE_DATADOG: False
+#     ENABLE_SPLUNKFORWARDER: False
+#     ENABLE_NEWRELIC: False
+#   roles:
+#     - aws
+#     - analytics_api
+#
+# ansible-playbook -i 'api.example.com,' ./analytics_api.yml  -e@/ansible/vars/deployment.yml -e@/ansible/vars/env-deployment.yml
+#
+
+- fail: msg="You must provide an private key for the analytics repo"
+  when: not ANALYTICS_API_GIT_IDENTITY
+
+- include: deploy.yml tags=deploy

playbooks/roles/analytics_api/templates/edx/app/analytics_api/analytics_api.sh.j2

+#!/usr/bin/env bash
+
+# {{ ansible_managed }}
+
+{% if COMMON_ENABLE_NEWRELIC %}
+{% set executable = analytics_api_venv_bin + '/newrelic-admin run-program ' + analytics_api_venv_bin + '/gunicorn' %}
+{% else %}
+{% set executable = analytics_api_venv_bin + '/gunicorn' %}
+{% endif %}
+
+{% if COMMON_ENABLE_NEWRELIC %}
+export NEW_RELIC_APP_NAME="{{ ANALYTICS_API_NEWRELIC_APPNAME }}"
+export NEW_RELIC_LICENSE_KEY="{{ NEWRELIC_LICENSE_KEY }}"
+{% endif -%}
+
+#
+# These blocks output environment vars that will
+# eventually be replaced by loading a yaml file in
+# the django settings.  That file is currently also
+# being created, but the var reading doesn't exist
+{% for name,value in ANALYTICS_ENV_VARS.items() %}
+{% if value %}
+export {{ name }}="{{ value }}"
+{% endif -%}
+{% endfor %}
+
+{% for name,value in ANALYTICS_DATABASES.default.items() %}
+{% if value %}
+export DATABASE_{{ name }}="{{ value }}"
+{% endif -%}
+{% endfor %}
+
+{{ executable }} -b {{ analytics_api_gunicorn_host }}:{{ analytics_api_gunicorn_port }} -w {{ analytics_api_gunicorn_workers }} --timeout={{ analytics_api_gunicorn_timeout }}  analyticsdataserver.wsgi:application

playbooks/roles/analytics_api/templates/edx/app/analytics_api/analytics_api.yaml

+---
+# {{ ansible_managed }}
+
+{{ ANALYTICS_API_CONFIG | to_nice_yaml }}
\ No newline at end of file

playbooks/roles/analytics_api/templates/edx/app/analytics_api/analytics_api.yaml.j2

+---
+# {{ ansible_managed }}
+
+{{ ANALYTICS_API_CONFIG | to_nice_yaml }}
\ No newline at end of file

playbooks/roles/analytics_api/templates/edx/app/supervisor/conf.d.available/analytics_api.conf.j2

+# {{ ansible_managed }}
+
+[program:{{ analytics_api_service_name }}]
+
+command={{ analytics_api_app_dir }}/analytics_api.sh
+user={{ common_web_user }}
+directory={{ analytics_api_code_dir }}
+stdout_logfile={{ supervisor_log_dir }}/%(program_name)-stdout.log
+stderr_logfile={{ supervisor_log_dir }}/%(program_name)-stderr.log
+killasgroup=true
+stopasgroup=true

playbooks/roles/edx_service/tasks/main.yml

@@ -10,7 +10,7 @@
 #
 #
 # Tasks for role edx_service
-# 
+#
 # Overview:
 #
 # This role performs the repetive tasks that most edX roles
@@ -45,6 +45,16 @@
     - "{{ COMMON_APP_DIR }}/{{ edx_service_name }}/venvs"
     - "{{ COMMON_APP_DIR }}/{{ edx_service_name }}/data"
 
+- name: create edx_service log dir
+  file: >
+    path="{{ item }}"
+    state=directory
+    owner="{{ common_web_user }}"
+    group="{{ common_web_group }}"
+  with_items:
+    - "{{ COMMON_LOG_DIR }}/{{ edx_service_name }}"
+
+
 - name: install a bunch of system packages on which edx_service relies
   apt: pkg={{ item }} state=present
   with_items: "{{ edx_service_name }}_debian_pkgs"
@@ -53,4 +63,4 @@
 - name: install a bunch of system packages on which edx_service relies
   yum: pkg={{ item }} state=present
   with_items: "{{ edx_service_name }}_redhat_pkgs"
-  when: ansible_distribution in common_redhat_variants
\ No newline at end of file
+  when: ansible_distribution in common_redhat_variants