Merge remote-tracking branch 'origin/master' into vik/ml-api

cloudformation_templates/edx-reference-architecture.json

@@ -1316,6 +1316,12 @@
             "CidrIp":{
               "Ref":"SSHLocation"
             }
+          },
+          {
+            "IpProtocol":"tcp",
+            "FromPort":"9418",
+            "ToPort":"9418",
+            "CidrIp":"0.0.0.0/0"
           }
         ],
         "SecurityGroupEgress":[
@@ -1338,6 +1344,12 @@
             "CidrIp":{
               "Ref":"SSHLocation"
             }
+          },
+          {
+            "IpProtocol":"tcp",
+            "FromPort":"9418",
+            "ToPort":"9418",
+            "CidrIp":"0.0.0.0/0"
           }
         ]
       }
@@ -3123,7 +3135,7 @@
                 "Endpoint.Address"
               ]
             },
-            "'\n",
+            "',\n",
             "        'PORT': '",
             {
               "Fn::GetAtt":[

playbooks/edxapp_prod.yml

@@ -6,5 +6,6 @@
   roles:
     - common
     - nginx
+    - gunicorn
     - lms
     - ruby

playbooks/edxapp_rolling_example.yml

@@ -8,21 +8,20 @@
   pre_tasks:
     - name: Gathering ec2 facts 
       ec2_facts: 
-    - name: Gathering ELB facts
-      local_action: ec2_elb_facts
-# These two modules "ec2_facts" and "ec2_elb_facts" are invoked in the 
-# pre_tasks and give us the $elbs and $ansible_ec2_isntance_id facts 
-# which are variables that can be used in the playbook
-    - local_action: command util/elb_reg.py -e {{ ",".join(elbs[ansible_ec2_instance_id]) }} -i {{ ansible_ec2_instance_id }} deregister
-# -e is the list of elbs that the current instances belong to and -i is 
-# the instance "active_elbs" are the elbs that are passed in.
+    - name: Removing instance from the ELB
+      local_action: ec2_elb
+      args:
+        instance_id: "{{ ansible_ec2_instance_id }}"
+        state: 'absent'
   roles:
     - common
     - nginx 
     - lms
-#    - ruby
+    - ruby
   post_tasks:
-    - local_action: command util/elb_reg.py -e {{ ",".join(elbs[ansible_ec2_instance_id]) }} -i {{ ansible_ec2_instance_id }} register
-# Register will pass in the same elb list and the same instance id 
-# to add it back to the pool
-
+    - name: Adding instance back to the ELB
+      local_action: ec2_elb
+      args:
+        instance_id: "{{ ansible_ec2_instance_id }}"
+        ec2_elbs: "{{ ec2_elbs }}"
+        state: 'present'

playbooks/edxapp_stage.yml

@@ -6,5 +6,6 @@
   roles:
     - common
     - nginx
+    - gunicorn
     - lms
     - ruby

playbooks/library/ec2_elb

+#!/usr/bin/python
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+DOCUMENTATION = """
+---
+module: ec2_elb
+short_description: De-registers or registers instances from EC2 ELB(s)
+description:
+  - This module de-registers or registers an AWS EC2 instance from the ELB(s)
+    that it belongs to.
+  - Returns fact "ec2_elbs" which is a list of elbs attached to the instance
+    if state=absent is passed as an argument.
+  - Will be marked changed when called only if there are ELBs found to operate on.
+version_added: "1.2"
+requirements: [ "boto" ]
+author: John Jarvis
+options:
+  state:
+    description:
+      - register or deregister the instance
+    required: true
+
+  instance_id:
+    description:
+      - EC2 Instance ID
+    required: true
+
+  ec2_elbs:
+    description:
+      - List of ELB names, required for registration. The ec2_elbs fact should be used if there was a previous de-register.
+    required: false
+    default: None
+  ec2_secret_key:
+    description:
+      - AWS Secret API key
+    required: false
+    default: None
+  ec2_access_key:
+    description:
+      - AWS Access API key
+    required: false
+    default: None
+
+"""
+
+EXAMPLES = """
+# basic pre_task and post_task example
+pre_tasks:
+  - name: Gathering ec2 facts
+    ec2_facts:
+  - name: Instance De-register
+    local_action: ec2_elb
+    args:
+      instance_id: "{{ ansible_ec2_instance_id }}"
+      state: 'absent'
+roles:
+  - myrole
+post_tasks:
+  - name: Instance Register
+    local_action: ec2_elb
+    args:
+      instance_id: "{{ ansible_ec2_instance_id }}"
+      ec2_elbs: "{{ ec2_elbs }}"
+      state: 'present'
+"""
+
+import time
+import sys
+import os
+
+try:
+    import boto
+except ImportError:
+    print "failed=True msg='boto required for this module'"
+    sys.exit(1)
+
+
+class ElbManager:
+    """Handles EC2 instance ELB registration and de-registration"""
+
+    def __init__(self, module, instance_id=None, ec2_elbs=None,
+                 ec2_access_key=None, ec2_secret_key=None):
+        self.ec2_access_key = ec2_access_key
+        self.ec2_secret_key = ec2_secret_key
+        self.module = module
+        self.instance_id = instance_id
+        self.lbs = self._get_instance_lbs(ec2_elbs)
+        # if there are no ELBs to operate on
+        # there will be no changes made
+        if len(self.lbs) > 0:
+            self.changed = True
+        else:
+            self.changed = False
+
+    def deregister(self):
+        """De-register the instance from all ELBs and wait for the ELB
+        to report it out-of-service"""
+
+        for lb in self.lbs:
+            lb.deregister_instances([self.instance_id])
+            self._await_elb_instance_state(lb, 'OutOfService')
+
+    def register(self):
+        """Register the instance for all ELBs and wait for the ELB
+        to report the instance in-service"""
+
+        for lb in self.lbs:
+            lb.register_instances([self.instance_id])
+            self._await_elb_instance_state(lb, 'InService')
+
+    def _await_elb_instance_state(self, lb, awaited_state):
+        """Wait for an ELB to change state
+        lb: load balancer
+        awaited_state : state to poll for (string)"""
+
+        while True:
+            state = lb.get_instance_health([self.instance_id])[0].state
+            if state == awaited_state:
+                break
+            else:
+                time.sleep(1)
+
+    def _get_instance_lbs(self, ec2_elbs=None):
+        """Returns a list of ELBs attached to self.instance_id
+        ec2_elbs: an optional list of elb names that will be used
+                  for elb lookup instead of returning what elbs
+                  are attached to self.instance_id"""
+
+        try:
+            elb = boto.connect_elb(self.ec2_access_key, self.ec2_secret_key)
+        except boto.exception.NoAuthHandlerFound, e:
+            self.module.fail_json(msg=str(e))
+        elbs = elb.get_all_load_balancers()
+
+        if ec2_elbs:
+            lbs = sorted(lb for lb in elbs if lb.name in ec2_elbs)
+        else:
+            lbs = []
+            for lb in elbs:
+                for info in lb.instances:
+                    if self.instance_id == info.id:
+                        lbs.append(lb)
+        return lbs
+
+
+def main():
+
+    module = AnsibleModule(
+        argument_spec=dict(
+            state={'required': True,
+                    'choices': ['present', 'absent']},
+            instance_id={'required': True},
+            ec2_elbs={'default': None, 'required': False},
+            ec2_secret_key={'default': None, 'aliases': ['EC2_SECRET_KEY']},
+            ec2_access_key={'default': None, 'aliases': ['EC2_ACCESS_KEY']}
+        )
+    )
+
+    ec2_secret_key = module.params['ec2_secret_key']
+    ec2_access_key = module.params['ec2_access_key']
+    ec2_elbs = module.params['ec2_elbs']
+
+    if module.params['state'] == 'present' and 'ec2_elbs' not in module.params:
+        module.fail_json(msg="ELBs are required for registration")
+
+    if not ec2_secret_key and 'EC2_SECRET_KEY' in os.environ:
+        ec2_secret_key = os.environ['EC2_SECRET_KEY']
+    if not ec2_access_key and 'EC2_ACCESS_KEY' in os.environ:
+        ec2_access_key = os.environ['EC2_ACCESS_KEY']
+
+    instance_id = module.params['instance_id']
+    elb_man = ElbManager(module, instance_id, ec2_elbs, ec2_access_key,
+                         ec2_secret_key)
+
+    if module.params['state'] == 'present':
+        elb_man.register()
+    elif module.params['state'] == 'absent':
+        elb_man.deregister()
+
+    ansible_facts = {'ec2_elbs': [lb.name for lb in elb_man.lbs]}
+    ec2_facts_result = dict(changed=elb_man.changed, ansible_facts=ansible_facts)
+
+    module.exit_json(**ec2_facts_result)
+
+# this is magic, see lib/ansible/module_common.py
+#<<INCLUDE_ANSIBLE_MODULE_COMMON>>
+
+main()

playbooks/library/ec2_elb_facts

-#!/usr/bin/python
-# -*- coding: utf-8 -*-
-
-# This file is part of Ansible
-#
-# Ansible is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Ansible is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-
-DOCUMENTATION="""
----
-module: ec2_elb_facts
-short_description: Creates a fact with instance to ELB mappings
-version_added: "1.0"
-options: {}
-description:
-     - This module fetches data from AWS using the boto api and returns 
-       a hash as a fact that maps instances to the ELB(s) that they belong to.
-examples:
-    - code: ansible all -m ec2_elb_facts
-      description: Obtain ELB info using boto
-requirements: [ "boto" ]
-author: "John Jarvis <john@jarv.org>"
-"""
-
-try:
-    import boto
-except ImportError:
-    print "failed=True msg='boto required for this module'"
-    sys.exit(1)
-
-
-from collections import defaultdict
-
-def get_elb_info(module):
-
-    try:
-        elb = boto.connect_elb()
-    except boto.exception.NoAuthHandlerFound, e:
-        module.fail_json(msg = str(e))
-
-    elbs = elb.get_all_load_balancers()
-    elb_info = defaultdict(set)
-    for lb in elbs:
-        for info in lb.instances:
-            elb_info[info.id].add(lb.name)
-    elb_info = {k: list(v) for k,v in elb_info.iteritems()}
-    return elb_info
-
-
-def main():
-
-
-    module = AnsibleModule(argument_spec = dict())
-    elb_info = get_elb_info(module)
-    ec2_facts_result = dict(changed=False, ansible_facts={'elbs': elb_info})
-    module.exit_json(**ec2_facts_result)
-
-# this is magic, see lib/ansible/module_common.py
-#<<INCLUDE_ANSIBLE_MODULE_COMMON>>
-
-main()

playbooks/roles/gunicorn/tasks/main.yml

+---
+- name:  pip install gunicorn
+  pip: name=gunicorn virtualenv="{{venv_dir}}" state=present
+  tags:
+  - gunicorn

playbooks/roles/gunicorn/tasks/upstart.yml

+#write the gunicorn upstart script for {{ service_variant }}
+- name: writing ${service_variant} upstart script to /etc/init
+  sudo: True
+  template: src={{ item }} dest=/etc/init/${service_variant}.conf owner=root group=root
+  first_available_file:
+  - "{{ local_dir }}/gunicorn/templates/{{ service_variant }}.conf.j2"
+  # seems like paths in first_available_file must be relative to the playbooks dir
+  - "roles/gunicorn/templates/{{ service_variant }}.conf.j2"
+  tags:
+  - upstart
+  - gunicorn

playbooks/roles/gunicorn/templates/lms.conf.j2

+# gunicorn
+
+description "gunicorn server"
+author "Calen Pennington <cpennington@mitx.mit.edu>"
+
+start on runlevel [2345]
+stop on runlevel [!2345]
+
+respawn
+respawn limit 3 30
+
+env PID=/var/tmp/lms.pid
+#env NEW_RELIC_CONFIG_FILE=${app_base_dir}/opt/wwc/newrelic.ini
+#env NEWRELIC=${app_base_dir}/bin/newrelic-admin
+env WORKERS=${lms_num_workers}
+env PORT=8000
+env LANG=en_US.UTF-8
+env DJANGO_SETTINGS_MODULE=lms.envs.aws
+env SERVICE_VARIANT="lms"
+
+
+chdir ${app_base_dir}/mitx
+setuid www-data
+
+exec ${venv_dir}/bin/gunicorn --preload -b 127.0.0.1:$PORT -w $WORKERS --timeout=300 --pythonpath=${app_base_dir}/mitx lms.wsgi
+
+post-start script
+  while true
+  do
+    if $(curl -s -i localhost:$PORT/heartbeat | egrep -q '200 OK'); then
+      break;
+    else
+      sleep 1;
+    fi
+  done
+end script
+

playbooks/roles/gunicorn/vars/main.yml

+---
+lms_num_workers: 1
\ No newline at end of file

playbooks/roles/lms/tasks/main.yml

@@ -8,38 +8,41 @@
   sudo: True
   tags:
   - lms
+  - lms-env
 
 - name: create lms auth file
   template: src=auth.json.j2 dest=$app_base_dir/lms.auth.json
   sudo: True
   tags:
   - lms
+  - lms-env
 
 - include: ../../nginx/tasks/nginx_site.yml state=link site_name=lms
 - include: ../../nginx/tasks/nginx_site.yml state=link site_name=lms-backend
 
+
 # Install ssh keys for ubuntu account to be able to check out from mitx
 # Temprory behavior, not needed after June 1. Perhaps still useful as a recipe.
 # {{ secure_dir }} is relative to the top-level playbooks dir so there is some
 # ugly relative pathing here
 
 - name: install read-only ssh key for mitx repo (private)
-  copy: src=../../../{{ secure_dir }}/files/git-identity dest=/etc/git-identity force=yes owner=root group=edx mode=640
+  copy: src=../../../{{ secure_dir }}/files/git-identity dest=/etc/git-identity force=yes owner=root group=adm mode=640
   sudo: True
   tags:
   - lms
   - cms
 
 - name: upload ssh script
-  copy: src=git_ssh.sh dest=/tmp/git_ssh.sh force=yes owner=root group=edx mode=750
+  copy: src=git_ssh.sh dest=/tmp/git_ssh.sh force=yes owner=root group=adm mode=750
   sudo: True
   tags:
   - lms
   - cms
 
 # Check out mitx repo to $app_base_dir
-- name: set permissions on $app_base_dir sgid for edx
-  file: path=$app_base_dir owner=root group=edx mode=2775 state=directory
+- name: set permissions on $app_base_dir sgid for adm
+  file: path=$app_base_dir owner=root group=adm mode=2775 state=directory
   sudo: True
   tags:
   - lms
@@ -102,7 +105,7 @@
   - cms
 
 # Install the python modules into $venv_dir 
-- name : install python pre-requirements
+- name : install python packages using the shell
   #pip: requirements="{{app_base_dir}}/mitx/requirements.txt" virtualenv="{{venv_dir}}" 
   # Need to use shell rather than pip so that we can maintain the context of our current working directory; some 
   # requirements are pathed relative to the mitx repo. Using the pip from inside the virtual environment implicitly
@@ -111,3 +114,6 @@
   tags:
   - lms
   - cms
+  
+# Creates LMS upstart file
+- include: ../../gunicorn/tasks/upstart.yml service_variant=lms

playbooks/roles/nginx/tasks/nginx_site.yml

@@ -11,6 +11,7 @@
   tags:
   - nginx
   - lms
+  - nginx-env
 
 - name: Creating nginx config link {{ site_name }}
   sudo: True
@@ -19,3 +20,4 @@
   tags:
   - nginx
   - lms
+  - nginx-env

playbooks/roles/ruby/tasks/main.yml

@@ -3,9 +3,13 @@
 - name: Create 'www' user (replicating historical environment)
   user: name=www state=present
   sudo: True
+  tags:
+  - ruby
 - name: Create ruby base
   sudo: True
   file: path=$ruby_base state=directory owner=www group=www
+  tags:
+  - ruby
 - name: rbenv | install build depends
   sudo: true
   apt: pkg=$item state=present install_recommends=no
@@ -19,66 +23,96 @@
     - libxml2-dev
     - libxslt1-dev
     - zlib1g-dev
+  tags:
+  - ruby
 
 - name: rbenv | update rbenv repo
   sudo: true
   git: repo=git://github.com/sstephenson/rbenv.git dest=$rbenv_root version=v0.4.0
+  tags:
+  - ruby
 
 - name: rbenv | add rbenv to path
   sudo: true
   file: path=/usr/local/bin/rbenv src=${rbenv_root}/bin/rbenv state=link
+  tags:
+  - ruby
 
 - name: rbenv | add rbenv initialization to profile
   sudo: true
   template: src=rbenv.sh.j2 dest=/etc/profile.d/rbenv.sh owner=root group=root mode=0755
+  tags:
+  - ruby
 
 - name: rbenv | check ruby-build installed
   sudo: true
   command: test -x /usr/local/bin/ruby-build
   register: rbuild_present
   ignore_errors: yes
+  tags:
+  - ruby
 
 - name: rbenv | create temporary directory
   command: mktemp -d
   register: tempdir
   when_failed: $rbuild_present
+  tags:
+  - ruby
 
 - name: rbenv | clone ruby-build repo
   git: repo=git://github.com/sstephenson/ruby-build.git dest=${tempdir.stdout}/ruby-build
   when_failed: $rbuild_present
+  tags:
+  - ruby
 
 - name: rbenv | install ruby-build
   sudo: true
   command: ./install.sh chdir=${tempdir.stdout}/ruby-build
   when_failed: $rbuild_present
+  tags:
+  - ruby
 
 - name: rbenv | remove temporary directory
   file: path=${tempdir.stdout} state=absent
   when_failed: $rbuild_present
+  tags:
+  - ruby
 
 - name: rbenv | check ruby $ruby_version installed
   shell: RBENV_ROOT=${rbenv_root} rbenv versions | grep $ruby_version
   register: ruby_installed
   ignore_errors: yes
+  tags:
+  - ruby
 
 - name: rbenv | install ruby $ruby_version
   shell: RBENV_ROOT=${rbenv_root} rbenv install $ruby_version
   sudo: true
   when_failed: $ruby_installed
+  tags:
+  - ruby
 
 - name: rbenv | set global ruby $ruby_version
   shell: RBENV_ROOT=${rbenv_root} rbenv global $ruby_version
   sudo: true
   when_failed: $ruby_installed
+  tags:
+  - ruby
 
 - name: rbenv | rehash
   shell: RBENV_ROOT=${rbenv_root} rbenv rehash
   sudo: true
   when_failed: $ruby_installed
+  tags:
+  - ruby
 
 - name: gem | gem install bundler
   shell: RBENV_ROOT=${rbenv_root} GEM_HOME=${gem_home} ${rbenv_root}/shims/gem install bundle chdir=${app_base_dir}/mitx
   sudo: true
+  tags:
+  - ruby
 
 - name: bundle | bundle install
-  shell: RBENV_ROOT=${rbenv_root} GEM_HOME=${gem_home} ${gem_home}/bin/bundle install --deployment --binstubs chdir=${app_base_dir}/mitx
+  shell: RBENV_ROOT=${rbenv_root} GEM_HOME=${gem_home} ${gem_home}/bin/bundle install --binstubs chdir=${app_base_dir}/mitx
+  tags:
+  - ruby

playbooks/roles/ruby/templates/rbenv.sh.j2

 export RBENV_ROOT="{{ rbenv_root }}"
 export GEM_HOME="{{ gem_home }}"
+export PATH="{{ gem_home }}/bin:$PATH"
 eval "$(rbenv init -)"

playbooks/secure_example/vars/edxapp_prod_vars.yml

@@ -46,6 +46,11 @@ env_config:
   'MEDIA_URL':  'hidden-prod'
   'BOOK_URL':  'hidden-prod'
   'ANALYTICS_SERVER_URL':  'hidden-prod'
+  'DEFAULT_FROM_EMAIL':  'hidden-stage'
+  'DEFAULT_FEEDBACK_EMAIL': 'hidden-stage'
+  'ADMINS' : 
+  - ['name', 'email']
+  'TIME_ZONE': 'America/New_York'
   'CACHES':
     'default':
       'KEY_PREFIX':  'hidden-prod'

playbooks/secure_example/vars/edxapp_stage_vars.yml

@@ -46,6 +46,11 @@ env_config:
   'MEDIA_URL':  'hidden-stage'
   'BOOK_URL':  'hidden-stage'
   'ANALYTICS_SERVER_URL':  'hidden-stage'
+  'DEFAULT_FROM_EMAIL':  'hidden-stage'
+  'DEFAULT_FEEDBACK_EMAIL': 'hidden-stage'
+  'ADMINS' : 
+  - ['name', 'email']
+  'TIME_ZONE': 'America/New_York'
   'CACHES':
     'default':
       'KEY_PREFIX':  'hidden-stage'