Merge pull request #76 from edx/vik/ml-api-v1

Vik/ml api v1

.gitignore

 *.swp
+.idea/

playbooks/ec2.py

@@ -380,7 +380,7 @@ class Ec2Inventory(object):
         the dict '''
 
         if key in my_dict:
-            my_dict[key].append(element);
+            my_dict[key].append(element)
         else:
             my_dict[key] = [element]
 

playbooks/mlapi_prod.yml

+- hosts:
+    - tag_Group_mlapi_prod
+  vars_files:
+    - "{{ secure_dir }}/vars/mlapi_prod_vars.yml"
+    - "{{ secure_dir }}/vars/users.yml"
+    - "{{ secure_dir }}/vars/mlapi_prod_users.yml"
+  roles:
+    - common
+    - discern
+  sudo: True
+- hosts:
+    - tag_Group_mlapi-bastion_prod
+    - tag_Group_mlapi-rabbitmq_prod
+  vars_files:
+    - "{{ secure_dir }}/vars/mlapi_prod_vars.yml"
+    - "{{ secure_dir }}/vars/users.yml"
+    - "{{ secure_dir }}/vars/mlapi_prod_users.yml"
+  roles:
+    - common
+  sudo: True
\ No newline at end of file

playbooks/mlapi_sandbox.yml

+- hosts:
+    - tag_Group_mlapi_sandbox
+  vars_files:
+    - "{{ secure_dir }}/vars/mlapi_sandbox_vars.yml"
+    - "{{ secure_dir }}/vars/users.yml"
+    - "{{ secure_dir }}/vars/mlapi_sandbox_users.yml"
+  roles:
+    - common
+    - discern
+  sudo: True
+- hosts:
+    - tag_Group_mlapi-bastion_sandbox
+    - tag_Group_mlapi-rabbitmq_sandbox
+  vars_files:
+    - "{{ secure_dir }}/vars/mlapi_sandbox_vars.yml"
+    - "{{ secure_dir }}/vars/users.yml"
+    - "{{ secure_dir }}/vars/mlapi_sandbox_users.yml"
+  roles:
+    - common
+  sudo: True
+

playbooks/mlapi_stage.yml

+- hosts:
+    - tag_Group_mlapi_stage
+  vars_files:
+    - "{{ secure_dir }}/vars/mlapi_stage_vars.yml"
+    - "{{ secure_dir }}/vars/users.yml"
+    - "{{ secure_dir }}/vars/mlapi_stage_users.yml"
+  roles:
+    - common
+    - discern
+  sudo: True
+- hosts:
+    - tag_Group_mlapi-bastion_stage
+    - tag_Group_mlapi-rabbitmq_stage
+  vars_files:
+    - "{{ secure_dir }}/vars/mlapi_stage_vars.yml"
+    - "{{ secure_dir }}/vars/users.yml"
+    - "{{ secure_dir }}/vars/mlapi_stage_users.yml"
+  roles:
+    - common
+  sudo: True

playbooks/roles/common/tasks/edx_logging_base.yml

@@ -13,7 +13,7 @@
   - update
 
 - name: Touch tracking file into existence
-  command: touch -a {{log_base_dir}}/tracking.log creates={{log_base_dir}}/tracking.log 
+  command: touch -a {{log_base_dir}}/tracking.log creates={{log_base_dir}}/tracking.log
   tags:
   - logging
   - update

playbooks/roles/discern/files/git_ssh.sh

+#!/bin/sh
+exec /usr/bin/ssh -o StrictHostKeyChecking=no  "$@"

playbooks/roles/discern/files/sudoers-discern

+Defaults        env_keep+=SSH_AUTH_SOCK

playbooks/roles/discern/handlers/main.yml

+---
+- name: restart discern
+  service: name=discern state=restarted
+
+- name: restart celery
+  service: name=celery state=restarted
+
+- name: restart nginx
+  service: name=nginx state=restarted

playbooks/roles/discern/tasks/main.yml

+---
+#Create the templates for upstart services
+- name: render celery service from template
+  template: src=celery.conf.j2 dest=/etc/init/celery.conf owner=root group=edx mode=0664
+  notify: restart celery
+
+- name: render discern service from template
+  template: src=discern.conf.j2 dest=/etc/init/discern.conf owner=root group=edx mode=0664
+  notify: restart discern
+
+#Allows us to recover from a bad sudoers file
+- name: Install policykit
+  apt: pkg=policykit-1 install_recommends=yes state=present update_cache=yes
+
+#Discern user is admin
+- name: Create discern user
+  user: name={{ discern_user }} append=yes groups="adm,edx" shell=/bin/bash
+
+- name: upload sudoers template to /tmp/sudoers
+  copy: src=sudoers-discern dest=/tmp/{{site_name}} owner=root group=root mode=0440
+
+#Verify file and move to sudoers.d folder
+- name: move temp file to sudoers.d
+  shell: visudo -q -c -f /tmp/{{site_name}} && cp /tmp/{{site_name}} /etc/sudoers.d/{{site_name}}
+
+#Upload config files for django (auth and env)
+- name: create discern application config env.json file
+  template: src=env.json.j2 dest={{app_base_dir}}/env.json owner={{discern_user}} group=edx mode=0640
+  notify:
+    - restart celery
+    - restart discern
+
+- name: create discern auth file auth.json
+  template: src=auth.json.j2 dest={{app_base_dir}}/auth.json owner={{discern_user}} group=edx mode=0640
+  notify:
+    - restart celery
+    - restart discern
+
+#Needed if using redis to prevent memory issues
+- name: change memory commit settings -- needed for redis
+  command: sysctl vm.overcommit_memory=1
+
+#Upload custom git ssh script
+- name: upload ssh script
+  copy: src=git_ssh.sh dest=/tmp/git_ssh.sh force=yes owner=root group=adm mode=750
+
+- name: set permissions on app_base_dir sgid for edx
+  file: path={{app_base_dir}} owner=root group=edx mode=2775 state=directory
+  file: path={{venv_dir}} owner=root group=edx mode=2775 state=directory
+
+- name: Install git so that we can clone repos
+  apt: pkg=git install_recommends=yes state=present
+
+#Create directories for repos
+- name: create discern and ease directories and set permissions
+  file: path={{item}} owner={{discern_user}} group=edx mode=2775 state=directory
+  with_items:
+   - ${discern_dir}
+   - ${ease_dir}
+
+#Grab both repos or update
+- name: git checkout discern repo into discern_dir
+  git: dest={{discern_dir}} repo={{discern_source_repo}} version={{discern_branch}}
+  environment:
+    GIT_SSH: /tmp/git_ssh.sh
+  notify:
+    - restart celery
+    - restart discern
+
+- name: git checkout ease repo into ease_dir
+  git: dest={{ease_dir}} repo={{ease_source_repo}} version={{ease_branch}}
+  environment:
+    GIT_SSH: /tmp/git_ssh.sh
+  notify:
+    - restart celery
+    - restart discern
+
+#Install system packages
+- name: install discern and ease apt packages
+  command: xargs -a {{item}}/apt-packages.txt apt-get install -y
+  with_items:
+    - ${discern_dir}
+    - ${ease_dir}
+
+#Numpy has to be a pre-requirement in order for scipy to build
+- name : install python pre-requirements for discern and ease
+  pip: requirements="{{item}}/pre-requirements.txt" virtualenv="{{venv_dir}}" state=present
+  with_items:
+    - ${discern_dir}
+    - ${ease_dir}
+
+- name : install python requirements for discern and ease
+  pip: requirements="{{item}}/requirements.txt" virtualenv="{{venv_dir}}" state=present
+  with_items:
+    - ${discern_dir}
+    - ${ease_dir}
+
+- name: install ease python package
+  shell: command="{{venv_dir}}/bin/activate; cd {{ease_dir}}; python setup.py install"
+
+#Needed for the ease package to work
+- name: install nltk data using rendered shell script
+  shell: command="{{venv_dir}}/bin/python -m nltk.downloader -d {{nltk_data_dir}} all"
+
+- name: set permissions on nltk data directory
+  file: path={{nltk_data_dir}} owner={{discern_user}} group=edx mode=2775 state=directory
+
+#Run this instead of using the ansible module because the ansible module only support syncdb of these three, and does not
+#support virtualenvs as of this comment
+- name: django syncdb, migrate, and collectstatic for discern
+  shell: ${venv_dir}/bin/python {{discern_dir}}/manage.py {{item}} --noinput --settings={{discern_settings}} --pythonpath={{discern_dir}}
+  with_items:
+    - syncdb
+    - migrate
+    - collectstatic
+
+#Have this separate from the other three because it doesn't take the noinput flag
+- name: django update_index for discern
+  shell: ${venv_dir}/bin/python {{discern_dir}}/manage.py update_index --settings={{discern_settings}} --pythonpath={{discern_dir}}
+
+- name: create nginx directory and set perms
+  file: path=/etc/nginx/sites-available owner=root group=edx mode=2775 state=directory
+
+#Install nginx sites available
+#remove default link, render template, link template to sites-enabled to activate
+- name: Removing default nginx config
+  file: path=/etc/nginx/sites-enabled/default state=absent
+  notify: restart nginx
+
+- name: render nginx sites available
+  template: src=nginx-discern.j2 dest=/etc/nginx/sites-available/{{ site_name }}
+  notify: restart nginx
+
+- name: Creating nginx config link {{ site_name }}
+  file: src=/etc/nginx/sites-available/{{ site_name }} dest=/etc/nginx/sites-enabled/{{ site_name }} state=link owner=root group=root
+  notify: restart nginx
\ No newline at end of file

playbooks/roles/discern/templates/auth.json.j2

+{{ auth_config | to_nice_json }}
\ No newline at end of file

playbooks/roles/discern/templates/celery.conf.j2

+#Celery task for ml api
+
+description "Celery ML api"
+author "Vik Paruchuri <vik@edx.org>"
+
+start on runlevel [2345]
+stop on runlevel [!2345]
+
+respawn
+respawn limit 3 30
+
+env DJANGO_SETTINGS_MODULE={{discern_settings}}
+
+chdir {{discern_dir}}
+setuid {{discern_user}}
+
+exec {{venv_dir}}/bin/python {{discern_dir}}/manage.py celeryd --loglevel=info --settings={{discern_settings}} --pythonpath={{discern_dir}} -B --autoscale={{ ansible_processor_cores * 2 }},1

playbooks/roles/discern/templates/discern.conf.j2

+# gunicorn
+
+description "ML API Server"
+author "Vik Paruchuri <vik@edx.org>"
+
+start on runlevel [2345]
+stop on runlevel [!2345]
+
+respawn
+respawn limit 3 30
+
+env PID=/var/run/gunicorn/discern.pid
+env WORKERS={{ ansible_processor_cores * 2 }}
+env PORT={{ gunicorn_port }}
+env LANG=en_US.UTF-8
+env DJANGO_SETTINGS_MODULE={{discern_settings}}
+
+chdir {{discern_dir}}
+setuid {{discern_user}}
+
+exec {{venv_dir}}/bin/gunicorn --preload -b 127.0.0.1:$PORT -w $WORKERS --timeout=30 --pythonpath={{discern_dir}} discern.wsgi
\ No newline at end of file

playbooks/roles/discern/templates/env.json.j2

+{{ env_config | to_nice_json }}
\ No newline at end of file

playbooks/roles/discern/templates/nginx-discern.j2

+server {
+    listen {{nginx_listen_port}};
+    server_name localhost;
+
+    set $my_host $http_host;
+    if ($host ~ "\d+\.\d+\.\d+\.\d+") {
+        set $my_host "127.0.0.1";
+    }
+
+    # https://docs.djangoproject.com/en/dev/howto/static-files/#serving-static-files-in-production
+    location /static/ { # STATIC_URL
+        alias {{discern_dir}}/staticfiles/;
+        expires 1m;
+	autoindex on;
+    }
+
+    location /media/ { # MEDIA_URL
+        alias /home/www/myhostname/static/; # MEDIA_ROOT
+        expires 30d;
+    }
+
+    location / {
+        proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto;
+        proxy_set_header X-Forwarded-Port $http_x_forwarded_port;
+        proxy_set_header X-Forwarded-For $http_x_forwarded_for;
+        proxy_set_header Host $my_host;
+
+        proxy_pass http://127.0.0.1:7999;
+    }
+}
+

playbooks/roles/discern/vars/main.yml

+discern_source_repo: git@github.com:edx/discern.git
+ease_source_repo: git@github.com:edx/ease.git
+ease_dir: $app_base_dir/ease
+discern_dir: $app_base_dir/discern
+discern_settings: discern.aws
+nltk_data_dir: /usr/share/nltk_data
+ease_branch: master
+discern_branch: dev
+nginx_listen_port: 80
+gunicorn_port: 7999
+discern_user: discern
+site_name: discern
\ No newline at end of file