resolving merge conflicts

.gitignore

 *.swp
 *.pyc
+.idea/

README.md

@@ -53,6 +53,26 @@ problems occur.
 
 Details on how to build the stack using Ansible are available below.
 
+
+### Post Bringup Manual Commands
+
+Unfortunately there is some infrastructure that we need that is currently not supported
+by CloudFormation.  So once your stack is created by CloudFormation you need to run
+a few manual commands to fill in those gaps.
+
+This requires that you've installed the command line utilities for [ElastiCache][cachecli]
+and [EC2][ec2cli].  Note that we requrire at least version 1.8 of the ElastiCache CLI due
+to some newer commands that we rely on.
+  [cachecli]: http://aws.amazon.com/developertools/2310261897259567
+  [ec2cli]: http://aws.amazon.com/developertools/351
+
+At the end of the CloudFormation run you should check the "Outputs" tab in
+Amazon UI and that will have the commands you need to run.  This screenshot 
+shows what that output looks like.
+
+![Amazon CloudFormation Output Screenshot](cfn-output-example.png)
+
+
 ### Connecting to Hosts in the Stack
 
 Because the reference architecture makes use of an Amazon VPC, you will not be able
@@ -64,16 +84,12 @@ Add something like the following to your `~/.ssh/config` file.
 
 ```
 Host *.us-west-1.compute-internal
-  ProxyCommand ssh -W %h:%p vpc-00000000-jumpbox
-  IdentityFile /path/to/aws/key.pem
+  ProxyCommand ssh -W %h:%p vpc-us-west-1-jumpbox
   ForwardAgent yes
-  User ubuntu
 
-Host vpc-00000000-jumpbox
-  HostName 54.236.224.226
-  IdentityFile /path/to/aws/key.pem
+Host vpc-us-west-1-jumpbox
+  HostName 54.236.202.101
   ForwardAgent yes
-  User ubuntu
 ```
 
 This assumes that you only have one VPC in the ```us-west-1``` region

playbooks/ec2.py

@@ -380,7 +380,7 @@ class Ec2Inventory(object):
         the dict '''
 
         if key in my_dict:
-            my_dict[key].append(element);
+            my_dict[key].append(element)
         else:
             my_dict[key] = [element]
 

playbooks/mlapi_prod.yml

+- hosts:
+    - tag_Group_mlapi_prod
+  vars_files:
+    - "{{ secure_dir }}/vars/mlapi_prod_vars.yml"
+    - "{{ secure_dir }}/vars/users.yml"
+    - "{{ secure_dir }}/vars/mlapi_prod_users.yml"
+  roles:
+    - common
+    - discern
+  sudo: True
+- hosts:
+    - tag_Group_mlapi-bastion_prod
+    - tag_Group_mlapi-rabbitmq_prod
+  vars_files:
+    - "{{ secure_dir }}/vars/mlapi_prod_vars.yml"
+    - "{{ secure_dir }}/vars/users.yml"
+    - "{{ secure_dir }}/vars/mlapi_prod_users.yml"
+  roles:
+    - common
+  sudo: True
\ No newline at end of file

playbooks/mlapi_sandbox.yml

+- hosts:
+    - tag_Group_mlapi_sandbox
+  vars_files:
+    - "{{ secure_dir }}/vars/mlapi_sandbox_vars.yml"
+    - "{{ secure_dir }}/vars/users.yml"
+    - "{{ secure_dir }}/vars/mlapi_sandbox_users.yml"
+  roles:
+    - common
+    - discern
+  sudo: True
+- hosts:
+    - tag_Group_mlapi-bastion_sandbox
+    - tag_Group_mlapi-rabbitmq_sandbox
+  vars_files:
+    - "{{ secure_dir }}/vars/mlapi_sandbox_vars.yml"
+    - "{{ secure_dir }}/vars/users.yml"
+    - "{{ secure_dir }}/vars/mlapi_sandbox_users.yml"
+  roles:
+    - common
+  sudo: True
+

playbooks/mlapi_stage.yml

+- hosts:
+    - tag_Group_mlapi_stage
+  vars_files:
+    - "{{ secure_dir }}/vars/mlapi_stage_vars.yml"
+    - "{{ secure_dir }}/vars/users.yml"
+    - "{{ secure_dir }}/vars/mlapi_stage_users.yml"
+  roles:
+    - common
+    - discern
+  sudo: True
+- hosts:
+    - tag_Group_mlapi-bastion_stage
+    - tag_Group_mlapi-rabbitmq_stage
+  vars_files:
+    - "{{ secure_dir }}/vars/mlapi_stage_vars.yml"
+    - "{{ secure_dir }}/vars/users.yml"
+    - "{{ secure_dir }}/vars/mlapi_stage_users.yml"
+  roles:
+    - common
+  sudo: True

playbooks/roles/common/tasks/edx_logging_base.yml

@@ -13,7 +13,7 @@
   - update
 
 - name: Touch tracking file into existence
-  command: touch -a {{log_base_dir}}/tracking.log creates={{log_base_dir}}/tracking.log 
+  command: touch -a {{log_base_dir}}/tracking.log creates={{log_base_dir}}/tracking.log
   tags:
   - logging
   - update

playbooks/roles/common/tasks/main.yml

@@ -3,13 +3,13 @@
 
 - name: Create application root
   # In the future consider making group edx r/t adm
-  file: path=$app_base_dir state=directory owner=root group=adm mode=2775 
+  file: path={{ app_base_dir }} state=directory owner=root group=adm mode=2775 
   tags:
   - pre_install
   - update
 
 - name: Create upload directory
-  file: path=$app_base_dir/uploads mode=2775 state=directory owner=root group=adm
+  file: path={{ app_base_dir }}/uploads mode=2775 state=directory owner=root group=adm
   tags:
   - pre_install
   - update
@@ -20,6 +20,12 @@
   - pre_install
   - update
   
+- name: Create staticfiles dir
+  file: path={{ app_base_dir }}/staticfiles state=directory owner=root group=adm mode=2775 
+  tags:
+  - pre_install
+  - update
+  
 - name: Install role-independent useful system packages
   # do this before log dir setup; rsyslog package guarantees syslog user present
   apt: pkg={{item}} install_recommends=yes state=present update_cache=yes

playbooks/roles/discern/files/git_ssh.sh

+#!/bin/sh
+exec /usr/bin/ssh -o StrictHostKeyChecking=no  "$@"

playbooks/roles/discern/files/sudoers-discern

+Defaults        env_keep+=SSH_AUTH_SOCK

playbooks/roles/discern/handlers/main.yml

+---
+- name: restart discern
+  service: name=discern state=restarted
+
+- name: restart celery
+  service: name=celery state=restarted
+
+- name: restart nginx
+  service: name=nginx state=restarted

playbooks/roles/discern/tasks/main.yml

+---
+#Create the templates for upstart services
+- name: render celery service from template
+  template: src=celery.conf.j2 dest=/etc/init/celery.conf owner=root group=edx mode=0664
+  notify: restart celery
+
+- name: render discern service from template
+  template: src=discern.conf.j2 dest=/etc/init/discern.conf owner=root group=edx mode=0664
+  notify: restart discern
+
+#Allows us to recover from a bad sudoers file
+- name: Install policykit
+  apt: pkg=policykit-1 install_recommends=yes state=present update_cache=yes
+
+#Discern user is admin
+- name: Create discern user
+  user: name={{ discern_user }} append=yes groups="adm,edx" shell=/bin/bash
+
+- name: upload sudoers template to /tmp/sudoers
+  copy: src=sudoers-discern dest=/tmp/{{site_name}} owner=root group=root mode=0440
+
+#Verify file and move to sudoers.d folder
+- name: move temp file to sudoers.d
+  shell: visudo -q -c -f /tmp/{{site_name}} && cp /tmp/{{site_name}} /etc/sudoers.d/{{site_name}}
+
+#Upload config files for django (auth and env)
+- name: create discern application config env.json file
+  template: src=env.json.j2 dest={{app_base_dir}}/env.json owner={{discern_user}} group=edx mode=0640
+  notify:
+    - restart celery
+    - restart discern
+
+- name: create discern auth file auth.json
+  template: src=auth.json.j2 dest={{app_base_dir}}/auth.json owner={{discern_user}} group=edx mode=0640
+  notify:
+    - restart celery
+    - restart discern
+
+#Needed if using redis to prevent memory issues
+- name: change memory commit settings -- needed for redis
+  command: sysctl vm.overcommit_memory=1
+
+#Upload custom git ssh script
+- name: upload ssh script
+  copy: src=git_ssh.sh dest=/tmp/git_ssh.sh force=yes owner=root group=adm mode=750
+
+- name: set permissions on app_base_dir sgid for edx
+  file: path={{app_base_dir}} owner=root group=edx mode=2775 state=directory
+  file: path={{venv_dir}} owner=root group=edx mode=2775 state=directory
+
+- name: Install git so that we can clone repos
+  apt: pkg=git install_recommends=yes state=present
+
+#Create directories for repos
+- name: create discern and ease directories and set permissions
+  file: path={{item}} owner={{discern_user}} group=edx mode=2775 state=directory
+  with_items:
+   - ${discern_dir}
+   - ${ease_dir}
+
+#Grab both repos or update
+- name: git checkout discern repo into discern_dir
+  git: dest={{discern_dir}} repo={{discern_source_repo}} version={{discern_branch}}
+  environment:
+    GIT_SSH: /tmp/git_ssh.sh
+  notify:
+    - restart celery
+    - restart discern
+
+- name: git checkout ease repo into ease_dir
+  git: dest={{ease_dir}} repo={{ease_source_repo}} version={{ease_branch}}
+  environment:
+    GIT_SSH: /tmp/git_ssh.sh
+  notify:
+    - restart celery
+    - restart discern
+
+#Install system packages
+- name: install discern and ease apt packages
+  command: xargs -a {{item}}/apt-packages.txt apt-get install -y
+  with_items:
+    - ${discern_dir}
+    - ${ease_dir}
+
+#Numpy has to be a pre-requirement in order for scipy to build
+- name : install python pre-requirements for discern and ease
+  pip: requirements="{{item}}/pre-requirements.txt" virtualenv="{{venv_dir}}" state=present
+  with_items:
+    - ${discern_dir}
+    - ${ease_dir}
+
+- name : install python requirements for discern and ease
+  pip: requirements="{{item}}/requirements.txt" virtualenv="{{venv_dir}}" state=present
+  with_items:
+    - ${discern_dir}
+    - ${ease_dir}
+
+- name: install ease python package
+  shell: command="{{venv_dir}}/bin/activate; cd {{ease_dir}}; python setup.py install"
+
+#Needed for the ease package to work
+- name: install nltk data using rendered shell script
+  shell: command="{{venv_dir}}/bin/python -m nltk.downloader -d {{nltk_data_dir}} all"
+
+- name: set permissions on nltk data directory
+  file: path={{nltk_data_dir}} owner={{discern_user}} group=edx mode=2775 state=directory
+
+#Run this instead of using the ansible module because the ansible module only support syncdb of these three, and does not
+#support virtualenvs as of this comment
+- name: django syncdb, migrate, and collectstatic for discern
+  shell: ${venv_dir}/bin/python {{discern_dir}}/manage.py {{item}} --noinput --settings={{discern_settings}} --pythonpath={{discern_dir}}
+  with_items:
+    - syncdb
+    - migrate
+    - collectstatic
+
+#Have this separate from the other three because it doesn't take the noinput flag
+- name: django update_index for discern
+  shell: ${venv_dir}/bin/python {{discern_dir}}/manage.py update_index --settings={{discern_settings}} --pythonpath={{discern_dir}}
+
+- name: create nginx directory and set perms
+  file: path=/etc/nginx/sites-available owner=root group=edx mode=2775 state=directory
+
+#Install nginx sites available
+#remove default link, render template, link template to sites-enabled to activate
+- name: Removing default nginx config
+  file: path=/etc/nginx/sites-enabled/default state=absent
+  notify: restart nginx
+
+- name: render nginx sites available
+  template: src=nginx-discern.j2 dest=/etc/nginx/sites-available/{{ site_name }}
+  notify: restart nginx
+
+- name: Creating nginx config link {{ site_name }}
+  file: src=/etc/nginx/sites-available/{{ site_name }} dest=/etc/nginx/sites-enabled/{{ site_name }} state=link owner=root group=root
+  notify: restart nginx
\ No newline at end of file

playbooks/roles/discern/templates/auth.json.j2

+{{ auth_config | to_nice_json }}
\ No newline at end of file

playbooks/roles/discern/templates/celery.conf.j2

+#Celery task for ml api
+
+description "Celery ML api"
+author "Vik Paruchuri <vik@edx.org>"
+
+start on runlevel [2345]
+stop on runlevel [!2345]
+
+respawn
+respawn limit 3 30
+
+env DJANGO_SETTINGS_MODULE={{discern_settings}}
+
+chdir {{discern_dir}}
+setuid {{discern_user}}
+
+exec {{venv_dir}}/bin/python {{discern_dir}}/manage.py celeryd --loglevel=info --settings={{discern_settings}} --pythonpath={{discern_dir}} -B --autoscale={{ ansible_processor_cores * 2 }},1

playbooks/roles/discern/templates/discern.conf.j2

+# gunicorn
+
+description "ML API Server"
+author "Vik Paruchuri <vik@edx.org>"
+
+start on runlevel [2345]
+stop on runlevel [!2345]
+
+respawn
+respawn limit 3 30
+
+env PID=/var/run/gunicorn/discern.pid
+env WORKERS={{ ansible_processor_cores * 2 }}
+env PORT={{ gunicorn_port }}
+env LANG=en_US.UTF-8
+env DJANGO_SETTINGS_MODULE={{discern_settings}}
+
+chdir {{discern_dir}}
+setuid {{discern_user}}
+
+exec {{venv_dir}}/bin/gunicorn --preload -b 127.0.0.1:$PORT -w $WORKERS --timeout=30 --pythonpath={{discern_dir}} discern.wsgi
\ No newline at end of file

playbooks/roles/discern/templates/env.json.j2

+{{ env_config | to_nice_json }}
\ No newline at end of file

playbooks/roles/discern/templates/nginx-discern.j2

+server {
+    listen {{nginx_listen_port}};
+    server_name localhost;
+
+    set $my_host $http_host;
+    if ($host ~ "\d+\.\d+\.\d+\.\d+") {
+        set $my_host "127.0.0.1";
+    }
+
+    # https://docs.djangoproject.com/en/dev/howto/static-files/#serving-static-files-in-production
+    location /static/ { # STATIC_URL
+        alias {{discern_dir}}/staticfiles/;
+        expires 1m;
+	autoindex on;
+    }
+
+    location /media/ { # MEDIA_URL
+        alias /home/www/myhostname/static/; # MEDIA_ROOT
+        expires 30d;
+    }
+
+    location / {
+        proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto;
+        proxy_set_header X-Forwarded-Port $http_x_forwarded_port;
+        proxy_set_header X-Forwarded-For $http_x_forwarded_for;
+        proxy_set_header Host $my_host;
+
+        proxy_pass http://127.0.0.1:7999;
+    }
+}
+

playbooks/roles/discern/vars/main.yml

+discern_source_repo: git@github.com:edx/discern.git
+ease_source_repo: git@github.com:edx/ease.git
+ease_dir: $app_base_dir/ease
+discern_dir: $app_base_dir/discern
+discern_settings: discern.aws
+nltk_data_dir: /usr/share/nltk_data
+ease_branch: master
+discern_branch: dev
+nginx_listen_port: 80
+gunicorn_port: 7999
+discern_user: discern
+site_name: discern
\ No newline at end of file

playbooks/roles/lms/tasks/main.yml

@@ -28,6 +28,15 @@
   - lms-env
   - update
 
+- name: Change owner on staticfiles
+  file: path={{ app_base_dir }}/staticfiles state=directory owner=www-data group=adm
+  tags:
+  - cms
+  - lms
+  - lms-env
+  - update
+
+
 - name: Create lms log target directory
   file: path={{log_base_dir}}/lms state=directory owner=syslog group=adm mode=2770
   tags:

playbooks/roles/nginx/tasks/main.yml

@@ -15,7 +15,7 @@
 - include: nginx_site.yml state={{nginx_cfg.sites_enabled.basic_auth}} site_name=basic-auth
 
 - name: Write out default htpasswd file
-  copy: content={{ nginx_cfg.htpasswd }} dest=/etc/nginx/nginx.htpasswd
+  copy: content={{ nginx_cfg.htpasswd }} dest=/etc/nginx/nginx.htpasswd owner=www-data group=www-data mode=0600
   tags:
   - nginx
   - update

playbooks/secure_example/vars/edxapp_ref_vars.yml

@@ -114,14 +114,10 @@ lms_env_config:  &lms_env
   'COMMENTS_SERVICE_KEY':  'hidden-prod'
 
 cms_auth_config:
-  'CONTENTSTORE':
-    'ENGINE':  'xmodule.contentstore.mongo.MongoContentStore'
-  'MODULESTORE':
-    'default':
-      'ENGINE':  'xmodule.modulestore.mongo.MongoModuleStore'
   'AWS_ACCESS_KEY_ID':  'hidden-prod'
   'AWS_SECRET_ACCESS_KEY':  'hidden-prod'
   'CONTENTSTORE':
+    'ENGINE':  'xmodule.contentstore.mongo.MongoContentStore'
     'OPTIONS':
       'db':  'hidden-prod'
       'host': [ 'hidden-prod',  'hidden-prod']