Merge remote-tracking branch 'origin/master' into vik/ml-api

Conflicts:
	playbooks/roles/common/tasks/main.yml

README.md

@@ -48,7 +48,7 @@ As a general policy we want to protect the following data:
 * Hostnames
 * Passwords, api keys
 
-The folowing yml files and examples serve as templates that should be overridden with your own
+The following yml files and examples serve as templates that should be overridden with your own
 environment specific configuration:
 
 * vars in `secure_example/vars` 
@@ -75,7 +75,7 @@ secure_example/
 └── vars
 ```
 
-The default `secure\_dir` is set in `group\_vars/all` and can be overridden by
+The default `secure_dir` is set in `group_vars/all` and can be overridden by
 adding another file in group_vars that corresponds to a deploy group name.
 
 
@@ -176,7 +176,7 @@ playbooks
 * _stack_name_: example: `EdxAppCustom`. Name of the stack, must not contain
   underscores or cloudformation will complain. Must be an unused name or
   otherwise the existing stack will update.
-* _group_name_: example: `edxapp\_stage`. The group name should correspond to
+* _group_name_: example: `edxapp_stage`. The group name should correspond to
   one of the yml files in the `playbooks/`. Used for grouping hosts.
 
 While this is running you see the cloudformation events in the AWS console as
@@ -238,7 +238,7 @@ any further than that, so to support multiple VPC's you'd have to get
 creative with subnets, for example ip-10-1 and ip-10-2...
 
 Test this by typing `ssh ip-10-0-10-1.us-west-1.compute.internal`, 
-(of coruse using a hostname exists in your environment.)  If things 
+(of course using a hostname exists in your environment.)  If things 
 are configured correctly you will ssh to 10.0.10.1, jumping 
 transparently via your basion host.
 

playbooks/edxapp_custom.yml

 - hosts: tag_Group_edxapp_custom
+  sudo: True
   vars_files:
   # using conditional loading to override defaults for site-specific installs
     - "{{ secure_dir }}/vars/edxapp_stage_vars.yml"
@@ -8,4 +9,7 @@
   roles:
     - common
     - nginx
+    - gunicorn
     - lms
+    - ruby
+    - npm

playbooks/edxapp_prod.yml

 - hosts: tag_Group_edxapp_prod
+  sudo: True
   vars_files:
     - "{{ secure_dir }}/vars/edxapp_prod_vars.yml"
     - "{{ secure_dir }}/vars/users.yml"
@@ -9,3 +10,6 @@
     - gunicorn
     - lms
     - ruby
+    - npm
+    # run this role last
+    - in_production

playbooks/edxapp_stage.yml

 - hosts: tag_Group_edxapp_stage
+  sudo: True
   vars_files:
     - "{{ secure_dir }}/vars/edxapp_stage_vars.yml"
     - "{{ secure_dir }}/vars/users.yml"
@@ -9,3 +10,4 @@
     - gunicorn
     - lms
     - ruby
+    - npm

playbooks/roles/cms/tasks/main.yml

+# requires:
+#  - group_vars/all
+#  - common/tasks/main.yml
+#  - nginx/tasks/main.yml
+---
+- name: create cms application config
+  template: src=env.json.j2 dest=$app_base_dir/cms.env.json
+  tags:
+  - cms-env
+  - cms
+
+- name: create cms auth file
+  template: src=auth.json.j2 dest=$app_base_dir/cms.auth.json
+  tags:
+  - cms-env
+  - cms
+
+- include: ../../nginx/tasks/nginx_site.yml state=link site_name=cms
+  tags:
+  - cms
+  - cms-env
+- include: ../../nginx/tasks/nginx_site.yml state=link site_name=cms-backend
+  tags:
+  - cms
+  - cms-env

playbooks/roles/cms/templates/auth.json.j2

+{# {% do auth_config.update( { < override some hash keys > } )  %} #}
+{{ cms_auth_config | to_nice_json }}

playbooks/roles/cms/templates/env.json.j2

+{{ cms_env_config | to_nice_json }}

playbooks/roles/cms/vars/main.yml

+# variables common to the cms role, automatically loaded
+# when the role is included
+---
+cms_auth_config:
+  'CONTENTSTORE':
+    'ENGINE':  'xmodule.contentstore.mongo.MongoContentStore'
+  'MODULESTORE':
+    'default':
+      'ENGINE':  'xmodule.modulestore.mongo.MongoModuleStore'
+ 
+cms_env_config:
+  'CACHES':
+    'default':
+      'BACKEND':  'django.core.cache.backends.memcached.MemcachedCache'
+      'KEY_FUNCTION':  'util.memcache.safe_key'
+      'KEY_PREFIX': 'cms.edx.org'
+      'LOCATION': [ "deploycache-large.foo-bar.amazonaws.com:11211" ]
+    'mongo_metadata_inheritance':
+      'BACKEND':  'django.core.cache.backends.memcached.MemcachedCache'
+      'KEY_FUNCTION':  'util.memcache.safe_key'
+      'TIMEOUT': 300
+      'KEY_PREFIX': 'cms.edx.org'
+      'LOCATION': [ "deploycache-large.foo-bar.amazonaws.com:11211" ]
+    'staticfiles':
+      'BACKEND':  'django.core.cache.backends.memcached.MemcachedCache'
+      'KEY_FUNCTION':  'util.memcache.safe_key'
+      'KEY_PREFIX': 'cms.edx.org'
+      'LOCATION': [ "deploycache-large.foo-bar.amazonaws.com:11211" ]
+  'LOG_DIR': '/mnt/logs/edx'
+  'LOGGING_ENV': 'cms-dev'
+  'SITE_NAME': 'studio.cms-dev.m.edx.org'
+  'SYSLOG_SERVER': 'syslog.a.m.i4x.org'
+  'LMS_BASE': 'cms-dev.m.edx.org'
+  'SESSION_COOKIE_DOMAIN': '.cms-dev.m.edx.org'
+

playbooks/roles/common/tasks/create_users.yml

 ---
 - name: Create 'edx' users group
   group: name=edx state=present
-  sudo: True
   tags:
   - users
   - admin_users
@@ -9,14 +8,12 @@
   # This is a temporary measure for initial configuration; after the last 
   # play is run and we've got a good set of users, ubuntu should no longer be used
   user: name=ubuntu append=yes groups="edx"
-  sudo: True
   tags:
   - users
   - admin_users
 - name: Creating admin users
   # Admin users, by definition, should be able to sudo w/ password, and read adm-only files
   user: name={{ item.user }} append=yes groups={{ "adm,edx,"+",".join(item.groups) }} shell=/bin/bash
-  sudo: True
   with_items: admin_users
   when: admin_users is defined
   tags:
@@ -24,21 +21,18 @@
   - admin_users
 - name: Copying ssh keys for admin users
   authorized_key: user={{ item.user }} key="{{lookup('file', item.path)}}"
-  sudo: True
   with_items: admin_keys
   tags:
   - users
   - admin_users
 - name: Creating env users
   user: name={{ item.user }} groups={{ ",".join(item.groups) }} shell=/bin/bash
-  sudo: True
   with_items: env_users
   when: env_users is defined
   tags:
   - users
 - name: Copying ssh keys for env users
   authorized_key: user={{ item.user }} key="{{lookup('file', item.path)}}"
-  sudo: True
   with_items: env_keys
   when: env_keys is defined
   tags:

playbooks/roles/common/tasks/create_venv.yml

@@ -2,22 +2,18 @@
 # create the 'edx' virtual environment in /opt so that roles can populate it
 - name: easy_install pip
   easy_install: name=pip
-  sudo: True
   tags:
   - venv_base
 - name: pip install virtualenv
   pip: name=virtualenv state=latest
-  sudo: True
   tags:
   - venv_base
 - name: pip install virtualenvwrapper
   pip: name=virtualenvwrapper state=latest
-  sudo: True
   tags:
   - venv_base
 - name: create edx virtualenv directory
-  file: path=$venv_dir owner=ubuntu group=edx mode=2775 state=directory
-  sudo: True
+  file: path=$venv_dir owner=ubuntu group=adm mode=2775 state=directory
   tags:
   - venv_base
 - name: create the edx virtualenv directory initial contents

playbooks/roles/common/tasks/main.yml

 ---
 - include: create_users.yml
 - name: Create application root
-  file: path=$app_base_dir state=directory owner=root group=root
-  sudo: True
+  # In the future consider making group edx r/t adm
+  file: path=$app_base_dir state=directory owner=root group=adm mode=2775 
+  tags:
+  - pre_install
 - name: Create log directory
-  file: path=/mnt/logs state=directory
-  sudo: True
+  file: path=/mnt/logs state=directory mode=2770 group=adm owner=root
+  tags:
+  - pre_install
+- name: Create aliases to the log directory
+  file: state=link src=/mnt/logs path=$app_base_dir/log
+  tags:
+  - pre_install
+- name: Touch the edx log file into place 
+  command: touch -a /mnt/logs/edx.log
+  tags:
+  - pre_install
 - name: Update apt cache
   apt: update_cache=yes
-  sudo: True
+  tags:
+  - pre_install
 
 - include: create_venv.yml
 
 - name: Install role-independent useful system packages
   apt: pkg={{item}} install_recommends=yes state=present
   with_items:
+  - ack-grep
+  - lynx-cur
   - mosh
-  - tmux
-  sudo: True
+  - most
+  - screen
\ No newline at end of file
+  tags:
+  - pre_install
\ No newline at end of file

playbooks/roles/in_production/tasks/main.yml

+# requires:
+#  - group_vars/all
+#  - common/tasks/main.yml
+#  - nginx/tasks/main.yml
+#  - lms/tasks/main.yml
+#  - ruby/tasks/main.yml
+#  - npm/tasks/main.yml
+---
+- name: Make sure LMS is running
+  service: name=lms state=started
+  tags:
+  - production
+
+- name: Disable HTTP Basic Auth on site
+  file: path=/etc/nginx/sites-enabled/basic-auth state=absent
+  notify: restart nginx
+  tags:
+  - production

playbooks/roles/lms/tasks/main.yml

@@ -5,14 +5,12 @@
 ---
 - name: create lms application config
   template: src=env.json.j2 dest=$app_base_dir/lms.env.json
-  sudo: True
   tags:
   - lms
   - lms-env
 
 - name: create lms auth file
   template: src=auth.json.j2 dest=$app_base_dir/lms.auth.json
-  sudo: True
   tags:
   - lms
   - lms-env
@@ -27,29 +25,20 @@
 # ugly relative pathing here
 
 - name: install read-only ssh key for mitx repo (private)
-  copy: src=../../../{{ secure_dir }}/files/git-identity dest=/etc/git-identity force=yes owner=root group=adm mode=640
-  sudo: True
+  copy: src=../../../{{ secure_dir }}/files/git-identity dest=/etc/git-identity force=yes owner=ubuntu group=adm mode=600
   tags:
   - lms
   - cms
 
 - name: upload ssh script
   copy: src=git_ssh.sh dest=/tmp/git_ssh.sh force=yes owner=root group=adm mode=750
-  sudo: True
   tags:
   - lms
   - cms
 
 # Check out mitx repo to $app_base_dir
-- name: set permissions on $app_base_dir sgid for adm
-  file: path=$app_base_dir owner=root group=adm mode=2775 state=directory
-  sudo: True
-  tags:
-  - lms
-  - cms
 - name: install git and its recommends
   apt: pkg=git state=present install_recommends=yes 
-  sudo: True
   tags:
   - lms
   - cms
@@ -92,7 +81,6 @@
 - name: install a bunch of system packages on which LMS and CMS rely
   apt: pkg={{item}} state=present
   with_items: lms_debian_pkgs
-  sudo: True
   tags:
   - lms
   - cms

playbooks/roles/lms/templates/auth.json.j2

-{{ auth_config | to_nice_json }}
+{{ lms_auth_config | to_nice_json }}

playbooks/roles/lms/templates/env.json.j2

-{{ env_config | to_nice_json }}
+{{ lms_env_config | to_nice_json }}

playbooks/roles/lms/vars/main.yml

 # variables common to the lms role, automatically loaded
 # when the role is included
 ---
-auth_config:
+lms_auth_config:
   'CONTENTSTORE':
     'ENGINE':  'xmodule.contentstore.mongo.MongoContentStore'
   'MODULESTORE':
     'default':
       'ENGINE':  'xmodule.modulestore.mongo.MongoModuleStore'
  
-env_config:
+lms_env_config:
   'CACHES':
     'default':
       'BACKEND':  'django.core.cache.backends.memcached.MemcachedCache'
@@ -89,7 +89,6 @@ lms_debian_pkgs:
   - mongodb
   - mongodb-clients
   - mysql-client
-  - nodejs
   - ntp
   - openjdk-7-jdk
   - openjdk-7-jre
@@ -97,11 +96,6 @@ lms_debian_pkgs:
   - perl
   - pkg-config
   - postfix
-  - puppet
-  - puppet-common
-  - puppet-lint
-  - puppetmaster
-  - puppetmaster-common
   - pylint
   - python-boto
   - python-coverage-test-runner
@@ -119,7 +113,6 @@ lms_debian_pkgs:
   - sqlite3
   - super
   - vagrant
-  - vim-puppet
   - yui-compressor
   - zip
   - zlib1g-dev

playbooks/roles/nginx/tasks/main.yml

@@ -2,18 +2,20 @@
 #   - common/tasks/main.yml
 ---
 - name: Install nginx
-  sudo: True
   apt: pkg=nginx state={{ pkgs.nginx.state }}
   notify: restart nginx
   tags:
   - nginx
 # removing default link
-- name: Removing default nginx config
-  sudo: True
-  file: path=/etc/nginx/sites-available/default state=absent
+- name: Removing default nginx config (enabled)
+  file: path=/etc/nginx/sites-enabled/default state=absent
   notify: restart nginx
   tags:
   - nginx
+- name: Removing default nginx config (available)
+  file: path=/etc/nginx/sites-available/default state=absent
+  tags:
+  - nginx
 # Standard configuration that is common across all roles
 # Default values for these variables are set in group_vars/all
 # Note: remove spaces in {{..}}, otherwise you will get a template parsing error.
@@ -21,11 +23,9 @@
 - include: nginx_site.yml state={{nginx_cfg.sites_enabled.basic_auth}} site_name=basic-auth
 # Default htpassword file, required for basic auth
 - copy: content={{ nginx_cfg.htpasswd }} dest=/etc/nginx/nginx.htpasswd
-  sudo: True
   tags:
   - nginx
 - name: Ensuring that nginx is running
-  sudo: True
   service: name=nginx state=started
   tags:
   - nginx

playbooks/roles/nginx/tasks/nginx_site.yml

 # Requires nginx package
 ---
 - name: Copying nginx config {{ site_name }}
-  sudo: True
   template: src={{ item }} dest=/etc/nginx/sites-available/{{ site_name }}
   first_available_file:
   - "{{ local_dir }}/nginx/templates/{{ site_name }}.j2"
@@ -14,7 +13,6 @@
   - nginx-env
 
 - name: Creating nginx config link {{ site_name }}
-  sudo: True
   file: src=/etc/nginx/sites-available/{{ site_name }} dest=/etc/nginx/sites-enabled/{{ site_name }} state={{ state }} owner=root group=root
   notify: restart nginx
   tags:

playbooks/roles/nginx/templates/cms-backend.j2

+upstream cms-backend {
+    # For a TCP configuration:
+    server 127.0.0.1:8010 fail_timeout=0;
+}

playbooks/roles/nginx/templates/cms.j2

+server {
+  # CMS configuration file for nginx, templated by ansible
+      
+  listen 80;
+
+  server_name trace-cms.*
+              studio.lms-dev.m.edx.org;
+
+  #
+  # Send error response when request host isn't under our control
+  # We will no longer respond to proxy attempts like this with 
+  # anything.
+  # curl -i -A '' -x http://www.edx.org:80 --proxy-negotiate -U u:p -u u:p http://chat.sdtz.com
+  #
+
+  set $reject 'no';
+
+  if ($host !~* (edx.org|edxonline.org)$ ) {
+    set $reject 'yes';
+  }
+
+  if ($request_uri ~ ^(/heartbeat)$) {
+    set $reject 'no';
+  }
+
+  if ( $reject = 'yes' ) {
+    return 444;
+  }
+
+  
+  # CS184 requires uploads of up to 4MB for submitting screenshots. 
+  # CMS requires larger value for course assest, values provided 
+  # via hiera.
+  client_max_body_size 100M;
+  
+  rewrite ^(.*)/favicon.ico$ /static/images/favicon.ico last;
+
+  location @proxy_to_cms_app {
+    proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto;
+    proxy_set_header X-Forwarded-Port $http_x_forwarded_port;
+    proxy_set_header X-Forwarded-For $http_x_forwarded_for;
+    proxy_set_header Host $http_host;
+
+    proxy_redirect off;
+    proxy_pass http://cms-backend;
+  }
+
+  location / {
+    try_files $uri @proxy_to_cms_app;
+  }
+
+  # No basic auth security on the github_service_hook url, so that github can use it for cms
+  location /github_service_hook {
+    try_files $uri @proxy_to_cms_app;
+  }
+
+  # No basic auth security on the heartbeat url, so that ELB can use it
+  location /heartbeat {
+    try_files $uri @proxy_to_cms_app;
+  }
+
+  # Check security on this
+  location ~ /static/(?P<file>.*) {
+    auth_basic            "Restricted";
+    auth_basic_user_file  /etc/nginx/nginx.htpasswd;
+    root {{app_base_dir}};
+
+    # return a 403 for static files that shouldn't be
+    # in the staticfiles directory
+    location ~ ^/static/(?:.*)(?:\.xml|\.json|README.TXT) {
+        return 403;
+    }
+    # Set django-pipelined files to maximum cache time
+    location ~ "/static/(?P<collected>.*\.[0-9a-f]{12}\..*)" {
+        expires max;
+
+        # Without this try_files, files that have been run through
+        # django-pipeline return 404s
+        try_files /staticfiles/$collected /course_static/$collected =404;
+    }
+
+    # Expire other static files immediately (there should be very few / none of these)
+    expires epoch;
+  }
+
+  # Forward to HTTPS if we're an HTTP request...
+  if ($http_x_forwarded_proto = "http") {
+    set $do_redirect "true";
+  }
+
+  # Run our actual redirect...
+  if ($do_redirect = "true") {
+    rewrite ^ https://$host$request_uri? permanent;
+  }
+
+}

playbooks/roles/npm/tasks/main.yml

@@ -3,7 +3,6 @@
 #   - ruby/tasks/main.yml
 ---
 - name: Install nodejs, and by extension npm
-  sudo: True
   apt: pkg=nodejs state=present install_recommends=no
   tags:
   - npm

playbooks/roles/ruby/tasks/main.yml

@@ -2,16 +2,13 @@
 #cribbed from https://github.com/mmoya/ansible-playbooks/blob/master/rbenv/main.yml
 - name: Create 'www' user (replicating historical environment)
   user: name=www state=present
-  sudo: True
   tags:
   - ruby
 - name: Create ruby base
-  sudo: True
   file: path=$ruby_base state=directory owner=www group=www
   tags:
   - ruby
 - name: rbenv | install build depends
-  sudo: true
   apt: pkg=$item state=present install_recommends=no
   with_items:
     - build-essential
@@ -27,25 +24,21 @@
   - ruby
 
 - name: rbenv | update rbenv repo
-  sudo: true
   git: repo=git://github.com/sstephenson/rbenv.git dest=$rbenv_root version=v0.4.0
   tags:
   - ruby
 
 - name: rbenv | add rbenv to path
-  sudo: true
   file: path=/usr/local/bin/rbenv src=${rbenv_root}/bin/rbenv state=link
   tags:
   - ruby
 
 - name: rbenv | add rbenv initialization to profile
-  sudo: true
   template: src=rbenv.sh.j2 dest=/etc/profile.d/rbenv.sh owner=root group=root mode=0755
   tags:
   - ruby
 
 - name: rbenv | check ruby-build installed
-  sudo: true
   command: test -x /usr/local/bin/ruby-build
   register: rbuild_present
   ignore_errors: yes
@@ -66,7 +59,6 @@
   - ruby
 
 - name: rbenv | install ruby-build
-  sudo: true
   command: ./install.sh chdir=${tempdir.stdout}/ruby-build
   when_failed: $rbuild_present
   tags:
@@ -87,28 +79,24 @@
 
 - name: rbenv | install ruby $ruby_version
   shell: RBENV_ROOT=${rbenv_root} rbenv install $ruby_version
-  sudo: true
   when_failed: $ruby_installed
   tags:
   - ruby
 
 - name: rbenv | set global ruby $ruby_version
   shell: RBENV_ROOT=${rbenv_root} rbenv global $ruby_version
-  sudo: true
   when_failed: $ruby_installed
   tags:
   - ruby
 
 - name: rbenv | rehash
   shell: RBENV_ROOT=${rbenv_root} rbenv rehash
-  sudo: true
   when_failed: $ruby_installed
   tags:
   - ruby
 
 - name: gem | gem install bundler
   shell: RBENV_ROOT=${rbenv_root} GEM_HOME=${gem_home} ${rbenv_root}/shims/gem install bundle chdir=${app_base_dir}/mitx
-  sudo: true
   tags:
   - ruby
 

playbooks/secure_example/vars/edxapp_custom_vars.yml

 ---
-auth_config:
+lms_auth_config:
   'DATABASES':
     'default': { 'ENGINE':  'custom',
       'HOST':  'custom',  'NAME':  'custom',

playbooks/secure_example/vars/edxapp_prod_vars.yml

 ---
-auth_config:
+lms_auth_config:
   'ANALYTICS_API_KEY':  'hidden-prod'
   'AWS_ACCESS_KEY_ID':  'hidden-prod'
   'AWS_SECRET_ACCESS_KEY':  'hidden-prod'
@@ -39,7 +39,7 @@ auth_config:
       'username':  'hidden-prod'}
     'url':  'hidden-prod'
  
-env_config:
+lms_env_config:
   'SYSLOG_SERVER':  'hidden-prod'
   'SITE_NAME':  'hidden-prod'
   'LOG_DIR':  'hidden-prod'

playbooks/secure_example/vars/edxapp_stage_vars.yml

 ---
-auth_config:
+lms_auth_config:
   'ANALYTICS_API_KEY':  'hidden-stage'
   'AWS_ACCESS_KEY_ID':  'hidden-stage'
   'AWS_SECRET_ACCESS_KEY':  'hidden-stage'
@@ -39,7 +39,7 @@ auth_config:
       'username':  'hidden-stage'}
     'url':  'hidden-stage'
  
-env_config:
+lms_env_config:
   'SYSLOG_SERVER':  'hidden-stage'
   'SITE_NAME':  'hidden-stage'
   'LOG_DIR':  'hidden-stage'

playbooks/vars/lms_vars.yml

 ---
-auth_config:
+lms_auth_config:
   'CONTENTSTORE':
     'ENGINE':  'xmodule.contentstore.mongo.MongoContentStore'
   'MODULESTORE':
     'default':
       'ENGINE':  'xmodule.modulestore.mongo.MongoModuleStore'
  
-env_config:
+lms_env_config:
   'CACHES':
     'default':
       'BACKEND':  'django.core.cache.backends.memcached.MemcachedCache'