consolidating lms/cms/lms-preview into edxapp role

Creates edxapp upstart job and combines lms, lms-preview. cms
into the edxapp role.

playbooks/roles/cms/vars/main.yml

-# variables common to the cms role, automatically loaded
-# when the role is included
----
-cms_auth_config: {}
-cms_env_config: {}

playbooks/roles/cms/tasks/main.yml → playbooks/roles/edxapp/tasks/cms.yml

@@ -4,14 +4,14 @@
 #  - nginx/tasks/main.yml
 ---
 - name: create cms application config
-  template: src=env.json.j2 dest=$app_base_dir/cms.env.json mode=640 owner=www-data group=adm
+  template: src=cms.env.json.j2 dest=$app_base_dir/cms.env.json mode=640 owner=www-data group=adm
   tags:
   - cms-env
   - cms
   - update
 
 - name: create cms auth file
-  template: src=auth.json.j2 dest=$app_base_dir/cms.auth.json mode=640 owner=www-data group=adm
+  template: src=cms.auth.json.j2 dest=$app_base_dir/cms.auth.json mode=640 owner=www-data group=adm
   tags:
   - cms-env
   - cms

playbooks/roles/lms-preview/tasks/main.yml → playbooks/roles/edxapp/tasks/lms-preview.yml

@@ -4,13 +4,13 @@
 #  - nginx/tasks/main.yml
 ---
 - name: create lms application config
-  template: src=env.json.j2 dest=$app_base_dir/lms-preview.env.json mode=640 owner=www-data group=adm
+  template: src=lms-preview.env.json.j2 dest=$app_base_dir/lms-preview.env.json mode=640 owner=www-data group=adm
   tags:
   - lms-preview
   - lms-preview-env
 
 - name: create lms auth file
-  template: src=auth.json.j2 dest=$app_base_dir/lms-preview.auth.json mode=640 owner=www-data group=adm
+  template: src=lms-preview.auth.json.j2 dest=$app_base_dir/lms-preview.auth.json mode=640 owner=www-data group=adm
   tags:
   - lms-preview
   - lms-preview-env

playbooks/roles/edxapp/tasks/lms-xml.yml

+# requires:
+#  - group_vars/all
+#  - common/tasks/main.yml
+#  - nginx/tasks/main.yml
+---
+- name: create lms-xml application config
+  template: src=lms-xml.env.json.j2 dest=$app_base_dir/lms-xml.env.json mode=640 owner=www-data group=adm
+  tags:
+  - cms-env
+  - cms
+  - update
+
+- name: create lms-xml auth file
+  template: src=lms-xml.auth.json.j2 dest=$app_base_dir/lms-xml.auth.json mode=640 owner=www-data group=adm
+  tags:
+  - cms-env
+  - cms
+  - update
+
+- include: ../../nginx/tasks/nginx_site.yml state=link site_name=lms-xml
+  
+- include: ../../nginx/tasks/nginx_site.yml state=link site_name=lms-xml-backend
+
+# Creates upstart file
+- include: ../../gunicorn/tasks/upstart.yml service_variant=lms-xml

playbooks/roles/edxapp/tasks/lms.yml

+---
+- name: create lms application config
+  template: src=lms.env.json.j2 dest=$app_base_dir/lms.env.json mode=640 owner=www-data group=adm
+  tags:
+  - lms
+  - lms-env
+  - update
+
+- name: create lms auth file
+  template: src=lms.auth.json.j2 dest=$app_base_dir/lms.auth.json mode=640 owner=www-data group=adm
+  tags:
+  - lms
+  - lms-env
+  - update
+
+- include: ../../nginx/tasks/nginx_site.yml state=link site_name=lms
+- include: ../../nginx/tasks/nginx_site.yml state=link site_name=lms-backend
+# Creates LMS upstart file
+- include: ../../gunicorn/tasks/upstart.yml service_variant=lms

playbooks/roles/lms/tasks/main.yml → playbooks/roles/edxapp/tasks/main.yml

@@ -3,23 +3,6 @@
 #  - common/tasks/main.yml
 #  - nginx/tasks/main.yml
 ---
-- name: create lms application config
-  template: src=env.json.j2 dest=$app_base_dir/lms.env.json mode=640 owner=www-data group=adm
-  tags:
-  - lms
-  - lms-env
-  - update
-
-- name: create lms auth file
-  template: src=auth.json.j2 dest=$app_base_dir/lms.auth.json mode=640 owner=www-data group=adm
-  tags:
-  - lms
-  - lms-env
-  - update
-
-- include: ../../nginx/tasks/nginx_site.yml state=link site_name=lms
-- include: ../../nginx/tasks/nginx_site.yml state=link site_name=lms-backend
-
 - name: Change permissions on datadir
   file: path={{ app_base_dir }}/data state=directory owner=www-data group=www-data
   tags:
@@ -90,43 +73,6 @@
   - cms
   - install
 
-
-## Install the debian package requirements system-wide
-- name: store remote apt_repos list for ansible use
-  command: cat {{ apt_sources_file }}
-  register: apt_repos_list
-  tags:
-  - lms
-  - cms
-  - install
-
-- name: add apt_repos to the remote hosts
-  apt_repository: repo="$item"
-  with_items: "{{apt_repos_list.stdout.split()}}"
-  register: apt_repos_list_repo_adds
-  sudo: True
-  tags:
-  - lms
-  - cms
-  - install
-
-- name: update apt cache (if necessary)
-  apt: update_cache=yes
-  sudo: True
-  only_if: "{{apt_repos_list_repo_adds.changed}}"
-  tags:
-  - lms
-  - cms
-  - install
-
-- name: store remote apt_packages list for ansible use
-  command: cat {{ apt_packages_file }}
-  register: apt_packages_list
-  tags:
-  - lms
-  - cms
-  - install
-
 - name: install a bunch of system packages on which LMS and CMS rely
   apt: pkg={{item}} state=present
   with_items: lms_debian_pkgs
@@ -147,10 +93,10 @@
   - cms
   - install
   
-  
+ 
 # Install the python post requirements into {{ venv_dir }}
 - name : install python post-requirements
-  pip: requirements="{{ post_requirements_file }}" virtualenv="{{ venv_dir }}" state=present
+  pip: requirements="{{post_requirements_file}}" virtualenv="{{venv_dir}}" state=present
   tags:
   - lms
   - cms
@@ -170,7 +116,13 @@
   - lms
   - cms
   - install
-  
 
-# Creates LMS upstart file
-- include: ../../gunicorn/tasks/upstart.yml service_variant=lms
+- include: lms.yml
+  when: lms_env_config != False
+- include: lms-xml.yml
+  when: lms_xml_env_config != False
+- include: cms.yml
+  when: cms_env_config != False
+- include: lms-preview.yml
+  when: lms_preview_env_config != False
+ 

playbooks/roles/edxapp/templates/lms-xml.auth.json.j2

+{{ lms_xml_auth_config | to_nice_json }}

playbooks/roles/edxapp/templates/lms-xml.env.json.j2

+{{ lms_xml_env_config | to_nice_json }}

playbooks/roles/lms/vars/main.yml → playbooks/roles/edxapp/vars/main.yml

 # variables common to the lms role, automatically loaded
 # when the role is included
 ---
-lms_auth_config: {}
-lms_env_config: {}
+lms_auth_config: false
+lms_env_config: false
+lms_preview_auth_config: false
+lms_preview_env_config: false
+cms_auth_config: false
+cms_env_config: false
+lms_xml_auth_config: false
+lms_xml_env_config: false
+
 lms_source_repo: git@github.com:edx/edx-platform.git
 lms_version: 'HEAD'
-
-apt_sources_file:  "{{ platform_code_dir }}/requirements/system/{{os_name}}/apt-repos.txt"
-apt_packages_file: "{{ platform_code_dir }}/requirements/system/{{os_name}}/apt-packages.txt"
-
 local_requirements_file:  "{{ platform_code_dir }}/requirements/edx/local.txt"
 post_requirements_file:   "{{ platform_code_dir }}/requirements/edx/post.txt"
 base_requirements_file:   "{{ platform_code_dir }}/requirements/edx/base.txt"
@@ -59,6 +62,7 @@ lms_debian_pkgs:
   - mongodb
   - mongodb-clients
   - mysql-client
+  - npm
   - ntp
   - openjdk-7-jdk
   - openjdk-7-jre

playbooks/roles/gunicorn/tasks/main.yml

@@ -4,3 +4,10 @@
   tags:
   - gunicorn
   - install
+- name: creating edxapp upstart script
+  sudo: True
+  template: src=edxapp.conf.j2 dest=/etc/init/edxapp.conf owner=root group=root
+  tags:
+  - upstart
+  - gunicorn
+  - update

playbooks/roles/gunicorn/tasks/upstart.yml

@@ -6,7 +6,6 @@
   - "{{ local_dir }}/gunicorn/templates/{{ service_variant }}.conf.j2"
   # seems like paths in first_available_file must be relative to the playbooks dir
   - "roles/gunicorn/templates/{{ service_variant }}.conf.j2"
-  notify: restart {{ service_variant }}
   tags:
   - upstart
   - gunicorn

playbooks/roles/gunicorn/templates/edxapp.conf.j2

+#/etc/init/edxapp.conf
+
+description "Starts and stops multiple edX services, e.g., lms, cms, etc., installed in a stacked configuration."
+
+start on runlevel [2345]
+stop on runlevel [!2345]
+
+##
+## Each awaited service is responsible for ensuring that it is ready
+## for service when it returns.
+##
+pre-start script
+
+{% if lms_xml_env_config %}
+if [ -e /etc/init/lms-xml.conf ]; then
+  start wait-for-state WAIT_FOR=lms-xml WAITER=$UPSTART_JOB
+fi
+{% endif %}
+
+{% if lms_env_config %}
+if [ -e /etc/init/lms.conf ]; then
+  start wait-for-state WAIT_FOR=lms WAITER=$UPSTART_JOB
+fi
+{% endif %}
+
+{% if lms_preview_env_config %}
+if [ -e /etc/init/lms-preview.conf ]; then
+  start wait-for-state WAIT_FOR=lms-preview WAITER=$UPSTART_JOB
+fi
+{% endif %}
+
+{% if cms_env_config %}
+if [ -e /etc/init/cms.conf ]; then
+  start wait-for-state WAIT_FOR=cms WAITER=$UPSTART_JOB
+fi
+{% endif %}
+
+end script
+
+script
+  # Noop process for other edX components to take their
+  # marching orders from.  In the edxapp deployment, 
+  # lms, cms, etc. will listen for this process to start
+  # and stop and follow suit.
+  while true
+    do
+      logger -t $0 "edX App Shell Daemon is running..."
+      sleep 600
+    done
+end script
+
+pre-stop script
+
+{% if lms_xml_env_config %}
+if [ -e /etc/init/lms-xml.conf ]; then
+  start wait-for-state WAIT_FOR=lms-xml WAITER=$UPSTART_JOB TARGET_GOAL="stop"
+fi
+{% endif %}
+
+{% if lms_env_config %}
+if [ -e /etc/init/lms.conf ]; then
+  start wait-for-state WAIT_FOR=lms WAITER=$UPSTART_JOB TARGET_GOAL="stop"
+fi
+{% endif %}
+
+{% if lms_preview_env_config %}
+if [ -e /etc/init/lms-preview.conf ]; then
+  start wait-for-state WAIT_FOR=lms-preview WAITER=$UPSTART_JOB TARGET_GOAL="stop"
+fi
+{% endif %}
+
+{% if cms_env_config %}
+if [ -e /etc/init/cms.conf ]; then
+  start wait-for-state WAIT_FOR=cms WAITER=$UPSTART_JOB TARGET_GOAL="stop"
+fi
+{% endif %}
+
+end script
+

playbooks/roles/gunicorn/templates/lms-xml.conf.j2

+# gunicorn
+
+description "gunicorn server"
+author "Calen Pennington <cpennington@mitx.mit.edu>"
+
+start on started edxapp
+stop on stopped edxapp
+
+respawn
+respawn limit 3 30
+
+env PID=/var/tmp/lms-xml.pid
+env NEW_RELIC_CONFIG_FILE=/opt/wwc/newrelic.ini
+env NEWRELIC=/opt/edx/bin/newrelic-admin
+env WORKERS=4
+env PORT=8030
+env LANG=en_US.UTF-8
+env DJANGO_SETTINGS_MODULE=lms.envs.aws
+env SERVICE_VARIANT="lms-xml"
+
+
+chdir /opt/wwc/edx-platform
+setuid www-data
+
+exec $NEWRELIC run-program /opt/edx/bin/gunicorn --preload -b 127.0.0.1:$PORT -w $WORKERS --timeout=300 --pythonpath=/opt/wwc/edx-platform lms.wsgi
+
+post-start script
+  while true
+  do
+    if $(curl -s -i localhost:$PORT/heartbeat | egrep -q '200 OK'); then
+      break;
+    else
+      sleep 1;
+    fi
+  done
+end script
+

playbooks/roles/nginx/templates/lms-xml-backend.j2

+upstream lms-xml-backend {
+    # For a TCP configuration:
+    server 127.0.0.1:8030 fail_timeout=0;
+}
+

playbooks/roles/nginx/templates/lms-xml.j2

+server {
+  # LMS-preview configuration file for nginx, templated by ansible
+
+  listen 80;
+
+  server_name trace-lms-preview.*
+              preview.lms-dev.m.edx.org;
+
+  #
+  # Send error response when request host isn't under our control
+  # We will no longer respond to proxy attempts like this with
+  # anything.
+  # curl -i -A '' -x http://www.edx.org:80 --proxy-negotiate -U u:p -u u:p http://chat.sdtz.com
+  #
+
+  set $reject 'no';
+
+  #if ($host !~* (edx.org|edxonline.org)$ ) {
+  #  set $reject 'yes';
+  #}
+
+  if ($request_uri ~ ^(/heartbeat)$) {
+    set $reject 'no';
+  }
+
+  if ( $reject = 'yes' ) {
+    return 444;
+  }
+  # CS184 requires uploads of up to 4MB for submitting screenshots.
+  # CMS requires larger value for course assest, values provided
+  # via hiera.
+  client_max_body_size 4M;
+
+  rewrite ^(.*)/favicon.ico$ /static/images/favicon.ico last;
+
+  location @proxy_to_lms-preview_app {
+    proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto;
+    proxy_set_header X-Forwarded-Port $http_x_forwarded_port;
+    proxy_set_header X-Forwarded-For $http_x_forwarded_for;
+    proxy_set_header Host $http_host;
+
+    proxy_redirect off;
+    proxy_pass http://lms-preview-backend;
+  }
+
+  location / {
+    try_files $uri @proxy_to_lms-preview_app;
+  }
+
+  # No basic auth security on the github_service_hook url, so that github can use it for cms
+  location /github_service_hook {
+    try_files $uri @proxy_to_lms-preview_app;
+  }
+
+  # No basic auth security on the heartbeat url, so that ELB can use it
+  location /heartbeat {
+    try_files $uri @proxy_to_lms-preview_app;
+  }
+
+  # Check security on this
+  location ~ /static/(?P<file>.*) {
+    root {{app_base_dir}};
+    try_files /staticfiles/$file /course_static/$file =404;
+
+    # return a 403 for static files that shouldn't be
+    # in the staticfiles directory
+    location ~ ^/static/(?:.*)(?:\.xml|\.json|README.TXT) {
+        return 403;
+    }
+    # Set django-pipelined files to maximum cache time
+    location ~ "/static/(?P<collected>.*\.[0-9a-f]{12}\..*)" {
+        expires max;
+        # Without this try_files, files that have been run through
+        # django-pipeline return 404s
+        try_files /staticfiles/$collected /course_static/$collected =404;
+    }
+
+    # Expire other static files immediately (there should be very few / none of these)
+    expires epoch;
+  }
+
+  # Forward to HTTPS if we're an HTTP request...
+  if ($http_x_forwarded_proto = "http") {
+    set $do_redirect "true";
+  }
+
+  # Run our actual redirect...
+  if ($do_redirect = "true") {
+    rewrite ^ https://$host$request_uri? permanent;
+  }
+
+  # Monitoring support for datadog.
+  location /nginx_status {
+    stub_status on;
+    access_log off;
+    allow 127.0.0.1/32;
+    deny all;
+  }
+}