theming support + playbooks/edx-west symlinks + redo edxapp deploy

This commit basically does 3 things:
* Creates symlinks within the playbooks/edx-west directory
  to allow ansible-playbook to be run in that directory
* Adds support for creating theming directory and checking out
  themed git repo into that directory at deploy time.
* Moves around some of the edxapp deploy handlers.
  Also tags them all with - deploy, so that running
  ansible-playbook with --tags=deploy basically just deploys.
  Also splits restart edxapp into stop and start edxapp b/c
  restart wasn't working on Stanford servers.  Could debug
  this later, probably best by looking at the edxapp upstart script.

playbooks/edx-west/edxapp_prod.yml

-- hosts: tag_environment_prod:&tag_function_webserver
+#- hosts: tag_environment_prod:&tag_function_webserver
+- hosts: ~tag_Name_app(11|21)_prod
+#- hosts: tag_environment_prod:&tag_function_webserver
+#- hosts: security_group_edx-prod-EdxappServerSecurityGroup-NSKCQTMZIPQB
   sudo: True
   vars_files:
     - "{{ secure_dir }}/vars/edxapp_prod_vars.yml"
     - "{{ secure_dir }}/vars/users.yml"
     - "{{ secure_dir }}/vars/edxapp_prod_users.yml"
+  vars:
+    secure_dir: '../../../configuration-secure/ansible'
+    # this indicates the path to site-specific (with precedence)
+    # things like nginx template files
+    local_dir:  '../../../configuration-secure/ansible/local'
+
   roles:
     - common
     - nginx
     - gunicorn
     - edxapp
-    - ruby
-    - npm
-    - edx-theme
     # run this role last
     - in_production

playbooks/edx-west/edxutil_prod.yml

+- hosts: tag_environment_prod:&tag_function_util
+  sudo: True
+  vars_files:
+    - "{{ secure_dir }}/vars/edxapp_prod_vars.yml"
+    - "{{ secure_dir }}/vars/users.yml"
+    - "{{ secure_dir }}/vars/edxapp_prod_users.yml"
+  roles:
+    - common
+    - edxapp
+    - edx_worker_upstart
\ No newline at end of file

playbooks/edx-west/secure_example

+../secure_example/
\ No newline at end of file

playbooks/group_vars/tag_environment_prod

+---
+secure_dir: '../../configuration-secure/ansible'
+# this indicates the path to site-specific (with precedence)
+# things like nginx template files
+local_dir:  '../../configuration-secure/ansible/local'

playbooks/roles/edxapp/handlers/main.yml

 ---
-# Restart Start all services.
-- name: restart edxapp
-  service: name=edxapp state=restarted
+- name: invoke all deploy handlers
+  shell: echo "running deploy handlers"
+  notify:
+  - gem | gem install bundler
+  - bundle | bundle install
+  - Install edx-platform npm dependencies
+  - install python pre-requirements
+  - install python base-requirements
+  - install python post-requirements
+  - install python post-post requirements
+  - check if rake gather_assets is available
+  - check if django can collect lms static data
+  - check if django can collect cms static data
+  - check if django can update cms templates
+  - gather lms static assets with rake
+  - gather lms static assets with django
+  - gather cms static assets with rake
+  - gather cms static assets with django
+  - update cms templates
+  - lms asset static failure check
+  - cms asset static failure check
+  - stop edxapp
+  - start edxapp
   tags:
   - lms
-  - lms-xml
-  - lms-preview
   - cms
+  - install
   - deploy
 
+
 # Ruby plays that need to be run after platform updates.
 - name: gem | gem install bundler
   shell: RBENV_ROOT={{ rbenv_root }} GEM_HOME={{ gem_home }} {{ rbenv_root }}/shims/gem install bundle chdir={{ platform_code_dir }}
@@ -30,6 +50,7 @@
   tags:
   - npm
   - update
+  - deploy
 
 # Python plays that need to be run after platform updates.
 
@@ -40,6 +61,7 @@
   - lms
   - cms
   - install
+  - deploy
 
 # Install the python modules into {{ venv_dir }}
 - name : install python base-requirements
@@ -51,7 +73,7 @@
   - lms
   - cms
   - install
-
+  - deploy
 
 # Install the python post requirements into {{ venv_dir }}
 - name : install python post-requirements
@@ -60,6 +82,7 @@
   - lms
   - cms
   - install
+  - deploy
 
 # Install the final python modules into {{ venv_dir }}
 - name : install python post-post requirements
@@ -75,6 +98,7 @@
   - lms
   - cms
   - install
+  - deploy
 
 # This check needs to be run to see if rake can be used but its failure should not stop the run.
 - name: check if rake gather_assets is available
@@ -126,7 +150,8 @@
   shell: executable=/bin/bash chdir={{ platform_code_dir }} SERVICE_VARIANT={{ lms_variant }} rake lms:gather_assets:aws
   when: grep_gather_assets.rc == 0
   notify:
-  - restart edxapp
+  - stop edxapp
+  - start edxapp
   sudo: yes
   sudo_user: www-data
   environment: "{{ deploy_environment }}"
@@ -141,7 +166,8 @@
   shell: SERVICE_VARIANT={{ lms_variant }} django-admin.py collectstatic --pythonpath={{ platform_code_dir }} --settings=lms.envs.aws --noinput --verbosity=0
   when: grep_gather_assets.rc != 0 and check_lms_collect_static.rc == 0
   notify:
-  - restart edxapp
+  - stop edxapp
+  - start edxapp
   sudo: yes
   sudo_user: www-data
   environment: "{{ deploy_environment }}"
@@ -158,7 +184,8 @@
   shell: executable=/bin/bash chdir={{ platform_code_dir }} SERVICE_VARIANT={{ cms_variant }} rake cms:gather_assets:aws
   when: grep_gather_assets.rc == 0
   notify:
-  - restart edxapp
+  - stop edxapp
+  - start edxapp
   sudo: yes
   sudo_user: www-data
   environment: "{{ deploy_environment }}"
@@ -170,7 +197,8 @@
   shell: SERVICE_VARIANT={{ cms_variant }} django-admin.py collectstatic --pythonpath={{ platform_code_dir }} --settings=lms.envs.aws --noinput --verbosity=0
   when: grep_gather_assets.rc != 0 and check_cms_collect_static.rc == 0
   notify:
-  - restart edxapp
+  - stop edxapp
+  - start edxapp
   sudo: yes
   sudo_user: www-data
   environment: "{{ deploy_environment }}"
@@ -182,7 +210,8 @@
   shell: SERVICE_VARIANT={{ cms_variant }} django-admin.py update_templates --pythonpath={{ platform_code_dir }} --settings=cms.envs.aws
   when: check_cms_update_templates.rc == 0
   notify:
-  - restart edxapp
+  - stop edxapp
+  - start edxapp
   sudo: yes
   sudo_user: www-data
   environment: "{{ deploy_environment }}"
@@ -206,3 +235,21 @@
   tags:
   - cms
   - deploy
+
+- name: stop edxapp
+  service: name=edxapp state=stopped
+  tags:
+  - lms
+  - lms-xml
+  - lms-preview
+  - cms
+  - deploy
+
+- name: start edxapp
+  service: name=edxapp state=started
+  tags:
+  - lms
+  - lms-xml
+  - lms-preview
+  - cms
+  - deploy

playbooks/roles/edxapp/tasks/deploy.yml

@@ -17,6 +17,7 @@
   - lms
   - cms
   - install
+  - deploy
 
 # Do Post Checkout Tasks.
 - name: create platform code dir
@@ -26,6 +27,18 @@
   - cms
   - install
 
+- name: checkout theme
+  git: dest={{app_base_dir}}/themes/{{edxapp_theme_name}} repo={{edxapp_theme_source_repo}} version={{edxapp_theme_version}}
+  when: edxapp_theme_name != ''
+  tags:
+  - cms
+  - lms-xml
+  - lms-preview
+  - lms
+  - update
+  - deploy
+
+
 # TODO: Check git.py _run_if_changed() to see if the logic there to skip running certain
 # portions of the deploy needs to be incorporated here.
 
@@ -33,27 +46,19 @@
   file: path={{platform_code_dir}} state=directory owner=www-data group=www-data recurse=yes
   # Post Checkout tasks will get run as handlers when the {{ platform_code_dir }} is ready.
   # Look at the handlers/main.yml in this role for a description of the tasks stated below.
-  notify:
-  - gem | gem install bundler
-  - bundle | bundle install
-  - Install edx-platform npm dependencies
-  - install python pre-requirements
-  - install python base-requirements
-  - install python post-requirements
-  - install python post-post requirements
-  - check if rake gather_assets is available
-  - check if django can collect lms static data
-  - check if django can collect cms static data
-  - check if django can update cms templates
-  - gather lms static assets with rake
-  - gather lms static assets with django
-  - gather cms static assets with rake
-  - gather cms static assets with django
-  - update cms templates
-  - lms asset static failure check
-  - cms asset static failure check
   tags:
   - lms
   - cms
   - install
+  - deploy
 
+# want deploy handlers to always run when deploy tag is specified, so move into a no op task
+- name: run deploy handlers
+  shell: echo "running deploy handlers"
+  notify:
+  - invoke all deploy handlers
+  tags:
+  - lms
+  - cms
+  - install
+  - deploy

playbooks/roles/edxapp/tasks/lms-preview.yml

@@ -15,6 +15,14 @@
   - lms-preview
   - lms-preview-env
 
+- name: Create lms-preview log target directory
+  file: path={{log_base_dir}}/lms-preview state=directory owner=syslog group=adm mode=2770
+  tags:
+  - lms-preview
+  - lms-preview-env
+  - logging
+  - update
+
 - include: ../../nginx/tasks/nginx_site.yml state=link site_name=lms-preview
 
 - include: ../../nginx/tasks/nginx_site.yml state=link site_name=lms-preview-backend

playbooks/roles/edxapp/tasks/lms-xml.yml

@@ -6,17 +6,25 @@
 - name: create lms-xml application config
   template: src=lms-xml.env.json.j2 dest=$app_base_dir/lms-xml.env.json mode=640 owner=www-data group=adm
   tags:
-  - cms-env
-  - cms
+  - lms-xml-env
+  - lmx-xml
   - update
 
 - name: create lms-xml auth file
   template: src=lms-xml.auth.json.j2 dest=$app_base_dir/lms-xml.auth.json mode=640 owner=www-data group=adm
   tags:
-  - cms-env
-  - cms
+  - lms-xml-env
+  - lmx-xml
   - update
-  
+
+- name: Create lms-xml log target directory
+  file: path={{log_base_dir}}/lms-xml state=directory owner=syslog group=adm mode=2770
+  tags:
+  - lms-xml
+  - lms-xml-env
+  - logging
+  - update
+
 - include: ../../nginx/tasks/nginx_site.yml state=link site_name=lms-xml
   
 - include: ../../nginx/tasks/nginx_site.yml state=link site_name=lms-xml-backend

playbooks/roles/edxapp/tasks/lms.yml

@@ -13,6 +13,14 @@
   - lms-env
   - update
 
+- name: Create lms log target directory
+  file: path={{log_base_dir}}/lms state=directory owner=syslog group=adm mode=2770
+  tags:
+  - lms
+  - lms-env
+  - logging
+  - update
+
 - include: ../../nginx/tasks/nginx_site.yml state=link site_name=lms
 - include: ../../nginx/tasks/nginx_site.yml state=link site_name=lms-backend
 # Creates LMS upstart file

playbooks/roles/edxapp/tasks/main.yml

@@ -19,13 +19,13 @@
   - lms-env
   - update
 
-
-- name: Create lms log target directory
-  file: path={{log_base_dir}}/lms state=directory owner=syslog group=adm mode=2770
+- name: Create theming directory
+  file: path={{ app_base_dir }}/themes state=directory mode=2775 group=adm owner=www-data
   tags:
+  - cms
   - lms
+  - cms-env
   - lms-env
-  - logging
   - update
 
 # Install ssh keys for ubuntu account to be able to check out from edx-platform

playbooks/roles/edxapp/vars/main.yml

@@ -24,10 +24,18 @@ cms_env_config: {}
 lms_xml_auth_config: {}
 lms_xml_env_config: {}
 
-lms_source_repo: git@github.com:edx/edx-platform.git
+#Theming
+#To turn off theming, specify edxapp_theme_name: ''
+#Stanford, for example, uses edxapp_theme_name: 'stanford'
+edxapp_theme_name: ''
+edxapp_theme_source_repo: 'git://github.com/Stanford-Online/edx-theme.git'
+edxapp_theme_version: 'HEAD'
+
+# make this the public URL instead of writable
+lms_source_repo: git://github.com/edx/edx-platform.git
 lms_version: 'HEAD'
 local_requirements_file:  "{{ platform_code_dir }}/requirements/edx/local.txt"
-pre_requirements_file:   "{{ platform_code_dir }}/requirements/edx/pre.txt"
+pre_requirements_file:    "{{ platform_code_dir }}/requirements/edx/pre.txt"
 post_requirements_file:   "{{ platform_code_dir }}/requirements/edx/post.txt"
 base_requirements_file:   "{{ platform_code_dir }}/requirements/edx/base.txt"
 github_requirements_file: "{{ platform_code_dir }}/requirements/edx/github.txt"
@@ -117,6 +125,7 @@ lms_variant: lms
 cms_variant: cms
 
 deploy_environment:
+  LANG: "en_US.UTF-8"
   NO_PREREQ_INSTALL: 1
   SKIP_WS_MIGRATIONS: 1
   RBENV_ROOT: "{{ rbenv_root }}"