Merge pull request #103 from edx/feanil/ansible_deploy

Feanil/ansible deploy

playbooks/ec2.ini

 [ec2]
 regions=all
 destination_variable=public_dns_name
-vpc_destination_variable=private_dns_name
+vpc_destination_variable=private_ip_address
 cache_path=/tmp
 cache_max_age=300

playbooks/edx-east/feanil_deploy.yml

+- hosts: tag_aws_cloudformation_stack-name_feanilpractice:&tag_group_edxapp
+  sudo: True
+  vars_files:
+    - "{{ secure_dir }}/vars/edxapp_continuous_integration_vars.yml"
+    - "{{ secure_dir }}/vars/users.yml"
+  roles:
+    - common
+    - nginx
+    - gunicorn
+    - edxapp

playbooks/roles/common/vars/main.yml

-# Variables for the common role
-# Automatically loaded when the common role is used
----
-nginx_cfg:
-  #   - link - turn on
-  #   - absent - turn off 
-  sites_enabled:
-    basic_auth: link
-    edx_release: link
-  # path to version files for the basic
-  # nginx configuration
-  version_html: $app_base_dir/versions.html
-  version_json: $app_base_dir/versions.json
-  # default htpasswd contents set to edx/edx
-  # this value can be overiden in vars/secure/<group>.yml
-  htpasswd: |
-    edx:$apr1$2gWcIvlc$Nu7b/KTwd5HoIDEkSPNUk/
-

playbooks/roles/edxapp/handlers/main.yml

+---
+# Restart Start all services.
+- name: restart edxapp
+  service: name=edxapp state=restarted
+  tags:
+  - lms
+  - lms-xml
+  - lms-preview
+  - cms
+  - deploy
+
+# Ruby plays that need to be run after platform updates.
+- name: gem | gem install bundler
+  shell: RBENV_ROOT={{ rbenv_root }} GEM_HOME={{ gem_home }} {{ rbenv_root }}/shims/gem install bundle chdir={{ platform_code_dir }}
+  tags:
+  - ruby
+  - deploy
+  - install
+
+- name: bundle | bundle install
+  shell: RBENV_ROOT={{ rbenv_root }} GEM_HOME={{ gem_home }} {{ gem_home }}/bin/bundle install --binstubs chdir={{ platform_code_dir }}
+  tags:
+  - ruby
+  - deploy
+  - install
+
+# Node play that need to be run after platform updates.
+- name: Install edx-platform npm dependencies
+  shell: npm install chdir={{ platform_code_dir }}
+  tags:
+  - npm
+  - update
+
+# Python plays that need to be run after platform updates.
+
+# Install the python pre requirements into {{ venv_dir }}
+- name : install python pre-requirements
+  pip: requirements="{{pre_requirements_file}}" virtualenv="{{venv_dir}}" state=present
+  tags:
+  - lms
+  - cms
+  - install
+
+# Install the python modules into {{ venv_dir }}
+- name : install python base-requirements
+  # Need to use shell rather than pip so that we can maintain the context of our current working directory; some
+  # requirements are pathed relative to the edx-platform repo. Using the pip from inside the virtual environment implicitly
+  # installs everything into that virtual environment.
+  shell: cd {{ platform_code_dir }} && {{ venv_dir }}/bin/pip install --use-mirrors -r {{ base_requirements_file }}
+  tags:
+  - lms
+  - cms
+  - install
+
+
+# Install the python post requirements into {{ venv_dir }}
+- name : install python post-requirements
+  pip: requirements="{{post_requirements_file}}" virtualenv="{{venv_dir}}" state=present
+  tags:
+  - lms
+  - cms
+  - install
+
+# Install the final python modules into {{ venv_dir }}
+- name : install python post-post requirements
+  # Need to use shell rather than pip so that we can maintain the context of our current working directory; some
+  # requirements are pathed relative to the edx-platform repo. Using the pip from inside the virtual environment implicitly
+  # installs everything into that virtual environment.
+  shell: cd {{ platform_code_dir }} && {{ venv_dir }}/bin/pip install --use-mirrors -r {{ item }}
+  with_items:
+  - "{{ repo_requirements_file }}"
+  - "{{ github_requirements_file }}"
+  - "{{ local_requirements_file }}"
+  tags:
+  - lms
+  - cms
+  - install
+
+# This check needs to be run to see if rake can be used but its failure should not stop the run.
+- name: check if rake gather_assets is available
+  shell: executable=/bin/bash chdir={{ platform_code_dir }} rake -T | grep gather_assets
+  environment: "{{ deploy_environment }}"
+  register: grep_gather_assets
+  ignore_errors: yes
+  tags:
+  - lms
+  - lms-preview
+  - lms-xml
+  - cms
+  - deploy
+
+- name: check if django can collect lms static data
+  shell: SERVICE_VARIANT={{ lms_variant }} django-admin.py help collectstatic --pythonpath={{ platform_code_dir }} --settings=lms.envs.aws
+  register: check_lms_collect_static
+  sudo: yes
+  sudo_user: www-data
+  environment: "{{ deploy_environment }}"
+  tags:
+  - lms
+  - lms-preview
+  - lms-xml
+  - deploy
+
+- name: check if django can collect cms static data
+  shell: SERVICE_VARIANT={{ lms_variant }} django-admin.py help collectstatic --pythonpath={{ platform_code_dir }} --settings=cms.envs.aws
+  register: check_cms_collect_static
+  sudo: yes
+  sudo_user: www-data
+  environment: "{{ deploy_environment }}"
+  tags:
+  - cms
+  - deploy
+
+- name: check if django can update cms templates
+  shell: SERVICE_VARIANT={{ cms_variant }} django-admin.py help update_templates --pythonpath={{ platform_code_dir }} --settings=cms.envs.aws
+  register: check_cms_update_templates
+  sudo: yes
+  sudo_user: www-data
+  environment: "{{ deploy_environment }}"
+  tags:
+  - cms
+  - deploy
+
+# Gather lms assets using rake if possible
+- name: gather lms static assets with rake
+  shell: executable=/bin/bash chdir={{ platform_code_dir }} SERVICE_VARIANT={{ lms_variant }} rake lms:gather_assets:aws
+  when: grep_gather_assets.rc == 0
+  notify:
+  - restart edxapp
+  sudo: yes
+  sudo_user: www-data
+  environment: "{{ deploy_environment }}"
+  tags:
+  - lms
+  - lms-preview
+  - lms-xml
+  - deploy
+
+# Gather lms assets using django if necessary(When rake doesn't know how)
+- name: gather lms static assets with django
+  shell: SERVICE_VARIANT={{ lms_variant }} django-admin.py collectstatic --pythonpath={{ platform_code_dir }} --settings=lms.envs.aws --noinput --verbosity=0
+  when: grep_gather_assets.rc != 0 and check_lms_collect_static.rc == 0
+  notify:
+  - restart edxapp
+  sudo: yes
+  sudo_user: www-data
+  environment: "{{ deploy_environment }}"
+  tags:
+  - lms
+  - lms-preview
+  - lms-xml
+  - deploy
+
+
+# Gather cms assets using rake if possible
+- name: gather cms static assets with rake
+#  script: gather_assets.sh
+  shell: executable=/bin/bash chdir={{ platform_code_dir }} SERVICE_VARIANT={{ cms_variant }} rake cms:gather_assets:aws
+  when: grep_gather_assets.rc == 0
+  notify:
+  - restart edxapp
+  sudo: yes
+  sudo_user: www-data
+  environment: "{{ deploy_environment }}"
+  tags:
+  - cms
+  - deploy
+
+- name: gather cms static assets with django
+  shell: SERVICE_VARIANT={{ cms_variant }} django-admin.py collectstatic --pythonpath={{ platform_code_dir }} --settings=lms.envs.aws --noinput --verbosity=0
+  when: grep_gather_assets.rc != 0 and check_cms_collect_static.rc == 0
+  notify:
+  - restart edxapp
+  sudo: yes
+  sudo_user: www-data
+  environment: "{{ deploy_environment }}"
+  tags:
+  - cms
+  - deploy
+
+- name: update cms templates
+  shell: SERVICE_VARIANT={{ cms_variant }} django-admin.py update_templates --pythonpath={{ platform_code_dir }} --settings=cms.envs.aws
+  when: check_cms_update_templates.rc == 0
+  notify:
+  - restart edxapp
+  sudo: yes
+  sudo_user: www-data
+  environment: "{{ deploy_environment }}"
+  tags:
+  - cms
+  - deploy
+
+# Add failure checks for if no static assets were deployed.
+- name: lms asset static failure check
+  shell: /bin/false
+  when: grep_gather_assets.rc != 0 and check_lms_collect_static.rc != 0
+  tags:
+  - lms
+  - lms-preview
+  - lms-xml
+  - deploy
+
+- name: cms asset static failure check
+  shell: /bin/false
+  when: grep_gather_assets.rc != 0 and check_cms_collect_static.rc != 0
+  tags:
+  - cms
+  - deploy

playbooks/roles/edxapp/tasks/cms.yml

@@ -29,8 +29,5 @@
   - logging
   - update
 
-# If we set up CMS, we have to set up edx logging
-- include: ../../common/tasks/edx_logging_base.yml
-
 # Creates LMS upstart file
 - include: ../../gunicorn/tasks/upstart.yml service_variant=cms

playbooks/roles/edxapp/tasks/deploy.yml

+# Stop all services.
+- name: stop edxapp services
+  service: name=edxapp state=stopped
+  tags:
+  - lms
+  - lms-xml
+  - lms-preview
+  - cms
+  - deploy
+
+# Do A Checkout
+- name: git checkout edx-platform repo into $app_base_dir
+  git: dest={{platform_code_dir}} repo={{lms_source_repo}} version={{lms_version}}
+  environment:
+    GIT_SSH: /tmp/git_ssh.sh
+  tags:
+  - lms
+  - cms
+  - install
+
+# Do Post Checkout Tasks.
+- name: create platform code dir
+  file: path={{platform_code_dir}} state=directory owner=www-data group=www-data mode=755
+  tags:
+  - lms
+  - cms
+  - install
+
+# TODO: Check git.py _run_if_changed() to see if the logic there to skip running certain
+# portions of the deploy needs to be incorporated here.
+
+- name: sets permissions on platform code dir and contents
+  file: path={{platform_code_dir}} state=directory owner=www-data group=www-data recurse=yes
+  # Post Checkout tasks will get run as handlers when the {{ platform_code_dir }} is ready.
+  # Look at the handlers/main.yml in this role for a description of the tasks stated below.
+  notify:
+  - gem | gem install bundler
+  - bundle | bundle install
+  - Install edx-platform npm dependencies
+  - install python pre-requirements
+  - install python base-requirements
+  - install python post-requirements
+  - install python post-post requirements
+  - check if rake gather_assets is available
+  - check if django can collect lms static data
+  - check if django can collect cms static data
+  - check if django can update cms templates
+  - gather lms static assets with rake
+  - gather lms static assets with django
+  - gather cms static assets with rake
+  - gather cms static assets with django
+  - update cms templates
+  - lms asset static failure check
+  - cms asset static failure check
+  tags:
+  - lms
+  - cms
+  - install
+

playbooks/roles/edxapp/tasks/main.yml

@@ -28,16 +28,12 @@
   - logging
   - update
 
-# If we set up LMS, we have to set up edx logging
-- include: ../../common/tasks/edx_logging_base.yml
-
 # Install ssh keys for ubuntu account to be able to check out from edx-platform
 # Temprory behavior, not needed after June 1. Perhaps still useful as a recipe.
 # {{ secure_dir }} is relative to the top-level playbooks dir so there is some
-# ugly relative pathing here
 
 - name: install read-only ssh key for edx-platform repo (private)
-  copy: src=../../../{{ secure_dir }}/files/git-identity dest=/etc/git-identity force=yes owner=ubuntu group=adm mode=600
+  copy: src={{ secure_dir }}/files/git-identity dest=/etc/git-identity force=yes owner=ubuntu group=adm mode=600
   tags:
   - lms
   - cms
@@ -57,22 +53,6 @@
   - lms
   - cms
 
-- name: git checkout edx-platform repo into $app_base_dir
-  git: dest={{platform_code_dir}} repo={{lms_source_repo}} version={{lms_version}}
-  environment:
-    GIT_SSH: /tmp/git_ssh.sh
-  tags:
-  - lms
-  - cms
-  - install
-  
-- name: sets permissions on platform code dir
-  file: path={{platform_code_dir}} state=directory owner=www-data group=www-data mode=755
-  tags:
-  - lms
-  - cms
-  - install
-
 - name: install a bunch of system packages on which LMS and CMS rely
   apt: pkg={{item}} state=present
   with_items: lms_debian_pkgs
@@ -82,49 +62,6 @@
   - install
 
 
-# Install the python pre requirements into {{ venv_dir }}
-- name : install python pre-requirements
-  pip: requirements="{{pre_requirements_file}}" virtualenv="{{venv_dir}}" state=present
-  tags:
-  - lms
-  - cms
-  - install
-
-# Install the python modules into {{ venv_dir }}
-- name : install base python packages using the shell
-  # Need to use shell rather than pip so that we can maintain the context of our current working directory; some 
-  # requirements are pathed relative to the edx-platform repo. Using the pip from inside the virtual environment implicitly
-  # installs everything into that virtual environment.
-  shell: cd {{ platform_code_dir }} && {{ venv_dir }}/bin/pip install --use-mirrors -r {{ base_requirements_file }}
-  tags:
-  - lms
-  - cms
-  - install
-  
- 
-# Install the python post requirements into {{ venv_dir }}
-- name : install python post-requirements
-  pip: requirements="{{post_requirements_file}}" virtualenv="{{venv_dir}}" state=present
-  tags:
-  - lms
-  - cms
-  - install
-
-# Install the final python modules into {{ venv_dir }}
-- name : install python post-post requirements using the shell
-  # Need to use shell rather than pip so that we can maintain the context of our current working directory; some 
-  # requirements are pathed relative to the edx-platform repo. Using the pip from inside the virtual environment implicitly
-  # installs everything into that virtual environment.
-  shell: cd {{ platform_code_dir }} && {{ venv_dir }}/bin/pip install --use-mirrors -r {{ item }}
-  with_items:
-  - "{{ repo_requirements_file }}"
-  - "{{ github_requirements_file }}"
-  - "{{ local_requirements_file }}"
-  tags:
-  - lms
-  - cms
-  - install
-
 - include: lms.yml
   when: "'lms' in service_variants_enabled"
 - include: lms-xml.yml
@@ -133,4 +70,7 @@
   when: "'cms' in service_variants_enabled"
 - include: lms-preview.yml
   when: "'lms-preview' in service_variants_enabled"
- 
+
+- include: npm.yml
+- include: ruby.yml
+- include: deploy.yml

playbooks/roles/npm/tasks/main.yml → playbooks/roles/edxapp/tasks/npm.yml

@@ -7,10 +7,3 @@
   tags:
   - npm
   - install
-  
-- name: Install edx-platform npm dependencies
-  shell: npm install chdir=${platform_code_dir}
-  tags:
-  - npm
-  - update
-

playbooks/roles/ruby/tasks/main.yml → playbooks/roles/edxapp/tasks/ruby.yml

@@ -112,14 +112,3 @@
   - ruby
   - install
 
-- name: gem | gem install bundler
-  shell: RBENV_ROOT=${rbenv_root} GEM_HOME=${gem_home} ${rbenv_root}/shims/gem install bundle chdir=${platform_code_dir}
-  tags:
-  - ruby
-  - install
-
-- name: bundle | bundle install
-  shell: RBENV_ROOT=${rbenv_root} GEM_HOME=${gem_home} ${gem_home}/bin/bundle install --binstubs chdir=${platform_code_dir}
-  tags:
-  - ruby
-  - install

playbooks/roles/edxapp/vars/main.yml

@@ -105,3 +105,20 @@ lms_debian_pkgs:
   - yui-compressor
   - zip
   - zlib1g-dev
+
+# Ruby Specific Vars
+ruby_base: /opt/www
+rbenv_root: "{{ ruby_base }}/.rbenv"
+ruby_version: "1.9.3-p374"
+gem_home: "{{ ruby_base }}/.gem"
+
+# Deploy Specific Vars
+lms_variant: lms
+cms_variant: cms
+
+deploy_environment:
+  NO_PREREQ_INSTALL: 1
+  SKIP_WS_MIGRATIONS: 1
+  RBENV_ROOT: "{{ rbenv_root }}"
+  GEM_HOME: "{{ gem_home }}"
+  PATH: "{{ venv_dir }}/bin:{{ platform_code_dir }}/bin:{{ rbenv_root }}/bin:{{ rbenv_root }}/shims:{{ gem_home }}/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games"

playbooks/roles/nginx/templates/lms-xml.j2

@@ -3,8 +3,8 @@ server {
 
   listen 80;
 
-  server_name trace-lms-preview.*
-              preview.lms-dev.m.edx.org;
+  server_name trace-lms-xml.*
+              xml.lms-dev.m.edx.org;
 
   #
   # Send error response when request host isn't under our control
@@ -40,21 +40,21 @@ server {
     proxy_set_header Host $http_host;
 
     proxy_redirect off;
-    proxy_pass http://lms-preview-backend;
+    proxy_pass http://lms-xml-backend;
   }
 
   location / {
-    try_files $uri @proxy_to_lms-preview_app;
+    try_files $uri @proxy_to_lms-xml_app;
   }
 
   # No basic auth security on the github_service_hook url, so that github can use it for cms
   location /github_service_hook {
-    try_files $uri @proxy_to_lms-preview_app;
+    try_files $uri @proxy_to_lms-xml_app;
   }
 
   # No basic auth security on the heartbeat url, so that ELB can use it
   location /heartbeat {
-    try_files $uri @proxy_to_lms-preview_app;
+    try_files $uri @proxy_to_lms-xml_app;
   }
 
   # Check security on this

playbooks/roles/nginx/templates/lms.j2

@@ -67,7 +67,7 @@ server {
   # Check security on this
   location ~ /static/(?P<file>.*) {
     auth_basic            "Restricted";
-    auth_basic_user_file  /etc/nginx/lms.htpasswd;
+    auth_basic_user_file  /etc/nginx/nginx.htpasswd;
     root {{app_base_dir}};
     try_files /staticfiles/$file /course_static/$file =404;
 

playbooks/roles/nginx/vars/main.yml

@@ -3,3 +3,19 @@
 pkgs:
   nginx:
     state: installed
+
+
+nginx_cfg:
+  #   - link - turn on
+  #   - absent - turn off
+  sites_enabled:
+    basic_auth: link
+    edx_release: link
+  # path to version files for the basic
+  # nginx configuration
+  version_html: $app_base_dir/versions.html
+  version_json: $app_base_dir/versions.json
+  # default htpasswd contents set to edx/edx
+  # this value can be overiden in vars/secure/<group>.yml
+  htpasswd: |
+    edx:$apr1$2gWcIvlc$Nu7b/KTwd5HoIDEkSPNUk/

playbooks/roles/ruby/vars/main.yml

----
-ruby_base: /opt/www
-rbenv_root: "{{ ruby_base }}/.rbenv"
-ruby_version: "1.9.3-p374"
-gem_home: "{{ ruby_base }}/.gem"