Merge pull request #256 from edx/feanil/update_vpc_tools

Feanil/update vpc tools

ansible-requirements.txt

@@ -2,7 +2,7 @@ Jinja2==2.6
 PyYAML==3.10
 ansible==1.2.2
 argparse==1.2.1
-boto==2.8.0
+boto==2.10.0
 paramiko==1.10.1
 pycrypto==2.6
 wsgiref==0.1.2

cloudformation_templates/edx-reference-architecture.json

@@ -2007,7 +2007,10 @@
                   "Effect":"Allow",
                   "Action":[
                     "cloudformation:DescribeStackResource",
-                    "s3:Put"
+                    "s3:Put",
+                    "ses:SendEmail",
+                    "ses:SendRawEmail",
+                    "ses:GetSendQuota"
                   ],
                   "Resource":"*"
                 }

fabric-requirements.txt

@@ -4,7 +4,7 @@ PyYAML==3.10
 WebOb==1.2.3
 argparse==1.2.1
 beautifulsoup4==4.1.3
-boto==2.7.0
+boto==2.10.0
 cloudformation==0.0.0
 decorator==3.4.0
 distribute==0.6.30

playbooks/edx-east/edx_dev2.yml

+---
+- hosts: tag_aws_cloudformation_stack-name_dev2:&tag_group_edxapp
+  sudo: True
+  vars_files:
+    - "{{ secure_dir }}/vars/dev/dev2.yml"
+    - "{{ secure_dir }}/vars/common/common.yml"
+    - "{{ secure_dir }}/vars/users.yml"
+  roles:
+    - common
+    - datadog
+    - nginx
+    - role: 'edxapp'
+      lms_nginx_port: 80
+      cms_nginx_port: 80
+      edxapp_lms_env: 'lms.envs.load_test'
+      lms_version: 'sarina/install-datadog'
+- hosts: tag_aws_cloudformation_stack-name_dev2:&tag_group_worker
+  sudo: True
+  vars_files:
+    - "{{ secure_dir }}/vars/dev/dev2.yml"
+    - "{{ secure_dir }}/vars/common/common.yml"
+    - "{{ secure_dir }}/vars/users.yml"
+  roles:
+    - common
+    - datadog
+    - nginx
+    - role: 'edxapp'
+      edxapp_lms_env: 'lms.envs.load_test'
+      celery_worker: True
+      lms_version: 'sarina/install-datadog'
+#- hosts: tag_aws_cloudformation_stack-name_dev2:&tag_group_xserver
+#  sudo: True
+#  vars_files:
+#    - "{{ secure_dir }}/vars/dev/dev2.yml"
+#    - "{{ secure_dir }}/vars/users.yml"
+#  roles:
+#    - common
+#    - nginx
+#    - xserver
+#- hosts: tag_aws_cloudformation_stack-name_dev2:&tag_group_rabbitmq
+#  serial: 1
+#  sudo: True
+#  vars_files:
+#    - "{{ secure_dir }}/vars/dev/dev2.yml"
+#    - "{{ secure_dir }}/vars/users.yml"
+#  roles:
+#    - common
+#    - rabbitmq
+#- hosts: tag_aws_cloudformation_stack-name_dev2:&tag_group_xqueue
+#  sudo: True
+#  vars_files:
+#    - "{{ secure_dir }}/vars/dev/dev2.yml"
+#    - "{{ secure_dir }}/vars/users.yml"
+#  roles:
+#    - common
+#    - nginx
+#    - xqueue

playbooks/edx-east/feanil_deploy.yml → playbooks/edx-east/edx_feanilsandbox.yml

 ---
-- hosts: tag_aws_cloudformation_stack-name_feanilpractice:&tag_group_edxapp
+- hosts: tag_aws_cloudformation_stack-name_feanilsandbox:&tag_group_edxapp
   sudo: True
   vars_files:
+    - "{{ secure_dir }}/vars/dev/feanilsandbox.yml"
+    - "{{ secure_dir }}/vars/common/common.yml"
     - "{{ secure_dir }}/vars/users.yml"
   roles:
     - common
+    - datadog
     - nginx
-    - edxapp
-    - { role: 'edxapp', celery_worker: True }
-
-- hosts: tag_aws_cloudformation_stack-name_feanilpractice:&tag_group_xserver
+    - role: 'edxapp'
+      lms_nginx_port: 80
+      cms_nginx_port: 80
+      edxapp_lms_env: 'lms.envs.load_test'
+      lms_version: 'HEAD'
+- hosts: tag_aws_cloudformation_stack-name_feanilsandbox:&tag_group_worker
   sudo: True
   vars_files:
+    - "{{ secure_dir }}/vars/dev/feanilsandbox.yml"
+    - "{{ secure_dir }}/vars/common/common.yml"
+    - "{{ secure_dir }}/vars/users.yml"
+  roles:
+    - common
+    - datadog
+    - nginx
+    - role: 'edxapp'
+      edxapp_lms_env: 'lms.envs.load_test'
+      celery_worker: True
+      lms_version: 'HEAD'
+- hosts: tag_aws_cloudformation_stack-name_feanilsandbox:&tag_group_xserver
+  sudo: True
+  vars_files:
+    - "{{ secure_dir }}/vars/dev/feanilsandbox.yml"
     - "{{ secure_dir }}/vars/users.yml"
   roles:
     - common
     - nginx
     - xserver
-- hosts: tag_aws_cloudformation_stack-name_feanilpractice:&tag_group_rabbitmq
+- hosts: tag_aws_cloudformation_stack-name_feanilsandbox:&tag_group_rabbitmq
   serial: 1
   sudo: True
   vars_files:
+    - "{{ secure_dir }}/vars/dev/feanilsandbox.yml"
     - "{{ secure_dir }}/vars/users.yml"
   roles:
     - common
     - rabbitmq
-- hosts: tag_aws_cloudformation_stack-name_feanilpractice:&tag_group_xqueue
+- hosts: tag_aws_cloudformation_stack-name_feanilsandbox:&tag_group_xqueue
   sudo: True
   vars_files:
+    - "{{ secure_dir }}/vars/dev/feanilsandbox.yml"
     - "{{ secure_dir }}/vars/users.yml"
   roles:
     - common

playbooks/roles/common/templates/edx_rsyslog.j2

 # custom edx syslog configuration
-# Put in place and templatized by ansible 
-#
-# Cliffs notes version: ansible uses local0 and local1, so they have to be
-# plumbed through appropriately.
- 
-#############
-# Change some global configuration
-#############
+#
+
+
+#  Default rules for rsyslog.
+#
+#                       For more information see rsyslog.conf(5) and /etc/rsyslog.conf
+
+#
+# First some standard log files.  Log by facility.
+
 # don't escape newlines
 $EscapeControlCharactersOnReceive off
+
 $SystemLogRateLimitInterval 0
 $RepeatedMsgReduction off
 $MaxMessageSize 32768
- 
-#############
-# Override default auth config so we can ignore local0 and local1 also
-#############
+
 auth,authpriv.*                                         /var/log/auth.log
 *.*;auth,authpriv.none,local0.none,local1.none          -/var/log/syslog
- 
-# According to the docs for rsyslog, "syslogtag" is the "TAG" from 
-# the message which in the case of tracking logs is interpreted to 
-# be everything before the first whitespace character. 
+
+# According to the docs for rsyslog, "syslogtag" is the "TAG" from
+# the message which in the case of tracking logs is interpreted to
+# be everything before the first whitespace character.
 # This is why we include "syslogtag."
-# Maybe one day this will be answered:
-# - http://stackoverflow.com/questions/10449447/how-to-avoid-syslogtag-from-rsyslog-template
+# Maybe one day this will be answered - http://stackoverflow.com/questions/10449447/how-to-avoid-syslogtag-from-rsyslog-template
 $template tracking,"%syslogtag%%msg%\n"
- 
+
 # looks for [service_name=<name>] in the beginning of the log message,
-# if it exists the log will go into {{log_base_dir}}/<name>/edx.log, otherwise
-# it will go into {{log_base_dir}}/edx.log
-$template DynaFile,"{{log_base_dir}}/%syslogtag:R,ERE,1,BLANK:\[service_variant=([a-zA-Z_-]*)\].*--end%/edx.log"
- 
+# if it exists the log will go into /mnt/logs/<name>/edx.log, otherwise
+# it will go into /mnt/logs/edx.log
+$template DynaFile,"/mnt/logs/%syslogtag:R,ERE,1,BLANK:\[service_variant=([a-zA-Z_-]*)\].*--end%/edx.log"
+
 local0.*                        -?DynaFile
-local1.*                        {{log_base_dir}}/tracking.log;tracking
+local1.*                        /mnt/logs/tracking.log;tracking
+#cron.*                         /var/log/cron.log
+#daemon.*                       -/var/log/daemon.log
+kern.*                          -/var/log/kern.log
+#lpr.*                          -/var/log/lpr.log
+mail.*                          -/var/log/mail.log
+#
+# Logging for the mail system.  Split it up so that
+# it is easy to write scripts to parse these files.
+#
+#mail.info                      -/var/log/mail.info
+#mail.warn                      -/var/log/mail.warn
+mail.err                        /var/log/mail.err
+
+#
+# Logging for INN news system.
+#
+news.crit                       /var/log/news/news.crit
+news.err                        /var/log/news/news.err
+news.notice                     -/var/log/news/news.notice
+
+#
+# Some "catch-all" log files.
+#
+#*.=debug;\
+#       auth,authpriv.none;\
+#       news.none;mail.none     -/var/log/debug
+#*.=info;*.=notice;*.=warn;\
+#       auth,authpriv.none;\
+#       cron,daemon.none;\
+#       mail,news.none          -/var/log/messages
+
+#
+# Emergencies are sent to everybody logged in.
+#
+*.emerg                                :omusrmsg:*
+
+#
+# I like to have messages displayed on the console, but only on a virtual
+# console I usually leave idle.
+#
+#daemon,mail.*;\
+#       news.=crit;news.=err;news.=notice;\
+#       *.=debug;*.=info;\
+#       *.=notice;*.=warn       /dev/tty8
+
+# The named pipe /dev/xconsole is for the `xconsole' utility.  To use it,
+# you must invoke `xconsole' with the `-file' option:
+#
+#    $ xconsole -file /dev/xconsole [...]
+#
+# NOTE: adjust the list below, or you'll go crazy if you have a reasonably
+#      busy site..
+#
+daemon.*;mail.*;\
+        news.err;\
+        *.=debug;*.=info;\
+        *.=notice;*.=warn       |/dev/xconsole
+

playbooks/roles/datadog/tasks/main.yml

@@ -16,7 +16,7 @@
 #
 
 - name: datadog | add apt key
-  shell: apt-key adv --keyserver keyserver.ubuntu.com --recv-keys C7A7DA52
+  apt_key: id=C7A7DA52 url={{datadog_apt_key}} state=present
   tags:
     - datadog
     - ubuntu
@@ -56,9 +56,9 @@
 - name: datadog | update api-key
   lineinfile: 
     dest="/etc/dd-agent/datadog.conf" 
-    "regexp=^api_key:$" 
-    "line=api_key:{{ common_datadog_api_key }}"
+    "regexp=^api_key:.*"
+    "line=api_key:{{ common_dd_api_key }}"
   notify:
     - datadog | restart the datadog service
   tags:
-    - datadog
+    - datadog
\ No newline at end of file

playbooks/roles/datadog/vars/main.yml

@@ -2,6 +2,8 @@
 
 datadog_api_key: "PUT_YOUR_API_KEY_HERE"
 
+datadog_apt_key: "http://keyserver.ubuntu.com/pks/lookup?op=get&search=0x226AE980C7A7DA52"
+
 datadog_debian_pkgs:
   - apparmor-utils
   - build-essential
@@ -10,4 +12,4 @@ datadog_debian_pkgs:
   - gcc
   - ipython
   - pkg-config
-  - rsyslog
+  - rsyslog
\ No newline at end of file

playbooks/roles/edxapp/tasks/deploy.yml

@@ -16,7 +16,7 @@
   - deploy
 
 # Do A Checkout
-- name: git checkout edx-platform repo into $app_base_dir
+- name: edxapp | checkout edx-platform repo into {{edx_platform_code_dir}}
   git: dest={{edx_platform_code_dir}} repo={{edx_platform_repo}} version={{edx_platform_commit}}
   tags:
   - lms
@@ -224,6 +224,17 @@
   - cms
   - syncdb
 
+- name: db migrate
+  shell: sudo -u www-data SERVICE_VARIANT=lms /opt/edx/bin/django-admin.py migrate --noinput --settings=lms.envs.aws --pythonpath=/opt/wwc/edx-platform
+  when: migrate_only is defined and migrate_only|lower == "yes"
+  tags:
+  - deploy
+  - lms
+  - lms-xml
+  - lms-preview
+  - cms
+  - syncdb
+
 - name: restart edxapp
   service: name=edxapp state=restarted
   when: celery_worker is not defined

playbooks/roles/edxapp/templates/edx-worker-cms.conf.j2

@@ -13,7 +13,7 @@ instance edx.${SERVICE_VARIANT}.core.${QUEUE}
 #env NEWRELIC={{venv_dir}}/bin/newrelic-admin
 env CONCURRENCY=${CONCURRENCY}
 env LOGLEVEL=info
-env DJANGO_SETTINGS_MODULE=cms.envs.aws
+env DJANGO_SETTINGS_MODULE={{worker_django_settings_module}}
 env PYTHONPATH={{edx_platform_code_dir}}
 env SERVICE_VARIANT=${SERVICE_VARIANT}
 
@@ -21,4 +21,4 @@ setuid www-data
 
 chdir {{edx_platform_code_dir}}
 
-exec {{venv_dir}}/bin/django-admin.py celery worker --settings=$DJANGO_SETTINGS_MODULE --pythonpath=$PYTHONPATH --loglevel=$LOGLEVEL --queues=edx.${SERVICE_VARIANT}.core.${QUEUE} --hostname=edx.${SERVICE_VARIANT}.core.${QUEUE}.`hostname` --concurrency=$CONCURRENCY
+exec {{venv_dir}}/bin/python {{edx_platform_code_dir}}/manage.py $SERVICE_VARIANT --settings=$DJANGO_SETTINGS_MODULE celery worker --loglevel=$LOGLEVEL --queues=edx.${SERVICE_VARIANT}.core.${QUEUE} --hostname=edx.${SERVICE_VARIANT}.core.${QUEUE}.`hostname` --concurrency=$CONCURRENCY

playbooks/roles/edxapp/templates/edx-worker-lms-xml.conf.j2

@@ -13,7 +13,7 @@ instance edx.${SERVICE_VARIANT}.core.${QUEUE}
 #env NEWRELIC={{venv_dir}}/bin/newrelic-admin
 env CONCURRENCY=${CONCURRENCY}
 env LOGLEVEL=info
-env DJANGO_SETTINGS_MODULE=lms.envs.aws
+env DJANGO_SETTINGS_MODULE={{worker_django_settings_module}}
 env PYTHONPATH={{edx_platform_code_dir}}
 env SERVICE_VARIANT=${SERVICE_VARIANT}
 
@@ -21,4 +21,4 @@ setuid www-data
 
 chdir {{edx_platform_code_dir}}
 
-exec {{venv_dir}}/bin/django-admin.py celery worker --settings=$DJANGO_SETTINGS_MODULE --pythonpath=$PYTHONPATH --loglevel=$LOGLEVEL --queues=edx.${SERVICE_VARIANT}.core.${QUEUE} --hostname=edx.${SERVICE_VARIANT}.core.${QUEUE}.`hostname` --concurrency=$CONCURRENCY
+exec {{venv_dir}}/bin/python {{edx_platform_code_dir}}/manage.py lms --service-variant=$SERVICE_VARIANT --settings=$DJANGO_SETTINGS_MODULE celery worker --loglevel=$LOGLEVEL --queues=edx.${SERVICE_VARIANT}.core.${QUEUE} --hostname=edx.${SERVICE_VARIANT}.core.${QUEUE}.`hostname` --concurrency=$CONCURRENCY

playbooks/roles/edxapp/templates/edx-worker-lms.conf.j2

@@ -13,7 +13,7 @@ instance edx.${SERVICE_VARIANT}.core.${QUEUE}
 #env NEWRELIC={{venv_dir}}/bin/newrelic-admin
 env CONCURRENCY=${CONCURRENCY}
 env LOGLEVEL=info
-env DJANGO_SETTINGS_MODULE=lms.envs.aws
+env DJANGO_SETTINGS_MODULE={{worker_django_settings_module}}
 env PYTHONPATH={{edx_platform_code_dir}}
 env SERVICE_VARIANT=${SERVICE_VARIANT}
 
@@ -21,4 +21,4 @@ setuid www-data
 
 chdir {{edx_platform_code_dir}}
 
-exec {{venv_dir}}/bin/django-admin.py celery worker --settings=$DJANGO_SETTINGS_MODULE --pythonpath=$PYTHONPATH --loglevel=$LOGLEVEL --queues=edx.${SERVICE_VARIANT}.core.${QUEUE} --hostname=edx.${SERVICE_VARIANT}.core.${QUEUE}.`hostname` --concurrency=$CONCURRENCY
+exec {{venv_dir}}/bin/python {{edx_platform_code_dir}}/manage.py lms --service-variant=$SERVICE_VARIANT --settings=$DJANGO_SETTINGS_MODULE celery worker --loglevel=$LOGLEVEL --queues=edx.${SERVICE_VARIANT}.core.${QUEUE} --hostname=edx.${SERVICE_VARIANT}.core.${QUEUE}.`hostname` --concurrency=$CONCURRENCY

playbooks/roles/edxapp/templates/lms.conf.j2

@@ -16,7 +16,7 @@ env WORKERS={{ worker_core_mult.lms }}
 {% endif %}
 env PORT={{edxapp_lms_app_port}}
 env LANG=en_US.UTF-8
-env DJANGO_SETTINGS_MODULE=lms.envs.aws
+env DJANGO_SETTINGS_MODULE={{ edxapp_lms_env }}
 env SERVICE_VARIANT="lms"
 
 chdir {{edx_platform_code_dir}}

playbooks/roles/edxapp/vars/main.yml

@@ -181,9 +181,9 @@ edxapp_lms_preview_app_port: 8020
 
 service_variants_enabled:
   - lms
-  - lms-xml
   - cms
-  - lms-preview
+
+edxapp_lms_env: 'lms.envs.aws'
 
 
 #Number of gunicorn worker processes to spawn, as a multiplier to number of virtual cores
@@ -307,3 +307,6 @@ deploy_environment:
   RBENV_ROOT: "{{ rbenv_root }}"
   GEM_HOME: "{{ gem_home }}"
   PATH: "{{ venv_dir }}/bin:{{ edx_platform_code_dir }}/bin:{{ rbenv_root }}/bin:{{ rbenv_root }}/shims:{{ gem_home }}/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+
+# Worker Settings
+worker_django_settings_module: 'aws'