consolidates and refactor common role

playbooks/group_vars/all

@@ -4,7 +4,7 @@
 
 data_dir: /edx/var
 app_dir: /edx/app
-log_base_dir: "{{ data_dir }}/log"
+log_dir: "{{ data_dir }}/log"
 os_name: ubuntu
 
 ENV_NAME: 'default_env'

playbooks/roles/common/defaults/main.yml

@@ -8,3 +8,10 @@ common_debian_pkgs:
   - tree
   - git
   - unzip
+  - python2.7
+  - python-pip
+  - python2.7-dev
+
+common_pip_pkgs:
+  - virtualenv
+  - virtualenvwrapper

playbooks/roles/common/tasks/create_venv.yml

----
-# create the 'edx' virtual environment in /opt so that roles can populate it
-- name: common | Install python and pip
-  apt: pkg={{item}} install_recommends=yes state=present update_cache=yes
-  with_items:
-  - python2.7
-  - python-pip
-  - python2.7-dev
-  tags:
-  - pre_install
-  - install
-
-- name: common | pip install virtualenv
-  pip: >
-    name=virtualenv 
-    state=present
-    extra_args="-i {{ PYPI_MIRROR_URL }}"
-  tags:
-  - venv_base
-  - install
-
-- name: common | pip install virtualenvwrapper
-  pip: >
-    name=virtualenvwrapper 
-    state=present
-    extra_args="-i {{ PYPI_MIRROR_URL }}"
-  tags:
-  - venv_base
-  - install
-
-- name: common | create edx virtualenv directory
-  file: path={{ venv_dir }} owner=ubuntu group=adm mode=2775 state=directory
-  tags:
-  - venv_base
-  - install
-
-- name: common | create the edx virtualenv directory initial contents
-  command: /usr/local/bin/virtualenv {{ venv_dir }} --distribute creates=$venv_dir/bin/activate
-  tags:
-  - venv_base
-  - install
-
-- name: common |  pip install gunicorn
-  pip: >
-    name=gunicorn 
-    virtualenv="{{venv_dir}}" 
-    state=present
-    extra_args="-i {{ PYPI_MIRROR_URL }}"
-  tags:
-  - gunicorn
-  - install

playbooks/roles/common/tasks/edx_logging_base.yml

----
-#- name: common | Install rsyslog configuration for ansible runs
-#  template: dest=/etc/rsyslog.d/90-edx.conf src=ansible_rsyslog.j2 owner=root group=root mode=644
-#  notify: common | restart rsyslogd
-#  tags:
-#  - lms-env
-#  - cms-env
-#  - logging
-#  - update
-
-- name: common | Install rsyslog configuration for edX
-  template: dest=/etc/rsyslog.d/99-edx.conf src=edx_rsyslog.j2 owner=root group=root mode=644
-  notify: common | restart rsyslogd
-  tags:
-  - logging
-  - update
-
-- name: common | Install logrotate configuration for edX
-  template: dest=/etc/logrotate.d/edx-services src=edx_logrotate.j2 owner=root group=root mode=644
-  tags:
-  - logging
-  - update
-
-- name: common | Touch tracking file into existence
-  command: touch -a {{log_base_dir}}/tracking.log creates={{log_base_dir}}/tracking.log
-  tags:
-  - logging
-  - update
-
-- name: common | Set permissions on tracking file
-  file: path={{log_base_dir}}/tracking.log owner=syslog group=adm mode=640
-  tags:
-  - logging
-  - update
-
-- name: common | Install logrotate configuration for tracking file
-  template: dest=/etc/logrotate.d/tracking.log src=edx_logrotate_tracking_log.j2 owner=root group=root mode=644
-  tags:
-  - logging
-  - update

playbooks/roles/common/tasks/main.yml

@@ -3,36 +3,17 @@
   # This is the default user for nginx
   user: name=www-data
 
-- name: common | Create the base directory for storage
+- name: common | Create common directories
   file: >
     path={{ data_dir }}
     state=directory
     owner=root
     group=root
     mode=0755
-
-- name: common | Create the base directory for the app
-  file: >
-    path={{ app_dir }}
-    state=directory
-    owner=root
-    group=root
-    mode=0755
-
-- name: common | Create upload directory
-  file: path={{ app_dir }}/uploads mode=2775 state=directory owner=root group=adm
-
-- name: common | Create data dir
-  file: path={{ app_dir }}/data state=directory owner=www-data group=root
-  tags:
-  - pre_install
-  - update
-
-- name: common | Create staticfiles dir
-  file: path={{ app_dir }}/staticfiles state=directory owner=www-data group=adm mode=2775
-  tags:
-  - pre_install
-  - update
+  with_items:
+    - "{{ data_dir }}"
+    - "{{ app_dir }}"
+    - "{{ log_dir }}"
 
 - name: common | Install role-independent useful system packages
   # do this before log dir setup; rsyslog package guarantees syslog user present
@@ -40,45 +21,37 @@
   tags:
   - pre_install
   - update
-
-- name: common | Create log directory
-  file: path={{log_base_dir}} state=directory mode=2755 group=adm owner=syslog
-  tags:
-  - pre_install
   - update
 
-- name: common | Create alias from app_dir to the log_base_dir
-  file: state=link src={{log_base_dir}} path={{app_dir}}/log
+- name: common | upload sudo config for key forwarding as root
+  copy: src=ssh_key_forward dest=/etc/sudoers.d/ssh_key_forward validate='visudo -c -f %s' owner=root group=root mode=0440
+
+- name: common | pip install virtualenv
+  pip: >
+    name="{{ item }}"
+    state=present
+    extra_args="-i {{ PYPI_MIRROR_URL }}"
+  with_items: common_pip_pkgs
+
+- name: common | Install rsyslog configuration for edX
+  template: dest=/etc/rsyslog.d/99-edx.conf src=edx_rsyslog.j2 owner=root group=root mode=644
+  notify: common | restart rsyslogd
   tags:
-  - pre_install
   - logging
   - update
+  notify: common | restart rsyslogd
 
-- name: common | Create convenience link from log_base_dir to system logs
-  file: state=link src=/var/log path=$log_base_dir/system
+- name: common | Install logrotate configuration for edX
+  template: dest=/etc/logrotate.d/edx-services src=edx_logrotate.j2 owner=root group=root mode=644
   tags:
-  - pre_install
   - logging
   - update
+  notify: common | restart logrotate
 
-- name: common | Touch edx log file into place
-  # This is done for the benefit of the rake commands, which expect it
-  command: touch -a {{log_base_dir}}/edx.log creates={{log_base_dir}}/edx.log
-  tags:
-  - pre_install
-  - logging
-  - install
 
-- name: common | Set permissions on edx log file
-  # This is done for the benefit of the rake commands, which expect it
-  file: path={{log_base_dir}}/edx.log owner=syslog group=adm mode=640
+- name: common | Install logrotate configuration for tracking file
+  template: dest=/etc/logrotate.d/tracking.log src=edx_logrotate_tracking_log.j2 owner=root group=root mode=644
   tags:
-  - pre_install
   - logging
   - update
-
-- name: common | upload sudo config for key forwarding as root
-  copy: src=ssh_key_forward dest=/etc/sudoers.d/ssh_key_forward validate='visudo -c -f %s' owner=root group=root mode=0440
-
-- include: create_venv.yml
-- include: edx_logging_base.yml
+  notify: common | restart logrotate

playbooks/roles/common/tasks/software_update.yml

----
-- name: common | edx-update.sh, manual lms/cms update script
-  template: src=edx-update.sh.j2 dest=/usr/local/bin/edx-update.sh owner=ubuntu group=adm mode=0775
-  tags:
-  - release
-  - update

playbooks/roles/common/templates/edx_logrotate.j2

-{{log_base_dir}}/*/edx.log {
+{{log_dir}}/*/edx.log {
   create
   compress
   copytruncate

playbooks/roles/common/templates/edx_logrotate_tracking_log.j2

-{{log_base_dir}}/tracking.log {
+{{log_dir}}/tracking.log {
   create
   compress
   delaycompress

playbooks/roles/common/templates/edx_rsyslog.j2

@@ -27,12 +27,12 @@ auth,authpriv.*                                         /var/log/auth.log
 $template tracking,"%syslogtag%%msg%\n"
 
 # looks for [service_name=<name>] in the beginning of the log message,
-# if it exists the log will go into {{log_base_dir}}/<name>/edx.log, otherwise
-# it will go into {{log_base_dir}}/edx.log
-$template DynaFile,"{{log_base_dir}}/%syslogtag:R,ERE,1,BLANK:\[service_variant=([a-zA-Z_-]*)\].*--end%/edx.log"
+# if it exists the log will go into {{log_dir}}/<name>/edx.log, otherwise
+# it will go into {{log_dir}}/edx.log
+$template DynaFile,"{{log_dir}}/%syslogtag:R,ERE,1,BLANK:\[service_variant=([a-zA-Z_-]*)\].*--end%/edx.log"
 
 local0.*                        -?DynaFile
-local1.*                        {{log_base_dir}}/tracking.log;tracking
+local1.*                        {{log_dir}}/tracking.log;tracking
 #cron.*                         /var/log/cron.log
 #daemon.*                       -/var/log/daemon.log
 kern.*                          -/var/log/kern.log

playbooks/roles/edxapp/defaults/main.yml

@@ -92,8 +92,11 @@ EDXAPP_LMS_PREVIEW_BASIC_AUTH: False
 #Use YAML references (& and *) and hash merge <<: to factor out shared settings
 #see http://atechie.net/2009/07/merging-hashes-in-yaml-conf-files/
 
-edxapp_data_dir:
-edxapp_app_dir:
+edxapp_data_dir: "{{ data_dir }}/edxapp"
+edxapp_app_dir: "{{ app_dir }}/edxapp"
+edxapp_log_dir: "{{ log_dir }}/edxapp"
+edxapp_user: edxapp
+
 edxapp_generic_auth_config:  &edxapp_generic_auth
   AWS_ACCESS_KEY_ID:  $EDXAPP_AWS_ACCESS_KEY_ID
   AWS_SECRET_ACCESS_KEY: $EDXAPP_AWS_SECRET_ACCESS_KEY
@@ -129,7 +132,6 @@ edxapp_generic_auth_config:  &edxapp_generic_auth
       OPTIONS:  *generic_modulestore_default_options
   DATABASES:
     default:
-      ENGINE: 'django.db.backends.mysql'
       NAME: $EDXAPP_MYSQL_DB_NAME
       USER: $EDXAPP_MYSQL_USER
       PASSWORD: $EDXAPP_MYSQL_PASSWORD
@@ -296,6 +298,7 @@ sandbox_post_requirements:  "{{ edx_platform_code_dir }}/requirements/edx-sandbo
 install_sandbox_reqs_into_regular_venv: true
 
 lms_debian_pkgs:
+  - npm
   # for compiling the virtualenv
   # (only needed if wheel files aren't available)
   - build-essential

playbooks/roles/edxapp/tasks/cms.yml

@@ -19,7 +19,7 @@
   - deploy
 
 - name: Create CMS log target directory
-  file: path={{log_base_dir}}/cms state=directory owner=syslog group=syslog mode=2750
+  file: path={{log_dir}}/cms state=directory owner=syslog group=syslog mode=2750
   tags:
   - cms 
   - cms-env

playbooks/roles/edxapp/tasks/lms-preview.yml

@@ -17,7 +17,7 @@
   - deploy
 
 - name: Create lms-preview log target directory
-  file: path={{log_base_dir}}/lms-preview state=directory owner=syslog group=syslog mode=2750
+  file: path={{log_dir}}/lms-preview state=directory owner=syslog group=syslog mode=2750
   tags:
   - lms-preview
   - lms-preview-env

playbooks/roles/edxapp/tasks/lms.yml

@@ -16,7 +16,7 @@
   - deploy
 
 - name: Create lms log target directory
-  file: path={{log_base_dir}}/lms state=directory owner=syslog group=syslog mode=2750
+  file: path={{log_dir}}/lms state=directory owner=syslog group=syslog mode=2750
   tags:
   - lms
   - lms-env

playbooks/roles/edxapp/tasks/main.yml

@@ -2,54 +2,41 @@
 #  - group_vars/all
 #  - common/tasks/main.yml
 ---
-- name: Change permissions on datadir
-  file: path={{ app_dir }}/data state=directory owner=www-data group=www-data
-  tags:
-  - cms
-  - lms
-  - lms-env
-  - update
+- name: edxapp | create application user
+  user: name="{{ edxapp_user }}"
 
-- name: Change owner on staticfiles
-  file: path={{ app_dir }}/staticfiles state=directory owner=www-data group=adm
-  tags:
-  - cms
-  - lms
-  - lms-env
-  - update
+- name: edxapp | create edxapp app dir
+  file: >
+    path="{{ item }}"
+    state=directory
+    owner=root
+    group="{{ edxapp_user }}"
+  with_items:
+    - "{{ edxapp_app_dir }}"
+    - "{{ edxapp_app_dir }}/venvs"
 
-- name: Create theming directory
-  file: path={{ app_dir }}/themes state=directory mode=2775 group=adm owner=www-data
-  tags:
-  - cms
-  - lms
-  - cms-env
-  - lms-env
-  - update
+- name: edxapp | create edxapp data dirs
+  file: >
+    path="{{ item }}"
+    state=directory
+    owner="{{ edxapp_user }}"
+  with_items:
+    - "{{ edxapp_log_dir }}"
+    - "{{ edxapp_data_dir }}/staticfiles"
+    - "{{ edxapp_data_dir }}/data"
+    - "{{ edxapp_data_dir }}/uploads"
+    - "{{ edxapp_data_dir }}/themes"
 
-- name: install a bunch of system packages on which LMS and CMS rely
+- name: edxapp | install system packages on which LMS and CMS rely
   apt: pkg={{','.join(lms_debian_pkgs)}} state=present
-  tags:
-  - lms
-  - cms
-  - install
 
-- name: creating edxapp upstart script
+- name: edxapp | creating edxapp upstart script
   sudo: True
   template: src=edxapp.conf.j2 dest=/etc/init/edxapp.conf owner=root group=root
   when: "celery_worker is not defined"
-  tags:
-  - upstart
-  - gunicorn
-  - update
 
-- name: create edx-workers upstart script
+- name: edxapp | create edx-workers upstart script
   template: src=edx-workers.conf.j2 dest=/etc/init/edx-workers.conf owner=root group=root
   when: "celery_worker is defined"
-  tags:
-  - upstart
-  - update
 
-- include: npm.yml
-- include: ruby.yml
 - include: deploy.yml

playbooks/roles/edxapp/tasks/npm.yml

-# requires:
-#   - common/tasks/main.yml
-#   - ruby/tasks/main.yml
----
-- name: Install npm
-  apt: pkg=npm state=present install_recommends=no
-  tags:
-  - npm
-  - install

playbooks/roles/edxapp/tasks/ruby.yml

-#
-#cribbed from https://github.com/mmoya/ansible-playbooks/blob/master/rbenv/main.yml
-- name: Create 'www' user (replicating historical environment)
-  user: name=www state=present
-  tags:
-  - ruby
-  - update
-
-- name: Create ruby base
-  file: path=$ruby_base state=directory owner=www group=www
-  tags:
-  - ruby
-  - update
-
-- name: rbenv | install build depends
-  apt: pkg=$item state=present install_recommends=no
-  with_items:
-    - build-essential
-    - git
-    - libcurl4-openssl-dev
-    - libmysqlclient-dev
-    - libreadline-dev
-    - libssl-dev
-    - libxml2-dev
-    - libxslt1-dev
-    - zlib1g-dev
-  tags:
-  - ruby
-  - install
-
-- name: rbenv | update rbenv repo
-  git: repo=https://github.com/sstephenson/rbenv.git dest=$rbenv_root version=v0.4.0
-  tags:
-  - ruby
-  - install
-
-- name: rbenv | add rbenv to path
-  file: path=/usr/local/bin/rbenv src=${rbenv_root}/bin/rbenv state=link
-  tags:
-  - ruby
-  - update
-
-- name: rbenv | add rbenv initialization to profile
-  template: src=rbenv.sh.j2 dest=/etc/profile.d/rbenv.sh owner=root group=root mode=0755
-  tags:
-  - ruby
-  - update
-
-- name: rbenv | check ruby-build installed
-  command: test -x /usr/local/bin/ruby-build
-  register: rbuild_present
-  ignore_errors: yes
-  tags:
-  - ruby
-  - install
-
-- name: rbenv | create temporary directory
-  command: mktemp -d
-  register: tempdir
-  when: rbuild_present|failed
-  tags:
-  - ruby
-  - install
-
-- name: rbenv | clone ruby-build repo
-  git: repo=https://github.com/sstephenson/ruby-build.git dest=${tempdir.stdout}/ruby-build
-  when: rbuild_present|failed
-  tags:
-  - ruby
-  - install
-
-- name: rbenv | install ruby-build
-  command: ./install.sh chdir=${tempdir.stdout}/ruby-build
-  when: rbuild_present|failed
-  tags:
-  - ruby
-  - install
-
-- name: rbenv | remove temporary directory
-  file: path=${tempdir.stdout} state=absent
-  when: rbuild_present|failed
-  tags:
-  - ruby
-  - install
-
-- name: rbenv | check ruby $ruby_version installed
-  shell: RBENV_ROOT=${rbenv_root} rbenv versions | grep $ruby_version
-  register: ruby_installed
-  ignore_errors: yes
-  tags:
-  - ruby
-  - install
-
-- name: rbenv | install ruby $ruby_version
-  shell: RBENV_ROOT=${rbenv_root} rbenv install $ruby_version
-  when: ruby_installed|failed
-  tags:
-  - ruby
-  - install
-
-- name: rbenv | set global ruby $ruby_version
-  shell: RBENV_ROOT=${rbenv_root} rbenv global $ruby_version
-  when: ruby_installed|failed
-  tags:
-  - ruby
-  - install
-
-- name: rbenv | rehash
-  shell: RBENV_ROOT=${rbenv_root} rbenv rehash
-  when: ruby_installed|failed
-  tags:
-  - ruby
-  - install
-

playbooks/roles/edxapp/tasks/upstart.yml

-# write the gunicorn upstart script for {{ service_variant }}
-- name: writing {{ basename }} upstart script to /etc/init
-  sudo: True
-  template: src={{ basename }}.conf.j2 dest=/etc/init/{{ basename }}.conf owner=root group=root
-  tags:
-  - upstart
-  - gunicorn
-  - update
-  - deploy

playbooks/roles/nginx/tasks/main.yml

@@ -49,7 +49,7 @@
   - update
 
 - name: nginx | Create nginx log file location (just in case)
-  file: path={{log_base_dir}}/nginx state=directory owner=syslog group=syslog mode=2770 recurse=yes
+  file: path={{log_dir}}/nginx state=directory owner=syslog group=syslog mode=2770 recurse=yes
   tags:
   - nginx
   - logging

playbooks/roles/nginx/templates/cms.j2

@@ -11,8 +11,8 @@ server {
 
   server_name studio.*;
 
-  access_log {{log_base_dir}}/nginx/access.log;
-  error_log {{log_base_dir}}/nginx/error.log error;
+  access_log {{log_dir}}/nginx/access.log;
+  error_log {{log_dir}}/nginx/error.log error;
 
   # CS184 requires uploads of up to 4MB for submitting screenshots. 
   # CMS requires larger value for course assest, values provided 

playbooks/roles/nginx/templates/edx_logrotate_nginx_access.j2

 # Put in place by ansible
 
-{{log_base_dir}}/nginx/access.log {
+{{log_dir}}/nginx/access.log {
   create 0640 www-data adm
   compress
   delaycompress

playbooks/roles/nginx/templates/edx_logrotate_nginx_error.j2

 # Put in place by ansible
 
-{{log_base_dir}}/nginx/error.log {
+{{log_dir}}/nginx/error.log {
   create 0640 www-data adm
   compress
   delaycompress

playbooks/roles/nginx/templates/lms.j2

@@ -9,8 +9,8 @@ server {
 
   listen {{EDXAPP_LMS_NGINX_PORT}} default;
 
-  access_log {{log_base_dir}}/nginx/access.log;
-  error_log {{log_base_dir}}/nginx/error.log error;
+  access_log {{log_dir}}/nginx/access.log;
+  error_log {{log_dir}}/nginx/error.log error;
 
   # CS184 requires uploads of up to 4MB for submitting screenshots.
   # CMS requires larger value for course assest, values provided

playbooks/roles/splunkforwarder/defaults/main.yml

@@ -23,7 +23,7 @@ SPLUNKFORWARDER_DEB: !!null
 SPLUNKFORWARDER_PASSWORD: !!null
 
 SPLUNKFORWARDER_LOG_ITEMS:
-  - directory: '{{log_base_dir}}'
+  - directory: '{{log_dir}}'
     recursive: true
     index: '{{ENV_TYPE}}-{{ENV_NAME}}'
     sourcetype: 'edx'
@@ -31,7 +31,7 @@ SPLUNKFORWARDER_LOG_ITEMS:
     recursive: true
     index: '{{ENV_TYPE}}-{{ENV_NAME}}'
     sourcetype: 'syslog'
-  - directory: '{{log_base_dir}}/nginx'
+  - directory: '{{log_dir}}/nginx'
     recursive: true
     index: '{{ENV_TYPE}}-{{ENV_NAME}}'
     sourcetype: 'nginx'

playbooks/secure_example/vars/edxapp_ref_vars.yml

@@ -178,7 +178,7 @@ cms_env_config:
       'KEY_PREFIX':  'hidden-prod'
       'LOCATION': [ 'hidden-prod',
         'hidden-prod']
-  'LOG_DIR': '{{log_base_dir}}/edx'
+  'LOG_DIR': '{{log_dir}}/edx'
   'LOGGING_ENV': 'cms-dev'
   'SITE_NAME': 'studio.cms-dev.m.edx.org'
   'SYSLOG_SERVER': 'syslog.a.m.i4x.org'