Merge pull request #408 from edx/jarv/edxapp-cleanup-and-certs

Some general cleanup of the edxapp role, adding certs to continuous integration

playbooks/edx-east/edx_continuous_integration.yml

@@ -30,3 +30,4 @@
     - xserver
     - ora
     - discern
+    - certs

playbooks/roles/edxapp/defaults/main.yml

@@ -138,13 +138,28 @@ edxapp_workers:
     service_variant: lms
     concurrency: 2
 
+# Requirement files we explicitely
+# check for changes before attempting
+# to update the venv
+edxapp_chksum_req_files:
+  - "{{ pre_requirements_file }}"
+  - "{{ post_requirements_file }}"
+  - "{{ base_requirements_file }}"
+  - "{{ sandbox_post_requirements }}"
+  - "{{ sandbox_base_requirements }}"
+
+# all edxapp requirements files
 edxapp_all_req_files:
   - "{{ pre_requirements_file }}"
   - "{{ post_requirements_file }}"
   - "{{ base_requirements_file }}"
+  - "{{ repo_requirements_file }}"
+  - "{{ github_requirements_file }}"
   - "{{ sandbox_post_requirements }}"
+  - "{{ sandbox_local_requirements }}"
   - "{{ sandbox_base_requirements }}"
 
+
 # TODO: old style variable syntax is necessary
 # for lists and dictionaries
 

playbooks/roles/edxapp/tasks/deploy.yml

@@ -13,6 +13,7 @@
     state=stopped
   when: celery_worker is not defined
   with_items: service_variants_enabled
+  sudo_user: "{{ common_web_user }}"
   tags:
     - deploy
 
@@ -24,6 +25,7 @@
     state=stopped
   when: celery_worker is defined
   with_items: edxapp_workers
+  sudo_user: "{{ common_web_user }}"
   tags:
     - deploy
 
@@ -48,7 +50,7 @@
 
 - name: edxapp | create checksum for requirements, package.json and Gemfile
   shell: >
-    /usr/bin/md5sum {{ " ".join(edxapp_all_req_files) }} 2>/dev/null > /var/tmp/edxapp.req.new
+    /usr/bin/md5sum {{ " ".join(edxapp_chksum_req_files) }} 2>/dev/null > /var/tmp/edxapp.req.new
   sudo_user: "{{ edxapp_user }}"
   ignore_errors: true
   tags:
@@ -56,22 +58,22 @@
 
 - stat: path=/var/tmp/edxapp.req.new
   register: new
+  sudo_user: "{{ edxapp_user }}"
   tags: deploy
 
 - stat: path=/var/tmp/edxapp.req.installed
   register: inst
+  sudo_user: "{{ edxapp_user }}"
   tags: deploy
 
 # Substitute github mirror in all requirements files
+# This is run on every single deploy
 - name: edxapp | Updating requirement files for git mirror
   command: |
-    /bin/sed -i -e 's/github\.com/{{ COMMON_GIT_MIRROR }}/g' {{ item }}
-  with_items: edxapp_all_req_files
+    /bin/sed -i -e 's/github\.com/{{ COMMON_GIT_MIRROR }}/g' {{ " ".join(edxapp_all_req_files) }}
   sudo_user: "{{ edxapp_user }}"
-  when: not inst.stat.exists or new.stat.md5 != inst.stat.md5
   tags: deploy
 
-
 # Ruby plays that need to be run after platform updates.
 - name: edxapp | gem install bundler
   shell: >
@@ -165,10 +167,17 @@
   when: not inst.stat.exists or new.stat.md5 != inst.stat.md5
   tags: deploy
 
-# For pip packages which create group-restricted venv files (httplib2)
-- name: edxapp | ensure all files are readable by any user
-  shell: chmod -R go+r {{ edxapp_venv_dir }}
-  when: not inst.stat.exists or new.stat.md5 != inst.stat.md5
+- name: edxapp | compiling all py files in the edx-platform repo
+  shell: "{{ edxapp_venv_bin }}/python -m compileall {{ edxapp_code_dir }}"
+  sudo_user: "{{ edxapp_user }}"
+  tags: deploy
+
+  # alternative would be to give {{ common_web_user }} read access
+  # to the virtualenv but that permission change will require
+  # root access.
+- name: edxapp | give other read permissions to the virtualenv
+  command: chmod -R o+r "{{ edxapp_venv_dir }}"
+  sudo_user: "{{ edxapp_user }}"
   tags: deploy
 
 - name: edxapp | create checksum for installed requirements
@@ -176,9 +185,6 @@
   sudo_user: "{{ edxapp_user }}"
   tags: deploy
 
-- name: edxapp | compiling all py files in the edx-platform repo
-  shell: "{{ edxapp_venv_bin }}/python -m compileall {{ edxapp_code_dir }}"
-  tags: deploy
 
 # https://code.launchpad.net/~wligtenberg/django-openid-auth/mysql_fix/+merge/22726
 # This is necessary for when syncdb is run and the django_openid_auth module is installed,
@@ -187,6 +193,7 @@
 - name: edxapp | openid workaround - NOT FOR PRODUCTION
   shell: sed -i -e 's/claimed_id = models.TextField(max_length=2047, unique=True/claimed_id = models.TextField(max_length=2047/' {{ edxapp_venv_dir }}/lib/python2.7/site-packages/django_openid_auth/models.py
   when: openid_workaround is defined
+  sudo_user: "{{ edxapp_user }}"
   tags:
   - deploy
 
@@ -203,6 +210,7 @@
     state=started
   when: celery_worker is not defined
   with_items: service_variants_enabled
+  sudo_user: "{{ common_web_user }}"
   tags:
     - deploy
 
@@ -214,5 +222,6 @@
     state=started
   when: celery_worker is defined
   with_items: edxapp_workers
+  sudo_user: "{{ common_web_user }}"
   tags:
     - deploy

playbooks/roles/edxapp/tasks/main.yml

@@ -18,6 +18,7 @@
     owner="{{ edxapp_user }}" group="{{ common_web_group }}"
   with_items:
     - "{{ edxapp_app_dir }}"
+    - "{{ edxapp_data_dir }}"
     - "{{ edxapp_venvs_dir }}"
     - "{{ edxapp_theme_dir }}"
     - "{{ edxapp_staticfile_dir }}"

playbooks/roles/edxapp/tasks/service_variant_config.yml

@@ -22,19 +22,21 @@
 - name: "writing {{ item }} supervisor script"
   template: >
     src={{ item }}.conf.j2 dest={{ supervisor_cfg_dir }}/{{ item }}.conf
-    owner={{ supervisor_user }} group={{ common_web_user }} mode=0644
+    owner={{ supervisor_user }}
   with_items: service_variants_enabled
   when: celery_worker is not defined
   notify: supervisor | reload supervisor
+  sudo_user: "{{ supervisor_user }}"
   tags:
   - deploy
 
-- name: "writing edxapp supervisor script"
+- name: edxapp | writing edxapp supervisor script
   template: >
     src=edxapp.conf.j2 dest={{ supervisor_cfg_dir }}/edxapp.conf
-    owner={{ supervisor_user }} group={{ common_web_user }} mode=0644
+    owner={{ supervisor_user }}
   when: celery_worker is not defined
   notify: supervisor | reload supervisor
+  sudo_user: "{{ supervisor_user }}"
   tags:
   - deploy
 
@@ -50,12 +52,13 @@
 
 # write the supervisor script for celery workers
 
-- name: writing celery worker supervisor script
+- name: edxapp | writing celery worker supervisor script
   template: >
     src=workers.conf.j2 dest={{ supervisor_cfg_dir }}/workers.conf
-    owner={{ supervisor_user }} group={{ common_web_user }} mode=0644
+    owner={{ supervisor_user }}
   when: celery_worker is defined
   notify: supervisor | reload supervisor
+  sudo_user: "{{ supervisor_user }}"
   tags:
   - deploy
 
@@ -85,14 +88,16 @@
 
 
 - name: edxapp | syncdb and migrate
-  shell: sudo -u {{ edxapp_user }} SERVICE_VARIANT=lms {{ edxapp_venv_bin}}/django-admin.py syncdb --migrate --noinput --settings=lms.envs.aws --pythonpath={{ edxapp_code_dir }}
+  shell: SERVICE_VARIANT=lms {{ edxapp_venv_bin}}/django-admin.py syncdb --migrate --noinput --settings=lms.envs.aws --pythonpath={{ edxapp_code_dir }}
   when: migrate_db is defined and migrate_db|lower == "yes"
+  sudo_user: "{{ edxapp_user }}"
   tags:
   - deploy
 
 - name: edxapp | db migrate
-  shell: sudo -u {{ edxapp_user }} SERVICE_VARIANT=lms {{ edxapp_venv_bin }}/django-admin.py migrate --noinput --settings=lms.envs.aws --pythonpath={{ edxapp_code_dir }}
+  shell: SERVICE_VARIANT=lms {{ edxapp_venv_bin }}/django-admin.py migrate --noinput --settings=lms.envs.aws --pythonpath={{ edxapp_code_dir }}
   when: migrate_only is defined and migrate_only|lower == "yes"
+  sudo_user: "{{ edxapp_user }}"
   tags:
   - deploy