Changes to unify edxapp and worker AMIs.

Always write the supervisor files and link them as necessary.

Drop the configs in the supervisor available directory. Instead of having a
seperate cfg dir for each service.

Seperate devstack logic from disabling supervisor tasks. Add the new flag so
devstack continues to not startup supervisor. Use new supervisor_available_dir
everywhere.

Remove mongo logic from abbey and remove stage_release. This should completely
remove our tie ins to mongo.

Also removing monitor repos script which depended on stage_release. We can add
in similar logic when we're ready to automate building the AMIs on rc branch
creation.

Updated requirements.

playbooks/roles/certs/handlers/main.yml

@@ -20,4 +20,4 @@
     supervisorctl_path={{ supervisor_ctl }}
     config={{ supervisor_cfg }}
     state=restarted
-  when: certs_installed is defined
+  when: certs_installed is defined and not disable_edx_services

playbooks/roles/certs/tasks/deploy.yml

@@ -16,9 +16,19 @@
 
 - name: writing supervisor script for certificates
   template: >
-    src=certs.conf.j2 dest={{ supervisor_cfg_dir }}/certs.conf
+    src=certs.conf.j2 dest={{ supervisor_available_dir }}/certs.conf
     owner={{ supervisor_user }} mode=0644
+
+- name: enable supervisor script for certificates
+  file: >
+    src={{ supervisor_available_dir }}/certs.conf
+    dest={{ supervisor_cfg_dir }}/certs.conf
+    owner={{ supervisor_user }}
+    state=link
+    force=yes
+    mode=0644
   notify: restart certs
+  when: not disable_edx_services
 
 - name: create ssh script for git
   template: >
@@ -58,6 +68,7 @@
   register: supervisor_update
   sudo_user: "{{ supervisor_service_user }}"
   changed_when: supervisor_update.stdout != ""
+  when: not disable_edx_services
 
 - name: ensure certs has started
   supervisorctl_local: >
@@ -66,6 +77,7 @@
     config={{ supervisor_cfg }}
     state=started
   sudo_user: "{{ supervisor_service_user }}"
+  when: not disable_edx_services
 
 - name: create a symlink for venv python
   file: >

playbooks/roles/common/defaults/main.yml

@@ -72,7 +72,14 @@ common_log_user: syslog
 common_git_ppa: "ppa:git-core/ppa"
 
 # Skip supervisor tasks
-# Useful when supervisor is not installed (local dev)
+# When set to true this flag will allow you to install everything but keep
+# supervisor from starting any of the services.
+# Service files will be placed in supervisor's conf.available.d but not linked
+# to supervisors 'conf.d' directory.
+disable_edx_services: False
+
+# Some apps run differently in dev mode(forums)
+# so different start scripts are generated in dev mode.
 devstack: False
 
 common_debian_variants:

playbooks/roles/discern/handlers/main.yml

@@ -9,3 +9,4 @@
   with_items:
   - discern
   - discern_celery
+  when: not disable_edx_services

playbooks/roles/discern/tasks/deploy.yml

@@ -2,11 +2,23 @@
 
 - name: create supervisor scripts - discern, discern_celery
   template: >
-    src={{ item }}.conf.j2 dest={{ supervisor_cfg_dir }}/{{ item }}.conf
+    src={{ item }}.conf.j2 dest={{ supervisor_available_dir }}/{{ item }}.conf
     owner={{ supervisor_user }} mode=0644
   sudo_user: "{{ supervisor_user }}"
   with_items: ['discern', 'discern_celery']
 
+- name: enable supervisor scripts - discern, discern_celery
+  file: >
+    dest={{ supervisor_available_dir }}/{{ item }}.conf
+    dest={{ supervisor_cfg_dir }}/{{ item }}.conf
+    owner={{ supervisor_user }}
+    state=link
+    force=yes
+    mode=0644
+  sudo_user: "{{ supervisor_user }}"
+  with_items: ['discern', 'discern_celery']
+  when: not disable_edx_services
+
 #Upload config files for django (auth and env)
 - name: create discern application config env.json file
   template: src=env.json.j2 dest={{ discern_app_dir }}/env.json
@@ -104,6 +116,7 @@
   register: supervisor_update
   sudo_user: "{{ supervisor_service_user }}"
   changed_when: supervisor_update.stdout != ""
+  when: not disable_edx_services
 
 - name: ensure discern, discern_celery has started
   supervisorctl_local: >
@@ -114,6 +127,7 @@
   with_items:
   - discern
   - discern_celery
+  when: not disable_edx_services
 
 - name: create a symlink for venv python
   file: >

playbooks/roles/edxapp/handlers/main.yml

@@ -5,7 +5,7 @@
     supervisorctl_path={{ supervisor_ctl }}
     config={{ supervisor_cfg }}
     name="edxapp:{{ item }}"
-  when: edxapp_installed is defined and celery_worker is not defined and not devstack
+  when: edxapp_installed is defined and celery_worker is not defined and not disable_edx_services
   sudo_user: "{{ supervisor_service_user }}"
   with_items: service_variants_enabled
 
@@ -15,6 +15,6 @@
     supervisorctl_path={{ supervisor_ctl }}
     config={{ supervisor_cfg }}
     state=restarted
-  when: edxapp_installed is defined and celery_worker is defined and not devstack
+  when: edxapp_installed is defined and celery_worker is defined and not disable_edx_services
   with_items: edxapp_workers
   sudo_user: "{{ common_web_user }}"

playbooks/roles/edxapp/tasks/deploy.yml

@@ -300,7 +300,7 @@
   register: supervisor_update
   sudo_user: "{{ supervisor_service_user }}"
   changed_when: supervisor_update.stdout != ""
-  when: not devstack
+  when: not disable_edx_services
 
 - name: ensure edxapp has started
   supervisorctl_local: >
@@ -309,7 +309,7 @@
     config={{ supervisor_cfg }}
     name="edxapp:{{ item }}"
   sudo_user: "{{ supervisor_service_user }}"
-  when: celery_worker is not defined and not devstack
+  when: celery_worker is not defined and not disable_edx_services
   with_items: service_variants_enabled
 
 - name: ensure edxapp_workers has started
@@ -318,7 +318,7 @@
     supervisorctl_path={{ supervisor_ctl }}
     config={{ supervisor_cfg }}
     state=started
-  when: celery_worker is defined and not devstack
+  when: celery_worker is defined and not disable_edx_services
   with_items: edxapp_workers
   sudo_user: "{{ supervisor_service_user }}"
 

playbooks/roles/edxapp/tasks/service_variant_config.yml

@@ -24,26 +24,55 @@
 
 - name: "writing {{ item }} supervisor script"
   template: >
-    src={{ item }}.conf.j2 dest={{ supervisor_cfg_dir }}/{{ item }}.conf
+    src={{ item }}.conf.j2 dest={{ supervisor_available_dir }}/{{ item }}.conf
     owner={{ supervisor_user }}
+    group={{ supervisor_user }}
   with_items: service_variants_enabled
-  when: celery_worker is not defined and not devstack
   sudo_user: "{{ supervisor_user }}"
 
 - name: writing edxapp supervisor script
   template: >
-    src=edxapp.conf.j2 dest={{ supervisor_cfg_dir }}/edxapp.conf
+    src=edxapp.conf.j2 dest={{ supervisor_available_dir }}/edxapp.conf
     owner={{ supervisor_user }}
-  when: celery_worker is not defined and not devstack
+    group={{ supervisor_user }}
   sudo_user: "{{ supervisor_user }}"
 
 # write the supervisor script for celery workers
 
 - name: writing celery worker supervisor script
   template: >
-    src=workers.conf.j2 dest={{ supervisor_cfg_dir }}/workers.conf
+    src=workers.conf.j2 dest={{ supervisor_available_dir }}/workers.conf
     owner={{ supervisor_user }}
-  when: celery_worker is defined and not devstack
+    group={{ supervisor_user }}
+  sudo_user: "{{ supervisor_user }}"
+
+# Enable the supervisor jobs
+- name: "enable {{ item }} supervisor script"
+  file: >
+    src={{ supervisor_available_dir }}/{{ item }}.conf
+    dest={{ supervisor_cfg_dir }}/{{ item }}.conf
+    state=link
+    force=yes
+  with_items: service_variants_enabled
+  when: celery_worker is not defined and not disable_edx_services
+  sudo_user: "{{ supervisor_user }}"
+
+- name: "enable edxapp supervisor script"
+  file: >
+    src={{ supervisor_available_dir }}/edxapp.conf
+    dest={{ supervisor_cfg_dir }}/edxapp.conf
+    state=link
+    force=yes
+  when: celery_worker is not defined and not disable_edx_services
+  sudo_user: "{{ supervisor_user }}"
+
+- name: "enable celery worker supervisor script"
+  file: >
+    src={{ supervisor_available_dir }}/workers.conf
+    dest={{ supervisor_cfg_dir }}/workers.conf
+    state=link
+    force=yes
+  when: celery_worker is defined and not disable_edx_services
   sudo_user: "{{ supervisor_user }}"
 
 # Fake syncdb with migrate, only when fake_migrations is defined

playbooks/roles/edxlocal/tasks/main.yml

@@ -31,14 +31,14 @@
     name={{ XQUEUE_MYSQL_USER }}
     password={{ XQUEUE_MYSQL_PASSWORD }}
     priv='{{XQUEUE_MYSQL_DB_NAME}}.*:ALL'
-  when: XQUEUE_MYSQL_USER is defined and not devstack
+  when: XQUEUE_MYSQL_USER is defined and not disable_edx_services
 
 - name: create a database for xqueue
   mysql_db: >
     db=xqueue
     state=present
     encoding=utf8
-  when: XQUEUE_MYSQL_USER is defined and not devstack
+  when: XQUEUE_MYSQL_USER is defined and not disable_edx_services
 
 - name: setup the ora db user
   mysql_user: >
@@ -58,7 +58,7 @@
     name={{ DISCERN_MYSQL_USER }}
     password={{ DISCERN_MYSQL_PASSWORD }}
     priv='{{DISCERN_MYSQL_DB_NAME}}.*:ALL'
-  when: DISCERN_MYSQL_USER is defined and not devstack
+  when: DISCERN_MYSQL_USER is defined and not disable_edx_services
 
 
 - name: create a database for discern
@@ -66,7 +66,7 @@
     db=discern
     state=present
     encoding=utf8
-  when: DISCERN_MYSQL_USER is defined and not devstack
+  when: DISCERN_MYSQL_USER is defined and not disable_edx_services
 
 
 - name: install memcached

playbooks/roles/forum/handlers/main.yml

@@ -5,4 +5,4 @@
     supervisorctl_path={{ supervisor_ctl }}
     config={{ supervisor_cfg }}
     state=restarted
-  when: forum_installed is defined and not devstack
+  when: forum_installed is defined and not disable_edx_services

playbooks/roles/forum/tasks/deploy.yml

@@ -2,11 +2,23 @@
 
 - name: create the supervisor config
   template: >
-    src=forum.conf.j2 dest={{ supervisor_cfg_dir }}/forum.conf
+    src=forum.conf.j2 dest={{ supervisor_available_dir }}/forum.conf
     owner={{ supervisor_user }}
+    group={{ supervisor_user }}
     mode=0644
   sudo_user: "{{ supervisor_user }}"
-  when: not devstack
+  register: forum_supervisor
+
+- name: enable the supervisor config
+  file: >
+    src={{ supervisor_available_dir }}/forum.conf
+    dest={{ supervisor_cfg_dir }}/forum.conf
+    owner={{ supervisor_user }}
+    state=link
+    force=yes
+    mode=0644
+  sudo_user: "{{ supervisor_user }}"
+  when: not disable_edx_services
   register: forum_supervisor
 
 - name: create the supervisor wrapper
@@ -15,7 +27,6 @@
     dest={{ forum_supervisor_wrapper }}
     mode=0755
   sudo_user: "{{ forum_user }}"
-  when: not devstack
   notify: restart the forum service
 
 - name:  git checkout forum repo into {{ forum_code_dir }}
@@ -41,7 +52,7 @@
   shell:  "{{ supervisor_ctl }} -c {{ supervisor_cfg }} update"
   register: supervisor_update
   changed_when: supervisor_update.stdout != ""
-  when: not devstack
+  when: not disable_edx_services
 
 - name: ensure forum is started
   supervisorctl_local: >
@@ -49,7 +60,7 @@
     supervisorctl_path={{ supervisor_ctl }}
     config={{ supervisor_cfg }}
     state=started
-  when: not devstack
+  when: not disable_edx_services
 
 - include: test.yml tags=deploy
 

playbooks/roles/forum/tasks/test.yml

@@ -3,9 +3,9 @@
 - name: test that the required service are listening
   wait_for: port={{ item.port }} host={{ item.host }} timeout=30
   with_items: forum_services
-  when: not devstack
+  when: not disable_edx_services
 
 - name: test that mongo replica set members are listing
   wait_for: port={{ FORUM_MONGO_PORT }} host={{ item }} timeout=30
   with_items: FORUM_MONGO_HOSTS
-  when: not devstack
\ No newline at end of file
+  when: not disable_edx_services

playbooks/roles/nginx/templates/forum.j2

@@ -49,4 +49,4 @@ location @proxy_to_app {
     proxy_redirect off;
     proxy_pass http://forum_app_server;
   }
-}
\ No newline at end of file
+}

playbooks/roles/notifier/handlers/main.yml

@@ -6,6 +6,7 @@
     state=restarted
     config={{ supervisor_cfg }}
     supervisorctl_path={{ supervisor_ctl }}
+  when: not disable_edx_services
 
 - name: restart notifier-celery-workers
   supervisorctl_local: >
@@ -13,3 +14,4 @@
     state=restarted
     config={{ supervisor_cfg }}
     supervisorctl_path={{ supervisor_ctl }}
+  when: not disable_edx_services

playbooks/roles/notifier/tasks/main.yml

@@ -85,18 +85,36 @@
   file:
     path="{{ NOTIFIER_HOME }}/bin" mode=2775 state=directory owner={{ NOTIFIER_USER }} group={{ NOTIFIER_USER }}
 
-- name: supervisord config for celery workers
+- name: write supervisord config for celery workers
   template: >
     src=edx/app/supervisor/conf.d/notifier-celery-workers.conf.j2
-    dest="{{ supervisor_cfg_dir }}/notifier-celery-workers.conf"
+    dest="{{ supervisor_available_dir }}/notifier-celery-workers.conf"
   sudo_user: "{{ supervisor_user }}"
-  notify: restart notifier-celery-workers
 
-- name: supervisord config for scheduler
+- name: write supervisord config for scheduler
   template: >
     src=edx/app/supervisor/conf.d/notifier-scheduler.conf.j2
+    dest="{{ supervisor_available_dir }}/notifier-scheduler.conf"
+  sudo_user: "{{ supervisor_user }}"
+
+- name: enable supervisord config for celery workers
+  file: >
+    src="{{ supervisor_available_dir }}/notifier-celery-workers.conf"
+    dest="{{ supervisor_cfg_dir }}/notifier-celery-workers.conf"
+    state=link
+    force=yes
+  sudo_user: "{{ supervisor_user }}"
+  notify: restart notifier-celery-workers
+  when: not disable_edx_services
+
+- name: enable supervisord config for scheduler
+  file: >
+    src="{{ supervisor_available_dir }}/notifier-scheduler.conf"
     dest="{{ supervisor_cfg_dir }}/notifier-scheduler.conf"
+    state=link
+    force=yes
   sudo_user: "{{ supervisor_user }}"
   notify: restart notifier-scheduler
+  when: not disable_edx_services
 
 - include: deploy.yml tags=deploy

playbooks/roles/ora/handlers/main.yml

@@ -5,7 +5,7 @@
     supervisorctl_path={{ supervisor_ctl }}
     config={{ supervisor_cfg }}
     state=restarted
-  when: ora_installed is defined and not devstack
+  when: ora_installed is defined and not disable_edx_services
 
 - name: restart ora_celery
   supervisorctl_local: >
@@ -13,4 +13,4 @@
     supervisorctl_path={{ supervisor_ctl }}
     config={{ supervisor_cfg }}
     state=restarted
-  when: ora_installed is defined and not devstack
+  when: ora_installed is defined and not disable_edx_services

playbooks/roles/ora/tasks/deploy.yml

 - name: create supervisor scripts - ora, ora_celery
   template: >
-    src={{ item }}.conf.j2 dest={{ supervisor_cfg_dir }}/{{ item }}.conf
+    src={{ item }}.conf.j2 dest={{ supervisor_available_dir }}/{{ item }}.conf
     owner={{ supervisor_user }} group={{ common_web_user }} mode=0644
+  with_items: ['ora', 'ora_celery']
+
+- name: enable supervisor scripts - ora, ora_celery
+  file: >
+    src={{ supervisor_available_dir }}/{{ item }}.conf
+    dest={{ supervisor_cfg_dir }}/{{ item }}.conf
+    state=link
+    force=yes
+    owner={{ supervisor_user }}
+    group={{ common_web_user }}
+    mode=0644
   notify:
     - restart ora
     - restart ora_celery
   with_items: ['ora', 'ora_celery']
-  when: not devstack
+  when: not disable_edx_services
 
 - include: ease.yml
 
@@ -86,7 +97,7 @@
 - name: update supervisor configuration
   shell:  "{{ supervisor_ctl }} -c {{ supervisor_cfg }} update"
   register: supervisor_update
-  when: not devstack
+  when: not disable_edx_services
   changed_when: supervisor_update.stdout != ""
 
 - name: ensure ora is started
@@ -95,7 +106,7 @@
     supervisorctl_path={{ supervisor_ctl }}
     config={{ supervisor_cfg }}
     state=started
-  when: not devstack
+  when: not disable_edx_services
 
 - name: ensure ora_celery is started
   supervisorctl_local: >
@@ -103,7 +114,7 @@
     supervisorctl_path={{ supervisor_ctl }}
     config={{ supervisor_cfg }}
     state=started
-  when: not devstack
+  when: not disable_edx_services
 
 - name: create a symlink for venv python
   file: >

playbooks/roles/supervisor/defaults/main.yml

@@ -19,6 +19,7 @@ SUPERVISOR_HTTP_BIND_IP: '127.0.0.1'
 supervisor_http_bind_port: '9001'
 supervisor_app_dir: "{{ COMMON_APP_DIR }}/supervisor"
 supervisor_cfg_dir: "{{ supervisor_app_dir }}/conf.d"
+supervisor_available_dir: "{{ supervisor_app_dir }}/conf.available.d"
 supervisor_data_dir: "{{ COMMON_DATA_DIR }}/supervisor"
 supervisor_venvs_dir: "{{ supervisor_app_dir }}/venvs"
 supervisor_venv_dir: "{{ supervisor_venvs_dir }}/supervisor"

playbooks/roles/supervisor/files/pre_supervisor_checks.py

+# Get the tags for this instance
+import argparse
+import boto
+import boto.utils
+import os
+import subprocess
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        description="Enable all services that are in the services tag of this ec2 instance.")
+    parser.add_argument("-a","--available",
+        help="The location of the available services.")
+    parser.add_argument("-e","--enabled",
+        help="The location of the enabled services.")
+
+    args = parser.parse_args()
+
+    ec2 = boto.connect_ec2()
+    instance_id = boto.utils.get_instance_metadata()['instance-id']
+    reservations = ec2.get_all_instances(instance_ids=[instance_id])
+    report = []
+    for reservation in reservations:
+        for instance in reservation.instances:
+            if instance.id == instance_id:
+                services = instance.tags['services'].split(',')
+                for service in services:
+                    # Link to available service.
+                    available_file = "{}/{}.conf".format(args.available, service)
+                    link_location = "{}/{}.conf".format(args.enabled, service)
+                    if os.path.exists(available_file):
+                        subprocess.call("ln -sf {} {}".format(available_file, link_location), shell=True)
+                        report.append("Linking service: {}".format(service))
+                    else:
+                        report.append("No conf available for service: {}".format(link_location))
+
+    print("\n".join(report))

playbooks/roles/supervisor/tasks/main.yml

@@ -68,10 +68,12 @@
     state=directory
     owner={{ supervisor_user }}
     group={{ supervisor_service_user }}
+    mode="755"
   with_items:
     - "{{ supervisor_app_dir }}"
     - "{{ supervisor_venv_dir }}"
     - "{{ supervisor_cfg_dir }}"
+    - "{{ supervisor_available_dir }}"
 
 - name: create supervisor directories
   file: >
@@ -88,11 +90,34 @@
   pip: name=supervisor virtualenv="{{supervisor_venv_dir}}" state=present
   sudo_user: "{{ supervisor_user }}"
 
+- name: install supervisor in its venv
+  pip: name=boto virtualenv="{{supervisor_venv_dir}}" state=present
+  sudo_user: "{{ supervisor_user }}"
+  when: supervisor_service == "supervisor" and disable_edx_services and not devstack
+
 - name: create supervisor upstart job
   template: >
     src=supervisor-upstart.conf.j2 dest=/etc/init/{{ supervisor_service }}.conf
     owner=root group=root
 
+# This script is aws specific and looks up instances
+# tags and enables services based on the 'services' tag
+# on instance startup.
+- name: create pre_supervisor upstart job
+  template: >
+    src=pre_supervisor.conf.j2 dest=/etc/init/pre_supervisor.conf
+    owner=root group=root
+  when: supervisor_service == "supervisor" and disable_edx_services and not devstack
+
+- name: write the pre_suprevisor python script
+  copy: >
+    src=pre_supervisor_checks.py 
+    dest={{ supervisor_app_dir }}/pre_supervisor_checks.py
+    mode=0750
+    owner={{ supervisor_user }}
+    group={{ supervisor_service_user }}
+  when: disable_edx_services
+
 - name: create supervisor master config
   template: >
     src=supervisord.conf.j2 dest={{ supervisor_cfg }}

playbooks/roles/supervisor/templates/pre_supervisor.conf.j2

+description     "Tasks before supervisord"
+
+start on runlevel [2345]
+task
+
+setuid {{ supervisor_service_user }}
+exec {{ supervisor_venv_dir }}/bin/python {{ supervisor_app_dir }}/pre_supervisor_checks.py --available={{supervisor_available_dir}} --enabled={{supervisor_cfg_dir}}

playbooks/roles/supervisor/templates/supervisor-upstart.conf.j2

 description     "supervisord"
 
+{% if disable_edx_services -%}
+  start on stopped pre_supervisor
+{% else %}
 start on runlevel [2345]
+{% endif %}
 stop on runlevel [!2345]
 
 setuid {{ supervisor_service_user }}

playbooks/roles/xqueue/handlers/main.yml

@@ -8,3 +8,4 @@
   with_items:
   - xqueue
   - xqueue_consumer
+  when: not disable_edx_services

playbooks/roles/xqueue/tasks/deploy.yml

 - name: "writing supervisor scripts - xqueue, xqueue consumer"
   template: >
-    src={{ item }}.conf.j2 dest={{ supervisor_cfg_dir }}/{{ item }}.conf
+    src={{ item }}.conf.j2 dest={{ supervisor_available_dir }}/{{ item }}.conf
     owner={{ supervisor_user }} group={{ common_web_user }} mode=0644
   with_items: ['xqueue', 'xqueue_consumer']
 
+- name: "enabling supervisor scripts - xqueue, xqueue consumer"
+  file: >
+    src={{ supervisor_available_dir }}/{{ item }}.conf
+    dest={{ supervisor_cfg_dir }}/{{ item }}.conf
+    owner={{ supervisor_user }} group={{ common_web_user }}
+    mode=0644 state=link force=yes
+  with_items: ['xqueue', 'xqueue_consumer']
+  when: not disable_edx_services
+
 
 - name: create xqueue application config
   template: src=xqueue.env.json.j2 dest={{ xqueue_app_dir }}/xqueue.env.json mode=0644
@@ -63,6 +72,7 @@
   shell:  "{{ supervisor_ctl }} -c {{ supervisor_cfg }} update"
   register: supervisor_update
   changed_when: supervisor_update.stdout != ""
+  when: not disable_edx_services
 
 - name: ensure xqueue, consumer is running
   supervisorctl_local: >
@@ -73,6 +83,7 @@
   with_items:
   - xqueue
   - xqueue_consumer
+  when: not disable_edx_services
 
 - name: create a symlink for venv python
   file: >

playbooks/roles/xserver/handlers/main.yml

@@ -20,4 +20,5 @@
     supervisorctl_path={{ supervisor_ctl }}
     config={{ supervisor_cfg }}
     state=restarted
+  when: not disable_edx_services
 

playbooks/roles/xserver/tasks/deploy.yml

 - name: "writing supervisor script"
   template: >
-    src=xserver.conf.j2 dest={{ supervisor_cfg_dir }}/xserver.conf
+    src=xserver.conf.j2 dest={{ supervisor_available_dir }}/xserver.conf
     owner={{ supervisor_user }} group={{ common_web_user }} mode=0644
 
+- name: "enable supervisor script"
+  template: >
+    src={{ supervisor_available_dir }}/xserver.conf
+    dest={{ supervisor_cfg_dir }}/xserver.conf
+    owner={{ supervisor_user }} group={{ common_web_user }} mode=0644
+    state=link force=yes
+  when: not disable_edx_services
+
 - name: checkout code
   git: dest={{xserver_code_dir}} repo={{xserver_source_repo}} version={{xserver_version}}
   sudo_user: "{{ xserver_user }}"
@@ -55,6 +63,7 @@
   shell:  "{{ supervisor_ctl }} -c {{ supervisor_cfg }} update"
   register: supervisor_update
   changed_when: supervisor_update.stdout != ""
+  when: not disable_edx_services
 
 - name: ensure xserver is started
   supervisorctl_local: >
@@ -62,6 +71,7 @@
     supervisorctl_path={{ supervisor_ctl }}
     config={{ supervisor_cfg }}
     state=started
+  when: not disable_edx_services
 
 - name: create a symlink for venv python
   file: >

playbooks/vagrant-devstack.yml

@@ -6,6 +6,7 @@
     migrate_db: "yes"
     openid_workaround: True
     devstack: True
+    disable_edx_services: True
     edx_platform_version: 'master'
     mongo_enable_journal: False
     EDXAPP_NO_PREREQ_INSTALL: 0

requirements.txt

@@ -8,7 +8,4 @@ ecdsa==0.10
 paramiko==1.12.0
 pycrypto==2.6.1
 wsgiref==0.1.2
-GitPython==0.3.2.RC1
-pymongo==2.4.1
-requests==2.2.1
 docopt==0.6.1

util/jenkins/monitor_repos.py

-import argparse
-import json
-import logging as log
-import pickle
-import requests
-import yaml
-from datetime import datetime
-from git import Repo
-from os import path
-from pprint import pformat
-from pymongo import MongoClient, DESCENDING
-from stage_release import uri_from
-
-def releases(repo):
-    """
-    Yield a list of all release candidates from the origin.
-    """
-    for ref in repo.refs:
-        if ref.name.startswith('origin/rc/'):
-            yield ref
-
-def candidates_since(repo, time):
-    """
-    Given a repo yield a list of release candidate refs that have a
-    commit on them after the passed in time
-    """
-    for rc in releases(repo):
-        last_update = datetime.utcfromtimestamp(rc.commit.committed_date)
-        if last_update > time:
-            # New or updated RC
-            yield rc
-
-def stage_release(url, token, repo, rc):
-    """
-    Submit a job to stage a new release for the new rc of the repo.
-    """
-    # Setup the Jenkins params.
-    params = []
-    params.append({'name': "{}_REF".format(repo), 'value': True})
-    params.append({'name': repo, 'value': rc.commit.hexsha})
-    build_params = {'parameter': params}
-    log.info("New rc found{}, staging new release.".format(rc.name))
-    r = requests.post(url,
-                      data={"token", token},
-                      params={"json": json.dumps(build_params)})
-    if r.status_code != 201:
-        msg = "Failed to submit request with params: {}"
-        raise Exception(msg.format(pformat(build_params)))
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(
-        description="Monitor git repos for new rc branches.")
-    parser.add_argument('-c', '--config', required=True,
-        help="Config file.")
-    parser.add_argument('-p', '--pickle', default="data.pickle",
-        help="Pickle of presistent data.")
-
-    args = parser.parse_args()
-
-    config = yaml.safe_load(open(args.config))
-
-    if path.exists(args.pickle):
-        data = pickle.load(open(args.pickle))
-    else:
-        data = {}
-
-    # Presist the last time we made this check.
-    if 'last_check' not in data:
-        last_check = datetime.utcnow()
-    else:
-        last_check = data['last_check']
-
-    data['last_check'] = datetime.utcnow()
-
-    # Find plays that are affected by this repo.
-    repos_with_changes = {}
-    for repo in config['repos']:
-        # Check for new rc candidates.
-        for rc in candidates_since(Repo(repo), last_check):
-            # Notify stage-release to build for the new repo.
-            stage_release(config['abbey_url'], config['abbey_token'], repo, rc)
-
-    pickle.dump(data, open(args.pickle, 'w'))

util/jenkins/stage_release.py

-"""
-Take in a YAML file with the basic data of all the things we could
-deploy and command line hashes for the repos that we want to deploy
-right now.
-
-Example Config YAML file:
----
-DOC_STORE_CONFIG:
-    hosts: [ list, of, mongo, hosts]
-    port: #
-    db: 'db'
-    user: 'jenkins'
-    password: 'password'
-
-configuration_repo: "/path/to/configuration/repo"
-configuration_secure_repo: "/path/to/configuration-secure"
-
-repos:
-    edxapp:
-        plays:
-        - edxapp
-        - worker
-    xqueue:
-        plays:
-        - xqueue
-    6.00x:
-        plays:
-        - xserver
-    xserver:
-        plays:
-        - xserver
-
-deployments:
-    edx:
-    - stage
-    - prod
-    edge:
-    - stage
-    - prod
-    loadtest:
-    - stage
-
-# A jenkins URL to post requests for building AMIs
-abbey_url: "http://...."
-
-# A mapping of plays to base AMIs
-base_ami:{}
-
-# The default AMI to use if there isn't one specific to your plays.
-default_base_ami: ''
----
-"""
-import argparse
-import json
-import yaml
-import logging as log
-import requests
-from datetime import datetime
-from git import Repo
-from pprint import pformat
-from pymongo import MongoClient, DESCENDING
-
-log.basicConfig(level=log.DEBUG)
-
-def uri_from(doc_store_config):
-    """
-    Convert the below structure to a mongodb uri.
-
-    DOC_STORE_CONFIG:
-      hosts:
-        - 'host1.com'
-        - 'host2.com'
-      port: 10012
-      db: 'devops'
-      user: 'username'
-      password: 'password'
-    """
-
-    uri_format = "mongodb://{user}:{password}@{hosts}/{db}"
-    host_format = "{host}:{port}"
-
-    port = doc_store_config['port']
-    host_uris = [host_format.format(host=host,port=port) for host in doc_store_config['hosts']]
-    return uri_format.format(
-        user=doc_store_config['user'],
-        password=doc_store_config['password'],
-        hosts=",".join(host_uris),
-        db=doc_store_config['db'])
-
-def prepare_release(args):
-    config = yaml.safe_load(open(args.config))
-    mongo_uri = uri_from(config['DOC_STORE_CONFIG'])
-    client = MongoClient(mongo_uri)
-    db = client[config['DOC_STORE_CONFIG']['db']]
-
-    # Get configuration repo versions
-    config_repo_ver = Repo(config['configuration_repo']).commit().hexsha
-    config_secure_ver = Repo(config['configuration_secure_repo']).commit().hexsha
-
-    # Parse the vars.
-    var_array = map(lambda key_value: key_value.split('='), args.REPOS)
-    update_repos = { item[0]:item[1] for item in var_array }
-    log.info("Update repos: {}".format(pformat(update_repos)))
-
-    release = {}
-    now = datetime.utcnow()
-    release['_id'] = args.release_id
-    release['date_created'] = now
-    release['date_modified'] = now
-    release['build_status'] = 'Unknown'
-    release['build_user'] = args.user
-
-
-    release_coll = db[args.deployment]
-    releases = release_coll.find({'build_status': 'Succeeded'}).sort('_id', DESCENDING)
-    all_plays = {}
-
-    try:
-        last_successful = releases.next()
-        all_plays = last_successful['plays']
-    except StopIteration:
-        # No successful builds.
-        log.warn("No Previously successful builds.")
-
-    # For all repos that were updated
-    for repo, ref in update_repos.items():
-        var_name = "{}_version".format(repo.replace('-','_'))
-        if repo not in config['repos']:
-            raise Exception("No info for repo with name '{}'".format(repo))
-
-        # For any play that uses the updated repo
-        for play in config['repos'][repo]:
-            if play not in all_plays:
-                all_plays[play] = {}
-
-            if 'vars' not in all_plays[play]:
-                all_plays[play]['vars'] = {}
-
-            all_plays[play]['vars'][var_name] = ref
-            # Configuration to use to build these AMIs
-            all_plays[play]['configuration_ref'] = config_repo_ver
-            all_plays[play]['configuration_secure_ref'] = config_secure_ver
-
-            # Set amis to None for all envs of this deployment
-            all_plays[play]['amis'] = {}
-            for env in config['deployments'][args.deployment]:
-                # Check the AMIs collection to see if an ami already exist
-                # for this configuration.
-                potential_ami = ami_for(db, env,
-                                        args.deployment,
-                                        play, config_repo_ver,
-                                        config_secure_ver,
-                                        ref)
-                if potential_ami:
-                    all_plays[play]['amis'][env] = potential_ami['_id']
-                else:
-                    all_plays[play]['amis'][env] = None
-
-    release['plays'] = all_plays
-    if args.noop:
-        print("Would insert into release collection: {}".format(pformat(release)))
-    else:
-        release_coll.insert(release)
-    # All plays that need new AMIs have been updated.
-    notify_abbey(config, args.deployment,
-                 all_plays, args.release_id, mongo_uri, config_repo_ver,
-                 config_secure_ver, args.noop)
-
-def ami_for(db, env, deployment, play, configuration,
-    configuration_secure, ansible_vars):
-
-    ami_signature = {
-        'env': env,
-        'deployment': deployment,
-        'play': play,
-        'configuration_ref': configuration,
-        'configuration_secure_ref': configuration_secure,
-        'vars': ansible_vars,
-    }
-
-    return db.amis.find_one(ami_signature)
-
-def notify_abbey(config, deployment, all_plays, release_id,
-                 mongo_uri, configuration_ref, configuration_secure_ref, noop=False):
-    abbey_url = config['abbey_url']
-    base_amis = config['base_amis']
-    default_base = config['default_base_ami']
-
-    for play_name, play in all_plays.items():
-        for env, ami in play['amis'].items():
-            if ami is None:
-                params = {}
-                params['play'] = play_name
-                params['deployment'] = deployment
-                params['environment'] = env
-                params['refs'] = yaml.safe_dump(play['vars'], default_flow_style=False)
-                params['release_id'] = release_id
-                params['mongo_uri'] = mongo_uri
-                params['configuration'] = configuration_ref
-                params['configuration_secure'] = configuration_secure_ref
-                params['base_ami'] = base_amis.get(play_name, default_base)
-
-                log.info("Need ami for {}".format(pformat(params)))
-                if noop:
-                    r = requests.Request('POST', abbey_url, params=params)
-                    url = r.prepare().url
-                    print("Would have posted: {}".format(url))
-                else:
-                    r = requests.post(abbey_url, params=params)
-
-                    log.info("Sent request got {}".format(r))
-                    if r.status_code != 200:
-                        # Something went wrong.
-                        msg = "Failed to submit request with params: {}"
-                        raise Exception(msg.format(pformat(params)))
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Prepare a new release.")
-    parser.add_argument('-c', '--config', required=True, help="Configuration for deploys")
-    parser.add_argument('-u', '--user', required=True, help="User staging the release.")
-    msg = "The deployment to build for eg. edx, edge, loadtest"
-    parser.add_argument('-d', '--deployment', required=True, help=msg)
-    parser.add_argument('-r', '--release-id', required=True, help="Id of Release.")
-    parser.add_argument('-n', '--noop', action='store_true',
-        help="Run without sending requests to abbey.")
-    parser.add_argument('REPOS', nargs='+',
-        help="Any number of var=value(no spcae around '='" + \
-             " e.g. 'edxapp=3233bac xqueue=92832ab'")
-
-    args = parser.parse_args()
-    log.debug(args)
-    prepare_release(args)

util/vpc-tools/abbey.py

@@ -15,8 +15,6 @@ except ImportError:
     print "boto required for script"
     sys.exit(1)
 
-from pymongo import MongoClient
-from pymongo.errors import ConnectionFailure, DuplicateKeyError
 from pprint import pprint
 
 AMI_TIMEOUT = 600  # time to wait for AMIs to complete
@@ -26,77 +24,6 @@ NUM_TASKS = 5  # number of tasks for time summary report
 NUM_PLAYBOOKS = 2
 
 
-class MongoConnection:
-
-    def __init__(self):
-        try:
-            mongo = MongoClient(host=args.mongo_uri)
-        except ConnectionFailure:
-            print "Unable to connect to the mongo database specified"
-            sys.exit(1)
-
-        mongo_db = getattr(mongo, args.mongo_db)
-        if args.mongo_ami_collection not in mongo_db.collection_names():
-            mongo_db.create_collection(args.mongo_ami_collection)
-        if args.mongo_deployment_collection not in mongo_db.collection_names():
-            mongo_db.create_collection(args.mongo_deployment_collection)
-        self.mongo_ami = getattr(mongo_db, args.mongo_ami_collection)
-        self.mongo_deployment = getattr(
-            mongo_db, args.mongo_deployment_collection)
-
-    def update_ami(self, ami):
-        """
-        Creates a new document in the AMI
-        collection with the ami id as the
-        id
-        """
-
-        query = {
-            '_id': ami,
-            'play': args.play,
-            'env': args.environment,
-            'deployment': args.deployment,
-            'configuration_ref': args.configuration_version,
-            'configuration_secure_ref': args.configuration_secure_version,
-            'vars': git_refs,
-        }
-        try:
-            self.mongo_ami.insert(query)
-        except DuplicateKeyError:
-            if not args.noop:
-                print "Entry already exists for {}".format(ami)
-                raise
-
-    def update_deployment(self, ami):
-        """
-        Adds the built AMI to the deployment
-        collection
-        """
-        query = {'_id': args.jenkins_build}
-        deployment = self.mongo_deployment.find_one(query)
-        try:
-            deployment['plays'][args.play]['amis'][args.environment] = ami
-        except KeyError:
-            msg = "Unexpected document structure, couldn't write " +\
-                  "to path deployment['plays']['{}']['amis']['{}']"
-            print msg.format(args.play, args.environment)
-            pprint(deployment)
-            if args.noop:
-                deployment = {
-                    'plays': {
-                        args.play: {
-                            'amis': {
-                                args.environment: ami,
-                            },
-                        },
-                    },
-                }
-            else:
-                raise
-
-        self.mongo_deployment.save(deployment)
-
-
 class Unbuffered:
     """
     For unbuffered output, not
@@ -156,8 +83,8 @@ def parse_args():
     parser.add_argument('--configuration-secure-repo', required=False,
                         default="git@github.com:edx-ops/prod-secure",
                         help="repo to use for the secure files")
-    parser.add_argument('-j', '--jenkins-build', required=False,
-                        help="jenkins build number to update")
+    parser.add_argument('-c', '--cache-id', required=True,
+                        help="unique id to use as part of cache prefix")
     parser.add_argument('-b', '--base-ami', required=False,
                         help="ami to use as a base ami",
                         default="ami-0568456c")
@@ -181,20 +108,6 @@ def parse_args():
                         default=5,
                         help="How long to delay message display from sqs "
                              "to ensure ordering")
-    parser.add_argument("--mongo-uri", required=False,
-                        default=None,
-                        help="Mongo uri for the host that contains"
-                             "the AMI collection")
-    parser.add_argument("--mongo-db", required=False,
-                        default="test",
-                        help="Mongo database")
-    parser.add_argument("--mongo-ami-collection", required=False,
-                        default="amis",
-                        help="Mongo ami collection")
-    parser.add_argument("--mongo-deployment-collection", required=False,
-                        default="deployment",
-                        help="Mongo deployment collection")
-
     return parser.parse_args()
 
 
@@ -329,7 +242,9 @@ fake_migrations: true
 
 # Use the build number an the dynamic cache key.
 EDXAPP_UPDATE_STATIC_FILES_KEY: true
-edxapp_dynamic_cache_key: {deployment}-{environment}-{play}-{build_id}
+edxapp_dynamic_cache_key: {deployment}-{environment}-{play}-{cache_id}
+
+disable_edx_services: true
 EOF
 
 chmod 400 $secure_identity
@@ -371,7 +286,7 @@ rm -rf $base_dir
                 extra_vars_yml=extra_vars_yml,
                 git_refs_yml=git_refs_yml,
                 secure_vars=secure_vars,
-                build_id=args.jenkins_build)
+                cache_id=args.cache_id)
 
     ec2_args = {
         'security_group_ids': [security_group_id],
@@ -530,7 +445,7 @@ def create_ami(instance_id, name, description):
                 time.sleep(AWS_API_WAIT_TIME)
                 img.add_tag("configuration_secure_repo", args.configuration_secure_repo)
                 time.sleep(AWS_API_WAIT_TIME)
-                img.add_tag("build_id", args.jenkins_build)
+                img.add_tag("cache_id", args.cache_id)
                 time.sleep(AWS_API_WAIT_TIME)
                 for repo, ref in git_refs.items():
                     key = "vars:{}".format(repo)
@@ -682,9 +597,6 @@ if __name__ == '__main__':
         print 'You must be able to connect to sqs and ec2 to use this script'
         sys.exit(1)
 
-    if args.mongo_uri:
-        mongo_con = MongoConnection()
-
     try:
         sqs_queue = None
         instance_id = None
@@ -708,9 +620,6 @@ if __name__ == '__main__':
                 print "{:<30} {:0>2.0f}:{:0>5.2f}".format(
                     run[0], run[1] / 60, run[1] % 60)
             print "AMI: {}".format(ami)
-        if args.mongo_uri:
-            mongo_con.update_ami(ami)
-            mongo_con.update_deployment(ami)
     finally:
         print
         if not args.no_cleanup and not args.noop: