general refactoring

docker/build/analytics_api/Dockerfile

+FROM edxops/precise-common:v2
+MAINTAINER edxops
+
+ENV CONFIGURATION_REPO="https://github.com/edx/configuration.git"
+ENV CONFIGURATION_VERSION="hack2015/docker"
+
+USER docker
+
+RUN sudo apt-get update
+
+WORKDIR /edx/app/edx_ansible/edx_ansible
+RUN sudo git fetch --all
+RUN sudo git checkout ${CONFIGURATION_VERSION}
+RUN sudo git reset --hard origin/${CONFIGURATION_VERSION}
+RUN sudo git pull
+
+WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays
+COPY ansible_overrides.yml /
+# Remove when branch is pushed
+COPY analytics_api-temp.yml /edx/app/edx_ansible/edx_ansible/docker/plays/analytics_api.yml
+#RUN sudo ansible-playbook analytics_api.yml -i '127.0.0.1,' -c local -t "install:base,install:system-requirements,install:configuration,install:app-requirements,install:code" -e@/ansible_overrides.yml
+
+CMD ["/edx/app/supervisor/venvs/supervisor/bin/supervisord -n --configuration /edx/app/supervisor/supervisord.conf"]
+EXPOSE 443 80

docker/build/analytics_api/analytics_api_tmp.yml

+../../plays/analytics_api.yml
\ No newline at end of file

docker/build/analytics_api/ansible_overrides.yml

+---
+
+DOCKER_TLD: "edx"
+
+INSIGHTS_LMS_BASE: "http://lms.{{ DOCKER_TLD }}:8000"
+INSIGHTS_CMS_BASE: "http://cms.{{ DOCKER_TLD }}:8010"
+INSIGHTS_BASE_URL: "http://insights.{{ DOCKER_TLD }}:8110"
+INSIGHTS_MEMCACHE:
+  - "memcache.{{ DOCKER_TLD }}:11211"
+ANALYTICS_API_ENDPOINT: "http://analtyicsapi.{{ DOCKER_TLD }}:8100/api/v0"
+
+INSIGHTS_DATABASES:
+  # rw user
+  default:
+    ENGINE: 'django.db.backends.mysql'
+    NAME: '{{ INSIGHTS_DATABASE_NAME }}'
+    USER: 'rosencrantz'
+    PASSWORD: 'secret'
+    HOST: "db.{{ DOCKER_TLD  }}"
+    PORT: '3306'

docker/build/edxapp/Dockerfile

 FROM edxops/precise-common
 MAINTAINER edxops
 
+ENV CONFIGURATION_REPO="https://github.com/edx/configuration.git"
+ENV CONFIGURATION_VERSION="hack2015/docker"
+
 USER docker
+
+RUN sudo apt-get update
+
 WORKDIR /edx/app/edx_ansible/edx_ansible
 RUN sudo git fetch --all
-RUN sudo git checkout hack2015/docker
-RUN sudo git reset --hard hack2015/docker
+RUN sudo git checkout ${CONFIGURATION_VERSION}
+RUN sudo git reset --hard origin/${CONFIGURATION_VERSION}
 RUN sudo git pull
 WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays
-ADD ansible.cfg /edx/app/edx_ansible/edx_ansible/docker/plays/ansible.cfg 
-RUN sudo ansible-playbook edxapp.yml -c local -e "EDXAPP_PYTHON_SANDBOX=false" -t 'install:base'
-RUN sudo ansible-playbook edxapp.yml -c local -e "EDXAPP_PYTHON_SANDBOX=false" -t 'install:code'
-RUN sudo ansible-playbook edxapp.yml -c local -e "EDXAPP_PYTHON_SANDBOX=false" -t 'install:configuration' -e EDXAPP_MYSQL_HOST='lms-mysql.local.edx.org' -e '{"EDXAPP_MONGO_HOSTS":["lms-mongo.local.edx.org"]}'
-USER root 
+COPY ansible_overrides.yml /
+
+RUN sudo ansible-playbook edxapp.yml -c local -e "EDXAPP_PYTHON_SANDBOX=false" -t "install:base,install:configuration,install:application-requirements,install:code" -e@/ansible_overrides.yml
+USER root
+WORKDIR /edx/app
 CMD ["/edx/app/supervisor/venvs/supervisor/bin/supervisord", "-n", "--configuration", "/edx/app/supervisor/supervisord.conf"]
-EXPOSE 18000 48000 18020 18010 48010 8010 8000 
+EXPOSE 8000 8010

docker/build/edxapp/ansible.cfg

-[defaults]
-
-jinja2_extensions=jinja2.ext.do
-

docker/build/edxapp/ansible_overrides.ymml

+---
+
+DOCKER_TLD: "edx"
+
+EDXAPP_MYSQL_HOST: "db.{{ DOCKER_TLD }}"
+EDXAPP_MONGO_HOSTS:
+	- "mongo.{{ DOCKER_TLD }}"
\ No newline at end of file

docker/build/edxapp/inventory

-[local]
-localhost

docker/build/forums/Dockerfile

 FROM edxops/precise-common:v2
 MAINTAINER edxops
 
+ENV CONFIGURATION_REPO="https://github.com/edx/configuration.git"
+ENV CONFIGURATION_VERSION="hack2015/docker"
+
 USER docker
 
 RUN sudo apt-get update
 
 WORKDIR /edx/app/edx_ansible/edx_ansible
 RUN sudo git fetch --all
-RUN sudo git checkout hack2015/docker
-RUN sudo git reset --hard hack2015/docker
+RUN sudo git checkout ${CONFIGURATION_VERSION}
+RUN sudo git reset --hard origin/${CONFIGURATION_VERSION}
 RUN sudo git pull
 WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays
-
-RUN sudo ansible-playbook forum.yml -c local -t 'install:base,install:configuration,install:code' -e '{"FORUM_MONGO_HOSTS":["mongo.forums"]}' -e FORUM_ELASTICSEARCH_HOST='es.forums' -e forum_source_repo='https://github.com/open-craft/cs_comments_service.git' -e forum_version='mongoid5' -e FORUM_USE_TCP='true' -e FORUM_RACK_ENV='staging' -e FORUM_SINATRA_ENV='staging'
-USER root 
+COPY ansible_overrides.yml /
+RUN sudo ansible-playbook forum.yml -i '127.0.0.1,' -c local -t "install:base,install:configuration,install:code" -e@/ansible_overrides.yml
+WORKDIR /edx/app
 CMD ["/edx/app/supervisor/venvs/supervisor/bin/supervisord", "-n", "--configuration", "/edx/app/supervisor/supervisord.conf"]
-EXPOSE 18080
+EXPOSE 4567
\ No newline at end of file

docker/build/forums/ansible_overrides.yml

+---
+
+FLOCK_TLD: "forums"
+
+FORUM_MONGO_HOSTS:
+  - mongo.{{ FLOCK_TLD }}
+  
+FORUM_ELASTICSEARCH_HOST: "es.{{ FLOCK_TLD }}"
+forum_source_repo: "https://github.com/open-craft/cs_comments_service.git"
+forum_version: "mongoid5"
+FORUM_USE_TCP: "true"
+FORUM_RACK_ENV: "staging"
+FORUM_SINATRA_ENV: "staging"
\ No newline at end of file

docker/build/forums/inventory

-[local]
-localhost

docker/build/insights/Dockerfile

 FROM edxops/precise-common:v2
-MAINTAINER e0d
+MAINTAINER edxops
+
+ENV CONFIGURATION_REPO="https://github.com/edx/configuration.git"
+ENV CONFIGURATION_VERSION="hack2015/docker"
 
 USER docker
-WORKDIR /edx/app/edx_ansible
-ENV PATH /tmp/ansible/bin:/bin:/sbin:/usr/sbin:/usr/bin
-#ENV ANSIBLE_LIBRARY /edx/app/edx_ansible/edx_ansible/playbooks/library
-#ENV PYTHONPATH /tmp/ansible/lib:$PYTHON_PATH
-ADD inventory /etc/ansible/hosts
-WORKDIR /edx/app/edx_ansible/edx_ansible/playbooks/edx-east
+
+RUN sudo apt-get update
+
+WORKDIR /edx/app/edx_ansible/edx_ansible
 RUN sudo git fetch --all
-RUN sudo git checkout e0d/docker
-RUN sudo ansible-playbook insights.yml -c local
-CMD ["/edx/app/supervisor/venvs/supervisor/bin/supervisord -n --configuration /edx/app/supervisor/supervisord.conf"]
-EXPOSE 22 443 80
+RUN sudo git checkout ${CONFIGURATION_VERSION}
+RUN sudo git reset --hard origin/${CONFIGURATION_VERSION}
+RUN sudo git pull
+
+WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays
+COPY ansible_overrides.yml /
+# Remove when branch is pushed
+COPY insights-temp.yml /edx/app/edx_ansible/edx_ansible/docker/plays/insights.yml
+#RUN sudo ansible-playbook insights.yml -i '127.0.0.1,' -c local -t "install:base,install:system-requirements,install:configuration,install:app-requirements,install:code" -e@/ansible_overrides.yml
+
+#CMD ["/edx/app/supervisor/venvs/supervisor/bin/supervisord -n --configuration /edx/app/supervisor/supervisord.conf"]
+#EXPOSE 443 80

docker/build/insights/ansible_overrides.yml

+---
+
+DOCKER_TLD: "edx"
+
+INSIGHTS_LMS_BASE: "http://lms.{{ DOCKER_TLD }}:8000"
+INSIGHTS_CMS_BASE: "http://cms.{{ DOCKER_TLD }}:8010"
+INSIGHTS_BASE_URL: "http://insights.{{ DOCKER_TLD }}:8110"
+INSIGHTS_MEMCACHE:
+  - "memcache.{{ DOCKER_TLD }}:11211"
+ANALYTICS_API_ENDPOINT: "http://analtyicsapi.{{ DOCKER_TLD }}:8100/api/v0"
+
+INSIGHTS_DATABASES:
+  # rw user
+  default:
+    ENGINE: 'django.db.backends.mysql'
+    NAME: '{{ INSIGHTS_DATABASE_NAME }}'
+    USER: 'rosencrantz'
+    PASSWORD: 'secret'
+    HOST: "db.{{ DOCKER_TLD  }}"
+    PORT: '3306'

docker/build/insights/insights-temp.yml

+- name: Deploy Insights
+  hosts: all
+  sudo: True
+  gather_facts: True
+  vars:
+    serial_count: 1
+  serial: "{{ serial_count }}"
+  roles:
+    - common_vars
+    - docker
+    - insights

docker/build/insights/inventory

-[local]
-localhost

docker/build/precise-common/Dockerfile

 FROM ubuntu:precise 
 MAINTAINER edxops 
+ENV ANSIBLE_REPO="https://github.com/edx/ansible"
+ENV CONFIGURATION_REPO="https://github.com/edx/configuration.git"
+ENV CONFIGURATION_VERSION="hack2015/docker"
 
+# system bootstrap
 RUN apt-get update
 RUN apt-get -y install sudo
 RUN useradd docker && echo "docker:docker" | chpasswd
@@ -9,22 +13,21 @@ RUN mkdir -p /home/docker && chown -R docker:docker /home/docker
 RUN apt-get install -y python2.7 python2.7-dev python-pip python-apt python-yaml python-jinja2 git
 USER docker
 
-# bootstrap
-RUN sudo git clone --recursive https://github.com/edx/ansible /tmp/ansible
+# ansible bootstrap
+RUN sudo git clone --recursive ${ANSIBLE_REPO} /tmp/ansible
 WORKDIR /tmp/ansible
 ENV PATH /tmp/ansible/bin:/bin:/sbin:/usr/sbin:/usr/bin
 
 # Install the configuration repository to install 
 # edx-ansible role
-RUN sudo git clone http://github.com/edx/configuration.git /tmp/configuration
-ADD inventory /etc/ansible/hosts
+RUN sudo git clone ${CONFIGURATION_REPO} /tmp/configuration
 WORKDIR /tmp/configuration
-RUN sudo git checkout hack2015/docker
+RUN sudo git checkout ${CONFIGURATION_VERSION}
 RUN sudo pip install -r pre-requirements.txt
 RUN sudo pip install -r requirements.txt
 WORKDIR /tmp/configuration/playbooks/edx-east
-RUN sudo /tmp/ansible/bin/ansible-playbook edx_ansible.yml -c local -e "configuration_version=hack2015/docker"
-
+RUN sudo /tmp/ansible/bin/ansible-playbook edx_ansible.yml -i '127.0.0.1,' -c local -e "configuration_version=${CONFIGURATION_VERSION}"
+WORKDIR /edx/app/edx_ansible
 # cleanup
 RUN sudo rm -rf /tmp/ansible
 RUN sudo rm -rf /tmp/configuration

docker/build/precise-common/inventory

-[local]
-localhost

docker/build/trusty-common/Dockerfile

 FROM ubuntu:trusty
 MAINTAINER edxops 
+ENV ANSIBLE_REPO="https://github.com/edx/ansible"
+ENV CONFIGURATION_REPO="https://github.com/edx/configuration.git"
+ENV CONFIGURATION_VERSION="hack2015/docker"
 
+# system bootstrap
 RUN apt-get update
 RUN apt-get -y install sudo
 RUN useradd docker && echo "docker:docker" | chpasswd
@@ -9,8 +13,8 @@ RUN mkdir -p /home/docker && chown -R docker:docker /home/docker
 RUN apt-get install -y python2.7 python2.7-dev python-pip python-apt python-yaml python-jinja2 git
 USER docker
 
-# bootstrap
-RUN sudo git clone --recursive https://github.com/edx/ansible /tmp/ansible
+# ansible bootstrap
+RUN sudo git clone --recursive ${ANSIBLE_REPO} /tmp/ansible
 WORKDIR /tmp/ansible
 ENV PATH /tmp/ansible/bin:/bin:/sbin:/usr/sbin:/usr/bin
 
@@ -24,6 +28,7 @@ RUN sudo pip install -r pre-requirements.txt
 RUN sudo pip install -r requirements.txt
 WORKDIR /tmp/configuration/playbooks/edx-east
 RUN sudo /tmp/ansible/bin/ansible-playbook edx_ansible.yml -c local -e "configuration_version=hack2015/docker"
+WORKDIR /edx/app/edx_ansible
 
 # cleanup
 RUN sudo rm -rf /tmp/ansible

docker/build/xqueue/Dockerfile

 FROM edxops/precise-common:v2
 MAINTAINER edxops
 
+ENV CONFIGURATION_REPO="https://github.com/edx/configuration.git"
+ENV CONFIGURATION_VERSION="hack2015/docker"
+
 USER docker
 
+RUN sudo apt-get update
+
 WORKDIR /edx/app/edx_ansible/edx_ansible
 RUN sudo git fetch --all
-RUN sudo git checkout e0d/docker-latest
-RUN sudo git reset --hard origin/e0d/docker-latest
+RUN sudo git checkout ${CONFIGURATION_VERSION}
+RUN sudo git reset --hard origin/${CONFIGURATION_VERSION}
 RUN sudo git pull
 
 WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays
-RUN echo "bust-cache"
-RUN sudo ansible-playbook xqueue.yml -c local
+COPY ansible_overrides.yml /
+RUN sudo ansible-playbook xqueue.yml -i '127.0.0.1,' -c local -t "install:base,install:system-requirements,install:configuration,install:application-requirements,install:code" -e@/ansible_overrides.yml
 USER root 
-COPY ./docker-run.sh /
+COPY docker-run.sh /
 ENTRYPOINT ["/docker-run.sh"]
-EXPOSE 18040 8040
+EXPOSE 8110 18110

docker/build/xqueue/ansible_overrides.yml

+---
+
+DOCKER_TLD: "xqueue"
+
+CONFIGURATION_REPO: "https://github.com/edx/configuration.git"
+CONFIGURATION_VERSION: "hack2015/docker"
+XQUEUE_SYSLOG_SERVER: "localhost"
+XQUEUE_RABBITMQ_HOSTNAME: "rabbit.{{ DOCKER_TLD }}"
+XQUEUE_MYSQL_HOST: "db.{{ DOCKER_TLD }}"

docker/plays/analytics_api.yml

+- name: Deploy Analytics API
+  hosts: all
+  sudo: True
+  gather_facts: True
+  vars:
+    serial_count: 1
+  serial: "{{ serial_count }}"
+  roles:
+    - common_vars
+    - docker
+    - analytics_api

docker/plays/edxapp.yml

@@ -6,17 +6,6 @@
     serial_count: 1
   serial: "{{ serial_count }}"
   roles:
+    - common_vars
     - docker
     - edxapp
-    - role: datadog
-      when: COMMON_ENABLE_DATADOG
-    - role: splunkforwarder
-      when: COMMON_ENABLE_SPLUNKFORWARDER
-    - role: newrelic
-      NEWRELIC_LOGWATCH:
-        - logwatch-503.j2
-        - logwatch-cms-errors.j2
-        - logwatch-lms-errors.j2
-      when: COMMON_ENABLE_NEWRELIC
-    - role: minos
-      when: COMMON_ENABLE_MINOS

docker/plays/forum.yml

@@ -6,9 +6,6 @@
     serial_count: 1
   serial: "{{ serial_count }}"
   roles:
+    - commmon_vars
     - docker
-    - role: nginx
-      nginx_sites:
-      - forum
     - forum
-    

docker/plays/insights.yml

+- name: Deploy Insights
+  hosts: all
+  sudo: True
+  gather_facts: True
+  vars:
+    serial_count: 1
+  serial: "{{ serial_count }}"
+  roles:
+    - common_vars
+    - docker
+    - insights

docker/plays/nginx.yml

@@ -6,6 +6,7 @@
     serial_count: 1
   serial: "{{ serial_count }}"
   roles:
+    - common_vars
     - docker 
     - role: nginx
       nginx_sites:

docker/plays/xqueue.yml

@@ -3,16 +3,6 @@
   sudo: True
   gather_facts: True
   roles:
+    - common_vars
     - docker
-    - role: xqueue
-    - role: nginx
-      nginx_sites:
-      - xqueue
-    - role: datadog
-      when: COMMON_ENABLE_DATADOG
-    - role: splunkforwarder
-      when: COMMON_ENABLE_SPLUNKFORWARDER
-    - role: newrelic
-      NEWRELIC_LOGWATCH:
-        - logwatch-xqueue-errors.j2
-      when: COMMON_ENABLE_NEWRELIC
+    - xqueue