Merge pull request #3800 from edx/jmbowman/edxapp-docker-devstack

Run edxapp on Docker for Devstack

docker/build/edxapp/Dockerfile

@@ -2,26 +2,25 @@
 #
 # From the root of configuration:
 #
-# docker build -f docker/build/edxapp/Dockerfile .
+# docker build -f docker/build/edxapp/Dockerfile . -t edxops/edxapp:latest
 #
 # This allows the dockerfile to update /edx/app/edx_ansible/edx_ansible
 # with the currently checked-out configuration repo.
 
+
 FROM edxops/precise-common:latest
 MAINTAINER edxops
-
 USER root
-RUN apt-get update
+CMD ["/edx/app/supervisor/venvs/supervisor/bin/supervisord", "-n", "--configuration", "/edx/app/supervisor/supervisord.conf"]
 
 ADD . /edx/app/edx_ansible/edx_ansible
-COPY docker/build/edxapp/ansible_overrides.yml /
 WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays
+
 COPY docker/build/edxapp/ansible_overrides.yml /
-RUN /edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook \
-    edxapp.yml -i '127.0.0.1,' -c local \
-    -e "EDXAPP_PYTHON_SANDBOX=false" \
-    -e@/ansible_overrides.yml \
-    -t "install:base,install:configuration,install:system-requirements,install:app-requirements,install:code"
-WORKDIR /edx/app
-CMD ["/edx/app/supervisor/venvs/supervisor/bin/supervisord", "-n", "--configuration", "/edx/app/supervisor/supervisord.conf"]
-EXPOSE 8000 8010
+
+RUN sudo /edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook edxapp.yml \
+    -c local -i '127.0.0.1,' \
+    -t 'install,assets,devstack' \
+    --extra-vars="@/ansible_overrides.yml"
+
+EXPOSE 18000 18010

docker/build/edxapp/ansible_overrides.yml

 ---
+EDXAPP_SETTINGS: 'devstack_docker'
+EDXAPP_MEMCACHE: ['edx.devstack.memcached:11211']
+EDXAPP_MYSQL_HOST: "edx.devstack.mysql"
+EDXAPP_MONGO_HOSTS:
+  - "edx.devstack.mongo"
 
-DOCKER_TLD: "edx"
+devstack: true
+migrate_db: false
+mongo_enable_journal: false
+edx_platform_version: 'master'
+edxapp_npm_production: "no"
 
-# prevents Travis from giving up on the build
-COMMON_PIP_VERBOSITY: "-vvvv"
+EDXAPP_LMS_GUNICORN_EXTRA_CONF: 'reload = True'
 
-EDXAPP_MYSQL_HOST: "db.{{ DOCKER_TLD }}"
-EDXAPP_MONGO_HOSTS:
-  - "mongo.{{ DOCKER_TLD }}"
+EDXAPP_NO_PREREQ_INSTALL: 0
+COMMON_SSH_PASSWORD_AUTH: "yes"
+EDXAPP_LMS_BASE: "edx.devstack.lms:18000"
+EDXAPP_CMS_BASE: "edx.devstack.studio:18010"
+EDXAPP_OAUTH_ENFORCE_SECURE: false
+EDXAPP_LMS_BASE_SCHEME: http
+COMMON_SECURITY_UPDATES: true
+SECURITY_UPGRADE_ON_ANSIBLE: true
+
+EDXAPP_INSTALL_PRIVATE_REQUIREMENTS: true
+
+EDXAPP_SEARCH_HOST: 'edx.devstack.elasticsearch'
+
+EDXAPP_PYTHON_SANDBOX: false

docker/plays/edxapp.yml

@@ -8,4 +8,13 @@
   roles:
     - common_vars
     - docker
+    - role: nginx
+      nginx_sites:
+      - lms
+      - cms
+      nginx_default_sites:
+      - lms
+      nginx_extra_sites: "{{ NGINX_EDXAPP_EXTRA_SITES }}"
+      nginx_extra_configs: "{{ NGINX_EDXAPP_EXTRA_CONFIGS }}"
+      nginx_redirects: "{{ NGINX_EDXAPP_CUSTOM_REDIRECTS }}"
     - edxapp

playbooks/roles/edxapp/defaults/main.yml

@@ -90,12 +90,15 @@ EDXAPP_MYSQL_CSMH_PORT: "{{ EDXAPP_MYSQL_PORT }}"
 EDXAPP_MYSQL_HOST: 'localhost'
 EDXAPP_MYSQL_PORT: '3306'
 
+EDXAPP_SEARCH_HOST: 'localhost'
+EDXAPP_SEARCH_PORT: '9200'
+
 # list of dictionaries of the format
 # { 'host': 'hostname', 'port': 'portnumber', 'otherconfigsuchas use_ssl': 'True' }
 # http://elasticsearch-py.readthedocs.org/en/master/api.html#elasticsearch
 EDXAPP_ELASTIC_SEARCH_CONFIG:
-    - host: "localhost"
-      port: 9200
+  - host: "{{ EDXAPP_SEARCH_HOST }}"
+    port: "{{ EDXAPP_SEARCH_PORT }}"
 
 EDXAPP_SETTINGS: '{{ COMMON_EDXAPP_SETTINGS }}'
 
@@ -678,6 +681,17 @@ edxapp_theme_dir: "{{ edxapp_data_dir }}/themes"
 edxapp_git_identity: "{{ edxapp_app_dir }}/edxapp-git-identity"
 edxapp_git_ssh: "/tmp/edxapp_git_ssh.sh"
 
+edxapp_devstack_logs:
+  - "{{ nginx_log_dir }}/access.log"
+  - "{{ nginx_log_dir }}/error.log"
+  - "{{ supervisor_log_dir }}/cms-stdout.log"
+  - "{{ supervisor_log_dir }}/cms-stderr.log"
+  - "{{ supervisor_log_dir }}/lms-stdout.log"
+  - "{{ supervisor_log_dir }}/lms-stderr.log"
+
+# Only install packages which are appropriate for this environment
+edxapp_npm_production: "yes"
+
 # TODO: This can be removed once VPC-122 is resolved
 edxapp_legacy_course_data_dir: "{{ edxapp_app_dir }}/data"
 

playbooks/roles/edxapp/tasks/deploy.yml

@@ -242,7 +242,7 @@
   npm:
     executable: "{{ edxapp_nodeenv_bin }}/npm"
     path: "{{ edxapp_code_dir }}"
-    production: yes
+    production: "{{ edxapp_npm_production }}"
     state: latest
   environment: "{{ edxapp_environment }}"
   become_user: "{{ edxapp_user }}"

playbooks/roles/edxapp/tasks/main.yml

@@ -36,6 +36,18 @@
     - install
     - install:base
 
+- name: write devstack script
+  template:
+    src: "devstack.sh.j2"
+    dest: "{{ edxapp_app_dir }}/devstack.sh"
+    owner: "{{ supervisor_user }}"
+    group: "{{ common_web_user }}"
+    mode: 0744
+  when: devstack is defined and devstack
+  tags:
+    - devstack
+    - devstack:install
+
 # This is a symlink that has to exist because
 # we currently can't override the DATA_DIR var
 # in edx-platform. TODO: This can be removed once

playbooks/roles/edxapp/templates/devstack.sh.j2

+#!/usr/bin/env bash
+
+# {{ ansible_managed }}
+
+source {{ edxapp_app_dir }}/edxapp_env
+COMMAND=$1
+
+case $COMMAND in
+    start)
+        {{ supervisor_venv_bin }}/supervisord --configuration {{ supervisor_cfg }}
+
+        # Needed to run bower as root. See explanation around 'edx_django_service_user=root'
+        echo '{ "allow_root": true }' > /root/.bowerrc
+
+        # Docker requires an active foreground task. Tail the logs to appease Docker and
+        # provide useful output for development.
+        tail -f {{ edxapp_devstack_logs | join(" -f ") }}
+        ;;
+    open)
+        . {{ edxapp_nodeenv_bin }}/activate
+        . {{ edxapp_venv_bin }}/activate
+        cd {{ edxapp_code_dir }}
+
+        /bin/bash
+        ;;
+esac

playbooks/roles/edxapp/templates/edx/bin/edxapp-update-assets-cms.j2

 {% include "edxapp_common.j2" %}
 
-sudo -E -u {{ edxapp_user }} env "PATH=$PATH" {{ edxapp_venv_bin }}/paver update_assets cms --settings $EDX_PLATFORM_SETTINGS
+sudo -E -H -u {{ edxapp_user }} env "PATH=$PATH" {{ edxapp_venv_bin }}/paver update_assets cms --settings $EDX_PLATFORM_SETTINGS

playbooks/roles/edxapp/templates/edx/bin/edxapp-update-assets-lms.j2

 {% include "edxapp_common.j2" %}
 
-sudo -E -u {{ edxapp_user }} env "PATH=$PATH" {{ edxapp_venv_bin }}/paver update_assets lms --settings $EDX_PLATFORM_SETTINGS
+sudo -E -H -u {{ edxapp_user }} env "PATH=$PATH" {{ edxapp_venv_bin }}/paver update_assets lms --settings $EDX_PLATFORM_SETTINGS