Make devstack-style behavior the default and launch with that

* Start docker-sync and use the local host volumes by default
* Make the devstack workspace configurable, make travis use /tmp
* Remove redundant env vars, set -d on make devstack.up
* Add sleep to travis.yml
* Set a default value for DEVSTACK_WORKSPACE, and export it to all subshells in Make
* Rename devstack prefix to dev, add comment to makefile target
* Edits to the README to encourage setting up docker-sync before getting
started. Also update references to make targets.

.gitignore

@@ -75,9 +75,6 @@ target/
 # celery beat schedule file
 celerybeat-schedule
 
-# dotenv
-.env
-
 # virtualenv
 venv/
 ENV/
@@ -95,3 +92,4 @@ ENV/
 .idea/
 
 .dev/
+.docker-sync/

.travis.yml

@@ -5,13 +5,20 @@ language: python
 python:
   - "3.5"
 
+env:
+  - DEVSTACK_WORKSPACE=/tmp
+
 services:
   - docker
 
 before_install:
   - make requirements
+  - make dev.clone
 
 script:
-  - make provision
+  - make dev.provision
+  - make dev.up
+  - sleep 60 # LMS needs like 60 seconds to come up
   - make healthchecks
+  - make validate-lms-volume
   - make up-marketing-detached

Makefile

 .DEFAULT_GOAL := help
 
+DEVSTACK_WORKSPACE ?= ..
+export DEVSTACK_WORKSPACE
+
 include *.mk
 
 # Generates a help message. Borrowed from https://github.com/pydanny/cookiecutter-djangopackage.
@@ -10,22 +13,36 @@ help: ## Display this help message
 requirements: ## Install requirements
 	pip install -r requirements.txt
 
-clone: ## Clone service repos to the parent directory
+dev.clone: ## Clone service repos to the parent directory
 	./clone.sh
 
-provision: ## Provision all services
-	./provision.sh
+dev.provision.run: ## Provision all services with local mounted directories
+	DOCKER_COMPOSE_FILES="-f docker-compose.yml -f docker-compose-host.yml" ./provision.sh
+
+dev.provision: | dev.provision.run stop ## Provision dev environment with all services stopped
+
+dev.up: ## Bring up all services with host volumes
+	docker-compose -f docker-compose.yml -f docker-compose-host.yml up -d
+
+dev.sync.daemon.start: ## Start the docker-sycn daemon
+	docker-sync-daemon start
 
-up: ## Bring up all services with host volumes
-	docker-compose -f docker-compose.yml -f docker-compose-host.yml up
+dev.sync.provision: | dev.sync.daemon.start dev.provision ## Provision with docker-sync enabled
 
-up-sync: ## Bring up all services with docker-sync
-	docker-sync-stack start
+dev.sync.requirements: ## Install requirements
+	gem install docker-sync
+
+dev.sync.up: | dev.sync.daemon.start dev.up ## Bring up all services with docker-sync enabled
+
+provision: ## Provision all services using the Docker volume
+	./provision.sh
 
 stop: ## Stop all services
+	(test -d .docker-sync && docker-sync-daemon stop) || true ## Ignore failure here
 	docker-compose stop
 
 down: ## Remove all service containers and networks
+	test -d .docker-sync && docker-sync-daemon clean
 	docker-compose down
 
 destroy: ## Remove all devstack-related containers, networks, and volumes
@@ -60,3 +77,8 @@ studio-shell: ## Run a shell on the Studio container
 
 healthchecks: ## Run a curl against all services' healthcheck endpoints to make sure they are up. This will eventually be parameterized
 	./healthchecks.sh
+
+validate-lms-volume: ## Validate that changes to the local workspace are reflected in the LMS container
+	touch $(DEVSTACK_WORKSPACE)/edx-platform/testfile
+	docker exec edx.devstack.lms ls /edx/app/edxapp/edx-platform/testfile
+	rm $(DEVSTACK_WORKSPACE)/edx-platform/testfile

README.md

@@ -22,6 +22,16 @@ boot2docker) are not supported.
 [Docker for Windows][] may work but has not been tested and is _not supported_.
 
 
+### Docker Sync
+
+Docker for Mac has known filesystem issues that significantly decrease
+performance. In order to mitigate these issues, we use [Docker Sync][] to
+synchronize file data from the host machine to the containers.
+
+If you are using macOS, please follow the [Docker Sync installation instructions][]
+before provisioning.
+
+
 ## Getting Started
 
 All of the services can be run by following the steps below. Note that since we
@@ -30,16 +40,19 @@ amount of resources. Our testing found that [configuring Docker for Mac][] with
 2 CPUs and 4GB of memory works well.
 
 1.  The Docker Compose file mounts a host volume for each service's executing
-    code. The host directory is expected to be a sibling of this directory. For
+    code. The host directory is defaults to be a sibling of this directory. For
     example, if this repo is cloned to `~/workspace/devstack`, host volumes
     will be expected in `~/workspace/course-discovery`,
     `~/workspace/ecommerce`, etc. These repos can be cloned with the command
     below.
 
     ```sh
-    make clone
+    make dev.clone
     ```
 
+    You may customize where the local repositories are found by setting the
+    DEVSTACK_WORKSPACE environment variable.
+
 2.  Run the provision command, if you haven't already, to configure the various
     services with superusers (for development without the auth service) and
     tenants (for multi-tenancy).
@@ -48,16 +61,29 @@ amount of resources. Our testing found that [configuring Docker for Mac][] with
     the services directly via Django admin at the `/admin/` path, or login via
     single sign-on at `/login/`.
 
+    Provision using docker-sync (recommended for macOS users)
     ```sh
-    make provision
+    make dev.sync.provision
     ```
 
-3.  Start the services. By default this command will use host directories for
-    source code. This is known to be slow on macOS. macOS users should follow
-    the steps below for Docker Sync to avoid this performance hit.
+    Default (non-macOS users)
+    ```sh
+    make dev.provision
+    ```
 
+3.  Start the services. This command will mount the repositories under the
+    DEVSTACK_WORKSPACE directory.
+
+    _Note: it may take up to 60 seconds for the LMS to start_
+
+    Start using docker-sync (recommended for macOS users)
+    ```sh
+    make dev.sync.up
+    ```
+
+    Default (non-macOS users)
     ```sh
-    make up
+    make dev.up
     ```
 
 After the services have started, if you need shell access to one of the
@@ -74,27 +100,6 @@ To reset your environment and start provisioning from scratch, you can run:
 make destroy
 ```
 
-### Docker Sync
-
-Docker for Mac has known filesystem issues that significantly decrease
-performance. In order to mitigate these issues, we use [Docker Sync][] to
-synchronize file data from the host machine to the containers. Follow the steps
-below to setup Docker Sync.
-
-1.  Ensure all containers are stopped.
-
-    ```sh
-    make stop
-    ```
-
-2.  Follow the [Docker Sync installation instructions][].
-
-3.  Run Docker Sync and devstack.
-
-    ```sh
-    make up-sync
-    ```
-
 
 ## Usernames and Passwords
 
@@ -134,21 +139,11 @@ meant to be user-facing, the "homepage" may be the API root.
 
 ## Useful Commands
 
-Sometimes you may need to restart a particular application server. Rather than
-restarting the entire devstack or the individual container, you can instruct
-[Supervisor][] to restart just the nginx and application servers with one
-command.
-
-If you are already working in the shell of a container run:
+Sometimes you may need to restart a particular application server. To do so,
+simply use the ```docker-compose restart``` command:
 
 ```sh
-/edx/app/supervisor/venvs/supervisor/bin/supervisorctl restart all
-```
-
-Alternatively, if you are not working from a shell inside a container run:
-
-```sh
-docker exec -t edx.devstack.<service> bash -c '/edx/app/supervisor/venvs/supervisor/bin/supervisorctl restart all'
+docker-compose restart <service>
 ```
 
 `<service>` should be replaced with one of the following:
@@ -165,16 +160,6 @@ Docker Compose files useful for integrating with the edx.org marketing site are
 those outside of edX. For details on getting things up and running, see
 https://openedx.atlassian.net/wiki/display/ENG/Marketing+Site.
 
-## Remaining Work
-
-There is still work to be done before this is ready for full release to the
-Open edX community. Here are the major items:
-
-* [ ] Align with or revise [OEP-5][]
-* [ ] Finish provisioning all services
-* [ ] Load demo data
-* [ ] PyCharm debugging
-
 
 [Docker Compose]: https://docs.docker.com/compose/
 [Docker Sync installation instructions]: https://github.com/EugenMayer/docker-sync/wiki/1.-Installation

clone.sh

@@ -4,6 +4,16 @@
 # data volumes into their corresponding Docker containers to facilitate development.
 # Repos are cloned to the directory above the one housing this file.
 
+if [ -z "$DEVSTACK_WORKSPACE" ]; then
+    echo "need to set workspace dir"
+    exit 1
+elif [ -d "$DEVSTACK_WORKSPACE" ]; then
+    cd $DEVSTACK_WORKSPACE
+else
+    echo "Workspace directory $DEVSTACK_WORKSPACE doesn't exist"
+    exit 1
+fi
+
 repos=(
     "https://github.com/edx/course-discovery.git"
     "https://github.com/edx/credentials.git"
@@ -20,11 +30,10 @@ do
     [[ $repo =~ $name_pattern ]]
     name="${BASH_REMATCH[1]}"
 
-    cd ..
     if [ -d "$name" ]; then
         printf "The [%s] repo is already checked out. Continuing.\n" $name
     else
         git clone --depth 1 $repo
     fi
-    cd - &> /dev/null
 done
+cd - &> /dev/null

destroy.sh

@@ -6,5 +6,11 @@ read -p "This will delete all data in your devstack. Would you like to proceed?
 if [[ $REPLY =~ ^[Yy]$ ]]
 then
     echo
+    if [[ "$OSTYPE" == "darwin"* ]]; then
+        set +e
+        docker-sync-daemon stop
+        docker-sync-daemon clean
+        set -e
+    fi
     docker-compose down -v
 fi

docker-compose-host.yml

@@ -6,28 +6,28 @@ services:
       - /edx/app/credentials/credentials/credentials/assets/
       - /edx/app/credentials/credentials/credentials/static/bundles/
       - /edx/app/credentials/credentials/node_modules/
-      - ../credentials:/edx/app/credentials/credentials
+      - ${DEVSTACK_WORKSPACE}/credentials:/edx/app/credentials/credentials
   discovery:
       volumes:
       - /edx/app/discovery/discovery/course_discovery/assets/
       - /edx/app/discovery/discovery/course_discovery/static/bower_components/
       - /edx/app/discovery/discovery/course_discovery/static/build/
       - /edx/app/discovery/discovery/node_modules/
-      - ../course-discovery:/edx/app/discovery/discovery
+      - ${DEVSTACK_WORKSPACE}/course-discovery:/edx/app/discovery/discovery
   ecommerce:
     volumes:
       - /edx/app/ecommerce/ecommerce/assets/
       - /edx/app/ecommerce/ecommerce/ecommerce/static/bower_components/
       - /edx/app/ecommerce/ecommerce/ecommerce/static/build/
       - /edx/app/ecommerce/ecommerce/node_modules/
-      - ../ecommerce:/edx/app/ecommerce/ecommerce
+      - ${DEVSTACK_WORKSPACE}/ecommerce:/edx/app/ecommerce/ecommerce
   lms:
     volumes:
       - /edx/app/edxapp/edx-platform/.prereqs_cache/
       - /edx/app/edxapp/edx-platform/node_modules/
-      - ../edx-platform:/edx/app/edxapp/edx-platform
+      - ${DEVSTACK_WORKSPACE}/edx-platform:/edx/app/edxapp/edx-platform
   studio:
     volumes:
       - /edx/app/edxapp/edx-platform/.prereqs_cache/
       - /edx/app/edxapp/edx-platform/node_modules/
-      - ../edx-platform:/edx/app/edxapp/edx-platform
+      - ${DEVSTACK_WORKSPACE}/edx-platform:/edx/app/edxapp/edx-platform

provision-ida.sh

 name=$1
 port=$2
 
-docker-compose up -d $name
+docker-compose $DOCKER_COMPOSE_FILES up -d $name
 
 echo -e "${GREEN}Installing requirements for ${name}...${NC}"
 docker exec -t edx.devstack.${name}  bash -c 'source /edx/app/$1/$1_env && cd /edx/app/$1/$1/ && make requirements' -- "$name"

provision-lms.sh

+set -e
+set -o pipefail
+set -x
+
 # Load database dumps for the largest databases to save time
 ./load-db.sh edxapp
 ./load-db.sh edxapp_csmh
 
-# Bring the rest of the services online
-docker-compose up -d lms
+# Bring LMS online
+docker-compose $DOCKER_COMPOSE_FILES up -d lms
+
+docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && paver install_prereqs'
+
+#Installing prereqs crashes the process
+docker-compose restart lms
 
 # Run edxapp migrations first since they are needed for the service users and OAuth clients
-docker exec -t edx.devstack.lms  bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && paver install_prereqs'
-docker exec -t edx.devstack.lms  bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && NO_PREREQ_INSTALL=1 paver update_db --settings devstack_docker'
+docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && NO_PREREQ_INSTALL=1 paver update_db --settings devstack_docker'
 
 # Create a superuser for edxapp
-docker exec -t edx.devstack.lms  bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user edx edx@example.com --superuser --staff'
-docker exec -t edx.devstack.lms  bash -c 'source /edx/app/edxapp/edxapp_env && echo "from django.contrib.auth import get_user_model; User = get_user_model(); user = User.objects.get(username=\"edx\"); user.set_password(\"edx\"); user.save()" | python /edx/app/edxapp/edx-platform/manage.py lms shell  --settings=devstack_docker'
+docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker manage_user edx edx@example.com --superuser --staff'
+docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && echo "from django.contrib.auth import get_user_model; User = get_user_model(); user = User.objects.get(username=\"edx\"); user.set_password(\"edx\"); user.save()" | python /edx/app/edxapp/edx-platform/manage.py lms shell  --settings=devstack_docker'
 
 # Enable the LMS-E-Commerce integration
-docker exec -t edx.devstack.lms  bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker configure_commerce'
+docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && python /edx/app/edxapp/edx-platform/manage.py lms --settings=devstack_docker configure_commerce'
 
 # Create demo course and users
-docker exec -t edx.devstack.lms  bash -c '/edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook /edx/app/edx_ansible/edx_ansible/playbooks/edx-east/demo.yml -v -c local -i "127.0.0.1," --extra-vars="COMMON_EDXAPP_SETTINGS=devstack_docker"'
-
-# @TODO Why is this necessary? Right now installing prereqs crashes the server
-docker restart edx.devstack.lms
+docker-compose exec lms bash -c '/edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook /edx/app/edx_ansible/edx_ansible/playbooks/edx-east/demo.yml -v -c local -i "127.0.0.1," --extra-vars="COMMON_EDXAPP_SETTINGS=devstack_docker"'
 
-docker exec -t edx.devstack.lms  bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && paver update_assets --settings devstack_docker'
+docker-compose exec lms bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && paver update_assets --settings devstack_docker'

provision.sh

@@ -41,7 +41,7 @@ docker exec -i edx.devstack.mongo mongo < mongo-provision.js
 ./provision-lms.sh
 
 # Nothing special needed for studio
-docker-compose up -d studio
+docker-compose $DOCKER_COMPOSE_FILES up -d studio
 
 ./provision-ecommerce.sh
 #./provision-discovery.sh