Merge pull request #2574 from cpennington/docker-ci-script

Add CI tasks for docker images

.dockerignore

 *
 !playbooks/
-!docker/
+!docker/build/*/ansible_overrides.yml
+!docker/plays/
 !util/install/

.gitignore

@@ -20,3 +20,7 @@ vagrant_ansible_inventory_default
 .AppleDouble
 :2e_*
 :2e#
+
+## Make artifacts
+.build
+playbooks/edx-east/travis-test.yml

.travis.yml

 # Travis CI configuration file for running tests
 language: python
+
 branches:
   only:
     - master
+
 python:
   - "2.7"
+
+services:
+  - docker
+  
+addons:
+  apt:
+    packages:
+      - nodejs
+      - python-demjson
+
 install:
-  - "sudo apt-get update -qq"
-  - "sudo apt-get install -y nodejs python-demjson"
   - "pip install --allow-all-external -r requirements.txt"
   - "pip install --allow-all-external demjson"
-script:
-  - |
-    for yml in $(find . -name "*.yml"); do
-      python -c "import sys,yaml; yaml.load(open(sys.argv[1]))" $yml >/dev/null
-      if [[ $? -ne 0 ]]; then
-        echo "ERROR parsing $yml"
-        exit 1
-      fi
-    done
-  - |
-    for json in $(find . -name "*.json"); do
-      jsonlint -v $json
-      if [[ $? -ne 0 ]]; then
-        echo "ERROR parsing $json"
-        exit 1
-      fi
-    done
-  - |
-    pushd playbooks
-    for j2 in $(git diff --name-only refs/heads/master..$(git rev-parse --abbrev-ref HEAD) |grep -E '.+\.j2$|.+\.yml'); do
-      python ../tests/jinja_check.py ../$j2
-      if [[ $? -ne 0 ]]; then
-          echo "Jinja parsing error on $j2"
-          exit 1
-      fi
-    done
-    popd
-  - |
-    set -e
-    ROOT_DIR=$PWD
-    cd playbooks/edx-east
-    ROLE_DIRS=$(/bin/ls -d roles/*)
-    cat <<EOF >travis-test.yml
-    - name: Play to test all roles
-      hosts: all
-      roles:
-    EOF
-    for role_dir in $ROLE_DIRS; do
-        echo "    - $(basename $role_dir)" >> travis-test.yml
-    done
-
-    ansible-playbook -i localhost, --syntax-check travis-test.yml
 
-    output_dir="$PWD/test_output/env-dep"
-    mkdir -p $output_dir
-    ansible-playbook -i localhost, -c local --tags edxapp_cfg edxapp.yml -e edxapp_user=`whoami` -e edxapp_app_dir=$output_dir -e edxapp_code_dir=$output_dir -e EDXAPP_CFG_DIR=$output_dir
-
-    root_dir=$output_dir
-    environment_deployments="."
-    source $ROOT_DIR/tests/validate_templates.sh
+env:
+  - MAKE_TARGET=docker.test.shard SHARD=0 SHARDS=3
+  - MAKE_TARGET=docker.test.shard SHARD=1 SHARDS=3
+  - MAKE_TARGET=docker.test.shard SHARD=2 SHARDS=3
+  - MAKE_TARGET=test.syntax
+  - MAKE_TARGET=test.edx_east_roles
 
+script:
+  - docker --version
+  - make --version
+  - make --keep-going $MAKE_TARGET SHARD=$SHARD SHARDS=$SHARDS

Makefile

+SHELL := /bin/bash
+.PHONY: help requirements clean build test pkg
+
+include *.mk
+
 help:
-	@echo '													'
-	@echo 'Makefile for the edX Configuration                                                               ' 
-	@echo '                                                                                     		'
-	@echo 'Usage:                                                                               		'
-	@echo '    make requirements                 install requirements					'
-	@echo '                                                                                     		'
+	@echo ''
+	@echo 'Makefile for the edX Configuration'
+	@echo ''
+	@echo 'Usage:'
+	@echo '    make requirements              install requirements'
+	@echo '    make test                      run all tests'
+	@echo '    make build                     build everything'
+	@echo '    make pkg                       package everything'
+	@echo '    make clean                     remove build by-products'
+	@echo ''
+	@echo '    Docker:'
+	@echo '        $$image: any dockerhub image'
+	@echo '        $$container: any container defined in docker/build/$$container/Dockerfile'
+	@echo ''
+	@echo '        make $(docker_pull)$$image        pull $$image from dockerhub'
+	@echo ''
+	@echo '        make $(docker_build)$$container   build $$container'
+	@echo '        make $(docker_test)$$container    test that $$container will build'
+	@echo '        make $(docker_pkg)$$container     package $$container for a push to dockerhub'
+	@echo '        make $(docker_push)$$container    push $$container to dockerhub '
+	@echo ''
+	@echo '        make docker.build          build all defined docker containers (based on dockerhub base images)'
+	@echo '        make docker.test           test all defined docker containers'
+	@echo '        make docker.pkg            package all defined docker containers (using local base images)'
+	@echo '        make docker.push           push all defined docker containers'
+	@echo ''
+	@echo '    Tests:'
+	@echo '        test.syntax                Run all syntax tests'
+	@echo '        test.syntax.json           Run syntax tests on .json files'
+	@echo '        test.syntax.yml            Run syntax tests on .yml files'
+	@echo '        test.syntax.jinja          Run syntax tests on .j2 files'
+	@echo '        test.edx_east_roles        Run validation on edx-east roles'
+	@echo ''
 
 requirements:
 	pip install -qr pre-requirements.txt --exists-action w
 	pip install -qr requirements.txt --exists-action w
-
-# Targets in a Makefile which do not produce an output file with the same name as the target name
-.PHONY: help requirements 

docker.mk

+.PHONY: docker.build docker.test docker.pkg
+
+SHARD=0
+SHARDS=1
+
+dockerfiles:=$(shell ls docker/build/*/Dockerfile)
+images:=$(patsubst docker/build/%/Dockerfile,%,$(dockerfiles))
+
+docker_build=docker.build.
+docker_test=docker.test.
+docker_pkg=docker.pkg.
+docker_push=docker.push.
+
+# N.B. / is used as a separator so that % will match the /
+# in something like 'edxops/trusty-common:latest'
+# Also, make can't handle ':' in filenames, so we instead '@'
+# which means the same thing to docker
+docker_pull=docker.pull/
+
+build: docker.build
+
+test: docker.test
+
+pkg: docker.pkg
+
+clean:
+	rm -rf .build
+
+docker.test.shard: $(foreach image,$(shell echo $(images) | tr ' ' '\n' | sed -n '$(SHARD)~$(SHARDS)p'),$(docker_test)$(image))
+
+docker.build: $(foreach image,$(images),$(docker_build)$(image))
+docker.test: $(foreach image,$(images),$(docker_test)$(image))
+docker.pkg: $(foreach image,$(images),$(docker_pkg)$(image))
+docker.push: $(foreach image,$(images),$(docker_push)$(image))
+
+$(docker_pull)%:
+	docker pull $*
+
+$(docker_build)%: docker/build/%/Dockerfile
+	docker build -f $< .
+
+$(docker_test)%: .build/%/Dockerfile.test
+	docker build -t $*:test -f $< .
+
+$(docker_pkg)%: .build/%/Dockerfile.pkg
+	docker build -t $*:latest -f $< .
+
+$(docker_push)%: $(docker_pkg)%
+	docker tag -f $*:latest edxops/$*:latest
+	docker push edxops/$*:latest
+
+
+.build/%/Dockerfile.d: docker/build/%/Dockerfile Makefile
+	@mkdir -p .build/$*
+	$(eval FROM=$(shell grep "FROM" $< | sed --regexp-extended "s/FROM //" | sed --regexp-extended "s/:/@/g"))
+	$(eval EDXOPS_FROM=$(shell echo "$(FROM)" | sed --regexp-extended "s#edxops/([^@]+)(@.*)?#\1#"))
+	@echo "$(docker_build)$*: $(docker_pull)$(FROM)" > $@
+	@if [ "$(EDXOPS_FROM)" != "$(FROM)" ]; then \
+	echo "$(docker_test)$*: $(docker_test)$(EDXOPS_FROM:@%=)" >> $@; \
+	echo "$(docker_pkg)$*: $(docker_pkg)$(EDXOPS_FROM:@%=)" >> $@; \
+	else \
+	echo "$(docker_test)$*: $(docker_pull)$(FROM)" >> $@; \
+	echo "$(docker_pkg)$*: $(docker_pull)$(FROM)" >> $@; \
+	fi
+
+.build/%/Dockerfile.test: docker/build/%/Dockerfile Makefile
+	@mkdir -p .build/$*
+	@sed --regexp-extended "s#FROM edxops/([^:]+)(:\S*)?#FROM \1:test#" $< > $@
+
+.build/%/Dockerfile.pkg: docker/build/%/Dockerfile Makefile
+	@mkdir -p .build/$*
+	@sed --regexp-extended "s#FROM edxops/([^:]+)(:\S*)?#FROM \1:test#" $< > $@
+
+include $(foreach image,$(images),.build/$(image)/Dockerfile.d)

docker/build/course-discovery/Dockerfile

@@ -7,19 +7,18 @@
 # This allows the dockerfile to update /edx/app/edx_ansible/edx_ansible
 # with the currently checked-out configuration repo.
 
-FROM edxops/trusty-common:hacking
+FROM edxops/trusty-common:latest
 MAINTAINER edxops
 
-ARG COURSE_DISCOVERY_VERSION=master
-ARG REPO_OWNER=edx
+ENV COURSE_DISCOVERY_VERSION=master
+ENV REPO_OWNER=edx
 
 ADD . /edx/app/edx_ansible/edx_ansible
 
-USER docker
 WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays
 
 COPY docker/build/course-discovery/ansible_overrides.yml /
-RUN sudo ansible-playbook course_discovery.yml \
+RUN sudo /edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook course_discovery.yml \
     -c local -i '127.0.0.1,' \
     -t 'install,assets,devstack:install' \
     --extra-vars="@/ansible_overrides.yml" \

docker/build/elasticsearch/Dockerfile → docker/build/elasticsearch/xDockerfile

 FROM edxops/precise-common:latest
 MAINTAINER edxops
 
-USER docker
 WORKDIR /edx/app/edx_ansible
 WORKDIR /edx/app/edx_ansible/edx_ansible/playbooks/edx-east
 RUN sudo git checkout e0d/docker-latest

playbooks/roles/ansible-role-django-ida/tasks/main.yml

@@ -12,7 +12,6 @@
   file: path=../docker/{{ item }}/{{ role_name }} state=directory
   with_items:
     - build
-    - plays
 
 - name: create role directories
   file: path=roles/{{ role_name }}/{{ item }} state=directory

playbooks/roles/ansible-role-django-ida/templates/defaults/main.yml.j2

@@ -73,6 +73,7 @@
   DATABASE_OPTIONS:
     connect_timeout: 10
   DATABASES: '{{ '{{' }} {{ role_name|upper }}_DATABASES }}'
+  CACHES: '{{ '{{' }} {{ role_name|upper }}_CACHES }}'
 
   PLATFORM_NAME: '{{ '{{' }} {{ role_name|upper }}_PLATFORM_NAME }}'
 

playbooks/roles/ansible-role-django-ida/templates/docker/build/ROLE_NAME/Dockerfile.j2

@@ -7,19 +7,18 @@
 # This allows the dockerfile to update /edx/app/edx_ansible/edx_ansible
 # with the currently checked-out configuration repo.
 
-FROM edxops/trusty-common
+FROM edxops/trusty-common:latest
 MAINTAINER edxops
 
-ARG {{ role_name|upper }}_VERSION=master
-ARG REPO_OWNER=edx
+ENV {{ role_name|upper }}_VERSION=master
+ENV REPO_OWNER=edx
 
 ADD . /edx/app/edx_ansible/edx_ansible
 
-USER docker
 WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays
 
 COPY docker/build/{{ role_name|replace("_","-") }}/ansible_overrides.yml /
-RUN sudo ansible-playbook {{ role_name }}.yml \
+RUN sudo /edx/app/edx_ansible/venvs/edx_ansible/bin/ansible-playbook {{ role_name }}.yml \
     -c local -i '127.0.0.1,' \
     -t 'install,assets,devstack:install' \
     --extra-vars="@/ansible_overrides.yml" \

playbooks/roles/ansible-role-django-ida/templates/meta/main.yml.j2

@@ -12,6 +12,8 @@
 #   my_role_var1: "bar"
 # }
 dependencies:
+  - common
+  - supervisor
   - role: edx_service
     edx_service_name: "{{ '{{' }} {{ role_name }}_service_name }}"
     edx_service_config: "{{ '{{' }} {{ role_name|upper }}_SERVICE_CONFIG }}"
@@ -21,4 +23,3 @@ dependencies:
     edx_service_packages:
       debian: "{{ '{{' }} {{ role_name }}_debian_pkgs }}"
       redhat: "{{ '{{' }} {{ role_name }}_redhat_pkgs }}"
-  - supervisor

playbooks/roles/ansible-role-django-ida/templates/templates/edx/app/ROLE_NAME/devstack.sh.j2

@@ -12,7 +12,7 @@ case $COMMAND in
 
         cd /edx/app/edx_ansible/edx_ansible/docker/plays
         sudo ansible-playbook {{ role_name }}.yml -c local -i '127.0.0.1,' \
-            -t 'install:app-requirements,assets:gather,devstack:migrate,devstack:manage' \
+            -t 'install:app-requirements,assets:gather,devstack' \
             --extra-vars="migrate_db=yes" \
             --extra-vars="@/ansible_overrides.yml"
 
@@ -23,7 +23,7 @@ case $COMMAND in
 
         ;;
     open)
-        cd {{ '{{' }} {{ role_name }}_home }}
+        cd {{ '{{' }} {{ role_name }}_code_dir }}/
         . {{ '{{' }} {{ role_name }}_venv_bin }}/activate
         /bin/bash
         ;;

playbooks/roles/course_discovery/defaults/main.yml

@@ -88,6 +88,7 @@ COURSE_DISCOVERY_SERVICE_CONFIG:
     connect_timeout: 10
   DATABASES: '{{ COURSE_DISCOVERY_DATABASES }}'
   ELASTICSEARCH: '{{ COURSE_DISCOVERY_ELASTICSEARCH }}'
+  CACHES: '{{ COURSE_DISCOVERY_CACHES }}'
 
   PLATFORM_NAME: '{{ COURSE_DISCOVERY_PLATFORM_NAME }}'
 

playbooks/roles/course_discovery/templates/edx/app/course_discovery/devstack.sh.j2

@@ -12,7 +12,7 @@ case $COMMAND in
 
         cd /edx/app/edx_ansible/edx_ansible/docker/plays
         sudo ansible-playbook course_discovery.yml -c local -i '127.0.0.1,' \
-            -t 'install:app-requirements,assets:gather,devstack:migrate,devstack:manage' \
+            -t 'install:app-requirements,assets:gather,devstack' \
             --extra-vars="migrate_db=yes" \
             --extra-vars="@/ansible_overrides.yml"
 
@@ -23,7 +23,7 @@ case $COMMAND in
 
         ;;
     open)
-        cd {{ course_discovery_home }}
+        cd {{ course_discovery_code_dir }}
         . {{ course_discovery_venv_bin }}/activate
         /bin/bash
         ;;

test.mk

+
+yml_files:=$(shell find . -name "*.yml")
+json_files:=$(shell find . -name "*.json")
+jinja_files:=$(shell find . -name "*.j2")
+
+test: test.syntax test.edx_east_roles
+
+test.syntax: test.syntax.yml test.syntax.json test.syntax.jinja
+
+test.syntax.yml: $(patsubst %,test.syntax.yml/%,$(yml_files))
+
+test.syntax.yml/%:
+	python -c "import sys,yaml; yaml.load(open(sys.argv[1]))" $* >/dev/null
+
+test.syntax.json: $(patsubst %,test.syntax.json/%,$(json_files))
+
+test.syntax.json/%:
+	jsonlint -v $*
+
+test.syntax.jinja: $(patsubst %,test.syntax.jinja/%,$(jinja_files))
+
+test.syntax.jinja/%:
+	cd playbooks && python ../tests/jinja_check.py ../$*
+
+test.edx_east_roles:
+	tests/test_edx_east_roles.sh

tests/test_edx_east_roles.sh

+set -e
+
+ROOT_DIR=$PWD
+cd playbooks/edx-east
+ROLE_DIRS=$(/bin/ls -d roles/*)
+cat <<EOF >travis-test.yml
+- name: Play to test all roles
+  hosts: all
+  roles:
+EOF
+for role_dir in $ROLE_DIRS; do
+    echo "    - $(basename $role_dir)" >> travis-test.yml
+done
+
+ansible-playbook -i localhost, --syntax-check travis-test.yml
+
+output_dir="$PWD/test_output/env-dep"
+mkdir -p $output_dir
+ansible-playbook -i localhost, -c local --tags edxapp_cfg edxapp.yml -e edxapp_user=`whoami` -e edxapp_app_dir=$output_dir -e edxapp_code_dir=$output_dir -e EDXAPP_CFG_DIR=$output_dir
+
+root_dir=$output_dir
+environment_deployments="."
+source $ROOT_DIR/tests/validate_templates.sh

tests/validate_templates.sh

@@ -3,6 +3,7 @@
 # The 'root_dir' and 'environment_deployments' variables
 # should be set when we source this.
 
+FAIL=0
 for e_d in $environment_deployments
 do
   GREP_DIR="$root_dir/${e_d}"