Merge pull request #2552 from cpennington/course-discovery-static-files

Course discovery static files

docker/build/course_discovery/Dockerfile

@@ -19,8 +19,9 @@ USER docker
 WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays
 
 COPY docker/build/course_discovery/ansible_overrides.yml /
-RUN sudo ansible-playbook course_discovery.yml -c local \
-    -t 'install:base,install:code,install:system-requirements,install:app-requirements,install:configuration,install:vhosts,install:devstack' \
+RUN sudo ansible-playbook course_discovery.yml \
+    -c local -i '127.0.0.1,' \
+    -t 'install,assets' \
     --extra-vars="@/ansible_overrides.yml" \
     --extra-vars="COURSE_DISCOVERY_VERSION=$COURSE_DISCOVERY_VERSION" \
     --extra-vars="COMMON_GIT_PATH=$REPO_OWNER"

docker/build/course_discovery/ansible_overrides.yml

 ---
 course_discovery_gunicorn_host: 0.0.0.0
 COURSE_DISCOVERY_MYSQL: 'db'
+COURSE_DISCOVERY_DJANGO_SETTINGS_MODULE: 'course_discovery.settings.devstack'
+

docker/build/ansible.cfg → docker/plays/ansible.cfg

 [defaults]
 
 jinja2_extensions=jinja2.ext.do
-roles_path=../playbooks/roles
-library=../playbooks/library
+roles_path=../plays:../../playbooks/roles
+library=../../playbooks/library
 

playbooks/roles/ansible-role-django-ida/templates/defaults/main.yml.j2

@@ -13,7 +13,7 @@
 {{ role_name|upper }}_SSL_NGINX_PORT: 48{{ port_suffix }}
 
 {{ role_name|upper }}_DEFAULT_DB_NAME: '{{ role_name }}'
-{{ role_name|upper }}_MYSQL: 'db'
+{{ role_name|upper }}_MYSQL: 'localhost'
 # MySQL usernames are limited to 16 characters
 {{ role_name|upper }}_MYSQL_USER: '{{ role_name[:13] }}001'
 {{ role_name|upper }}_MYSQL_PASSWORD: 'password'
@@ -106,7 +106,14 @@ nginx_{{ role_name }}_gunicorn_hosts:
 {{ role_name }}_environment:
   DJANGO_SETTINGS_MODULE: "{{ '{{' }} {{ role_name|upper }}_DJANGO_SETTINGS_MODULE }}"
   {{ role_name|upper }}_CFG: "{{ '{{' }} COMMON_CFG_DIR }}/{{ '{{' }} {{ role_name }}_service_name }}.yml"
-  PATH: "{{ '{{' }} {{ role_name|lower }}_venv_dir }}/bin:${PATH}"
+  PATH: "{{ '{{' }} {{ role_name|lower }}_venv_dir }}/bin:{{ '{{' }} ansible_env.PATH }}"
+
+{{ role_name }}_migration_environment:
+  DJANGO_SETTINGS_MODULE: "{{ '{{' }} {{ role_name|upper }}_DJANGO_SETTINGS_MODULE }}"
+  {{ role_name|upper }}_CFG: "{{ '{{' }} COMMON_CFG_DIR }}/{{ '{{' }} {{ role_name }}_service_name }}.yml"
+  PATH: "{{ '{{' }} {{ role_name|lower }}_venv_dir }}/bin:{{ '{{' }} ansible_env.PATH }}"
+  DB_MIGRATION_USER: "{{ '{{' }} COMMON_MYSQL_MIGRATE_USER }}"
+  DB_MIGRATION_PASS: "{{ '{{' }} COMMON_MYSQL_MIGRATE_PASS }}"
 
 {{ role_name }}_service_name: "{{ role_name }}"
 {{ role_name }}_user: "{{ '{{' }} {{ role_name }}_service_name }}"

playbooks/roles/ansible-role-django-ida/templates/docker/build/ROLE_NAME/Dockerfile.j2

@@ -19,8 +19,9 @@ USER docker
 WORKDIR /edx/app/edx_ansible/edx_ansible/docker/plays
 
 COPY docker/build/{{ role_name }}/ansible_overrides.yml /
-RUN sudo ansible-playbook {{ role_name }}.yml -c local \
-    -t 'install:base,install:code,install:system-requirements,install:app-requirements,install:configuration,install:vhosts,install:devstack' \
+RUN sudo ansible-playbook {{ role_name }}.yml \
+    -c local -i '127.0.0.1,' \
+    -t 'install,assets' \
     --extra-vars="@/ansible_overrides.yml" \
     --extra-vars="{{ role_name|upper }}_VERSION=${{ role_name|upper }}_VERSION" \
     --extra-vars="COMMON_GIT_PATH=$REPO_OWNER"

playbooks/roles/ansible-role-django-ida/templates/docker/build/ROLE_NAME/ansible_overrides.yml.j2

 ---
-{{ role_name }}_gunicorn_host: 0.0.0.0
\ No newline at end of file
+{{ role_name }}_gunicorn_host: 0.0.0.0
+{{ role_name|upper }}_MYSQL: 'db'
+{{ role_name|upper }}_DJANGO_SETTINGS_MODULE: '{{ role_name }}.settings.devstack'
+

playbooks/roles/ansible-role-django-ida/templates/tasks/main.yml.j2

@@ -24,21 +24,21 @@
     - install:configuration
 
 - name: install application requirements
-  shell: >
-    chdir=/edx/app/{{ role_name }}/{{ role_name }}
-    . {{ '{{' }} {{ role_name }}_venv_dir }}/bin/activate;
-    make requirements
+  command: make requirements
+  args:
+    chdir: "{{ '{{' }} {{ role_name }}_code_dir }}"
   sudo_user: "{{ '{{' }} {{ role_name }}_user }}"
+  environment: "{{ '{{' }} {{ role_name }}_environment }}"
   tags:
     - install
     - install:app-requirements
 
 - name: install development requirements
-  shell: >
-    chdir=/edx/app/{{ role_name }}/{{ role_name }}
-    . {{ '{{' }} {{ role_name }}_venv_dir }}/bin/activate;
-    make local-requirements
+  command: make local-requirements
+  args:
+    chdir: "{{ '{{' }} {{ role_name }}_code_dir }}"
   sudo_user: "{{ '{{' }} {{ role_name }}_user }}"
+  environment: "{{ '{{' }} {{ role_name }}_environment }}"
   tags:
     - install
     - install:devstack
@@ -54,8 +54,8 @@
 - name: create databases
   mysql_db:
     login_host: "{{ '{{' }} {{ role_name|upper }}_DATABASES.default.HOST }}"
-    login_user: "{{ '{{' }} {{ role_name|upper }}_DB_ADMIN.default.USER }}"
-    login_password: "{{ '{{' }} {{ role_name|upper }}_DB_ADMIN.default.PASSWORD }}"
+    login_user: "{{ '{{' }} {{ role_name|upper }}_DB_ADMIN_USER }}"
+    login_password: "{{ '{{' }} {{ role_name|upper }}_DB_ADMIN_PASSWORD }}"
     db: "{{ '{{' }} {{ role_name|upper }}_DEFAULT_DB_NAME }}"
     state: present
     encoding: utf8
@@ -65,11 +65,11 @@
 - name: create database users
   mysql_user:
     login_host: "{{ '{{' }} {{ role_name|upper }}_DATABASES.default.HOST }}"
-    login_user: "{{ '{{' }} {{ role_name|upper }}_DB_ADMIN.default.USER }}"
-    login_password: "{{ '{{' }} {{ role_name|upper }}_DB_ADMIN.default.PASSWORD }}"
+    login_user: "{{ '{{' }} {{ role_name|upper }}_DB_ADMIN_USER }}"
+    login_password: "{{ '{{' }} {{ role_name|upper }}_DB_ADMIN_PASSWORD }}"
     name: "{{ '{{' }} item.name }}"
     host: "%"
-    name: "{{ '{{' }} item.password }}"
+    password: "{{ '{{' }} item.password }}"
     priv: "{{ '{{' }} {{ role_name|upper }}_DEFAULT_DB_NAME }}.*:ALL"
   with_items:
     - name: "{{ '{{' }} {{ role_name|upper }}_DATABASES.default.USER }}"
@@ -79,14 +79,12 @@
   tags:
     - migrate:devstack
 
-- name: migrate
-  shell: >
-    chdir={{ '{{' }} {{ role_name }}_code_dir }}
-    DB_MIGRATION_USER={{ '{{' }} COMMON_MYSQL_MIGRATE_USER }}
-    DB_MIGRATION_PASS={{ '{{' }} COMMON_MYSQL_MIGRATE_PASS }}
-    {{ '{{' }} {{ role_name|lower }}_venv_dir }}/bin/python ./manage.py migrate --noinput
+- name: migrate database
+  command: make migrate
+  args:
+    chdir: "{{ '{{' }} {{ role_name }}_code_dir }}"
   sudo_user: "{{ '{{' }} {{ role_name }}_user }}"
-  environment: "{{ '{{' }} {{ role_name }}_environment }}"
+  environment: "{{ '{{' }} {{ role_name }}_migration_environment }}"
   when: migrate_db is defined and migrate_db|lower == "yes"
   tags:
     - migrate
@@ -148,7 +146,7 @@
     - install:configuration
 
 - name: update supervisor configuration
-  shell: "{{ '{{' }} supervisor_ctl }} -c {{ '{{' }} supervisor_cfg }} update"
+  command: "{{ '{{' }} supervisor_ctl }} -c {{ '{{' }} supervisor_cfg }} update"
   when: not disable_edx_services
   tags:
     - manage
@@ -177,6 +175,16 @@
   tags:
     - install
     - install:app-requirements
+    
+- name: run collectstatic
+  command: make static
+  args:
+    chdir: "{{ '{{' }} {{ role_name }}_code_dir }}"
+  sudo_user: "{{ '{{' }} {{ role_name }}_user }}"
+  environment: "{{ '{{' }} {{ role_name }}_environment }}"
+  tags:
+    - assets
+    - assets:gather
 
 - name: restart the application
   supervisorctl: 

playbooks/roles/ansible-role-django-ida/templates/templates/edx/app/ROLE_NAME/devstack.sh.j2

@@ -2,12 +2,7 @@
 
 # {{ '{{' }} ansible_managed }}
 
-cd /edx/app/edx_ansible/edx_ansible/docker/plays
-sudo ansible-playbook {{ role_name }}.yml -c local -t 'install:app-requirements'
-sudo ansible-playbook {{ role_name }}.yml -c local -t 'migrate:devstack' \
-                      --extra-vars="migrate_db=yes"
-sudo ansible-playbook {{ role_name }}.yml -c local -t 'manage:devstack'
-
+source {{ '{{' }} {{ role_name }}_home }}/{{ role_name }}_env
 COMMAND=$1
 
 case $COMMAND in
@@ -15,6 +10,12 @@ case $COMMAND in
         {{ '{%' }} set {{ role_name }}_venv_bin = {{ role_name }}_home + "/venvs/" + {{ role_name }}_service_name + "/bin" %}
         {{ '{%' }} set executable = {{ role_name }}_venv_bin + '/gunicorn' %}
 
+        cd /edx/app/edx_ansible/edx_ansible/docker/plays
+        sudo ansible-playbook {{ role_name }}.yml -c local -i '127.0.0.1,' \
+            -t 'install:app-requirements,assets:gather,migrate:devstack,manage:devstack' \
+            --extra-vars="migrate_db=yes" \
+            --extra-vars="@/ansible_overrides.yml"
+
         {{ '{{' }} executable }} -c {{ '{{' }} {{ role_name }}_home }}/{{ role_name }}_gunicorn.py \
                                  {{ '{{' }} {{ role_name|upper }}_GUNICORN_EXTRA }} \
                                  --reload \

playbooks/roles/course_discovery/defaults/main.yml

@@ -113,7 +113,14 @@ course_discovery_venv_dir: "{{ course_discovery_home }}/venvs/{{ course_discover
 course_discovery_environment:
   DJANGO_SETTINGS_MODULE: "{{ COURSE_DISCOVERY_DJANGO_SETTINGS_MODULE }}"
   COURSE_DISCOVERY_CFG: "{{ COMMON_CFG_DIR }}/{{ course_discovery_service_name }}.yml"
-  PATH: "{{ course_discovery_venv_dir }}/bin:${PATH}"
+  PATH: "{{ course_discovery_venv_dir }}/bin:{{ ansible_env.PATH }}"
+
+course_discovery_migration_environment:
+  DJANGO_SETTINGS_MODULE: "{{ COURSE_DISCOVERY_DJANGO_SETTINGS_MODULE }}"
+  COURSE_DISCOVERY_CFG: "{{ COMMON_CFG_DIR }}/{{ course_discovery_service_name }}.yml"
+  PATH: "{{ course_discovery_venv_dir }}/bin:{{ ansible_env.PATH }}"
+  DB_MIGRATION_USER: "{{ COMMON_MYSQL_MIGRATE_USER }}"
+  DB_MIGRATION_PASS: "{{ COMMON_MYSQL_MIGRATE_PASS }}"
 
 course_discovery_service_name: "course_discovery"
 course_discovery_user: "{{ course_discovery_service_name }}"

playbooks/roles/course_discovery/tasks/main.yml

@@ -47,7 +47,7 @@
     - install:system-requirements
 
 - name: build virtualenv
-  shell: "virtualenv --python=python3.5 {{ course_discovery_venv_dir }}"
+  command: "virtualenv --python=python3.5 {{ course_discovery_venv_dir }}"
   args:
     creates: "{{ course_discovery_venv_dir }}/bin/pip"
   sudo_user: "{{ course_discovery_user }}"
@@ -56,21 +56,21 @@
     - install:system-requirements
 
 - name: install application requirements
-  shell: >
-    chdir=/edx/app/course_discovery/course_discovery
-    . {{ course_discovery_venv_dir }}/bin/activate;
-    make requirements
+  command: make requirements
+  args:
+    chdir: "{{ course_discovery_code_dir }}"
   sudo_user: "{{ course_discovery_user }}"
+  environment: "{{ course_discovery_environment }}"
   tags:
     - install
     - install:app-requirements
 
 - name: install development requirements
-  shell: >
-    chdir=/edx/app/course_discovery/course_discovery
-    . {{ course_discovery_venv_dir }}/bin/activate;
-    make local-requirements
+  command: make local-requirements
+  args:
+    chdir: "{{ course_discovery_code_dir }}"
   sudo_user: "{{ course_discovery_user }}"
+  environment: "{{ course_discovery_environment }}"
   tags:
     - install
     - install:devstack
@@ -101,7 +101,6 @@
     login_password: "{{ COURSE_DISCOVERY_DB_ADMIN_PASSWORD }}"
     name: "{{ item.name }}"
     host: "%"
-    name: "{{ item.name }}"
     password: "{{ item.password }}"
     priv: "{{ COURSE_DISCOVERY_DEFAULT_DB_NAME }}.*:ALL"
   with_items:
@@ -112,14 +111,12 @@
   tags:
     - migrate:devstack
 
-- name: migrate
-  shell: >
-    chdir={{ course_discovery_code_dir }}
-    DB_MIGRATION_USER={{ COMMON_MYSQL_MIGRATE_USER }}
-    DB_MIGRATION_PASS={{ COMMON_MYSQL_MIGRATE_PASS }}
-    {{ course_discovery_venv_dir }}/bin/python ./manage.py migrate --noinput
+- name: migrate database
+  command: make migrate
+  args:
+    chdir: "{{ course_discovery_code_dir }}"
   sudo_user: "{{ course_discovery_user }}"
-  environment: "{{ course_discovery_environment }}"
+  environment: "{{ course_discovery_migration_environment }}"
   when: migrate_db is defined and migrate_db|lower == "yes"
   tags:
     - migrate
@@ -181,7 +178,7 @@
     - install:configuration
 
 - name: update supervisor configuration
-  shell: "{{ supervisor_ctl }} -c {{ supervisor_cfg }} update"
+  command: "{{ supervisor_ctl }} -c {{ supervisor_cfg }} update"
   when: not disable_edx_services
   tags:
     - manage
@@ -210,6 +207,16 @@
   tags:
     - install
     - install:app-requirements
+    
+- name: run collectstatic
+  command: make static
+  args:
+    chdir: "{{ course_discovery_code_dir }}"
+  sudo_user: "{{ course_discovery_user }}"
+  environment: "{{ course_discovery_environment }}"
+  tags:
+    - assets
+    - assets:gather
 
 - name: restart the application
   supervisorctl: 

playbooks/roles/course_discovery/templates/edx/app/course_discovery/devstack.sh.j2

@@ -2,12 +2,7 @@
 
 # {{ ansible_managed }}
 
-cd /edx/app/edx_ansible/edx_ansible/docker/plays
-sudo ansible-playbook course_discovery.yml -c local -t 'install:app-requirements'
-sudo ansible-playbook course_discovery.yml -c local -t 'migrate:devstack' \
-                      --extra-vars="migrate_db=yes"
-sudo ansible-playbook course_discovery.yml -c local -t 'manage:devstack'
-
+source {{ course_discovery_home }}/course_discovery_env
 COMMAND=$1
 
 case $COMMAND in
@@ -15,6 +10,12 @@ case $COMMAND in
         {% set course_discovery_venv_bin = course_discovery_home + "/venvs/" + course_discovery_service_name + "/bin" %}
         {% set executable = course_discovery_venv_bin + '/gunicorn' %}
 
+        cd /edx/app/edx_ansible/edx_ansible/docker/plays
+        sudo ansible-playbook course_discovery.yml -c local -i '127.0.0.1,' \
+            -t 'install:app-requirements,assets:gather,migrate:devstack,manage:devstack' \
+            --extra-vars="migrate_db=yes" \
+            --extra-vars="@/ansible_overrides.yml"
+
         {{ executable }} -c {{ course_discovery_home }}/course_discovery_gunicorn.py \
                                  {{ COURSE_DISCOVERY_GUNICORN_EXTRA }} \
                                  --reload \

playbooks/roles/ecommerce/defaults/main.yml

@@ -195,7 +195,7 @@ ecommerce_requirements:
 ecommerce_environment:
   DJANGO_SETTINGS_MODULE: "{{ ECOMMERCE_DJANGO_SETTINGS_MODULE }}"
   ECOMMERCE_CFG: "{{ COMMON_CFG_DIR }}/{{ ecommerce_service_name }}.yml"
-  PATH: "{{ ecommerce_venv_dir }}/bin:${PATH}"
+  PATH: "{{ ecommerce_venv_dir }}/bin:{{ ansible_env.PATH }}"
 
 #
 # OS packages

playbooks/roles/programs/defaults/main.yml

@@ -93,7 +93,7 @@ programs_venv_dir: "{{ programs_home }}/venvs/{{ programs_service_name }}"
 programs_environment:
   DJANGO_SETTINGS_MODULE: "{{ PROGRAMS_DJANGO_SETTINGS_MODULE }}"
   PROGRAMS_CFG: "{{ COMMON_CFG_DIR }}/{{ programs_service_name }}.yml"
-  PATH: "{{ programs_venv_dir }}/bin:${PATH}"
+  PATH: "{{ programs_venv_dir }}/bin:{{ ansible_env.PATH }}"
 
 programs_service_name: "programs"
 programs_user: "{{ programs_service_name }}"