Merge branch 'jibsheet/test-courses-more' of github.com:edx/configuration into…

Merge branch 'jibsheet/test-courses-more' of github.com:edx/configuration into jlajoie/move-elasticsearch-init

playbooks/roles/go-server/defaults/main.yml

@@ -53,6 +53,14 @@ GO_SERVER_BACKUP_PASSWORD: ""
 GO_SERVER_GOMATIC_USERNAME: ""
 GO_SERVER_GOMATIC_PASSWORD: ""
 
+GO_SERVER_USERS:
+    - username: "{{ GO_SERVER_ADMIN_USERNAME }}"
+      password: "{{ GO_SERVER_ADMIN_PASSWORD }}"
+    - username: "{{ GO_SERVER_BACKUP_USERNAME }}"
+      password: "{{ GO_SERVER_BACKUP_PASSWORD }}"
+    - username: "{{ GO_SERVER_GOMATIC_USERNAME }}"
+      password: "{{ GO_SERVER_GOMATIC_PASSWORD }}"
+
 # go-server network settings
 GO_SERVER_PORT: 8153
 GO_SERVER_SECURE_PORT: 8154

playbooks/roles/go-server/tasks/main.yml

@@ -74,31 +74,6 @@
     - { url: "{{ GO_SERVER_OAUTH_LOGIN_JAR_URL }}", md5: "{{ GO_SERVER_OAUTH_LOGIN_MD5 }}" }
     - { url: "{{ GO_SERVER_GITHUB_PR_PLUGIN_JAR_URL }}", md5: "{{ GO_SERVER_GITHUB_PR_PLUGIN_MD5 }}" }
 
-- name: generate line for go-server password file for admin user
-  command: "/usr/bin/htpasswd -nbs \"{{ GO_SERVER_ADMIN_USERNAME }}\" \"{{ GO_SERVER_ADMIN_PASSWORD }}\""
-  register: admin_user_password_line
-  when: GO_SERVER_ADMIN_USERNAME and GO_SERVER_ADMIN_PASSWORD
-
-- name: generate line for go-server password file for backup user
-  command: "/usr/bin/htpasswd -nbs \"{{ GO_SERVER_BACKUP_USERNAME }}\" \"{{ GO_SERVER_BACKUP_PASSWORD }}\""
-  register: backup_user_password_line
-  when: GO_SERVER_BACKUP_USERNAME and GO_SERVER_BACKUP_PASSWORD
-
-- name: generate line for go-server password file for gomatic user
-  command: "/usr/bin/htpasswd -nbs \"{{ GO_SERVER_GOMATIC_USERNAME }}\" \"{{ GO_SERVER_GOMATIC_PASSWORD }}\""
-  register: gomatic_user_password_line
-  when: GO_SERVER_GOMATIC_USERNAME and GO_SERVER_GOMATIC_PASSWORD
-
-- name: setup password file
-  template:
-    src: edx/app/go-server/password.txt.j2
-    dest: "{{ GO_SERVER_CONF_HOME }}/{{ GO_SERVER_PASSWORD_FILE_NAME }}"
-    mode: 0600
-    owner: "{{ GO_SERVER_USER }}"
-    group: "{{ GO_SERVER_GROUP }}"
-    force: no
-  when: GO_SERVER_ADMIN_PASSWORD and GO_SERVER_BACKUP_PASSWORD and GO_SERVER_GOMATIC_PASSWORD
-
 - name: install go-server configuration
   template:
     src: edx/app/go-server/cruise-config.xml.j2
@@ -111,16 +86,29 @@
 - include: download_backup.yml
   when: GO_SERVER_BACKUP_S3_BUCKET and GO_SERVER_BACKUP_S3_OBJECT and GO_SERVER_RESTORE_BACKUP
 
-- name: replace the password lines in the password file post-restore
+- name: generate lines for go-server password file for users
+  command: "/usr/bin/htpasswd -nbs '{{ item.username }}' '{{ item.password }}'"
+  with_items: "{{ GO_SERVER_USERS }}"
+  when: item.username and item.password
+  register: go_server_users_htpasswds
+  tags:
+    - install
+    - install:app-configuration
+
+- name: replace the password lines in the password file
   lineinfile:
     dest: "{{ GO_SERVER_CONF_HOME }}/{{ GO_SERVER_PASSWORD_FILE_NAME }}"
-    regexp: "^{{ item.username }}"
-    line: "{{ item.password_hash.stdout }}"
-  with_items:
-    - { username: "{{ GO_SERVER_ADMIN_USERNAME }}", password: "{{ GO_SERVER_ADMIN_PASSWORD }}", password_hash: "{{ admin_user_password_line }}" }
-    - { username: "{{ GO_SERVER_BACKUP_USERNAME }}", password: "{{ GO_SERVER_BACKUP_PASSWORD }}", password_hash: "{{ backup_user_password_line }}" }
-    - { username: "{{ GO_SERVER_GOMATIC_USERNAME }}", password: "{{ GO_SERVER_GOMATIC_PASSWORD }}", password_hash: "{{ gomatic_user_password_line }}" }
-  when: item.username and item.password and item.password_hash
+    regexp: "^{{ item.item.username }}"
+    line: "{{ item.stdout }}"
+    create: yes
+    mode: "0600"
+    owner: "{{ GO_SERVER_USER }}"
+    group: "{{ GO_SERVER_GROUP }}"
+  with_items: "{{ go_server_users_htpasswds.results }}"
+  when: item.item.username and item.item.password and item.stdout
+  tags:
+    - install
+    - install:app-configuration
 
 - name: install ssh key for the secure repos
   copy:

playbooks/roles/go-server/templates/edx/app/go-server/password.txt.j2

-{{ admin_user_password_line.stdout }}
-{{ backup_user_password_line.stdout }}
-{{ gomatic_user_password_line.stdout }}

playbooks/roles/rabbitmq/tasks/main.yml

@@ -51,6 +51,8 @@
     owner: "{{ rabbitmq_user }}"
     group: "{{ rabbitmq_group }}"
     mode: "0755"
+  tags:
+    - "monitoring"
 
 - name: Add RabbitMQ memory usage script
   template:
@@ -59,16 +61,22 @@
     owner: "{{ rabbitmq_user }}"
     group: "{{ rabbitmq_group }}"
     mode: "0775"
+  tags:
+    - "monitoring"
 
 - name: Set up a cron job to run queue script
   cron:
     name: "log-queue-lenghts"
     job: "{{ rabbitmq_app_dir }}/log-rabbitmq-queues.sh >/dev/null 2>&1"
+  tags:
+    - "monitoring"
 
 - name: Set up a cron job to run the script
   cron:
     name: "log-rabbitmq-memory-usage"
     job: "{{ rabbitmq_app_dir }}/log-rabbitmq-memory.sh >/dev/null 2>&1"
+  tags:
+    - "monitoring"
 
 - name: install logrotate configuration
   template:

playbooks/roles/testcourses/tasks/deploy.yml

 ---
-- name: import the test courses from github
-  shell: "{{ demo_edxapp_venv_bin }}/python /edx/bin/manage.edxapp lms git_add_course --settings={{ demo_edxapp_settings }} \"{{ item.github_url }}\""
-  become_user: "{{ common_web_user }}"
-  when: item.install == True
+- include: import_course.yml
+  when: course.install == True
   with_items: "{{ TESTCOURSES_EXPORTS }}"
+  loop_control:
+    loop_var: course
 
 - name: enroll test users in the testcourses
   shell: "{{ demo_edxapp_venv_bin }}/python ./manage.py lms --settings={{ demo_edxapp_settings }} --service-variant lms enroll_user_in_course -e {{ item[0].email }} -c {{ item[1].course_id }}"

playbooks/roles/testcourses/tasks/import_course.yml

+- name: check out testcourses
+  git:
+    dest: "{{ demo_app_dir }}/{{ course.course_id }}"
+    repo: "{{ course.github_url }}"
+    accept_hostkey: yes
+  become_user: "{{ demo_edxapp_user }}"
+  register: testcourse_checkout
+
+- name: import testcourse
+  shell: "{{ demo_edxapp_venv_bin }}/python ./manage.py cms --settings={{ demo_edxapp_settings }} import {{ demo_app_dir }} {{ course.course_id }}"
+  args:
+    chdir: "{{ demo_edxapp_code_dir }}"
+  when: testcourse_checkout.changed
+  become_user: "{{ common_web_user }}"

playbooks/vagrant-fullstack.yml

-- name: Configure instance(s)
-  hosts: all
-  become: True
-  gather_facts: True
-  vars:
-    migrate_db: 'yes'
-    EDXAPP_LMS_NGINX_PORT: '80'
-    EDX_ANSIBLE_DUMP_VARS: true
-    CERTS_DOWNLOAD_URL: 'http://192.168.33.10:18090'
-    CERTS_VERIFY_URL: 'http://192.168.33.10:18090'
-    # used for releases
-    edx_platform_version: '{{ OPENEDX_RELEASE | default("master") }}'
-    certs_version: '{{ OPENEDX_RELEASE | default("master") }}'
-    forum_version: '{{ OPENEDX_RELEASE | default("master") }}'
-    xqueue_version: '{{ OPENEDX_RELEASE | default("master") }}'
-    demo_version: '{{ OPENEDX_RELEASE | default("master") }}'
-  roles:
-    - common
-    - vhost
-    - edx_ansible
-    - user
-    - role: nginx
-      nginx_sites:
-      - cms
-      - lms
-      - forum
-      - xqueue
-      - certs
-      nginx_default_sites:
-      - lms
-      - cms
-    - mysql
-    - edxlocal
-    - memcache
-    - mongo
-    - { role: 'rabbitmq', rabbitmq_ip: '127.0.0.1' }
-    - edxapp
-    - { role: 'edxapp', celery_worker: True }
-    - demo
-    - oraclejdk
-    - elasticsearch
-    - forum
-    - { role: "xqueue", update_users: True }
-    - certs
-    - role: analytics_api
-      when: ANALYTICS_API_GIT_IDENTITY