Merge branch 'mtyaka/optional-aws-opencraft-roles' of…

Merge branch 'mtyaka/optional-aws-opencraft-roles' of git://github.com/open-craft/configuration into open-craft-mtyaka/optional-aws-opencraft-roles

CHANGELOG.md

+- Role: common_vars
+  - Added `COMMON_ENABLE_AWS_INTEGRATION` to run the `aws` role when enabled. Default: `False`
+  - Added `COMMON_ENABLE_OPENSTACK_INTEGRATION` to run the `openstack` role when enabled. Default: `False`
 - Role: credentials
   - Added `CREDENTIALS_EXTRA_APPS` to enable the inclusion of additional Django apps in the Credentials Service.
 - Role: common

playbooks/edx-east/retire_host.yml

@@ -40,7 +40,7 @@
         - "edx-services"
     # This catches the case where tracking.log is 0b
     - name: Sync again
-      command: /edx/bin/send-logs-to-object-store -d "{{ COMMON_LOG_DIR }}/tracking/" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}/logs/tracking"
+      command: /edx/bin/send-logs-to-s3 -d "{{ COMMON_LOG_DIR }}/tracking/" -b "{{ COMMON_S3_LOG_SYNC_BUCKET }}/logs/tracking"
     
 - name: Run minos verification
   hosts: "{{TARGET}}"

playbooks/edx-stateless.yml

@@ -38,6 +38,12 @@
       celery_worker: True
     - edxapp
 
+    # AWS / OpenStack libraries & tools
+    - role: aws
+      when: COMMON_ENABLE_AWS_INTEGRATION
+    - role: openstack
+      when: COMMON_ENABLE_OPENSTACK_INTEGRATION
+
     # Discussion forums
     # https://github.com/edx/cs_comments_service
     - forum

playbooks/edx_sandbox.yml

@@ -42,6 +42,10 @@
     - { role: 'rabbitmq', rabbitmq_ip: '127.0.0.1' }
     - { role: 'edxapp', celery_worker: True }
     - edxapp
+    - role: aws
+      when: COMMON_ENABLE_AWS_INTEGRATION
+    - role: openstack
+      when: COMMON_ENABLE_OPENSTACK_INTEGRATION
     - role: ecommerce
       when: SANDBOX_ENABLE_ECOMMERCE
     - role: ecomworker

playbooks/roles/aws/defaults/main.yml

@@ -11,14 +11,6 @@
 # Defaults for role aws
 #
 
-# Both of these vars are required to work-around
-# some ansible variable precedence issues with
-# circular dependencies introduced in the openstack PR.
-# More investigation is required to determine the optimal
-# solution.
-vhost_name: aws
-VHOST_NAME: "{{ vhost_name }}"
-
 # If there are any issues with the s3 sync an error
 # log will be sent to the following address.
 # This relies on your server being able to send mail
@@ -29,8 +21,22 @@ AWS_S3_LOGS_FROM_EMAIL: dummy@example.com
 AWS_S3_LOGS_ACCESS_KEY_ID: ""
 AWS_S3_LOGS_SECRET_KEY: ""
 
-aws_s3_sync_script: "{{ vhost_dirs.home.path }}/send-logs-to-s3"
-aws_s3_logfile: "{{ vhost_dirs.logs.path }}/s3-log-sync.log"
+aws_role_name: aws
+
+aws_dirs:
+  home:
+    path: "{{ COMMON_APP_DIR }}/{{ aws_role_name }}"
+    owner: "root"
+    group: "root"
+    mode: "0755"
+  logs:
+    path: "{{ COMMON_LOG_DIR }}/{{ aws_role_name }}"
+    owner: "syslog"
+    group: "syslog"
+    mode: "0650"
+
+aws_s3_sync_script: "{{ aws_dirs.home.path }}/send-logs-to-s3"
+aws_s3_logfile: "{{ aws_dirs.logs.path }}/s3-log-sync.log"
 aws_region: "us-east-1"
 # default path to the aws binary
 aws_s3cmd: "/usr/local/bin/s3cmd"

playbooks/roles/aws/meta/main.yml

@@ -11,5 +11,4 @@
 # Role includes for role aws
 # 
 dependencies:
-  - role: vhost
-    VHOST_NAME: "{{ vhost_name }}"
+  - vhost

playbooks/roles/aws/tasks/main.yml

@@ -67,6 +67,15 @@
     extra_args: "-i {{ COMMON_PYPI_MIRROR_URL }}"
   with_items: "{{ aws_pip_pkgs }}"
 
+- name: Create required directories
+  file:
+    path: "{{ item.value.path }}"
+    state: directory
+    owner: "{{ item.value.owner }}"
+    group: "{{ item.value.group }}"
+    mode: "{{ item.value.mode }}"
+  with_dict: "{{ aws_dirs }}"
+
 - name: Create s3 log sync script
   template:
     dest: "{{ aws_s3_sync_script }}"
@@ -74,14 +83,14 @@
     mode: 0755
     owner: root
     group: root
-  when: COMMON_OBJECT_STORE_LOG_SYNC
+  when: COMMON_S3_LOG_SYNC
 
 - name: Create symlink for s3 log sync script
   file:
     state: link
     src: "{{ aws_s3_sync_script }}"
-    dest: "{{ COMMON_OBJECT_STORE_LOG_SYNC_SCRIPT }}"
-  when: COMMON_OBJECT_STORE_LOG_SYNC
+    dest: "{{ COMMON_S3_LOG_SYNC_SCRIPT }}"
+  when: COMMON_S3_LOG_SYNC
 
  # update the ssh motd on Ubuntu
  # Remove some of the default motd display on ubuntu

playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate_tracking_log.j2

@@ -14,8 +14,11 @@
     /usr/bin/killall -HUP rsyslogd
   endscript
   lastaction
-    {% if COMMON_OBJECT_STORE_LOG_SYNC -%}
-    {{ COMMON_OBJECT_STORE_LOG_SYNC_SCRIPT }} -d "{{ COMMON_LOG_DIR }}/tracking" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_LOG_SYNC_PREFIX }}"
+    {% if COMMON_S3_LOG_SYNC -%}
+    {{ COMMON_S3_LOG_SYNC_SCRIPT }} -d "{{ COMMON_LOG_DIR }}/tracking" -b "{{ COMMON_S3_LOG_SYNC_BUCKET }}" -p "{{ COMMON_S3_LOG_SYNC_PREFIX }}"
+    {% endif -%}
+    {% if COMMON_SWIFT_LOG_SYNC -%}
+    {{ COMMON_SWIFT_LOG_SYNC_SCRIPT }} -d "{{ COMMON_LOG_DIR }}/tracking" -b "{{ COMMON_SWIFT_LOG_SYNC_BUCKET }}" -p "{{ COMMON_SWIFT_LOG_SYNC_PREFIX }}"
     {% endif -%}
   endscript
 }

playbooks/roles/common_vars/defaults/main.yml

@@ -14,13 +14,27 @@ COMMON_BASIC_AUTH_EXCEPTIONS:
 # Settings to use for calls to edxapp manage.py
 COMMON_EDXAPP_SETTINGS: 'aws'
 
+# Set to True to install aws/openstack role
+# when running edx_sandbox or edx-stateless playbook.
+COMMON_ENABLE_AWS_INTEGRATION: False
+COMMON_ENABLE_OPENSTACK_INTEGRATION: False
+
 # Turn on syncing logs on rotation for edx
 # application and tracking logs, must also
 # have the aws or openstack role installed
 COMMON_OBJECT_STORE_LOG_SYNC: False
 COMMON_OBJECT_STORE_LOG_SYNC_BUCKET: "edx-{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}"
 COMMON_OBJECT_STORE_LOG_SYNC_PREFIX: "logs/tracking/"
-COMMON_OBJECT_STORE_LOG_SYNC_SCRIPT: "{{ COMMON_BIN_DIR }}/send-logs-to-object-store"
+
+COMMON_S3_LOG_SYNC: "{{ COMMON_OBJECT_STORE_LOG_SYNC and COMMON_ENABLE_AWS_INTEGRATION }}"
+COMMON_S3_LOG_SYNC_BUCKET: "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}"
+COMMON_S3_LOG_SYNC_PREFIX: "{{ COMMON_OBJECT_STORE_LOG_SYNC_PREFIX }}"
+COMMON_S3_LOG_SYNC_SCRIPT: "{{ COMMON_BIN_DIR }}/send-logs-to-s3"
+
+COMMON_SWIFT_LOG_SYNC: "{{ COMMON_ENABLE_OPENSTACK_INTEGRATION and COMMON_OBJECT_STORE_LOG_SYNC }}"
+COMMON_SWIFT_LOG_SYNC_BUCKET: "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}"
+COMMON_SWIFT_LOG_SYNC_PREFIX: "{{ COMMON_OBJECT_STORE_LOG_SYNC_PREFIX }}"
+COMMON_SWIFT_LOG_SYNC_SCRIPT: "{{ COMMON_BIN_DIR }}/send-logs-to-swift"
 
 COMMON_BASE_DIR: /edx
 COMMON_DATA_DIR: "{{ COMMON_BASE_DIR}}/var"
@@ -88,6 +102,7 @@ COMMON_MYSQL_MIGRATE_PASS: 'password'
 
 COMMON_MONGO_READ_ONLY_USER: 'read_only'
 COMMON_MONGO_READ_ONLY_PASS: !!null
+
 COMMON_ENABLE_DATADOG: False
 COMMON_ENABLE_NGINXTRA: False
 COMMON_ENABLE_SPLUNKFORWARDER: False

playbooks/roles/edxapp/defaults/main.yml

@@ -1126,6 +1126,7 @@ base_requirements_file:   "{{ edxapp_code_dir }}/requirements/edx/base.txt"
 post_requirements_file:   "{{ edxapp_code_dir }}/requirements/edx/post.txt"
 paver_requirements_file:   "{{ edxapp_code_dir }}/requirements/edx/paver.txt"
 private_requirements_file:   "{{ edxapp_code_dir }}/requirements/edx/edx-private.txt"
+openstack_requirements_file: "{{ edxapp_code_dir }}/requirements/edx/openstack.txt"
 
 sandbox_base_requirements:  "{{ edxapp_code_dir }}/requirements/edx-sandbox/base.txt"
 sandbox_local_requirements: "{{ edxapp_code_dir }}/requirements/edx-sandbox/local.txt"
@@ -1136,6 +1137,8 @@ sandbox_post_requirements:  "{{ edxapp_code_dir }}/requirements/edx-sandbox/post
 # edx-solutions fork of edx-platform when if you consider reordering this list.)
 # private_requirements_file is not included in this list, since it is only installed
 # conditionally based on the value of EDXAPP_INSTALL_PRIVATE_REQUIREMENTS.
+# openstack_requirements_file is not included in this list, because it is only installed
+# conditionally based on the value of COMMON_ENABLE_OPENSTACK_INTEGRATION.
 edxapp_requirements_files:
   - "{{ pre_requirements_file }}"
   - "{{ github_requirements_file }}"
@@ -1156,6 +1159,7 @@ edxapp_requirements_with_github_urls:
   - "{{ post_requirements_file }}"
   - "{{ paver_requirements_file }}"
   - "{{ private_requirements_file }}"
+  - "{{ openstack_requirements_file }}"
   - "{{ sandbox_post_requirements }}"
   - "{{ sandbox_local_requirements }}"
   - "{{ sandbox_base_requirements }}"

playbooks/roles/edxapp/tasks/deploy.yml

@@ -164,6 +164,21 @@
     - install
     - install:app-requirements
 
+# Conditinally install openstack requirements.
+- name: install python private requirements
+  shell: "{{ edxapp_venv_dir }}/bin/pip install {{ COMMON_PIP_VERBOSITY }} -i {{ COMMON_PYPI_MIRROR_URL }} --exists-action w -r {{ item }}"
+  args:
+    chdir: "{{ edxapp_code_dir }}"
+  with_items:
+  - "{{ openstack_requirements_file }}"
+  become_user: "{{ edxapp_user }}"
+  environment:
+    GIT_SSH: "{{ edxapp_git_ssh }}"
+  when: COMMON_ENABLE_OPENSTACK_INTEGRATION
+  tags:
+    - install
+    - install:app-requirements
+
 # Install any custom extra requirements if defined in EDXAPP_EXTRA_REQUIREMENTS.
 - name: install python extra requirements
   pip:

playbooks/roles/minos/defaults/main.yml

@@ -15,7 +15,7 @@ MINOS_GIT_IDENTITY: !!null
 MINOS_SERVICE_CONFIG:
   aws_profile: !!null
   aws_region: "{{ MINOS_AWS_REGION }}"
-  s3_bucket: "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}"
+  s3_bucket: "{{ COMMON_S3_LOG_SYNC_BUCKET }}"
   bucket_path: lifecycle/minos
   voter_conf_d: "{{ minos_voter_cfg }}"
 

playbooks/roles/minos/templates/edx/etc/minos/conf.d/TrackingLogVoter.yml.j2

@@ -2,5 +2,5 @@ TrackingLogVoter:
   config:
     aws_profile: !!null
     local_directory: '{{ COMMON_LOG_DIR }}/tracking'
-    s3_bucket: '{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}'
+    s3_bucket: '{{ COMMON_S3_LOG_SYNC_BUCKET }}'
     bucket_path_prefix: 'logs/tracking'

playbooks/roles/openstack/defaults/main.yml

@@ -11,14 +11,6 @@
 # Defaults for role openstack
 #
 
-# Both of these vars are required to work-around
-# some ansible variable precedence issues with
-# circular dependencies introduced in the openstack PR.
-# More investigation is required to determine the optimal
-# solution.
-vhost_name: openstack
-VHOST_NAME: "{{ vhost_name }}"
-
 # Credentials for log sync script
 SWIFT_LOG_SYNC_USERNAME: ''
 SWIFT_LOG_SYNC_PASSWORD: ''
@@ -27,11 +19,23 @@ SWIFT_LOG_SYNC_TENANT_NAME: ''
 SWIFT_LOG_SYNC_AUTH_URL: ''
 SWIFT_LOG_SYNC_REGION_NAME: ''
 
-openstack_requirements_file: "{{ edxapp_code_dir }}/requirements/edx/openstack.txt"
+openstack_role_name: openstack
+
+openstack_dirs:
+  home:
+    path: "{{ COMMON_APP_DIR }}/{{ openstack_role_name }}"
+    owner: "root"
+    group: "root"
+    mode: "0755"
+  logs:
+    path: "{{ COMMON_LOG_DIR }}/{{ openstack_role_name }}"
+    owner: "syslog"
+    group: "syslog"
+    mode: "0650"
 
-openstack_log_sync_script: "{{ vhost_dirs.home.path }}/send-logs-to-swift"
-openstack_log_sync_script_environment: "{{ vhost_dirs.home.path }}/log-sync-env.sh"
-openstack_swift_logfile: "{{ vhost_dirs.logs.path }}/log-sync.log"
+openstack_log_sync_script: "{{ openstack_dirs.home.path }}/send-logs-to-swift"
+openstack_log_sync_script_environment: "{{ openstack_dirs.home.path }}/log-sync-env.sh"
+openstack_swift_logfile: "{{ openstack_dirs.logs.path }}/log-sync.log"
 
 openstack_debian_pkgs:
   - python-setuptools

playbooks/roles/openstack/meta/main.yml

@@ -11,5 +11,4 @@
 # Role includes for role openstack
 #
 dependencies:
-  - role: vhost
-    VHOST_NAME: "{{ vhost_name }}"
+  - vhost

playbooks/roles/openstack/tasks/main.yml

@@ -25,6 +25,15 @@
     extra_args: "-i {{ COMMON_PYPI_MIRROR_URL }}"
   with_items: "{{ openstack_pip_pkgs }}"
 
+- name: Create required directories
+  file:
+    path: "{{ item.value.path }}"
+    state: directory
+    owner: "{{ item.value.owner }}"
+    group: "{{ item.value.group }}"
+    mode: "{{ item.value.mode }}"
+  with_dict: "{{ openstack_dirs }}"
+
 - name: Create log sync script
   template:
     src: send-logs-to-swift.j2
@@ -32,7 +41,7 @@
     mode: 0755
     owner: root
     group: root
-  when: COMMON_OBJECT_STORE_LOG_SYNC
+  when: COMMON_SWIFT_LOG_SYNC
 
 - name: Upload openstack credentials for log script
   template:
@@ -41,26 +50,11 @@
     mode: 0600
     owner: root
     group: root
-  when: COMMON_OBJECT_STORE_LOG_SYNC
+  when: COMMON_SWIFT_LOG_SYNC
 
 - name: Create symlink for log sync script
   file:
     state: link
     src: "{{ openstack_log_sync_script }}"
-    dest: "{{ COMMON_OBJECT_STORE_LOG_SYNC_SCRIPT }}"
-  when: COMMON_OBJECT_STORE_LOG_SYNC
-
-# Install openstack python requirements into {{ edxapp_venv_dir }}
-- name : Install python requirements
-  # Need to use command rather than pip so that we can maintain the context of our current working directory;
-  # some requirements are pathed relative to the edx-platform repo.
-  # Using the pip from inside the virtual environment implicitly installs everything into that virtual environment.
-  command: "{{ edxapp_venv_dir }}/bin/pip install {{ COMMON_PIP_VERBOSITY }} -i {{ COMMON_PYPI_MIRROR_URL }} --exists-action w -r {{ openstack_requirements_file }}"
-  args:
-    chdir: "{{ edxapp_code_dir }}"
-  sudo_user: "{{ edxapp_user }}"
-  environment: "{{ edxapp_environment }}"
-  when: edxapp_code_dir is defined
-  tags:
-    - install
-    - install:app-requirements
+    dest: "{{ COMMON_SWIFT_LOG_SYNC_SCRIPT }}"
+  when: COMMON_SWIFT_LOG_SYNC

playbooks/roles/vhost/defaults/main.yml

----
-#
-# edX Configuration
-#
-# github:     https://github.com/edx/configuration
-# wiki:       https://openedx.atlassian.net/wiki/display/OpenOPS
-# code style: https://openedx.atlassian.net/wiki/display/OpenOPS/Ansible+Code+Conventions
-# license:    https://github.com/edx/configuration/blob/master/LICENSE.TXT
-#
-##
-# Defaults for role vhost
-# 
-
-# Specify a name for vhost deployments, e.g. aws or openstack. Service files
-# specific to the vhost will be namespaced in directories with this name.
-VHOST_NAME: 'vhost'
-
-vhost_dirs:
-  home:
-    path: "{{ COMMON_APP_DIR }}/{{ VHOST_NAME }}"
-    owner: "root"
-    group: "root"
-    mode: "0755"
-  logs:
-    path: "{{ COMMON_LOG_DIR }}/{{ VHOST_NAME }}"
-    owner: "syslog"
-    group: "syslog"
-    mode: "0650"
-  data:
-    path: "{{ COMMON_DATA_DIR }}/{{ VHOST_NAME }}"
-    owner: "root"
-    group: "root"
-    mode: "0700"

playbooks/roles/vhost/tasks/main.yml

@@ -22,15 +22,6 @@
 # - common
 #
 
-- name: Create all service directories
-  file:
-    path: "{{ item.value.path }}"
-    state: directory
-    owner: "{{ item.value.owner }}"
-    group: "{{ item.value.group }}"
-    mode: "{{ item.value.mode }}"
-  with_dict: "{{ vhost_dirs }}"
-
 - name: Force logrotate on supervisor stop
   template:
     src: etc/init/sync-on-stop.conf.j2
@@ -38,7 +29,7 @@
     owner: root
     group: root
     mode: 0644
-  when: COMMON_OBJECT_STORE_LOG_SYNC
+  when: COMMON_S3_LOG_SYNC or COMMON_SWIFT_LOG_SYNC
 
 - name: Update /etc/dhcp/dhclient.conf
   template: