Make S3 and SWIFT log syncing independent.

While COMMON_OBJECT_STORE_LOG_SYNC and related variables are still
supported, this patch offers a finer grained control over tracking log
syncing.

Sometimes you want to host edX on openstack (and therefore need to run
the openstack role), but you still want to sync tracking logs to S3, so
it does not make sense to tie the choice of log sync store to the
presence of either aws or openstack role in the playbook.

playbooks/edx-east/retire_host.yml

@@ -40,7 +40,7 @@
         - "edx-services"
     # This catches the case where tracking.log is 0b
     - name: Sync again
-      command: /edx/bin/send-logs-to-object-store -d "{{ COMMON_LOG_DIR }}/tracking/" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}/logs/tracking"
+      command: /edx/bin/send-logs-to-s3 -d "{{ COMMON_LOG_DIR }}/tracking/" -b "{{ COMMON_S3_LOG_SYNC_BUCKET }}/logs/tracking"
     
 - name: Run minos verification
   hosts: "{{TARGET}}"

playbooks/roles/aws/tasks/main.yml

@@ -83,14 +83,14 @@
     mode: 0755
     owner: root
     group: root
-  when: COMMON_OBJECT_STORE_LOG_SYNC
+  when: COMMON_S3_LOG_SYNC
 
 - name: Create symlink for s3 log sync script
   file:
     state: link
     src: "{{ aws_s3_sync_script }}"
-    dest: "{{ COMMON_OBJECT_STORE_LOG_SYNC_SCRIPT }}"
-  when: COMMON_OBJECT_STORE_LOG_SYNC
+    dest: "{{ COMMON_S3_LOG_SYNC_SCRIPT }}"
+  when: COMMON_S3_LOG_SYNC
 
  # update the ssh motd on Ubuntu
  # Remove some of the default motd display on ubuntu

playbooks/roles/common/templates/etc/logrotate.d/hourly/edx_logrotate_tracking_log.j2

@@ -14,8 +14,11 @@
     /usr/bin/killall -HUP rsyslogd
   endscript
   lastaction
-    {% if COMMON_OBJECT_STORE_LOG_SYNC -%}
-    {{ COMMON_OBJECT_STORE_LOG_SYNC_SCRIPT }} -d "{{ COMMON_LOG_DIR }}/tracking" -b "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}" -p "{{ COMMON_OBJECT_STORE_LOG_SYNC_PREFIX }}"
+    {% if COMMON_S3_LOG_SYNC -%}
+    {{ COMMON_S3_LOG_SYNC_SCRIPT }} -d "{{ COMMON_LOG_DIR }}/tracking" -b "{{ COMMON_S3_LOG_SYNC_BUCKET }}" -p "{{ COMMON_S3_LOG_SYNC_PREFIX }}"
+    {% endif -%}
+    {% if COMMON_SWIFT_LOG_SYNC -%}
+    {{ COMMON_SWIFT_LOG_SYNC_SCRIPT }} -d "{{ COMMON_LOG_DIR }}/tracking" -b "{{ COMMON_SWIFT_LOG_SYNC_BUCKET }}" -p "{{ COMMON_SWIFT_LOG_SYNC_PREFIX }}"
     {% endif -%}
   endscript
 }

playbooks/roles/common_vars/defaults/main.yml

@@ -25,7 +25,16 @@ COMMON_ENABLE_OPENSTACK_INTEGRATION: False
 COMMON_OBJECT_STORE_LOG_SYNC: False
 COMMON_OBJECT_STORE_LOG_SYNC_BUCKET: "edx-{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}"
 COMMON_OBJECT_STORE_LOG_SYNC_PREFIX: "logs/tracking/"
-COMMON_OBJECT_STORE_LOG_SYNC_SCRIPT: "{{ COMMON_BIN_DIR }}/send-logs-to-object-store"
+
+COMMON_S3_LOG_SYNC: "{{ COMMON_OBJECT_STORE_LOG_SYNC and COMMON_ENABLE_AWS_INTEGRATION }}"
+COMMON_S3_LOG_SYNC_BUCKET: "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}"
+COMMON_S3_LOG_SYNC_PREFIX: "{{ COMMON_OBJECT_STORE_LOG_SYNC_PREFIX }}"
+COMMON_S3_LOG_SYNC_SCRIPT: "{{ COMMON_BIN_DIR }}/send-logs-to-s3"
+
+COMMON_SWIFT_LOG_SYNC: "{{ COMMON_ENABLE_OPENSTACK_INTEGRATION and COMMON_OBJECT_STORE_LOG_SYNC }}"
+COMMON_SWIFT_LOG_SYNC_BUCKET: "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}"
+COMMON_SWIFT_LOG_SYNC_PREFIX: "{{ COMMON_OBJECT_STORE_LOG_SYNC_PREFIX }}"
+COMMON_SWIFT_LOG_SYNC_SCRIPT: "{{ COMMON_BIN_DIR }}/send-logs-to-swift"
 
 COMMON_BASE_DIR: /edx
 COMMON_DATA_DIR: "{{ COMMON_BASE_DIR}}/var"

playbooks/roles/minos/defaults/main.yml

@@ -15,7 +15,7 @@ MINOS_GIT_IDENTITY: !!null
 MINOS_SERVICE_CONFIG:
   aws_profile: !!null
   aws_region: "{{ MINOS_AWS_REGION }}"
-  s3_bucket: "{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}"
+  s3_bucket: "{{ COMMON_S3_LOG_SYNC_BUCKET }}"
   bucket_path: lifecycle/minos
   voter_conf_d: "{{ minos_voter_cfg }}"
 

playbooks/roles/minos/templates/edx/etc/minos/conf.d/TrackingLogVoter.yml.j2

@@ -2,5 +2,5 @@ TrackingLogVoter:
   config:
     aws_profile: !!null
     local_directory: '{{ COMMON_LOG_DIR }}/tracking'
-    s3_bucket: '{{ COMMON_OBJECT_STORE_LOG_SYNC_BUCKET }}'
+    s3_bucket: '{{ COMMON_S3_LOG_SYNC_BUCKET }}'
     bucket_path_prefix: 'logs/tracking'

playbooks/roles/openstack/tasks/main.yml

@@ -41,7 +41,7 @@
     mode: 0755
     owner: root
     group: root
-  when: COMMON_OBJECT_STORE_LOG_SYNC
+  when: COMMON_SWIFT_LOG_SYNC
 
 - name: Upload openstack credentials for log script
   template:
@@ -50,14 +50,14 @@
     mode: 0600
     owner: root
     group: root
-  when: COMMON_OBJECT_STORE_LOG_SYNC
+  when: COMMON_SWIFT_LOG_SYNC
 
 - name: Create symlink for log sync script
   file:
     state: link
     src: "{{ openstack_log_sync_script }}"
-    dest: "{{ COMMON_OBJECT_STORE_LOG_SYNC_SCRIPT }}"
-  when: COMMON_OBJECT_STORE_LOG_SYNC
+    dest: "{{ COMMON_SWIFT_LOG_SYNC_SCRIPT }}"
+  when: COMMON_SWIFT_LOG_SYNC
 
 # Install openstack python requirements into {{ edxapp_venv_dir }}
 - name : Install python requirements

playbooks/roles/vhost/tasks/main.yml

@@ -29,7 +29,7 @@
     owner: root
     group: root
     mode: 0644
-  when: COMMON_OBJECT_STORE_LOG_SYNC
+  when: COMMON_S3_LOG_SYNC or COMMON_SWIFT_LOG_SYNC
 
 - name: Update /etc/dhcp/dhclient.conf
   template: