Merge pull request #1110 from edx/jarv/connect-to-prod-clone

Jarv/connect to prod clone

playbooks/edx-east/connect_sandbox.yml

+- name: connect a sandbox to production data
+  hosts: all
+  gather_facts: False
+  sudo: True
+  tasks:
+
+    - name: Switch the mongo db to use ephemeral
+      file: >
+        name=/mnt/mongodb
+        state=directory
+        owner=mongodb
+        group=mongodb
+      tags: update_mongo_data
+
+    - name: update the mongo config to use the new mongo dir
+      shell: >
+        sed -i 's#^dbpath=.*#dbpath=/mnt/mongodb#' /etc/mongodb.conf
+      tags: update_mongo_data
+
+    - name: restart mongodb
+      service: >
+        name=mongodb
+        state=restarted
+      tags: update_mongo_data
+
+    - name: grab the most recent backup from s3 for forums
+      shell : >
+        /edx/bin/s3cmd ls s3://edx-mongohq/mongohq_backups/ | grep comment | sort | tail -1  | awk '{ print $4 }'
+      register: s3cmd_out_forum
+      tags: update_mongo_data
+
+    - name: grab the most recent backup from s3 for forums
+      shell : >
+        /edx/bin/s3cmd get {{ s3cmd_out_forum.stdout }} --skip-existing
+        chdir=/mnt
+      tags: update_mongo_data
+
+    - name: untar the s3 backup
+      shell: >
+        tar zxf {{ s3cmd_out_forum.stdout|basename }}
+        chdir=/mnt
+      tags: update_mongo_data
+
+
+    - name: grab the most recent backup from s3 for prod-edx
+      shell : >
+        /edx/bin/s3cmd ls s3://edx-mongohq/mongohq_backups/ | grep prod-edx | sort | tail -1  | awk '{ print $4 }'
+      register: s3cmd_out_modulestore
+      tags: update_mongo_data
+
+    - name: grab the most recent backup from s3 for prod-edx
+      shell : >
+        /edx/bin/s3cmd get {{ s3cmd_out_modulestore.stdout }} --skip-existing
+        chdir=/mnt
+      tags: update_mongo_data
+
+    - name: untar the s3 backup
+      shell: >
+        tar zxf {{ s3cmd_out_modulestore.stdout|basename }}
+        chdir=/mnt
+      tags: update_mongo_data
+
+    - name: Restore the mongo data for the forums
+      shell: >
+        mongorestore --drop -d cs_comments_service  /mnt/comments-prod
+      tags: update_mongo_data
+
+    - name: Restore the mongo data for the modulestore
+      shell: >
+        mongorestore --drop -d edxapp /mnt/prod-edx
+      tags: update_mongo_data
+
+    # recreate users after the restore
+    - name: create a mongodb users
+      mongodb_user: >
+        database={{ item.database }}
+        name={{ item.user }}
+        password={{ item.password }}
+        state=present
+      with_items:
+        - user: cs_comments_service
+          password: password
+          database: cs_comments_service
+        - user: exdapp
+          password: password
+          database: edxapp
+
+      # WARNING - calling lineinfile on a symlink
+      # will convert the symlink to a file!
+      # don't use /edx/etc/server-vars.yml here
+      #
+      # What we are doing here is updating the sandbox
+      # server-vars config file so that when update
+      # is called it will use the new MYSQL connection
+      # info.
+    - name: Update RDS to point to the sandbox clone
+      lineinfile: >
+        dest=/edx/app/edx_ansible/server-vars.yml
+        line="{{ item }}"
+      with_items:
+        - "EDXAPP_MYSQL_HOST: {{ EDXAPP_MYSQL_HOST }}"
+        - "EDXAPP_MYSQL_DB_NAME: {{ EDXAPP_MYSQL_DB_NAME }}"
+        - "EDXAPP_MYSQL_USER: {{ EDXAPP_MYSQL_USER }}"
+        - "EDXAPP_MYSQL_PASSWORD: {{ EDXAPP_MYSQL_PASSWORD }}"
+      tags: update_edxapp_mysql_host
+
+    - name: call update on edx-platform
+      shell: >
+        /edx/bin/update edx-platform master
+      tags: update_edxapp_mysql_host

playbooks/roles/edxapp/tasks/deploy.yml

@@ -359,6 +359,7 @@
 # service variants configured, runs
 # gather_assets and db migrations
 - include: service_variant_config.yml
+  tags: service_variant_config
 
   # call supervisorctl update. this reloads
   # the supervisorctl config and restarts

util/jenkins/ansible-provision.sh

@@ -72,7 +72,7 @@ if [[ -z $ami ]]; then
 fi
 
 if [[ -z $instance_type ]]; then
-  instance_type="m3.medium"
+  instance_type="m1.medium"
 fi
 
 deploy_host="${dns_name}.${dns_zone}"

util/jenkins/ascii-convert.sh

@@ -9,5 +9,3 @@ BUILD_USER_LAST_NAME=$(ascii_convert $BUILD_USER_LAST_NAME)
 BUILD_USER_FIRST_NAME=$(ascii_convert $BUILD_USER_FIRST_NAME)
 BUILD_USER_ID=$(ascii_convert $BUILD_USER_ID)
 BUILD_USER=$(ascii_convert $BUILD_USER)
-
-

util/jenkins/connect-sandbox-to-prod-clone.sh

+#!/usr/bin/env bash
+
+# Ansible provisioning wrapper script that
+# assumes the following parameters set
+# as environment variables
+#
+# - github_username
+# - server_type
+# - instance_type
+# - region
+# - aws_account
+# - keypair
+# - ami
+# - root_ebs_size
+# - security_group
+# - dns_zone
+# - dns_name
+# - environment
+# - name_tag
+
+export PYTHONUNBUFFERED=1
+export BOTO_CONFIG=/var/lib/jenkins/${aws_account}.boto
+
+if [[ -z $WORKSPACE ]]; then
+    dir=$(dirname $0)
+    source "$dir/ascii-convert.sh"
+else
+    source "$WORKSPACE/configuration/util/jenkins/ascii-convert.sh"
+fi
+
+if [[ ! -f $BOTO_CONFIG ]]; then
+  echo "AWS credentials not found for $aws_account"
+  exit 1
+fi
+
+if [[ -z $sandbox_to_update ]]; then
+  sandbox_to_update="${BUILD_USER_ID}.m.sandbox.edx.org"
+fi
+
+cd $WORKSPACE/configuration/playbooks/edx-east
+ansible-playbook connect_sandbox.yml  -i $sandbox_to_update, -e@${WORKSPACE}/configuration-secure/ansible/vars/clone-db.yml -e EDXAPP_MYSQL_HOST=$EDXAPP_MYSQL_HOST --user ubuntu  -v