Merge pull request #642 from edx/feanil/abbey_common

Feanil/abbey common

playbooks/roles/datadog/meta/main.yml

+---
+dependencies:
+  - common

playbooks/roles/splunkforwarder/meta/main.yml

+---
+dependencies:
+  - common

util/vpc-tools/abbey.py

@@ -22,13 +22,14 @@ AMI_TIMEOUT = 600  # time to wait for AMIs to complete
 EC2_RUN_TIMEOUT = 180  # time to wait for ec2 state transition
 EC2_STATUS_TIMEOUT = 300  # time to wait for ec2 system status checks
 NUM_TASKS = 5  # number of tasks for time summary report
+NUM_PLAYBOOKS = 3
 
 
 class MongoConnection:
 
     def __init__(self):
         try:
-            mongo = MongoClient(host=args.mongo_host, port=args.mongo_port)
+            mongo = MongoClient(host=args.mongo_uri)
         except ConnectionFailure:
             print "Unable to connect to the mongo database specified"
             sys.exit(1)
@@ -109,11 +110,11 @@ def parse_args():
                         default=False)
     parser.add_argument('--secure-vars', required=False,
                         metavar="SECURE_VAR_FILE",
-                        help="path to secure-vars, defaults to "
-                        "../../../configuration-secure/ansible/"
-                        "vars/DEPLOYMENT/ENVIRONMENT.yml")
+                        help="path to secure-vars from the root of "
+                        "configuration-secure, defaults to ansible/"
+                        "vars/DEPLOYMENT/ENVIRONMENT-DEPLOYMENT.yml")
     parser.add_argument('--stack-name',
-                        help="defaults to DEPLOYMENT-ENVIRONMENT",
+                        help="defaults to ENVIRONMENT-DEPLOYMENT",
                         metavar="STACK_NAME",
                         required=False)
     parser.add_argument('-p', '--play',
@@ -133,10 +134,10 @@ def parse_args():
                         help="Application for subnet, defaults to admin",
                         default="admin")
     parser.add_argument('--configuration-version', required=False,
-                        help="configuration repo version",
+                        help="configuration repo branch(no hashes)",
                         default="master")
     parser.add_argument('--configuration-secure-version', required=False,
-                        help="configuration-secure repo version",
+                        help="configuration-secure repo branch(no hashes)",
                         default="master")
     parser.add_argument('-j', '--jenkins-build', required=False,
                         help="jenkins build number to update")
@@ -165,15 +166,10 @@ def parse_args():
                         default=5,
                         help="How long to delay message display from sqs "
                              "to ensure ordering")
-    parser.add_argument("--mongo-host", required=False,
+    parser.add_argument("--mongo-uri", required=False,
                         default=None,
-                        help="Mongo host that contains the AMI collection")
-    parser.add_argument("--mongo-pass", required=False,
-                        default=None,
-                        help="Mongo password")
-    parser.add_argument("--mongo-port", required=False,
-                        default=27017,
-                        help="Mongo port")
+                        help="Mongo uri for the host that contains"
+                             "the AMI collection")
     parser.add_argument("--mongo-db", required=False,
                         default="test",
                         help="Mongo database")
@@ -242,8 +238,8 @@ environment="{environment}"
 deployment="{deployment}"
 play="{play}"
 config_secure={config_secure}
-secure_vars_file="$base_dir/configuration-secure\\
-/ansible/vars/$environment/$environment-$deployment.yml"
+secure_vars_file="$base_dir/configuration-secure/{secure_vars}"
+common_vars_file="$base_dir/configuration-secure/ansible/vars/common/common.yml"
 instance_id=\\
 $(curl http://169.254.169.254/latest/meta-data/instance-id 2>/dev/null)
 instance_ip=\\
@@ -314,9 +310,9 @@ sudo pip install -r requirements.txt
 
 cd $playbook_dir
 
-ansible-playbook -vvvv -c local -i "localhost," $play.yml -e@$extra_vars
-ansible-playbook -vvvv -c local -i "localhost," datadog.yml -e@$extra_vars
-ansible-playbook -vvvv -c local -i "localhost," splunkforwarder.yml -e@$extra_vars
+ansible-playbook -vvvv -c local -i "localhost," $play.yml -e@$extra_vars -e@$common_vars_file
+ansible-playbook -vvvv -c local -i "localhost," datadog.yml -e@$extra_vars -e@$common_vars_file
+ansible-playbook -vvvv -c local -i "localhost," splunkforwarder.yml -e@$extra_vars -e@$common_vars_file
 
 rm -rf $base_dir
 
@@ -329,7 +325,8 @@ rm -rf $base_dir
                 config_secure=config_secure,
                 identity_file=identity_file,
                 queue_name=run_id,
-                extra_vars_yml=extra_vars_yml)
+                extra_vars_yml=extra_vars_yml,
+                secure_vars=secure_vars)
 
     ec2_args = {
         'security_group_ids': [security_group_id],
@@ -362,6 +359,7 @@ def poll_sqs_ansible():
     buf = []
     task_report = []  # list of tasks for reporting
     last_task = None
+    completed = 0
     while True:
         messages = []
         while True:
@@ -438,7 +436,14 @@ def poll_sqs_ansible():
                         to_disp['msg']['TS'] / 60,
                         to_disp['msg']['TS'] % 60,
                         to_disp['msg']['PREFIX'])
-                    return (to_disp['msg']['TS'], task_report)
+
+                    # Since 3 ansible plays get run.
+                    # We see the COMPLETE message 3 times
+                    # wait till the last one to end listening
+                    # for new messages.
+                    completed += 1
+                    if completed >= NUM_PLAYBOOKS:
+                        return (to_disp['msg']['TS'], task_report)
 
         if not messages:
             # wait 1 second between sqs polls
@@ -581,9 +586,8 @@ if __name__ == '__main__':
     if args.secure_vars:
         secure_vars = args.secure_vars
     else:
-        secure_vars = "../../../configuration-secure/" \
-                      "ansible/vars/{}/{}.yml".format(
-                      args.deployment, args.environment)
+        secure_vars = "ansible/vars/{}/{}-{}.yml".format(
+                      args.environment, args.environment, args.deployment)
     if args.stack_name:
         stack_name = args.stack_name
     else: