Merge pull request #1096 from edx/e0d/xqwatcher

changes to add xqwatcher asg

cloudformation_templates/edx-reference-architecture.json

@@ -254,6 +254,40 @@
       ],
       "ConstraintDescription":"must be a valid EC2 instance type."
     },
+    "XQWatcherInstanceType":{
+      "Description":"Xserver server EC2 instance type",
+      "Type":"String",
+      "Default":"m1.small",
+      "AllowedValues":[
+        "t1.micro",
+        "m1.small",
+        "m1.medium",
+        "m1.large",
+        "m1.xlarge",
+        "m2.xlarge",
+        "m2.2xlarge",
+        "m2.4xlarge",
+        "cr1.8xlarge",
+        "cc2.8xlarge",
+        "c1.medium",
+        "c1.xlarge",
+        "m3.medium",
+        "m3.large",
+        "m3.xlarge",
+        "m3.2xlarge",
+        "c3.large",
+        "c3.xlarge",
+        "c3.2xlarge",
+        "c3.4xlarge",
+        "c3.8xlarge",
+        "r3.large",
+        "r3.xlarge",
+        "r3.2xlarge",
+        "r3.4xlarge",
+        "r3.8xlarge"
+      ],
+      "ConstraintDescription":"must be a valid EC2 instance type."
+    },
     "SSHLocation":{
       "Description":"The IP address range that can be used to SSH to the EC2 instances",
       "Type":"String",
@@ -371,6 +405,11 @@
       "Type":"Number",
       "Default":"2"
     },
+    "XQWatcherDesiredCapacity":{
+      "Description":"The Auto-scaling group desired capacity for the xqueue watcher hosts",
+      "Type":"Number",
+      "Default":"2"
+    },
     "CommonClusterDesiredCapacity":{
       "Description":"The Auto-scaling group desired capacity for the CommonCluster hosts",
       "Type":"Number",
@@ -587,9 +626,11 @@
       "Edxapp02":        { "CIDR":".11.0/24" },
       "XServerJail01":   { "CIDR":".20.0/24" },
       "XServerJail02":   { "CIDR":".21.0/24" },
-      "CommonCluster01": { "CIDR":".46.0/24"},
-      "CommonCluster02": { "CIDR":".47.0/24"},
-      "CommonCluster03": { "CIDR":".48.0/24"},
+      "XQWatcherJail01": { "CIDR":".30.0/24" },
+      "XQWatcherJail02": { "CIDR":".31.0/24" },
+      "CommonCluster01": { "CIDR":".46.0/24" },
+      "CommonCluster02": { "CIDR":".47.0/24" },
+      "CommonCluster03": { "CIDR":".48.0/24" },
       "Data01":          { "CIDR":".50.0/24" },
       "Data02":          { "CIDR":".51.0/24" },
       "Cache01":         { "CIDR":".60.0/24" },
@@ -601,8 +642,8 @@
       "Mongo01":         { "CIDR":".90.0/24" },
       "Mongo02":         { "CIDR":".91.0/24" },
       "Mongo03":         { "CIDR":".92.0/24" },
-      "Notifier01":      { "CIDR":".100.0/24" },
-      "Admin":           { "CIDR":".200.0/24" }
+      "Notifier01":      { "CIDR":".100.0/24"},
+      "Admin":           { "CIDR":".200.0/24"}
     },
     "MapRegionsToAvailZones":{
       "us-east-1":       { "AZone2":"us-east-1d",      "AZone0":"us-east-1b",      "AZone1":"us-east-1c" },
@@ -1076,6 +1117,102 @@
         ]
       }
     },
+    "XQWatcherSubnet01":{
+      "Type":"AWS::EC2::Subnet",
+      "Properties":{
+        "VpcId":{
+          "Ref":"EdxVPC"
+        },
+        "CidrBlock":{
+          "Fn::Join": ["", [
+            "10.", { "Ref": "ClassB"},
+            {"Fn::FindInMap":[
+              "SubnetConfig",
+              "XQWatcherJail01",
+              "CIDR"
+            ]}
+          ]]
+        },
+        "AvailabilityZone":{
+          "Fn::FindInMap":[
+            "MapRegionsToAvailZones",
+            { "Ref":"AWS::Region" },
+            "AZone0"
+          ]
+        },
+        "Tags":[
+          {
+            "Key":"play",
+            "Value":"xqwatcher"
+          },
+          {
+            "Key":"Network",
+            "Value":"Private"
+          },
+          {
+            "Key" : "immutable_metadata",
+            "Value":{"Fn::Join":["",
+         ["{'purpose':'",
+          {"Ref":"EnvironmentTag"},
+          "-",
+          {"Ref":"DeploymentTag"},
+          "-",
+          "internal-xqwatcher','target':'ec2'}"
+         ]
+        ]
+        }
+          }
+        ]
+      }
+    },
+    "XQWatcherSubnet02":{
+      "Type":"AWS::EC2::Subnet",
+      "Properties":{
+        "VpcId":{
+          "Ref":"EdxVPC"
+        },
+        "CidrBlock":{
+          "Fn::Join": ["", [
+            "10.", { "Ref": "ClassB"},
+            {"Fn::FindInMap":[
+              "SubnetConfig",
+              "XQWatcherJail02",
+              "CIDR"
+            ]}
+          ]]
+        },
+        "AvailabilityZone":{
+          "Fn::FindInMap":[
+            "MapRegionsToAvailZones",
+            { "Ref":"AWS::Region" },
+            "AZone1"
+          ]
+        },
+        "Tags":[
+          {
+            "Key":"play",
+            "Value":"xqwatcher"
+          },
+          {
+            "Key":"Network",
+            "Value":"Private"
+          },
+          {
+            "Key" : "immutable_metadata",
+            "Value":{"Fn::Join":["",
+         ["{'purpose':'",
+          {"Ref":"EnvironmentTag"},
+          "-",
+          {"Ref":"DeploymentTag"},
+          "-",
+          "internal-xqwatcher','target':'ec2'}"
+         ]
+        ]
+        }
+          }
+        ]
+      }
+    },
     "Data01":{
       "Type":"AWS::EC2::Subnet",
       "Properties":{
@@ -1999,6 +2136,28 @@
         }
       }
     },
+    "PrivateSubnetRouteTableAssociationXQWatcher01":{
+      "Type":"AWS::EC2::SubnetRouteTableAssociation",
+      "Properties":{
+        "SubnetId":{
+          "Ref":"XQWatcherSubnet01"
+        },
+        "RouteTableId":{
+          "Ref":"PrivateRouteTable"
+        }
+      }
+    },
+    "PrivateSubnetRouteTableAssociationXQWatcher02":{
+      "Type":"AWS::EC2::SubnetRouteTableAssociation",
+      "Properties":{
+        "SubnetId":{
+          "Ref":"XQWatcherSubnet02"
+        },
+        "RouteTableId":{
+          "Ref":"PrivateRouteTable"
+        }
+      }
+    },
     "PrivateSubnetRouteTableAssociationData01":{
       "Type":"AWS::EC2::SubnetRouteTableAssociation",
       "Properties":{
@@ -2273,6 +2432,28 @@
         }
       }
     },
+    "PrivateSubnetNetworkAclAssociationXQWatcher01":{
+      "Type":"AWS::EC2::SubnetNetworkAclAssociation",
+      "Properties":{
+        "SubnetId":{
+          "Ref":"XQWatcherSubnet01"
+        },
+        "NetworkAclId":{
+          "Ref":"PrivateNetworkAcl"
+        }
+      }
+    },
+    "PrivateSubnetNetworkAclAssociationXQWatcher02":{
+      "Type":"AWS::EC2::SubnetNetworkAclAssociation",
+      "Properties":{
+        "SubnetId":{
+          "Ref":"XQWatcherSubnet02"
+        },
+        "NetworkAclId":{
+          "Ref":"PrivateNetworkAcl"
+        }
+      }
+    },
     "PrivateSubnetNetworkAclAssociationData01":{
       "Type":"AWS::EC2::SubnetNetworkAclAssociation",
       "Properties":{
@@ -2961,6 +3142,44 @@
             } ]
         }
     },
+    "XQWatcherRole": {
+        "Type": "AWS::IAM::Role",
+        "Properties": {
+            "AssumeRolePolicyDocument": {
+                "Statement": [ {
+                    "Effect": "Allow",
+                    "Principal": {
+                        "Service": [ "ec2.amazonaws.com" ]
+                    },
+                    "Action": [ "sts:AssumeRole" ]
+                } ]
+            },
+            "Path": "/",
+            "Policies": [ {
+              "PolicyName": "XQWatcherBasePolicy",
+              "PolicyDocument": {
+                "Statement":[
+                  {
+                    "Effect":"Allow",
+                    "Action":[
+                      "ec2:DescribeTags"
+                    ],
+                    "Resource":"*"
+                  }
+                ]
+              }
+            } ]
+        }
+    },
+    "XQWatcherInstanceProfile": {
+        "Type": "AWS::IAM::InstanceProfile",
+        "Properties": {
+            "Path": "/",
+            "Roles": [ {
+                "Ref": "XQWatcherRole"
+            } ]
+        }
+    },
     "ForumRole": {
         "Type": "AWS::IAM::Role",
         "Properties": {
@@ -4360,6 +4579,229 @@
         ]
       }
     },
+    "XQWatcherServer":{
+      "Type":"AWS::AutoScaling::LaunchConfiguration",
+      "Properties":{
+        "IamInstanceProfile":{ "Ref":"XQWatcherInstanceProfile" },
+        "SecurityGroups":[
+          {
+            "Ref":"XQWatcherServerSecurityGroup"
+          }
+        ],
+        "ImageId":{
+          "Fn::FindInMap":[
+            "AWSRegionArch2AMI",
+            {
+              "Ref":"AWS::Region"
+            },
+            {
+              "Fn::FindInMap":[
+                "AWSInstanceType2Arch",
+                {
+                  "Ref":"XQWatcherInstanceType"
+                },
+                "Arch"
+              ]
+            }
+          ]
+        },
+        "KeyName":{
+          "Ref":"KeyName"
+        },
+        "InstanceType":{
+          "Ref":"XQWatcherInstanceType"
+        },
+        "BlockDeviceMappings":[
+          {
+            "DeviceName":"/dev/sda1",
+            "Ebs":{
+              "VolumeSize":"100"
+            }
+          }
+        ]
+      }
+    },
+
+
+    "XQWatcherServerAsGroup":{
+      "Type":"AWS::AutoScaling::AutoScalingGroup",
+      "Properties":{
+        "AvailabilityZones":[
+          {
+            "Fn::GetAtt":[
+              "XQWatcherSubnet01",
+              "AvailabilityZone"
+            ]
+          },
+          {
+            "Fn::GetAtt":[
+              "XQWatcherSubnet02",
+              "AvailabilityZone"
+            ]
+          }
+        ],
+        "VPCZoneIdentifier":[
+          {
+            "Ref":"XQWatcherSubnet01"
+          },
+          {
+            "Ref":"XQWatcherSubnet02"
+          }
+        ],
+        "Tags":[
+          {
+            "Key":"Name",
+            "Value": {"Fn::Join": ["-",[{"Ref": "EnvironmentTag"},{"Ref": "DeploymentTag"},"xqwatcher"]]},
+            "PropagateAtLaunch":true
+          },
+          {
+            "Key":"play",
+            "Value":"xqwatcher",
+            "PropagateAtLaunch":true
+          },
+          {
+            "Key":"services",
+            "Value":"xqwatcher",
+            "PropagateAtLaunch":true
+          },
+          {
+            "Key":"environment",
+            "Value":{
+              "Ref":"EnvironmentTag"
+            },
+            "PropagateAtLaunch":true
+          },
+          {
+            "Key":"deployment",
+            "Value":{
+              "Ref":"DeploymentTag"
+            },
+            "PropagateAtLaunch":true
+          }
+        ],
+        "LaunchConfigurationName":{
+          "Ref":"XQWatcherServer"
+        },
+        "MinSize":{
+          "Ref":"XQWatcherDesiredCapacity"
+        },
+        "MaxSize":{
+          "Ref":"XQWatcherDesiredCapacity"
+        },
+        "DesiredCapacity":{
+          "Ref":"XQWatcherDesiredCapacity"
+        }
+      }
+    },
+    "XQWatcherServerScaleUpPolicy":{
+      "Type":"AWS::AutoScaling::ScalingPolicy",
+      "Properties":{
+        "AdjustmentType":"ChangeInCapacity",
+        "AutoScalingGroupName":{
+          "Ref":"XQWatcherServerAsGroup"
+        },
+        "Cooldown":"60",
+        "ScalingAdjustment":"1"
+      }
+    },
+    "XQWatcherServerScaleDownPolicy":{
+      "Type":"AWS::AutoScaling::ScalingPolicy",
+      "Properties":{
+        "AdjustmentType":"ChangeInCapacity",
+        "AutoScalingGroupName":{
+          "Ref":"XQWatcherServerAsGroup"
+        },
+        "Cooldown":"60",
+        "ScalingAdjustment":"-1"
+      }
+    },
+    "XQWatcherCPUAlarmHigh":{
+      "Type":"AWS::CloudWatch::Alarm",
+      "Properties":{
+        "AlarmDescription":"Scale-up if CPU > 90% for 10 minutes",
+        "MetricName":"CPUUtilization",
+        "Namespace":"AWS/EC2",
+        "Statistic":"Average",
+        "Period":"300",
+        "EvaluationPeriods":"2",
+        "Threshold":"90",
+        "AlarmActions":[
+          {
+            "Ref":"XQWatcherServerScaleUpPolicy"
+          }
+        ],
+        "Dimensions":[
+          {
+            "Name":"AutoScalingGroupName",
+            "Value":{
+              "Ref":"XQWatcherServerAsGroup"
+            }
+          }
+        ],
+        "ComparisonOperator":"GreaterThanThreshold"
+      }
+    },
+    "XQWatcherCPUAlarmLow":{
+      "Type":"AWS::CloudWatch::Alarm",
+      "Properties":{
+        "AlarmDescription":"Scale-down if CPU < 70% for 10 minutes",
+        "MetricName":"CPUUtilization",
+        "Namespace":"AWS/EC2",
+        "Statistic":"Average",
+        "Period":"300",
+        "EvaluationPeriods":"2",
+        "Threshold":"70",
+        "AlarmActions":[
+          {
+            "Ref":"XQWatcherServerScaleDownPolicy"
+          }
+        ],
+        "Dimensions":[
+          {
+            "Name":"AutoScalingGroupName",
+            "Value":{
+              "Ref":"XQWatcherServerAsGroup"
+            }
+          }
+        ],
+        "ComparisonOperator":"LessThanThreshold"
+      }
+    },
+    "XQWatcherServerSecurityGroup":{
+      "Type":"AWS::EC2::SecurityGroup",
+      "Properties":{
+        "GroupDescription":"Open up SSH access.",
+        "VpcId":{
+          "Ref":"EdxVPC"
+        },
+        "SecurityGroupIngress":[
+          {
+            "IpProtocol":"tcp",
+            "FromPort":"22",
+            "ToPort":"22",
+            "CidrIp": { "Fn::Join": ["", ["10.", { "Ref": "ClassB" }, { "Fn::FindInMap": [ "SubnetConfig", "VPC", "CIDR"]}]]}
+          }
+        ],
+        "Tags":[
+          {
+            "Key":"play",
+            "Value":"xqwatcher"
+          },
+          {
+            "Key":"environment",
+            "Value":{
+              "Ref":"EnvironmentTag"
+            }
+          },
+          {
+            "Key":"deployment",
+            "Value":{
+              "Ref":"DeploymentTag"
+            }
+          }
+        ]
+      }
+    },
     "EdxDataSecurityGroup":{
       "Type":"AWS::EC2::SecurityGroup",
       "Properties":{