Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
C
configuration
Overview
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
OpenEdx
configuration
Commits
e7f51684
Commit
e7f51684
authored
7 years ago
by
Hannah Chen
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
upgrade minos scripts to boto3
parent
1aa63d30
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
52 additions
and
55 deletions
+52
-55
playbooks/edx-east/lifecycle_inventory.py
+23
-26
util/vpc-tools/asg_lifcycle_watcher.py
+29
-29
No files found.
playbooks/edx-east/lifecycle_inventory.py
View file @
e7f51684
...
...
@@ -29,71 +29,68 @@ group and state.
}
"""
import
argparse
import
boto
import
boto.ec2.autoscale
import
boto3
import
json
from
collections
import
defaultdict
from
os
import
environ
class
LifecycleInventory
():
profile
=
None
def
__init__
(
self
,
profile
):
def
__init__
(
self
,
region
):
parser
=
argparse
.
ArgumentParser
()
self
.
profile
=
profile
self
.
region
=
region
def
get_e_d_from_tags
(
self
,
group
):
environment
=
"default_environment"
deployment
=
"default_deployment"
for
r
in
group
.
tags
:
if
r
.
key
==
"environment"
:
environment
=
r
.
value
elif
r
.
key
==
"deployment"
:
deployment
=
r
.
value
for
r
in
group
[
'Tags'
]
:
if
r
[
'Key'
]
==
"environment"
:
environment
=
r
[
'Value'
]
elif
r
[
'Key'
]
==
"deployment"
:
deployment
=
r
[
'Value'
]
return
environment
,
deployment
def
get_instance_dict
(
self
):
ec2
=
boto
.
ec2
.
connect_to_region
(
region
,
profile_name
=
self
.
profile
)
reservations
=
ec2
.
get_all_instances
()
ec2
=
boto
3
.
client
(
'ec2'
,
region_name
=
self
.
region
)
reservations
=
ec2
.
describe_instances
()[
'Reservations'
]
dict
=
{}
for
instance
in
[
i
for
r
in
reservations
for
i
in
r
.
instances
]:
dict
[
instance
.
id
]
=
instance
for
instance
in
[
i
for
r
in
reservations
for
i
in
r
[
'Instances'
]
]:
dict
[
instance
[
'InstanceId'
]
]
=
instance
return
dict
def
run
(
self
):
asg
=
boto
.
ec2
.
autoscale
.
connect_to_region
(
region
,
profile_name
=
self
.
profile
)
groups
=
asg
.
get_all_groups
()
asg
=
boto3
.
client
(
'autoscaling'
,
region_name
=
self
.
region
)
groups
=
asg
.
describe_auto_scaling_groups
()[
'AutoScalingGroups'
]
instances
=
self
.
get_instance_dict
()
inventory
=
defaultdict
(
list
)
for
group
in
groups
:
for
instance
in
group
.
instances
:
for
instance
in
group
[
'Instances'
]
:
private_ip_address
=
instances
[
instance
.
instance_id
]
.
private_ip_address
private_ip_address
=
instances
[
instance
[
'InstanceId'
]][
'PrivateIpAddress'
]
if
private_ip_address
:
environment
,
deployment
=
self
.
get_e_d_from_tags
(
group
)
inventory
[
environment
+
"_"
+
deployment
+
"_"
+
instance
.
lifecycle_state
.
replace
(
":"
,
"_"
)]
.
append
(
private_ip_address
)
inventory
[
group
.
name
]
.
append
(
private_ip_address
)
inventory
[
group
.
name
+
"_"
+
instance
.
lifecycle_state
.
replace
(
":"
,
"_"
)]
.
append
(
private_ip_address
)
inventory
[
instance
.
lifecycle_state
.
replace
(
":"
,
"_"
)]
.
append
(
private_ip_address
)
inventory
[
environment
+
"_"
+
deployment
+
"_"
+
instance
[
'LifecycleState'
]
.
replace
(
":"
,
"_"
)]
.
append
(
private_ip_address
)
inventory
[
group
[
'AutoScalingGroupName'
]
]
.
append
(
private_ip_address
)
inventory
[
group
[
'AutoScalingGroupName'
]
+
"_"
+
instance
[
'LifecycleState'
]
.
replace
(
":"
,
"_"
)]
.
append
(
private_ip_address
)
inventory
[
instance
[
'LifecycleState'
]
.
replace
(
":"
,
"_"
)]
.
append
(
private_ip_address
)
print
json
.
dumps
(
inventory
,
sort_keys
=
True
,
indent
=
2
)
if
__name__
==
"__main__"
:
parser
=
argparse
.
ArgumentParser
()
parser
.
add_argument
(
'-
p'
,
'--profile'
,
help
=
'The aws profile to use when connecting.
'
)
parser
.
add_argument
(
'-
r'
,
'--region'
,
help
=
'The aws region to use when connecting.'
,
default
=
'us-east-1
'
)
parser
.
add_argument
(
'-l'
,
'--list'
,
help
=
'Ansible passes this, we ignore it.'
,
action
=
'store_true'
,
default
=
True
)
args
=
parser
.
parse_args
()
region
=
environ
.
get
(
'AWS_REGION'
,
'us-east-1'
)
LifecycleInventory
(
args
.
profile
)
.
run
()
LifecycleInventory
(
args
.
region
)
.
run
()
This diff is collapsed.
Click to expand it.
util/vpc-tools/asg_lifcycle_watcher.py
View file @
e7f51684
...
...
@@ -17,12 +17,9 @@ It relies on some component applying the proper tags and performing pre-retireme
"""
import
argparse
import
boto
import
boto.ec2
import
boto.sqs
import
boto3
import
json
import
subprocess
from
boto.sqs.message
import
RawMessage
import
logging
import
os
from
distutils
import
spawn
...
...
@@ -35,40 +32,37 @@ class LifecycleHandler:
INSTANCE_TERMINATION
=
'autoscaling:EC2_INSTANCE_TERMINATING'
TEST_NOTIFICATION
=
'autoscaling:TEST_NOTIFICATION'
NUM_MESSAGES
=
10
WAIT_TIME_SECONDS
=
10
WAIT_TIME_SECONDS
=
1
VISIBILITY_TIMEOUT
=
10
def
__init__
(
self
,
profile
,
queue
,
hook
,
dry_run
,
bin_directory
=
None
):
def
__init__
(
self
,
region
,
queue
,
hook
,
dry_run
,
bin_directory
=
None
):
logging
.
basicConfig
(
level
=
logging
.
INFO
)
self
.
queue
=
queue
self
.
hook
=
hook
self
.
profile
=
profile
self
.
region
=
region
if
bin_directory
:
os
.
environ
[
"PATH"
]
=
bin_directory
+
os
.
pathsep
+
os
.
environ
[
"PATH"
]
self
.
aws_bin
=
spawn
.
find_executable
(
'aws'
)
self
.
python_bin
=
spawn
.
find_executable
(
'python'
)
self
.
region
=
os
.
environ
.
get
(
'AWS_REGION'
,
'us-east-1'
)
self
.
base_cli_command
=
"{python_bin} {aws_bin} "
.
format
(
python_bin
=
self
.
python_bin
,
aws_bin
=
self
.
aws_bin
)
if
self
.
profile
:
self
.
base_cli_command
+=
"--profile {profile} "
.
format
(
profile
=
self
.
profile
)
if
self
.
region
:
self
.
base_cli_command
+=
"--region {region} "
.
format
(
region
=
self
.
region
)
self
.
dry_run
=
dry_run
self
.
ec2_con
=
boto
.
ec2
.
connect_to_region
(
self
.
region
)
self
.
sqs_con
=
boto
.
sqs
.
connect_to_region
(
self
.
region
)
self
.
dry_run
=
args
.
dry_run
self
.
ec2_con
=
boto
3
.
client
(
'ec2'
,
region_name
=
self
.
region
)
self
.
sqs_con
=
boto
3
.
client
(
'sqs'
,
region_name
=
self
.
region
)
def
process_lifecycle_messages
(
self
):
queue
=
self
.
sqs_con
.
get_queue
(
self
.
queue
)
queue_url
=
self
.
sqs_con
.
get_queue_url
(
QueueName
=
self
.
queue
)[
'QueueUrl'
]
queue
=
boto3
.
resource
(
'sqs'
,
region_name
=
self
.
region
)
.
Queue
(
queue_url
)
# Needed to get unencoded message for ease of processing
queue
.
set_message_class
(
RawMessage
)
for
sqs_message
in
queue
.
get_messages
(
LifecycleHandler
.
NUM_MESSAGES
,
wait_time_seconds
=
LifecycleHandler
.
WAIT_TIME_SECONDS
):
body
=
json
.
loads
(
sqs_message
.
get_body_encoded
())
for
sqs_message
in
self
.
sqs_con
.
receive_message
(
QueueUrl
=
queue_url
,
MaxNumberOfMessages
=
LifecycleHandler
.
NUM_MESSAGES
,
VisibilityTimeout
=
LifecycleHandler
.
VISIBILITY_TIMEOUT
,
WaitTimeSeconds
=
LifecycleHandler
.
WAIT_TIME_SECONDS
)
.
get
(
'Messages'
,
[]):
body
=
json
.
loads
(
sqs_message
[
'Body'
])
as_message
=
json
.
loads
(
body
[
'Message'
])
logging
.
info
(
"Proccessing message {message}."
.
format
(
message
=
as_message
))
...
...
@@ -113,7 +107,7 @@ class LifecycleHandler:
def
delete_sqs_message
(
self
,
queue
,
sqs_message
,
as_message
,
dry_run
):
if
not
dry_run
:
logging
.
info
(
"Deleting message with body {message}"
.
format
(
message
=
as_message
))
self
.
sqs_con
.
delete_message
(
queue
,
sqs_message
)
self
.
sqs_con
.
delete_message
(
QueueUrl
=
queue
.
url
,
ReceiptHandle
=
sqs_message
[
'ReceiptHandle'
]
)
else
:
logging
.
info
(
"Would have deleted message with body {message}"
.
format
(
message
=
as_message
))
...
...
@@ -154,10 +148,12 @@ class LifecycleHandler:
"""
Simple boto call to get the instance based on the instance-id
"""
instances
=
self
.
ec2_con
.
get_only_instances
([
instance_id
])
reservations
=
self
.
ec2_con
.
describe_instances
(
InstanceIds
=
[
instance_id
])
.
get
(
'Reservations'
,
[])
instances
=
[]
if
len
(
reservations
)
==
1
:
instances
=
reservations
[
0
]
.
get
(
'Instances'
,
[])
if
len
(
instances
)
==
1
:
return
self
.
ec2_con
.
get_only_instances
([
instance_id
])
[
0
]
return
self
.
ec2_con
.
describe_instances
(
InstanceIds
=
[
instance_id
])[
'Reservations'
][
0
][
'Instances'
]
[
0
]
else
:
return
None
...
...
@@ -167,9 +163,13 @@ class LifecycleHandler:
with the value 'true'
"""
instance
=
self
.
get_ec2_instance_by_id
(
instance_id
)
tags_dict
=
{}
if
instance
:
if
'safe_to_retire'
in
instance
.
tags
and
instance
.
tags
[
'safe_to_retire'
]
.
lower
()
==
'true'
:
tags_dict
=
{}
for
t
in
instance
[
'Tags'
]:
tags_dict
[
t
[
'Key'
]]
=
t
[
'Value'
]
if
'safe_to_retire'
in
tags_dict
and
tags_dict
[
'safe_to_retire'
]
.
lower
()
==
'true'
:
logging
.
info
(
"Instance with id {id} is safe to retire."
.
format
(
id
=
instance_id
))
return
True
else
:
...
...
@@ -184,9 +184,9 @@ class LifecycleHandler:
if
__name__
==
"__main__"
:
parser
=
argparse
.
ArgumentParser
()
parser
.
add_argument
(
'-
p'
,
'--profile
'
,
help
=
'The
boto profile
to use '
'per line.'
,
default
=
None
)
parser
.
add_argument
(
'-
r'
,
'--region
'
,
help
=
'The
aws region
to use '
'per line.'
,
default
=
'us-east-1'
)
parser
.
add_argument
(
'-b'
,
'--bin-directory'
,
required
=
False
,
default
=
None
,
help
=
'The bin directory of the virtual env '
'from which to run the AWS cli (optional)'
)
...
...
@@ -201,5 +201,5 @@ if __name__=="__main__":
parser
.
set_defaults
(
dry_run
=
False
)
args
=
parser
.
parse_args
()
lh
=
LifecycleHandler
(
args
.
profile
,
args
.
queue
,
args
.
hook
,
args
.
dry_run
,
args
.
bin_directory
)
lh
=
LifecycleHandler
(
args
.
region
,
args
.
queue
,
args
.
hook
,
args
.
dry_run
,
args
.
bin_directory
)
lh
.
process_lifecycle_messages
()
This diff is collapsed.
Click to expand it.
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment