PoS_EVM
/
woop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

added request_spots

Browse Source
pull/27/head
Richard Liu 6 years ago
parent 01fa20ecbc
commit 9997b6c0ff
1 changed files with 160 additions and 87 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 103
      aws-experiment-launch/create_and_deploy.py

103
aws-experiment-launch/create_and_deploy.py
Unescape View File

@ -4,6 +4,7 @@ import sys
import json
import time
import datetime
import base64
REGION_NAME = 'region_name'
REGION_KEY = 'region_key'
@ -11,12 +12,17 @@ REGION_SECURITY_GROUP = 'region_security_group'
REGION_HUMAN_NAME = 'region_human_name'
INSTANCE_TYPE = 't2.micro'
REGION_AMI = 'region_ami'
USER_DATA = 'user-data.sh'
# USER_DATA = 'user-data.sh'
# UserData must be base64 encoded.
with open("user-data.sh", "rb") as userdata_file:
USER_DATA = base64.b64encode(userdata_file.read())
IAM_INSTANCE_PROFILE = 'BenchMarkCodeDeployInstanceProfile'
REPO = "simple-rules/harmony-benchmark"
APPLICATION_NAME = 'benchmark-experiments'
time_stamp = time.time()
CURRENT_SESSION = datetime.datetime.fromtimestamp(time_stamp).strftime('%H-%M-%S-%Y-%m-%d')
CURRENT_SESSION = datetime.datetime.fromtimestamp(
time_stamp).strftime('%H-%M-%S-%Y-%m-%d')
PLACEMENT_GROUP = "PLACEMENT-" + CURRENT_SESSION
NODE_VALUE = "NODE-" + CURRENT_SESSION
@ -32,6 +38,8 @@ Build (argparse,functions) support for
3. run create instance followed by codedeploy
"""
def get_instance_ids(response):
instance_ids = []
for reservation in response["Reservations"]:
@ -39,6 +47,7 @@ def get_instance_ids(response):
instance_ids.append(instance["InstanceId"])
return instance_ids
def run_one_region_instances(config, region_number, number_of_instances):
#todo: explore the use ec2 resource and not client. e.g. create_instances -- Might make for better code.
"""
@ -47,10 +56,24 @@ def run_one_region_instances(config,region_number,number_of_instances):
region_name = config[region_number][REGION_NAME]
session = boto3.Session(region_name=region_name)
ec2_client = session.client('ec2')
response,placement = create_instances(config,ec2_client,region_number,int(number_of_instances))
# response, placement = create_instances(
# config, ec2_client, region_number, int(number_of_instances))
response, placement = request_spots(
config, ec2_client, region_number, int(number_of_instances))
print(placement)
return session, placement
def run_one_region_spots(config, region_number, number_of_instances):
region_name = config[region_number][REGION_NAME]
session = boto3.Session(region_name=region_name)
ec2_client = session.client('ec2')
response, placement = create_instances(
config, ec2_client, region_number, int(number_of_instances))
print(placement)
return session, placement
def run_one_region_codedeploy(region_number, placement_group, commitId):
#todo: explore the use ec2 resource and not client. e.g. create_instances -- Might make for better code.
"""
@ -96,10 +119,13 @@ def run_one_region_codedeploy(region_number,placement_group,commitId):
deployment_group = APPLICATION_NAME + "-" + str(commitId)
repo = REPO
response = get_application(codedeploy, application_name)
response = get_deployment_group(codedeploy,application_name,deployment_group)
deploy(codedeploy, application_name, deployment_group, repo, commitId, wait=True)
response = get_deployment_group(
codedeploy, application_name, deployment_group)
deploy(codedeploy, application_name,
deployment_group, repo, commitId, wait=True)
return response
def get_availability_zones(ec2_client):
response = ec2_client.describe_availability_zones()
all_zones = []
@ -110,6 +136,7 @@ def get_availability_zones(ec2_client):
all_zones.append(info['ZoneName'])
return all_zones
def get_one_availability_zone(ec2_client):
all_zones = get_availability_zones(ec2_client)
if len(all_zones) > 0:
@ -118,6 +145,7 @@ def get_one_availability_zone(ec2_client):
print("No availability zone for this region")
sys.exit()
def create_instances(config, ec2_client, region_number, number_of_instances):
placement_group = region_number + "-" + PLACEMENT_GROUP
response = ec2_client.create_placement_group(
@ -149,10 +177,45 @@ def create_instances(config,ec2_client,region_number,number_of_instances):
},
]
},
]
],
# InstanceMarketOptions={
# 'MarketType': 'spot',
# 'SpotOptions': {
# 'BlockDurationMinutes': 60,
# }
# }
)
return response, placement_group
def request_spots(config, ec2_client, region_number, number_of_instances):
placement_group = region_number + "-" + PLACEMENT_GROUP
response = ec2_client.create_placement_group(
GroupName=placement_group,
Strategy='spread'
)
response = ec2_client.request_spot_instances(
# DryRun=True,
BlockDurationMinutes=60,
InstanceCount=number_of_instances,
LaunchSpecification={
'SecurityGroups': [config[region_number][REGION_SECURITY_GROUP]],
'IamInstanceProfile': {
'Name': IAM_INSTANCE_PROFILE
},
'UserData': USER_DATA,
'ImageId': config[region_number][REGION_AMI],
'InstanceType': INSTANCE_TYPE,
'KeyName': config[region_number][REGION_KEY],
'Placement': {
'AvailabilityZone': get_one_availability_zone(ec2_client),
'GroupName': placement_group
}
}
)
return response, placement_group
def get_deployment_group(codedeploy, application_name, deployment_group):
response = codedeploy.list_deployment_groups(
applicationName=application_name
@ -183,6 +246,7 @@ def get_deployment_group(codedeploy,application_name,deployment_group):
)
return response
def get_commitId(commitId):
if commitId is None:
commitId = run("git rev-list --max-count=1 HEAD",
@ -190,6 +254,7 @@ def get_commitId(commitId):
print("Got newest commitId as " + commitId)
return commitId
def get_application(codedeploy, application_name):
response = codedeploy.list_applications()
if application_name in response['applications']:
@ -201,6 +266,7 @@ def get_application(codedeploy,application_name):
)
return response
def deploy(codedeploy, application_name, deployment_group, repo, commitId, wait=True):
"""Deploy new code at specified revision to instance.
@ -242,6 +308,7 @@ def deploy(codedeploy, application_name,deployment_group,repo, commitId, wait=Tr
print("\nDeploy Failed")
print(info)
def read_configuration_file(filename):
config = {}
with open(filename, 'r') as f:
@ -256,29 +323,35 @@ def read_configuration_file(filename):
config[region_num][REGION_AMI] = mylist[5]
return config
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='This script helps you start instances across multiple regions')
parser.add_argument('--regions',type=str,dest='regions',default='3',help="Supply a csv list of all regions")
parser.add_argument('--instances', type=str,dest='numInstances',default='1',help='number of instances')
parser.add_argument('--configuration',type=str,dest='config',default='configuration.txt')
parser.add_argument('--commitId',type=str,dest='commitId',default='1f7e6e7ca7cf1c1190cedec10e791c01a29971cf')
parser = argparse.ArgumentParser(
description='This script helps you start instances across multiple regions')
parser.add_argument('--regions', type=str, dest='regions',
default='3', help="Supply a csv list of all regions")
parser.add_argument('--instances', type=str, dest='numInstances',
default='1', help='number of instances')
parser.add_argument('--configuration', type=str,
dest='config', default='configuration.txt')
parser.add_argument('--commitId', type=str, dest='commitId',
default='1f7e6e7ca7cf1c1190cedec10e791c01a29971cf')
args = parser.parse_args()
config = read_configuration_file(args.config)
region_list = args.regions.split(',')
instances_list = args.numInstances.split(',')
assert len(region_list) == len(instances_list),"number of regions: %d != number of instances per region: %d" % (len(region_list),len(intances_list))
assert len(region_list) == len(instances_list), "number of regions: %d != number of instances per region: %d" % (
len(region_list), len(intances_list))
commitId = args.commitId
placement_groups = []
for i in range(len(region_list)):
region_number = region_list[i]
number_of_instances = instances_list[i]
session,placement_group = run_one_region_instances(config,region_number,number_of_instances)
session, placement_group = run_one_region_instances(
config, region_number, number_of_instances)
placement_groups.append(placement_group)
for i in range(len(region_list)):
region_number = region_list[i]
placement_group = placement_groups[i]
run_one_region_codedeploy(region_number,placement_group,commitId)
Write Preview
Loading…
Cancel
Save