PoS_EVM
/
woop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' into commander

Browse Source
pull/37/head
Minh Doan 6 years ago
parent ef7720f983 58fd904b99
commit ad78ab5fce
8 changed files with 297 additions and 89 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 16
      aws-experiment-launch/configuration.txt
  2. 121
      aws-experiment-launch/create_and_deploy.py
  3. 58
      aws-experiment-launch/create_instances.py
  4. 16
      aws-experiment-launch/utils/configuration.txt
  5. 46
      aws-experiment-launch/utils/launch_template.py
  6. 9
      aws-experiment-launch/utils/logger.py
  7. 103
      aws-experiment-launch/utils/spot_fleet.py
  8. 17
      aws-experiment-launch/utils/utils.py

16
aws-experiment-launch/configuration.txt
Unescape View File

@ -1,8 +1,8 @@
1,us-east-1,virginia-key-benchmark,virginia-security-group,virginia,ami-b70554c8
2,us-east-2,ohio-key-benchmark,ohio-security-group,ohio,ami-8c122be9
3,us-west-1,california-key-benchmark,california-security-group,california,ami-e0ba5c83
4,us-west-2,oregon-key-benchmark,oregon-security-group,oregon,ami-a9d09ed1
5,ap-northeast-1,tokyo-key-benchmark,tokyo-security-group,tokyo,ami-e99f4896
6,ap-southeast-1,singapore-key-benchmark,singapore-security-group,singapore,ami-05868579
7,eu-central-1,frankfurt-key-benchmark,frankfurt-security-group,frankfurt,ami-7c4f7097
8,eu-west-1,ireland-key-benchmark,ireland-security-group,ireland,ami-466768ac
1,us-east-1,virginia-key-benchmark,virginia-security-group,virginia,ami-b70554c8,sg-04d0b62ee08ce8800
2,us-east-2,ohio-key-benchmark,ohio-security-group,ohio,ami-8c122be9,sg-0789078f1c76defbe
3,us-west-1,california-key-benchmark,california-security-group,california,ami-e0ba5c83,sg-0a66ccb6ab9161a14
4,us-west-2,oregon-key-benchmark,oregon-security-group,oregon,ami-a9d09ed1,sg-020cb5729fa212d43
5,ap-northeast-1,tokyo-key-benchmark,tokyo-security-group,tokyo,ami-e99f4896,sg-009aeb97f675c1ad5
6,ap-southeast-1,singapore-key-benchmark,singapore-security-group,singapore,ami-05868579,sg-05f9b60044a19dfb2
7,eu-central-1,frankfurt-key-benchmark,frankfurt-security-group,frankfurt,ami-7c4f7097,sg-0bb06fcd8b25b5910
8,eu-west-1,ireland-key-benchmark,ireland-security-group,ireland,ami-466768ac,sg-0aa8954acb79fdb58

121
aws-experiment-launch/create_and_deploy.py
Unescape View File

@ -18,6 +18,7 @@ class InstanceResource:
REGION_NAME = 'region_name'
REGION_KEY = 'region_key'
REGION_SECURITY_GROUP = 'region_security_group'
REGION_SECURITY_GROUP_ID = 'region_security_group_id'
REGION_HUMAN_NAME = 'region_human_name'
INSTANCE_TYPE = 't2.micro'
REGION_AMI = 'region_ami'
@ -36,6 +37,49 @@ CURRENT_SESSION = datetime.datetime.fromtimestamp(
PLACEMENT_GROUP = "PLACEMENT-" + CURRENT_SESSION
NODE_NAME_SUFFIX = "NODE-" + CURRENT_SESSION
def create_launch_specification(region_number, instanceType):
NODE_NAME = region_number + "-" + NODE_NAME_SUFFIX
return {
# Region irrelevant fields
'IamInstanceProfile': {
'Name': IAM_INSTANCE_PROFILE
},
'InstanceType': instanceType,
'UserData': USER_DATA_BASE64,
# Region relevant fields
'SecurityGroups': [
{
# In certain scenarios, we have to use group id instead of group name
# https://github.com/boto/boto/issues/350#issuecomment-27359492
'GroupId': config[region_number][REGION_SECURITY_GROUP_ID]
}
],
'ImageId': config[region_number][REGION_AMI],
'KeyName': config[region_number][REGION_KEY],
'UserData': USER_DATA_BASE64,
'TagSpecifications': [
{
'ResourceType': 'instance',
'Tags': [
{
'Key': 'Name',
'Value': NODE_NAME
}
]
}
],
# 'WeightedCapacity': 123.0,
# 'Placement': {
# # 'AvailabilityZone': get_one_availability_zone(ec2_client)
# }
}
def create_launch_specification_list(region_number, instance_type_list):
return list(map(lambda type: create_launch_specification(region_number, type), instance_type_list))
"""
TODO:
@ -60,13 +104,16 @@ def run_one_region_instances(config, region_number, number_of_instances, instanc
if instance_resource == InstanceResource.ON_DEMAND:
NODE_NAME = create_instances(
config, ec2_client, region_number, int(number_of_instances))
print("Created %s in region %s"%(NODE_NAME,region_number)) ##REPLACE ALL print with logger
# REPLACE ALL print with logger
print("Created %s in region %s" % (NODE_NAME, region_number))
elif instance_resource == InstanceResource.SPOT_INSTANCE:
response = request_spot_instances(
config, ec2_client, region_number, int(number_of_instances))
else:
instance_type_list = ['t2.micro', 't2.small', 'm3.medium']
response = request_spot_fleet(
config, ec2_client, region_number, int(number_of_instances))
config, ec2_client, region_number, int(number_of_instances), instance_type_list)
return
return session
@ -133,7 +180,8 @@ def request_spot_instances(config, ec2_client, region_number, number_of_instance
)
return response
def request_spot_fleet(config, ec2_client, region_number, number_of_instances):
def request_spot_fleet(config, ec2_client, region_number, number_of_instances, instance_type_list):
NODE_NAME = region_number + "-" + NODE_NAME_SUFFIX
# https://boto3.readthedocs.io/en/latest/reference/services/ec2.html#EC2.Client.request_spot_fleet
response = ec2_client.request_spot_fleet(
@ -142,40 +190,7 @@ def request_spot_fleet(config, ec2_client, region_number, number_of_instances):
# https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-fleet.html#spot-fleet-allocation-strategy
'AllocationStrategy': 'diversified',
'IamFleetRole': 'arn:aws:iam::656503231766:role/RichardFleetRole',
'LaunchSpecifications': [
{
'SecurityGroups': [
{
# In certain scenarios, we have to use group id instead of group name
# https://github.com/boto/boto/issues/350#issuecomment-27359492
# 'GroupName': config[region_number][REGION_SECURITY_GROUP]
'GroupId': 'sg-06f90158506dca54f'
}
],
'IamInstanceProfile': {
'Name': IAM_INSTANCE_PROFILE
},
'ImageId': config[region_number][REGION_AMI],
'InstanceType': INSTANCE_TYPE,
'KeyName': config[region_number][REGION_KEY],
# 'Placement': {
# # 'AvailabilityZone': get_one_availability_zone(ec2_client)
# },
'UserData': USER_DATA_BASE64,
# 'WeightedCapacity': 123.0,
'TagSpecifications': [
{
'ResourceType': 'instance',
'Tags': [
{
'Key': 'Name',
'Value': NODE_NAME
},
]
}
]
},
],
'LaunchSpecifications': create_launch_specification_list(region_number, instance_type_list),
# 'SpotPrice': 'string', # The maximum price per unit hour that you are willing to pay for a Spot Instance. The default is the On-Demand price.
'TargetCapacity': number_of_instances,
'OnDemandTargetCapacity': 0,
@ -224,29 +239,35 @@ def run_one_region_codedeploy(region_number, commitId):
NODE_NAME = region_number + "-" + NODE_NAME_SUFFIX
session = boto3.Session(region_name=region_name)
ec2_client = session.client('ec2')
filters = [{'Name': 'tag:Name','Values': [NODE_NAME]}]
instance_ids = get_instance_ids(ec2_client.describe_instances(Filters=filters))
filters = [{'Name': 'tag:Name', 'Values': [NODE_NAME]}]
instance_ids = get_instance_ids(
ec2_client.describe_instances(Filters=filters))
print("Number of instances: %d" % len(instance_ids))
print("Waiting for all %d instances in region %s to start running"%(len(instance_ids),region_number))
print("Waiting for all %d instances in region %s to start running" %
(len(instance_ids), region_number))
waiter = ec2_client.get_waiter('instance_running')
waiter.wait(InstanceIds=instance_ids)
print("Waiting for all %d instances in region %s to be INSTANCE STATUS OK"%(len(instance_ids),region_number))
print("Waiting for all %d instances in region %s to be INSTANCE STATUS OK" % (
len(instance_ids), region_number))
waiter = ec2_client.get_waiter('instance_status_ok')
waiter.wait(InstanceIds=instance_ids)
print("Waiting for all %d instances in region %s to be SYSTEM STATUS OK"%(len(instance_ids),region_number))
print("Waiting for all %d instances in region %s to be SYSTEM STATUS OK" %
(len(instance_ids), region_number))
waiter = ec2_client.get_waiter('system_status_ok')
waiter.wait(InstanceIds=instance_ids)
codedeploy = session.client('codedeploy')
application_name = APPLICATION_NAME
deployment_group = APPLICATION_NAME + "-" + commitId[:6] + "-" + CURRENT_SESSION
deployment_group = APPLICATION_NAME + "-" + \
commitId[:6] + "-" + CURRENT_SESSION
repo = REPO
print("Setting up to deploy commitId %s on region %s"%(commitId,region_number))
print("Setting up to deploy commitId %s on region %s" %
(commitId, region_number))
response = get_application(codedeploy, application_name)
deployment_group = get_deployment_group(
codedeploy, region_number, application_name, deployment_group)
@ -255,7 +276,7 @@ def run_one_region_codedeploy(region_number, commitId):
return region_number, depId
def get_deployment_group(codedeploy, region_number,application_name, deployment_group):
def get_deployment_group(codedeploy, region_number, application_name, deployment_group):
NODE_NAME = region_number + "-" + NODE_NAME_SUFFIX
response = codedeploy.create_deployment_group(
applicationName=application_name,
@ -266,7 +287,7 @@ def get_deployment_group(codedeploy, region_number,application_name, deployment_
'deploymentType': 'IN_PLACE',
'deploymentOption': 'WITHOUT_TRAFFIC_CONTROL'
},
ec2TagFilters = [
ec2TagFilters=[
{
'Key': 'Name',
'Value': NODE_NAME,
@ -329,10 +350,12 @@ def deploy(codedeploy, application_name, deployment_group, repo, commitId):
print(info)
return depId
def run_one_region_codedeploy_wrapper(region_number, commitId, queue):
region_number, depId = run_one_region_codedeploy(region_number, commitId)
queue.put((region_number, depId))
def launch_code_deploy(region_list, commitId):
queue = Queue()
jobs = []
@ -357,6 +380,7 @@ def get_instance_ids(describe_instances_response):
instance_ids.append(instance["InstanceId"])
return instance_ids
def read_configuration_file(filename):
config = {}
with open(filename, 'r') as f:
@ -369,6 +393,7 @@ def read_configuration_file(filename):
config[region_num][REGION_SECURITY_GROUP] = mylist[3]
config[region_num][REGION_HUMAN_NAME] = mylist[4]
config[region_num][REGION_AMI] = mylist[5]
config[region_num][REGION_SECURITY_GROUP_ID] = mylist[6]
return config
##### UTILS ####
@ -396,6 +421,6 @@ if __name__ == "__main__":
region_number = region_list[i]
number_of_instances = instances_list[i]
session = run_one_region_instances(
config, region_number, number_of_instances,InstanceResource.ON_DEMAND)
config, region_number, number_of_instances, InstanceResource.SPOT_FLEET)
results = launch_code_deploy(region_list, commitId)
print(results)

58
aws-experiment-launch/create_instances.py
Unescape View File

@ -3,30 +3,20 @@ import base64
import boto3
import datetime
import json
import logging
import sys
import threading
import time
from utils import utils
from utils import utils, spot_fleet, logger
LOGGER = logger.getLogger(__file__)
logging.basicConfig(level=logging.INFO, format='%(threadName)s %(asctime)s - %(name)s - %(levelname)s - %(message)s')
LOGGER = logging.getLogger(__file__)
LOGGER.setLevel(logging.INFO)
class InstanceResource:
ON_DEMAND = 1
SPOT_INSTANCE = 2
SPOT_FLEET = 3
with open("user-data.sh", "r") as userdata_file:
USER_DATA = userdata_file.read()
IAM_INSTANCE_PROFILE = 'BenchMarkCodeDeployInstanceProfile'
time_stamp = time.time()
CURRENT_SESSION = datetime.datetime.fromtimestamp(
time_stamp).strftime('%H-%M-%S-%Y-%m-%d')
NODE_NAME_SUFFIX = "NODE-" + CURRENT_SESSION
def run_one_region_instances(config, region_number, number_of_instances, instance_resource=InstanceResource.ON_DEMAND):
region_name = config[region_number][utils.REGION_NAME]
@ -40,12 +30,19 @@ def run_one_region_instances(config, region_number, number_of_instances, instanc
config, ec2_client, region_number, int(number_of_instances))
LOGGER.info("Created %s in region %s" % (node_name_tag, region_number))
return node_name_tag, ec2_client
elif instance_resource == InstanceResource.SPOT_FLEET:
instance_type_list = ['t2.micro', 't2.small', 'm3.medium']
node_name_tag = spot_fleet.request_spot_fleet_with_on_demand(
config, ec2_client, region_number, int(number_of_instances), 1, instance_type_list)
# node_name_tag = spot_fleet.request_spot_fleet(
# config, ec2_client, region_number, int(number_of_instances), instance_type_list)
return node_name_tag, ec2_client
else:
return None, None
def create_instances(config, ec2_client, region_number, number_of_instances):
node_name_tag = region_number + "-" + NODE_NAME_SUFFIX
node_name_tag = utils.get_node_name_tag(region_number)
LOGGER.info("Creating node_name_tag: %s" % node_name_tag)
available_zone = utils.get_one_availability_zone(ec2_client)
LOGGER.info("Looking at zone %s to create instances." % available_zone)
@ -59,10 +56,10 @@ def create_instances(config, ec2_client, region_number, number_of_instances):
},
SecurityGroups=[config[region_number][utils.REGION_SECURITY_GROUP]],
IamInstanceProfile={
'Name': IAM_INSTANCE_PROFILE
'Name': utils.IAM_INSTANCE_PROFILE
},
KeyName=config[region_number][utils.REGION_KEY],
UserData=USER_DATA,
UserData=utils.USER_DATA,
InstanceType=utils.INSTANCE_TYPE,
TagSpecifications=[
{
@ -78,7 +75,8 @@ def create_instances(config, ec2_client, region_number, number_of_instances):
)
instance_ids = utils.get_instance_ids2(ec2_client, node_name_tag)
LOGGER.info("Waiting for all %d instances in region %s to be in RUNNING" % (len(instance_ids), region_number))
LOGGER.info("Waiting for all %d instances in region %s to be in RUNNING" % (
len(instance_ids), region_number))
waiter = ec2_client.get_waiter('instance_running')
waiter.wait(InstanceIds=instance_ids)
@ -86,7 +84,8 @@ def create_instances(config, ec2_client, region_number, number_of_instances):
while count < 40:
time.sleep(5)
LOGGER.info("Waiting ...")
ip_list = utils.collect_public_ips_from_ec2_client(ec2_client, node_name_tag)
ip_list = utils.collect_public_ips_from_ec2_client(
ec2_client, node_name_tag)
if len(ip_list) == number_of_instances:
LOGGER.info("Created %d instances" % number_of_instances)
return node_name_tag
@ -94,22 +93,28 @@ def create_instances(config, ec2_client, region_number, number_of_instances):
LOGGER.info("Can not create %d instances" % number_of_instances)
return None
lock = threading.Lock()
def run_for_one_region(config, region_number, number_of_instances, instance_resouce, fout, fout2):
node_name_tag, ec2_client = run_one_region_instances(config, region_number, number_of_instances, InstanceResource.ON_DEMAND)
node_name_tag, ec2_client = run_one_region_instances(
config, region_number, number_of_instances, instance_resouce)
if node_name_tag:
LOGGER.info("Managed to create instances for region %s" % region_number )
LOGGER.info("Managed to create instances for region %s" %
region_number)
instance_ids = utils.get_instance_ids2(ec2_client, node_name_tag)
lock.acquire()
try:
fout.write("%s %s\n" % (node_name_tag, region_number))
for instance_id in instance_ids:
fout2.write(instance_id + " " + node_name_tag + " " + region_number + " " + config[region_number][utils.REGION_NAME] + "\n")
fout2.write(instance_id + " " + node_name_tag + " " + region_number +
" " + config[region_number][utils.REGION_NAME] + "\n")
finally:
lock.release()
else:
LOGGER.info("Failed to create instances for region %s" % region_number )
LOGGER.info("Failed to create instances for region %s" % region_number)
if __name__ == "__main__":
parser = argparse.ArgumentParser(
@ -118,7 +123,8 @@ if __name__ == "__main__":
default='3', help="Supply a csv list of all regions")
parser.add_argument('--instances', type=str, dest='num_instance_list',
default='1', help='number of instances in respective of region')
parser.add_argument('--region_config', type=str, dest='region_config', default='configuration.txt')
parser.add_argument('--region_config', type=str,
dest='region_config', default='configuration.txt')
parser.add_argument('--instance_output', type=str, dest='instance_output',
default='instance_output.txt', help='the file to append or write')
parser.add_argument('--instance_ids_output', type=str, dest='instance_ids_output',
@ -129,7 +135,8 @@ if __name__ == "__main__":
config = utils.read_region_config(args.region_config)
region_list = args.regions.split(',')
num_instance_list = args.num_instance_list.split(',')
assert len(region_list) == len(num_instance_list), "number of regions: %d != number of instances per region: %d" % (len(region_list), len(num_instance_list))
assert len(region_list) == len(num_instance_list), "number of regions: %d != number of instances per region: %d" % (
len(region_list), len(num_instance_list))
write_mode = "a" if args.append else "w"
with open(args.instance_output, write_mode) as fout, open(args.instance_ids_output, write_mode) as fout2:
@ -137,7 +144,8 @@ if __name__ == "__main__":
for i in range(len(region_list)):
region_number = region_list[i]
number_of_instances = num_instance_list[i]
t = threading.Thread(target=run_for_one_region, args=(config, region_number, number_of_instances, InstanceResource.ON_DEMAND, fout, fout2))
t = threading.Thread(target=run_for_one_region, args=(
config, region_number, number_of_instances, InstanceResource.SPOT_FLEET, fout, fout2))
LOGGER.info("creating thread for region %s" % region_number)
t.start()
thread_pool.append(t)

16
aws-experiment-launch/utils/configuration.txt
Unescape View File

@ -1,8 +1,8 @@
1,us-east-1,virginia-key-benchmark,virginia-security-group,virginia,ami-b70554c8
2,us-east-2,ohio-key-benchmark,ohio-security-group,ohio,ami-8c122be9
3,us-west-1,california-key-benchmark,california-security-group,california,ami-e0ba5c83
4,us-west-2,oregon-key-benchmark,oregon-security-group,oregon,ami-a9d09ed1
5,ap-northeast-1,tokyo-key-benchmark,tokyo-security-group,tokyo,ami-e99f4896
6,ap-southeast-1,singapore-key-benchmark,singapore-security-group,singapore,ami-05868579
7,eu-central-1,frankfurt-key-benchmark,frankfurt-security-group,frankfurt,ami-7c4f7097
8,eu-west-1,ireland-key-benchmark,ireland-security-group,ireland,ami-466768ac
1,us-east-1,virginia-key-benchmark,virginia-security-group,virginia,ami-b70554c8,sg-04d0b62ee08ce8800
2,us-east-2,ohio-key-benchmark,ohio-security-group,ohio,ami-8c122be9,sg-0789078f1c76defbe
3,us-west-1,california-key-benchmark,california-security-group,california,ami-e0ba5c83,sg-0a66ccb6ab9161a14
4,us-west-2,oregon-key-benchmark,oregon-security-group,oregon,ami-a9d09ed1,sg-020cb5729fa212d43
5,ap-northeast-1,tokyo-key-benchmark,tokyo-security-group,tokyo,ami-e99f4896,sg-009aeb97f675c1ad5
6,ap-southeast-1,singapore-key-benchmark,singapore-security-group,singapore,ami-05868579,sg-05f9b60044a19dfb2
7,eu-central-1,frankfurt-key-benchmark,frankfurt-security-group,frankfurt,ami-7c4f7097,sg-0bb06fcd8b25b5910
8,eu-west-1,ireland-key-benchmark,ireland-security-group,ireland,ami-466768ac,sg-0aa8954acb79fdb58

46
aws-experiment-launch/utils/launch_template.py
Unescape View File

@ -0,0 +1,46 @@
import utils
def get_launch_template_name(config, region_number):
return 'benchmark-' + config[region_number][utils.REGION_NAME]
def create(config, ec2_client, region_number):
return ec2_client.create_launch_template(
# DryRun=True,
LaunchTemplateName=get_launch_template_name(config, region_number),
LaunchTemplateData={
'IamInstanceProfile': {
'Name': utils.IAM_INSTANCE_PROFILE
},
'ImageId': config[region_number][utils.REGION_AMI],
# 'InstanceType': instance_type,
'KeyName': config[region_number][utils.REGION_KEY],
'UserData': utils.USER_DATA_BASE64,
'SecurityGroupIds': [
config[region_number][utils.REGION_SECURITY_GROUP_ID]
],
# 'InstanceInitiatedShutdownBehavior': 'stop',
'TagSpecifications': [
{
'ResourceType': 'instance',
'Tags': [
{
'Key': 'LaunchTemplate',
'Value': 'Yes'
}
]
}
],
# 'InstanceMarketOptions': {
# 'MarketType': 'spot',
# 'SpotOptions': {
# 'MaxPrice': 'string',
# 'SpotInstanceType': 'one-time'|'persistent',
# 'BlockDurationMinutes': 123,
# 'InstanceInterruptionBehavior': 'hibernate'|'stop'|'terminate'
# }
# },
}
)

9
aws-experiment-launch/utils/logger.py
Unescape View File

@ -0,0 +1,9 @@
import logging
logging.basicConfig(level=logging.INFO,
format='%(threadName)s %(asctime)s - %(name)s - %(levelname)s - %(message)s')
def getLogger(file):
LOGGER = logging.getLogger(file)
LOGGER.setLevel(logging.INFO)
return LOGGER

103
aws-experiment-launch/utils/spot_fleet.py
Unescape View File

@ -0,0 +1,103 @@
import utils
import logger
import launch_template
LOGGER = logger.getLogger(__file__)
def create_launch_specification(config, region_number, instanceType):
return {
# Region irrelevant fields
'IamInstanceProfile': {
'Name': utils.IAM_INSTANCE_PROFILE
},
'InstanceType': instanceType,
'UserData': utils.USER_DATA_BASE64,
# Region relevant fields
'SecurityGroups': [
{
# In certain scenarios, we have to use group id instead of group name
# https://github.com/boto/boto/issues/350#issuecomment-27359492
'GroupId': config[region_number][utils.REGION_SECURITY_GROUP_ID]
}
],
'ImageId': config[region_number][utils.REGION_AMI],
'KeyName': config[region_number][utils.REGION_KEY],
'TagSpecifications': [
{
'ResourceType': 'instance',
'Tags': [
{
'Key': 'Name',
'Value': utils.get_node_name_tag(region_number)
}
]
}
],
# 'WeightedCapacity': 123.0,
# 'Placement': {
# # 'AvailabilityZone': get_one_availability_zone(ec2_client)
# }
}
def create_launch_specification_list(config, region_number, instance_type_list):
return list(map(lambda type: create_launch_specification(config, region_number, type), instance_type_list))
def get_launch_template(config, region_number, instance_type):
return {
'LaunchTemplateSpecification': {
'LaunchTemplateName': launch_template.get_launch_template_name(config, region_number),
'Version': '1'
},
'Overrides': [
{
'InstanceType': instance_type
}
]
}
def get_launch_template_list(config, region_number, instance_type_list):
return list(map(lambda type: get_launch_template(config, region_number, type), instance_type_list))
def request_spot_fleet(config, ec2_client, region_number, number_of_instances, instance_type_list):
LOGGER.info("Requesting spot fleet")
LOGGER.info("Creating node_name_tag: %s" %
utils.get_node_name_tag(region_number))
# https://boto3.readthedocs.io/en/latest/reference/services/ec2.html#EC2.Client.request_spot_fleet
response = ec2_client.request_spot_fleet(
# DryRun=True,
SpotFleetRequestConfig={
# https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-fleet.html#spot-fleet-allocation-strategy
'AllocationStrategy': 'diversified',
'IamFleetRole': 'arn:aws:iam::656503231766:role/RichardFleetRole',
'LaunchSpecifications': create_launch_specification_list(config, region_number, instance_type_list),
# 'SpotPrice': 'string', # The maximum price per unit hour that you are willing to pay for a Spot Instance. The default is the On-Demand price.
'TargetCapacity': number_of_instances,
'Type': 'maintain'
}
)
return response
def request_spot_fleet_with_on_demand(config, ec2_client, region_number, number_of_instances, number_of_on_demand, instance_type_list):
LOGGER.info("Requesting spot fleet")
LOGGER.info("Creating node_name_tag: %s" %
utils.get_node_name_tag(region_number))
# https://boto3.readthedocs.io/en/latest/reference/services/ec2.html#EC2.Client.request_spot_fleet
response = ec2_client.request_spot_fleet(
# DryRun=True,
SpotFleetRequestConfig={
# https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-fleet.html#spot-fleet-allocation-strategy
'AllocationStrategy': 'diversified',
'IamFleetRole': 'arn:aws:iam::656503231766:role/RichardFleetRole',
'LaunchTemplateConfigs': get_launch_template_list(config, region_number, instance_type_list),
# 'SpotPrice': 'string', # The maximum price per unit hour that you are willing to pay for a Spot Instance. The default is the On-Demand price.
'TargetCapacity': number_of_instances,
'OnDemandTargetCapacity': number_of_on_demand,
'Type': 'maintain'
}
)
return response

17
aws-experiment-launch/utils/utils.py
Unescape View File

@ -3,14 +3,30 @@ import datetime
import json
import sys
import time
import base64
REGION_NAME = 'region_name'
REGION_KEY = 'region_key'
REGION_SECURITY_GROUP = 'region_security_group'
REGION_SECURITY_GROUP_ID = 'region_security_group_id'
REGION_HUMAN_NAME = 'region_human_name'
INSTANCE_TYPE = 't2.micro'
REGION_AMI = 'region_ami'
IAM_INSTANCE_PROFILE = 'BenchMarkCodeDeployInstanceProfile'
time_stamp = time.time()
CURRENT_SESSION = datetime.datetime.fromtimestamp(
time_stamp).strftime('%H-%M-%S-%Y-%m-%d')
NODE_NAME_SUFFIX = "NODE-" + CURRENT_SESSION
def get_node_name_tag(region_number):
return region_number + "-" + NODE_NAME_SUFFIX
with open("user-data.sh", "r") as userdata_file:
USER_DATA = userdata_file.read()
# UserData must be base64 encoded for spot instances.
USER_DATA_BASE64 = base64.b64encode(USER_DATA)
def read_region_config(region_config='configuration.txt'):
config = {}
@ -24,6 +40,7 @@ def read_region_config(region_config='configuration.txt'):
config[region_num][REGION_SECURITY_GROUP] = mylist[3]
config[region_num][REGION_HUMAN_NAME] = mylist[4]
config[region_num][REGION_AMI] = mylist[5]
config[region_num][REGION_SECURITY_GROUP_ID] = mylist[6]
return config
def get_ip_list(response):

Write Preview
Loading…
Cancel
Save