From 6a4f66bad8410e71db103f56c29d9719bcf2ae6b Mon Sep 17 00:00:00 2001 From: Minh Doan Date: Wed, 11 Jul 2018 22:27:02 -0700 Subject: [PATCH 1/4] fix bug to get ondemand working --- aws-experiment-launch/create_instances.py | 9 ++++++--- aws-experiment-launch/utils/user-data.sh | 11 +++++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) create mode 100755 aws-experiment-launch/utils/user-data.sh diff --git a/aws-experiment-launch/create_instances.py b/aws-experiment-launch/create_instances.py index b97b2f1e1..01fd7a127 100644 --- a/aws-experiment-launch/create_instances.py +++ b/aws-experiment-launch/create_instances.py @@ -6,19 +6,20 @@ import json import sys import threading import time +import enum from utils import utils, spot_fleet, logger LOGGER = logger.getLogger(__file__) -class InstanceResource: +class InstanceResource(enum.Enum): ON_DEMAND = 1 SPOT_INSTANCE = 2 SPOT_FLEET = 3 -def run_one_region_instances(config, region_number, number_of_instances, instance_resource=InstanceResource.ON_DEMAND): +def run_one_region_instances(config, region_number, number_of_instances, instance_resource): region_name = config[region_number][utils.REGION_NAME] # Create session. session = boto3.Session(region_name=region_name) @@ -129,6 +130,8 @@ if __name__ == "__main__": default='instance_output.txt', help='the file to append or write') parser.add_argument('--instance_ids_output', type=str, dest='instance_ids_output', default='instance_ids_output.txt', help='the file to append or write') + parser.add_argument('--instance_resource', dest='instance_resource', type=InstanceResource, + default=InstanceResource.ON_DEMAND, choices=list(InstanceResource)) parser.add_argument('--append', dest='append', type=bool, default=False, help='append to the current instance_output') args = parser.parse_args() @@ -145,7 +148,7 @@ if __name__ == "__main__": region_number = region_list[i] number_of_instances = num_instance_list[i] t = threading.Thread(target=run_for_one_region, args=( - config, region_number, number_of_instances, InstanceResource.SPOT_FLEET, fout, fout2)) + config, region_number, number_of_instances, InstanceResource.ON_DEMAND, fout, fout2)) LOGGER.info("creating thread for region %s" % region_number) t.start() thread_pool.append(t) diff --git a/aws-experiment-launch/utils/user-data.sh b/aws-experiment-launch/utils/user-data.sh new file mode 100755 index 000000000..165aab479 --- /dev/null +++ b/aws-experiment-launch/utils/user-data.sh @@ -0,0 +1,11 @@ +#!/bin/bash -x +REGION=$(curl 169.254.169.254/latest/meta-data/placement/availability-zone/ | sed 's/[a-z]$//') +#yum update -y #This breaking codedeploy right now +yum install ruby wget -y +cd /home/ec2-user +touch yum-not-updated.txt +wget https://aws-codedeploy-$REGION.s3.amazonaws.com/latest/install +chmod +x ./install +./install auto +mkdir projects +mkdir projects/src \ No newline at end of file From d2222607a6a7e1b99982cd274a38c5203f07cd52 Mon Sep 17 00:00:00 2001 From: Richard Liu Date: Wed, 11 Jul 2018 22:34:28 -0700 Subject: [PATCH 2/4] separate on demand and fleet. --- aws-experiment-launch/create_instances.py | 43 +++++------ .../spot-instance/create_launch_specs.py | 72 ------------------- .../utils/launch_template.py | 14 ++-- aws-experiment-launch/utils/spot_fleet.py | 43 +++++++---- aws-experiment-launch/utils/utils.py | 9 +++ 5 files changed, 61 insertions(+), 120 deletions(-) delete mode 100644 aws-experiment-launch/spot-instance/create_launch_specs.py diff --git a/aws-experiment-launch/create_instances.py b/aws-experiment-launch/create_instances.py index b97b2f1e1..934215dab 100644 --- a/aws-experiment-launch/create_instances.py +++ b/aws-experiment-launch/create_instances.py @@ -17,29 +17,13 @@ class InstanceResource: SPOT_INSTANCE = 2 SPOT_FLEET = 3 +def run_one_region_on_demand_instances(config, region_number, number_of_instances): + ec2_client = utils.create_client(config, region_number) -def run_one_region_instances(config, region_number, number_of_instances, instance_resource=InstanceResource.ON_DEMAND): - region_name = config[region_number][utils.REGION_NAME] - # Create session. - session = boto3.Session(region_name=region_name) - # Create a client. - ec2_client = session.client('ec2') - - if instance_resource == InstanceResource.ON_DEMAND: - node_name_tag = create_instances( - config, ec2_client, region_number, int(number_of_instances)) - LOGGER.info("Created %s in region %s" % (node_name_tag, region_number)) - return node_name_tag, ec2_client - elif instance_resource == InstanceResource.SPOT_FLEET: - instance_type_list = ['t2.micro', 't2.small', 'm3.medium'] - node_name_tag = spot_fleet.request_spot_fleet_with_on_demand( - config, ec2_client, region_number, int(number_of_instances), 1, instance_type_list) - # node_name_tag = spot_fleet.request_spot_fleet( - # config, ec2_client, region_number, int(number_of_instances), instance_type_list) - return node_name_tag, ec2_client - else: - return None, None - + node_name_tag = create_instances( + config, ec2_client, region_number, int(number_of_instances)) + LOGGER.info("Created %s in region %s" % (node_name_tag, region_number)) + return node_name_tag, ec2_client def create_instances(config, ec2_client, region_number, number_of_instances): node_name_tag = utils.get_node_name_tag(region_number) @@ -97,9 +81,9 @@ def create_instances(config, ec2_client, region_number, number_of_instances): lock = threading.Lock() -def run_for_one_region(config, region_number, number_of_instances, instance_resouce, fout, fout2): - node_name_tag, ec2_client = run_one_region_instances( - config, region_number, number_of_instances, instance_resouce) +def run_for_one_region_on_demand(config, region_number, number_of_instances, fout, fout2): + node_name_tag, ec2_client = run_one_region_on_demand_instances( + config, region_number, number_of_instances) if node_name_tag: LOGGER.info("Managed to create instances for region %s" % region_number) @@ -131,6 +115,7 @@ if __name__ == "__main__": default='instance_ids_output.txt', help='the file to append or write') parser.add_argument('--append', dest='append', type=bool, default=False, help='append to the current instance_output') + instanceResource = InstanceResource.SPOT_FLEET args = parser.parse_args() config = utils.read_region_config(args.region_config) region_list = args.regions.split(',') @@ -144,8 +129,12 @@ if __name__ == "__main__": for i in range(len(region_list)): region_number = region_list[i] number_of_instances = num_instance_list[i] - t = threading.Thread(target=run_for_one_region, args=( - config, region_number, number_of_instances, InstanceResource.SPOT_FLEET, fout, fout2)) + if instanceResource == InstanceResource.ON_DEMAND: + t = threading.Thread(target=run_for_one_region_on_demand, args=( + config, region_number, number_of_instances, fout, fout2)) + elif instanceResource == InstanceResource.SPOT_FLEET: + t = threading.Thread(target=spot_fleet.run_one_region, args=( + region_number, number_of_instances, fout, fout2)) LOGGER.info("creating thread for region %s" % region_number) t.start() thread_pool.append(t) diff --git a/aws-experiment-launch/spot-instance/create_launch_specs.py b/aws-experiment-launch/spot-instance/create_launch_specs.py deleted file mode 100644 index 185a648d9..000000000 --- a/aws-experiment-launch/spot-instance/create_launch_specs.py +++ /dev/null @@ -1,72 +0,0 @@ -import os -import argparse -import json -import time -import datetime -import base64 - -REGION_NAME = 'region_name' -REGION_KEY = 'region_key' -REGION_SECURITY_GROUP = 'region_security_group' -REGION_HUMAN_NAME = 'region_human_name' - -INSTANCE_TYPE = 'm3.medium' # 't2.micro' -AMI = 'ami-f2d3638a' # 'ami-a9d09ed1' -# UserData must be base64 encoded. -with open("userdata.sh", "rb") as userdata_file: - USER_DATA = base64.b64encode(userdata_file.read()) -IAM_INSTANCE_PROFILE = 'BenchMarkCodeDeployInstanceProfile' - -def read_configuration_file(filename): - config = {} - with open(filename,'r') as f: - for line in f: - vals = line.strip().split(',') - region_num = vals[0] - config[region_num] = {} - config[region_num][REGION_NAME] = vals[1] - config[region_num][REGION_KEY] = vals[2] - config[region_num][REGION_SECURITY_GROUP] = vals[3] - config[region_num][REGION_HUMAN_NAME] = vals[4] - return config - -def create_launch_specification(region_num): - input_cli = {} - input_cli['ImageId'] = AMI - # input_cli['Placement'] = { - # "AvailabilityZone": config[region_num][REGION_NAME] +"a" - # } - input_cli['SecurityGroups'] = [ "richard-spot-instance SSH" ] # [ config[region_num][REGION_SECURITY_GROUP] ] - input_cli['IamInstanceProfile'] = { - "Name": IAM_INSTANCE_PROFILE - } - input_cli['KeyName'] = "richard-spot-instance" # config[region_num][REGION_KEY] - input_cli['UserData'] = USER_DATA - input_cli['InstanceType'] = INSTANCE_TYPE - # folder = "launch_specs/" + "session-"+ current_session - folder = "launch_specs/latest" - if not os.path.exists(folder): - os.makedirs(folder) - launch_spec_file = os.path.join(folder,config[region_num][REGION_HUMAN_NAME]+".json") - with open(launch_spec_file,'w') as g: - json.dump(input_cli,g) - print("Launch spec: %s" % launch_spec_file) - return launch_spec_file - -def create_instances(region_list): - for i in range(len(region_list)): - region_num = region_list[i] - launch_spec_file = create_launch_specification(region_num) - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description='This script helps you start instances across multiple regions') - parser.add_argument('--regions',type=str,dest='regions',default='3',help="a comma-separated-value list of all regions") - # configuration file contains the number/name/security-group etc. information of each region. - parser.add_argument('--config',type=str,dest='config',default='configuration.txt') - args = parser.parse_args() - config = read_configuration_file(args.config) - region_list = args.regions.split(',') - time_stamp = time.time() - current_session = datetime.datetime.fromtimestamp(time_stamp).strftime('%Y-%m-%d-%H-%M-%S') - print("current session is %s" % current_session) - create_instances(region_list) \ No newline at end of file diff --git a/aws-experiment-launch/utils/launch_template.py b/aws-experiment-launch/utils/launch_template.py index 1ca9e73da..6c915abc6 100644 --- a/aws-experiment-launch/utils/launch_template.py +++ b/aws-experiment-launch/utils/launch_template.py @@ -2,24 +2,24 @@ import utils -def get_launch_template_name(config, region_number): - return 'benchmark-' + config[region_number][utils.REGION_NAME] +def get_launch_template_name(region_number): + return 'benchmark-' + utils.CONFIG[region_number][utils.REGION_NAME] -def create(config, ec2_client, region_number): +def create(ec2_client, region_number): return ec2_client.create_launch_template( # DryRun=True, - LaunchTemplateName=get_launch_template_name(config, region_number), + LaunchTemplateName=get_launch_template_name(region_number), LaunchTemplateData={ 'IamInstanceProfile': { 'Name': utils.IAM_INSTANCE_PROFILE }, - 'ImageId': config[region_number][utils.REGION_AMI], + 'ImageId': utils.CONFIG[region_number][utils.REGION_AMI], # 'InstanceType': instance_type, - 'KeyName': config[region_number][utils.REGION_KEY], + 'KeyName': utils.CONFIG[region_number][utils.REGION_KEY], 'UserData': utils.USER_DATA_BASE64, 'SecurityGroupIds': [ - config[region_number][utils.REGION_SECURITY_GROUP_ID] + utils.CONFIG[region_number][utils.REGION_SECURITY_GROUP_ID] ], # 'InstanceInitiatedShutdownBehavior': 'stop', 'TagSpecifications': [ diff --git a/aws-experiment-launch/utils/spot_fleet.py b/aws-experiment-launch/utils/spot_fleet.py index f98bb0937..3c41bd9d2 100644 --- a/aws-experiment-launch/utils/spot_fleet.py +++ b/aws-experiment-launch/utils/spot_fleet.py @@ -4,7 +4,7 @@ import launch_template LOGGER = logger.getLogger(__file__) -def create_launch_specification(config, region_number, instanceType): +def create_launch_specification(region_number, instanceType): return { # Region irrelevant fields 'IamInstanceProfile': { @@ -17,11 +17,11 @@ def create_launch_specification(config, region_number, instanceType): { # In certain scenarios, we have to use group id instead of group name # https://github.com/boto/boto/issues/350#issuecomment-27359492 - 'GroupId': config[region_number][utils.REGION_SECURITY_GROUP_ID] + 'GroupId': utils.CONFIG[region_number][utils.REGION_SECURITY_GROUP_ID] } ], - 'ImageId': config[region_number][utils.REGION_AMI], - 'KeyName': config[region_number][utils.REGION_KEY], + 'ImageId': utils.CONFIG[region_number][utils.REGION_AMI], + 'KeyName': utils.CONFIG[region_number][utils.REGION_KEY], 'TagSpecifications': [ { 'ResourceType': 'instance', @@ -40,14 +40,14 @@ def create_launch_specification(config, region_number, instanceType): } -def create_launch_specification_list(config, region_number, instance_type_list): - return list(map(lambda type: create_launch_specification(config, region_number, type), instance_type_list)) +def create_launch_specification_list(region_number, instance_type_list): + return list(map(lambda type: create_launch_specification(region_number, type), instance_type_list)) -def get_launch_template(config, region_number, instance_type): +def get_launch_template(region_number, instance_type): return { 'LaunchTemplateSpecification': { - 'LaunchTemplateName': launch_template.get_launch_template_name(config, region_number), + 'LaunchTemplateName': launch_template.get_launch_template_name(region_number), 'Version': '1' }, 'Overrides': [ @@ -58,11 +58,11 @@ def get_launch_template(config, region_number, instance_type): } -def get_launch_template_list(config, region_number, instance_type_list): - return list(map(lambda type: get_launch_template(config, region_number, type), instance_type_list)) +def get_launch_template_list(region_number, instance_type_list): + return list(map(lambda type: get_launch_template(region_number, type), instance_type_list)) -def request_spot_fleet(config, ec2_client, region_number, number_of_instances, instance_type_list): +def request_spot_fleet(ec2_client, region_number, number_of_instances, instance_type_list): LOGGER.info("Requesting spot fleet") LOGGER.info("Creating node_name_tag: %s" % utils.get_node_name_tag(region_number)) @@ -73,7 +73,7 @@ def request_spot_fleet(config, ec2_client, region_number, number_of_instances, i # https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-fleet.html#spot-fleet-allocation-strategy 'AllocationStrategy': 'diversified', 'IamFleetRole': 'arn:aws:iam::656503231766:role/RichardFleetRole', - 'LaunchSpecifications': create_launch_specification_list(config, region_number, instance_type_list), + 'LaunchSpecifications': create_launch_specification_list(region_number, instance_type_list), # 'SpotPrice': 'string', # The maximum price per unit hour that you are willing to pay for a Spot Instance. The default is the On-Demand price. 'TargetCapacity': number_of_instances, 'Type': 'maintain' @@ -82,7 +82,7 @@ def request_spot_fleet(config, ec2_client, region_number, number_of_instances, i return response -def request_spot_fleet_with_on_demand(config, ec2_client, region_number, number_of_instances, number_of_on_demand, instance_type_list): +def request_spot_fleet_with_on_demand(ec2_client, region_number, number_of_instances, number_of_on_demand, instance_type_list): LOGGER.info("Requesting spot fleet") LOGGER.info("Creating node_name_tag: %s" % utils.get_node_name_tag(region_number)) @@ -93,7 +93,7 @@ def request_spot_fleet_with_on_demand(config, ec2_client, region_number, number_ # https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-fleet.html#spot-fleet-allocation-strategy 'AllocationStrategy': 'diversified', 'IamFleetRole': 'arn:aws:iam::656503231766:role/RichardFleetRole', - 'LaunchTemplateConfigs': get_launch_template_list(config, region_number, instance_type_list), + 'LaunchTemplateConfigs': get_launch_template_list(region_number, instance_type_list), # 'SpotPrice': 'string', # The maximum price per unit hour that you are willing to pay for a Spot Instance. The default is the On-Demand price. 'TargetCapacity': number_of_instances, 'OnDemandTargetCapacity': number_of_on_demand, @@ -101,3 +101,18 @@ def request_spot_fleet_with_on_demand(config, ec2_client, region_number, number_ } ) return response + +def get_instance_ids(ec2_client, request_id): + res = ec2_client.describe_spot_fleet_instances( + SpotFleetRequestId=request_id + ) + return [ inst.InstanceId for inst in res.ActiveInstances ] + +def run_one_region(region_number, number_of_instances, fout, fout2): + client = utils.create_client(utils.CONFIG, region_number) + instance_type_list = ['t2.micro', 't2.small', 'm3.medium'] + # node_name_tag = request_spot_fleet_with_on_demand( + # client, region_number, int(number_of_instances), 1, instance_type_list) + node_name_tag = request_spot_fleet( + client, region_number, int(number_of_instances), instance_type_list) + return node_name_tag, client \ No newline at end of file diff --git a/aws-experiment-launch/utils/utils.py b/aws-experiment-launch/utils/utils.py index cfc44b6a3..91ae501e4 100644 --- a/aws-experiment-launch/utils/utils.py +++ b/aws-experiment-launch/utils/utils.py @@ -28,7 +28,15 @@ with open("user-data.sh", "r") as userdata_file: # UserData must be base64 encoded for spot instances. USER_DATA_BASE64 = base64.b64encode(USER_DATA) +def create_client(config, region_number): + region_name = config[region_number][REGION_NAME] + # Create session. + session = boto3.Session(region_name=region_name) + # Create a client. + return session.client('ec2') + def read_region_config(region_config='configuration.txt'): + global CONFIG config = {} with open(region_config, 'r') as f: for myline in f: @@ -41,6 +49,7 @@ def read_region_config(region_config='configuration.txt'): config[region_num][REGION_HUMAN_NAME] = mylist[4] config[region_num][REGION_AMI] = mylist[5] config[region_num][REGION_SECURITY_GROUP_ID] = mylist[6] + CONFIG = config return config def get_ip_list(response): From 8fe9368f25de07528c2acd6367afdf184fecb8f7 Mon Sep 17 00:00:00 2001 From: Richard Liu Date: Wed, 11 Jul 2018 22:40:12 -0700 Subject: [PATCH 3/4] read instance resource from args. --- aws-experiment-launch/create_instances.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/aws-experiment-launch/create_instances.py b/aws-experiment-launch/create_instances.py index 10c1bcf26..a2db7b824 100644 --- a/aws-experiment-launch/create_instances.py +++ b/aws-experiment-launch/create_instances.py @@ -117,11 +117,11 @@ if __name__ == "__main__": default=InstanceResource.ON_DEMAND, choices=list(InstanceResource)) parser.add_argument('--append', dest='append', type=bool, default=False, help='append to the current instance_output') - instanceResource = InstanceResource.SPOT_FLEET args = parser.parse_args() config = utils.read_region_config(args.region_config) region_list = args.regions.split(',') num_instance_list = args.num_instance_list.split(',') + instance_resource = args.instance_resource assert len(region_list) == len(num_instance_list), "number of regions: %d != number of instances per region: %d" % ( len(region_list), len(num_instance_list)) @@ -131,10 +131,10 @@ if __name__ == "__main__": for i in range(len(region_list)): region_number = region_list[i] number_of_instances = num_instance_list[i] - if instanceResource == InstanceResource.ON_DEMAND: + if instance_resource == InstanceResource.ON_DEMAND: t = threading.Thread(target=run_for_one_region_on_demand, args=( config, region_number, number_of_instances, fout, fout2)) - elif instanceResource == InstanceResource.SPOT_FLEET: + elif instance_resource == InstanceResource.SPOT_FLEET: t = threading.Thread(target=spot_fleet.run_one_region, args=( region_number, number_of_instances, fout, fout2)) LOGGER.info("creating thread for region %s" % region_number) From 7f48e2270aa6fe004b2ac75fb72b0d6729887b73 Mon Sep 17 00:00:00 2001 From: Richard Liu Date: Wed, 11 Jul 2018 22:44:07 -0700 Subject: [PATCH 4/4] let instance resource supports cli reading --- aws-experiment-launch/create_instances.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/aws-experiment-launch/create_instances.py b/aws-experiment-launch/create_instances.py index a2db7b824..0a4b2ec46 100644 --- a/aws-experiment-launch/create_instances.py +++ b/aws-experiment-launch/create_instances.py @@ -14,9 +14,12 @@ LOGGER = logger.getLogger(__file__) class InstanceResource(enum.Enum): - ON_DEMAND = 1 - SPOT_INSTANCE = 2 - SPOT_FLEET = 3 + ON_DEMAND = 'ondemand' + SPOT_INSTANCE = 'spot' + SPOT_FLEET = 'fleet' + + def __str__(self): + return self.value def run_one_region_on_demand_instances(config, region_number, number_of_instances): ec2_client = utils.create_client(config, region_number)