diff --git a/aws-experiment-launch/create_instances.py b/aws-experiment-launch/create_instances.py index 49bf1f52d..b97b2f1e1 100644 --- a/aws-experiment-launch/create_instances.py +++ b/aws-experiment-launch/create_instances.py @@ -11,11 +11,13 @@ from utils import utils, spot_fleet, logger LOGGER = logger.getLogger(__file__) + class InstanceResource: ON_DEMAND = 1 SPOT_INSTANCE = 2 SPOT_FLEET = 3 + def run_one_region_instances(config, region_number, number_of_instances, instance_resource=InstanceResource.ON_DEMAND): region_name = config[region_number][utils.REGION_NAME] # Create session. @@ -30,8 +32,10 @@ def run_one_region_instances(config, region_number, number_of_instances, instanc return node_name_tag, ec2_client elif instance_resource == InstanceResource.SPOT_FLEET: instance_type_list = ['t2.micro', 't2.small', 'm3.medium'] - node_name_tag = spot_fleet.request_spot_fleet( - config, ec2_client, region_number, int(number_of_instances), instance_type_list) + node_name_tag = spot_fleet.request_spot_fleet_with_on_demand( + config, ec2_client, region_number, int(number_of_instances), 1, instance_type_list) + # node_name_tag = spot_fleet.request_spot_fleet( + # config, ec2_client, region_number, int(number_of_instances), instance_type_list) return node_name_tag, ec2_client else: return None, None diff --git a/aws-experiment-launch/utils/launch_template.py b/aws-experiment-launch/utils/launch_template.py index 7939b41ca..1ca9e73da 100644 --- a/aws-experiment-launch/utils/launch_template.py +++ b/aws-experiment-launch/utils/launch_template.py @@ -2,34 +2,37 @@ import utils -def create(config, ec2_client, region_number, instance_type): +def get_launch_template_name(config, region_number): + return 'benchmark-' + config[region_number][utils.REGION_NAME] + + +def create(config, ec2_client, region_number): return ec2_client.create_launch_template( # DryRun=True, - LaunchTemplateName='richard-benchmark-' + \ - config[region_number][utils.REGION_NAME] + '-' + instance_type, + LaunchTemplateName=get_launch_template_name(config, region_number), LaunchTemplateData={ 'IamInstanceProfile': { 'Name': utils.IAM_INSTANCE_PROFILE }, 'ImageId': config[region_number][utils.REGION_AMI], - 'InstanceType': instance_type, + # 'InstanceType': instance_type, 'KeyName': config[region_number][utils.REGION_KEY], 'UserData': utils.USER_DATA_BASE64, 'SecurityGroupIds': [ config[region_number][utils.REGION_SECURITY_GROUP_ID] ], # 'InstanceInitiatedShutdownBehavior': 'stop', - # 'TagSpecifications': [ - # { - # 'ResourceType': 'instance', - # 'Tags': [ - # { - # 'Key': 'Name', - # 'Value': 'string' - # } - # ] - # } - # ], + 'TagSpecifications': [ + { + 'ResourceType': 'instance', + 'Tags': [ + { + 'Key': 'LaunchTemplate', + 'Value': 'Yes' + } + ] + } + ], # 'InstanceMarketOptions': { # 'MarketType': 'spot', # 'SpotOptions': { @@ -40,4 +43,4 @@ def create(config, ec2_client, region_number, instance_type): # } # }, } - ) + ) \ No newline at end of file diff --git a/aws-experiment-launch/utils/spot_fleet.py b/aws-experiment-launch/utils/spot_fleet.py index 12b1c929f..f98bb0937 100644 --- a/aws-experiment-launch/utils/spot_fleet.py +++ b/aws-experiment-launch/utils/spot_fleet.py @@ -1,7 +1,9 @@ import utils import logger +import launch_template LOGGER = logger.getLogger(__file__) + def create_launch_specification(config, region_number, instanceType): return { # Region irrelevant fields @@ -37,12 +39,33 @@ def create_launch_specification(config, region_number, instanceType): # } } + def create_launch_specification_list(config, region_number, instance_type_list): return list(map(lambda type: create_launch_specification(config, region_number, type), instance_type_list)) + +def get_launch_template(config, region_number, instance_type): + return { + 'LaunchTemplateSpecification': { + 'LaunchTemplateName': launch_template.get_launch_template_name(config, region_number), + 'Version': '1' + }, + 'Overrides': [ + { + 'InstanceType': instance_type + } + ] + } + + +def get_launch_template_list(config, region_number, instance_type_list): + return list(map(lambda type: get_launch_template(config, region_number, type), instance_type_list)) + + def request_spot_fleet(config, ec2_client, region_number, number_of_instances, instance_type_list): LOGGER.info("Requesting spot fleet") - LOGGER.info("Creating node_name_tag: %s" % utils.get_node_name_tag(region_number)) + LOGGER.info("Creating node_name_tag: %s" % + utils.get_node_name_tag(region_number)) # https://boto3.readthedocs.io/en/latest/reference/services/ec2.html#EC2.Client.request_spot_fleet response = ec2_client.request_spot_fleet( # DryRun=True, @@ -53,8 +76,28 @@ def request_spot_fleet(config, ec2_client, region_number, number_of_instances, i 'LaunchSpecifications': create_launch_specification_list(config, region_number, instance_type_list), # 'SpotPrice': 'string', # The maximum price per unit hour that you are willing to pay for a Spot Instance. The default is the On-Demand price. 'TargetCapacity': number_of_instances, - 'OnDemandTargetCapacity': 0, 'Type': 'maintain' } ) - return response \ No newline at end of file + return response + + +def request_spot_fleet_with_on_demand(config, ec2_client, region_number, number_of_instances, number_of_on_demand, instance_type_list): + LOGGER.info("Requesting spot fleet") + LOGGER.info("Creating node_name_tag: %s" % + utils.get_node_name_tag(region_number)) + # https://boto3.readthedocs.io/en/latest/reference/services/ec2.html#EC2.Client.request_spot_fleet + response = ec2_client.request_spot_fleet( + # DryRun=True, + SpotFleetRequestConfig={ + # https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-fleet.html#spot-fleet-allocation-strategy + 'AllocationStrategy': 'diversified', + 'IamFleetRole': 'arn:aws:iam::656503231766:role/RichardFleetRole', + 'LaunchTemplateConfigs': get_launch_template_list(config, region_number, instance_type_list), + # 'SpotPrice': 'string', # The maximum price per unit hour that you are willing to pay for a Spot Instance. The default is the On-Demand price. + 'TargetCapacity': number_of_instances, + 'OnDemandTargetCapacity': number_of_on_demand, + 'Type': 'maintain' + } + ) + return response