Merge pull request #27 from simple-rules/ricl-aws

added request_spots

aws-experiment-launch/create_and_deploy.py

@@ -4,6 +4,7 @@ import sys
 import json
 import time
 import datetime
+import base64
 
 REGION_NAME = 'region_name'
 REGION_KEY = 'region_key'
@@ -11,12 +12,17 @@ REGION_SECURITY_GROUP = 'region_security_group'
 REGION_HUMAN_NAME = 'region_human_name'
 INSTANCE_TYPE = 't2.micro'
 REGION_AMI = 'region_ami'
-USER_DATA = 'user-data.sh'
+# USER_DATA = 'user-data.sh'
+# UserData must be base64 encoded.
+with open("user-data.sh", "rb") as userdata_file:
+    USER_DATA = base64.b64encode(userdata_file.read())
+
 IAM_INSTANCE_PROFILE = 'BenchMarkCodeDeployInstanceProfile'
 REPO = "simple-rules/harmony-benchmark"
 APPLICATION_NAME = 'benchmark-experiments'
 time_stamp = time.time()
-CURRENT_SESSION = datetime.datetime.fromtimestamp(time_stamp).strftime('%H-%M-%S-%Y-%m-%d')
+CURRENT_SESSION = datetime.datetime.fromtimestamp(
+    time_stamp).strftime('%H-%M-%S-%Y-%m-%d')
 PLACEMENT_GROUP = "PLACEMENT-" + CURRENT_SESSION
 NODE_VALUE = "NODE-" + CURRENT_SESSION
 
@@ -32,6 +38,8 @@ Build (argparse,functions) support for
 3. run create instance followed by codedeploy
 
 """
+
+
 def get_instance_ids(response):
     instance_ids = []
     for reservation in response["Reservations"]:
@@ -39,7 +47,8 @@ def get_instance_ids(response):
             instance_ids.append(instance["InstanceId"])
     return instance_ids
 
-def run_one_region_instances(config,region_number,number_of_instances):
+
+def run_one_region_instances(config, region_number, number_of_instances):
     #todo: explore the use ec2 resource and not client. e.g. create_instances --  Might make for better code.
     """
     e.g. ec2.create_instances
@@ -47,11 +56,25 @@ def run_one_region_instances(config,region_number,number_of_instances):
     region_name = config[region_number][REGION_NAME]
     session = boto3.Session(region_name=region_name)
     ec2_client = session.client('ec2')
-    response,placement = create_instances(config,ec2_client,region_number,int(number_of_instances))
+    # response, placement = create_instances(
+    #     config, ec2_client, region_number, int(number_of_instances))
+    response, placement = request_spots(
+        config, ec2_client, region_number, int(number_of_instances))
+    print(placement)
+    return session, placement
+
+
+def run_one_region_spots(config, region_number, number_of_instances):
+    region_name = config[region_number][REGION_NAME]
+    session = boto3.Session(region_name=region_name)
+    ec2_client = session.client('ec2')
+    response, placement = create_instances(
+        config, ec2_client, region_number, int(number_of_instances))
     print(placement)
-    return session,placement
+    return session, placement
 
-def run_one_region_codedeploy(region_number,placement_group,commitId):
+
+def run_one_region_codedeploy(region_number, placement_group, commitId):
     #todo: explore the use ec2 resource and not client. e.g. create_instances --  Might make for better code.
     """
     for getting instance ids:---
@@ -68,10 +91,10 @@ def run_one_region_codedeploy(region_number,placement_group,commitId):
     session = boto3.Session(region_name=region_name)
     ec2_client = session.client('ec2')
     response = ec2_client.describe_instances(
-        Filters = [
+        Filters=[
             {
                 'Name': 'placement-group-name',
-                'Values' : [
+                'Values': [
                         placement_group
                 ]
             }
@@ -95,21 +118,25 @@ def run_one_region_codedeploy(region_number,placement_group,commitId):
     application_name = APPLICATION_NAME
     deployment_group = APPLICATION_NAME + "-" + str(commitId)
     repo = REPO
-    response = get_application(codedeploy,application_name)
+    response = get_application(codedeploy, application_name)
-    response  = get_deployment_group(codedeploy,application_name,deployment_group)
+    response = get_deployment_group(
-    deploy(codedeploy, application_name, deployment_group, repo, commitId, wait=True)
+        codedeploy, application_name, deployment_group)
+    deploy(codedeploy, application_name,
+           deployment_group, repo, commitId, wait=True)
     return response
 
+
 def get_availability_zones(ec2_client):
     response = ec2_client.describe_availability_zones()
     all_zones = []
-    if response.get('AvailabilityZones',None):
+    if response.get('AvailabilityZones', None):
        region_info = response.get('AvailabilityZones')
        for info in region_info:
            if info['State'] == 'available':
                all_zones.append(info['ZoneName'])
     return all_zones
 
+
 def get_one_availability_zone(ec2_client):
     all_zones = get_availability_zones(ec2_client)
     if len(all_zones) > 0:
@@ -118,71 +145,108 @@ def get_one_availability_zone(ec2_client):
         print("No availability zone for this region")
         sys.exit()
 
-def create_instances(config,ec2_client,region_number,number_of_instances):
+
+def create_instances(config, ec2_client, region_number, number_of_instances):
     placement_group = region_number + "-" + PLACEMENT_GROUP
     response = ec2_client.create_placement_group(
-    GroupName = placement_group,
+        GroupName=placement_group,
-    Strategy = 'spread'
+        Strategy='spread'
     )
     response = ec2_client.run_instances(
-        MinCount = number_of_instances,
+        MinCount=number_of_instances,
-        MaxCount = number_of_instances,
+        MaxCount=number_of_instances,
-        ImageId = config[region_number][REGION_AMI],
+        ImageId=config[region_number][REGION_AMI],
-        Placement = {
+        Placement={
-        'AvailabilityZone': get_one_availability_zone(ec2_client),
+            'AvailabilityZone': get_one_availability_zone(ec2_client),
-        'GroupName': placement_group
+            'GroupName': placement_group
         },
-        SecurityGroups = [config[region_number][REGION_SECURITY_GROUP]],
+        SecurityGroups=[config[region_number][REGION_SECURITY_GROUP]],
-        IamInstanceProfile = {
+        IamInstanceProfile={
-            'Name' : IAM_INSTANCE_PROFILE
+            'Name': IAM_INSTANCE_PROFILE
         },
-        KeyName = config[region_number][REGION_KEY],
+        KeyName=config[region_number][REGION_KEY],
-        UserData = USER_DATA,
+        UserData=USER_DATA,
-        InstanceType = INSTANCE_TYPE,
+        InstanceType=INSTANCE_TYPE,
-        TagSpecifications = [
+        TagSpecifications=[
-          {
+            {
-            'ResourceType' : 'instance',
+                'ResourceType': 'instance',
-            'Tags': [
+                'Tags': [
-                {
+                    {
-                    'Key': 'Name',
+                        'Key': 'Name',
-                    'Value': 'Node'
+                        'Value': 'Node'
-                },
+                    },
-            ]
+                ]
-          },
+            },
-        ]
+        ],
+        # InstanceMarketOptions={
+        #     'MarketType': 'spot',
+        #     'SpotOptions': {
+        #         'BlockDurationMinutes': 60,
+        #     }
+        # }
+    )
+    return response, placement_group
+
+
+def request_spots(config, ec2_client, region_number, number_of_instances):
+    placement_group = region_number + "-" + PLACEMENT_GROUP
+    response = ec2_client.create_placement_group(
+        GroupName=placement_group,
+        Strategy='spread'
+    )
+    response = ec2_client.request_spot_instances(
+        # DryRun=True,
+        BlockDurationMinutes=60,
+        InstanceCount=number_of_instances,
+        LaunchSpecification={
+            'SecurityGroups': [config[region_number][REGION_SECURITY_GROUP]],
+            'IamInstanceProfile': {
+                'Name': IAM_INSTANCE_PROFILE
+            },
+            'UserData': USER_DATA,
+            'ImageId': config[region_number][REGION_AMI],
+            'InstanceType': INSTANCE_TYPE,
+            'KeyName': config[region_number][REGION_KEY],
+            'Placement': {
+                'AvailabilityZone': get_one_availability_zone(ec2_client),
+                'GroupName': placement_group
+            }
+        }
     )
-    return response,placement_group
+    return response, placement_group
 
-def get_deployment_group(codedeploy,application_name,deployment_group):
+
+def get_deployment_group(codedeploy, application_name, deployment_group):
     response = codedeploy.list_deployment_groups(
-        applicationName = application_name
+        applicationName=application_name
     )
     if deployment_group in response['deploymentGroups']:
         return response
     else:
         response = codedeploy.create_deployment_group(
-                applicationName = application_name,
+            applicationName=application_name,
-                deploymentGroupName = deployment_group,
+            deploymentGroupName=deployment_group,
-                deploymentConfigName = 'CodeDeployDefault.AllAtOnce',
+            deploymentConfigName='CodeDeployDefault.AllAtOnce',
-                serviceRoleArn = 'arn:aws:iam::656503231766:role/BenchMarkCodeDeployServiceRole',
+            serviceRoleArn='arn:aws:iam::656503231766:role/BenchMarkCodeDeployServiceRole',
-                deploymentStyle={
+            deploymentStyle={
-                    'deploymentType': 'IN_PLACE',
+                'deploymentType': 'IN_PLACE',
-                    'deploymentOption': 'WITHOUT_TRAFFIC_CONTROL'
+                'deploymentOption': 'WITHOUT_TRAFFIC_CONTROL'
-                },
+            },
-                ec2TagSet={
+            ec2TagSet={
-                    'ec2TagSetList': [
+                'ec2TagSetList': [
-                        [
+                    [
                         {
                             'Key': 'Name',
                             'Value': 'Node',
                             'Type': 'KEY_AND_VALUE'
                         },
-                        ],
+                    ],
-                    ]
+                ]
-                }
+            }
         )
         return response
 
+
 def get_commitId(commitId):
     if commitId is None:
         commitId = run("git rev-list --max-count=1 HEAD",
@@ -190,18 +254,20 @@ def get_commitId(commitId):
         print("Got newest commitId as " + commitId)
     return commitId
 
-def get_application(codedeploy,application_name):
+
+def get_application(codedeploy, application_name):
     response = codedeploy.list_applications()
     if application_name in response['applications']:
         return response
     else:
         response = codedeploy.create_application(
-        applicationName= application_name,
+            applicationName=application_name,
-        computePlatform='Server'
+            computePlatform='Server'
-    )
+        )
     return response
 
-def deploy(codedeploy, application_name,deployment_group,repo, commitId, wait=True):
+
+def deploy(codedeploy, application_name, deployment_group, repo, commitId, wait=True):
     """Deploy new code at specified revision to instance.
 
     arguments:
@@ -211,18 +277,18 @@ def deploy(codedeploy, application_name,deployment_group,repo, commitId, wait=Tr
     """
     print("Launching CodeDeploy with commit " + commitId)
     res = codedeploy.create_deployment(
-            applicationName = application_name,
+        applicationName=application_name,
-            deploymentGroupName = deployment_group,
+        deploymentGroupName=deployment_group,
-            deploymentConfigName = 'CodeDeployDefault.AllAtOnce',
+        deploymentConfigName='CodeDeployDefault.AllAtOnce',
-            description = 'benchmark experiments',
+        description='benchmark experiments',
-            revision = {
+        revision={
-                'revisionType': 'GitHub',
+            'revisionType': 'GitHub',
-                'gitHubLocation': {
+            'gitHubLocation': {
-                'repository': repo,
+                    'repository': repo,
-                'commitId': commitId,
+                    'commitId': commitId,
-                }
             }
-        )
+        }
+    )
     depId = res["deploymentId"]
     print("Deployment ID: " + depId)
 
@@ -242,9 +308,10 @@ def deploy(codedeploy, application_name,deployment_group,repo, commitId, wait=Tr
         print("\nDeploy Failed")
         print(info)
 
+
 def read_configuration_file(filename):
     config = {}
-    with open(filename,'r') as f:
+    with open(filename, 'r') as f:
         for myline in f:
             mylist = myline.strip().split(',')
             region_num = mylist[0]
@@ -256,29 +323,35 @@ def read_configuration_file(filename):
             config[region_num][REGION_AMI] = mylist[5]
     return config
 
+
 if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description='This script helps you start instances across multiple regions')
+    parser = argparse.ArgumentParser(
-    parser.add_argument('--regions',type=str,dest='regions',default='3',help="Supply a csv list of all regions")
+        description='This script helps you start instances across multiple regions')
-    parser.add_argument('--instances', type=str,dest='numInstances',default='1',help='number of instances')
+    parser.add_argument('--regions', type=str, dest='regions',
-    parser.add_argument('--configuration',type=str,dest='config',default='configuration.txt')
+                        default='3', help="Supply a csv list of all regions")
-    parser.add_argument('--commitId',type=str,dest='commitId',default='1f7e6e7ca7cf1c1190cedec10e791c01a29971cf')
+    parser.add_argument('--instances', type=str, dest='numInstances',
+                        default='1', help='number of instances')
+    parser.add_argument('--configuration', type=str,
+                        dest='config', default='configuration.txt')
+    parser.add_argument('--commitId', type=str, dest='commitId',
+                        default='1f7e6e7ca7cf1c1190cedec10e791c01a29971cf')
     args = parser.parse_args()
     config = read_configuration_file(args.config)
     region_list = args.regions.split(',')
     instances_list = args.numInstances.split(',')
-    assert len(region_list) == len(instances_list),"number of regions: %d != number of instances per region: %d" % (len(region_list),len(intances_list))
+    assert len(region_list) == len(instances_list), "number of regions: %d != number of instances per region: %d" % (
+        len(region_list), len(intances_list))
     commitId = args.commitId
     placement_groups = []
 
     for i in range(len(region_list)):
         region_number = region_list[i]
         number_of_instances = instances_list[i]
-        session,placement_group = run_one_region_instances(config,region_number,number_of_instances)
+        session, placement_group = run_one_region_instances(
+            config, region_number, number_of_instances)
         placement_groups.append(placement_group)
 
     for i in range(len(region_list)):
         region_number = region_list[i]
         placement_group = placement_groups[i]
         run_one_region_codedeploy(region_number,placement_group,commitId)
-
-