working off node-name because new codedeploys deploy to old instances as well

aws-experiment-launch/create_and_deploy.py

@@ -4,6 +4,8 @@ import sys
 import json
 import time
 import datetime
+from threading import Thread
+from Queue import Queue
 
 REGION_NAME = 'region_name'
 REGION_KEY = 'region_key'
@@ -18,7 +20,7 @@ APPLICATION_NAME = 'benchmark-experiments'
 time_stamp = time.time()
 CURRENT_SESSION = datetime.datetime.fromtimestamp(time_stamp).strftime('%H-%M-%S-%Y-%m-%d')
 PLACEMENT_GROUP = "PLACEMENT-" + CURRENT_SESSION
-NODE_VALUE = "NODE-" + CURRENT_SESSION
+NODE_NAME_SUFFIX = "NODE-" + CURRENT_SESSION
 
 """
 TODO:
@@ -32,12 +34,8 @@ Build (argparse,functions) support for
 3. run create instance followed by codedeploy
 
 """
-def get_instance_ids(response):
-    instance_ids = []
-    for reservation in response["Reservations"]:
-        for instance in reservation["Instances"]:
-            instance_ids.append(instance["InstanceId"])
-    return instance_ids   
+
+### CREATE INSTANCES ###
 
 def run_one_region_instances(config,region_number,number_of_instances):
     #todo: explore the use ec2 resource and not client. e.g. create_instances --  Might make for better code.
@@ -47,11 +45,60 @@ def run_one_region_instances(config,region_number,number_of_instances):
     region_name = config[region_number][REGION_NAME]
     session = boto3.Session(region_name=region_name)
     ec2_client = session.client('ec2')
-    response,placement = create_instances(config,ec2_client,region_number,int(number_of_instances))
-    print(placement)
-    return session,placement
+    response = create_instances(config,ec2_client,region_number,int(number_of_instances))
+    return session
+
+def create_instances(config,ec2_client,region_number,number_of_instances):
+    NODE_NAME = region_number + "-" + NODE_NAME_SUFFIX
+    response = ec2_client.run_instances(
+        MinCount = number_of_instances,
+        MaxCount = number_of_instances,
+        ImageId = config[region_number][REGION_AMI],
+        Placement = {
+        'AvailabilityZone': get_one_availability_zone(ec2_client),
+        },
+        SecurityGroups = [config[region_number][REGION_SECURITY_GROUP]],
+        IamInstanceProfile = {
+            'Name' : IAM_INSTANCE_PROFILE
+        },
+        KeyName = config[region_number][REGION_KEY],
+        UserData = USER_DATA,
+        InstanceType = INSTANCE_TYPE,
+        TagSpecifications = [
+          {
+            'ResourceType' : 'instance',
+            'Tags': [
+                {
+                    'Key': 'Name',
+                    'Value': NODE_NAME
+                },
+            ]
+          },
+        ]
+    )
+    return response
 
-def run_one_region_codedeploy(region_number,placement_group,commitId):
+def get_availability_zones(ec2_client):
+    response = ec2_client.describe_availability_zones()
+    all_zones = []
+    if response.get('AvailabilityZones',None):
+       region_info = response.get('AvailabilityZones')
+       for info in region_info:
+           if info['State'] == 'available':
+               all_zones.append(info['ZoneName'])
+    return all_zones
+
+def get_one_availability_zone(ec2_client):
+    all_zones = get_availability_zones(ec2_client)
+    if len(all_zones) > 0:
+        return all_zones[0]
+    else:
+        print("No availability zone for this region")
+        sys.exit() 
+
+#### CODEDEPLOY ###
+
+def run_one_region_codedeploy(region_number,commitId):
     #todo: explore the use ec2 resource and not client. e.g. create_instances --  Might make for better code.
     """
     for getting instance ids:---
@@ -65,31 +112,27 @@ def run_one_region_codedeploy(region_number,placement_group,commitId):
         instance
     """
     region_name = config[region_number][REGION_NAME]
+    NODE_NAME = region_number + "-" + NODE_NAME_SUFFIX
     session = boto3.Session(region_name=region_name)
     ec2_client = session.client('ec2')
-    response = ec2_client.describe_instances(
-        Filters = [
+    filters =  [
             {
-                'Name': 'placement-group-name',
-                'Values' : [
-                        placement_group
-                ]
+                'Name': 'tag:Name',
+                'Values' : [NODE_NAME]
             }
         ]
-    )
-    instance_ids = get_instance_ids(response)
     
     print("Waiting for all instances to start running")
     waiter = ec2_client.get_waiter('instance_running')
-    waiter.wait(InstanceIds=instance_ids)
+    waiter.wait(Filters=filters)
 
     print("Waiting for all instances to be status ok")
     waiter = ec2_client.get_waiter('instance_status_ok')
-    waiter.wait(InstanceIds=instance_ids)
+    waiter.wait(Filters=filters)
 
     print("Waiting for system to be status ok")
     waiter = ec2_client.get_waiter('system_status_ok')
-    waiter.wait(InstanceIds=instance_ids)
+    waiter.wait(Filters=filters)
 
     codedeploy = session.client('codedeploy')
     application_name = APPLICATION_NAME
@@ -97,63 +140,11 @@ def run_one_region_codedeploy(region_number,placement_group,commitId):
     repo = REPO
     response = get_application(codedeploy,application_name)
     response  = get_deployment_group(codedeploy,application_name,deployment_group)
-    deploy(codedeploy, application_name, deployment_group, repo, commitId, wait=True)
-    return response
+    depId = deploy(codedeploy, application_name, deployment_group, repo, commitId)
+    return region_number,depId
     
-def get_availability_zones(ec2_client):
-    response = ec2_client.describe_availability_zones()
-    all_zones = []
-    if response.get('AvailabilityZones',None):
-       region_info = response.get('AvailabilityZones')
-       for info in region_info:
-           if info['State'] == 'available':
-               all_zones.append(info['ZoneName'])
-    return all_zones
-
-def get_one_availability_zone(ec2_client):
-    all_zones = get_availability_zones(ec2_client)
-    if len(all_zones) > 0:
-        return all_zones[0]
-    else:
-        print("No availability zone for this region")
-        sys.exit()
-
-def create_instances(config,ec2_client,region_number,number_of_instances):
-    placement_group = region_number + "-" + PLACEMENT_GROUP
-    response = ec2_client.create_placement_group(
-    GroupName = placement_group,
-    Strategy = 'spread'
-    ) 
-    response = ec2_client.run_instances(
-        MinCount = number_of_instances,
-        MaxCount = number_of_instances,
-        ImageId = config[region_number][REGION_AMI],
-        Placement = {
-        'AvailabilityZone': get_one_availability_zone(ec2_client),
-        'GroupName': placement_group
-        },
-        SecurityGroups = [config[region_number][REGION_SECURITY_GROUP]],
-        IamInstanceProfile = {
-            'Name' : IAM_INSTANCE_PROFILE
-        },
-        KeyName = config[region_number][REGION_KEY],
-        UserData = USER_DATA,
-        InstanceType = INSTANCE_TYPE,
-        TagSpecifications = [
-          {
-            'ResourceType' : 'instance',
-            'Tags': [
-                {
-                    'Key': 'Name',
-                    'Value': 'Node'
-                },
-            ]
-          },
-        ]
-    )
-    return response,placement_group
-
 def get_deployment_group(codedeploy,application_name,deployment_group):
+    NODE_NAME = region_number + "-" + NODE_NAME_SUFFIX
     response = codedeploy.list_deployment_groups(
         applicationName = application_name
     )
@@ -174,7 +165,7 @@ def get_deployment_group(codedeploy,application_name,deployment_group):
                         [
                         {
                             'Key': 'Name',
-                            'Value': 'Node',
+                            'Value': NODE_NAME,
                             'Type': 'KEY_AND_VALUE'
                         },
                         ],
@@ -183,13 +174,6 @@ def get_deployment_group(codedeploy,application_name,deployment_group):
         )
         return response
 
-def get_commitId(commitId):
-    if commitId is None:
-        commitId = run("git rev-list --max-count=1 HEAD",
-                       hide=True).stdout.strip()
-        print("Got newest commitId as " + commitId)
-    return commitId
-
 def get_application(codedeploy,application_name):
     response = codedeploy.list_applications()
     if application_name in response['applications']:
@@ -201,7 +185,7 @@ def get_application(codedeploy,application_name):
     )
     return response
 
-def deploy(codedeploy, application_name,deployment_group,repo, commitId, wait=True):
+def deploy(codedeploy, application_name,deployment_group,repo, commitId):
     """Deploy new code at specified revision to instance.
 
     arguments:
@@ -225,11 +209,8 @@ def deploy(codedeploy, application_name,deployment_group,repo, commitId, wait=Tr
         )
     depId = res["deploymentId"]
     print("Deployment ID: " + depId)
-
     # The deployment is launched at this point, so exit unless asked to wait
     # until it finishes
-    if not wait:
-        return
     info = {'status': 'Created'}
     start = time.time()
     while info['status'] not in ('Succeeded', 'Failed', 'Stopped',) and (time.time() - start < 300.0):
@@ -238,9 +219,37 @@ def deploy(codedeploy, application_name,deployment_group,repo, commitId, wait=Tr
         time.sleep(15)
     if info['status'] == 'Succeeded':
         print("\nDeploy Succeeded")
+        return depId
     else:
         print("\nDeploy Failed")
         print(info)
+        return depId
+
+def run_one_region_codedeploy_wrapper(region_number,commitId,queue):
+    region_number,depId = run_one_region_codedeploy(region_number,commitId)
+    queue.put((region_number,depId))
+
+def launch_code_deploy(region_list,commitId):
+    queue = Queue()
+    jobs = []
+    for i in range(len(region_list)):
+        region_number = region_list[i]
+        my_thread = Thread(target=run_one_region_codedeploy_wrapper, args=(region_number,commitId,queue))
+        my_thread.start()
+        jobs.append(my_thread)
+    for my_thread in jobs:
+        my_thread.join()
+    results = [queue.get() for job in jobs]
+    return results
+
+##### UTILS ####
+
+def get_instance_ids(describe_instances_response):
+    instance_ids = []
+    for reservation in describe_instances_response["Reservations"]:
+        for instance in reservation["Instances"]:
+            instance_ids.append(instance["InstanceId"])
+    return instance_ids  
 
 def read_configuration_file(filename):
     config = {}
@@ -256,6 +265,15 @@ def read_configuration_file(filename):
             config[region_num][REGION_AMI] = mylist[5]
     return config
 
+def get_commitId(commitId):
+    if commitId is None:
+        commitId = run("git rev-list --max-count=1 HEAD",
+                       hide=True).stdout.strip()
+        print("Got newest commitId as " + commitId)
+    return commitId
+
+##### UTILS ####
+
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description='This script helps you start instances across multiple regions')
     parser.add_argument('--regions',type=str,dest='regions',default='3',help="Supply a csv list of all regions")
@@ -268,17 +286,11 @@ if __name__ == "__main__":
     instances_list = args.numInstances.split(',')
     assert len(region_list) == len(instances_list),"number of regions: %d != number of instances per region: %d" % (len(region_list),len(intances_list))
     commitId = args.commitId
-    placement_groups = []
-
     for i in range(len(region_list)):
         region_number = region_list[i]
         number_of_instances = instances_list[i]
-        session,placement_group = run_one_region_instances(config,region_number,number_of_instances)
-        placement_groups.append(placement_group)
-        
-    for i in range(len(region_list)):
-        region_number = region_list[i]
-        placement_group = placement_groups[i]
-        run_one_region_codedeploy(region_number,placement_group,commitId)
-
+        session = run_one_region_instances(config,region_number,number_of_instances)
+    results = launch_code_deploy(region_list,commitId)
+    print(results)
+