From ef1346c00c1a46b919e012fe528bbd637b3628e1 Mon Sep 17 00:00:00 2001 From: alok Date: Wed, 4 Jul 2018 15:53:49 -0700 Subject: [PATCH] working off node-name because new codedeploys deploy to old instances as well --- aws-experiment-launch/create_and_deploy.py | 210 +++++++++++---------- 1 file changed, 111 insertions(+), 99 deletions(-) diff --git a/aws-experiment-launch/create_and_deploy.py b/aws-experiment-launch/create_and_deploy.py index 530fbf3ab..03189ff42 100644 --- a/aws-experiment-launch/create_and_deploy.py +++ b/aws-experiment-launch/create_and_deploy.py @@ -4,6 +4,8 @@ import sys import json import time import datetime +from threading import Thread +from Queue import Queue REGION_NAME = 'region_name' REGION_KEY = 'region_key' @@ -18,7 +20,7 @@ APPLICATION_NAME = 'benchmark-experiments' time_stamp = time.time() CURRENT_SESSION = datetime.datetime.fromtimestamp(time_stamp).strftime('%H-%M-%S-%Y-%m-%d') PLACEMENT_GROUP = "PLACEMENT-" + CURRENT_SESSION -NODE_VALUE = "NODE-" + CURRENT_SESSION +NODE_NAME_SUFFIX = "NODE-" + CURRENT_SESSION """ TODO: @@ -32,12 +34,8 @@ Build (argparse,functions) support for 3. run create instance followed by codedeploy """ -def get_instance_ids(response): - instance_ids = [] - for reservation in response["Reservations"]: - for instance in reservation["Instances"]: - instance_ids.append(instance["InstanceId"]) - return instance_ids + +### CREATE INSTANCES ### def run_one_region_instances(config,region_number,number_of_instances): #todo: explore the use ec2 resource and not client. e.g. create_instances -- Might make for better code. @@ -47,11 +45,60 @@ def run_one_region_instances(config,region_number,number_of_instances): region_name = config[region_number][REGION_NAME] session = boto3.Session(region_name=region_name) ec2_client = session.client('ec2') - response,placement = create_instances(config,ec2_client,region_number,int(number_of_instances)) - print(placement) - return session,placement + response = create_instances(config,ec2_client,region_number,int(number_of_instances)) + return session + +def create_instances(config,ec2_client,region_number,number_of_instances): + NODE_NAME = region_number + "-" + NODE_NAME_SUFFIX + response = ec2_client.run_instances( + MinCount = number_of_instances, + MaxCount = number_of_instances, + ImageId = config[region_number][REGION_AMI], + Placement = { + 'AvailabilityZone': get_one_availability_zone(ec2_client), + }, + SecurityGroups = [config[region_number][REGION_SECURITY_GROUP]], + IamInstanceProfile = { + 'Name' : IAM_INSTANCE_PROFILE + }, + KeyName = config[region_number][REGION_KEY], + UserData = USER_DATA, + InstanceType = INSTANCE_TYPE, + TagSpecifications = [ + { + 'ResourceType' : 'instance', + 'Tags': [ + { + 'Key': 'Name', + 'Value': NODE_NAME + }, + ] + }, + ] + ) + return response -def run_one_region_codedeploy(region_number,placement_group,commitId): +def get_availability_zones(ec2_client): + response = ec2_client.describe_availability_zones() + all_zones = [] + if response.get('AvailabilityZones',None): + region_info = response.get('AvailabilityZones') + for info in region_info: + if info['State'] == 'available': + all_zones.append(info['ZoneName']) + return all_zones + +def get_one_availability_zone(ec2_client): + all_zones = get_availability_zones(ec2_client) + if len(all_zones) > 0: + return all_zones[0] + else: + print("No availability zone for this region") + sys.exit() + +#### CODEDEPLOY ### + +def run_one_region_codedeploy(region_number,commitId): #todo: explore the use ec2 resource and not client. e.g. create_instances -- Might make for better code. """ for getting instance ids:--- @@ -65,31 +112,27 @@ def run_one_region_codedeploy(region_number,placement_group,commitId): instance """ region_name = config[region_number][REGION_NAME] + NODE_NAME = region_number + "-" + NODE_NAME_SUFFIX session = boto3.Session(region_name=region_name) ec2_client = session.client('ec2') - response = ec2_client.describe_instances( - Filters = [ + filters = [ { - 'Name': 'placement-group-name', - 'Values' : [ - placement_group - ] + 'Name': 'tag:Name', + 'Values' : [NODE_NAME] } ] - ) - instance_ids = get_instance_ids(response) print("Waiting for all instances to start running") waiter = ec2_client.get_waiter('instance_running') - waiter.wait(InstanceIds=instance_ids) + waiter.wait(Filters=filters) print("Waiting for all instances to be status ok") waiter = ec2_client.get_waiter('instance_status_ok') - waiter.wait(InstanceIds=instance_ids) + waiter.wait(Filters=filters) print("Waiting for system to be status ok") waiter = ec2_client.get_waiter('system_status_ok') - waiter.wait(InstanceIds=instance_ids) + waiter.wait(Filters=filters) codedeploy = session.client('codedeploy') application_name = APPLICATION_NAME @@ -97,63 +140,11 @@ def run_one_region_codedeploy(region_number,placement_group,commitId): repo = REPO response = get_application(codedeploy,application_name) response = get_deployment_group(codedeploy,application_name,deployment_group) - deploy(codedeploy, application_name, deployment_group, repo, commitId, wait=True) - return response + depId = deploy(codedeploy, application_name, deployment_group, repo, commitId) + return region_number,depId -def get_availability_zones(ec2_client): - response = ec2_client.describe_availability_zones() - all_zones = [] - if response.get('AvailabilityZones',None): - region_info = response.get('AvailabilityZones') - for info in region_info: - if info['State'] == 'available': - all_zones.append(info['ZoneName']) - return all_zones - -def get_one_availability_zone(ec2_client): - all_zones = get_availability_zones(ec2_client) - if len(all_zones) > 0: - return all_zones[0] - else: - print("No availability zone for this region") - sys.exit() - -def create_instances(config,ec2_client,region_number,number_of_instances): - placement_group = region_number + "-" + PLACEMENT_GROUP - response = ec2_client.create_placement_group( - GroupName = placement_group, - Strategy = 'spread' - ) - response = ec2_client.run_instances( - MinCount = number_of_instances, - MaxCount = number_of_instances, - ImageId = config[region_number][REGION_AMI], - Placement = { - 'AvailabilityZone': get_one_availability_zone(ec2_client), - 'GroupName': placement_group - }, - SecurityGroups = [config[region_number][REGION_SECURITY_GROUP]], - IamInstanceProfile = { - 'Name' : IAM_INSTANCE_PROFILE - }, - KeyName = config[region_number][REGION_KEY], - UserData = USER_DATA, - InstanceType = INSTANCE_TYPE, - TagSpecifications = [ - { - 'ResourceType' : 'instance', - 'Tags': [ - { - 'Key': 'Name', - 'Value': 'Node' - }, - ] - }, - ] - ) - return response,placement_group - def get_deployment_group(codedeploy,application_name,deployment_group): + NODE_NAME = region_number + "-" + NODE_NAME_SUFFIX response = codedeploy.list_deployment_groups( applicationName = application_name ) @@ -174,7 +165,7 @@ def get_deployment_group(codedeploy,application_name,deployment_group): [ { 'Key': 'Name', - 'Value': 'Node', + 'Value': NODE_NAME, 'Type': 'KEY_AND_VALUE' }, ], @@ -183,13 +174,6 @@ def get_deployment_group(codedeploy,application_name,deployment_group): ) return response -def get_commitId(commitId): - if commitId is None: - commitId = run("git rev-list --max-count=1 HEAD", - hide=True).stdout.strip() - print("Got newest commitId as " + commitId) - return commitId - def get_application(codedeploy,application_name): response = codedeploy.list_applications() if application_name in response['applications']: @@ -201,7 +185,7 @@ def get_application(codedeploy,application_name): ) return response -def deploy(codedeploy, application_name,deployment_group,repo, commitId, wait=True): +def deploy(codedeploy, application_name,deployment_group,repo, commitId): """Deploy new code at specified revision to instance. arguments: @@ -225,11 +209,8 @@ def deploy(codedeploy, application_name,deployment_group,repo, commitId, wait=Tr ) depId = res["deploymentId"] print("Deployment ID: " + depId) - # The deployment is launched at this point, so exit unless asked to wait # until it finishes - if not wait: - return info = {'status': 'Created'} start = time.time() while info['status'] not in ('Succeeded', 'Failed', 'Stopped',) and (time.time() - start < 300.0): @@ -238,9 +219,37 @@ def deploy(codedeploy, application_name,deployment_group,repo, commitId, wait=Tr time.sleep(15) if info['status'] == 'Succeeded': print("\nDeploy Succeeded") + return depId else: print("\nDeploy Failed") print(info) + return depId + +def run_one_region_codedeploy_wrapper(region_number,commitId,queue): + region_number,depId = run_one_region_codedeploy(region_number,commitId) + queue.put((region_number,depId)) + +def launch_code_deploy(region_list,commitId): + queue = Queue() + jobs = [] + for i in range(len(region_list)): + region_number = region_list[i] + my_thread = Thread(target=run_one_region_codedeploy_wrapper, args=(region_number,commitId,queue)) + my_thread.start() + jobs.append(my_thread) + for my_thread in jobs: + my_thread.join() + results = [queue.get() for job in jobs] + return results + +##### UTILS #### + +def get_instance_ids(describe_instances_response): + instance_ids = [] + for reservation in describe_instances_response["Reservations"]: + for instance in reservation["Instances"]: + instance_ids.append(instance["InstanceId"]) + return instance_ids def read_configuration_file(filename): config = {} @@ -256,6 +265,15 @@ def read_configuration_file(filename): config[region_num][REGION_AMI] = mylist[5] return config +def get_commitId(commitId): + if commitId is None: + commitId = run("git rev-list --max-count=1 HEAD", + hide=True).stdout.strip() + print("Got newest commitId as " + commitId) + return commitId + +##### UTILS #### + if __name__ == "__main__": parser = argparse.ArgumentParser(description='This script helps you start instances across multiple regions') parser.add_argument('--regions',type=str,dest='regions',default='3',help="Supply a csv list of all regions") @@ -268,17 +286,11 @@ if __name__ == "__main__": instances_list = args.numInstances.split(',') assert len(region_list) == len(instances_list),"number of regions: %d != number of instances per region: %d" % (len(region_list),len(intances_list)) commitId = args.commitId - placement_groups = [] - for i in range(len(region_list)): region_number = region_list[i] number_of_instances = instances_list[i] - session,placement_group = run_one_region_instances(config,region_number,number_of_instances) - placement_groups.append(placement_group) - - for i in range(len(region_list)): - region_number = region_list[i] - placement_group = placement_groups[i] - run_one_region_codedeploy(region_number,placement_group,commitId) - + session = run_one_region_instances(config,region_number,number_of_instances) + results = launch_code_deploy(region_list,commitId) + print(results) + \ No newline at end of file