PoS_EVM
/
woop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/simple-rules/harmony-benchmark

Browse Source
pull/37/head
Richard Liu 6 years ago
parent 1ea2bef61a d073e10d15
commit d7a15cc9d2
2 changed files with 86 additions and 101 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 143
      aws-experiment-launch/deploy.py
  2. 44
      aws-experiment-launch/utils/utils.py

143
aws-experiment-launch/deploy.py
Unescape View File

@ -1,12 +1,14 @@
import boto3
import argparse import argparse
import sys import base64
import boto3
import datetime
import json import json
import os
import sys
import time import time
import datetime
from threading import Thread
from Queue import Queue from Queue import Queue
import base64 from threading import Thread
from utils import utils from utils import utils
@ -30,25 +32,10 @@ CURRENT_SESSION = datetime.datetime.fromtimestamp(
PLACEMENT_GROUP = "PLACEMENT-" + CURRENT_SESSION PLACEMENT_GROUP = "PLACEMENT-" + CURRENT_SESSION
NODE_NAME_SUFFIX = "NODE-" + CURRENT_SESSION NODE_NAME_SUFFIX = "NODE-" + CURRENT_SESSION
def run_one_region_codedeploy(region_number, commit_id): def run_one_region_codedeploy(region_number, region_config, node_name_tag, commit_id):
#todo: explore the use ec2 resource and not client. e.g. create_instances -- Might make for better code. ec2_client, session = utils.create_ec2_client(region_number, region_config)
""" filters = [{'Name': 'tag:Name','Values': [node_name_tag]}]
for getting instance ids:--- instance_ids = utils.get_instance_ids(ec2_client.describe_instances(Filters=filters))
ec2 = boto3.resource('ec2', region_name=region_name])
result = ec2.instances.filter(Filters=[{'Name': 'instance-state-name', 'Values': ['running']}])
for instance in result:
instances.append(instance.id)
for getting public ips : --
ec2 = boto3.resource('ec2')
instance
"""
region_name = config[region_number][REGION_NAME]
NODE_NAME = region_number + "-" + NODE_NAME_SUFFIX
session = boto3.Session(region_name=region_name)
ec2_client = session.client('ec2')
filters = [{'Name': 'tag:Name','Values': [NODE_NAME]}]
instance_ids = get_instance_ids(ec2_client.describe_instances(Filters=filters))
print("Number of instances: %d" % len(instance_ids)) print("Number of instances: %d" % len(instance_ids))
@ -66,50 +53,15 @@ def run_one_region_codedeploy(region_number, commit_id):
codedeploy = session.client('codedeploy') codedeploy = session.client('codedeploy')
application_name = APPLICATION_NAME application_name = APPLICATION_NAME
deployment_group = APPLICATION_NAME + "-" + str(commit_id)[6] + "-" + CURRENT_SESSION deployment_group = APPLICATION_NAME + "-" + commit_id[:6] + "-" + CURRENT_SESSION
repo = REPO repo = REPO
print("Setting up to deploy commit_id %s on region %s"%(commit_id,region_number)) print("Setting up to deploy commit_id %s on region %s" % (commit_id, region_number))
response = get_application(codedeploy, application_name) utils.get_application(codedeploy, application_name)
deployment_group = get_deployment_group( deployment_group = utils.create_deployment_group(
codedeploy, region_number, application_name, deployment_group) codedeploy, region_number, application_name, deployment_group, node_name_tag)
depId = deploy(codedeploy, application_name, deployment_id = deploy(codedeploy, application_name, deployment_group, repo, commit_id)
deployment_group, repo, commit_id) return region_number, deployment_id
return region_number, depId
def get_deployment_group(codedeploy, region_number,application_name, deployment_group):
NODE_NAME = region_number + "-" + NODE_NAME_SUFFIX
response = codedeploy.create_deployment_group(
applicationName=application_name,
deploymentGroupName=deployment_group,
deploymentConfigName='CodeDeployDefault.AllAtOnce',
serviceRoleArn='arn:aws:iam::656503231766:role/BenchMarkCodeDeployServiceRole',
deploymentStyle={
'deploymentType': 'IN_PLACE',
'deploymentOption': 'WITHOUT_TRAFFIC_CONTROL'
},
ec2TagFilters = [
{
'Key': 'Name',
'Value': NODE_NAME,
'Type': 'KEY_AND_VALUE'
}
]
)
return deployment_group
def get_application(codedeploy, application_name):
response = codedeploy.list_applications()
if application_name in response['applications']:
return response
else:
response = codedeploy.create_application(
applicationName=application_name,
computePlatform='Server'
)
return response
def deploy(codedeploy, application_name, deployment_group, repo, commit_id): def deploy(codedeploy, application_name, deployment_group, repo, commit_id):
@ -130,39 +82,39 @@ def deploy(codedeploy, application_name, deployment_group, repo, commit_id):
'revisionType': 'GitHub', 'revisionType': 'GitHub',
'gitHubLocation': { 'gitHubLocation': {
'repository': repo, 'repository': repo,
'commit_id': commit_id, 'commitId': commit_id,
} }
} }
) )
depId = res["deploymentId"] deployment_id = res["deploymentId"]
print("Deployment ID: " + depId) print("Deployment ID: " + deployment_id)
# The deployment is launched at this point, so exit unless asked to wait # The deployment is launched at this point, so exit unless asked to wait
# until it finishes # until it finishes
info = {'status': 'Created'} info = {'status': 'Created'}
start = time.time() start = time.time()
while info['status'] not in ('Succeeded', 'Failed', 'Stopped',) and (time.time() - start < 600.0): while info['status'] not in ('Succeeded', 'Failed', 'Stopped',) and (time.time() - start < 600.0):
info = codedeploy.get_deployment(deploymentId=depId)['deploymentInfo'] info = codedeploy.get_deployment(deploymentId=deployment_id)['deploymentInfo']
print(info['status']) print(info['status'])
time.sleep(15) time.sleep(15)
if info['status'] == 'Succeeded': if info['status'] == 'Succeeded':
print("\nDeploy Succeeded") print("\nDeploy Succeeded")
return depId
else: else:
print("\nDeploy Failed") print("\nDeploy Failed")
print(info) print(info)
return depId return deployment_id
def run_one_region_codedeploy_wrapper(region_number, commit_id, queue): def run_one_region_codedeploy_wrapper(region_number, region_config, node_name_tag, commit_id, queue):
region_number, depId = run_one_region_codedeploy(region_number, commit_id) region_number, deployment_id = run_one_region_codedeploy(region_number, region_config, node_name_tag, commit_id)
queue.put((region_number, depId)) queue.put((region_number, deployment_id))
def launch_code_deploy(region_list, commit_id): def launch_code_deploy(region_list, region_config, commit_id):
queue = Queue() queue = Queue()
jobs = [] jobs = []
for i in range(len(region_list)): for region_tuppple in region_list:
region_number = region_list[i] # node_name_tag comes first.
node_name_tag, region_number = region_tuppple
my_thread = Thread(target=run_one_region_codedeploy_wrapper, args=( my_thread = Thread(target=run_one_region_codedeploy_wrapper, args=(
region_number, commit_id, queue)) region_number, region_config, node_name_tag, commit_id, queue))
my_thread.start() my_thread.start()
jobs.append(my_thread) jobs.append(my_thread)
for my_thread in jobs: for my_thread in jobs:
@ -170,31 +122,24 @@ def launch_code_deploy(region_list, commit_id):
results = [queue.get() for job in jobs] results = [queue.get() for job in jobs]
return results return results
def get_instance_ids(describe_instances_response):
instance_ids = []
for reservation in describe_instances_response["Reservations"]:
for instance in reservation["Instances"]:
instance_ids.append(instance["InstanceId"])
return instance_ids
if __name__ == "__main__": if __name__ == "__main__":
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description='This script helps you start instances across multiple regions') description='This script helps you start instances across multiple regions')
parser.add_argument('--regions', type=str, dest='regions', parser.add_argument('--instance_output', type=str, dest='instance_output',
default='3', help="Supply a csv list of all regions") default='instance_output.txt',
parser.add_argument('--instances', type=str, dest='numInstances', help='the file contains node_name_tag and region number of created instances.')
default='1', help='number of instances') parser.add_argument('--region_config', type=str, dest='region_config', default='configuration.txt')
parser.add_argument('--configuration', type=str,
dest='config', default='configuration.txt')
parser.add_argument('--commit_id', type=str, dest='commit_id', parser.add_argument('--commit_id', type=str, dest='commit_id',
default='1f7e6e7ca7cf1c1190cedec10e791c01a29971cf') default='1f7e6e7ca7cf1c1190cedec10e791c01a29971cf')
args = parser.parse_args() args = parser.parse_args()
config = utils.read_region_config(args.config)
region_list = [item.strip() for item in args.regions.split(',')]
instances_list = [item.strip() for item in args.numInstances.split(',')]
assert len(region_list) == len(instances_list), "number of regions: %d != number of instances per region: %d" % (
len(region_list), len(instances_list))
commit_id = args.commit_id commit_id = args.commit_id
results = launch_code_deploy(region_list, commit_id) if not os.path.isfile(args.instance_output) or not commit_id:
print(results) print "%s does not exist" % args.instance_output
sys.exit(1)
with open(args.instance_output, "r") as fin:
region_list = [line.split(" ") for line in fin.readlines()]
region_list = [(item[0].strip(), item[1].strip()) for item in region_list]
results = launch_code_deploy(region_list, args.region_config, commit_id)
print(results)

44
aws-experiment-launch/utils/utils.py
Unescape View File

@ -36,7 +36,7 @@ def create_ec2_client(region_number, region_config):
config = read_region_config(region_config) config = read_region_config(region_config)
region_name = config[region_number][REGION_NAME] region_name = config[region_number][REGION_NAME]
session = boto3.Session(region_name=region_name) session = boto3.Session(region_name=region_name)
return session.client('ec2') return session.client('ec2'), session
def collect_public_ips_from_ec2_client(ec2_client, node_name_tag): def collect_public_ips_from_ec2_client(ec2_client, node_name_tag):
filters = [{'Name': 'tag:Name','Values': [node_name_tag]}] filters = [{'Name': 'tag:Name','Values': [node_name_tag]}]
@ -48,10 +48,42 @@ def collect_public_ips_from_ec2_client(ec2_client, node_name_tag):
return ip_list return ip_list
def collect_public_ips(region_number, node_name_tag, region_config): def collect_public_ips(region_number, node_name_tag, region_config):
ec2_client = create_ec2_client(region_number, region_config) ec2_client, _ = create_ec2_client(region_number, region_config)
ip_list = collect_public_ips_from_ec2_client(ec2_client, node_name_tag) ip_list = collect_public_ips_from_ec2_client(ec2_client, node_name_tag)
return ip_list return ip_list
def get_application(codedeploy, application_name):
response = codedeploy.list_applications()
if application_name in response['applications']:
return response
else:
response = codedeploy.create_application(
applicationName=application_name,
computePlatform='Server'
)
return response
def create_deployment_group(codedeploy, region_number,application_name, deployment_group, node_name_tag):
_ = codedeploy.create_deployment_group(
applicationName=application_name,
deploymentGroupName=deployment_group,
deploymentConfigName='CodeDeployDefault.AllAtOnce',
serviceRoleArn='arn:aws:iam::656503231766:role/BenchMarkCodeDeployServiceRole',
deploymentStyle={
'deploymentType': 'IN_PLACE',
'deploymentOption': 'WITHOUT_TRAFFIC_CONTROL'
},
ec2TagFilters = [
{
'Key': 'Name',
'Value': node_name_tag,
'Type': 'KEY_AND_VALUE'
}
]
)
# Checking??
return deployment_group
def generate_distribution_config2(region_number, node_name_tag, region_config, def generate_distribution_config2(region_number, node_name_tag, region_config,
shard_number, client_number, distribution_config): shard_number, client_number, distribution_config):
ip_list = collect_public_ips(region_number, node_name_tag, region_config) ip_list = collect_public_ips(region_number, node_name_tag, region_config)
@ -96,6 +128,14 @@ def get_one_availability_zone(ec2_client):
else: else:
return None return None
# Get instance_ids from describe_instances_response.
def get_instance_ids(describe_instances_response):
instance_ids = []
if describe_instances_response["Reservations"]:
for reservation in describe_instances_response["Reservations"]:
instance_ids.extend(instance["InstanceId"] for instance in reservation["Instances"] if instance.get("InstanceId"))
return instance_ids
# used for testing only. # used for testing only.
# if __name__ == "__main__": # if __name__ == "__main__":
# ip_list = collect_public_ips('4', "4-NODE-23-36-01-2018-07-05", "configuration.txt") # ip_list = collect_public_ips('4', "4-NODE-23-36-01-2018-07-05", "configuration.txt")

Write Preview
Loading…
Cancel
Save