PoS_EVM
/
woop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

remove minh_aws

Browse Source
pull/37/head
Minh Doan 6 years ago
parent c444f5bc5c
commit 7a4ffa3cad
2 changed files with 0 additions and 438 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 8
      aws-experiment-launch/minh_aws/configuration.txt
  2. 430
      aws-experiment-launch/minh_aws/create_and_deploy.py

8
aws-experiment-launch/minh_aws/configuration.txt
Unescape View File

@ -1,8 +0,0 @@
1,us-east-1,virginia-key-benchmark,virginia-security-group,virginia,ami-b70554c8
2,us-east-2,ohio-key-benchmark,ohio-security-group,ohio,ami-8c122be9
3,us-west-1,california-key-benchmark,california-security-group,california,ami-e0ba5c83
4,us-west-2,oregon-key-benchmark,oregon-security-group,oregon,ami-a9d09ed1
5,ap-northeast-1,tokyo-key-benchmark,tokyo-security-group,tokyo,ami-e99f4896
6,ap-southeast-1,singapore-key-benchmark,singapore-security-group,singapore,ami-05868579
7,eu-central-1,frankfurt-key-benchmark,frankfurt-security-group,frankfurt,ami-7c4f7097
8,eu-west-1,ireland-key-benchmark,ireland-security-group,ireland,ami-466768ac

430
aws-experiment-launch/minh_aws/create_and_deploy.py
Unescape View File

@ -1,430 +0,0 @@
import boto3
import argparse
import sys
import json
import time
import datetime
from threading import Thread
from Queue import Queue
import base64
class InstanceResource:
ON_DEMAND = 1
SPOT_INSTANCE = 2
SPOT_FLEET = 3
REGION_NAME = 'region_name'
REGION_KEY = 'region_key'
REGION_SECURITY_GROUP = 'region_security_group'
REGION_HUMAN_NAME = 'region_human_name'
INSTANCE_TYPE = 't2.small'
REGION_AMI = 'region_ami'
with open("user-data.sh", "r") as userdata_file:
USER_DATA = userdata_file.read()
# UserData must be base64 encoded for spot instances.
USER_DATA_BASE64 = base64.b64encode(USER_DATA)
IAM_INSTANCE_PROFILE = 'BenchMarkCodeDeployInstanceProfile'
REPO = "simple-rules/harmony-benchmark"
APPLICATION_NAME = 'benchmark-experiments'
time_stamp = time.time()
CURRENT_SESSION = datetime.datetime.fromtimestamp(
time_stamp).strftime('%H-%M-%S-%Y-%m-%d')
PLACEMENT_GROUP = "PLACEMENT-" + CURRENT_SESSION
NODE_NAME_SUFFIX = "NODE-" + CURRENT_SESSION
"""
TODO:
save NODE to disk, so that you can selectively only run deploy (not recreate instances).
Right now all instances have "NODE" so this has uninted consequences of running on instances that were previous created.
Build (argparse,functions) support for
1. run only create instance (multiple times)
2. run only codedeploy (multiple times)
3. run create instance followed by codedeploy
"""
### CREATE INSTANCES ###
def run_one_region_instances(config, region_number, number_of_instances, instance_resource=InstanceResource.ON_DEMAND):
#todo: explore the use ec2 resource and not client. e.g. create_instances -- Might make for better code.
"""
e.g. ec2.create_instances
"""
region_name = config[region_number][REGION_NAME]
session = boto3.Session(region_name=region_name)
ec2_client = session.client('ec2')
if instance_resource == InstanceResource.ON_DEMAND:
NODE_NAME = create_instances(
config, ec2_client, region_number, int(number_of_instances))
print("Created %s in region %s"%(NODE_NAME,region_number)) ##REPLACE ALL print with logger
elif instance_resource == InstanceResource.SPOT_INSTANCE:
response = request_spot_instances(
config, ec2_client, region_number, int(number_of_instances))
else:
response = request_spot_fleet(
config, ec2_client, region_number, int(number_of_instances))
return session
def create_instances(config, ec2_client, region_number, number_of_instances):
NODE_NAME = region_number + "-" + NODE_NAME_SUFFIX
response = ec2_client.run_instances(
MinCount=number_of_instances,
MaxCount=number_of_instances,
ImageId=config[region_number][REGION_AMI],
Placement={
'AvailabilityZone': get_one_availability_zone(ec2_client),
},
SecurityGroups=[config[region_number][REGION_SECURITY_GROUP]],
IamInstanceProfile={
'Name': IAM_INSTANCE_PROFILE
},
KeyName=config[region_number][REGION_KEY],
UserData=USER_DATA,
InstanceType=INSTANCE_TYPE,
TagSpecifications=[
{
'ResourceType': 'instance',
'Tags': [
{
'Key': 'Name',
'Value': NODE_NAME
},
]
},
],
# We can also request spot instances this way but this way will block the
# process until spot requests are fulfilled, otherwise it will throw exception
# after 4 failed re-try.
# InstanceMarketOptions= {
# 'MarketType': 'spot',
# 'SpotOptions': {
# 'SpotInstanceType': 'one-time',
# 'BlockDurationMinutes': 60,
# }
# }
)
return NODE_NAME
def request_spot_instances(config, ec2_client, region_number, number_of_instances):
NODE_NAME = region_number + "-" + NODE_NAME_SUFFIX
response = ec2_client.request_spot_instances(
# DryRun=True,
BlockDurationMinutes=60,
InstanceCount=number_of_instances,
LaunchSpecification={
'SecurityGroups': [config[region_number][REGION_SECURITY_GROUP]],
'IamInstanceProfile': {
'Name': IAM_INSTANCE_PROFILE
},
'UserData': USER_DATA_BASE64,
'ImageId': config[region_number][REGION_AMI],
'InstanceType': INSTANCE_TYPE,
'KeyName': config[region_number][REGION_KEY],
'Placement': {
'AvailabilityZone': get_one_availability_zone(ec2_client)
}
}
)
return response
def request_spot_fleet(config, ec2_client, region_number, number_of_instances):
NODE_NAME = region_number + "-" + NODE_NAME_SUFFIX
# https://boto3.readthedocs.io/en/latest/reference/services/ec2.html#EC2.Client.request_spot_fleet
response = ec2_client.request_spot_fleet(
# DryRun=True,
SpotFleetRequestConfig={
# https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-fleet.html#spot-fleet-allocation-strategy
'AllocationStrategy': 'diversified',
# 'IamFleetRole': IAM_INSTANCE_PROFILE, // TODO@ricl, create fleet role.
'LaunchSpecifications': [
{
'SecurityGroups': [
{
'GroupName': config[region_number][REGION_SECURITY_GROUP]
}
],
'IamInstanceProfile': {
'Name': IAM_INSTANCE_PROFILE
},
'ImageId': config[region_number][REGION_AMI],
'InstanceType': INSTANCE_TYPE,
'KeyName': config[region_number][REGION_KEY],
'Placement': {
'AvailabilityZone': get_one_availability_zone(ec2_client)
},
'UserData': USER_DATA,
# 'WeightedCapacity': 123.0,
'TagSpecifications': [
{
'ResourceType': 'instance',
'Tags': [
{
'Key': 'Name',
'Value': NODE_NAME
},
]
}
]
},
],
# 'SpotPrice': 'string', # The maximum price per unit hour that you are willing to pay for a Spot Instance. The default is the On-Demand price.
'TargetCapacity': 1,
'OnDemandTargetCapacity': 0,
'Type': 'maintain',
}
)
return response
def get_availability_zones(ec2_client):
response = ec2_client.describe_availability_zones()
all_zones = []
if response.get('AvailabilityZones', None):
region_info = response.get('AvailabilityZones')
for info in region_info:
if info['State'] == 'available':
all_zones.append(info['ZoneName'])
return all_zones
def get_one_availability_zone(ec2_client):
all_zones = get_availability_zones(ec2_client)
if len(all_zones) > 0:
return all_zones[0]
else:
print("No availability zone for this region")
sys.exit()
#### CODEDEPLOY ###
def run_one_region_codedeploy(region_number, commitId):
#todo: explore the use ec2 resource and not client. e.g. create_instances -- Might make for better code.
"""
for getting instance ids:---
ec2 = boto3.resource('ec2', region_name=region_name])
result = ec2.instances.filter(Filters=[{'Name': 'instance-state-name', 'Values': ['running']}])
for instance in result:
instances.append(instance.id)
for getting public ips : --
ec2 = boto3.resource('ec2')
instance
"""
region_name = config[region_number][REGION_NAME]
NODE_NAME = region_number + "-" + NODE_NAME_SUFFIX
session = boto3.Session(region_name=region_name)
ec2_client = session.client('ec2')
filters = [{'Name': 'tag:Name','Values': [NODE_NAME]}]
instance_ids = get_instance_ids(ec2_client.describe_instances(Filters=filters))
print("Number of instances: %d" % len(instance_ids))
print("Waiting for all %d instances in region %s to start running"%(len(instance_ids),region_number))
waiter = ec2_client.get_waiter('instance_running')
waiter.wait(InstanceIds=instance_ids)
print("Waiting for all %d instances in region %s to be INSTANCE STATUS OK"%(len(instance_ids),region_number))
waiter = ec2_client.get_waiter('instance_status_ok')
waiter.wait(InstanceIds=instance_ids)
print("Waiting for all %d instances in region %s to be SYSTEM STATUS OK"%(len(instance_ids),region_number))
waiter = ec2_client.get_waiter('system_status_ok')
waiter.wait(InstanceIds=instance_ids)
codedeploy = session.client('codedeploy')
application_name = APPLICATION_NAME
deployment_group = APPLICATION_NAME + "-" + str(commitId)[6] + "-" + CURRENT_SESSION
repo = REPO
print("Setting up to deploy commitId %s on region %s"%(commitId,region_number))
response = get_application(codedeploy, application_name)
deployment_group = get_deployment_group(
codedeploy, region_number, application_name, deployment_group)
depId = deploy(codedeploy, application_name,
deployment_group, repo, commitId)
return region_number, depId
def get_deployment_group(codedeploy, region_number,application_name, deployment_group):
NODE_NAME = region_number + "-" + NODE_NAME_SUFFIX
response = codedeploy.create_deployment_group(
applicationName=application_name,
deploymentGroupName=deployment_group,
deploymentConfigName='CodeDeployDefault.AllAtOnce',
serviceRoleArn='arn:aws:iam::656503231766:role/BenchMarkCodeDeployServiceRole',
deploymentStyle={
'deploymentType': 'IN_PLACE',
'deploymentOption': 'WITHOUT_TRAFFIC_CONTROL'
},
ec2TagFilters = [
{
'Key': 'Name',
'Value': NODE_NAME,
'Type': 'KEY_AND_VALUE'
}
]
)
return deployment_group
def get_application(codedeploy, application_name):
response = codedeploy.list_applications()
if application_name in response['applications']:
return response
else:
response = codedeploy.create_application(
applicationName=application_name,
computePlatform='Server'
)
return response
def deploy(codedeploy, application_name, deployment_group, repo, commitId):
"""Deploy new code at specified revision to instance.
arguments:
- repo: GitHub repository path from which to get the code
- commitId: commit ID to be deployed
- wait: wait until the CodeDeploy finishes
"""
print("Launching CodeDeploy with commit " + commitId)
res = codedeploy.create_deployment(
applicationName=application_name,
deploymentGroupName=deployment_group,
deploymentConfigName='CodeDeployDefault.AllAtOnce',
description='benchmark experiments',
revision={
'revisionType': 'GitHub',
'gitHubLocation': {
'repository': repo,
'commitId': commitId,
}
}
)
depId = res["deploymentId"]
print("Deployment ID: " + depId)
# The deployment is launched at this point, so exit unless asked to wait
# until it finishes
info = {'status': 'Created'}
start = time.time()
while info['status'] not in ('Succeeded', 'Failed', 'Stopped',) and (time.time() - start < 600.0):
info = codedeploy.get_deployment(deploymentId=depId)['deploymentInfo']
print(info['status'])
time.sleep(15)
if info['status'] == 'Succeeded':
print("\nDeploy Succeeded")
return depId
else:
print("\nDeploy Failed")
print(info)
return depId
def run_one_region_codedeploy_wrapper(region_number, commitId, queue):
region_number, depId = run_one_region_codedeploy(region_number, commitId)
queue.put((region_number, depId))
def launch_code_deploy(region_list, commitId):
queue = Queue()
jobs = []
for i in range(len(region_list)):
region_number = region_list[i]
my_thread = Thread(target=run_one_region_codedeploy_wrapper, args=(
region_number, commitId, queue))
my_thread.start()
jobs.append(my_thread)
for my_thread in jobs:
my_thread.join()
results = [queue.get() for job in jobs]
return results
##### UTILS ####
def get_instance_ids(describe_instances_response):
instance_ids = []
for reservation in describe_instances_response["Reservations"]:
for instance in reservation["Instances"]:
instance_ids.append(instance["InstanceId"])
return instance_ids
def read_configuration_file(filename):
config = {}
with open(filename, 'r') as f:
for myline in f:
mylist = myline.strip().split(',')
region_num = mylist[0]
config[region_num] = {}
config[region_num][REGION_NAME] = mylist[1]
config[region_num][REGION_KEY] = mylist[2]
config[region_num][REGION_SECURITY_GROUP] = mylist[3]
config[region_num][REGION_HUMAN_NAME] = mylist[4]
config[region_num][REGION_AMI] = mylist[5]
return config
def get_commitId(commitId):
if commitId is None:
commitId = run("git rev-list --max-count=1 HEAD",
hide=True).stdout.strip()
print("Got newest commitId as " + commitId)
return commitId
def get_public_ips(response):
return [instance['PublicIpAddress'] for instance in response['Instances']]
def generate_config_file(shard_num, client_num, ips, config_filename):
if len(ips) < shard_num * 2 + client_num:
print("Not enough nodes to generate a config file")
return False
# Create ip for clients.
client_id, leader_id, validator_id = 0, 0, 0
with open(config_filename, "w") as fout:
for i in range(len(ips)):
if client_id < client_num:
fout.write("%s 9000 client %d\n" % (ips[i], client_id % shard_num))
client_id = client_id + 1
elif leader_id < shard_num:
fout.write("%s 9000 leader %d\n" % (ips[i], leader_id))
leader_id = leader_id + 1
else:
fout.write("%s 9000 validator %d\n" % (ips[i], validator_id % shard_num))
validator_id = validator_id + 1
##### UTILS ####
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description='This script helps you start instances across multiple regions')
parser.add_argument('--regions', type=str, dest='regions',
default='3', help="Supply a csv list of all regions")
parser.add_argument('--instances', type=str, dest='numInstances',
default='1', help='number of instances')
parser.add_argument('--configuration', type=str,
dest='config', default='configuration.txt')
parser.add_argument('--commitId', type=str, dest='commitId',
default='1f7e6e7ca7cf1c1190cedec10e791c01a29971cf')
args = parser.parse_args()
config = read_configuration_file(args.config)
region_list = args.regions.split(',')
instances_list = args.numInstances.split(',')
assert len(region_list) == len(instances_list), "number of regions: %d != number of instances per region: %d" % (
len(region_list), len(instances_list))
commitId = args.commitId
for i in range(len(region_list)):
region_number = region_list[i]
number_of_instances = instances_list[i]
session = run_one_region_instances(
config, region_number, number_of_instances, InstanceResource.ON_DEMAND)
results = launch_code_deploy(region_list, commitId)
print(results)
Write Preview
Loading…
Cancel
Save