PoS_EVM
/
woop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' into benchmark_update

Browse Source
pull/37/head
Minh Doan 6 years ago
parent f0fd5e6180 7f48e2270a
commit 0c1756298c
6 changed files with 82 additions and 125 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 58
      aws-experiment-launch/create_instances.py
  2. 72
      aws-experiment-launch/spot-instance/create_launch_specs.py
  3. 14
      aws-experiment-launch/utils/launch_template.py
  4. 43
      aws-experiment-launch/utils/spot_fleet.py
  5. 11
      aws-experiment-launch/utils/user-data.sh
  6. 9
      aws-experiment-launch/utils/utils.py

58
aws-experiment-launch/create_instances.py
Unescape View File

@ -6,40 +6,27 @@ import json
import sys
import threading
import time
import enum
from utils import utils, spot_fleet, logger
LOGGER = logger.getLogger(__file__)
class InstanceResource:
ON_DEMAND = 1
SPOT_INSTANCE = 2
SPOT_FLEET = 3
def run_one_region_instances(config, region_number, number_of_instances, instance_resource=InstanceResource.ON_DEMAND):
region_name = config[region_number][utils.REGION_NAME]
# Create session.
session = boto3.Session(region_name=region_name)
# Create a client.
ec2_client = session.client('ec2')
if instance_resource == InstanceResource.ON_DEMAND:
node_name_tag = create_instances(
config, ec2_client, region_number, int(number_of_instances))
LOGGER.info("Created %s in region %s" % (node_name_tag, region_number))
return node_name_tag, ec2_client
elif instance_resource == InstanceResource.SPOT_FLEET:
instance_type_list = ['t2.micro', 't2.small', 'm3.medium']
node_name_tag = spot_fleet.request_spot_fleet_with_on_demand(
config, ec2_client, region_number, int(number_of_instances), 1, instance_type_list)
# node_name_tag = spot_fleet.request_spot_fleet(
# config, ec2_client, region_number, int(number_of_instances), instance_type_list)
return node_name_tag, ec2_client
else:
return None, None
class InstanceResource(enum.Enum):
ON_DEMAND = 'ondemand'
SPOT_INSTANCE = 'spot'
SPOT_FLEET = 'fleet'
def __str__(self):
return self.value
def run_one_region_on_demand_instances(config, region_number, number_of_instances):
ec2_client = utils.create_client(config, region_number)
node_name_tag = create_instances(
config, ec2_client, region_number, int(number_of_instances))
LOGGER.info("Created %s in region %s" % (node_name_tag, region_number))
return node_name_tag, ec2_client
def create_instances(config, ec2_client, region_number, number_of_instances):
node_name_tag = utils.get_node_name_tag(region_number)
@ -97,9 +84,9 @@ def create_instances(config, ec2_client, region_number, number_of_instances):
lock = threading.Lock()
def run_for_one_region(config, region_number, number_of_instances, instance_resouce, fout, fout2):
node_name_tag, ec2_client = run_one_region_instances(
config, region_number, number_of_instances, instance_resouce)
def run_for_one_region_on_demand(config, region_number, number_of_instances, fout, fout2):
node_name_tag, ec2_client = run_one_region_on_demand_instances(
config, region_number, number_of_instances)
if node_name_tag:
LOGGER.info("Managed to create instances for region %s" %
region_number)
@ -129,12 +116,15 @@ if __name__ == "__main__":
default='instance_output.txt', help='the file to append or write')
parser.add_argument('--instance_ids_output', type=str, dest='instance_ids_output',
default='instance_ids_output.txt', help='the file to append or write')
parser.add_argument('--instance_resource', dest='instance_resource', type=InstanceResource,
default=InstanceResource.ON_DEMAND, choices=list(InstanceResource))
parser.add_argument('--append', dest='append', type=bool, default=False,
help='append to the current instance_output')
args = parser.parse_args()
config = utils.read_region_config(args.region_config)
region_list = args.regions.split(',')
num_instance_list = args.num_instance_list.split(',')
instance_resource = args.instance_resource
assert len(region_list) == len(num_instance_list), "number of regions: %d != number of instances per region: %d" % (
len(region_list), len(num_instance_list))
@ -144,8 +134,12 @@ if __name__ == "__main__":
for i in range(len(region_list)):
region_number = region_list[i]
number_of_instances = num_instance_list[i]
t = threading.Thread(target=run_for_one_region, args=(
config, region_number, number_of_instances, InstanceResource.SPOT_FLEET, fout, fout2))
if instance_resource == InstanceResource.ON_DEMAND:
t = threading.Thread(target=run_for_one_region_on_demand, args=(
config, region_number, number_of_instances, fout, fout2))
elif instance_resource == InstanceResource.SPOT_FLEET:
t = threading.Thread(target=spot_fleet.run_one_region, args=(
region_number, number_of_instances, fout, fout2))
LOGGER.info("creating thread for region %s" % region_number)
t.start()
thread_pool.append(t)

72
aws-experiment-launch/spot-instance/create_launch_specs.py
Unescape View File

@ -1,72 +0,0 @@
import os
import argparse
import json
import time
import datetime
import base64
REGION_NAME = 'region_name'
REGION_KEY = 'region_key'
REGION_SECURITY_GROUP = 'region_security_group'
REGION_HUMAN_NAME = 'region_human_name'
INSTANCE_TYPE = 'm3.medium' # 't2.micro'
AMI = 'ami-f2d3638a' # 'ami-a9d09ed1'
# UserData must be base64 encoded.
with open("userdata.sh", "rb") as userdata_file:
USER_DATA = base64.b64encode(userdata_file.read())
IAM_INSTANCE_PROFILE = 'BenchMarkCodeDeployInstanceProfile'
def read_configuration_file(filename):
config = {}
with open(filename,'r') as f:
for line in f:
vals = line.strip().split(',')
region_num = vals[0]
config[region_num] = {}
config[region_num][REGION_NAME] = vals[1]
config[region_num][REGION_KEY] = vals[2]
config[region_num][REGION_SECURITY_GROUP] = vals[3]
config[region_num][REGION_HUMAN_NAME] = vals[4]
return config
def create_launch_specification(region_num):
input_cli = {}
input_cli['ImageId'] = AMI
# input_cli['Placement'] = {
# "AvailabilityZone": config[region_num][REGION_NAME] +"a"
# }
input_cli['SecurityGroups'] = [ "richard-spot-instance SSH" ] # [ config[region_num][REGION_SECURITY_GROUP] ]
input_cli['IamInstanceProfile'] = {
"Name": IAM_INSTANCE_PROFILE
}
input_cli['KeyName'] = "richard-spot-instance" # config[region_num][REGION_KEY]
input_cli['UserData'] = USER_DATA
input_cli['InstanceType'] = INSTANCE_TYPE
# folder = "launch_specs/" + "session-"+ current_session
folder = "launch_specs/latest"
if not os.path.exists(folder):
os.makedirs(folder)
launch_spec_file = os.path.join(folder,config[region_num][REGION_HUMAN_NAME]+".json")
with open(launch_spec_file,'w') as g:
json.dump(input_cli,g)
print("Launch spec: %s" % launch_spec_file)
return launch_spec_file
def create_instances(region_list):
for i in range(len(region_list)):
region_num = region_list[i]
launch_spec_file = create_launch_specification(region_num)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='This script helps you start instances across multiple regions')
parser.add_argument('--regions',type=str,dest='regions',default='3',help="a comma-separated-value list of all regions")
# configuration file contains the number/name/security-group etc. information of each region.
parser.add_argument('--config',type=str,dest='config',default='configuration.txt')
args = parser.parse_args()
config = read_configuration_file(args.config)
region_list = args.regions.split(',')
time_stamp = time.time()
current_session = datetime.datetime.fromtimestamp(time_stamp).strftime('%Y-%m-%d-%H-%M-%S')
print("current session is %s" % current_session)
create_instances(region_list)

14
aws-experiment-launch/utils/launch_template.py
Unescape View File

@ -2,24 +2,24 @@
import utils
def get_launch_template_name(config, region_number):
return 'benchmark-' + config[region_number][utils.REGION_NAME]
def get_launch_template_name(region_number):
return 'benchmark-' + utils.CONFIG[region_number][utils.REGION_NAME]
def create(config, ec2_client, region_number):
def create(ec2_client, region_number):
return ec2_client.create_launch_template(
# DryRun=True,
LaunchTemplateName=get_launch_template_name(config, region_number),
LaunchTemplateName=get_launch_template_name(region_number),
LaunchTemplateData={
'IamInstanceProfile': {
'Name': utils.IAM_INSTANCE_PROFILE
},
'ImageId': config[region_number][utils.REGION_AMI],
'ImageId': utils.CONFIG[region_number][utils.REGION_AMI],
# 'InstanceType': instance_type,
'KeyName': config[region_number][utils.REGION_KEY],
'KeyName': utils.CONFIG[region_number][utils.REGION_KEY],
'UserData': utils.USER_DATA_BASE64,
'SecurityGroupIds': [
config[region_number][utils.REGION_SECURITY_GROUP_ID]
utils.CONFIG[region_number][utils.REGION_SECURITY_GROUP_ID]
],
# 'InstanceInitiatedShutdownBehavior': 'stop',
'TagSpecifications': [

43
aws-experiment-launch/utils/spot_fleet.py
Unescape View File

@ -4,7 +4,7 @@ import launch_template
LOGGER = logger.getLogger(__file__)
def create_launch_specification(config, region_number, instanceType):
def create_launch_specification(region_number, instanceType):
return {
# Region irrelevant fields
'IamInstanceProfile': {
@ -17,11 +17,11 @@ def create_launch_specification(config, region_number, instanceType):
{
# In certain scenarios, we have to use group id instead of group name
# https://github.com/boto/boto/issues/350#issuecomment-27359492
'GroupId': config[region_number][utils.REGION_SECURITY_GROUP_ID]
'GroupId': utils.CONFIG[region_number][utils.REGION_SECURITY_GROUP_ID]
}
],
'ImageId': config[region_number][utils.REGION_AMI],
'KeyName': config[region_number][utils.REGION_KEY],
'ImageId': utils.CONFIG[region_number][utils.REGION_AMI],
'KeyName': utils.CONFIG[region_number][utils.REGION_KEY],
'TagSpecifications': [
{
'ResourceType': 'instance',
@ -40,14 +40,14 @@ def create_launch_specification(config, region_number, instanceType):
}
def create_launch_specification_list(config, region_number, instance_type_list):
return list(map(lambda type: create_launch_specification(config, region_number, type), instance_type_list))
def create_launch_specification_list(region_number, instance_type_list):
return list(map(lambda type: create_launch_specification(region_number, type), instance_type_list))
def get_launch_template(config, region_number, instance_type):
def get_launch_template(region_number, instance_type):
return {
'LaunchTemplateSpecification': {
'LaunchTemplateName': launch_template.get_launch_template_name(config, region_number),
'LaunchTemplateName': launch_template.get_launch_template_name(region_number),
'Version': '1'
},
'Overrides': [
@ -58,11 +58,11 @@ def get_launch_template(config, region_number, instance_type):
}
def get_launch_template_list(config, region_number, instance_type_list):
return list(map(lambda type: get_launch_template(config, region_number, type), instance_type_list))
def get_launch_template_list(region_number, instance_type_list):
return list(map(lambda type: get_launch_template(region_number, type), instance_type_list))
def request_spot_fleet(config, ec2_client, region_number, number_of_instances, instance_type_list):
def request_spot_fleet(ec2_client, region_number, number_of_instances, instance_type_list):
LOGGER.info("Requesting spot fleet")
LOGGER.info("Creating node_name_tag: %s" %
utils.get_node_name_tag(region_number))
@ -73,7 +73,7 @@ def request_spot_fleet(config, ec2_client, region_number, number_of_instances, i
# https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-fleet.html#spot-fleet-allocation-strategy
'AllocationStrategy': 'diversified',
'IamFleetRole': 'arn:aws:iam::656503231766:role/RichardFleetRole',
'LaunchSpecifications': create_launch_specification_list(config, region_number, instance_type_list),
'LaunchSpecifications': create_launch_specification_list(region_number, instance_type_list),
# 'SpotPrice': 'string', # The maximum price per unit hour that you are willing to pay for a Spot Instance. The default is the On-Demand price.
'TargetCapacity': number_of_instances,
'Type': 'maintain'
@ -82,7 +82,7 @@ def request_spot_fleet(config, ec2_client, region_number, number_of_instances, i
return response
def request_spot_fleet_with_on_demand(config, ec2_client, region_number, number_of_instances, number_of_on_demand, instance_type_list):
def request_spot_fleet_with_on_demand(ec2_client, region_number, number_of_instances, number_of_on_demand, instance_type_list):
LOGGER.info("Requesting spot fleet")
LOGGER.info("Creating node_name_tag: %s" %
utils.get_node_name_tag(region_number))
@ -93,7 +93,7 @@ def request_spot_fleet_with_on_demand(config, ec2_client, region_number, number_
# https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-fleet.html#spot-fleet-allocation-strategy
'AllocationStrategy': 'diversified',
'IamFleetRole': 'arn:aws:iam::656503231766:role/RichardFleetRole',
'LaunchTemplateConfigs': get_launch_template_list(config, region_number, instance_type_list),
'LaunchTemplateConfigs': get_launch_template_list(region_number, instance_type_list),
# 'SpotPrice': 'string', # The maximum price per unit hour that you are willing to pay for a Spot Instance. The default is the On-Demand price.
'TargetCapacity': number_of_instances,
'OnDemandTargetCapacity': number_of_on_demand,
@ -101,3 +101,18 @@ def request_spot_fleet_with_on_demand(config, ec2_client, region_number, number_
}
)
return response
def get_instance_ids(ec2_client, request_id):
res = ec2_client.describe_spot_fleet_instances(
SpotFleetRequestId=request_id
)
return [ inst.InstanceId for inst in res.ActiveInstances ]
def run_one_region(region_number, number_of_instances, fout, fout2):
client = utils.create_client(utils.CONFIG, region_number)
instance_type_list = ['t2.micro', 't2.small', 'm3.medium']
# node_name_tag = request_spot_fleet_with_on_demand(
# client, region_number, int(number_of_instances), 1, instance_type_list)
node_name_tag = request_spot_fleet(
client, region_number, int(number_of_instances), instance_type_list)
return node_name_tag, client

11
aws-experiment-launch/utils/user-data.sh
Unescape View File

@ -0,0 +1,11 @@
#!/bin/bash -x
REGION=$(curl 169.254.169.254/latest/meta-data/placement/availability-zone/ | sed 's/[a-z]$//')
#yum update -y #This breaking codedeploy right now
yum install ruby wget -y
cd /home/ec2-user
touch yum-not-updated.txt
wget https://aws-codedeploy-$REGION.s3.amazonaws.com/latest/install
chmod +x ./install
./install auto
mkdir projects
mkdir projects/src

9
aws-experiment-launch/utils/utils.py
Unescape View File

@ -28,7 +28,15 @@ with open("user-data.sh", "r") as userdata_file:
# UserData must be base64 encoded for spot instances.
USER_DATA_BASE64 = base64.b64encode(USER_DATA)
def create_client(config, region_number):
region_name = config[region_number][REGION_NAME]
# Create session.
session = boto3.Session(region_name=region_name)
# Create a client.
return session.client('ec2')
def read_region_config(region_config='configuration.txt'):
global CONFIG
config = {}
with open(region_config, 'r') as f:
for myline in f:
@ -41,6 +49,7 @@ def read_region_config(region_config='configuration.txt'):
config[region_num][REGION_HUMAN_NAME] = mylist[4]
config[region_num][REGION_AMI] = mylist[5]
config[region_num][REGION_SECURITY_GROUP_ID] = mylist[6]
CONFIG = config
return config
def get_ip_list(response):

Write Preview
Loading…
Cancel
Save