PoS_EVM
/
woop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

move code from new-pipeline

Browse Source
pull/43/head
Minh Doan 6 years ago
parent 48d15578af
commit c8a83c9a9e
13 changed files with 537 additions and 0 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 35
      aws-experiment-launch/collect_public_ips.py
  2. 62
      aws-experiment-launch/commander_prepare.py
  3. 8
      aws-experiment-launch/configuration-git.txt
  4. 28
      aws-experiment-launch/create_deploy_soldiers.sh
  5. 193
      aws-experiment-launch/create_solider_instances.py
  6. 37
      aws-experiment-launch/generate_distribution_config.py
  7. 1
      aws-experiment-launch/logs_download.sh
  8. 105
      aws-experiment-launch/terminate_instances.py
  9. 15
      aws-experiment-launch/upload_binaries.py
  10. 10
      aws-experiment-launch/upload_config.py
  11. 9
      aws-experiment-launch/upload_s3.py
  12. 11
      aws-experiment-launch/userdata-commander.sh
  13. 23
      aws-experiment-launch/userdata-soldier.sh

35
aws-experiment-launch/collect_public_ips.py
Unescape View File

@ -0,0 +1,35 @@
import argparse
import os
import random
import sys
from utils import utils
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='This script helps you to collect public ips')
parser.add_argument('--instance_output', type=str, dest='instance_output',
default='instance_output.txt',
help='the file contains node_name_tag and region number of created instances.')
parser.add_argument('--region_config', type=str,
dest='region_config', default='configuration.txt')
parser.add_argument('--file_output', type=str,
dest='file_output', default='raw_ip.txt')
args = parser.parse_args()
if not args.instance_output or not os.path.isfile(args.instance_output):
print "%s or %s are not existed" % (args.file_output, args.instance_output)
sys.exit(1)
if args.instance_output:
with open(args.instance_output, "r") as fin, open(args.file_output, "w") as fout:
total_ips = []
node_name_tag_list = []
for line in fin.readlines():
items = line.split(" ")
region_number = items[1].strip()
node_name_tag = items[0].strip()
ip_list = utils.collect_public_ips(region_number, node_name_tag, args.region_config)
total_ips.extend([(ip, node_name_tag) for ip in ip_list])
random.shuffle(total_ips)
for tuple in total_ips:
fout.write(tuple[0] + " " + tuple[1] + "\n")
print "Done collecting public ips %s" % args.file_output

62
aws-experiment-launch/commander_prepare.py
Unescape View File

@ -0,0 +1,62 @@
import argparse
import logging
import os
import stat
import sys
from utils import utils
logging.basicConfig(level=logging.INFO, format='%(threadName)s %(asctime)s - %(name)s - %(levelname)s - %(message)s')
LOGGER = logging.getLogger(__file__)
LOGGER.setLevel(logging.INFO)
PEMS = [
"virginia-key-benchmark.pem",
"ohio-key-benchmark.pem",
"california-key-benchmark.pem",
"oregon-key-benchmark.pem",
"tokyo-key-benchmark.pem",
"singapore-key-benchmark.pem",
"frankfurt-key-benchmark.pem",
"ireland-key-benchmark.pem",
]
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='This script helps you to genereate distribution config')
parser.add_argument('--distribution_config', type=str,
dest='distribution_config', default='distribution_config.txt')
parser.add_argument('--commander_logging', type=str,
dest='commander_logging', default='commander_logging.sh')
args = parser.parse_args()
if not os.path.exists(args.distribution_config):
sys.exit(1)
with open(args.distribution_config, "r") as fin:
lines = fin.readlines()
commander_address = None
commander_region = None
commander_output = None
with open(args.distribution_config, "w") as fout:
for line in lines:
if "commander" in line:
items = [item.strip() for item in line.split(" ")]
commander_address = items[0]
commander_region = int(items[4][0])
commander_output = "\n".join(items)
else:
fout.write(line.strip() + "\n")
LOGGER.info("Generated %s" % args.distribution_config)
if not commander_address or not commander_region:
LOGGER.info("Failed to extract commander address and commander region.")
sys.exit(1)
with open(args.commander_logging, "w") as fout:
fout.write("ssh -i ../keys/%s ec2-user@%s\n" % (PEMS[commander_region - 1], commander_address))
st = os.stat(args.commander_logging)
os.chmod(args.commander_logging, st.st_mode | stat.S_IEXEC)
LOGGER.info("Generated %s" % args.commander_logging)
LOGGER.info("DONE.")

8
aws-experiment-launch/configuration-git.txt
Unescape View File

@ -0,0 +1,8 @@
1,us-east-1,virginia-key-benchmark,virginia-security-group,virginia,ami-97785bed,sg-04d0b62ee08ce8800
2,us-east-2,ohio-key-benchmark,ohio-security-group,ohio,ami-f63b1193,sg-0789078f1c76defbe
3,us-west-1,california-key-benchmark,california-security-group,california,ami-824c4ee2,sg-0a66ccb6ab9161a14
4,us-west-2,oregon-key-benchmark,oregon-security-group,oregon,ami-f2d3638a,sg-020cb5729fa212d43
5,ap-northeast-1,tokyo-key-benchmark,tokyo-security-group,tokyo,ami-ceafcba8,sg-009aeb97f675c1ad5
6,ap-southeast-1,singapore-key-benchmark,singapore-security-group,singapore,ami-68097514,sg-05f9b60044a19dfb2
7,eu-central-1,frankfurt-key-benchmark,frankfurt-security-group,frankfurt,ami-5652ce39,sg-0bb06fcd8b25b5910
8,eu-west-1,ireland-key-benchmark,ireland-security-group,ireland,ami-d834aba1,sg-0aa8954acb79fdb58

28
aws-experiment-launch/create_deploy_soldiers.sh
Unescape View File

@ -0,0 +1,28 @@
if [ $# -lt 3 ]; then
echo "Please provide # of instances, # of shards, # of clients"
exit 1
fi
INSTANCE_NUM=$1
SHARD_NUM=$2
CLIENT_NUM=$3
SLEEP_TIME=10
echo "Creating $INSTANCE_NUM instances at 8 regions"
python create_solider_instances.py --regions 1,2,3,4,5,6,7,8 --instances $INSTANCE_NUM,$INSTANCE_NUM,$INSTANCE_NUM,$INSTANCE_NUM,$INSTANCE_NUM,$INSTANCE_NUM,$INSTANCE_NUM,$INSTANCE_NUM
echo "Sleep for $SLEEP_TIME seconds"
sleep $SLEEP_TIME
echo "Rung collecint raw ips"
python collect_public_ips.py --instance_output instance_output.txt
# sleep 10
echo "Generate distribution_config"
python generate_distribution_config.py --ip_list_file raw_ip.txt --shard_number $SHARD_NUM --client_number $CLIENT_NUM
echo "Run commander prepare"
python commander_prepare.py
aws s3 cp distribution_config.txt s3://unique-bucket-bin/distribution_config.txt --acl public-read-write

193
aws-experiment-launch/create_solider_instances.py
Unescape View File

@ -0,0 +1,193 @@
import argparse
import base64
import boto3
import datetime
import json
import sys
import threading
import time
import enum
#TODO REMOVE UTILS
from utils import utils, spot_fleet, logger
LOGGER = logger.getLogger(__file__)
REGION_NAME = 'region_name'
REGION_KEY = 'region_key'
REGION_SECURITY_GROUP = 'region_security_group'
REGION_SECURITY_GROUP_ID = 'region_security_group_id'
REGION_HUMAN_NAME = 'region_human_name'
INSTANCE_TYPE = 't2.micro'
REGION_AMI = 'region_ami'
class InstanceResource(enum.Enum):
ON_DEMAND = 'ondemand'
SPOT_INSTANCE = 'spot'
SPOT_FLEET = 'fleet'
def __str__(self):
return self.value
def run_one_region_on_demand_instances(config, region_number, number_of_instances, tag):
ec2_client = utils.create_client(config, region_number)
node_name_tag = create_instances(
config, ec2_client, region_number, number_of_instances, tag)
LOGGER.info("Created %s in region %s" % (node_name_tag, region_number))
return node_name_tag, ec2_client
def create_instances(config, ec2_client, region_number, number_of_instances, tag):
node_name_tag = utils.get_node_name_tag2(region_number, tag)
LOGGER.info("Creating node_name_tag: %s" % node_name_tag)
available_zone = utils.get_one_availability_zone(ec2_client)
LOGGER.info("Looking at zone %s to create instances." % available_zone)
time.sleep(2)
ec2_client.run_instances(
MinCount=number_of_instances,
MaxCount=number_of_instances,
ImageId=config[region_number][utils.REGION_AMI],
Placement={
'AvailabilityZone': available_zone,
},
SecurityGroups=[config[region_number][utils.REGION_SECURITY_GROUP]],
IamInstanceProfile={
'Name': utils.IAM_INSTANCE_PROFILE
},
KeyName=config[region_number][utils.REGION_KEY],
UserData=utils.USER_DATA,
InstanceType=utils.INSTANCE_TYPE,
TagSpecifications=[
{
'ResourceType': 'instance',
'Tags': [
{
'Key': 'Name',
'Value': node_name_tag
},
]
},
],
)
retry_count = 10
while retry_count > 0:
try:
time.sleep(20)
instance_ids = utils.get_instance_ids2(ec2_client, node_name_tag)
LOGGER.info("Waiting for all %d instances in region %s with node_name_tag %s to be in RUNNING" % (
len(instance_ids), region_number, node_name_tag))
break
except:
retry_count -= 1
LOGGER.info("Failed to get instance ids. Retry again.")
retry_count = 10
while retry_count > 0:
try:
time.sleep(20)
waiter = ec2_client.get_waiter('instance_running')
waiter.wait(InstanceIds=instance_ids)
break
except:
retry_count -= 1
LOGGER.info("Failed to wait.")
retry_count = 10
while retry_count > 0:
time.sleep(10)
LOGGER.info("Waiting ...")
ip_list = utils.collect_public_ips_from_ec2_client(
ec2_client, node_name_tag)
if len(ip_list) == number_of_instances:
LOGGER.info("Created %d instances" % number_of_instances)
return node_name_tag
retry_count -= 10
LOGGER.info("Can not create %d instances" % number_of_instances)
return None
LOCK_FOR_RUN_ONE_REGION = threading.Lock()
def run_for_one_region_on_demand(config, region_number, number_of_instances, fout, fout2):
tag = 0
number_of_instances = int(number_of_instances)
while number_of_instances > 0:
number_of_creation = min(utils.MAX_INSTANCES_FOR_DEPLOYMENT, number_of_instances)
node_name_tag, ec2_client = run_one_region_on_demand_instances(
config, region_number, number_of_creation, tag)
if node_name_tag:
LOGGER.info("Managed to create instances for region %s with name_name_tag %s" %
(region_number, node_name_tag))
instance_ids = utils.get_instance_ids2(ec2_client, node_name_tag)
LOCK_FOR_RUN_ONE_REGION.acquire()
try:
fout.write("%s %s\n" % (node_name_tag, region_number))
for instance_id in instance_ids:
fout2.write(instance_id + " " + node_name_tag + " " + region_number +
" " + config[region_number][utils.REGION_NAME] + "\n")
finally:
LOCK_FOR_RUN_ONE_REGION.release()
else:
LOGGER.info("Failed to create instances for region %s" % region_number)
number_of_instances -= number_of_creation
tag += 1
def read_region_config(region_config='configuration.txt'):
global CONFIG
config = {}
with open(region_config, 'r') as f:
for myline in f:
mylist = [item.strip() for item in myline.strip().split(',')]
region_num = mylist[0]
config[region_num] = {}
config[region_num][REGION_NAME] = mylist[1]
config[region_num][REGION_KEY] = mylist[2]
config[region_num][REGION_SECURITY_GROUP] = mylist[3]
config[region_num][REGION_HUMAN_NAME] = mylist[4]
config[region_num][REGION_AMI] = mylist[5]
config[region_num][REGION_SECURITY_GROUP_ID] = mylist[6]
CONFIG = config
return config
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description='This script helps you start instances across multiple regions')
parser.add_argument('--regions', type=str, dest='regions',
default='3', help="Supply a csv list of all regions")
parser.add_argument('--instances', type=str, dest='num_instance_list',
default='1', help='number of instances in respective of region')
parser.add_argument('--region_config', type=str,
dest='region_config', default='configuration.txt')
parser.add_argument('--instance_output', type=str, dest='instance_output',
default='instance_output.txt', help='the file to append or write')
parser.add_argument('--instance_ids_output', type=str, dest='instance_ids_output',
default='instance_ids_output.txt', help='the file to append or write')
parser.add_argument('--instance_resource', dest='instance_resource', type=InstanceResource,
default=InstanceResource.ON_DEMAND, choices=list(InstanceResource))
parser.add_argument('--append', dest='append', type=bool, default=False,
help='append to the current instance_output')
args = parser.parse_args()
config = read_region_config(args.region_config)
region_list = args.regions.split(',')
num_instance_list = args.num_instance_list.split(',')
instance_resource = args.instance_resource
assert len(region_list) == len(num_instance_list), "number of regions: %d != number of instances per region: %d" % (
len(region_list), len(num_instance_list))
write_mode = "a" if args.append else "w"
with open(args.instance_output, write_mode) as fout, open(args.instance_ids_output, write_mode) as fout2:
thread_pool = []
for i in range(len(region_list)):
region_number = region_list[i]
number_of_instances = num_instance_list[i]
if instance_resource == InstanceResource.ON_DEMAND:
t = threading.Thread(target=run_for_one_region_on_demand, args=(
config, region_number, number_of_instances, fout, fout2))
elif instance_resource == InstanceResource.SPOT_FLEET:
t = threading.Thread(target=spot_fleet.run_one_region, args=(
region_number, number_of_instances, fout, fout2))
LOGGER.info("creating thread for region %s" % region_number)
t.start()
thread_pool.append(t)
for t in thread_pool:
t.join()
LOGGER.info("done.")

37
aws-experiment-launch/generate_distribution_config.py
Unescape View File

@ -0,0 +1,37 @@
import argparse
import logging
import sys
from utils import utils
logging.basicConfig(level=logging.INFO, format='%(threadName)s %(asctime)s - %(name)s - %(levelname)s - %(message)s')
LOGGER = logging.getLogger(__file__)
LOGGER.setLevel(logging.INFO)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='This script helps you to genereate distribution config')
parser.add_argument('--ip_list_file', type=str, dest='ip_list_file',
default='raw_ip.txt', help="the file containing available raw ips")
# If the ip_list_file is None we need to use the region, node_name_tag and region_config to collect raw_ip
parser.add_argument('--region', type=str, dest='region_number',
default="4", help="region number")
parser.add_argument('--node_name_tag', type=str,
dest='node_name_tag', default='4-NODE-23-36-01-2018-07-05')
parser.add_argument('--region_config', type=str,
dest='region_config', default='configuration.txt')
parser.add_argument('--shard_number', type=int, dest='shard_number', default=1)
parser.add_argument('--client_number', type=int, dest='client_number', default=1)
parser.add_argument('--distribution_config', type=str,
dest='distribution_config', default='distribution_config.txt')
args = parser.parse_args()
if args.ip_list_file == None:
utils.generate_distribution_config2(
args.region_number, args.node_name_tag, args.region_config,
args.shard_number, args.client_number, args.distribution_config)
else:
utils.generate_distribution_config3(args.shard_number, args.client_number,
args.ip_list_file, args.distribution_config)
LOGGER.info("Done writing %s" % args.distribution_config)

1
aws-experiment-launch/logs_download.sh
Unescape View File

@ -0,0 +1 @@
scp -i ../keys/ohio-key-benchmark.pem ec2-user@18.219.217.193:~/projects/src/harmony-benchmark/bin/upload tmp/

105
aws-experiment-launch/terminate_instances.py
Unescape View File

@ -0,0 +1,105 @@
import argparse
import logging
import os
import random
import sys
import threading
import boto3
logging.basicConfig(level=logging.INFO, format='%(threadName)s %(asctime)s - %(name)s - %(levelname)s - %(message)s')
LOGGER = logging.getLogger(__file__)
LOGGER.setLevel(logging.INFO)
REGION_NAME = 'region_name'
REGION_KEY = 'region_key'
REGION_SECURITY_GROUP = 'region_security_group'
REGION_SECURITY_GROUP_ID = 'region_security_group_id'
REGION_HUMAN_NAME = 'region_human_name'
INSTANCE_TYPE = 't2.micro'
REGION_AMI = 'region_ami'
def read_configuration_file(filename):
config = {}
with open(filename, 'r') as f:
for myline in f:
mylist = myline.strip().split(',')
region_num = mylist[0]
config[region_num] = {}
config[region_num][REGION_NAME] = mylist[1]
config[region_num][REGION_KEY] = mylist[2]
config[region_num][REGION_SECURITY_GROUP] = mylist[3]
config[region_num][REGION_HUMAN_NAME] = mylist[4]
config[region_num][REGION_AMI] = mylist[5]
config[region_num][REGION_SECURITY_GROUP_ID] = mylist[6]
return config
def get_instance_ids(describe_instances_response):
instance_ids = []
if describe_instances_response["Reservations"]:
for reservation in describe_instances_response["Reservations"]:
instance_ids.extend(instance["InstanceId"] for instance in reservation["Instances"] if instance.get("InstanceId"))
return instance_ids
def get_instance_ids2(ec2_client, node_name_tag):
time.sleep(5)
filters = [{'Name': 'tag:Name','Values': [node_name_tag]}]
return get_instance_ids(ec2_client.describe_instances(Filters=filters))
def create_ec2_client(region_number, region_config):
config = read_configuration_file(region_config)
region_name = config[region_number][REGION_NAME]
session = boto3.Session(region_name=region_name)
return session.client('ec2'), session
def terminate_instances_by_region(region_number, region_config, node_name_tag):
ec2_client, _ = create_ec2_client(region_number, region_config)
filters = [{'Name': 'tag:Name','Values': [node_name_tag]}]
instance_ids = get_instance_ids(ec2_client.describe_instances(Filters=filters))
if instance_ids:
ec2_client.terminate_instances(InstanceIds=instance_ids)
LOGGER.info("waiting until instances with tag %s died." % node_name_tag)
waiter = ec2_client.get_waiter('instance_terminated')
waiter.wait(InstanceIds=instance_ids)
LOGGER.info("instances with node name tag %s terminated." % node_name_tag)
else:
pass
LOGGER.warn("there is no instances to terminate")
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='This script helps you to collect public ips')
parser.add_argument('--instance_output', type=str, dest='instance_output',
default='instance_output.txt',
help='the file contains node_name_tag and region number of created instances.')
parser.add_argument('--node_name_tag', type=str, dest='node_name_tag')
parser.add_argument('--region_config', type=str,
dest='region_config', default='configuration.txt')
args = parser.parse_args()
if args.node_name_tag:
node_name_tag_items = args.node_name_tag.split(",")
region_number_items = [item[:1] for item in node_name_tag_items]
thread_pool = []
for i in range(len(region_number_items)):
region_number = region_number_items[i]
node_name_tag = node_name_tag_items[i]
t = threading.Thread(target=terminate_instances_by_region, args=(region_number, args.region_config, node_name_tag))
t.start()
thread_pool.append(t)
for t in thread_pool:
t.join()
LOGGER.info("done.")
elif args.instance_output:
with open(args.instance_output, "r") as fin:
thread_pool = []
for line in fin.readlines():
items = line.split(" ")
region_number = items[1].strip()
node_name_tag = items[0].strip()
t = threading.Thread(target=terminate_instances_by_region, args=(region_number, args.region_config, node_name_tag))
t.start()
thread_pool.append(t)
for t in thread_pool:
t.join()
LOGGER.info("done.")

15
aws-experiment-launch/upload_binaries.py
Unescape View File

@ -0,0 +1,15 @@
import boto3
import os
GOHOME ='/Users/alok/Documents/goworkspace/'
s3 = boto3.client('s3')
bucket_name = 'unique-bucket-bin'
#response = s3.create_bucket(Bucket=bucket_name,ACL='public-read-write', CreateBucketConfiguration={'LocationConstraint': 'us-west-2'})
response = s3.list_buckets()
buckets = [bucket['Name'] for bucket in response['Buckets']]
print("Bucket List: %s" % buckets)
dirname = GOHOME + 'src/harmony-benchmark/bin/'
for myfile in os.listdir(dirname):
with open('distribution_config.txt','r') as f:
f = open(os.path.join(dirname,myfile))
response = s3.put_object(ACL='public-read-write',Body=f.read(),Bucket=bucket_name,Key=myfile)
print(response)

10
aws-experiment-launch/upload_config.py
Unescape View File

@ -0,0 +1,10 @@
import boto3
import os
#from boto3.session import Session
s3 = boto3.client('s3')
bucket_name = 'unique-bucket-bin'
#response = s3.create_bucket(Bucket=bucket_name,ACL='public-read-write', CreateBucketConfiguration={'LocationConstraint': 'us-west-2'})
myfile = 'distribution_config.txt'
with open('distribution_config.txt','r') as f:
response = s3.put_object(ACL='public-read-write',Body=f.read(),Bucket=bucket_name,Key=myfile)
print(response)

9
aws-experiment-launch/upload_s3.py
Unescape View File

@ -0,0 +1,9 @@
import boto3
s3 = boto3.client('s3')
bucket_name = 'first-try'
s3.create_bucket(Bucket=bucket_name,ACL='public-read-write')
response = s3.list_buckets()
buckets = [bucket['Name'] for bucket in response['Buckets']]
print("Bucket List: %s" % buckets)
filename='myfirst.txt'
s3.upload_file(filename, bucket_name, filename)

11
aws-experiment-launch/userdata-commander.sh
Unescape View File

@ -0,0 +1,11 @@
#!/bin/bash -x
REGION=$(curl 169.254.169.254/latest/meta-data/placement/availability-zone/ | sed 's/[a-z]$//')
#yum update -y #This breaking codedeploy right now
yum install ruby wget -y
cd /home/ec2-user
touch yum-not-updated.txt
wget https://aws-codedeploy-$REGION.s3.amazonaws.com/latest/install
chmod +x ./install
./install auto
mkdir projects
mkdir projects/src

23
aws-experiment-launch/userdata-soldier.sh
Unescape View File

@ -0,0 +1,23 @@
#!/bin/bash
yum install ruby -y
cd /home/ec2-user/
curl http://unique-bucket-bin.s3.amazonaws.com/txgen -o txgen
curl http://unique-bucket-bin.s3.amazonaws.com/soldier -o soldier
curl http://unique-bucket-bin.s3.amazonaws.com/commander -o commander
curl http://unique-bucket-bin.s3.amazonaws.com/benchmark -o benchmark
curl http://unique-bucket-bin.s3.amazonaws.com/kill_node.sh -o kill_node.sh
chmod +x ./soldier
chmod +x ./txgen
chmod +x ./benchmark
chmod +x ./commander
chmod +x ./kill_node.sh
# Get My IP
ip=`curl http://169.254.169.254/latest/meta-data/public-ipv4`
SOLDIER_PORT=9000
# Kill existing soldier
fuser -k -n tcp $SOLDIER_PORT
# Run soldier
./soldier -ip $ip -port $SOLDIER_PORT > soldier_log 2>&1 &
Write Preview
Loading…
Cancel
Save