move code from new-pipeline

aws-experiment-launch/collect_public_ips.py

@@ -0,0 +1,35 @@
+import argparse
+import os
+import random
+import sys
+
+from utils import utils
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description='This script helps you to collect public ips')
+    parser.add_argument('--instance_output', type=str, dest='instance_output',
+                        default='instance_output.txt',
+                        help='the file contains node_name_tag and region number of created instances.')
+    parser.add_argument('--region_config', type=str,
+                        dest='region_config', default='configuration.txt')
+    parser.add_argument('--file_output', type=str,
+                        dest='file_output', default='raw_ip.txt')
+    args = parser.parse_args()
+
+    if not args.instance_output or not os.path.isfile(args.instance_output):
+        print "%s or %s are not existed" % (args.file_output, args.instance_output)
+        sys.exit(1)
+    if args.instance_output:
+        with open(args.instance_output, "r") as fin, open(args.file_output, "w") as fout:
+            total_ips = []
+            node_name_tag_list = []
+            for line in fin.readlines():
+                items = line.split(" ")
+                region_number = items[1].strip()
+                node_name_tag = items[0].strip()
+                ip_list = utils.collect_public_ips(region_number, node_name_tag, args.region_config)
+                total_ips.extend([(ip, node_name_tag) for ip in ip_list])
+            random.shuffle(total_ips)
+            for tuple in total_ips:
+                fout.write(tuple[0] + " " + tuple[1] + "\n")
+        print "Done collecting public ips %s" % args.file_output

aws-experiment-launch/commander_prepare.py

@@ -0,0 +1,62 @@
+import argparse
+import logging
+import os
+import stat
+import sys
+
+from utils import utils
+
+logging.basicConfig(level=logging.INFO, format='%(threadName)s %(asctime)s - %(name)s - %(levelname)s - %(message)s')
+LOGGER = logging.getLogger(__file__)
+LOGGER.setLevel(logging.INFO)
+
+PEMS = [
+    "virginia-key-benchmark.pem",
+    "ohio-key-benchmark.pem",
+    "california-key-benchmark.pem",
+    "oregon-key-benchmark.pem",
+    "tokyo-key-benchmark.pem",
+    "singapore-key-benchmark.pem",
+    "frankfurt-key-benchmark.pem",
+    "ireland-key-benchmark.pem",
+]
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description='This script helps you to genereate distribution config')
+    parser.add_argument('--distribution_config', type=str,
+                        dest='distribution_config', default='distribution_config.txt')
+    parser.add_argument('--commander_logging', type=str,
+                        dest='commander_logging', default='commander_logging.sh')
+
+    args = parser.parse_args()
+
+    if not os.path.exists(args.distribution_config):
+        sys.exit(1)
+    with open(args.distribution_config, "r") as fin:
+        lines = fin.readlines()
+
+    commander_address = None
+    commander_region = None
+    commander_output = None
+    with open(args.distribution_config, "w") as fout:
+        for line in lines:
+            if "commander" in line:
+                items = [item.strip() for item in line.split(" ")]
+                commander_address = items[0]
+                commander_region = int(items[4][0])
+                commander_output = "\n".join(items)
+            else:
+                fout.write(line.strip() + "\n")
+    LOGGER.info("Generated %s" % args.distribution_config)
+
+    if not commander_address or not commander_region:
+        LOGGER.info("Failed to extract commander address and commander region.")
+        sys.exit(1)
+
+    with open(args.commander_logging, "w") as fout:
+        fout.write("ssh -i ../keys/%s ec2-user@%s\n" % (PEMS[commander_region - 1], commander_address))
+    st = os.stat(args.commander_logging)
+    os.chmod(args.commander_logging, st.st_mode | stat.S_IEXEC)
+    LOGGER.info("Generated %s" % args.commander_logging)
+    LOGGER.info("DONE.")
+

aws-experiment-launch/configuration-git.txt

@@ -0,0 +1,8 @@
+1,us-east-1,virginia-key-benchmark,virginia-security-group,virginia,ami-97785bed,sg-04d0b62ee08ce8800
+2,us-east-2,ohio-key-benchmark,ohio-security-group,ohio,ami-f63b1193,sg-0789078f1c76defbe
+3,us-west-1,california-key-benchmark,california-security-group,california,ami-824c4ee2,sg-0a66ccb6ab9161a14
+4,us-west-2,oregon-key-benchmark,oregon-security-group,oregon,ami-f2d3638a,sg-020cb5729fa212d43
+5,ap-northeast-1,tokyo-key-benchmark,tokyo-security-group,tokyo,ami-ceafcba8,sg-009aeb97f675c1ad5
+6,ap-southeast-1,singapore-key-benchmark,singapore-security-group,singapore,ami-68097514,sg-05f9b60044a19dfb2
+7,eu-central-1,frankfurt-key-benchmark,frankfurt-security-group,frankfurt,ami-5652ce39,sg-0bb06fcd8b25b5910
+8,eu-west-1,ireland-key-benchmark,ireland-security-group,ireland,ami-d834aba1,sg-0aa8954acb79fdb58

aws-experiment-launch/create_deploy_soldiers.sh

@@ -0,0 +1,28 @@
+if [ $# -lt 3 ]; then
+    echo "Please provide # of instances, # of shards, # of clients"
+    exit 1
+fi
+
+INSTANCE_NUM=$1
+SHARD_NUM=$2
+CLIENT_NUM=$3
+SLEEP_TIME=10
+
+echo "Creating $INSTANCE_NUM instances at 8 regions"
+python create_solider_instances.py --regions 1,2,3,4,5,6,7,8 --instances $INSTANCE_NUM,$INSTANCE_NUM,$INSTANCE_NUM,$INSTANCE_NUM,$INSTANCE_NUM,$INSTANCE_NUM,$INSTANCE_NUM,$INSTANCE_NUM
+
+
+echo "Sleep for $SLEEP_TIME seconds"
+sleep $SLEEP_TIME
+
+echo "Rung collecint raw ips"
+python collect_public_ips.py --instance_output instance_output.txt
+
+# sleep 10
+echo "Generate distribution_config"
+python generate_distribution_config.py --ip_list_file raw_ip.txt --shard_number $SHARD_NUM --client_number $CLIENT_NUM
+
+echo "Run commander prepare"
+python commander_prepare.py
+
+aws s3 cp distribution_config.txt s3://unique-bucket-bin/distribution_config.txt --acl public-read-write

aws-experiment-launch/create_solider_instances.py

@@ -0,0 +1,193 @@
+import argparse
+import base64
+import boto3
+import datetime
+import json
+import sys
+import threading
+import time
+import enum
+
+#TODO REMOVE UTILS 
+from utils import utils, spot_fleet, logger
+
+LOGGER = logger.getLogger(__file__)
+REGION_NAME = 'region_name'
+REGION_KEY = 'region_key'
+REGION_SECURITY_GROUP = 'region_security_group'
+REGION_SECURITY_GROUP_ID = 'region_security_group_id'
+REGION_HUMAN_NAME = 'region_human_name'
+INSTANCE_TYPE = 't2.micro'
+REGION_AMI = 'region_ami'
+
+class InstanceResource(enum.Enum):
+    ON_DEMAND = 'ondemand'
+    SPOT_INSTANCE = 'spot'
+    SPOT_FLEET = 'fleet'
+
+    def __str__(self):
+        return self.value
+
+def run_one_region_on_demand_instances(config, region_number, number_of_instances, tag):
+    ec2_client = utils.create_client(config, region_number)
+    node_name_tag = create_instances(
+        config, ec2_client, region_number, number_of_instances, tag)
+    LOGGER.info("Created %s in region %s" % (node_name_tag, region_number))
+    return node_name_tag, ec2_client
+
+def create_instances(config, ec2_client, region_number, number_of_instances, tag):
+    node_name_tag = utils.get_node_name_tag2(region_number, tag)
+    LOGGER.info("Creating node_name_tag: %s" % node_name_tag)
+    available_zone = utils.get_one_availability_zone(ec2_client)
+    LOGGER.info("Looking at zone %s to create instances." % available_zone)
+
+    time.sleep(2)
+    ec2_client.run_instances(
+        MinCount=number_of_instances,
+        MaxCount=number_of_instances,
+        ImageId=config[region_number][utils.REGION_AMI],
+        Placement={
+            'AvailabilityZone': available_zone,
+        },
+        SecurityGroups=[config[region_number][utils.REGION_SECURITY_GROUP]],
+        IamInstanceProfile={
+            'Name': utils.IAM_INSTANCE_PROFILE
+        },
+        KeyName=config[region_number][utils.REGION_KEY],
+        UserData=utils.USER_DATA,
+        InstanceType=utils.INSTANCE_TYPE,
+        TagSpecifications=[
+            {
+                'ResourceType': 'instance',
+                'Tags': [
+                    {
+                        'Key': 'Name',
+                        'Value': node_name_tag
+                    },
+                ]
+            },
+        ],
+    )
+
+    retry_count = 10
+    while retry_count > 0:
+        try:
+            time.sleep(20)
+            instance_ids = utils.get_instance_ids2(ec2_client, node_name_tag)
+            LOGGER.info("Waiting for all %d instances in region %s with node_name_tag %s to be in RUNNING" % (
+                len(instance_ids), region_number, node_name_tag))
+            break
+        except:
+            retry_count -= 1
+            LOGGER.info("Failed to get instance ids. Retry again.")
+    retry_count = 10
+    while retry_count > 0:
+        try:
+            time.sleep(20)
+            waiter = ec2_client.get_waiter('instance_running')
+            waiter.wait(InstanceIds=instance_ids)
+            break
+        except:
+            retry_count -= 1
+            LOGGER.info("Failed to wait.")
+
+    retry_count = 10
+    while retry_count > 0:
+        time.sleep(10)
+        LOGGER.info("Waiting ...")
+        ip_list = utils.collect_public_ips_from_ec2_client(
+            ec2_client, node_name_tag)
+        if len(ip_list) == number_of_instances:
+            LOGGER.info("Created %d instances" % number_of_instances)
+            return node_name_tag
+        retry_count -= 10
+    LOGGER.info("Can not create %d instances" % number_of_instances)
+    return None
+
+
+LOCK_FOR_RUN_ONE_REGION = threading.Lock()
+
+def run_for_one_region_on_demand(config, region_number, number_of_instances, fout, fout2):
+    tag = 0
+    number_of_instances = int(number_of_instances)
+    while number_of_instances > 0:
+        number_of_creation = min(utils.MAX_INSTANCES_FOR_DEPLOYMENT, number_of_instances)
+        node_name_tag, ec2_client = run_one_region_on_demand_instances(
+            config, region_number, number_of_creation, tag)
+        if node_name_tag:
+            LOGGER.info("Managed to create instances for region %s with name_name_tag %s" %
+                        (region_number, node_name_tag))
+            instance_ids = utils.get_instance_ids2(ec2_client, node_name_tag)
+            LOCK_FOR_RUN_ONE_REGION.acquire()
+            try:
+                fout.write("%s %s\n" % (node_name_tag, region_number))
+                for instance_id in instance_ids:
+                    fout2.write(instance_id + " " + node_name_tag + " " + region_number +
+                                " " + config[region_number][utils.REGION_NAME] + "\n")
+            finally:
+                LOCK_FOR_RUN_ONE_REGION.release()
+        else:
+            LOGGER.info("Failed to create instances for region %s" % region_number)
+        number_of_instances -= number_of_creation
+        tag += 1
+
+def read_region_config(region_config='configuration.txt'):
+    global CONFIG
+    config = {}
+    with open(region_config, 'r') as f:
+        for myline in f:
+            mylist = [item.strip() for item in myline.strip().split(',')]
+            region_num = mylist[0]
+            config[region_num] = {}
+            config[region_num][REGION_NAME] = mylist[1]
+            config[region_num][REGION_KEY] = mylist[2]
+            config[region_num][REGION_SECURITY_GROUP] = mylist[3]
+            config[region_num][REGION_HUMAN_NAME] = mylist[4]
+            config[region_num][REGION_AMI] = mylist[5]
+            config[region_num][REGION_SECURITY_GROUP_ID] = mylist[6]
+    CONFIG = config
+    return config
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description='This script helps you start instances across multiple regions')
+    parser.add_argument('--regions', type=str, dest='regions',
+                        default='3', help="Supply a csv list of all regions")
+    parser.add_argument('--instances', type=str, dest='num_instance_list',
+                        default='1', help='number of instances in respective of region')
+    parser.add_argument('--region_config', type=str,
+                        dest='region_config', default='configuration.txt')
+    parser.add_argument('--instance_output', type=str, dest='instance_output',
+                        default='instance_output.txt', help='the file to append or write')
+    parser.add_argument('--instance_ids_output', type=str, dest='instance_ids_output',
+                        default='instance_ids_output.txt', help='the file to append or write')
+    parser.add_argument('--instance_resource', dest='instance_resource', type=InstanceResource,
+                        default=InstanceResource.ON_DEMAND, choices=list(InstanceResource))
+    parser.add_argument('--append', dest='append', type=bool, default=False,
+                        help='append to the current instance_output')
+    args = parser.parse_args()
+    config = read_region_config(args.region_config)
+    region_list = args.regions.split(',')
+    num_instance_list = args.num_instance_list.split(',')
+    instance_resource = args.instance_resource
+    assert len(region_list) == len(num_instance_list), "number of regions: %d != number of instances per region: %d" % (
+        len(region_list), len(num_instance_list))
+
+    write_mode = "a" if args.append else "w"
+    with open(args.instance_output, write_mode) as fout, open(args.instance_ids_output, write_mode) as fout2:
+        thread_pool = []
+        for i in range(len(region_list)):
+            region_number = region_list[i]
+            number_of_instances = num_instance_list[i]
+            if instance_resource == InstanceResource.ON_DEMAND:
+                t = threading.Thread(target=run_for_one_region_on_demand, args=(
+                    config, region_number, number_of_instances, fout, fout2))
+            elif instance_resource == InstanceResource.SPOT_FLEET:
+                t = threading.Thread(target=spot_fleet.run_one_region, args=(
+                    region_number, number_of_instances, fout, fout2))
+            LOGGER.info("creating thread for region %s" % region_number)
+            t.start()
+            thread_pool.append(t)
+        for t in thread_pool:
+            t.join()
+        LOGGER.info("done.")

aws-experiment-launch/generate_distribution_config.py

@@ -0,0 +1,37 @@
+import argparse
+import logging
+import sys
+
+from utils import utils
+
+logging.basicConfig(level=logging.INFO, format='%(threadName)s %(asctime)s - %(name)s - %(levelname)s - %(message)s')
+LOGGER = logging.getLogger(__file__)
+LOGGER.setLevel(logging.INFO)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description='This script helps you to genereate distribution config')
+    parser.add_argument('--ip_list_file', type=str, dest='ip_list_file',
+                        default='raw_ip.txt', help="the file containing available raw ips")
+    # If the ip_list_file is None we need to use the region, node_name_tag and region_config to collect raw_ip                        
+    parser.add_argument('--region', type=str, dest='region_number',
+                        default="4", help="region number")
+    parser.add_argument('--node_name_tag', type=str,
+                        dest='node_name_tag', default='4-NODE-23-36-01-2018-07-05')
+    parser.add_argument('--region_config', type=str,
+                        dest='region_config', default='configuration.txt')
+
+    parser.add_argument('--shard_number', type=int, dest='shard_number', default=1)
+    parser.add_argument('--client_number', type=int, dest='client_number', default=1)
+    parser.add_argument('--distribution_config', type=str,
+                        dest='distribution_config', default='distribution_config.txt')
+    args = parser.parse_args()
+
+    if args.ip_list_file == None:
+        utils.generate_distribution_config2(
+                args.region_number, args.node_name_tag, args.region_config,
+                args.shard_number, args.client_number, args.distribution_config)
+    else:
+        utils.generate_distribution_config3(args.shard_number, args.client_number,
+                                            args.ip_list_file, args.distribution_config)
+    LOGGER.info("Done writing %s" % args.distribution_config)

aws-experiment-launch/logs_download.sh

@@ -0,0 +1 @@
+scp -i ../keys/ohio-key-benchmark.pem ec2-user@18.219.217.193:~/projects/src/harmony-benchmark/bin/upload tmp/

aws-experiment-launch/terminate_instances.py

@@ -0,0 +1,105 @@
+import argparse
+import logging
+import os
+import random
+import sys
+import threading
+import boto3
+
+
+
+logging.basicConfig(level=logging.INFO, format='%(threadName)s %(asctime)s - %(name)s - %(levelname)s - %(message)s')
+LOGGER = logging.getLogger(__file__)
+LOGGER.setLevel(logging.INFO)
+
+REGION_NAME = 'region_name'
+REGION_KEY = 'region_key'
+REGION_SECURITY_GROUP = 'region_security_group'
+REGION_SECURITY_GROUP_ID = 'region_security_group_id'
+REGION_HUMAN_NAME = 'region_human_name'
+INSTANCE_TYPE = 't2.micro'
+REGION_AMI = 'region_ami'
+
+def read_configuration_file(filename):
+    config = {}
+    with open(filename, 'r') as f:
+        for myline in f:
+            mylist = myline.strip().split(',')
+            region_num = mylist[0]
+            config[region_num] = {}
+            config[region_num][REGION_NAME] = mylist[1]
+            config[region_num][REGION_KEY] = mylist[2]
+            config[region_num][REGION_SECURITY_GROUP] = mylist[3]
+            config[region_num][REGION_HUMAN_NAME] = mylist[4]
+            config[region_num][REGION_AMI] = mylist[5]
+            config[region_num][REGION_SECURITY_GROUP_ID] = mylist[6]
+    return config
+
+def get_instance_ids(describe_instances_response):
+    instance_ids = []
+    if describe_instances_response["Reservations"]:
+        for reservation in describe_instances_response["Reservations"]:
+            instance_ids.extend(instance["InstanceId"] for instance in reservation["Instances"] if instance.get("InstanceId"))
+    return instance_ids
+    
+def get_instance_ids2(ec2_client, node_name_tag):
+    time.sleep(5)
+    filters = [{'Name': 'tag:Name','Values': [node_name_tag]}]
+    return get_instance_ids(ec2_client.describe_instances(Filters=filters))
+
+def create_ec2_client(region_number, region_config):
+    config = read_configuration_file(region_config)
+    region_name = config[region_number][REGION_NAME]
+    session = boto3.Session(region_name=region_name)
+    return session.client('ec2'), session
+
+def terminate_instances_by_region(region_number, region_config, node_name_tag):
+    ec2_client, _ = create_ec2_client(region_number, region_config)
+    filters = [{'Name': 'tag:Name','Values': [node_name_tag]}]
+    instance_ids = get_instance_ids(ec2_client.describe_instances(Filters=filters))
+    if instance_ids:
+        ec2_client.terminate_instances(InstanceIds=instance_ids)
+        LOGGER.info("waiting until instances with tag %s died." % node_name_tag)
+        waiter = ec2_client.get_waiter('instance_terminated')
+        waiter.wait(InstanceIds=instance_ids)
+        LOGGER.info("instances with node name tag %s terminated." % node_name_tag)
+    else:
+        pass
+        LOGGER.warn("there is no instances to terminate")
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description='This script helps you to collect public ips')
+    parser.add_argument('--instance_output', type=str, dest='instance_output',
+                        default='instance_output.txt',
+                        help='the file contains node_name_tag and region number of created instances.')
+    parser.add_argument('--node_name_tag', type=str, dest='node_name_tag')
+    parser.add_argument('--region_config', type=str,
+                        dest='region_config', default='configuration.txt')
+    args = parser.parse_args()
+
+    if args.node_name_tag:
+        node_name_tag_items = args.node_name_tag.split(",")
+        region_number_items = [item[:1] for item in node_name_tag_items]
+        thread_pool = []
+        for i in range(len(region_number_items)):
+            region_number = region_number_items[i]
+            node_name_tag = node_name_tag_items[i]
+            t = threading.Thread(target=terminate_instances_by_region, args=(region_number, args.region_config, node_name_tag))
+            t.start()
+            thread_pool.append(t)
+        for t in thread_pool:
+            t.join()
+        LOGGER.info("done.")
+    elif args.instance_output:
+        with open(args.instance_output, "r") as fin:
+            thread_pool = []
+            for line in fin.readlines():
+                items = line.split(" ")
+                region_number = items[1].strip()
+                node_name_tag = items[0].strip()
+                t = threading.Thread(target=terminate_instances_by_region, args=(region_number, args.region_config, node_name_tag))
+                t.start()
+                thread_pool.append(t)
+            for t in thread_pool:
+                t.join()
+            LOGGER.info("done.")

aws-experiment-launch/upload_binaries.py

@@ -0,0 +1,15 @@
+import boto3
+import os
+GOHOME ='/Users/alok/Documents/goworkspace/'
+s3 = boto3.client('s3')
+bucket_name = 'unique-bucket-bin'
+#response = s3.create_bucket(Bucket=bucket_name,ACL='public-read-write', CreateBucketConfiguration={'LocationConstraint': 'us-west-2'})
+response = s3.list_buckets()
+buckets = [bucket['Name'] for bucket in response['Buckets']]
+print("Bucket List: %s" % buckets)
+dirname = GOHOME + 'src/harmony-benchmark/bin/'
+for myfile in os.listdir(dirname):
+    with open('distribution_config.txt','r') as f:
+        f = open(os.path.join(dirname,myfile))
+        response = s3.put_object(ACL='public-read-write',Body=f.read(),Bucket=bucket_name,Key=myfile)
+        print(response)

aws-experiment-launch/upload_config.py

@@ -0,0 +1,10 @@
+import boto3
+import os
+#from boto3.session import Session
+s3 = boto3.client('s3')
+bucket_name = 'unique-bucket-bin'
+#response = s3.create_bucket(Bucket=bucket_name,ACL='public-read-write', CreateBucketConfiguration={'LocationConstraint': 'us-west-2'})
+myfile = 'distribution_config.txt'
+with open('distribution_config.txt','r') as f:
+    response  = s3.put_object(ACL='public-read-write',Body=f.read(),Bucket=bucket_name,Key=myfile)
+    print(response)

aws-experiment-launch/upload_s3.py

@@ -0,0 +1,9 @@
+import boto3
+s3 = boto3.client('s3')
+bucket_name = 'first-try'
+s3.create_bucket(Bucket=bucket_name,ACL='public-read-write')
+response = s3.list_buckets()
+buckets = [bucket['Name'] for bucket in response['Buckets']]
+print("Bucket List: %s" % buckets)
+filename='myfirst.txt'
+s3.upload_file(filename, bucket_name, filename)

aws-experiment-launch/userdata-commander.sh

@@ -0,0 +1,11 @@
+#!/bin/bash -x
+REGION=$(curl 169.254.169.254/latest/meta-data/placement/availability-zone/ | sed 's/[a-z]$//')
+#yum update -y  #This breaking codedeploy right now
+yum install ruby wget -y
+cd /home/ec2-user
+touch yum-not-updated.txt
+wget https://aws-codedeploy-$REGION.s3.amazonaws.com/latest/install
+chmod +x ./install
+./install auto
+mkdir projects
+mkdir projects/src

aws-experiment-launch/userdata-soldier.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+yum install ruby -y
+cd /home/ec2-user/
+curl http://unique-bucket-bin.s3.amazonaws.com/txgen -o txgen
+curl http://unique-bucket-bin.s3.amazonaws.com/soldier -o soldier
+curl http://unique-bucket-bin.s3.amazonaws.com/commander -o commander
+curl http://unique-bucket-bin.s3.amazonaws.com/benchmark -o benchmark
+curl http://unique-bucket-bin.s3.amazonaws.com/kill_node.sh -o kill_node.sh
+chmod +x ./soldier
+chmod +x ./txgen
+chmod +x ./benchmark
+chmod +x ./commander
+chmod +x ./kill_node.sh
+
+# Get My IP
+ip=`curl http://169.254.169.254/latest/meta-data/public-ipv4`
+
+SOLDIER_PORT=9000
+# Kill existing soldier
+fuser -k -n tcp $SOLDIER_PORT
+
+# Run soldier
+./soldier -ip $ip -port $SOLDIER_PORT > soldier_log 2>&1 &