diff --git a/aws-experiment-launch/new-pipeline/create_solider_instances.py b/aws-experiment-launch/new-pipeline/create_solider_instances.py index e1f3edaec..bdcdcbd09 100644 --- a/aws-experiment-launch/new-pipeline/create_solider_instances.py +++ b/aws-experiment-launch/new-pipeline/create_solider_instances.py @@ -41,6 +41,8 @@ def create_instances(config, ec2_client, region_number, number_of_instances, tag available_zone = utils.get_one_availability_zone(ec2_client) LOGGER.info("Looking at zone %s to create instances." % available_zone) + + LOGGER.info("USER_DATA: \n%s" % utils\) time.sleep(2) ec2_client.run_instances( MinCount=number_of_instances, @@ -68,13 +70,24 @@ def create_instances(config, ec2_client, region_number, number_of_instances, tag }, ], ) - time.sleep(30) - instance_ids = utils.get_instance_ids2(ec2_client, node_name_tag) - LOGGER.info("Waiting for all %d instances in region %s with node_name_tag %s to be in RUNNING" % ( - len(instance_ids), region_number, node_name_tag)) - time.sleep(20) - waiter = ec2_client.get_waiter('instance_running') - waiter.wait(InstanceIds=instance_ids) + + while True: + try: + time.sleep(10) + instance_ids = utils.get_instance_ids2(ec2_client, node_name_tag) + LOGGER.info("Waiting for all %d instances in region %s with node_name_tag %s to be in RUNNING" % ( + len(instance_ids), region_number, node_name_tag)) + break + except: + LOGGER.info("Failed to get instance ids. Retry again.") + while True: + try: + time.sleep(20) + waiter = ec2_client.get_waiter('instance_running') + waiter.wait(InstanceIds=instance_ids) + break + except: + LOGGER.info("Failed to wait.") count = 0 while count < 40: diff --git a/aws-experiment-launch/new-pipeline/userdata-soldier.sh b/aws-experiment-launch/new-pipeline/userdata-soldier.sh index 7436bd578..f9b42d9bb 100644 --- a/aws-experiment-launch/new-pipeline/userdata-soldier.sh +++ b/aws-experiment-launch/new-pipeline/userdata-soldier.sh @@ -1,20 +1,19 @@ #!/bin/bash -cd /home/ec2-user -commanderIP=52.53.162.27 # <- Put the commander IP here. -curl http://$commanderIP:8080/soldier -o soldier +yum install ruby wget -y +cd $HOME +wget http://unique-bucket-bin.s3.amazonaws.com/txgen +wget http://unique-bucket-bin.s3.amazonaws.com/soldier +wget http://unique-bucket-bin.s3.amazonaws.com/benchmark chmod +x ./soldier -curl http://$commanderIP:8080/benchmark -o benchmark -chmod +x ./benchmark -curl http://$commanderIP:8080/txgen -o txgen chmod +x ./txgen +chmod +x ./benchmark # Get My IP ip=`curl http://169.254.169.254/latest/meta-data/public-ipv4` -node_port=9000 -soldier_port=1$node_port +SOLDIER_PORT=9000 # Kill existing soldier -fuser -k -n tcp $soldier_port +fuser -k -n tcp $SOLDIER_PORT # Run soldier -./soldier -ip $ip -port $node_port > soldier_log 2>&1 & \ No newline at end of file +./soldier -ip $ip -port $SOLDIER_PORT > soldier_log 2>&1 & \ No newline at end of file