unblocked wait failure

aws-experiment-launch/create_and_deploy.py

@@ -30,8 +30,7 @@ NODE_NAME_SUFFIX = "NODE-" + CURRENT_SESSION
 
 """
 TODO:
-NODE = region_number + NODE_VALUE
-Use that to retrieve ids, only deploy on specific nodes (right now it deploys everywere), remove placement group. 
+
 save NODE to disk, so that you can selectively only run deploy (not recreate instances). 
 Right now all instances have "NODE" so this has uninted consequences of running on instances that were previous created.
 Build (argparse,functions) support for 
@@ -53,8 +52,9 @@ def run_one_region_instances(config, region_number, number_of_instances, isOnDem
     session = boto3.Session(region_name=region_name)
     ec2_client = session.client('ec2')
     if isOnDemand:
-        response = create_instances(
+        NODE_NAME = create_instances(
             config, ec2_client, region_number, int(number_of_instances))
+        print("Created %s in region %s"%(NODE_NAME,region_number))  ##REPLACE ALL print with logger
     else:
         response = request_spots(
             config, ec2_client, region_number, int(number_of_instances))
@@ -89,7 +89,7 @@ def create_instances(config, ec2_client, region_number, number_of_instances):
             },
         ]
     )
-    return response
+    return NODE_NAME
 
 
 def request_spots(config, ec2_client, region_number, number_of_instances):
@@ -159,38 +159,38 @@ def run_one_region_codedeploy(region_number, commitId):
     NODE_NAME = region_number + "-" + NODE_NAME_SUFFIX
     session = boto3.Session(region_name=region_name)
     ec2_client = session.client('ec2')
-    filters = [
-        {
-            'Name': 'tag:Name',
-            'Values': [NODE_NAME]
-        }
-    ]
+    filters = [{'Name': 'tag:Name','Values': [NODE_NAME]}]
+    instance_ids = get_instance_ids(ec2_client.describe_instances(Filters=filters))
+    
+    print("Number of instances: %d" % len(instance_ids))
 
-    print("Waiting for all instances to start running")
+    print("Waiting for all %d instances in region %s to start running"%(len(instance_ids),region_number))
     waiter = ec2_client.get_waiter('instance_running')
-    waiter.wait(Filters=filters)
+    waiter.wait(InstanceIds=instance_ids)
 
-    print("Waiting for all instances to be status ok")
+    print("Waiting for all %d instances in region %s to be INSTANCE STATUS OK"%(len(instance_ids),region_number))
     waiter = ec2_client.get_waiter('instance_status_ok')
-    waiter.wait(Filters=filters)
+    waiter.wait(InstanceIds=instance_ids)
 
-    print("Waiting for system to be status ok")
+    print("Waiting for all %d instances in region %s to be SYSTEM STATUS OK"%(len(instance_ids),region_number))
     waiter = ec2_client.get_waiter('system_status_ok')
-    waiter.wait(Filters=filters)
+    waiter.wait(InstanceIds=instance_ids)
 
     codedeploy = session.client('codedeploy')
     application_name = APPLICATION_NAME
-    deployment_group = APPLICATION_NAME + "-" + str(commitId)
+    deployment_group = APPLICATION_NAME + "-" + str(commitId) + "-1"
     repo = REPO
+
+    print("Setting up to deploy commitId %s on region %s"%(commitId,region_number))
     response = get_application(codedeploy, application_name)
     response = get_deployment_group(
-        codedeploy, application_name, deployment_group)
+        codedeploy, region_number, application_name, deployment_group)
     depId = deploy(codedeploy, application_name,
                    deployment_group, repo, commitId)
     return region_number, depId
 
 
-def get_deployment_group(codedeploy, application_name, deployment_group):
+def get_deployment_group(codedeploy, region_number,application_name, deployment_group):
     NODE_NAME = region_number + "-" + NODE_NAME_SUFFIX
     response = codedeploy.list_deployment_groups(
         applicationName=application_name
@@ -207,17 +207,13 @@ def get_deployment_group(codedeploy, application_name, deployment_group):
                 'deploymentType': 'IN_PLACE',
                 'deploymentOption': 'WITHOUT_TRAFFIC_CONTROL'
             },
-            ec2TagSet={
-                'ec2TagSetList': [
-                    [
+            ec2TagFilters = [
                 {
                     'Key': 'Name',
                     'Value': NODE_NAME,
                     'Type': 'KEY_AND_VALUE'
-                        },
-                    ],
-                ]
                 }
+            ]
         )
         return response
 
@@ -274,12 +270,10 @@ def deploy(codedeploy, application_name, deployment_group, repo, commitId):
         print(info)
         return depId
 
-
 def run_one_region_codedeploy_wrapper(region_number, commitId, queue):
     region_number, depId = run_one_region_codedeploy(region_number, commitId)
     queue.put((region_number, depId))
 
-
 def launch_code_deploy(region_list, commitId):
     queue = Queue()
     jobs = []
@@ -304,7 +298,6 @@ def get_instance_ids(describe_instances_response):
             instance_ids.append(instance["InstanceId"])
     return instance_ids
 
-
 def read_configuration_file(filename):
     config = {}
     with open(filename, 'r') as f:
@@ -319,7 +312,6 @@ def read_configuration_file(filename):
             config[region_num][REGION_AMI] = mylist[5]
     return config
 
-
 def get_commitId(commitId):
     if commitId is None:
         commitId = run("git rev-list --max-count=1 HEAD",