Merge pull request #28 from simple-rules/ricl-aws

Ricl aws

aws-experiment-launch/create_and_deploy.py

@@ -4,12 +4,9 @@ import sys
 import json
 import time
 import datetime
-<<<<<<< HEAD
 from threading import Thread
 from Queue import Queue
-=======
 import base64
->>>>>>> 2580d1fc425fc25917ebb637328a62ddc45b084a
 
 REGION_NAME = 'region_name'
 REGION_KEY = 'region_key'
@@ -47,7 +44,7 @@ Build (argparse,functions) support for
 ### CREATE INSTANCES ###
 
 
-def run_one_region_instances(config, region_number, number_of_instances):
+def run_one_region_instances(config, region_number, number_of_instances, isOnDemand=True):
     #todo: explore the use ec2 resource and not client. e.g. create_instances --  Might make for better code.
     """
     e.g. ec2.create_instances
@@ -55,9 +52,15 @@ def run_one_region_instances(config, region_number, number_of_instances):
     region_name = config[region_number][REGION_NAME]
     session = boto3.Session(region_name=region_name)
     ec2_client = session.client('ec2')
-    response = create_instances(config,ec2_client,region_number,int(number_of_instances))
+    if isOnDemand:
+        response = create_instances(
+            config, ec2_client, region_number, int(number_of_instances))
+    else:
+        response = request_spots(
+            config, ec2_client, region_number, int(number_of_instances))
     return session
 
+
 def create_instances(config, ec2_client, region_number, number_of_instances):
     NODE_NAME = region_number + "-" + NODE_NAME_SUFFIX
     response = ec2_client.run_instances(
@@ -88,6 +91,35 @@ def create_instances(config,ec2_client,region_number,number_of_instances):
     )
     return response
 
+
+def request_spots(config, ec2_client, region_number, number_of_instances):
+    placement_group = region_number + "-" + PLACEMENT_GROUP
+    response = ec2_client.create_placement_group(
+        GroupName=placement_group,
+        Strategy='spread'
+    )
+    response = ec2_client.request_spot_instances(
+        # DryRun=True,
+        BlockDurationMinutes=60,
+        InstanceCount=number_of_instances,
+        LaunchSpecification={
+            'SecurityGroups': [config[region_number][REGION_SECURITY_GROUP]],
+            'IamInstanceProfile': {
+                'Name': IAM_INSTANCE_PROFILE
+            },
+            'UserData': USER_DATA,
+            'ImageId': config[region_number][REGION_AMI],
+            'InstanceType': INSTANCE_TYPE,
+            'KeyName': config[region_number][REGION_KEY],
+            'Placement': {
+                'AvailabilityZone': get_one_availability_zone(ec2_client),
+                'GroupName': placement_group
+            }
+        }
+    )
+    return response
+
+
 def get_availability_zones(ec2_client):
     response = ec2_client.describe_availability_zones()
     all_zones = []
@@ -98,6 +130,7 @@ def get_availability_zones(ec2_client):
                all_zones.append(info['ZoneName'])
     return all_zones
 
+
 def get_one_availability_zone(ec2_client):
     all_zones = get_availability_zones(ec2_client)
     if len(all_zones) > 0:
@@ -108,6 +141,7 @@ def get_one_availability_zone(ec2_client):
 
 #### CODEDEPLOY ###
 
+
 def run_one_region_codedeploy(region_number, commitId):
     #todo: explore the use ec2 resource and not client. e.g. create_instances --  Might make for better code.
     """
@@ -149,10 +183,13 @@ def run_one_region_codedeploy(region_number,commitId):
     deployment_group = APPLICATION_NAME + "-" + str(commitId)
     repo = REPO
     response = get_application(codedeploy, application_name)
-    response  = get_deployment_group(codedeploy,application_name,deployment_group)
-    depId = deploy(codedeploy, application_name, deployment_group, repo, commitId)
+    response = get_deployment_group(
+        codedeploy, application_name, deployment_group)
+    depId = deploy(codedeploy, application_name,
+                   deployment_group, repo, commitId)
     return region_number, depId
 
+
 def get_deployment_group(codedeploy, application_name, deployment_group):
     NODE_NAME = region_number + "-" + NODE_NAME_SUFFIX
     response = codedeploy.list_deployment_groups(
@@ -184,6 +221,7 @@ def get_deployment_group(codedeploy,application_name,deployment_group):
         )
         return response
 
+
 def get_application(codedeploy, application_name):
     response = codedeploy.list_applications()
     if application_name in response['applications']:
@@ -195,6 +233,7 @@ def get_application(codedeploy,application_name):
         )
     return response
 
+
 def deploy(codedeploy, application_name, deployment_group, repo, commitId):
     """Deploy new code at specified revision to instance.
 
@@ -235,16 +274,19 @@ def deploy(codedeploy, application_name,deployment_group,repo, commitId):
         print(info)
         return depId
 
+
 def run_one_region_codedeploy_wrapper(region_number, commitId, queue):
     region_number, depId = run_one_region_codedeploy(region_number, commitId)
     queue.put((region_number, depId))
 
+
 def launch_code_deploy(region_list, commitId):
     queue = Queue()
     jobs = []
     for i in range(len(region_list)):
         region_number = region_list[i]
-        my_thread = Thread(target=run_one_region_codedeploy_wrapper, args=(region_number,commitId,queue))
+        my_thread = Thread(target=run_one_region_codedeploy_wrapper, args=(
+            region_number, commitId, queue))
         my_thread.start()
         jobs.append(my_thread)
     for my_thread in jobs:
@@ -254,6 +296,7 @@ def launch_code_deploy(region_list,commitId):
 
 ##### UTILS ####
 
+
 def get_instance_ids(describe_instances_response):
     instance_ids = []
     for reservation in describe_instances_response["Reservations"]:
@@ -276,6 +319,7 @@ def read_configuration_file(filename):
             config[region_num][REGION_AMI] = mylist[5]
     return config
 
+
 def get_commitId(commitId):
     if commitId is None:
         commitId = run("git rev-list --max-count=1 HEAD",
@@ -285,6 +329,7 @@ def get_commitId(commitId):
 
 ##### UTILS ####
 
+
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(
         description='This script helps you start instances across multiple regions')
@@ -306,8 +351,7 @@ if __name__ == "__main__":
     for i in range(len(region_list)):
         region_number = region_list[i]
         number_of_instances = instances_list[i]
-        session = run_one_region_instances(config,region_number,number_of_instances)
+        session = run_one_region_instances(
+            config, region_number, number_of_instances, False)
     results = launch_code_deploy(region_list, commitId)
     print(results)
-   
-