commit
57b6fcc863
@ -1,8 +0,0 @@ |
||||
# security_group_name=$1 |
||||
# security_group_description=$2 |
||||
# instanceName=$3 |
||||
aws ec2 create-security-group --group-name mcDG --description "mcdg" |
||||
aws ec2 authorize-security-group-ingress --group-name MySecurityGroup --protocol tcp --port all --cidr 0.0.0.0/0 |
||||
|
||||
aws ec2 run-instances --image-id ami-a9d09ed1 --count 5 --instance-type t2.micro --key-name main --security-group-ids mcDG \ |
||||
--user-data user-data.sh --iam-instance-profile Name=CodeDeployDemo-EC2-Instance-Profile --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=Client}]' |
@ -1,8 +0,0 @@ |
||||
# security_group_name=$1 |
||||
# security_group_description=$2 |
||||
# instanceName=$3 |
||||
aws ec2 create-security-group --group-name mcDG --description "mcdg" |
||||
aws ec2 authorize-security-group-ingress --group-name MySecurityGroup --protocol tcp --port all --cidr 0.0.0.0/0 |
||||
|
||||
aws ec2 run-instances --image-id ami-a9d09ed1 --count 5 --instance-type t2.micro --key-name main --security-group-ids mcDG \ |
||||
--user-data user-data.sh --iam-instance-profile Name=CodeDeployDemo-EC2-Instance-Profile --tag-specifications 'ResourceType=instance,Tags=[{Key=Name,Value=Client}]' |
@ -1,84 +0,0 @@ |
||||
import os |
||||
import argparse |
||||
import json |
||||
import time |
||||
import datetime |
||||
|
||||
REGION_NAME = 'region_name' |
||||
REGION_KEY = 'region_key' |
||||
REGION_SECURITY_GROUP = 'region_security_group' |
||||
REGION_HUMAN_NAME = 'region_human_name' |
||||
INSTANCE_TYPE = 't2.micro' |
||||
REGION_AMI = 'region_ami' |
||||
USER_DATA = 'user-data.sh' |
||||
IAM_INSTANCE_PROFILE = 'BenchMarkCodeDeployInstanceProfile' |
||||
|
||||
def read_configuration_file(filename): |
||||
config = {} |
||||
with open(filename,'r') as f: |
||||
for myline in f: |
||||
mylist = myline.strip().split(',') |
||||
region_num = mylist[0] |
||||
config[region_num] = {} |
||||
config[region_num][REGION_NAME] = mylist[1] |
||||
config[region_num][REGION_KEY] = mylist[2] |
||||
config[region_num][REGION_SECURITY_GROUP] = mylist[3] |
||||
config[region_num][REGION_HUMAN_NAME] = mylist[4] |
||||
config[region_num][REGION_AMI] = mylist[5] |
||||
return config |
||||
|
||||
def region_variant(region_name): |
||||
return region_name + "a" |
||||
|
||||
def create_custom_json(config,num_instances,region_num,current_session): |
||||
print(num_instances) |
||||
input_cli = {} |
||||
input_cli['MinCount'] = num_instances |
||||
input_cli['MaxCount'] = num_instances |
||||
input_cli['ImageId'] = config[region_num][REGION_AMI] |
||||
input_cli['Placement'] = {} |
||||
input_cli['Placement']['AvailabilityZone'] = region_variant(config[region_num][REGION_NAME]) |
||||
input_cli['SecurityGroups'] = [] |
||||
input_cli['SecurityGroups'].append(config[region_num][REGION_SECURITY_GROUP]) |
||||
input_cli['IamInstanceProfile'] = {} |
||||
input_cli['IamInstanceProfile']['Name'] = IAM_INSTANCE_PROFILE |
||||
input_cli['KeyName'] = config[region_num][REGION_KEY] |
||||
#input_cli['KeyName'] = "main" |
||||
input_cli['UserData'] = USER_DATA |
||||
input_cli['InstanceType'] = INSTANCE_TYPE |
||||
input_cli['TagSpecifications'] = [] |
||||
input_cli['TagSpecifications'].append({"ResourceType": "instance","Tags":[{"Key":"Name","Value":"Node"}]}) |
||||
my_dir = "input_jsons/" + "session-"+ current_session |
||||
if not os.path.exists(my_dir): |
||||
os.makedirs(my_dir) |
||||
#cli_input_file = os.path.join(my_dir,config[region_num][REGION_HUMAN_NAME]+".json") |
||||
cli_input_file = "local.json" |
||||
with open(cli_input_file,'w') as g: |
||||
json.dump(input_cli,g) |
||||
print("INPUT CLI JSON FILE: %s" % cli_input_file) |
||||
return cli_input_file |
||||
|
||||
def create_instances(config,region_list,instances_list,current_session): |
||||
for i in range(len(region_list)): |
||||
region_num = region_list[i] |
||||
num_instances = int(instances_list[i]) |
||||
cli_input_file = create_custom_json(config,num_instances,region_num,current_session) |
||||
cmd_str = "aws ec2 --region " + config[region_num][REGION_NAME] + " run-instances " + " --cli-input-json " + str(cli_input_file) |
||||
print(cmd_str) |
||||
|
||||
if __name__ == "__main__": |
||||
parser = argparse.ArgumentParser(description='This script helps you start instances across multiple regions') |
||||
parser.add_argument('--regions',type=str,dest='regions',default='3',help="Supply a csv list of all regions") |
||||
parser.add_argument('--instances', type=str,dest='numInstances',default=1,help='number of instances') |
||||
parser.add_argument('--configuration',type=str,dest='config',default='configuration.txt') |
||||
args = parser.parse_args() |
||||
config = read_configuration_file(args.config) |
||||
region_list = args.regions.split(',') |
||||
instances_list = args.numInstances.split(',') |
||||
assert len(region_list) == len(instances_list),"number of regions: %d != number of instances per region: %d" % (len(region_list),len(intances_list)) |
||||
time_stamp = time.time() |
||||
current_session = datetime.datetime.fromtimestamp(time_stamp).strftime('%H-%M-%S-%Y-%m-%d') |
||||
print("current session is %s" % current_session) |
||||
create_instances(config,region_list,instances_list,current_session) |
||||
|
||||
|
@ -1,13 +0,0 @@ |
||||
echo "Getting list of Availability Zones" |
||||
all_regions=$(aws ec2 describe-regions --output text --query 'Regions[*].[RegionName]' | sort) |
||||
all_az=() |
||||
|
||||
echo $all_regions |
||||
while read -r region; do |
||||
az_per_region=$(aws ec2 describe-availability-zones --region $region --query 'AvailabilityZones[*].[ZoneName]' --output text | sort) |
||||
echo $region $az_per_region |
||||
while read -r az; do |
||||
all_az+=($az) |
||||
done <<< "$az_per_region" |
||||
done <<< "$all_regions" |
||||
echo $all_az |
@ -0,0 +1,200 @@ |
||||
import boto3 |
||||
import argparse |
||||
import sys |
||||
import json |
||||
import time |
||||
import datetime |
||||
|
||||
REGION_NAME = 'region_name' |
||||
REGION_KEY = 'region_key' |
||||
REGION_SECURITY_GROUP = 'region_security_group' |
||||
REGION_HUMAN_NAME = 'region_human_name' |
||||
INSTANCE_TYPE = 't2.micro' |
||||
REGION_AMI = 'region_ami' |
||||
USER_DATA = 'user-data.sh' |
||||
IAM_INSTANCE_PROFILE = 'BenchMarkCodeDeployInstanceProfile' |
||||
REPO = "simple-rules/harmony-benchmark" |
||||
APPLICATION_NAME = 'benchmark-experiments' |
||||
|
||||
def run_one_region(config,region_number,number_of_instances,commitId): |
||||
region_name = config[region_number][REGION_NAME] |
||||
session = boto3.Session(region_name=region_name) |
||||
# ec2_client = session.client('ec2') |
||||
# response = create_instances(config,ec2_client,region_number,int(number_of_instances)) |
||||
codedeploy = session.client('codedeploy') |
||||
#commitId = get_commitId(commitId) |
||||
application_name = APPLICATION_NAME |
||||
deployment_group = APPLICATION_NAME + str(commitId) |
||||
repo = REPO |
||||
response = get_application(codedeploy,application_name) |
||||
response = get_deployment_group(codedeploy,application_name,deployment_group) |
||||
print(response) |
||||
# deploy(codedeploy, application_name, deployment_group, repo, commitId, wait=True) |
||||
# return response |
||||
|
||||
def get_availability_zones(ec2_client): |
||||
response = ec2_client.describe_availability_zones() |
||||
all_zones = [] |
||||
if response.get('AvailabilityZones',None): |
||||
region_info = response.get('AvailabilityZones') |
||||
for info in region_info: |
||||
if info['State'] == 'available': |
||||
all_zones.append(info['ZoneName']) |
||||
return all_zones |
||||
|
||||
def get_one_availability_zone(ec2_client): |
||||
all_zones = get_availability_zones(ec2_client) |
||||
if len(all_zones) > 0: |
||||
return all_zones[0] |
||||
else: |
||||
print("No availability zone for this region") |
||||
sys.exit() |
||||
|
||||
def create_instances(config,ec2_client,region_number,number_of_instances): |
||||
response = ec2_client.run_instances( |
||||
MinCount = number_of_instances, |
||||
MaxCount = number_of_instances, |
||||
ImageId = config[region_number][REGION_AMI], |
||||
Placement = { |
||||
'AvailabilityZone': get_one_availability_zone(ec2_client) |
||||
}, |
||||
SecurityGroups = [config[region_number][REGION_SECURITY_GROUP]], |
||||
IamInstanceProfile = { |
||||
'Name' : IAM_INSTANCE_PROFILE |
||||
}, |
||||
KeyName = config[region_number][REGION_KEY], |
||||
UserData = USER_DATA, |
||||
InstanceType = INSTANCE_TYPE, |
||||
TagSpecifications = [ |
||||
{ |
||||
'ResourceType' : 'instance', |
||||
'Tags': [ |
||||
{ |
||||
'Key': 'Name', |
||||
'Value': 'Node' |
||||
}, |
||||
] |
||||
}, |
||||
] |
||||
) |
||||
return response |
||||
|
||||
def get_deployment_group(codedeploy,application_name,deployment_group): |
||||
response = codedeploy.create_deployment_group( |
||||
applicationName = application_name, |
||||
deploymentGroupName = deployment_group, |
||||
deploymentConfigName='CodeDeployDefault.AllAtAOnce', |
||||
serviceRoleArn = 'arn:aws:iam::656503231766:role/BenchMarkCodeDeployServiceRole', |
||||
deploymentStyle={ |
||||
'deploymentType': 'IN_PLACE' |
||||
}, |
||||
ec2TagSet={ |
||||
'ec2TagSetList': [ |
||||
[ |
||||
{ |
||||
'Key': 'Name', |
||||
'Value': 'Node', |
||||
'Type': 'KEY_AND_VALUE' |
||||
}, |
||||
], |
||||
] |
||||
} |
||||
) |
||||
return response |
||||
|
||||
def get_commitId(commitId): |
||||
if commitId is None: |
||||
commitId = run("git rev-list --max-count=1 HEAD", |
||||
hide=True).stdout.strip() |
||||
print("Got newest commitId as " + commitId) |
||||
return commitId |
||||
|
||||
def get_application(codedeploy,application_name): |
||||
response = codedeploy.list_applications() |
||||
if application_name in response['applications']: |
||||
return response |
||||
else: |
||||
response = codedeploy.create_application( |
||||
applicationName= application_name, |
||||
computePlatform='Server' |
||||
) |
||||
return response |
||||
|
||||
def deploy(codedeploy, application_name,deployment_group,repo, commitId, wait=True): |
||||
"""Deploy new code at specified revision to instance. |
||||
|
||||
arguments: |
||||
- repo: GitHub repository path from which to get the code |
||||
- commitId: commit ID to be deployed |
||||
- wait: wait until the CodeDeploy finishes |
||||
""" |
||||
print("Launching CodeDeploy with commit " + commitId) |
||||
res = codedeploy.create_deployment( |
||||
applicationName = application_name, |
||||
deploymentGroupName = deployment_group, |
||||
deploymentConfigName = 'CodeDeployDefault.AllAtAOnce', |
||||
description = 'benchmark experiments', |
||||
revision = { |
||||
'revisionType': 'GitHub', |
||||
'gitHubLocation': { |
||||
'repository': repo, |
||||
'commitId': commitId, |
||||
} |
||||
} |
||||
) |
||||
depId = res["deploymentId"] |
||||
print("Deployment ID: " + depId) |
||||
|
||||
# The deployment is launched at this point, so exit unless asked to wait |
||||
# until it finishes |
||||
if not wait: |
||||
return |
||||
|
||||
# This should use a boto3 waiter instead, but that hasn't been |
||||
# implemented yet: https://github.com/boto/boto3/issues/708 |
||||
# So instead we check the status every few seconds manually |
||||
info = {'status': 'Created'} |
||||
start = time.time() |
||||
while info['status'] not in ('Succeeded', 'Failed', 'Stopped',) and (time.time() - start < 300.0): |
||||
info = codedeploy.get_deployment(deploymentId=depId)['deploymentInfo'] |
||||
print(info) |
||||
print(info['status']) |
||||
time.sleep(15) |
||||
if info['status'] == 'Succeeded': |
||||
print("\nDeploy Succeeded") |
||||
else: |
||||
print("\nDeploy Failed") |
||||
print(info) |
||||
|
||||
def read_configuration_file(filename): |
||||
config = {} |
||||
with open(filename,'r') as f: |
||||
for myline in f: |
||||
mylist = myline.strip().split(',') |
||||
region_num = mylist[0] |
||||
config[region_num] = {} |
||||
config[region_num][REGION_NAME] = mylist[1] |
||||
config[region_num][REGION_KEY] = mylist[2] |
||||
config[region_num][REGION_SECURITY_GROUP] = mylist[3] |
||||
config[region_num][REGION_HUMAN_NAME] = mylist[4] |
||||
config[region_num][REGION_AMI] = mylist[5] |
||||
return config |
||||
|
||||
if __name__ == "__main__": |
||||
parser = argparse.ArgumentParser(description='This script helps you start instances across multiple regions') |
||||
parser.add_argument('--regions',type=str,dest='regions',default='3',help="Supply a csv list of all regions") |
||||
parser.add_argument('--instances', type=str,dest='numInstances',default='1',help='number of instances') |
||||
parser.add_argument('--configuration',type=str,dest='config',default='configuration.txt') |
||||
parser.add_argument('--commitId',type=str,dest='commitId',default='1f7e6e7ca7cf1c1190cedec10e791c01a29971cf') |
||||
args = parser.parse_args() |
||||
config = read_configuration_file(args.config) |
||||
region_list = args.regions.split(',') |
||||
instances_list = args.numInstances.split(',') |
||||
assert len(region_list) == len(instances_list),"number of regions: %d != number of instances per region: %d" % (len(region_list),len(intances_list)) |
||||
time_stamp = time.time() |
||||
current_session = datetime.datetime.fromtimestamp(time_stamp).strftime('%H-%M-%S-%Y-%m-%d') |
||||
print("current session is %s" % current_session) |
||||
region_number = '1' |
||||
number_of_instances = '2' |
||||
run_one_region(config,region_number,number_of_instances,args.commitId) |
||||
|
@ -0,0 +1,72 @@ |
||||
import os |
||||
import argparse |
||||
import json |
||||
import time |
||||
import datetime |
||||
import base64 |
||||
|
||||
REGION_NAME = 'region_name' |
||||
REGION_KEY = 'region_key' |
||||
REGION_SECURITY_GROUP = 'region_security_group' |
||||
REGION_HUMAN_NAME = 'region_human_name' |
||||
|
||||
INSTANCE_TYPE = 'm3.medium' # 't2.micro' |
||||
AMI = 'ami-f2d3638a' # 'ami-a9d09ed1' |
||||
# UserData must be base64 encoded. |
||||
with open("userdata.sh", "rb") as userdata_file: |
||||
USER_DATA = base64.b64encode(userdata_file.read()) |
||||
IAM_INSTANCE_PROFILE = 'BenchMarkCodeDeployInstanceProfile' |
||||
|
||||
def read_configuration_file(filename): |
||||
config = {} |
||||
with open(filename,'r') as f: |
||||
for line in f: |
||||
vals = line.strip().split(',') |
||||
region_num = vals[0] |
||||
config[region_num] = {} |
||||
config[region_num][REGION_NAME] = vals[1] |
||||
config[region_num][REGION_KEY] = vals[2] |
||||
config[region_num][REGION_SECURITY_GROUP] = vals[3] |
||||
config[region_num][REGION_HUMAN_NAME] = vals[4] |
||||
return config |
||||
|
||||
def create_launch_specification(region_num): |
||||
input_cli = {} |
||||
input_cli['ImageId'] = AMI |
||||
# input_cli['Placement'] = { |
||||
# "AvailabilityZone": config[region_num][REGION_NAME] +"a" |
||||
# } |
||||
input_cli['SecurityGroups'] = [ "richard-spot-instance SSH" ] # [ config[region_num][REGION_SECURITY_GROUP] ] |
||||
input_cli['IamInstanceProfile'] = { |
||||
"Name": IAM_INSTANCE_PROFILE |
||||
} |
||||
input_cli['KeyName'] = "richard-spot-instance" # config[region_num][REGION_KEY] |
||||
input_cli['UserData'] = USER_DATA |
||||
input_cli['InstanceType'] = INSTANCE_TYPE |
||||
# folder = "launch_specs/" + "session-"+ current_session |
||||
folder = "launch_specs/latest" |
||||
if not os.path.exists(folder): |
||||
os.makedirs(folder) |
||||
launch_spec_file = os.path.join(folder,config[region_num][REGION_HUMAN_NAME]+".json") |
||||
with open(launch_spec_file,'w') as g: |
||||
json.dump(input_cli,g) |
||||
print("Launch spec: %s" % launch_spec_file) |
||||
return launch_spec_file |
||||
|
||||
def create_instances(region_list): |
||||
for i in range(len(region_list)): |
||||
region_num = region_list[i] |
||||
launch_spec_file = create_launch_specification(region_num) |
||||
|
||||
if __name__ == "__main__": |
||||
parser = argparse.ArgumentParser(description='This script helps you start instances across multiple regions') |
||||
parser.add_argument('--regions',type=str,dest='regions',default='3',help="a comma-separated-value list of all regions") |
||||
# configuration file contains the number/name/security-group etc. information of each region. |
||||
parser.add_argument('--config',type=str,dest='config',default='configuration.txt') |
||||
args = parser.parse_args() |
||||
config = read_configuration_file(args.config) |
||||
region_list = args.regions.split(',') |
||||
time_stamp = time.time() |
||||
current_session = datetime.datetime.fromtimestamp(time_stamp).strftime('%Y-%m-%d-%H-%M-%S') |
||||
print("current session is %s" % current_session) |
||||
create_instances(region_list) |
@ -0,0 +1,5 @@ |
||||
aws ec2 request-spot-instances \ |
||||
--instance-count 1 \ |
||||
--block-duration-minutes 60 \ |
||||
--launch-specification file://launch_specs/latest/ohio.json |
||||
# --dry-run # uncomment this line to send a real request. |
@ -0,0 +1,8 @@ |
||||
#!/bin/bash |
||||
REGION=$(curl 169.254.169.254/latest/meta-data/placement/availability-zone/ | sed 's/[a-z]$//') |
||||
yum -y update |
||||
yum install -y ruby |
||||
cd /home/ec2-user |
||||
curl -O https://aws-codedeploy-$REGION.s3.amazonaws.com/latest/install |
||||
chmod +x ./install |
||||
./install auto |
@ -1,7 +0,0 @@ |
||||
#!/bin/bash |
||||
yum -y update |
||||
yum install -y ruby |
||||
cd /home/ec2-user |
||||
curl -O https://aws-codedeploy-us-west-2.s3.amazonaws.com/latest/install |
||||
chmod +x ./install |
||||
./install auto |
@ -0,0 +1,31 @@ |
||||
package consensus |
||||
|
||||
// Consensus state enum for both leader and validator
|
||||
// States for leader:
|
||||
// FINISHED, ANNOUNCE_DONE, CHALLENGE_DONE
|
||||
// States for validator:
|
||||
// FINISHED, COMMIT_DONE, RESPONSE_DONE
|
||||
type ConsensusState int |
||||
|
||||
const ( |
||||
FINISHED ConsensusState = iota // initial state or state after previous consensus is done.
|
||||
ANNOUNCE_DONE |
||||
COMMIT_DONE |
||||
CHALLENGE_DONE |
||||
RESPONSE_DONE |
||||
) |
||||
|
||||
// Returns string name for the ConsensusState enum
|
||||
func (state ConsensusState) String() string { |
||||
names := [...]string{ |
||||
"FINISHED", |
||||
"ANNOUNCE_DONE", |
||||
"COMMIT_DONE", |
||||
"CHALLENGE_DONE", |
||||
"RESPONSE_DONE"} |
||||
|
||||
if state < FINISHED || state > RESPONSE_DONE { |
||||
return "Unknown" |
||||
} |
||||
return names[state] |
||||
} |
@ -0,0 +1,5 @@ |
||||
package utils |
||||
|
||||
func BToMb(b uint64) uint64 { |
||||
return b / 1024 / 1024 |
||||
} |
Loading…
Reference in new issue