This starts with an abstracted config file…
[python]
### variables (abstract these further and pull in as a seprate file – then create a new file for a new environment)
region = ‘aws region’
cluster_name = ‘cluster’
amiid = ‘ami-123456789123’
instance_type = ‘t2.medium’
key_name = ‘aws key name’
subnet_id_1 = ‘subnet-123456789123’
subnet_id_2 = ‘subnet-123456789125’
sec_group = ‘sg-123456789123′
user_data_script = """#!/bin/bash
CLUSTER=’cluster’
echo "ECS_CLUSTER=${CLUSTER}" >> /etc/ecs/ecs.config"""
instance_profile_arn = ‘arn:aws:iam::accountid:instance-profile/aws-role-name’
container_name = ‘ecs container name’
task_family = ‘task family’
docker_image = ‘accountid.dkr.ecr.us-east-1.amazonaws.com/repo_name:git-short-commit-hash’
env_name = ‘production’
task_port = 3030
# checking containers after creating…
wait_time = 22 # pause 22 seconds
check_retries = 42 # test 42 times
# internet-facing load balancer
elb_name = ‘name for external elb’
elb_subnet_1 = ‘subnet-123456789126’
elb_subnet_2 = ‘subnet-123456789127’
elb_subnet_3 = ‘subnet-123456789128’
elb_sec_group = ‘sg-123456789567’
target_group_name = ‘target name’
vpc_id = ‘vpc-123456789890’
certificate_arn = ‘arn:aws:acm:us-east-1:accountid:certificate/amazon-random-number’
service_desired_count = 2
service_name = ‘name for ecs service’
# service rollover tuning
max_percent = 100
min_healthy_percent = 50
[/python]
Which then gets imports as “cfg.variable” and used in the script…
[python]
#!/bin/python
import sys, boto3, logging, json, time
# bring in config file
import prod_sso_api_ecs_config as cfg
# this comes after populating ECR with the docker image, by hand at first and then through jenkins once all this comes together…
# add id_rsa key to access bitbucket
# git clone <repo url>
# cd repo
# docker build -t catapult:<app> .
# get the git shorthash
# git rev-parse HEAD
# docker tag image
# aws ecr get-login –region us-east-1
# then enter the returned login creds – MAY have to remove "-e none" from return
# docker push image
# then add_tags <shorthash> to tag as app too
# now the initial image to work with is populated
# script to create
# cluster
# container instances
# task definition
# elb
# listener
# target group
# then service
def main():
"""
call create_cluster()
"""
result = str(create_cluster())
print("result create_cluster: " + result)
instance_1, instance_2 = create_instances()
# I need to save each instance
print("instances:")
print(instance_1)
print(instance_2)
check_instances()
task_def_arn = create_task_def()
print("result of create_task_def: " + str(task_def_arn))
lb_arn = create_elb()
print("result of create_elb: " + str(lb_arn))
target_group_arn = create_target_group()
print("result of create_target_group: " + str(result))
register_targets(instance_1, instance_2, target_group_arn)
print("result of register targets: " + str(result))
create_listener(target_group_arn, lb_arn)
print("result of create_listener: " + str(result))
create_service(task_def_arn, target_group_arn, cfg.container_name, cfg.task_port)
print("result of create_service: " + str(result))
def create_cluster():
"""
create new cluster
"""
client = boto3.client(‘ecs’, region_name=cfg.region)
response = client.create_cluster(
clusterName=cfg.cluster_name
)
return response
def create_instances():
"""
create container instances for cluster
this requires an ECS aware amazon image
"""
ec2 = boto3.client(‘ec2’, region_name=cfg.region)
instance_1 = ec2.run_instances(
ImageId=cfg.amiid,
MinCount=1,
MaxCount=1,
InstanceType=cfg.instance_type,
KeyName=cfg.key_name,
SubnetId=cfg.subnet_id_1,
SecurityGroupIds=[
cfg.sec_group,
],
IamInstanceProfile={
‘Arn’:cfg.instance_profile_arn,
},
UserData=cfg.user_data_script,
TagSpecifications=[
{
‘ResourceType’:’instance’,
‘Tags’: [
{
‘Key’: ‘Name’,
‘Value’: cfg.container_name
},
]
},
]
)
instance_2 = ec2.run_instances(
ImageId=cfg.amiid,
MinCount=1,
MaxCount=1,
InstanceType=cfg.instance_type,
KeyName=cfg.key_name,
SubnetId=cfg.subnet_id_2,
SecurityGroupIds=[
cfg.sec_group,
],
IamInstanceProfile={
‘Arn’:cfg.instance_profile_arn,
},
UserData=cfg.user_data_script,
TagSpecifications=[
{
‘ResourceType’:’instance’,
‘Tags’: [
{
‘Key’: ‘Name’,
‘Value’: cfg.container_name
},
]
},
]
)
id_1_raw = extract_values(instance_1, ‘InstanceId’)
id_1 = id_1_raw[0]
id_2_raw = extract_values(instance_2, ‘InstanceId’)
id_2 = id_2_raw[0]
return id_1, id_2
def check_instances():
"""
check for instances up and registered
"""
count = 0
ecs_client = boto3.client(‘ecs’, region_name=cfg.region)
while count < cfg.check_retries:
container_instances_response = ecs_client.list_container_instances(
cluster=cfg.cluster_name
)
# isolate containerInstanceArns
containerarns = container_instances_response[‘containerInstanceArns’]
if len(containerarns) == 2:
print("found instances")
return 0
else:
print(len(containerarns))
count = count + 1
time.sleep(cfg.wait_time)
else:
print("failed to find instances spun up for container use…")
return 1
def create_task_def():
"""
create a task definition
"""
client = boto3.client(‘ecs’, region_name=cfg.region)
response = client.register_task_definition(
family=cfg.task_family,
containerDefinitions=[
{
‘name’: cfg.container_name,
‘image’: cfg.docker_image,
‘cpu’: 0,
‘memory’: 1024,
‘essential’: True,
‘environment’: [{
‘name’: ‘NODE_ENV’,
‘value’: cfg.env_name
}],
‘portMappings’: [
{
‘containerPort’: cfg.task_port,
‘hostPort’: cfg.task_port,
‘protocol’: ‘tcp’
},
]
}
]
)
task_def_arn_list = extract_values(response, ‘taskDefinitionArn’)
task_def_arn = task_def_arn_list[0]
return task_def_arn
def create_elb():
"""
create elbv2 load balancer
"""
conn = boto3.client(‘elbv2′, region_name=cfg.region)
response = conn.create_load_balancer(
Name=cfg.elb_name,
Subnets=[cfg.elb_subnet_1, cfg.elb_subnet_2, cfg.elb_subnet_3],
SecurityGroups=[cfg.elb_sec_group],
Scheme=’internet-facing’
)
lb_arn_raw = extract_values(response, ‘LoadBalancerArn’)
lb_arn = lb_arn_raw[0]
return lb_arn
def create_target_group():
"""
create target group
"""
conn = boto3.client(‘elbv2′, region_name=cfg.region)
response = conn.create_target_group(
Name=cfg.target_group_name,
Protocol=’HTTP’,
Port=cfg.task_port,
VpcId=cfg.vpc_id,
HealthCheckProtocol=’HTTP’,
HealthCheckPort=str(cfg.task_port),
HealthCheckPath=’/health’,
HealthCheckIntervalSeconds=12,
HealthCheckTimeoutSeconds=10,
HealthyThresholdCount=3,
UnhealthyThresholdCount=3,
Matcher={‘HttpCode’: ‘200’})
target_group = response.get(‘TargetGroups’)[0]
# target_group_arn is global (defined outside of and before main)
# and reused to register targets below
target_group_arn = target_group[‘TargetGroupArn’]
return target_group_arn
def register_targets(id_1, id_2, target_group_arn):
"""
register containers as targets
"""
conn = boto3.client(‘elbv2’, region_name=cfg.region)
response = conn.register_targets(
TargetGroupArn=target_group_arn,
Targets=[
{
‘Id’: id_1,
‘Port’: cfg.task_port,
},
{
‘Id’: id_2,
‘Port’: cfg.task_port,
},
])
def create_listener(target_group_arn, lb_arn):
"""
create a listener
"""
conn = boto3.client(‘elbv2′, region_name=cfg.region)
response = conn.create_listener(
LoadBalancerArn=lb_arn,
Protocol=’HTTPS’,
Port=443,
Certificates=[
{‘CertificateArn’: cfg.certificate_arn}],
DefaultActions=[{‘Type’: ‘forward’, ‘TargetGroupArn’: target_group_arn}]
)
return response
def create_service(task_def_arn, target_group_arn, container_name, task_port):
"""
create the service
"""
client = boto3.client(‘ecs’, region_name=cfg.region)
response = client.create_service(
cluster=cfg.cluster_name,
serviceName=cfg.service_name,
taskDefinition=task_def_arn,
desiredCount=cfg.service_desired_count,
loadBalancers=[
{
‘targetGroupArn’: target_group_arn,
‘containerName’: cfg.container_name,
‘containerPort’: cfg.task_port
},
],
deploymentConfiguration={
‘maximumPercent’: cfg.max_percent,
‘minimumHealthyPercent’: cfg.min_healthy_percent
},
healthCheckGracePeriodSeconds=30,
launchType=’EC2′,
)
# extract value given key from boto3 return
def extract_values(obj, key):
"""Pull all values of specified key from nested JSON."""
arr = []
def extract(obj, arr, key):
"""Recursively search for values of key in JSON tree."""
if isinstance(obj, dict):
for k, v in obj.items():
if isinstance(v, (dict, list)):
extract(v, arr, key)
elif k == key:
arr.append(v)
elif isinstance(obj, list):
for item in obj:
extract(item, arr, key)
return arr
results = extract(obj, arr, key)
return results
if __name__ == ‘__main__’:
main()
[/python]
— doug