diff --git a/README.md b/README.md index d4af9c7..9842c9e 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,16 @@ # LambCI ECS cluster and Docker image More documentation should be coming soon, but to get up and running quickly, -launch the `cluster.template` file in CloudFormation and give your stack a name like `lambci-ecs` +launch the `cluster.template` file in CloudFormation and give your stack a name like `lambci-ecs`. +You can also use `cluster.spot.template` to use ECS under Spot Instances. (You should have already created a LambCI stack as documented at https://github.com/lambci/lambci) This will create an auto-scaling group and an ECS cluster and task definition, which you can find in the AWS console from `Services > EC2 Container Service` +LambCI-ECS will look for a `Dockerfile.test` is in the root of the repository. This is where you put your test/build instructions. + ## LambCI configuration You'll need to give the Lambda function in your LambCI stack access to run the task, so will need add to IAM @@ -15,13 +18,21 @@ permissions something like this: ```json { - "Effect": "Allow", - "Action": "ecs:RunTask", - "Resource": "arn:aws:ecs:*:*:task-definition/lambci-ecs-BuildTask-1PVABCDEFKFT" + "Statement": { + "Effect": "Allow", + "Action": "ecs:RunTask", + "Resource": "arn:aws:ecs:*:*:task-definition/lambci-ecs-BuildTask-1PVABCDEFKFT" + } } ``` -Where you replace the resource with the name of the ECS task definition created in your `lambci-ecs` stack. +This block should be added as part of the `LambdaExecution > Properties > Policies` section of the `lambci` template. + +Replace the `Resource` value with the name of the ECS task definition created in your `lambci-ecs` stack. + +![Example resource location](http://i.imgur.com/3U7NHQr.png) + +## Project configuration Then in the project you want to build using ECS, you'll need to ensure the following LambCI config settings are given: @@ -34,6 +45,9 @@ Then in the project you want to build using ECS, you'll need to ensure the follo } ``` -(replacing with the actual names of your cluster and task) +(replacing with the actual names of your ECS cluster and task) + +![Example cluster and task location](http://i.imgur.com/DKgcdBU.png) These are normal LambCI config settings which you can set in your `.lambci.js[on]` file or in the config DB. + diff --git a/cluster.spot.template b/cluster.spot.template new file mode 100644 index 0000000..69769c0 --- /dev/null +++ b/cluster.spot.template @@ -0,0 +1,170 @@ +{ + "AWSTemplateFormatVersion": "2010-09-09", + "Description": "LambCI build servers running on ECS", + "Parameters": { + "InstanceType": { + "Description": "EC2 instance type (t2.micro, t2.medium, t2.large, etc)", + "Type": "String", + "Default": "t2.micro", + "ConstraintDescription": "must be a valid EC2 instance type." + }, + + "SpotPrice" : { + "Type" : Number, + "Description" : "Spot Price based on your EC2 type", + }, + + "VpcId" : { + "Type" : "AWS::EC2::VPC::Id", + "Description" : "VpcId of your existing Virtual Private Cloud (VPC)", + "ConstraintDescription" : "must be the VPC Id of an existing Virtual Private Cloud." + }, + + "Subnets" : { + "Type" : "List", + "Description" : "The list of SubnetIds in your Virtual Private Cloud (VPC)", + "ConstraintDescription" : "must be a list of an existing subnets in the selected Virtual Private Cloud." + }, + + "AZs" : { + "Type" : "List", + "Description" : "The list of AvailabilityZones for your Virtual Private Cloud (VPC)", + "ConstraintDescription" : "must be a list if valid EC2 availability zones for the selected Virtual Private Cloud" + } + }, + "Mappings": { + "EcsAmisByRegion": { + "us-east-1": {"ami": "ami-a88a46c5"}, + "us-west-1": {"ami": "ami-34a7e354"}, + "us-west-2": {"ami": "ami-ae0acdce"}, + "eu-west-1": {"ami": "ami-ccd942bf"}, + "eu-central-1": {"ami": "ami-4a5eb625"}, + "ap-northeast-1": {"ami": "ami-4aab5d2b"}, + "ap-southeast-1": {"ami": "ami-24c71547"}, + "ap-southeast-2": {"ami": "ami-0bf2da68"} + } + }, + "Resources": { + "Cluster": { + "Type": "AWS::ECS::Cluster" + }, + "BuildTask": { + "Type": "AWS::ECS::TaskDefinition", + "Properties": { + "ContainerDefinitions": [{ + "Name": "build", + "Image": "lambci/ecs", + "Memory": 450, + "LogConfiguration": { + "LogDriver": "awslogs", + "Options": { + "awslogs-group": {"Ref": "EcsLogs"}, + "awslogs-region": {"Ref": "AWS::Region"} + } + }, + "Environment": [{"Name": "LOG_GROUP", "Value": {"Ref": "EcsLogs"}}], + "MountPoints": [{"SourceVolume": "docker-socket", "ContainerPath": "/var/run/docker.sock"}] + }], + "Volumes": [{"Name": "docker-socket", "Host": {"SourcePath": "/var/run/docker.sock"}}] + } + }, + "EcsLogs": { + "Type": "AWS::Logs::LogGroup" + }, + "AutoScalingGroup": { + "Type": "AWS::AutoScaling::AutoScalingGroup", + "Properties": { + "AvailabilityZones": {"Ref": "AZs"}, + "VPCZoneIdentifier" : { "Ref" : "Subnets" }, + "LaunchConfigurationName": {"Ref": "LaunchConfig"}, + "DesiredCapacity": "1", + "MinSize": "0", + "MaxSize": "4" + }, + "CreationPolicy": { + "ResourceSignal": { + "Count": "1" + } + } + }, + "LaunchConfig": { + "Type": "AWS::AutoScaling::LaunchConfiguration", + "Properties": { + "AssociatePublicIpAddress" : "true", + "SpotPrice": { "Ref" : "SpotPrice" }, + "ImageId": {"Fn::FindInMap": ["EcsAmisByRegion", {"Ref": "AWS::Region"}, "ami"]}, + "IamInstanceProfile": {"Ref": "InstanceProfile"}, + "InstanceType": {"Ref": "InstanceType"}, + "SecurityGroups" : [ { "Ref" : "InstanceSecurityGroup" } ], + "UserData": { + "Fn::Base64": { + "Fn::Join": ["", [ + "#!/bin/bash\n", + "echo ECS_CLUSTER=", {"Ref": "Cluster"}, " >> /etc/ecs/ecs.config\n", + "yum install -y aws-cfn-bootstrap\n", + "/opt/aws/bin/cfn-signal -e $? --resource AutoScalingGroup --stack ", {"Ref": "AWS::StackName"}, " --region ", {"Ref": "AWS::Region"} + ]] + } + } + } + }, + "InstanceSecurityGroup" : { + "Type" : "AWS::EC2::SecurityGroup", + "Properties" : { + "GroupDescription" : "Enable HTTP access and SSH access", + "VpcId" : { "Ref" : "VpcId" }, + "SecurityGroupIngress" : [ + { "IpProtocol" : "tcp", "FromPort" : "22", "ToPort" : "22", "CidrIp" : "0.0.0.0/0" } + ] + } + }, + "InstanceProfile": { + "Type": "AWS::IAM::InstanceProfile", + "Properties": { + "Path": "/", + "Roles": [{"Ref": "InstanceRole"}] + } + }, + "InstanceRole": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": { + "Effect": "Allow", + "Principal": {"Service": "ec2.amazonaws.com"}, + "Action": "sts:AssumeRole" + } + }, + "Policies": [{ + "PolicyName": "RunEcs", + "PolicyDocument": { + "Statement": { + "Effect": "Allow", + "Action": [ + "ecs:DeregisterContainerInstance", + "ecs:DiscoverPollEndpoint", + "ecs:Poll", + "ecs:RegisterContainerInstance", + "ecs:StartTelemetrySession", + "ecs:Submit*" + ], + "Resource": "*" + } + } + },{ + "PolicyName": "WriteLogs", + "PolicyDocument": { + "Statement": { + "Effect": "Allow", + "Action": [ + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "Resource": "*" + } + } + }] + } + } + } +} diff --git a/cluster.template b/cluster.template index 53556bf..836960f 100644 --- a/cluster.template +++ b/cluster.template @@ -7,6 +7,24 @@ "Type": "String", "Default": "t2.micro", "ConstraintDescription": "must be a valid EC2 instance type." + }, + + "VpcId" : { + "Type" : "AWS::EC2::VPC::Id", + "Description" : "VpcId of your existing Virtual Private Cloud (VPC)", + "ConstraintDescription" : "must be the VPC Id of an existing Virtual Private Cloud." + }, + + "Subnets" : { + "Type" : "List", + "Description" : "The list of SubnetIds in your Virtual Private Cloud (VPC)", + "ConstraintDescription" : "must be a list of an existing subnets in the selected Virtual Private Cloud." + }, + + "AZs" : { + "Type" : "List", + "Description" : "The list of AvailabilityZones for your Virtual Private Cloud (VPC)", + "ConstraintDescription" : "must be a list if valid EC2 availability zones for the selected Virtual Private Cloud" } }, "Mappings": { @@ -57,7 +75,8 @@ "AutoScalingGroup": { "Type": "AWS::AutoScaling::AutoScalingGroup", "Properties": { - "AvailabilityZones": {"Fn::GetAZs": ""}, + "AvailabilityZones": {"Ref": "AZs"}, + "VPCZoneIdentifier" : { "Ref" : "Subnets" }, "LaunchConfigurationName": {"Ref": "LaunchConfig"}, "DesiredCapacity": "1", "MinSize": "0", @@ -72,9 +91,11 @@ "LaunchConfig": { "Type": "AWS::AutoScaling::LaunchConfiguration", "Properties": { + "AssociatePublicIpAddress" : "true", "ImageId": {"Fn::FindInMap": ["EcsAmisByRegion", {"Ref": "AWS::Region"}, "ami"]}, "IamInstanceProfile": {"Ref": "InstanceProfile"}, "InstanceType": {"Ref": "InstanceType"}, + "SecurityGroups" : [ { "Ref" : "InstanceSecurityGroup" } ], "UserData": { "Fn::Base64": { "Fn::Join": ["", [ @@ -86,6 +107,16 @@ } } } + }, + "InstanceSecurityGroup" : { + "Type" : "AWS::EC2::SecurityGroup", + "Properties" : { + "GroupDescription" : "Enable HTTP access and SSH access", + "VpcId" : { "Ref" : "VpcId" }, + "SecurityGroupIngress" : [ + { "IpProtocol" : "tcp", "FromPort" : "22", "ToPort" : "22", "CidrIp" : "0.0.0.0/0" } + ] + } }, "InstanceProfile": { "Type": "AWS::IAM::InstanceProfile",