Skip to content

Commit 22e21f1

Browse files
alldoamialldoami
authored
[feature] Add GitHub Webhooks archiver and S3 private bucket modules (#112)
1 parent 1b38a02 commit 22e21f1

File tree

15 files changed

+521
-2
lines changed

15 files changed

+521
-2
lines changed

‎.gitignore

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
.terraform
22
*.tfstate
33
*.tfstate.backup
4-
bin
4+
bin
5+
.idea

‎README.md

+7
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,13 @@ This creates a security-audit role, based off the AWS-managed policy, but with a
9292

9393
[Read More](aws-iam-role-security-audit/README.md)
9494

95+
### GitHub Webhooks to S3
96+
97+
Accept GitHub webhooks and store them in S3
98+
99+
[Read More](github-webhooks-to-s3/README.md)
100+
101+
95102
## Contributing
96103

97104
### Writing tests

‎aws-s3-private-bucket/README.md

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
<!-- START -->
2+
## Inputs
3+
4+
| Name | Description | Type | Default | Required |
5+
|------|-------------|:----:|:-----:|:-----:|
6+
| bucket\_name | | string | n/a | yes |
7+
| bucket\_policy | | string | `""` | no |
8+
| env | | string | n/a | yes |
9+
| owner | | string | n/a | yes |
10+
| project | | string | n/a | yes |
11+
| service | | string | n/a | yes |
12+
13+
## Outputs
14+
15+
| Name | Description |
16+
|------|-------------|
17+
| arn | |
18+
| domain\_name | |
19+
| id | |
20+
| name | We do this to hint TF dependency graph since modules can't depend_on |
21+
22+
<!-- END -->

‎aws-s3-private-bucket/main.tf

+66
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
locals {
2+
tags = {
3+
project = "${var.project}"
4+
env = "${var.env}"
5+
service = "${var.service}"
6+
owner = "${var.owner}"
7+
managedBy = "terraform"
8+
}
9+
}
10+
11+
resource "aws_s3_bucket" "bucket" {
12+
bucket = "${var.bucket_name}"
13+
acl = "private"
14+
15+
policy = "${data.aws_iam_policy_document.bucket_policy.json}"
16+
17+
versioning {
18+
enabled = true
19+
}
20+
21+
# TODO
22+
# logging {
23+
# target_bucket = ""
24+
# target_prefix = ""
25+
# }
26+
27+
server_side_encryption_configuration {
28+
rule {
29+
apply_server_side_encryption_by_default {
30+
sse_algorithm = "AES256"
31+
}
32+
}
33+
}
34+
tags = "${local.tags}"
35+
}
36+
37+
resource "aws_s3_bucket_public_access_block" "bucket" {
38+
bucket = "${aws_s3_bucket.bucket.id}"
39+
40+
block_public_acls = true
41+
block_public_policy = true
42+
}
43+
44+
data "aws_iam_policy_document" "bucket_policy" {
45+
# Deny access to bucket if it's not accessed through HTTPS
46+
source_json = "${var.bucket_policy}"
47+
48+
statement {
49+
sid = "EnforceHTTPS"
50+
actions = ["s3:GetObject"]
51+
resources = ["arn:aws:s3:::${var.bucket_name}/*"]
52+
53+
principals {
54+
type = "*"
55+
identifiers = ["*"]
56+
}
57+
58+
effect = "Deny"
59+
60+
condition {
61+
test = "Bool"
62+
variable = "aws:SecureTransport"
63+
values = ["false"]
64+
}
65+
}
66+
}

‎aws-s3-private-bucket/module_test.go

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
package test
2+
3+
import (
4+
"testing"
5+
6+
"github.com/gruntwork-io/terratest/modules/terraform"
7+
)
8+
9+
func TestPrivateBucket(t *testing.T) {
10+
options := &terraform.Options{
11+
TerraformDir: ".",
12+
}
13+
terraform.Init(t, options)
14+
}

‎aws-s3-private-bucket/outputs.tf

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
// HACK(el): we do this to hint TF dependency graph since modules can't depend_on
2+
output "name" {
3+
value = "${var.bucket_name.id}"
4+
}
5+
6+
output "domain_name" {
7+
value = "${aws_s3_bucket.bucket.bucket_domain_name}"
8+
}
9+
10+
output "arn" {
11+
value = "${aws_s3_bucket.bucket.arn}"
12+
}
13+
14+
output "id" {
15+
value = "${aws_s3_bucket.bucket.id}"
16+
}

‎aws-s3-private-bucket/variables.tf

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
variable "bucket_name" {
2+
type = "string"
3+
}
4+
5+
variable "bucket_policy" {
6+
type = "string"
7+
default = ""
8+
}
9+
10+
variable "project" {
11+
type = "string"
12+
}
13+
14+
variable "env" {
15+
type = "string"
16+
}
17+
18+
variable "service" {
19+
type = "string"
20+
}
21+
22+
variable "owner" {
23+
type = "string"
24+
}

‎github-webhooks-to-s3/README.md

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
<!-- START -->
2+
Accept GitHub webhooks to S3. Keeps track of events such as pushing code to a repository.
3+
4+
## Inputs
5+
6+
| Name | Description | Type | Default | Required |
7+
|------|-------------|:----:|:-----:|:-----:|
8+
| certificate\_arn | A certificate in us-east-1 for var.fqdn | string | n/a | yes |
9+
| env | Env for tagging and naming. | string | n/a | yes |
10+
| fqdn | The fqdn to expose the api gateway as | string | n/a | yes |
11+
| iam\_path | | string | `"/"` | no |
12+
| lambda\_source\_s3\_bucket | The s3 bucket where to find the lambda executable | string | `"shared-infra-prod-assets"` | no |
13+
| lambda\_source\_s3\_key | The s3 key where to find the lambda executable | string | `"go-misc/lambdas/2019/06/03/github_to_firehose.zip"` | no |
14+
| owner | Owner for tagging and naming. | string | n/a | yes |
15+
| project | Project for tagging and naming. | string | n/a | yes |
16+
| route53\_zone\_id | The route53 zone id for fqdn's domain | string | n/a | yes |
17+
| service | Service for tagging and naming. | string | n/a | yes |
18+
19+
## Outputs
20+
21+
| Name | Description |
22+
|------|-------------|
23+
24+
<!-- END -->

‎github-webhooks-to-s3/api_gateway.tf

+64
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
// https://learn.hashicorp.com/terraform/aws/lambda-api-gateway
2+
resource "aws_api_gateway_rest_api" "github" {
3+
name = "${local.name}"
4+
description = "Github webhook ingestion"
5+
}
6+
7+
resource "aws_api_gateway_resource" "github" {
8+
rest_api_id = "${aws_api_gateway_rest_api.github.id}"
9+
parent_id = "${aws_api_gateway_rest_api.github.root_resource_id}"
10+
path_part = "{proxy+}"
11+
}
12+
13+
resource "aws_api_gateway_method" "github" {
14+
rest_api_id = "${aws_api_gateway_rest_api.github.id}"
15+
resource_id = "${aws_api_gateway_resource.github.id}"
16+
http_method = "POST"
17+
authorization = "NONE"
18+
}
19+
20+
resource "aws_api_gateway_integration" "lambda" {
21+
rest_api_id = "${aws_api_gateway_rest_api.github.id}"
22+
resource_id = "${aws_api_gateway_method.github.resource_id}"
23+
http_method = "${aws_api_gateway_method.github.http_method}"
24+
25+
integration_http_method = "POST"
26+
type = "AWS_PROXY"
27+
uri = "${aws_lambda_function.lambda.invoke_arn}"
28+
}
29+
30+
resource "aws_api_gateway_method" "github_root" {
31+
rest_api_id = "${aws_api_gateway_rest_api.github.id}"
32+
resource_id = "${aws_api_gateway_rest_api.github.root_resource_id}"
33+
http_method = "ANY"
34+
authorization = "NONE"
35+
}
36+
37+
resource "aws_api_gateway_integration" "lambda_root" {
38+
rest_api_id = "${aws_api_gateway_rest_api.github.id}"
39+
resource_id = "${aws_api_gateway_method.github_root.resource_id}"
40+
http_method = "${aws_api_gateway_method.github_root.http_method}"
41+
42+
integration_http_method = "POST"
43+
type = "AWS_PROXY"
44+
uri = "${aws_lambda_function.lambda.invoke_arn}"
45+
}
46+
47+
resource "aws_api_gateway_deployment" "github" {
48+
depends_on = [
49+
"aws_api_gateway_integration.lambda",
50+
"aws_api_gateway_integration.lambda_root",
51+
]
52+
53+
rest_api_id = "${aws_api_gateway_rest_api.github.id}"
54+
stage_name = "${var.env}"
55+
}
56+
57+
resource "aws_lambda_permission" "apigw" {
58+
statement_id = "AllowAPIGatewayInvoke"
59+
action = "lambda:InvokeFunction"
60+
function_name = "${aws_lambda_function.lambda.arn}"
61+
principal = "apigateway.amazonaws.com"
62+
63+
source_arn = "${aws_api_gateway_deployment.github.execution_arn}/*/*"
64+
}

‎github-webhooks-to-s3/certs.tf

+34
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
resource "aws_api_gateway_domain_name" "github" {
2+
certificate_arn = "${var.certificate_arn}"
3+
domain_name = "${var.fqdn}"
4+
}
5+
6+
resource "aws_api_gateway_base_path_mapping" "github" {
7+
api_id = "${aws_api_gateway_rest_api.github.id}"
8+
stage_name = "${aws_api_gateway_deployment.github.stage_name}"
9+
domain_name = "${aws_api_gateway_domain_name.github.domain_name}"
10+
}
11+
12+
resource "aws_route53_record" "github" {
13+
name = "${aws_api_gateway_domain_name.github.domain_name}"
14+
type = "A"
15+
zone_id = "${var.route53_zone_id}"
16+
17+
alias {
18+
evaluate_target_health = true
19+
name = "${aws_api_gateway_domain_name.github.cloudfront_domain_name}"
20+
zone_id = "${aws_api_gateway_domain_name.github.cloudfront_zone_id}"
21+
}
22+
}
23+
24+
resource "aws_route53_record" "github-ipv6" {
25+
name = "${aws_api_gateway_domain_name.github.domain_name}"
26+
type = "AAAA"
27+
zone_id = "${var.route53_zone_id}"
28+
29+
alias {
30+
evaluate_target_health = true
31+
name = "${aws_api_gateway_domain_name.github.cloudfront_domain_name}"
32+
zone_id = "${aws_api_gateway_domain_name.github.cloudfront_zone_id}"
33+
}
34+
}

‎github-webhooks-to-s3/firehose.tf

+102
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
module "bucket" {
2+
source = "../aws-s3-private-bucket"
3+
4+
bucket_name = "${local.name}"
5+
bucket_policy = ""
6+
7+
project = "${var.project}"
8+
env = "${var.env}"
9+
service = "${var.service}"
10+
owner = "${var.owner}"
11+
}
12+
13+
data "aws_iam_policy_document" "firehose" {
14+
statement {
15+
sid = "EnableFirehoseAssumeRole"
16+
effect = "Allow"
17+
actions = ["sts:AssumeRole"]
18+
19+
principals {
20+
type = "Service"
21+
identifiers = ["firehose.amazonaws.com"]
22+
}
23+
}
24+
}
25+
26+
resource "aws_iam_role" "firehose" {
27+
name = "${local.name}-firehose"
28+
29+
assume_role_policy = "${data.aws_iam_policy_document.firehose.json}"
30+
tags = "${local.tags}"
31+
}
32+
33+
data "aws_iam_policy_document" "firehose-to-s3" {
34+
statement {
35+
effect = "Allow"
36+
37+
actions = [
38+
"s3:AbortMultipartUpload",
39+
"s3:GetBucketLocation",
40+
"s3:GetObject",
41+
"s3:ListBucket",
42+
"s3:ListBucketMultipartUploads",
43+
"s3:PutObject",
44+
]
45+
46+
resources = [
47+
"${module.bucket.arn}",
48+
"${module.bucket.arn}/*",
49+
]
50+
}
51+
52+
statement {
53+
effect = "Allow"
54+
55+
actions = [
56+
"logs:CreateLogStream",
57+
"logs:PutLogEvents",
58+
"logs:GetLogEvents",
59+
]
60+
61+
resources = [
62+
"${aws_cloudwatch_log_group.firehose.arn}",
63+
"${aws_cloudwatch_log_group.firehose.arn}/*",
64+
]
65+
}
66+
}
67+
68+
resource "aws_iam_role_policy" "firehose-s3" {
69+
name = "firehose-s3"
70+
role = "${aws_iam_role.firehose.id}"
71+
policy = "${data.aws_iam_policy_document.firehose-to-s3.json}"
72+
}
73+
74+
resource "aws_kinesis_firehose_delivery_stream" "firehose" {
75+
name = "${local.name}"
76+
destination = "s3"
77+
78+
s3_configuration {
79+
role_arn = "${aws_iam_role.firehose.arn}"
80+
bucket_arn = "${module.bucket.arn}"
81+
prefix = ""
82+
compression_format = "GZIP"
83+
84+
cloudwatch_logging_options {
85+
enabled = true
86+
log_group_name = "${aws_cloudwatch_log_group.firehose.name}"
87+
log_stream_name = "${aws_cloudwatch_log_stream.firehose.name}"
88+
}
89+
}
90+
91+
tags = "${local.tags}"
92+
}
93+
94+
resource "aws_cloudwatch_log_group" "firehose" {
95+
name = "${local.name}-firehose"
96+
tags = "${local.tags}"
97+
}
98+
99+
resource "aws_cloudwatch_log_stream" "firehose" {
100+
name = "status"
101+
log_group_name = "${aws_cloudwatch_log_group.firehose.name}"
102+
}

0 commit comments

Comments
 (0)