Skip to content

Commit f803dc0

Browse files
githubjianliJianLi-ExpediaDiego Armando Vargas Salazar
authored
fix: added new variable to support use existing SQS queue for s3 logs bucket data event (#334)
* fix: added new variable to support use existing SQS queue for s3 logs bucket data event * Decouple the ´apiary_log_bucket´ variable for s3 logs and s3 logs hive buckets creation This will decouple the old apiary_log_bucket variable and split out in two, this would decouple the behavior of creating the s3 logs and the s3 logs hive bucket depending on the same variable. * Revert "Decouple the ´apiary_log_bucket´ variable for s3 logs and s3 logs hive buckets creation" This reverts commit fd3f972. * fix: update condition * Added flag to enable or disable s3 access logging of the buckets * fix: update s3 bucket logging configuration --------- Co-authored-by: janli <janli@expediagroup.com> Co-authored-by: Diego Armando Vargas Salazar <v-dievargas@expediagroup.com>
1 parent 0d1eb30 commit f803dc0

File tree

7 files changed

+22
-7
lines changed

7 files changed

+22
-7
lines changed

‎CHANGELOG.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file.
33

44
The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
55

6+
## [7.12.6] - 2026-01-22
7+
### Added
8+
- Added var `apiary_managed_s3_logs_queue_arn` to support use existing SQS queue for s3 logs bucket data event.
9+
610
## [7.12.5] - 2025-11-12
711
### Added
812
- Added var `lf_catalog_data_location_access_producer_arns` that is used to give `DATA_LOCATION_ACCESS` permission in LakeFormation on s3 locations of all databases.

‎VARIABLES.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
| apiary\_governance\_iamroles | AWS IAM governance roles allowed read and tagging access to managed Apiary S3 buckets. | `list(string)` | `[]` | no |
1818
| apiary\_log\_bucket | Bucket for Apiary logs.If this is blank, module will create a bucket. | `string` | `""` | no |
1919
| apiary\_log\_prefix | Prefix for Apiary logs. | `string` | `""` | no |
20+
| apiary\_managed\_s3\_logs_queue\_arn | Apiary Managed S3 Logs SQS Queue ARN. | `string` | `""` | no |
2021
| apiary\_managed\_schemas | List of maps, each map contains schema name from which S3 bucket names will be derived, and various properties. The corresponding S3 bucket will be named as apiary\_instance-aws\_account-aws\_region-schema\_name. | `list(map(string))` | `[]` | no |
2122
| apiary\_producer\_iamroles | AWS IAM roles allowed write access to managed Apiary S3 buckets. | `map(any)` | `{}` | no |
2223
| apiary\_rds\_additional\_sg | Comma-separated string containing additional security groups to attach to RDS. | `list(any)` | `[]` | no |

‎common.tf‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ locals {
4141
create_sqs_data_event_queue = contains([for schema in local.schemas_info : lookup(schema, "enable_data_events_sqs", "0")], "1") ? true : false
4242
enable_apiary_s3_log_management = var.apiary_log_bucket == "" ? true : false
4343
enable_apiary_s3_log_hive = var.apiary_log_bucket == "" && var.enable_apiary_s3_log_hive ? true : false
44+
create_sqs_s3_logs_queue = var.apiary_managed_s3_logs_queue_arn == "" && local.enable_apiary_s3_log_management ? true : false
4445
apiary_s3_logs_bucket = local.enable_apiary_s3_log_management ? "${local.apiary_bucket_prefix}-s3-logs" : ""
4546
apiary_s3_hive_logs_bucket = local.enable_apiary_s3_log_management ? "${local.apiary_s3_logs_bucket}-hive" : ""
4647
apiary_system_bucket = "${local.apiary_bucket_prefix}-${replace(var.system_schema_name, "_", "-")}"

‎s3-other.tf‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -208,7 +208,7 @@ resource "aws_s3_bucket_notification" "apiary_managed_logs_bucket" {
208208
bucket = aws_s3_bucket.apiary_managed_logs_bucket[0].bucket
209209

210210
queue {
211-
queue_arn = aws_sqs_queue.apiary_managed_logs_queue[0].arn
211+
queue_arn = local.create_sqs_s3_logs_queue ? aws_sqs_queue.apiary_managed_logs_queue[0].arn : var.apiary_managed_s3_logs_queue_arn
212212
events = ["s3:ObjectCreated:*"]
213213
}
214214
}

‎s3.tf‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -45,11 +45,6 @@ resource "aws_s3_bucket" "apiary_data_bucket" {
4545
tags = merge(tomap({"Name"=each.value["data_bucket"]}),
4646
var.apiary_tags,
4747
jsondecode(lookup(each.value, "tags", "{}")))
48-
49-
logging {
50-
target_bucket = local.enable_apiary_s3_log_management ? aws_s3_bucket.apiary_managed_logs_bucket[0].id : var.apiary_log_bucket
51-
target_prefix = "${var.apiary_log_prefix}${each.value["data_bucket"]}/"
52-
}
5348
}
5449

5550
resource "aws_s3_bucket_versioning" "apiary_data_bucket_versioning" {
@@ -188,6 +183,14 @@ resource "aws_s3_bucket_notification" "data_queue_events" {
188183
}
189184
}
190185

186+
resource "aws_s3_bucket_logging" "apiary_bucket" {
187+
for_each = {
188+
for schema in local.schemas_info : "${schema["schema_name"]}" => schema
189+
}
190+
bucket = each.value["data_bucket"]
191+
target_bucket = local.enable_apiary_s3_log_management ? aws_s3_bucket.apiary_managed_logs_bucket[0].id : var.apiary_log_bucket
192+
target_prefix = "${var.apiary_log_prefix}${each.value["data_bucket"]}/"
193+
}
191194

192195
resource "aws_s3_bucket_metric" "paid_metrics" {
193196
for_each = var.enable_s3_paid_metrics ? {

‎sns.tf‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ POLICY
7777
}
7878

7979
resource "aws_sqs_queue" "apiary_managed_logs_queue" {
80-
count = local.enable_apiary_s3_log_management ? 1 : 0
80+
count = local.create_sqs_s3_logs_queue ? 1 : 0
8181
name = "${local.instance_alias}-s3-logs-queue"
8282
tags = var.apiary_tags
8383

‎variables.tf‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1251,6 +1251,12 @@ variable "additional_s3_log_buckets" {
12511251
default = []
12521252
}
12531253

1254+
variable "apiary_managed_s3_logs_queue_arn" {
1255+
description = "Apiary Managed Log SQS Queue ARN"
1256+
type = string
1257+
default = ""
1258+
}
1259+
12541260
variable "hms_ro_k8s_log4j_properties" {
12551261
description = "Custom Log4j properties for apiary readonly metastore."
12561262
type = string

0 commit comments

Comments
 (0)