Skip to content

Commit 6f2155c

Browse files
mollonadogivanovexpe
mollonado
and
givanovexpe
authored
Added variables to support adding custom log4j configuration for readonly,readwrite and housekeeper hms when deployed in Kubernetes. (#313)
Co-authored-by: Georgi Ivanov <givanov@expediagroup.com>
1 parent 1e988f0 commit 6f2155c

File tree

7 files changed

+141
-2
lines changed

7 files changed

+141
-2
lines changed

‎CHANGELOG.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file.
33

44
The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
55

6+
## [7.10.11] - 2025-05-16
7+
### Added
8+
- Added variables to support adding custom log4j configuration for readonly,readwrite and housekeeper hms when deployed in Kubernetes.
9+
610
## [7.10.10] - 2025-05-14
711
### Added
812
- Added variable `s3_logs_customer_accounts` to support allowing read access on s3 logs bucket.

‎README.md‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ For more information please refer to the main [Apiary](https://github.com/Expedi
1616
* Metastore authorization - A metastore pre-event listener to handle authorization using Ranger.
1717
* Grafana dashboard - If deployed in EKS, a Grafana dashboard will be created that shows S3 bucket sizes for each Apiary bucket.
1818
* Lake Formation - Databases will be synced in Lake formation as resources to enhance access control.
19+
* Custom Log4j Configuration - When deployed in Kubernetes, custom Log4j properties can be provided for read-only, read/write, and housekeeper metastores either inline or from files.
1920

2021
## Variables
2122
Please refer to [VARIABLES.md](VARIABLES.md).
@@ -74,6 +75,21 @@ module "apiary" {
7475
"StringLike": {"s3:ExistingObjectTag/type": "image*" }
7576
EOF
7677
ingress_cidr = ["10.0.0.0/8"]
78+
# Custom Log4j properties (for Kubernetes deployment)
79+
# You can use heredoc syntax for inline configuration
80+
hms_ro_k8s_log4j_properties = <<EOF
81+
rootLogger.level = INFO
82+
appender.console.type = Console
83+
appender.console.name = console
84+
appender.console.layout.type = PatternLayout
85+
appender.console.layout.pattern = %d{ISO8601} %p [%t] %c{1}: %m%n
86+
EOF
87+
88+
# Or you can load configuration from external files (recommended for complex configurations)
89+
hms_rw_k8s_log4j_properties = file("${path.module}/files/rw-hive-log4j2.properties")
90+
91+
# Using file function to read from a file
92+
hms_housekeeper_k8s_log4j_properties = file("${path.module}/files/housekeeper-hive-log4j2.properties")
7793
apiary_assume_roles = [
7894
{
7995
name = "client_name"

‎VARIABLES.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -153,6 +153,9 @@
153153
| splunk_hec_token | The token used for authentication with the Splunk HTTP Event Collector (HEC). This is required for sending logs to Splunk. Compatible with both EC2 and FARGATE ECS task definitions. | `string` | | no |
154154
| splunk_hec_host | The hostname or URL of the Splunk HTTP Event Collector (HEC) endpoint to which logs will be sent. | `string` | | no |
155155
| splunk_hec_index | The index in Splunk where logs will be stored. This is used to organize and manage logs within Splunk. | `string` | | no |
156+
| hms\_housekeeper\_k8s\_log4j\_properties | Custom Log4j properties for the Hive Metastore Housekeeper when deployed in Kubernetes. | `string` | `""` | no |
157+
| hms\_ro\_k8s\_log4j\_properties | Custom Log4j properties for the read-only Hive Metastore when deployed in Kubernetes. | `string` | `""` | no |
158+
| hms\_rw\_k8s\_log4j\_properties | Custom Log4j properties for the read-write Hive Metastore when deployed in Kubernetes. | `string` | `""` | no |
156159

157160
### apiary_assume_roles
158161

‎k8s-housekeeper.tf‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,17 @@ resource "kubernetes_deployment_v1" "apiary_hms_housekeeper" {
4242
spec {
4343
service_account_name = kubernetes_service_account_v1.hms_readwrite[0].metadata.0.name
4444
automount_service_account_token = true
45+
46+
dynamic "volume" {
47+
for_each = length(trimspace(var.hms_housekeeper_k8s_log4j_properties)) > 0 ? [1] : []
48+
content {
49+
name = "log4j-config"
50+
config_map {
51+
name = "${local.hms_alias}-housekeeper-log4j-properties"
52+
}
53+
}
54+
}
55+
4556
dynamic "init_container" {
4657
for_each = var.external_database_host == "" ? ["enabled"] : []
4758
content {
@@ -90,6 +101,16 @@ resource "kubernetes_deployment_v1" "apiary_hms_housekeeper" {
90101
container {
91102
image = "${var.hms_docker_image}:${var.hms_docker_version}"
92103
name = "${local.hms_alias}-housekeeper"
104+
105+
dynamic "volume_mount" {
106+
for_each = length(trimspace(var.hms_housekeeper_k8s_log4j_properties)) > 0 ? [1] : []
107+
content {
108+
name = "log4j-config"
109+
mount_path = "/etc/hive/conf/hive-log4j2.properties"
110+
sub_path = "hive-log4j2.properties"
111+
}
112+
}
113+
93114
port {
94115
container_port = var.hive_metastore_port
95116
}
@@ -193,3 +214,16 @@ resource "kubernetes_deployment_v1" "apiary_hms_housekeeper" {
193214
}
194215
}
195216
}
217+
218+
resource "kubernetes_config_map_v1" "housekeeper_log4j_config" {
219+
count = var.hms_instance_type == "k8s" && length(trimspace(var.hms_housekeeper_k8s_log4j_properties)) > 0 ? 1 : 0
220+
221+
metadata {
222+
name = "${local.hms_alias}-housekeeper-log4j-properties"
223+
namespace = var.metastore_namespace
224+
}
225+
226+
data = {
227+
"hive-log4j2.properties" = var.hms_housekeeper_k8s_log4j_properties
228+
}
229+
}

‎k8s-readonly.tf‎

Lines changed: 33 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -287,6 +287,15 @@ resource "kubernetes_deployment_v1" "apiary_hms_readonly" {
287287
period_seconds = 20
288288
}
289289

290+
dynamic "volume_mount" {
291+
for_each = length(trimspace(var.hms_ro_k8s_log4j_properties)) > 0 ? [1] : []
292+
content {
293+
name = "log4j-config"
294+
mount_path = "/etc/hive/conf/hive-log4j2.properties"
295+
sub_path = "hive-log4j2.properties"
296+
}
297+
}
298+
290299
resources {
291300
limits = {
292301
cpu = local.k8s_ro_cpu_limit
@@ -298,6 +307,17 @@ resource "kubernetes_deployment_v1" "apiary_hms_readonly" {
298307
}
299308
}
300309
}
310+
311+
dynamic "volume" {
312+
for_each = length(trimspace(var.hms_ro_k8s_log4j_properties)) > 0 ? [1] : []
313+
content {
314+
name = "log4j-config"
315+
config_map {
316+
name = "${local.hms_alias}-readonly-log4j-properties"
317+
}
318+
}
319+
}
320+
301321
image_pull_secrets {
302322
name = var.k8s_docker_registry_secret
303323
}
@@ -375,4 +395,16 @@ resource "kubernetes_pod_disruption_budget_v1" "hms_readonly" {
375395
max_unavailable = var.hms_ro_k8s_pdb_settings.max_unavailable != null ? var.hms_ro_k8s_pdb_settings.max_unavailable : "1"
376396
min_available = var.hms_ro_k8s_pdb_settings.min_available != null ? var.hms_ro_k8s_pdb_settings.min_available : null
377397
}
378-
}
398+
}
399+
400+
resource "kubernetes_config_map" "k8s_hms_ro_log4j_properties" {
401+
count = var.hms_instance_type == "k8s" && length(trimspace(var.hms_ro_k8s_log4j_properties)) > 0 ? 1 : 0
402+
metadata {
403+
name = "${local.hms_alias}-readonly-log4j-properties"
404+
namespace = var.metastore_namespace
405+
}
406+
407+
data = {
408+
"hive-log4j2.properties" = var.hms_ro_k8s_log4j_properties
409+
}
410+
}

‎k8s-readwrite.tf‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -349,7 +349,27 @@ resource "kubernetes_deployment_v1" "apiary_hms_readwrite" {
349349
memory = "${var.hms_rw_heapsize}Mi"
350350
}
351351
}
352+
353+
dynamic "volume_mount" {
354+
for_each = length(trimspace(var.hms_rw_k8s_log4j_properties)) > 0 ? [1] : []
355+
content {
356+
name = "log4j-config"
357+
mount_path = "/etc/hive/conf/hive-log4j2.properties"
358+
sub_path = "hive-log4j2.properties"
359+
}
360+
}
361+
}
362+
363+
dynamic "volume" {
364+
for_each = length(trimspace(var.hms_rw_k8s_log4j_properties)) > 0 ? [1] : []
365+
content {
366+
name = "log4j-config"
367+
config_map {
368+
name = "${local.hms_alias}-readwrite-log4j-properties"
369+
}
370+
}
352371
}
372+
353373
image_pull_secrets {
354374
name = var.k8s_docker_registry_secret
355375
}
@@ -406,3 +426,15 @@ resource "kubernetes_pod_disruption_budget_v1" "hms_readwrite" {
406426
min_available = var.hms_rw_k8s_pdb_settings.min_available != null ? var.hms_rw_k8s_pdb_settings.min_available : null
407427
}
408428
}
429+
430+
resource "kubernetes_config_map" "k8s_hms_rw_log4j_properties" {
431+
count = var.hms_instance_type == "k8s" && length(trimspace(var.hms_rw_k8s_log4j_properties)) > 0 ? 1 : 0
432+
metadata {
433+
name = "${local.hms_alias}-readwrite-log4j-properties"
434+
namespace = var.metastore_namespace
435+
}
436+
437+
data = {
438+
"hive-log4j2.properties" = var.hms_rw_k8s_log4j_properties
439+
}
440+
}

‎variables.tf‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1207,4 +1207,22 @@ variable "additional_s3_log_buckets" {
12071207
description = "Additional S3 log buckets"
12081208
type = list(string)
12091209
default = []
1210-
}
1210+
}
1211+
1212+
variable "hms_ro_k8s_log4j_properties" {
1213+
description = "Custom Log4j properties for apiary readonly metastore."
1214+
type = string
1215+
default = ""
1216+
}
1217+
1218+
variable "hms_rw_k8s_log4j_properties" {
1219+
description = "Custom Log4j properties for apiary readwrite metastore."
1220+
type = string
1221+
default = ""
1222+
}
1223+
1224+
variable "hms_housekeeper_k8s_log4j_properties" {
1225+
description = "Custom Log4j properties for apiary housekeeper metastore."
1226+
type = string
1227+
default = ""
1228+
}

0 commit comments

Comments
 (0)