Merge pull request #37 from useparagon/feat/PARA-11102/openobserve-lo…

…gging

Replaces Kibana+Elasticsearch with OpenObserve

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@useparagon/aws-on-prem",
-  "version": "2.4.0",
+  "version": "2.5.0",
   "description": "Deploy Paragon to your own AWS cloud.",
   "repository": "[email protected]:useparagon/aws-on-prem.git",
   "author": "Paragon Engineering",

terraform/workspaces/infra/s3/s3-logs.tf

@@ -28,19 +28,42 @@ data "aws_iam_policy_document" "logs_bucket_policy" {
   count = var.disable_logs ? 0 : 1
 
   statement {
-    sid     = "AllowPutObjects"
+    sid     = "AllowAccessLogs"
     actions = ["s3:PutObject"]
     effect  = "Allow"
     resources = [
       "${aws_s3_bucket.logs[count.index].arn}",
       "${aws_s3_bucket.logs[count.index].arn}/access_logs/AWSLogs/${data.aws_caller_identity.current.account_id}/*",
     ]
-
     principals {
       type        = "AWS"
       identifiers = [data.aws_elb_service_account.main.arn]
     }
   }
+
+  statement {
+    sid = "AllowAppLogs"
+    actions = [
+      "s3:DeleteObject",
+      "s3:GetObject",
+      "s3:ListBucket",
+      "s3:PutObject"
+    ]
+    effect = "Allow"
+    resources = [
+      "${aws_s3_bucket.logs[count.index].arn}",
+      "${aws_s3_bucket.logs[count.index].arn}/*",
+    ]
+    principals {
+      type        = "AWS"
+      identifiers = ["*"]
+    }
+    condition {
+      test     = "StringEquals"
+      variable = "aws:PrincipalAccount"
+      values   = [data.aws_caller_identity.current.account_id]
+    }
+  }
 }
 
 resource "aws_s3_bucket_policy" "logs_bucket" {
@@ -69,7 +92,7 @@ resource "aws_s3_bucket_lifecycle_configuration" "logs" {
     status = "Enabled"
 
     transition {
-      days          = 7
+      days          = 30
       storage_class = "GLACIER"
     }
   }

terraform/workspaces/paragon/helm/helm.tf

@@ -262,7 +262,7 @@ module "helm_hash_logging" {
   chart_directory = "./charts/paragon-logging"
 }
 
-# paragon logging stack fluent bit , kibana , elasticsearch
+# paragon logging stack fluent bit and openobserve
 resource "helm_release" "paragon_logging" {
   name             = "paragon-logging"
   description      = "Paragon logging services"
@@ -286,6 +286,31 @@ resource "helm_release" "paragon_logging" {
     }))
   ]
 
+  set {
+    name  = "global.env.ZO_S3_PROVIDER"
+    value = "s3"
+  }
+
+  set {
+    name  = "global.env.ZO_S3_BUCKET_NAME"
+    value = var.logs_bucket
+  }
+
+  set {
+    name  = "global.env.ZO_S3_REGION_NAME"
+    value = var.aws_region
+  }
+
+  set {
+    name  = "global.env.ZO_ROOT_USER_EMAIL"
+    value = local.openobserve_email
+  }
+
+  set_sensitive {
+    name  = "global.env.ZO_ROOT_USER_PASSWORD"
+    value = local.openobserve_password
+  }
+
   depends_on = [
     helm_release.ingress,
     kubernetes_secret.docker_login,

terraform/workspaces/paragon/helm/openobserve.tf

@@ -0,0 +1,24 @@
+resource "random_string" "openobserve_email" {
+  count = var.openobserve_email == null ? 1 : 0
+
+  length  = 12
+  lower   = true
+  numeric = true
+  special = false
+  upper   = false
+}
+
+resource "random_password" "openobserve_password" {
+  count = var.openobserve_password == null ? 1 : 0
+
+  length  = 32
+  lower   = true
+  numeric = true
+  special = false
+  upper   = true
+}
+
+locals {
+  openobserve_email    = var.openobserve_email != null ? var.openobserve_email : "${random_string.openobserve_email[0].result}@useparagon.com"
+  openobserve_password = var.openobserve_password != null ? var.openobserve_password : random_password.openobserve_password[0].result
+}

terraform/workspaces/paragon/helm/outputs.tf

@@ -4,4 +4,13 @@ output "release_ingress" {
 
 output "release_paragon_on_prem" {
   value = helm_release.paragon_on_prem
-}
+}
+
+output "openobserve_email" {
+  value = local.openobserve_email
+}
+
+output "openobserve_password" {
+  value     = local.openobserve_password
+  sensitive = true
+}

terraform/workspaces/paragon/helm/variables.tf

@@ -33,6 +33,18 @@ variable "docker_email" {
   type        = string
 }
 
+variable "openobserve_email" {
+  description = "OpenObserve admin login email."
+  type        = string
+  default     = null
+}
+
+variable "openobserve_password" {
+  description = "OpenObserve admin login password."
+  type        = string
+  default     = null
+}
+
 variable "logs_bucket" {
   description = "Bucket to store system logs."
   type        = string

terraform/workspaces/paragon/modules.tf

@@ -32,6 +32,8 @@ module "helm" {
   monitor_version        = local.monitor_version
   monitors               = local.monitors
   monitors_enabled       = var.monitors_enabled
+  openobserve_email      = var.openobserve_email
+  openobserve_password   = var.openobserve_password
   public_monitors        = local.public_monitors
 
   acm_certificate_arn = module.alb.acm_certificate_arn

terraform/workspaces/paragon/outputs.tf

@@ -37,3 +37,12 @@ output "uptime_webhook" {
   value       = module.uptime.webhook
   sensitive   = true
 }
+
+output "openobserve_email" {
+  value = module.helm.openobserve_email
+}
+
+output "openobserve_password" {
+  value     = module.helm.openobserve_password
+  sensitive = true
+}

terraform/workspaces/paragon/variables.tf

@@ -152,6 +152,18 @@ variable "uptime_company" {
   default     = null
 }
 
+variable "openobserve_email" {
+  description = "OpenObserve admin login email."
+  type        = string
+  default     = null
+}
+
+variable "openobserve_password" {
+  description = "OpenObserve admin login password."
+  type        = string
+  default     = null
+}
+
 locals {
   raw_helm_env = jsondecode(base64decode(var.helm_env))
   raw_helm_values = try(yamldecode(
@@ -458,7 +470,7 @@ locals {
             WORKER_TRIGGERS_PORT    = try(local.microservices["worker-triggers"].port, null)
             WORKER_WORKFLOWS_PORT   = try(local.microservices["worker-workflows"].port, null)
 
-            ACCOUNT_PRIVATE_URL  = try("http://account:${local.microservices.account.port}", null)
+            ACCOUNT_PRIVATE_URL   = try("http://account:${local.microservices.account.port}", null)
             CERBERUS_PRIVATE_URL  = try("http://cerberus:${local.microservices.cerberus.port}", null)
             CHRONOS_PRIVATE_URL   = try("http://chronos:${local.microservices.chronos.port}", null)
             CONNECT_PRIVATE_URL   = try("http://connect:${local.microservices.connect.port}", null)