feat: add alerts for onhost canaries [NR-374147] (#1120)

Author: danielorihuela

.github/workflows/component_onhost_canaries.yml

@@ -47,7 +47,7 @@ jobs:
         uses: newrelic/fargate-runner-action@main
         with:
           aws_region: us-east-2
-          container_make_target: "TERRAFORM_DIR=test/onhost-canaries/terraform ANSIBLE_FOLDER=test/onhost-canaries/ansible ENVIRONMENT=${{ inputs.environment }} test/onhost-canaries/terraform-${{ inputs.operation }}"
+          container_make_target: "TERRAFORM_DIR=test/onhost-canaries/terraform ONHOST_ANSIBLE_FOLDER=test/onhost-canaries/ansible ENVIRONMENT=${{ inputs.environment }} test/onhost-canaries/terraform-${{ inputs.operation }}"
           ecs_cluster_name: agent_control
           task_definition_name: agent_control
           cloud_watch_logs_group_name: /ecs/test-prerelease-agent_control

.github/workflows/push_pr_onhost_canaries_apply.yml

@@ -15,8 +15,9 @@ concurrency:
 
 jobs:
   onhost_canaries:
-    matrix:
-      environment: [ staging, production ]
+    strategy:
+      matrix:
+        environment: [ staging, production ]
 
     uses: ./.github/workflows/component_onhost_canaries.yml
     with:

.github/workflows/push_pr_onhost_canaries_plan.yml

@@ -13,8 +13,9 @@ concurrency:
 
 jobs:
   onhost_canaries:
-    matrix:
-      environment: [ staging, production ]
+    strategy:
+      matrix:
+        environment: [ staging, production ]
 
     uses: ./.github/workflows/component_onhost_canaries.yml
     with:

test/k8s-canaries/terraform/production/.terraform.lock.hcl

@@ -1,7 +1,7 @@
 SELECT count(${metric})
 FROM ${sample}
 WHERE (
-  clusterName = '${cluster_name}'
+  clusterName = '${instance_id}'
   AND `label.app.kubernetes.io/name` = 'agent-control'
   %{ for k, v in wheres }
   AND ${k}='${v}'

test/k8s-canaries/terraform/production/alert_nrql_templates/generic_metric_count.tftpl

@@ -1,7 +1,7 @@
 SELECT max(${metric})
 FROM ${sample}
 WHERE (
-  clusterName = '${cluster_name}'
+  clusterName = '${instance_id}'
   AND `label.app.kubernetes.io/name` = 'agent-control'
   %{ for k, v in wheres }
   AND ${k}='${v}'

test/k8s-canaries/terraform/production/alert_nrql_templates/generic_metric_threshold.tftpl

@@ -4,6 +4,10 @@ terraform {
       source  = "hashicorp/aws"
       version = ">= 5.48"
     }
+
+    newrelic = {
+      source = "newrelic/newrelic"
+    }
   }
 
   backend "s3" {
@@ -23,3 +27,9 @@ provider "aws" {
     }
   }
 }
+
+provider "newrelic" {
+  account_id = var.account_id
+  api_key    = var.api_key
+  region     = "Staging" # To change to US when production is stable
+}

test/k8s-canaries/terraform/production/backend.tf

@@ -11,12 +11,16 @@ variable "account_id" {}
 variable "api_key" {}
 variable "slack_webhook_url" {}
 module "alerts" {
-  source = "../modules/nr_alerts"
+  source = "../../../terraform/modules/nr_alerts"
 
   api_key         = var.api_key
   account_id      = var.account_id
   slack_webhook_url = var.slack_webhook_url
   policies_prefix = "Agent Control canaries metric monitoring"
+
+  region       = "Staging" # To change to US when production is stable
+  instance_id = "Agent_Control_Canaries_Production-Cluster"
+
   conditions = [
     {
       name          = "CPU usage (cores)"
@@ -74,6 +78,4 @@ module "alerts" {
       template_name = "./alert_nrql_templates/generic_metric_count.tftpl"
     },
   ]
-  region       = "Staging" # To change to US when production is stable
-  cluster_name = "Agent_Control_Canaries_Production-Cluster"
 }

test/k8s-canaries/terraform/production/main.tf

@@ -1,7 +1,7 @@
 SELECT count(${metric})
 FROM ${sample}
 WHERE (
-  clusterName = '${cluster_name}'
+  clusterName = '${instance_id}'
   AND `label.app.kubernetes.io/name` = 'agent-control'
   %{ for k, v in wheres }
   AND ${k}='${v}'

test/k8s-canaries/terraform/staging/alert_nrql_templates/generic_metric_count.tftpl

@@ -1,7 +1,7 @@
 SELECT max(${metric})
 FROM ${sample}
 WHERE (
-  clusterName = '${cluster_name}'
+  clusterName = '${instance_id}'
   AND `label.app.kubernetes.io/name` = 'agent-control'
   %{ for k, v in wheres }
   AND ${k}='${v}'