Merge pull request #641 from govuk-one-login/OJ-3243-jwt-alarms

OJ-3243: Refactor JWT verify alarms

Author: mikebeeby

infrastructure/template.yaml

@@ -351,59 +351,185 @@ Resources:
           MetricNamespace: !Sub "${AWS::StackName}/LogMessages"
           MetricName: "OTGFunction-Fatalerror"
 
-  SessionLambdaFailedToVerifyJWTAlarm:
+  SessionLambdaFailedToVerifyJWTWarningAlarm:
     Type: AWS::CloudWatch::Alarm
+    Condition: DeployAlarms
     Properties:
       AlarmDescription: !Sub
-        - "Errors verifying JWTs that have been been received by the session lambda. Runbook: ${SupportManualURL}"
+        - "Errors verifying JWTs (jwt_verification_failed) rate exceeds 10% of Session Lambda invocations consecutively for 3, 5 minute periods. Runbook: ${SupportManualURL}"
         - SupportManualURL: !FindInMap [StaticVariables, Urls, SupportManualURL]
-      ActionsEnabled: true
-      AlarmActions:
-        # - !ImportValue core-infrastructure-AlarmTopic # OJ-3243: turning off pager duty notifications while we are seeing false positives
-        - !ImportValue platform-alarm-critical-alert-topic
-      OKActions:
-        # - !ImportValue core-infrastructure-AlarmTopic # OJ-3243: turning off pager duty notifications while we are seeing false positives
-        - !ImportValue platform-alarm-critical-alert-topic
-      InsufficientDataActions: []
-      MetricName: jwt_verification_failed
-      Namespace: !Sub "${CriIdentifier}"
-      Statistic: Sum
-      Dimensions:
-        - Name: service
-          Value: !Sub "${CriIdentifier}-sessionTS"
-      Period: 300
+      ComparisonOperator: GreaterThanThreshold
+      Threshold: 10
       DatapointsToAlarm: 3
       EvaluationPeriods: 3
-      Threshold: 1
-      ComparisonOperator: GreaterThanThreshold
       TreatMissingData: notBreaching
+      AlarmActions:
+        - !ImportValue platform-alarm-warning-alert-topic
+      OKActions:
+        - !ImportValue platform-alarm-warning-alert-topic
+      Metrics:
+        - Id: errors
+          Expression: IF(m2 != 0, (m1 / m2) * 100, 0)
+          Label: JWTErrorRate
+          ReturnData: true
+        - Id: m1
+          ReturnData: false
+          MetricStat:
+            Metric:
+              Namespace: !Sub "${CriIdentifier}"
+              MetricName: jwt_verification_failed
+              Dimensions:
+                - Name: service
+                  Value: !Sub "${CriIdentifier}-sessionTS"
+            Period: 300
+            Stat: Sum
+        - Id: m2
+          ReturnData: false
+          MetricStat:
+            Metric:
+              Namespace: AWS/Lambda
+              MetricName: Invocations
+              Dimensions:
+                - Name: FunctionName
+                  Value: !Sub ${CommonStackName}-SessionFunctionTS
+            Period: 300
+            Stat: Sum
 
-  TokenLambdaFailedToVerifyJWTAlarm:
+  SessionLambdaFailedToVerifyJWTCriticalAlarm:
     Type: AWS::CloudWatch::Alarm
+    Condition: DeployAlarms
     Properties:
       AlarmDescription: !Sub
-        - "Errors verifying JWTs that have been been received by the token lambda. Runbook: ${SupportManualURL}"
+        - "Errors verifying JWTs (jwt_verification_failed) rate exceeds 80% of Session Lambda invocations consecutively for 3, 5 minute periods. Runbook: ${SupportManualURL}"
         - SupportManualURL: !FindInMap [StaticVariables, Urls, SupportManualURL]
-      ActionsEnabled: true
+      ComparisonOperator: GreaterThanThreshold
+      Threshold: 80
+      DatapointsToAlarm: 3
+      EvaluationPeriods: 3
+      TreatMissingData: notBreaching
       AlarmActions:
-        # - !ImportValue core-infrastructure-AlarmTopic # OJ-3243: turning off pager duty notifications while we are seeing false positives
+        - !ImportValue core-infrastructure-AlarmTopic
         - !ImportValue platform-alarm-critical-alert-topic
       OKActions:
-        # - !ImportValue core-infrastructure-AlarmTopic # OJ-3243: turning off pager duty notifications while we are seeing false positives
+        - !ImportValue core-infrastructure-AlarmTopic
         - !ImportValue platform-alarm-critical-alert-topic
-      InsufficientDataActions: []
-      MetricName: jwt_verification_failed
-      Namespace: !Sub "${CriIdentifier}"
-      Statistic: Sum
-      Dimensions:
-        - Name: service
-          Value: !Sub "${CriIdentifier}-access-token-2"
-      Period: 300
+      Metrics:
+        - Id: errors
+          Expression: IF(m2 != 0, (m1 / m2) * 100, 0)
+          Label: JWTErrorRate
+          ReturnData: true
+        - Id: m1
+          ReturnData: false
+          MetricStat:
+            Metric:
+              Namespace: !Sub "${CriIdentifier}"
+              MetricName: jwt_verification_failed
+              Dimensions:
+                - Name: service
+                  Value: !Sub "${CriIdentifier}-sessionTS"
+            Period: 300
+            Stat: Sum
+        - Id: m2
+          ReturnData: false
+          MetricStat:
+            Metric:
+              Namespace: AWS/Lambda
+              MetricName: Invocations
+              Dimensions:
+                - Name: FunctionName
+                  Value: !Sub ${CommonStackName}-SessionFunctionTS
+            Period: 300
+            Stat: Sum
+
+  TokenLambdaFailedToVerifyJWTWarningAlarm:
+    Type: AWS::CloudWatch::Alarm
+    Condition: DeployAlarms
+    Properties:
+      AlarmDescription: !Sub
+        - "Errors verifying JWTs (jwt_verification_failed) rate exceeds 10% of Token Lambda invocations consecutively for 3, 5 minute periods. Runbook: ${SupportManualURL}"
+        - SupportManualURL: !FindInMap [StaticVariables, Urls, SupportManualURL]
+      ComparisonOperator: GreaterThanThreshold
+      Threshold: 10
       DatapointsToAlarm: 3
       EvaluationPeriods: 3
-      Threshold: 1
+      TreatMissingData: notBreaching
+      AlarmActions:
+        - !ImportValue platform-alarm-warning-alert-topic
+      OKActions:
+        - !ImportValue platform-alarm-warning-alert-topic
+      Metrics:
+        - Id: errors
+          Expression: IF(m2 != 0, (m1 / m2) * 100, 0)
+          Label: JWTErrorRate
+          ReturnData: true
+        - Id: m1
+          ReturnData: false
+          MetricStat:
+            Metric:
+              Namespace: !Sub "${CriIdentifier}"
+              MetricName: jwt_verification_failed
+              Dimensions:
+                - Name: service
+                  Value: !Sub "${CriIdentifier}-access-token-2"
+            Period: 300
+            Stat: Sum
+        - Id: m2
+          ReturnData: false
+          MetricStat:
+            Metric:
+              Namespace: AWS/Lambda
+              MetricName: Invocations
+              Dimensions:
+                - Name: FunctionName
+                  Value: !Sub ${CommonStackName}-AccessTokenFunctionTS
+            Period: 300
+            Stat: Sum
+
+  TokenLambdaFailedToVerifyJWTCriticalAlarm:
+    Type: AWS::CloudWatch::Alarm
+    Condition: DeployAlarms
+    Properties:
+      AlarmDescription: !Sub
+        - "Errors verifying JWTs (jwt_verification_failed) rate exceeds 80% of Token Lambda invocations consecutively for 3, 5 minute periods. Runbook: ${SupportManualURL}"
+        - SupportManualURL: !FindInMap [StaticVariables, Urls, SupportManualURL]
       ComparisonOperator: GreaterThanThreshold
+      Threshold: 80
+      DatapointsToAlarm: 3
+      EvaluationPeriods: 3
       TreatMissingData: notBreaching
+      AlarmActions:
+        - !ImportValue core-infrastructure-AlarmTopic
+        - !ImportValue platform-alarm-critical-alert-topic
+      OKActions:
+        - !ImportValue core-infrastructure-AlarmTopic
+        - !ImportValue platform-alarm-critical-alert-topic
+      Metrics:
+        - Id: errors
+          Expression: IF(m2 != 0, (m1 / m2) * 100, 0)
+          Label: JWTErrorRate
+          ReturnData: true
+        - Id: m1
+          ReturnData: false
+          MetricStat:
+            Metric:
+              Namespace: !Sub "${CriIdentifier}"
+              MetricName: jwt_verification_failed
+              Dimensions:
+                - Name: service
+                  Value: !Sub "${CriIdentifier}-access-token-2"
+            Period: 300
+            Stat: Sum
+        - Id: m2
+          ReturnData: false
+          MetricStat:
+            Metric:
+              Namespace: AWS/Lambda
+              MetricName: Invocations
+              Dimensions:
+                - Name: FunctionName
+                  Value: !Sub ${CommonStackName}-AccessTokenFunctionTS
+            Period: 300
+            Stat: Sum
 
   CheckHmrcLambdaConcurrency80Alarm:
     Type: AWS::CloudWatch::Alarm