Merge pull request #1098 from erikdarlingdata/feature/1096-exclude-cdc-long-running

Exclude CDC capture jobs from long-running query alerts (Dashboard + Lite) (#1096)

Author: erikdarlingdata

Dashboard/MainWindow.xaml.cs

@@ -1472,7 +1472,7 @@ private async Task CheckAllServerAlertsAsync()
                     var connectionString = server.GetConnectionString(_credentialService);
                     var databaseService = new DatabaseService(connectionString);
                     var connStatus = _serverManager.GetConnectionStatus(server.Id);
-                    var health = await databaseService.GetAlertHealthAsync(connStatus.SqlEngineEdition, prefs.LongRunningQueryThresholdMinutes, prefs.LongRunningJobMultiplier, prefs.LongRunningQueryMaxResults, prefs.LongRunningQueryExcludeSpServerDiagnostics, prefs.LongRunningQueryExcludeWaitFor, prefs.LongRunningQueryExcludeBackups, prefs.LongRunningQueryExcludeMiscWaits, prefs.AlertExcludedDatabases);
+                    var health = await databaseService.GetAlertHealthAsync(connStatus.SqlEngineEdition, prefs.LongRunningQueryThresholdMinutes, prefs.LongRunningJobMultiplier, prefs.LongRunningQueryMaxResults, prefs.LongRunningQueryExcludeSpServerDiagnostics, prefs.LongRunningQueryExcludeWaitFor, prefs.LongRunningQueryExcludeBackups, prefs.LongRunningQueryExcludeMiscWaits, prefs.LongRunningQueryExcludeCdc, prefs.AlertExcludedDatabases);
 
                     if (health.IsOnline)
                     {

Dashboard/Models/UserPreferences.cs

@@ -102,6 +102,7 @@ public class UserPreferences
         public bool LongRunningQueryExcludeWaitFor { get; set; } = true;
         public bool LongRunningQueryExcludeBackups { get; set; } = true;
         public bool LongRunningQueryExcludeMiscWaits { get; set; } = true;
+        public bool LongRunningQueryExcludeCdc { get; set; } = true; // Exclude CDC capture jobs (sp_MScdc_capture_job / sp_cdc_scan)
         public bool NotifyOnTempDbSpace { get; set; } = true;
         public int TempDbSpaceThresholdPercent { get; set; } = 80; // Alert when TempDB used > X%
         public bool NotifyOnLongRunningJobs { get; set; } = true;

Dashboard/Services/DatabaseService.NocHealth.cs

@@ -131,6 +131,7 @@ public async Task<AlertHealthResult> GetAlertHealthAsync(
             bool excludeWaitFor = true,
             bool excludeBackups = true,
             bool excludeMiscWaits = true,
+            bool excludeCdc = true,
             IReadOnlyList<string>? excludedDatabases = null)
         {
             var result = new AlertHealthResult();
@@ -149,7 +150,7 @@ public async Task<AlertHealthResult> GetAlertHealthAsync(
                     ? GetFilteredDeadlockCountAsync(connection, excludedDatabases)
                     : null;
                 var poisonWaitTask = GetPoisonWaitDeltasAsync(connection);
-                var longRunningTask = GetLongRunningQueriesAsync(connection, longRunningQueryThresholdMinutes, longRunningQueryMaxResults, excludeSpServerDiagnostics, excludeWaitFor, excludeBackups, excludeMiscWaits);
+                var longRunningTask = GetLongRunningQueriesAsync(connection, longRunningQueryThresholdMinutes, longRunningQueryMaxResults, excludeSpServerDiagnostics, excludeWaitFor, excludeBackups, excludeMiscWaits, excludeCdc);
                 var tempDbTask = GetTempDbSpaceAsync(connection);
                 var anomalousJobTask = GetAnomalousJobsAsync(connection, longRunningJobMultiplier);
                 var missingCaptureTask = GetMissingCaptureSessionsAsync(connection);
@@ -751,7 +752,8 @@ private async Task<List<LongRunningQueryInfo>> GetLongRunningQueriesAsync(
             bool excludeSpServerDiagnostics = true,
             bool excludeWaitFor = true,
             bool excludeBackups = true,
-            bool excludeMiscWaits = true)
+            bool excludeMiscWaits = true,
+            bool excludeCdc = true)
         {
             maxResults = Math.Clamp(maxResults, 1, 1000);
 
@@ -763,9 +765,45 @@ private async Task<List<LongRunningQueryInfo>> GetLongRunningQueriesAsync(
                 ? "AND r.wait_type NOT IN (N'BACKUPTHREAD', N'BACKUPIO')" : "";
             string miscWaitsFilter = excludeMiscWaits
                 ? "AND r.wait_type NOT IN (N'XE_LIVE_TARGET_TVF')" : "";
+            // CDC capture runs continuously as a SQL Agent job (EXEC sys.sp_MScdc_capture_job -> sys.sp_cdc_scan),
+            // so it permanently exceeds the duration threshold and none of the wait_type filters above catch it.
+            //
+            // Primary signal: resolve the capture job_id(s) from msdb.dbo.cdc_jobs and match the running session via
+            // its SQL Agent program_name ('SQLAgent - TSQL JobStep (Job 0x<job_id> : Step N)'). This is CDC-specific
+            // and never hides unrelated Agent jobs. The msdb reference is deferred through sp_executesql inside
+            // TRY/CATCH so a login without msdb access gets a *catchable* error (not an uncatchable cross-db 916) and
+            // cleanly falls back to a text match on the whole batch/object text.
+            string cdcSetup = excludeCdc ? @"
+                DECLARE @cdc_capture_jobs TABLE (job_id uniqueidentifier PRIMARY KEY);
+                DECLARE @cdc_readable bit = 0;
+                BEGIN TRY
+                    INSERT @cdc_capture_jobs (job_id)
+                    EXEC sys.sp_executesql N'SELECT cj.job_id FROM msdb.dbo.cdc_jobs AS cj WHERE cj.job_type = N''capture'';';
+                    SET @cdc_readable = 1;
+                END TRY
+                BEGIN CATCH
+                    SET @cdc_readable = 0;
+                END CATCH;
+" : "";
+            string cdcFilter = excludeCdc ? @"
+                    AND NOT
+                    (
+                        (
+                            @cdc_readable = 1
+                            AND s.program_name LIKE N'SQLAgent - TSQL JobStep (Job 0x%'
+                            AND TRY_CONVERT(uniqueidentifier, TRY_CONVERT(binary(16), SUBSTRING(s.program_name, 32, 32), 2))
+                                IN (SELECT j.job_id FROM @cdc_capture_jobs AS j)
+                        )
+                        OR
+                        (
+                            @cdc_readable = 0
+                            AND t.text IS NOT NULL
+                            AND (t.text LIKE N'%sp_MScdc_capture_job%' OR t.text LIKE N'%sp_cdc_scan%')
+                        )
+                    )" : "";
 
             string query = @$"SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;
-
+                {cdcSetup}
                 SELECT TOP(@maxResults)
                     r.session_id,
                     DB_NAME(r.database_id) AS database_name,
@@ -787,6 +825,7 @@ CROSS APPLY sys.dm_exec_sql_text(r.sql_handle) AS t
                     {waitForFilter}
                     {backupsFilter}
                     {miscWaitsFilter}
+                    {cdcFilter}
                 ORDER BY r.total_elapsed_time DESC
                 OPTION(MAXDOP 1, RECOMPILE);";
 

Dashboard/SettingsWindow.xaml

@@ -246,6 +246,9 @@
                                       Margin="0,0,0,4"/>
                             <CheckBox x:Name="LrqExcludeMiscWaitsCheckBox"
                                       Content="Exclude miscellaneous waits (XE_LIVE_TARGET_TVF)"
+                                      Margin="0,0,0,4"/>
+                            <CheckBox x:Name="LrqExcludeCdcCheckBox"
+                                      Content="Exclude CDC capture jobs (sp_MScdc_capture_job, sp_cdc_scan)"
                                       Margin="0,0,0,0"/>
                             <StackPanel Orientation="Horizontal" Margin="0,8,0,0">
                                 <CheckBox x:Name="NotifyOnTempDbSpaceCheckBox"

Dashboard/SettingsWindow.xaml.cs

@@ -186,6 +186,7 @@ private void LoadSettings()
             LrqExcludeWaitForCheckBox.IsChecked = prefs.LongRunningQueryExcludeWaitFor;
             LrqExcludeBackupsCheckBox.IsChecked = prefs.LongRunningQueryExcludeBackups;
             LrqExcludeMiscWaitsCheckBox.IsChecked = prefs.LongRunningQueryExcludeMiscWaits;
+            LrqExcludeCdcCheckBox.IsChecked = prefs.LongRunningQueryExcludeCdc;
             AlertExcludedDatabasesTextBox.Text = string.Join(", ", prefs.AlertExcludedDatabases);
             NotifyOnTempDbSpaceCheckBox.IsChecked = prefs.NotifyOnTempDbSpace;
             TempDbSpaceThresholdTextBox.Text = prefs.TempDbSpaceThresholdPercent.ToString(CultureInfo.InvariantCulture);
@@ -683,6 +684,7 @@ private async void OkButton_Click(object sender, RoutedEventArgs e)
             prefs.LongRunningQueryExcludeWaitFor = LrqExcludeWaitForCheckBox.IsChecked == true;
             prefs.LongRunningQueryExcludeBackups = LrqExcludeBackupsCheckBox.IsChecked == true;
             prefs.LongRunningQueryExcludeMiscWaits = LrqExcludeMiscWaitsCheckBox.IsChecked == true;
+            prefs.LongRunningQueryExcludeCdc = LrqExcludeCdcCheckBox.IsChecked == true;
             prefs.AlertExcludedDatabases = AlertExcludedDatabasesTextBox.Text
                 .Split(',')
                 .Select(s => s.Trim())

Lite/App.xaml.cs

@@ -101,6 +101,7 @@ public partial class App : Application
     public static bool AlertLongRunningQueryExcludeWaitFor { get; set; } = true;
     public static bool AlertLongRunningQueryExcludeBackups { get; set; } = true;
     public static bool AlertLongRunningQueryExcludeMiscWaits { get; set; } = true;
+    public static bool AlertLongRunningQueryExcludeCdc { get; set; } = true;
     public static List<string> AlertExcludedDatabases { get; set; } = new();
     public static bool AlertTempDbSpaceEnabled { get; set; } = true;
     public static int AlertTempDbSpaceThresholdPercent { get; set; } = 80;
@@ -472,6 +473,7 @@ public static void LoadAlertSettings()
             if (root.TryGetProperty("alert_long_running_query_exclude_waitfor", out v)) AlertLongRunningQueryExcludeWaitFor = v.GetBoolean();
             if (root.TryGetProperty("alert_long_running_query_exclude_backups", out v)) AlertLongRunningQueryExcludeBackups = v.GetBoolean();
             if (root.TryGetProperty("alert_long_running_query_exclude_misc_waits", out v)) AlertLongRunningQueryExcludeMiscWaits = v.GetBoolean();
+            if (root.TryGetProperty("alert_long_running_query_exclude_cdc", out v)) AlertLongRunningQueryExcludeCdc = v.GetBoolean();
             if (root.TryGetProperty("alert_excluded_databases", out v) && v.ValueKind == System.Text.Json.JsonValueKind.Array)
             {
                 AlertExcludedDatabases = new List<string>();

Lite/Database/DuckDbInitializer.cs

@@ -97,7 +97,7 @@ public void Dispose()
     /// <summary>
     /// Current schema version. Increment this when schema changes require table rebuilds.
     /// </summary>
-    internal const int CurrentSchemaVersion = 27;
+    internal const int CurrentSchemaVersion = 28;
 
     private readonly string _archivePath;
 
@@ -732,6 +732,23 @@ New tables only — no existing table changes needed. Tables created by
                 throw;
             }
         }
+
+        if (fromVersion < 28)
+        {
+            /* v28: Added is_cdc_capture flag to query_snapshots so the long-running query
+                    alert can exclude CDC capture sessions. The collector computes the flag
+                    server-side (program_name -> job_id via msdb.dbo.cdc_jobs, text fallback).
+                    Appended at the end to match the DuckDB appender's positional order. */
+            _logger?.LogInformation("Running migration to v28: adding is_cdc_capture column to query_snapshots");
+            try
+            {
+                await ExecuteNonQueryAsync(connection, "ALTER TABLE query_snapshots ADD COLUMN IF NOT EXISTS is_cdc_capture BOOLEAN DEFAULT false");
+            }
+            catch (Exception ex)
+            {
+                _logger?.LogWarning("Migration to v28 encountered an error (non-fatal): {Error}", ex.Message);
+            }
+        }
     }
 
     /// <summary>

Lite/Database/Schema.cs

@@ -346,7 +346,8 @@ granted_query_memory_gb DECIMAL(18,2),
     host_name VARCHAR,
     program_name VARCHAR,
     open_transaction_count INTEGER,
-    percent_complete DECIMAL(5,2)
+    percent_complete DECIMAL(5,2),
+    is_cdc_capture BOOLEAN DEFAULT false
 )";
 
     public const string CreateTempdbStatsTable = @"

Lite/MainWindow.xaml.cs

@@ -1745,7 +1745,7 @@ await _emailAlertService.TrySendAlertEmailAsync(
         {
             try
             {
-                var longRunning = await _dataService.GetLongRunningQueriesAsync(summary.ServerId, App.AlertLongRunningQueryThresholdMinutes, App.AlertLongRunningQueryMaxResults, App.AlertLongRunningQueryExcludeSpServerDiagnostics, App.AlertLongRunningQueryExcludeWaitFor, App.AlertLongRunningQueryExcludeBackups, App.AlertLongRunningQueryExcludeMiscWaits);
+                var longRunning = await _dataService.GetLongRunningQueriesAsync(summary.ServerId, App.AlertLongRunningQueryThresholdMinutes, App.AlertLongRunningQueryMaxResults, App.AlertLongRunningQueryExcludeSpServerDiagnostics, App.AlertLongRunningQueryExcludeWaitFor, App.AlertLongRunningQueryExcludeBackups, App.AlertLongRunningQueryExcludeMiscWaits, App.AlertLongRunningQueryExcludeCdc);
 
                 if (App.AlertExcludedDatabases.Count > 0)
                 {

Lite/Services/LocalDataService.WaitStats.cs

@@ -436,7 +436,8 @@ public async Task<List<LongRunningQueryInfo>> GetLongRunningQueriesAsync(
         bool excludeSpServerDiagnostics = true,
         bool excludeWaitFor = true,
         bool excludeBackups = true,
-        bool excludeMiscWaits = true)
+        bool excludeMiscWaits = true,
+        bool excludeCdc = true)
     {
         using var connection = await OpenConnectionAsync();
         using var command = connection.CreateCommand();
@@ -451,6 +452,10 @@ public async Task<List<LongRunningQueryInfo>> GetLongRunningQueriesAsync(
             ? "AND r.wait_type NOT IN (N'BACKUPTHREAD', N'BACKUPIO')" : "";
         string miscWaitsFilter = excludeMiscWaits
             ? "AND r.wait_type NOT IN (N'XE_LIVE_TARGET_TVF')" : "";
+        // CDC capture sessions are flagged server-side by the collector (is_cdc_capture). COALESCE
+        // guards pre-migration / archived rows where the column is NULL.
+        string cdcFilter = excludeCdc
+            ? "AND COALESCE(r.is_cdc_capture, FALSE) = FALSE" : "";
         maxResults = Math.Clamp(maxResults, 1, 1000);
 
         command.CommandText = @$"
@@ -473,6 +478,7 @@ AND r.session_id > 50
                     {waitForFilter}
                     {backupsFilter}
                     {miscWaitsFilter}
+                    {cdcFilter}
                     AND r.total_elapsed_time_ms >= $2
                 ORDER BY r.total_elapsed_time_ms DESC
                 LIMIT $3;";