Merge pull request #13577 from rabbitmq/mergify/bp/v4.1.x/pr-13571

michaelklishin · web-flow · commit f5dcc91eb936 · 2025-03-19T15:18:50.000-04:00
New HTTP API health check endpoints (backport #13571)
diff --git a/deps/rabbitmq_management/src/rabbit_mgmt_dispatcher.erl b/deps/rabbitmq_management/src/rabbit_mgmt_dispatcher.erl
@@ -202,6 +202,10 @@ dispatcher() ->
      {"/health/checks/port-listener/:port",                    rabbit_mgmt_wm_health_check_port_listener, []},
      {"/health/checks/protocol-listener/:protocol",            rabbit_mgmt_wm_health_check_protocol_listener, []},
      {"/health/checks/virtual-hosts",                          rabbit_mgmt_wm_health_check_virtual_hosts, []},
+     {"/health/checks/quorum-queues-without-elected-leaders/all-vhosts/",    rabbit_mgmt_wm_health_check_quorum_queues_without_elected_leaders_across_all_vhosts, []},
+     {"/health/checks/quorum-queues-without-elected-leaders/vhost/:vhost/",  rabbit_mgmt_wm_health_check_quorum_queues_without_elected_leaders, []},
+     {"/health/checks/quorum-queues-without-elected-leaders/all-vhosts/pattern/:pattern",    rabbit_mgmt_wm_health_check_quorum_queues_without_elected_leaders_across_all_vhosts, []},
+     {"/health/checks/quorum-queues-without-elected-leaders/vhost/:vhost/pattern/:pattern",  rabbit_mgmt_wm_health_check_quorum_queues_without_elected_leaders, []},
      {"/health/checks/node-is-quorum-critical",                rabbit_mgmt_wm_health_check_node_is_quorum_critical, []},
      {"/reset",                                                rabbit_mgmt_wm_reset, []},
      {"/reset/:node",                                          rabbit_mgmt_wm_reset, []},
diff --git a/deps/rabbitmq_management/src/rabbit_mgmt_wm_health_check_quorum_queues_without_elected_leaders.erl b/deps/rabbitmq_management/src/rabbit_mgmt_wm_health_check_quorum_queues_without_elected_leaders.erl
@@ -0,0 +1,68 @@
+%% This Source Code Form is subject to the terms of the Mozilla Public
+%% License, v. 2.0. If a copy of the MPL was not distributed with this
+%% file, You can obtain one at https://mozilla.org/MPL/2.0/.
+%%
+%% Copyright (c) 2007-2025 Broadcom. All Rights Reserved. The term “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. All rights reserved.
+%%
+
+%% An HTTP API counterpart of 'rabbitmq-diagnostics check_for_quorum_queues_without_an_elected_leader'
+-module(rabbit_mgmt_wm_health_check_quorum_queues_without_elected_leaders).
+
+-export([init/2, to_json/2, content_types_provided/2, is_authorized/2]).
+-export([resource_exists/2]).
+-export([variances/2]).
+
+-include_lib("rabbitmq_management_agent/include/rabbit_mgmt_records.hrl").
+
+-define(DEFAULT_PATTERN, <<".*">>).
+
+%%--------------------------------------------------------------------
+
+init(Req, _State) ->
+    {cowboy_rest, rabbit_mgmt_headers:set_common_permission_headers(Req, ?MODULE), #context{}}.
+
+variances(Req, Context) ->
+    {[<<"accept-encoding">>, <<"origin">>], Req, Context}.
+
+content_types_provided(ReqData, Context) ->
+    {rabbit_mgmt_util:responder_map(to_json), ReqData, Context}.
+
+resource_exists(ReqData, Context) ->
+    Result = case {vhost(ReqData), pattern(ReqData)} of
+                 {none, _} -> false;
+                 {_, none} -> false;
+                 _         -> true
+             end,
+    {Result, ReqData, Context}.
+
+to_json(ReqData, Context) ->
+    case rabbit_quorum_queue:leader_health_check(pattern(ReqData), vhost(ReqData)) of
+        [] ->
+            rabbit_mgmt_util:reply(#{status => ok}, ReqData, Context);
+        Qs when length(Qs) > 0 ->
+            Msg = <<"Detected quorum queues without an elected leader">>,
+            failure(Msg, Qs, ReqData, Context)
+    end.
+
+failure(Message, Qs, ReqData, Context) ->
+    Body = #{status => failed,
+             reason => Message,
+             queues => Qs},
+    {Response, ReqData1, Context1} = rabbit_mgmt_util:reply(Body, ReqData, Context),
+    {stop, cowboy_req:reply(503, #{}, Response, ReqData1), Context1}.
+
+is_authorized(ReqData, Context) ->
+    rabbit_mgmt_util:is_authorized(ReqData, Context).
+
+%%
+%% Implementation
+%%
+
+vhost(ReqData) ->
+    rabbit_mgmt_util:id(vhost, ReqData).
+
+pattern(ReqData) ->
+    case rabbit_mgmt_util:id(pattern, ReqData) of
+        none  -> ?DEFAULT_PATTERN;
+        Other -> Other
+    end.
diff --git a/deps/rabbitmq_management/src/rabbit_mgmt_wm_health_check_quorum_queues_without_elected_leaders_across_all_vhosts.erl b/deps/rabbitmq_management/src/rabbit_mgmt_wm_health_check_quorum_queues_without_elected_leaders_across_all_vhosts.erl
@@ -0,0 +1,61 @@
+%% This Source Code Form is subject to the terms of the Mozilla Public
+%% License, v. 2.0. If a copy of the MPL was not distributed with this
+%% file, You can obtain one at https://mozilla.org/MPL/2.0/.
+%%
+%% Copyright (c) 2007-2025 Broadcom. All Rights Reserved. The term “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. All rights reserved.
+%%
+
+%% An HTTP API counterpart of 'rabbitmq-diagnostics check_for_quorum_queues_without_an_elected_leader --across-all-vhosts'
+-module(rabbit_mgmt_wm_health_check_quorum_queues_without_elected_leaders_across_all_vhosts).
+
+-export([init/2, to_json/2, content_types_provided/2, is_authorized/2]).
+-export([resource_exists/2]).
+-export([variances/2]).
+
+-include_lib("rabbitmq_management_agent/include/rabbit_mgmt_records.hrl").
+
+-define(ACROSS_ALL_VHOSTS, across_all_vhosts).
+-define(DEFAULT_PATTERN, <<".*">>).
+
+%%--------------------------------------------------------------------
+
+init(Req, _State) ->
+    {cowboy_rest, rabbit_mgmt_headers:set_common_permission_headers(Req, ?MODULE), #context{}}.
+
+variances(Req, Context) ->
+    {[<<"accept-encoding">>, <<"origin">>], Req, Context}.
+
+content_types_provided(ReqData, Context) ->
+    {rabbit_mgmt_util:responder_map(to_json), ReqData, Context}.
+
+resource_exists(ReqData, Context) ->
+    {true, ReqData, Context}.
+
+to_json(ReqData, Context) ->
+    case rabbit_quorum_queue:leader_health_check(pattern(ReqData), ?ACROSS_ALL_VHOSTS) of
+        [] ->
+            rabbit_mgmt_util:reply(#{status => ok}, ReqData, Context);
+        Qs when length(Qs) > 0 ->
+            Msg = <<"Detected quorum queues without an elected leader">>,
+            failure(Msg, Qs, ReqData, Context)
+    end.
+
+failure(Message, Qs, ReqData, Context) ->
+    Body = #{status => failed,
+        reason => Message,
+        queues => Qs},
+    {Response, ReqData1, Context1} = rabbit_mgmt_util:reply(Body, ReqData, Context),
+    {stop, cowboy_req:reply(503, #{}, Response, ReqData1), Context1}.
+
+is_authorized(ReqData, Context) ->
+    rabbit_mgmt_util:is_authorized(ReqData, Context).
+
+%%
+%% Implementation
+%%
+
+pattern(ReqData) ->
+    case rabbit_mgmt_util:id(pattern, ReqData) of
+        none  -> ?DEFAULT_PATTERN;
+        Other -> Other
+    end.
diff --git a/deps/rabbitmq_management/test/rabbit_mgmt_http_health_checks_SUITE.erl b/deps/rabbitmq_management/test/rabbit_mgmt_http_health_checks_SUITE.erl
@@ -37,7 +37,10 @@ groups() ->
                         local_alarms_test,
                         metadata_store_initialized_test,
                         metadata_store_initialized_with_data_test,
-                        is_quorum_critical_single_node_test]}
+                        is_quorum_critical_single_node_test,
+                        quorum_queues_without_elected_leader_single_node_test,
+                        quorum_queues_without_elected_leader_across_all_virtual_hosts_single_node_test
+     ]}
     ].
 
 all_tests() -> [
@@ -165,7 +168,8 @@ local_alarms_test(Config) ->
 
 
 is_quorum_critical_single_node_test(Config) ->
-    Check0 = http_get(Config, "/health/checks/node-is-quorum-critical", ?OK),
+    EndpointPath = "/health/checks/node-is-quorum-critical",
+    Check0 = http_get(Config, EndpointPath, ?OK),
     ?assertEqual(<<"single node cluster">>, maps:get(reason, Check0)),
     ?assertEqual(<<"ok">>, maps:get(status, Check0)),
 
@@ -178,13 +182,14 @@ is_quorum_critical_single_node_test(Config) ->
                                                         durable   = true,
                                                         auto_delete = false,
                                                         arguments = Args})),
-    Check1 = http_get(Config, "/health/checks/node-is-quorum-critical", ?OK),
+    Check1 = http_get(Config, EndpointPath, ?OK),
     ?assertEqual(<<"single node cluster">>, maps:get(reason, Check1)),
 
     passed.
 
 is_quorum_critical_test(Config) ->
-    Check0 = http_get(Config, "/health/checks/node-is-quorum-critical", ?OK),
+    EndpointPath = "/health/checks/node-is-quorum-critical",
+    Check0 = http_get(Config, EndpointPath, ?OK),
     ?assertEqual(false, maps:is_key(reason, Check0)),
     ?assertEqual(<<"ok">>, maps:get(status, Check0)),
 
@@ -198,7 +203,7 @@ is_quorum_critical_test(Config) ->
                                                         durable   = true,
                                                         auto_delete = false,
                                                         arguments = Args})),
-    Check1 = http_get(Config, "/health/checks/node-is-quorum-critical", ?OK),
+    Check1 = http_get(Config, EndpointPath, ?OK),
     ?assertEqual(false, maps:is_key(reason, Check1)),
 
     RaName = binary_to_atom(<<"%2F_", QName/binary>>, utf8),
@@ -207,7 +212,104 @@ is_quorum_critical_test(Config) ->
     ok = rabbit_ct_broker_helpers:stop_node(Config, Server2),
     ok = rabbit_ct_broker_helpers:stop_node(Config, Server3),
 
-    Body = http_get_failed(Config, "/health/checks/node-is-quorum-critical"),
+    Body = http_get_failed(Config, EndpointPath),
+    ?assertEqual(<<"failed">>, maps:get(<<"status">>, Body)),
+    ?assertEqual(true, maps:is_key(<<"reason">>, Body)),
+    Queues = maps:get(<<"queues">>, Body),
+    ?assert(lists:any(
+        fun(Item) ->
+            QName =:= maps:get(<<"name">>, Item)
+        end, Queues)),
+
+    passed.
+
+quorum_queues_without_elected_leader_single_node_test(Config) ->
+    EndpointPath = "/health/checks/quorum-queues-without-elected-leaders/all-vhosts/",
+    Check0 = http_get(Config, EndpointPath, ?OK),
+    ?assertEqual(false, maps:is_key(reason, Check0)),
+    ?assertEqual(<<"ok">>, maps:get(status, Check0)),
+
+    [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename),
+    Ch = rabbit_ct_client_helpers:open_channel(Config, Server),
+    Args = [{<<"x-queue-type">>, longstr, <<"quorum">>},
+            {<<"x-quorum-initial-group-size">>, long, 3}],
+    QName = <<"quorum_queues_without_elected_leader">>,
+    ?assertEqual({'queue.declare_ok', QName, 0, 0},
+        amqp_channel:call(Ch, #'queue.declare'{
+            queue       = QName,
+            durable     = true,
+            auto_delete = false,
+            arguments   = Args
+        })),
+
+    Check1 = http_get(Config, EndpointPath, ?OK),
+    ?assertEqual(false, maps:is_key(reason, Check1)),
+
+    RaSystem = quorum_queues,
+    QResource = rabbit_misc:r(<<"/">>, queue, QName),
+    {ok, Q1} = rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_db_queue, get, [QResource]),
+
+    _ = rabbit_ct_broker_helpers:rpc(Config, 0, ra, stop_server, [RaSystem, amqqueue:get_pid(Q1)]),
+
+    Body = http_get_failed(Config, EndpointPath),
+    ?assertEqual(<<"failed">>, maps:get(<<"status">>, Body)),
+    ?assertEqual(true, maps:is_key(<<"reason">>, Body)),
+    Queues = maps:get(<<"queues">>, Body),
+    ?assert(lists:any(
+        fun(Item) ->
+            QName =:= maps:get(<<"name">>, Item)
+        end, Queues)),
+
+    _ = rabbit_ct_broker_helpers:rpc(Config, 0, ra, restart_server, [RaSystem, amqqueue:get_pid(Q1)]),
+    rabbit_ct_helpers:await_condition(
+        fun() ->
+            try
+                Check2 = http_get(Config, EndpointPath, ?OK),
+                false =:= maps:is_key(reason, Check2)
+            catch _:_ ->
+                false
+            end
+        end),
+
+    passed.
+
+quorum_queues_without_elected_leader_across_all_virtual_hosts_single_node_test(Config) ->
+    VH2 = <<"vh-2">>,
+    rabbit_ct_broker_helpers:add_vhost(Config, VH2),
+
+    EndpointPath1 = "/health/checks/quorum-queues-without-elected-leaders/vhost/%2f/",
+    EndpointPath2 = "/health/checks/quorum-queues-without-elected-leaders/vhost/vh-2/",
+    %% ^other
+    EndpointPath3 = "/health/checks/quorum-queues-without-elected-leaders/vhost/vh-2/pattern/%5Eother",
+
+    Check0 = http_get(Config, EndpointPath1, ?OK),
+    Check0 = http_get(Config, EndpointPath2, ?OK),
+    ?assertEqual(false, maps:is_key(reason, Check0)),
+    ?assertEqual(<<"ok">>, maps:get(status, Check0)),
+
+    [Server | _] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename),
+    Ch = rabbit_ct_client_helpers:open_channel(Config, Server),
+    Args = [{<<"x-queue-type">>, longstr, <<"quorum">>},
+        {<<"x-quorum-initial-group-size">>, long, 3}],
+    QName = <<"quorum_queues_without_elected_leader_across_all_virtual_hosts_single_node_test">>,
+    ?assertEqual({'queue.declare_ok', QName, 0, 0},
+        amqp_channel:call(Ch, #'queue.declare'{
+            queue       = QName,
+            durable     = true,
+            auto_delete = false,
+            arguments   = Args
+        })),
+
+    Check1 = http_get(Config, EndpointPath1, ?OK),
+    ?assertEqual(false, maps:is_key(reason, Check1)),
+
+    RaSystem = quorum_queues,
+    QResource = rabbit_misc:r(<<"/">>, queue, QName),
+    {ok, Q1} = rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_db_queue, get, [QResource]),
+
+    _ = rabbit_ct_broker_helpers:rpc(Config, 0, ra, stop_server, [RaSystem, amqqueue:get_pid(Q1)]),
+
+    Body = http_get_failed(Config, EndpointPath1),
     ?assertEqual(<<"failed">>, maps:get(<<"status">>, Body)),
     ?assertEqual(true, maps:is_key(<<"reason">>, Body)),
     Queues = maps:get(<<"queues">>, Body),
@@ -216,8 +318,30 @@ is_quorum_critical_test(Config) ->
             QName =:= maps:get(<<"name">>, Item)
         end, Queues)),
 
+    %% virtual host vh-2 is still fine
+    Check2 = http_get(Config, EndpointPath2, ?OK),
+    ?assertEqual(false, maps:is_key(reason, Check2)),
+
+    %% a different queue name pattern succeeds
+    Check3 = http_get(Config, EndpointPath3, ?OK),
+    ?assertEqual(false, maps:is_key(reason, Check3)),
+
+    _ = rabbit_ct_broker_helpers:rpc(Config, 0, ra, restart_server, [RaSystem, amqqueue:get_pid(Q1)]),
+    rabbit_ct_helpers:await_condition(
+        fun() ->
+            try
+                Check4 = http_get(Config, EndpointPath1, ?OK),
+                false =:= maps:is_key(reason, Check4)
+            catch _:_ ->
+                false
+            end
+        end),
+
+    rabbit_ct_broker_helpers:delete_vhost(Config, VH2),
+
     passed.
 
+
 virtual_hosts_test(Config) ->
     VHost1 = <<"vhost1">>,
     VHost2 = <<"vhost2">>,
