rabbit_mnesia: add retries

Occasionally, clustering will fail with the log
as pasted before. I believe it's because of the parallel
node startup, sometimes leading to crashes.

Hopefully, with retries, we'll handle this more gracefully.

```
Feature flags: nodes `rmq-ct-cluster_size_3_2-2-21072@localhost` and `rmq-ct-cluster_size_3_2-1-21000@localhost` are compatible

Mnesia('rmq-ct-cluster_size_3_2-2-21072@localhost'): ** ERROR ** (ignoring core) ** FATAL ** mnesia_monitor crashed:
{{badmatch, <0.203.0>, Ref<0.1988436133.884998146.137464>}},
{mnesia_monitor, handle_info, 2, [{file, "mnesia_monitor.erl"}, {line, 583}]},
gen_server, try_handle_info, 3, [{file, "gen_server.erl"}, {line, 2434}]},
gen_server, handle_msg, 3, [{file, "gen_server.erl"}, {line, 2420}]},
proc_lib, init_p_do_apply, 3, [{file, "proc_lib.erl"}, {line, 333}]}]}

Error in process <0.300.0> on node 'rmq-ct-cluster_size_3_2-2-21072@localhost' with exit value:
{badarg,[{erlang,send,
                [mnesia_locker,{release_tid,{tid,142,<24815.431.0>}}],
                [{error_info,#{module => erl_erts_errors}}]},
        {mnesia_locker,release_tid,1,[{file,"mnesia_locker.erl"},{line,128}]},
        {mnesia_tm,commit_participant,7,
                   [{file,"mnesia_tm.erl"},{line,1828}]}]}

Application mnesia exited with reason: stopped

BOOT FAILED
===========
Exception during startup:

Exit:{killed,{gen_server,call,[<0.280.0>,{negotiate_protocol,['rmq-ct-cluster_size_3_2-1-21000@localhost']},infinity]}}

   gen_server:call/3, line 1301
   mnesia_monitor:call/1, line 232
   rabbit_mnesia:-check_mnesia_consistency/2-fun-0-/2, line 1002
   rabbit_mnesia:with_running_or_clean_mnesia/1, line 1036
   rabbit_mnesia:check_cluster_consistency/2, line 719
   lists:foldl/3, line 2466
   rabbit_mnesia:check_cluster_consistency/0, line 680
   rabbit_prelaunch_cluster:setup/1, line 27
```

Author: mkuratczyk

deps/rabbit/src/rabbit_mnesia.erl

@@ -965,13 +965,27 @@ stop_mnesia() ->
     ensure_mnesia_not_running().
 
 change_extra_db_nodes(ClusterNodes0, CheckOtherNodes) ->
+    change_extra_db_nodes(ClusterNodes0, CheckOtherNodes, 3).
+
+change_extra_db_nodes(ClusterNodes0, CheckOtherNodes, RetriesLeft) ->
     ClusterNodes = rabbit_nodes:nodes_excl_me(ClusterNodes0),
     case {mnesia:change_config(extra_db_nodes, ClusterNodes), ClusterNodes} of
         {{ok, []}, [_|_]} when CheckOtherNodes ->
             throw({error, {failed_to_cluster_with, ClusterNodes,
                            "Mnesia could not connect to any nodes."}});
         {{ok, Nodes}, _} ->
-            Nodes
+            Nodes;
+        {{error, _} = Error, _} when RetriesLeft > 0 ->
+            ?LOG_WARNING(
+               "Failed to add extra Mnesia db nodes ~tp: ~tp. "
+               "Retrying (~b attempts left).",
+               [ClusterNodes, Error, RetriesLeft - 1]),
+            timer:sleep(1000),
+            change_extra_db_nodes(ClusterNodes0, CheckOtherNodes,
+                                  RetriesLeft - 1);
+        {{error, _} = Error, _} ->
+            throw({error, {failed_to_cluster_with, ClusterNodes,
+                           rabbit_misc:format("~tp", [Error])}})
     end.
 
 check_nodes_consistency(Node, {RemoteAllNodes, _, _}) ->
@@ -1011,8 +1025,27 @@ check_mnesia_consistency(Node, ProtocolVersion) ->
         end
     end).
 
-negotiate_protocol([Node]) ->
-    mnesia_monitor:negotiate_protocol([Node]).
+negotiate_protocol(Nodes) ->
+    negotiate_protocol(Nodes, 3).
+
+negotiate_protocol([Node], RetriesLeft) ->
+    try
+        mnesia_monitor:negotiate_protocol([Node])
+    catch
+        exit:Reason when RetriesLeft > 1 ->
+            ?LOG_WARNING(
+               "Mnesia protocol negotiation with node ~tp "
+               "failed: ~tp. Retrying (~b attempts left).",
+               [Node, Reason, RetriesLeft - 1]),
+            timer:sleep(1000),
+            negotiate_protocol([Node], RetriesLeft - 1);
+        exit:Reason ->
+            ?LOG_WARNING(
+               "Mnesia protocol negotiation with node ~tp "
+               "failed: ~tp. No retries left.",
+               [Node, Reason]),
+            []
+    end.
 
 with_running_or_clean_mnesia(Fun) ->
     IsMnesiaRunning = case mnesia:system_info(is_running) of