Skip to content

michal/mnesia/fix-mnesia-force-load-table-getting-stuck/OTP-20256 #11426

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Mikaka27 wants to merge 3 commits into
base: maint
Choose a base branch
from
Open

michal/mnesia/fix-mnesia-force-load-table-getting-stuck/OTP-20256 #11426

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
08c4bd7
Skip not implemented test and make clean_up_suite last
Mikaka27 Aug 22, 2026
90263eb
Add possibility to simulate network outage between nodes
Mikaka27 Aug 25, 2026
c4c796d
Fix mnesia:force_load_table/1 getting stuck
Mikaka27 Aug 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
120 changes: 62 additions & 58 deletions lib/mnesia/src/mnesia_controller.erl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1184,71 +1184,75 @@ handle_info(#dumper_done{worker_pid=Pid, worker_res=Res}, State) ->
{stop, fatal, State}
end;

handle_info(Done = #loader_done{worker_pid=WPid, table_name=Tab}, State0) ->
handle_info(Done = #loader_done{worker_pid=WPid, table_name=Tab, reply=Reply}, State0) ->
LateQueue0 = State0#state.late_loader_queue,
State1 = State0#state{loader_pid = lists:keydelete(WPid,1,get_loaders(State0))},
{value, {WPid, Worker}, Loaders} = lists:keytake(WPid, 1, get_loaders(State0)),
State1 = State0#state{loader_pid = Loaders},

State2 =
case Done#loader_done.is_loaded of
true ->
%% Optional table announcement
if
Done#loader_done.needs_announce == true,
Done#loader_done.needs_reply == true ->
i_have_tab(Tab),
%% Should be {dumper,{add_table_copy, _}} only
reply(Done#loader_done.reply_to,
Done#loader_done.reply);
Done#loader_done.needs_reply == true ->
%% Should be {dumper,{add_table_copy,_}} only
reply(Done#loader_done.reply_to,
Done#loader_done.reply);
Done#loader_done.needs_announce == true, Tab == schema ->
i_have_tab(Tab);
Done#loader_done.needs_announce == true ->
i_have_tab(Tab),
%% Local node needs to perform user_sync_tab/1
Ns = val({current, db_nodes}),
abcast(Ns, {i_have_tab, Tab, node()});
Tab == schema ->
ignore;
true ->
%% Local node needs to perform user_sync_tab/1
Ns = val({current, db_nodes}),
AlreadyKnows = val({Tab, active_replicas}),
abcast(Ns -- AlreadyKnows, {i_have_tab, Tab, node()})
end,
%% Optional user sync
case Done#loader_done.needs_sync of
true -> user_sync_tab(Tab);
false -> ignore
end,
State1#state{late_loader_queue=gb_trees:delete_any(Tab, LateQueue0)};
false ->
%% Either the node went down or table was not
%% loaded remotly yet
case Done#loader_done.needs_reply of
true ->
reply(Done#loader_done.reply_to,
Done#loader_done.reply);
false ->
ignore
end,

case {?catch_val({Tab, storage_type}), val({Tab, active_replicas})} of
{unknown, _} -> %% Should not have a local copy anymore
case Done#loader_done.is_loaded of
true ->
%% Optional table announcement
if
Done#loader_done.needs_announce == true,
Done#loader_done.needs_reply == true ->
i_have_tab(Tab),
%% Should be {dumper,{add_table_copy, _}} only
reply(Done#loader_done.reply_to, Reply);
Done#loader_done.needs_reply == true ->
%% Should be {dumper,{add_table_copy,_}} only
reply(Done#loader_done.reply_to, Reply);
Done#loader_done.needs_announce == true, Tab == schema ->
i_have_tab(Tab);
Done#loader_done.needs_announce == true ->
i_have_tab(Tab),
%% Local node needs to perform user_sync_tab/1
Ns = val({current, db_nodes}),
abcast(Ns, {i_have_tab, Tab, node()});
Tab == schema ->
ignore;
true ->
%% Local node needs to perform user_sync_tab/1
Ns = val({current, db_nodes}),
AlreadyKnows = val({Tab, active_replicas}),
abcast(Ns -- AlreadyKnows, {i_have_tab, Tab, node()})
end,
%% Optional user sync
case Done#loader_done.needs_sync of
true -> user_sync_tab(Tab);
false -> ignore
end,
State1#state{late_loader_queue=gb_trees:delete_any(Tab, LateQueue0)};
false ->
%% Either the node went down or table was not
%% loaded remotly yet
case Done#loader_done.needs_reply of
true ->
reply(Done#loader_done.reply_to, Reply);
false ->
ignore
end,

case {?catch_val({Tab, storage_type}), val({Tab, active_replicas}), ?catch_val({Tab, load_by_force})} of
{unknown, _, _} -> %% Should not have a local copy anymore
State1#state{late_loader_queue=gb_trees:delete_any(Tab, LateQueue0)};
{_, [_|_]} -> % still available elsewhere
{value,{_,Worker}} = lists:keysearch(WPid,1,get_loaders(State0)),
add_loader(Tab,Worker,State1);
{ram_copies, []} ->
DelState = State1#state{late_loader_queue=gb_trees:delete_any(Tab, LateQueue0)},
{_, [_|_], _} -> % still available elsewhere
add_loader(Tab,Worker,State1);
{ram_copies, [], _} ->
DelState = State1#state{late_loader_queue=gb_trees:delete_any(Tab, LateQueue0)},
cast({disc_load, Tab, ram_only}),
DelState;
{_, []} -> %% Table deleted or not loaded anywhere
{_, [], true} when is_record(Worker, net_load), Reply =:= {not_loaded, none_active} ->
%% Network load could have been aborted by Sender node going down,
%% if user has forced the load, we retry loading from disc
DelState = State1#state{late_loader_queue=gb_trees:delete_any(Tab, LateQueue0)},
cast({disc_load, Tab, forced_by_user}),
DelState;
{_, [], _} ->
%% Table deleted or not loaded anywhere
State1#state{late_loader_queue=gb_trees:delete_any(Tab, LateQueue0)}
end
end,
end
end,
State3 = opt_start_worker(State2),
noreply(State3);

Expand Down
108 changes: 83 additions & 25 deletions lib/mnesia/src/mnesia_loader.erl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@
net_load_table/4,
send_table/4]).

-export([spawned_receiver/8]). %% Spawned lock taking process
-export([spawned_receiver/7]). %% Spawned lock taking process

-import(mnesia_lib, [set/2, fatal/2, verbose/2, dbg_out/2]).

Expand Down Expand Up @@ -281,9 +281,8 @@ init_receiver(Node, Tab,Storage,Cs,Reason) ->
true = lists:member(Node, Active),
{SenderPid, TabSize, DetsData} =
start_remote_sender(Node,Tab,Storage,load),
Init = table_init_fun(SenderPid, Storage),
Args = [self(),Tab,Storage,Cs,SenderPid,
TabSize,DetsData,Init],
Args = [self(),Tab,Storage,Cs,SenderPid,
TabSize,DetsData],
Pid = spawn_link(?MODULE, spawned_receiver, Args),
put(mnesia_real_loader, Pid),
wait_on_load_complete(Pid)
Expand Down Expand Up @@ -339,7 +338,18 @@ start_remote_sender(Node,Tab,Storage, Why) ->
end.

table_init_fun(SenderPid, Storage) ->
Parent = self(),
fun(read) ->
case Storage of
disc_only_copies ->
%% For disc_only_copies this is running inside dets_server process,
%% Parent is the process which is registered as receiver, and must
%% have our pid in order to forward us any {copier_done, Node} sent to it.
put(mnesia_table_dets_receiver, {Parent, SenderPid}),
Parent ! {dets_receiver_pid, self()};
_ ->
ok
end,
% We want to store subscribed mnesia table events received during
% table copying for later processing to not let receiver message queue
% to grow too much (which in consequence would slow down the whole copying process)
Expand All @@ -364,7 +374,8 @@ start_receiver(Tab,Storage,Cs,SenderPid,TabSize,DetsData,{dumper,{add_table_copy
Else
end.

spawned_receiver(ReplyTo,Tab,Storage,Cs, SenderPid,TabSize,DetsData, Init) ->
spawned_receiver(ReplyTo,Tab,Storage,Cs, SenderPid,TabSize,DetsData) ->
Init = table_init_fun(SenderPid, Storage),
process_flag(trap_exit, true),
Done = do_init_table(Tab,Storage,Cs,
SenderPid,TabSize,DetsData,
Expand Down Expand Up @@ -546,26 +557,43 @@ init_table(Tab, {ext,Alias,Mod}, Fun, State, Sender) ->
ext_init_table(Alias, Mod, Tab, Fun, State, Sender);
init_table(Tab, disc_only_copies, Fun, DetsInfo,Sender) ->
ErtsVer = erlang:system_info(version),
case DetsInfo of
{ErtsVer, DetsData} ->
try dets:is_compatible_bchunk_format(Tab, DetsData) of
false ->
Sender ! {self(), {old_protocol, Tab}},
dets:init_table(Tab, Fun); %% Old dets version
true ->
dets:init_table(Tab, Fun, [{format, bchunk}])
catch
error:{undef,[{dets,_,_,_}|_]} ->
Sender ! {self(), {old_protocol, Tab}},
dets:init_table(Tab, Fun); %% Old dets version
error:What ->
What
end;
Old when Old /= false ->
Sender ! {self(), {old_protocol, Tab}},
dets:init_table(Tab, Fun); %% Old dets version
_ ->
dets:init_table(Tab, Fun)
Parent = self(),
DetsInit = fun(Opts) ->
fun() ->
put(mnesia_dets_worker, {Tab, node(Sender), Sender}),
Res = dets:init_table(Tab, Fun, Opts),
Parent ! {self(), Res},
unlink(Parent),
exit(normal)
end
end,
Worker =
case DetsInfo of
{ErtsVer, DetsData} ->
try dets:is_compatible_bchunk_format(Tab, DetsData) of
false ->
Sender ! {self(), {old_protocol, Tab}},
spawn_link(DetsInit([])); %% Old dets version
true ->
spawn_link(DetsInit([{format, bchunk}]))
catch
error:{undef,[{dets,_,_,_}|_]} ->
Sender ! {self(), {old_protocol, Tab}},
spawn_link(DetsInit([])); %% Old dets version
error:What ->
What
end;
Old when Old /= false ->
Sender ! {self(), {old_protocol, Tab}},
spawn_link(DetsInit([])); %% Old dets version
_ ->
spawn_link(DetsInit([]))
end,
case is_pid(Worker) of
true ->
init_dets_receiver(Worker);
_ ->
Worker
end;
init_table(Tab, _, Fun, _DetsInfo,_) ->
try
Expand All @@ -574,6 +602,36 @@ init_table(Tab, _, Fun, _DetsInfo,_) ->
catch _:Else:Stacktrace -> {Else, Stacktrace}
end.

init_dets_receiver(Worker) ->
receive
{dets_receiver_pid, Receiver} ->
%% We got the pid of the dets receiver, we can start forwarding {copier_done, Node}
handle_dets_receiver(Worker, Receiver);
{Worker, Result} ->
%% Dets worker has finished with ok | {error, Reason} before we got receiver pid
Result;
{'EXIT', Pid, Reason} ->
%% If some local pid crashes we also crash
handle_exit(Pid, Reason),
init_dets_receiver(Worker)
end.

handle_dets_receiver(Worker, Receiver) ->
receive
{Worker, Result} ->
%% Dets worker has finished with ok | {error, Reason}
Result;
{copier_done, Node} ->
%% Forward any {copier_done, Node} blindly
%% they will either be ignored, or cause Worker
%% to finish with ok | {error, Reason}
Receiver ! {copier_done, Node},
handle_dets_receiver(Worker, Receiver);
{'EXIT', Pid, Reason} ->
%% If some local pid crashes we also crash
handle_exit(Pid, Reason),
handle_dets_receiver(Worker, Receiver)
end.

finish_copy(Storage,Tab,Cs,SenderPid,DatBin,OrigTabRec) ->
TabRef = {Storage, Tab},
Expand Down
5 changes: 3 additions & 2 deletions lib/mnesia/test/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,8 @@ MODULES= \
ext_test \
ext_test_server \
mnesia_index_plugin_test \
mnesia_external_backend_test
mnesia_external_backend_test \
gen_tcp_blocking_dist

DocExamplesDir := ../doc/src/

Expand All @@ -78,7 +79,7 @@ ExampleModules = \

ERL_FILES= $(MODULES:%=%.erl) $(DocExampleModules:%=$(DocExamplesDir)/%.erl) $(ExampleModules:%=$(ExamplesDir)/%.erl)

HRL_FILES= mnesia_test_lib.hrl ext_test_server.hrl $(DocExamplesHrl:%=$(DocExamplesDir)/%)
HRL_FILES= mnesia_test_lib.hrl ext_test_server.hrl gen_tcp_blocking_dist.hrl $(DocExamplesHrl:%=$(DocExamplesDir)/%)

TARGET_FILES= \
$(MODULES:%=$(EBIN)/%.$(EMULATOR)) $(DocExampleModules:%=$(EBIN)/%.$(EMULATOR)) $(ExampleModules:%=$(EBIN)/%.$(EMULATOR))
Expand Down
Loading
Loading