Skip to content

Commit 0c3c705

Browse files
Fix concurrency issues on TransactionScope timeout (#3483)
1 parent f938239 commit 0c3c705

22 files changed

+610
-70
lines changed

.editorconfig

+1
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ root=true
22

33
[*]
44
insert_final_newline = true
5+
charset = utf-8
56

67
[*.cs]
78
indent_style = tab

build-common/teamcity-hibernate.cfg.xml

+2
Original file line numberDiff line numberDiff line change
@@ -26,5 +26,7 @@
2626
<property name="odbc.explicit_datetime_scale"></property>
2727
<property name="oracle.use_n_prefixed_types_for_unicode"></property>
2828
<property name="query.default_cast_length"></property>
29+
<property name="transaction.ignore_session_synchronization_failures"></property>
30+
<property name="transaction.system_completion_lock_timeout"></property>
2931
</session-factory>
3032
</hibernate-configuration>

default.build

+20
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,26 @@
139139
<namespace prefix="hbm" uri="urn:nhibernate-configuration-2.2" />
140140
</namespaces>
141141
</xmlpoke>
142+
143+
<!-- Make sure the property exists - it's only set for some scenarios. -->
144+
<property name="nhibernate.transaction.ignore_session_synchronization_failures" value="" unless="${property::exists('nhibernate.transaction.ignore_session_synchronization_failures')}"/>
145+
<xmlpoke file="${app.config}"
146+
xpath="//*/hbm:property[@name='transaction.ignore_session_synchronization_failures']"
147+
value="${nhibernate.transaction.ignore_session_synchronization_failures}">
148+
<namespaces>
149+
<namespace prefix="hbm" uri="urn:nhibernate-configuration-2.2" />
150+
</namespaces>
151+
</xmlpoke>
152+
153+
<!-- Make sure the property exists - it's only set for some scenarios. -->
154+
<property name="nhibernate.transaction.system_completion_lock_timeout" value="" unless="${property::exists('nhibernate.transaction.system_completion_lock_timeout')}"/>
155+
<xmlpoke file="${app.config}"
156+
xpath="//*/hbm:property[@name='transaction.system_completion_lock_timeout']"
157+
value="${nhibernate.transaction.system_completion_lock_timeout}">
158+
<namespaces>
159+
<namespace prefix="hbm" uri="urn:nhibernate-configuration-2.2" />
160+
</namespaces>
161+
</xmlpoke>
142162
</target>
143163

144164
<target name="put-connection-settings-into-app-config">

doc/reference/modules/configuration.xml

+29-2
Original file line numberDiff line numberDiff line change
@@ -1050,8 +1050,8 @@ var session = sessions.OpenSession(conn);
10501050
after scope disposal. This occurs when the transaction is distributed.
10511051
This notably concerns <literal>ISessionImplementor.AfterTransactionCompletion(bool, ITransaction)</literal>.
10521052
NHibernate protects the session from being concurrently used by the code following the scope disposal
1053-
with a lock. To prevent any application freeze, this lock has a default timeout of five seconds. If the
1054-
application appears to require longer (!) running transaction completion events, this setting allows to
1053+
with a lock. To prevent any application freeze, this lock has a default timeout of one second. If the
1054+
application appears to require longer running transaction completion events, this setting allows to
10551055
raise this timeout. <literal>-1</literal> disables the timeout.
10561056
</para>
10571057
<para>
@@ -1060,6 +1060,33 @@ var session = sessions.OpenSession(conn);
10601060
</para>
10611061
</entry>
10621062
</row>
1063+
<row>
1064+
<entry>
1065+
<literal>transaction.ignore_session_synchronization_failures</literal>
1066+
</entry>
1067+
<entry>
1068+
Whether session synchronisation failures occuring during finalizations of system transaction should be
1069+
ignored or not. <literal>false</literal> by default.
1070+
<para>
1071+
When a system transaction terminates abnormaly, especially through timeouts, it may have its
1072+
completion events running on concurrent threads while the session is still performing some processing.
1073+
To prevent threading concurrency failures, NHibernate then wait for the session to end its processing,
1074+
up to <literal>transaction.system_completion_lock_timeout</literal>. If the session processing is still ongoing
1075+
afterwards, it will by default log an error, perform transaction finalization processing concurrently,
1076+
then throw a synchronization error. This setting allows to disable that later throw.
1077+
</para>
1078+
<para>
1079+
Disabling the throw can be useful if the used data provider has its own locking mechanism applied
1080+
during transaction completion, preventing the session to end its processing. It may then be safe to
1081+
ignore this synchronization failure. In case of threading concurrency failure, you may then need to
1082+
raise <literal>transaction.system_completion_lock_timeout</literal>.
1083+
</para>
1084+
<para>
1085+
<emphasis role="strong">eg.</emphasis>
1086+
<literal>true</literal> | <literal>false</literal>
1087+
</para>
1088+
</entry>
1089+
</row>
10631090
<row>
10641091
<entry>
10651092
<literal>transaction.auto_join</literal>

psake.ps1

+2
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,8 @@ Task Set-Configuration {
2929
'connection.connection_string' = 'Server=(local)\SQL2017;Uid=sa;Pwd=Password12!;Database=nhibernateOdbc;Driver={SQL Server Native Client 11.0};Mars_Connection=yes;';
3030
'connection.driver_class' = 'NHibernate.Driver.OdbcDriver';
3131
'odbc.explicit_datetime_scale' = '3';
32+
'transaction.ignore_session_synchronization_failures' = 'true';
33+
'transaction.system_completion_lock_timeout' = '200';
3234
<# We need to use a dialect that avoids mapping DbType.Time to TIME on MSSQL. On modern SQL Server
3335
this becomes TIME(7). Later, such values cannot be read back over ODBC. The
3436
error we get is "System.ArgumentException : Unknown SQL type - SS_TIME_EX.". I don't know for certain

src/NHibernate.Config.Templates/SapSQLAnywhere.cfg.xml

+2
Original file line numberDiff line numberDiff line change
@@ -17,5 +17,7 @@ for your own use before compiling tests in Visual Studio.
1717
</property>
1818
<property name="dialect">NHibernate.Dialect.SybaseSQLAnywhere12Dialect</property>
1919
<property name="query.substitutions">true=1;false=0</property>
20+
<property name="transaction.ignore_session_synchronization_failures">true</property>
21+
<property name="transaction.system_completion_lock_timeout">200</property>
2022
</session-factory>
2123
</hibernate-configuration>

src/NHibernate.Test/Async/SystemTransactions/SystemTransactionFixture.cs

+154
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99

1010

1111
using System;
12+
using System.Collections.Concurrent;
1213
using System.Collections.Generic;
1314
using System.Diagnostics;
1415
using System.Linq;
@@ -30,6 +31,13 @@ public class SystemTransactionFixtureAsync : SystemTransactionFixtureBase
3031
protected override bool UseConnectionOnSystemTransactionPrepare => true;
3132
protected override bool AutoJoinTransaction => true;
3233

34+
protected override void OnTearDown()
35+
{
36+
base.OnTearDown();
37+
// The SupportsTransactionTimeout test may change this, restore it to its default value.
38+
FailOnNotClosedSession = true;
39+
}
40+
3341
[Test]
3442
public async Task WillNotCrashOnPrepareFailureAsync()
3543
{
@@ -524,6 +532,152 @@ public async Task EnforceConnectionUsageRulesOnTransactionCompletionAsync()
524532
// Currently always forbidden, whatever UseConnectionOnSystemTransactionEvents.
525533
Assert.That(interceptor.AfterException, Is.TypeOf<HibernateException>());
526534
}
535+
536+
// This test check a concurrency issue hard to reproduce. If it is flaky, it has to be considered failing.
537+
// In such case, raise triesCount to investigate it locally with more chances of triggering the trouble.
538+
[Test]
539+
public async Task SupportsTransactionTimeoutAsync()
540+
{
541+
Assume.That(TestDialect.SupportsTransactionScopeTimeouts, Is.True, "The tested dialect is not supported for transaction scope timeouts.");
542+
// Other special cases: ODBC and SAP SQL Anywhere succeed this test only with transaction.ignore_session_synchronization_failures
543+
// enabled.
544+
// They freeze the session during the transaction cancellation. To avoid the test to be very long, the synchronization
545+
// lock timeout should be lowered too.
546+
547+
// A concurrency issue exists with the legacy setting allowing to use the session from transaction completion, which
548+
// may cause session leaks. Ignore them.
549+
FailOnNotClosedSession = !UseConnectionOnSystemTransactionPrepare;
550+
551+
// Test case adapted from https://github.com/kaksmet/NHibBugRepro
552+
553+
// Create some test data.
554+
const int entitiesCount = 5000;
555+
using (var s = OpenSession())
556+
using (var t = s.BeginTransaction())
557+
{
558+
for (var i = 0; i < entitiesCount; i++)
559+
{
560+
var person = new Person
561+
{
562+
NotNullData = Guid.NewGuid().ToString()
563+
};
564+
565+
await (s.SaveAsync(person));
566+
}
567+
568+
await (t.CommitAsync());
569+
}
570+
571+
// Setup unhandled exception catcher.
572+
_unhandledExceptions = new ConcurrentBag<object>();
573+
AppDomain.CurrentDomain.UnhandledException += CurrentDomain_UnhandledException;
574+
try
575+
{
576+
// Generate transaction timeouts.
577+
const int triesCount = 100;
578+
var txOptions = new TransactionOptions { Timeout = TimeSpan.FromMilliseconds(1) };
579+
var timeoutsCount = 0;
580+
for (var i = 0; i < triesCount; i++)
581+
{
582+
try
583+
{
584+
using var txScope = new TransactionScope(TransactionScopeOption.Required, txOptions, TransactionScopeAsyncFlowOption.Enabled);
585+
using var session = OpenSession();
586+
var data = await (session.CreateCriteria<Person>().ListAsync());
587+
Assert.That(data, Has.Count.EqualTo(entitiesCount), "Unexpected count of loaded entities.");
588+
await (Task.Delay(2));
589+
var count = await (session.Query<Person>().CountAsync());
590+
Assert.That(count, Is.EqualTo(entitiesCount), "Unexpected entities count.");
591+
txScope.Complete();
592+
}
593+
catch
594+
{
595+
// Assume that is a transaction timeout. It may cause various failures, of which some are hard to identify.
596+
timeoutsCount++;
597+
}
598+
// If in need of checking some specific failures, the following code may be used instead:
599+
/*
600+
catch (Exception ex)
601+
{
602+
var currentEx = ex;
603+
// Depending on where the transaction aborption has broken NHibernate processing, we may
604+
// get various exceptions, like directly a TransactionAbortedException with an inner
605+
// TimeoutException, or a HibernateException encapsulating a TransactionException with a
606+
// timeout, ...
607+
bool isTransactionException, isTimeout;
608+
do
609+
{
610+
isTransactionException = currentEx is System.Transactions.TransactionException;
611+
isTimeout = isTransactionException && currentEx is TransactionAbortedException;
612+
currentEx = currentEx.InnerException;
613+
}
614+
while (!isTransactionException && currentEx != null);
615+
while (!isTimeout && currentEx != null)
616+
{
617+
isTimeout = currentEx is TimeoutException;
618+
currentEx = currentEx?.InnerException;
619+
}
620+
621+
if (!isTimeout)
622+
{
623+
// We may also get a GenericADOException with an InvalidOperationException stating the
624+
// transaction associated to the connection is no more active but not yet suppressed,
625+
// and that for executing some SQL, we need to suppress it. That is a weak way of
626+
// identifying the case, especially with the many localizations of the message.
627+
currentEx = ex;
628+
do
629+
{
630+
isTimeout = currentEx is InvalidOperationException && currentEx.Message.Contains("SQL");
631+
currentEx = currentEx?.InnerException;
632+
}
633+
while (!isTimeout && currentEx != null);
634+
}
635+
636+
if (isTimeout)
637+
timeoutsCount++;
638+
else
639+
throw;
640+
}
641+
*/
642+
}
643+
644+
Assert.That(
645+
_unhandledExceptions.Count,
646+
Is.EqualTo(0),
647+
"Unhandled exceptions have occurred: {0}",
648+
string.Join(@"
649+
650+
", _unhandledExceptions));
651+
652+
// Despite the Thread sleep and the count of entities to load, this test may get the timeout only for slightly
653+
// more than 10% of the attempts.
654+
Warn.Unless(timeoutsCount, Is.GreaterThan(5), "The test should have generated more timeouts.");
655+
}
656+
finally
657+
{
658+
AppDomain.CurrentDomain.UnhandledException -= CurrentDomain_UnhandledException;
659+
}
660+
}
661+
662+
private ConcurrentBag<object> _unhandledExceptions;
663+
664+
private void CurrentDomain_UnhandledException(object sender, UnhandledExceptionEventArgs e)
665+
{
666+
if (e.ExceptionObject is Exception exception)
667+
{
668+
// Ascertain NHibernate is involved. Some unhandled exceptions occur due to the
669+
// TransactionScope timeout operating on an unexpected thread for the data provider.
670+
var isNHibernateInvolved = false;
671+
while (exception != null && !isNHibernateInvolved)
672+
{
673+
isNHibernateInvolved = exception.StackTrace != null && exception.StackTrace.ToLowerInvariant().Contains("nhibernate");
674+
exception = exception.InnerException;
675+
}
676+
if (!isNHibernateInvolved)
677+
return;
678+
}
679+
_unhandledExceptions.Add(e.ExceptionObject);
680+
}
527681
}
528682

529683
[TestFixture]

0 commit comments

Comments
 (0)