Perf[MQB]: less allocations for ElectorEvent (#986)

Signed-off-by: Evgeny Malygin <[email protected]>

Author: 678098

src/groups/bmq/bmqp/bmqp_protocolutil.h

@@ -524,11 +524,11 @@ int ProtocolUtil::encodeMessage(bsl::ostream&      errorDescription,
         balber::BerEncoderOptions options;
         balber::BerEncoder        encoder(&options, allocator);
 
-        int rc = encoder.encode(out, message);
+        const int rc = encoder.encode(out, message);
 
         // Debug print message if any
         bslstl::StringRef logStr = encoder.loggedMessages();
-        if (BSLS_PERFORMANCEHINT_PREDICT_UNLIKELY(logStr.length() != 0)) {
+        if (BSLS_PERFORMANCEHINT_PREDICT_UNLIKELY(!logStr.empty())) {
             BSLS_PERFORMANCEHINT_UNLIKELY_HINT;
             errorDescription << "BER encoder returned the following "
                              << "[rc: " << rc << "]\n"
@@ -541,11 +541,11 @@ int ProtocolUtil::encodeMessage(bsl::ostream&      errorDescription,
         baljsn::EncoderOptions options;
         baljsn::Encoder        encoder(allocator);
 
-        int rc = encoder.encode(out, message, options);
+        const int rc = encoder.encode(out, message, options);
 
         // Debug print message if any
         bsl::string logStr = encoder.loggedMessages();
-        if (BSLS_PERFORMANCEHINT_PREDICT_UNLIKELY(logStr.length() != 0)) {
+        if (BSLS_PERFORMANCEHINT_PREDICT_UNLIKELY(!logStr.empty())) {
             BSLS_PERFORMANCEHINT_UNLIKELY_HINT;
             errorDescription << "JSON encoder returned the following "
                              << "[rc: " << rc << "]\n"
@@ -593,15 +593,15 @@ int ProtocolUtil::decodeMessage(bsl::ostream&      errorDescription,
     switch (encodingType) {
     case EncodingType::e_BER: {
         balber::BerDecoderOptions options;
-        balber::BerDecoder        decoder(&options, allocator);
         options.setSkipUnknownElements(true);
         options.setDefaultEmptyStrings(false);
+        balber::BerDecoder decoder(&options, allocator);
 
-        int rc = decoder.decode(stream, message);
+        const int rc = decoder.decode(stream, message);
 
         // Debug print message if any
         bslstl::StringRef logStr = decoder.loggedMessages();
-        if (BSLS_PERFORMANCEHINT_PREDICT_UNLIKELY(logStr.length() != 0)) {
+        if (BSLS_PERFORMANCEHINT_PREDICT_UNLIKELY(!logStr.empty())) {
             BSLS_PERFORMANCEHINT_UNLIKELY_HINT;
             errorDescription << "BER decoder returned the following "
                              << "[rc: " << rc << "]\n"
@@ -615,11 +615,11 @@ int ProtocolUtil::decodeMessage(bsl::ostream&      errorDescription,
         baljsn::DecoderOptions options;
         options.setSkipUnknownElements(true);
 
-        int rc = decoder.decode(stream, message, options);
+        const int rc = decoder.decode(stream, message, options);
 
         // Debug print message if any
         bsl::string logStr = decoder.loggedMessages();
-        if (BSLS_PERFORMANCEHINT_PREDICT_UNLIKELY(logStr.length() != 0)) {
+        if (BSLS_PERFORMANCEHINT_PREDICT_UNLIKELY(!logStr.empty())) {
             BSLS_PERFORMANCEHINT_UNLIKELY_HINT;
             errorDescription << "JSON decoder returned the following "
                              << "[rc: " << rc << "]\n"

src/groups/mqb/mqbblp/mqbblp_clusterorchestrator.cpp

@@ -62,6 +62,33 @@ const bsls::Types::Int64 k_WATCHDOG_TIMEOUT_DURATION = 60 * 5;
 
 }  // close unnamed namespace
 
+// ------------------------------------------------
+// class ClusterOrchestrator::OnElectorEventFunctor
+// ------------------------------------------------
+
+ClusterOrchestrator::OnElectorEventFunctor::OnElectorEventFunctor(
+    ClusterOrchestrator*           orchestrator_p,
+    bslmf::MovableRef<bmqp::Event> event,
+    mqbnet::ClusterNode*           source_p)
+: d_orchestrator_p(orchestrator_p)
+, d_event(bslmf::MovableRefUtil::move(event))
+, d_source_p(source_p)
+{
+    // PRECONDITIONS
+    BSLS_ASSERT_SAFE(d_orchestrator_p);
+}
+
+ClusterOrchestrator::OnElectorEventFunctor::~OnElectorEventFunctor()
+{
+    // NOTHING
+}
+
+void ClusterOrchestrator::OnElectorEventFunctor::operator()() const
+{
+    // Thread: CLUSTER dispatcher
+    d_orchestrator_p->processElectorEventDispatched(d_event, d_source_p);
+}
+
 // -------------------------
 // class ClusterOrchestrator
 // -------------------------
@@ -665,7 +692,7 @@ int ClusterOrchestrator::start(bsl::ostream& errorDescription)
                                  _4),  // Term
             ledgerLSN.electorTerm(),
             &d_clusterData_p->blobSpPool(),
-            d_allocator_p),
+            d_allocators.get("Elector")),
         d_allocator_p);
 
     rc = d_elector_mp->start();
@@ -1241,17 +1268,19 @@ void ClusterOrchestrator::processElectorEvent(const bmqp::Event&   event,
     // important that elector events are processed in the dispatcher thread
     // too, otherwise, depending upon thread scheduling, a new node may get
     // certain events "out of order" (some cases were found out while testing).
-    // Note that 'bindA' instead of 'bind' is needed below because we need to
-    // pass allocator to one of the 'bmqp::Event' instances created below
-    // (allocator is *not* optional for 'bmqp::Event')
-    dispatcher()->execute(
-        bdlf::BindUtil::bindS(
-            d_allocator_p,
-            &ClusterOrchestrator::processElectorEventDispatched,
+
+    mqbi::DispatcherEvent* clusterEvent = dispatcher()->getEvent(
+        mqbi::DispatcherClientType::e_CLUSTER);
+
+    (*clusterEvent).setType(mqbi::DispatcherEventType::e_CALLBACK);
+
+    clusterEvent->callback()
+        .createInplace<ClusterOrchestrator::OnElectorEventFunctor>(
             this,
             event.clone(d_allocator_p),
-            source),
-        d_cluster_p);
+            source);
+
+    dispatcher()->dispatchEvent(clusterEvent, d_cluster_p);
 }
 
 void ClusterOrchestrator::processLeaderSyncStateQuery(

src/groups/mqb/mqbblp/mqbblp_clusterorchestrator.h

@@ -112,6 +112,27 @@ class ClusterOrchestrator {
 
     typedef mqbc::ClusterStateQueueInfo::AppInfos AppInfos;
 
+    /// The purpose is to avoid memory allocation by bdlf::BindUtil::bind
+    /// when dispatching ElectorEvent from IO to Cluster.
+    class OnElectorEventFunctor
+    : public bmqu::ManagedCallback::CallbackFunctor {
+      private:
+        // PRIVATE DATA
+        ClusterOrchestrator* d_orchestrator_p;
+        const bmqp::Event    d_event;
+        mqbnet::ClusterNode* d_source_p;
+
+      public:
+        // CREATORS
+        explicit OnElectorEventFunctor(ClusterOrchestrator* orchestrator,
+                                       bslmf::MovableRef<bmqp::Event> event,
+                                       mqbnet::ClusterNode* source_p);
+
+        ~OnElectorEventFunctor() BSLS_KEYWORD_OVERRIDE;
+
+        void operator()() const BSLS_KEYWORD_OVERRIDE;
+    };
+
   private:
     // DATA
 

src/groups/mqb/mqbnet/mqbnet_elector.cpp

@@ -2079,9 +2079,10 @@ void Elector::emitIOEvent(const ElectorStateMachineOutput& output)
     // 'emitIOEvent' is currently always called while 'd_lock' is held, but
     // that's an implementation side effect, not part of contract.  That's why
     // we create the builder on stack instead of making it a class member.
-    bmqp::SchemaEventBuilder builder(d_blobSpPool_p,
+    bdlma::LocalSequentialAllocator<256> localAllocator(d_allocator_p);
+    bmqp::SchemaEventBuilder             builder(d_blobSpPool_p,
                                      bmqp::EncodingType::e_BER,
-                                     d_allocator_p);
+                                     &localAllocator);
 
     int rc = builder.setMessage(message, bmqp::EventType::e_ELECTOR);
     if (0 != rc) {