diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 3718ff8743..42ef6aa7c2 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -15,6 +15,7 @@ set(REALM_SOURCES activemsg.cc + multicast.cc network.cc codedesc.cc logging.cc diff --git a/src/realm/activemsg.cc b/src/realm/activemsg.cc index 671db37ced..0fe534f0f0 100644 --- a/src/realm/activemsg.cc +++ b/src/realm/activemsg.cc @@ -19,6 +19,9 @@ #include "realm/atomics.h" #include "realm/activemsg.h" +#include "realm/module_config.h" +#include "realm/multicast.h" +#include "realm/runtime_impl.h" #include "realm/mutex.h" #include "realm/cmdline.h" #include "realm/logging.h" @@ -903,4 +906,903 @@ namespace Realm { } } + //////////////////////////////////////////////////////////////////////// + // + // multicast envelope and bounded-radix forwarding (plan sections 7.3 and 7.4) + // + + Logger log_multicast("multicast"); + + MulticastMetricsSink::~MulticastMetricsSink(void) {} + + MulticastTransport::~MulticastTransport(void) {} + + MulticastCompletionCallback::~MulticastCompletionCallback(void) {} + + //////////////////////////////////////////////////////////////////////// + // + // struct MulticastFatalContext + // + + void MulticastFatalContext::describe(std::ostream &os) const + { + os << rule << " (local=" << local_node << " sender=" << sender + << " origin=" << origin_node << " multicast_id=" << multicast_id + << " message_id=" << original_message_id + << " encoding_kind=" << target_encoding_kind + << " encoding_size=" << target_encoding_size + << " header_size=" << original_header_size + << " payload_size=" << original_payload_size + << " received_bytes=" << received_payload_size << " depth=" << depth + << " decode_status=" << status << ")"; + } + + std::string MulticastFatalContext::to_string(void) const + { + std::ostringstream os; + describe(os); + return os.str(); + } + + namespace { + + class DefaultMulticastFatalReporter : public MulticastFatalReporter { + public: + virtual void report(const MulticastFatalContext &ctx) + { + log_multicast.fatal() << "multicast protocol violation: " << ctx.to_string(); + abort(); + } + }; + + DefaultMulticastFatalReporter default_multicast_fatal_reporter; + + // installed only by tests, and only while no multicast traffic is in flight, so a + // plain pointer is sufficient here + MulticastFatalReporter *installed_multicast_fatal_reporter = nullptr; + + }; // namespace + + MulticastFatalReporter *set_multicast_fatal_reporter(MulticastFatalReporter *reporter) + { + MulticastFatalReporter *previous = installed_multicast_fatal_reporter; + installed_multicast_fatal_reporter = reporter; + return previous; + } + + MulticastFatalReporter *get_multicast_fatal_reporter(void) + { + return installed_multicast_fatal_reporter; + } + + void report_multicast_fatal(const MulticastFatalContext &ctx) + { + if(installed_multicast_fatal_reporter != nullptr) { + installed_multicast_fatal_reporter->report(ctx); + return; + } + default_multicast_fatal_reporter.report(ctx); + } + + //////////////////////////////////////////////////////////////////////// + // + // class MulticastCompletionState + // + + MulticastCompletionState::MulticastCompletionState(void) {} + + MulticastCompletionState::~MulticastCompletionState(void) + { + // plan section 2 (final bullet) and 7.1: nothing may outlive the multicasts it + // belongs to, so a nonempty table here means a subtree never acknowledged + assert(pending.empty() && + "multicast completion state outlived the multicasts it was tracking"); + } + + void MulticastCompletionState::begin_origin(NodeID origin, uint64_t multicast_id, + size_t outstanding, + MulticastCompletionCallback *callback) + { + assert(outstanding > 0); + assert(callback != nullptr); + MulticastCompletionKey key; + key.origin = origin; + key.multicast_id = multicast_id; + + Record rec; + rec.parent = origin; + rec.outstanding = outstanding; + rec.callback = callback; + rec.is_origin = true; + + AutoLock<> al(mutex); + bool inserted = pending.insert(std::make_pair(key, rec)).second; + assert(inserted && "duplicate multicast completion record at the origin"); + (void)inserted; + if(pending.size() > peak) + peak = pending.size(); + } + + void MulticastCompletionState::begin_relay(NodeID origin, uint64_t multicast_id, + NodeID parent, size_t outstanding) + { + assert(outstanding > 0); + MulticastCompletionKey key; + key.origin = origin; + key.multicast_id = multicast_id; + + Record rec; + rec.parent = parent; + rec.outstanding = outstanding; + rec.callback = nullptr; + rec.is_origin = false; + + AutoLock<> al(mutex); + bool inserted = pending.insert(std::make_pair(key, rec)).second; + assert(inserted && "duplicate multicast completion record at a relay"); + (void)inserted; + if(pending.size() > peak) + peak = pending.size(); + } + + MulticastCompletionState::Notification + MulticastCompletionState::note_completion(NodeID origin, uint64_t multicast_id) + { + MulticastCompletionKey key; + key.origin = origin; + key.multicast_id = multicast_id; + + Notification result; + { + AutoLock<> al(mutex); + std::map::iterator it = pending.find(key); + if(it == pending.end()) { + result.action = Notification::UNKNOWN; + return result; + } + assert(it->second.outstanding > 0); + if(--it->second.outstanding > 0) + return result; // NOTHING + + // the subtree is complete: reclaim the record BEFORE telling anyone, so that no + // state survives the acknowledgement (plan section 7.5) + result.action = (it->second.is_origin ? Notification::INVOKE_CALLBACK + : Notification::ACK_PARENT); + result.parent = it->second.parent; + result.callback = it->second.callback; + pending.erase(it); + } + return result; + } + + size_t MulticastCompletionState::num_pending(void) const + { + AutoLock<> al(mutex); + return pending.size(); + } + + size_t MulticastCompletionState::peak_pending(void) const + { + AutoLock<> al(mutex); + return peak; + } + + void MulticastCompletionState::reset_peak(void) + { + AutoLock<> al(mutex); + peak = pending.size(); + } + + //////////////////////////////////////////////////////////////////////// + // + // forwarding helpers + // + + namespace { + +#ifdef DEBUG_REALM + // Plan section 21.1 requires partition overlap (or a slice that is not actually a + // subset of what we were asked to forward) to be fatal in debug builds. The check + // is exact and linear in the number of runs: walking the slices in order, every + // run must start after the previous one ended and must fit inside a run of the + // source, and the cardinalities must add up. Containment plus disjointness plus + // equal cardinality is exactly "the slices partition the source". + void validate_partition(const MulticastTargetSet &source, + const std::vector &slices) + { + typedef MulticastTargetSet::Range Range; + size_t src_idx = 0; + size_t total = 0; + bool have_prev = false; + NodeID prev_last = 0; + for(size_t i = 0; i < slices.size(); i++) { + assert(!slices[i].empty() && "multicast partition produced an empty slice"); + const std::vector &runs = slices[i].ranges(); + for(size_t j = 0; j < runs.size(); j++) { + assert((!have_prev || (runs[j].first > prev_last)) && + "multicast partition slices overlap"); + while((src_idx < source.num_ranges()) && + (source.ranges()[src_idx].last < runs[j].first)) + src_idx++; + assert((src_idx < source.num_ranges()) && + "multicast partition slice is not part of the target set"); + assert((runs[j].first >= source.ranges()[src_idx].first) && + (runs[j].last <= source.ranges()[src_idx].last) && + "multicast partition slice is not part of the target set"); + prev_last = runs[j].last; + have_prev = true; + } + total += slices[i].size(); + } + assert((total == source.size()) && "multicast partition dropped targets"); + } +#endif + + // Everything that is identical for every child envelope of one forwarding step - + // the original message, the multicast identity, and whether this multicast is + // completion tracked. + struct OutboundMulticast { + NodeID origin = 0; + uint64_t multicast_id = 0; + ActiveMessageHandlerTable::MessageID msgid = 0; + const void *hdr = nullptr; + size_t hdr_size = 0; + const void *payload = nullptr; + size_t payload_size = 0; + uint32_t flags = 0; + unsigned depth = 0; + // node the child subtree must acknowledge to, i.e. the node doing the sending - + // only meaningful when COMPLETION_TRACKED is set in 'flags' + NodeID completion_parent = 0; + + bool tracked(void) const + { + return ((flags & MulticastEnvelopeFlags::COMPLETION_TRACKED) != 0); + } + }; + + // Builds one envelope for 'slice' and hands it to the transport. The relay is the + // first node of the slice (plan section 7.3 step 3). + void send_one_slice(MulticastTransport &transport, const MulticastTargetSet &slice, + const OutboundMulticast &out, MulticastMetricsSink *metrics) + { + assert(!slice.empty()); + + EncodedMulticastTargets enc = + EncodedMulticastTargets::encode(slice, transport.num_nodes()); + if(metrics != nullptr) + metrics->record_encoding_choice(enc.kind()); + + // a fire-and-forget multicast carries no completion metadata whatsoever + const size_t comp_size = + (out.tracked() + ? MulticastWire::varint_size(static_cast(out.completion_parent)) + : 0); + + MulticastEnvelopeMessage env; + env.multicast_id = out.multicast_id; + env.origin_node = out.origin; + env.original_payload_size = static_cast(out.payload_size); + env.target_encoding_size = static_cast(enc.bytes()); + env.completion_size = static_cast(comp_size); + env.flags = out.flags; + env.original_message_id = out.msgid; + env.original_header_size = static_cast(out.hdr_size); + env.depth = out.depth; + env.target_encoding_kind = static_cast(enc.kind()); + + // the variable portion is copied here, which is what makes the caller's + // PAYLOAD_KEEP lifetime guarantee hold across commit() (plan section 7.5) + std::vector buf; + buf.reserve(enc.bytes() + out.hdr_size + out.payload_size + comp_size); + buf.insert(buf.end(), enc.wire_bytes().begin(), enc.wire_bytes().end()); + if(out.hdr_size > 0) { + const unsigned char *hdr_bytes = static_cast(out.hdr); + buf.insert(buf.end(), hdr_bytes, hdr_bytes + out.hdr_size); + } + if(out.payload_size > 0) { + const unsigned char *body = static_cast(out.payload); + buf.insert(buf.end(), body, body + out.payload_size); + } + if(comp_size > 0) + MulticastWire::append_varint(buf, static_cast(out.completion_parent)); + + transport.send_envelope(slice.first_node(), env, buf.data(), buf.size()); + } + + // Partitions 'remaining' into at most R slices WITHOUT sending anything. Splitting + // planning from sending matters for completion tracking: the number of children + // has to be known (and the record retained) before the first child can possibly + // acknowledge. + void plan_children(MulticastTransport &transport, const MulticastTargetSet &remaining, + std::vector &slices) + { + slices.clear(); + if(remaining.empty()) + return; + + const size_t radix = transport.radix(); + assert(radix >= 1); + + remaining.partition(radix, slices); +#ifdef DEBUG_REALM + validate_partition(remaining, slices); +#endif + assert(slices.size() <= radix); + } + + }; // namespace + + //////////////////////////////////////////////////////////////////////// + // + // class MulticastForwarder + // + + /*static*/ uint64_t MulticastForwarder::next_multicast_id(void) + { + // only the local half of (origin_node, counter) - the origin node is carried + // separately in the envelope, so this only has to be unique per origin + static atomic counter(0); + return counter.fetch_add(1) + 1; + } + + namespace { + + // Post-handler notification for a local delivery that could not be handled inline. + // The token is heap allocated because IncomingMessageManager only carries two + // uintptr_t of callback data, and it is captured rather than rederived because + // this runs on a handler thread, potentially long after dispatch_local returned. + void deferred_local_delivery_callback(NodeID /*sender*/, + IncomingMessageManager::CallbackData data1, + IncomingMessageManager::CallbackData /*data2*/) + { + MulticastCompletionToken *token = + reinterpret_cast(data1); + MulticastCompletionToken copy = *token; + delete token; + MulticastForwarder::settle(copy); + } + + }; // namespace + + /*static*/ void MulticastForwarder::settle(const MulticastCompletionToken &token) + { + assert(token.state != nullptr); + MulticastCompletionState::Notification note = + token.state->note_completion(token.origin, token.multicast_id); + + switch(note.action) { + case MulticastCompletionState::Notification::NOTHING: + break; + + case MulticastCompletionState::Notification::ACK_PARENT: + { + // exactly one acknowledgement per subtree, sent after the record was reclaimed + assert(token.transport != nullptr); + token.transport->send_ack(token.local, note.parent, token.origin, + token.multicast_id); + break; + } + + case MulticastCompletionState::Notification::INVOKE_CALLBACK: + { + // the origin's callback runs exactly once, after every target has HANDLED the + // message, and the state is already gone (plan section 7.5) + assert(note.callback != nullptr); + note.callback->invoke(); + delete note.callback; + break; + } + + case MulticastCompletionState::Notification::UNKNOWN: + { + MulticastFatalContext ctx; + ctx.local_node = token.local; + ctx.sender = token.local; + ctx.origin_node = token.origin; + ctx.multicast_id = token.multicast_id; + ctx.rule = "multicast acknowledgement does not match any multicast in flight"; + report_multicast_fatal(ctx); + break; + } + } + } + + /*static*/ void MulticastForwarder::handle_ack(MulticastTransport &transport, + NodeID sender, + const MulticastAckMessage &ack) + { + MulticastCompletionToken token; + token.transport = &transport; + token.state = &transport.completion_state(); + token.local = transport.my_node_id(); + token.origin = ack.origin_node; + token.multicast_id = ack.multicast_id; + (void)sender; + settle(token); + } + + /*static*/ bool MulticastForwarder::dispatch_local( + IncomingMessageManager *manager, NodeID sender, + ActiveMessageHandlerTable::MessageID msgid, const void *hdr, size_t hdr_size, + const void *payload, size_t payload_size, TimeLimit work_until, + const MulticastCompletionToken *completion) + { + assert(manager != nullptr); + + IncomingMessageManager::CallbackFnptr fnptr = nullptr; + IncomingMessageManager::CallbackData data1 = 0; + MulticastCompletionToken *token = nullptr; + if(completion != nullptr) { +#ifdef DEBUG_REALM + // A message type that carries its own FragmentInfo is reassembled inside + // add_incoming_message, and an incomplete fragment returns without ever reaching + // a handler - so "handled" would be unobservable and the notification would be + // lost. Such a type must not be the ORIGINAL message of a completion-tracked + // multicast; multicast its unfragmented form and let the envelope be chunked. + ActiveMessageHandlerTable::HandlerEntry *entry = + activemsg_handler_table.lookup_message_handler(msgid); + assert((entry != nullptr) && !entry->extract_frag_info.has_value() && + "completion-tracked multicast of a fragment-carrying message type"); +#endif + token = new MulticastCompletionToken(*completion); + fnptr = &deferred_local_delivery_callback; + data1 = reinterpret_cast(token); + } + + // Reuse of the ordinary incoming path is deliberate: it looks the handler up by + // message ID in the same ActiveMessageHandlerTable, honors an inline handler when + // there is time for one, and otherwise queues the message for a handler thread + // with the same TimeLimit semantics as a message that arrived off the wire. No + // handler-signature detection is duplicated here (plan section 7.4). + bool handled = manager->add_incoming_message( + sender, msgid, hdr, hdr_size, PAYLOAD_COPY, payload, payload_size, PAYLOAD_COPY, + fnptr, data1, 0, work_until); + + if(token != nullptr) { + if(handled) { + // handled inline, so the post-handler callback will NOT run - settle here + MulticastCompletionToken copy = *token; + delete token; + settle(copy); + } + // otherwise deferred_local_delivery_callback owns and frees it + } + + return handled; + } + + /*static*/ void MulticastForwarder::send( + MulticastTransport &transport, const MulticastTargetSet &targets, + ActiveMessageHandlerTable::MessageID msgid, const void *hdr, size_t hdr_size, + const void *payload, size_t payload_size, TimeLimit work_until, + MulticastMetricsSink *metrics, MulticastCompletionCallback *on_remote_complete) + { + // the envelope's length fields are 16/32 bits wide + assert(hdr_size <= 0xffff); + assert(payload_size <= 0xffffffffULL); + + // plan section 7.5: an empty target set is a successful no-op. Every one of the + // zero targets has trivially already handled the message, so a requested remote + // completion fires immediately and no state is created. + if(targets.empty()) { + if(on_remote_complete != nullptr) { + on_remote_complete->invoke(); + delete on_remote_complete; + } + return; + } + + const NodeID local = transport.my_node_id(); + const NodeID num_nodes = transport.num_nodes(); + + if(!targets.fits_node_count(num_nodes)) { + MulticastFatalContext ctx; + ctx.local_node = local; + ctx.sender = local; + ctx.origin_node = local; + ctx.original_message_id = msgid; + ctx.original_header_size = hdr_size; + ctx.original_payload_size = payload_size; + ctx.status = MulticastDecodeStatus::NODE_OUT_OF_RANGE; + ctx.rule = "multicast target set contains a node outside the configured node " + "count"; + report_multicast_fatal(ctx); + // nothing was sent, so the callback must not claim that everyone handled it + delete on_remote_complete; + return; + } + + // 1. if the origin is a target, arrange local delivery and remove it from the + // forwarding set (plan section 7.3) + MulticastTargetSet remaining(targets); + const bool deliver_here = remaining.remove(local); + + OutboundMulticast out; + out.origin = local; + out.msgid = msgid; + out.hdr = hdr; + out.hdr_size = hdr_size; + out.payload = payload; + out.payload_size = payload_size; + out.depth = 1; + out.completion_parent = local; + // every multicast gets the globally unique (origin_node, counter) identity plan + // section 7.4 requires, whether or not it is completion tracked - it is what a + // fatal diagnostic names, and what an acknowledgement is keyed on + out.multicast_id = next_multicast_id(); + + // A fire-and-forget multicast sets no flag, puts no completion metadata on the wire + // and creates no state anywhere (plan section 7.5). + const bool tracked = (on_remote_complete != nullptr); + if(tracked) + out.flags |= MulticastEnvelopeFlags::COMPLETION_TRACKED; + + // decide the whole shape of the first hop before anything is transmitted, so that + // the origin's record can be retained before a child can possibly acknowledge + const bool unicast_fast_path = + ((remaining.size() == 1) && transport.can_send_original(hdr_size, payload_size)); + std::vector slices; + if(!remaining.empty() && !unicast_fast_path) + plan_children(transport, remaining, slices); + + const size_t first_hops = (unicast_fast_path ? 1 : slices.size()); + + MulticastCompletionToken token; + if(tracked) { + token.transport = &transport; + token.state = &transport.completion_state(); + token.local = local; + token.origin = local; + token.multicast_id = out.multicast_id; + // one unit per first hop, plus one for our own delivery if we are a target + token.state->begin_origin(local, out.multicast_id, + first_hops + (deliver_here ? 1 : 0), on_remote_complete); + } + + if(unicast_fast_path) { + // plan section 7.5: a singleton target uses the ordinary unicast fast path. This + // is only safe at the origin, where the local node already IS the sender the + // handler must see. A tracked send rides the ordinary active-message remote + // completion, which by definition fires once the target has handled it. + transport.send_original(remaining.first_node(), msgid, hdr, hdr_size, payload, + payload_size, (tracked ? &token : nullptr)); + } else { + // 2-4. one envelope per slice, addressed to that slice's first node + for(size_t i = 0; i < slices.size(); i++) + send_one_slice(transport, slices[i], out, metrics); + } + if((metrics != nullptr) && (first_hops > 0)) + metrics->record_first_hops(first_hops); + + // forward-before-deliver (plan section 7.3 step 4) applies at the origin too: a + // handler such as runtime shutdown may stop progress + if(deliver_here) + transport.deliver_local(local, msgid, hdr, hdr_size, payload, payload_size, + work_until, (tracked ? &token : nullptr)); + } + + /*static*/ void MulticastForwarder::forward(MulticastTransport &transport, + NodeID sender, + const MulticastEnvelopeMessage &env, + const void *payload, size_t payload_size, + TimeLimit work_until, + MulticastMetricsSink *metrics) + { + const NodeID local = transport.my_node_id(); + const NodeID num_nodes = transport.num_nodes(); + + MulticastFatalContext ctx; + ctx.local_node = local; + ctx.sender = sender; + ctx.origin_node = env.origin_node; + ctx.multicast_id = env.multicast_id; + ctx.original_message_id = env.original_message_id; + ctx.target_encoding_kind = env.target_encoding_kind; + ctx.target_encoding_size = env.target_encoding_size; + ctx.original_header_size = env.original_header_size; + ctx.original_payload_size = env.original_payload_size; + ctx.received_payload_size = payload_size; + ctx.depth = env.depth; + + // the four variable-length pieces must account for exactly the bytes we received - + // nothing below is sized from an unvalidated remote length (plan section 22) + const size_t targets_bytes = env.target_encoding_size; + const size_t hdr_bytes = env.original_header_size; + const size_t body_bytes = env.original_payload_size; + const size_t comp_bytes = env.completion_size; + if((targets_bytes + hdr_bytes + body_bytes + comp_bytes) != payload_size) { + ctx.status = MulticastDecodeStatus::TRUNCATED; + ctx.rule = "multicast envelope length fields do not match the received payload"; + report_multicast_fatal(ctx); + return; + } + + if((env.flags & ~static_cast(MulticastEnvelopeFlags::ALL_KNOWN)) != 0) { + ctx.rule = "multicast envelope carries unknown flags"; + report_multicast_fatal(ctx); + return; + } + + const unsigned char *base = static_cast(payload); + + // 0. completion metadata is present if and only if this multicast is tracked, and + // names the node this whole subtree must acknowledge to (plan section 7.5) + const bool tracked = ((env.flags & MulticastEnvelopeFlags::COMPLETION_TRACKED) != 0); + NodeID parent = 0; + if(tracked) { + size_t pos = 0; + uint64_t raw = 0; + MulticastDecodeStatus cstat = MulticastWire::read_varint( + base + targets_bytes + hdr_bytes + body_bytes, comp_bytes, pos, raw); + if((cstat != MulticastDecodeStatus::OK) || (pos != comp_bytes) || + (raw >= static_cast(num_nodes))) { + ctx.status = ((cstat != MulticastDecodeStatus::OK) + ? cstat + : MulticastDecodeStatus::NODE_OUT_OF_RANGE); + ctx.rule = "malformed multicast completion metadata"; + report_multicast_fatal(ctx); + return; + } + parent = static_cast(raw); + if(parent != sender) { + ctx.rule = "multicast completion metadata names a parent that is not the sender"; + report_multicast_fatal(ctx); + return; + } + } else if(comp_bytes != 0) { + ctx.rule = "multicast envelope carries completion metadata without the completion " + "flag"; + report_multicast_fatal(ctx); + return; + } + + // 1a. decode and fully validate the target slice + MulticastTargetSet slice; + MulticastDecodeStatus status = + EncodedMulticastTargets::decode(base, targets_bytes, num_nodes, slice); + if(status != MulticastDecodeStatus::OK) { + ctx.status = status; + ctx.rule = "malformed multicast target encoding"; + report_multicast_fatal(ctx); + return; + } + if(base[0] != env.target_encoding_kind) { + ctx.status = MulticastDecodeStatus::UNKNOWN_KIND; + ctx.rule = "multicast envelope encoding kind disagrees with its payload"; + report_multicast_fatal(ctx); + return; + } + + // 1b. validate that the relay is included in the received slice (plan sections 7.3 + // and 21.1) + if(!slice.contains(local)) { + ctx.rule = "multicast relay is not a member of the slice it was sent"; + report_multicast_fatal(ctx); + return; + } + + // 2. save the original sender from the envelope - a relay must never become the + // apparent sender just because it transmitted the final hop (plan section 7.4) + const NodeID origin = env.origin_node; + + if(metrics != nullptr) + metrics->record_tree_depth(env.depth); + + // 3. remove ourselves from the slice + bool was_present = slice.remove(local); + assert(was_present); + (void)was_present; + + OutboundMulticast out; + out.origin = origin; + out.multicast_id = env.multicast_id; + out.msgid = env.original_message_id; + out.hdr = base + targets_bytes; + out.hdr_size = hdr_bytes; + out.payload = (body_bytes > 0) ? (base + targets_bytes + hdr_bytes) : nullptr; + out.payload_size = body_bytes; + out.flags = env.flags; + out.depth = env.depth + 1; + // a child acknowledges to US, not to whoever sent us this envelope + out.completion_parent = local; + + // 4. partition and enqueue child envelopes BEFORE invoking the original local + // handler. Forward-before-deliver matters for handlers such as runtime shutdown + // whose handler may stop progress. We are on the ordinary (non-inline) handler + // path, so these sends are not recursive forwarding out of an inline handler + // (plan section 22). + std::vector slices; + plan_children(transport, slice, slices); + + // The relay's transient record: its parent, its own (not yet finished) local + // delivery, and one outstanding acknowledgement per child. It has to exist before + // the first child envelope goes out, because a child can acknowledge at any time + // after that (plan section 7.5). + MulticastCompletionToken token; + if(tracked) { + token.transport = &transport; + token.state = &transport.completion_state(); + token.local = local; + token.origin = origin; + token.multicast_id = env.multicast_id; + token.state->begin_relay(origin, env.multicast_id, parent, slices.size() + 1); + } + + for(size_t i = 0; i < slices.size(); i++) + send_one_slice(transport, slices[i], out, metrics); + + // 5. deliver the original typed message locally exactly once + transport.deliver_local(origin, env.original_message_id, out.hdr, out.hdr_size, + out.payload, out.payload_size, work_until, + (tracked ? &token : nullptr)); + } + + //////////////////////////////////////////////////////////////////////// + // + // struct MulticastEnvelopeMessage + // + + /*static*/ void MulticastEnvelopeMessage::handle_message( + NodeID sender, const MulticastEnvelopeMessage &hdr, const void *payload, + size_t payload_size, TimeLimit work_until) + { + MulticastForwarder::forward(get_runtime_multicast_transport(), sender, hdr, payload, + payload_size, work_until); + } + + ActiveMessageHandlerReg multicast_envelope_message_handler; + + //////////////////////////////////////////////////////////////////////// + // + // struct MulticastAckMessage + // + + /*static*/ void MulticastAckMessage::handle_message(NodeID sender, + const MulticastAckMessage &hdr, + const void * /*payload*/, + size_t /*payload_size*/, + TimeLimit /*work_until*/) + { + MulticastForwarder::handle_ack(get_runtime_multicast_transport(), sender, hdr); + } + + ActiveMessageHandlerReg multicast_ack_message_handler; + + //////////////////////////////////////////////////////////////////////// + // + // the runtime's multicast transport + // + + namespace { + + class RuntimeMulticastTransport : public MulticastTransport { + public: + virtual NodeID my_node_id(void) const { return Network::my_node_id; } + + virtual NodeID num_nodes(void) const { return Network::max_node_id + 1; } + + virtual size_t radix(void) const + { + size_t cached = cached_radix.load(); + if(cached != 0) + return cached; + + // initially the existing -ll:barrier_radix / barrier_broadcast_radix value + // (plan section 7.3). The answer is only cached once the core module config + // actually exists, so a call made before the runtime is up falls back to the + // default without poisoning the cache. + RuntimeImpl *runtime = get_runtime(); + if(runtime != nullptr) { + ModuleConfig *core = runtime->get_module_config("core"); + if(core != nullptr) { + int configured = 0; + if((core->get_property("barrier_broadcast_radix", configured) == + REALM_SUCCESS) && + (configured >= 1)) { + cached_radix.store(static_cast(configured)); + return static_cast(configured); + } + } + } + return MULTICAST_DEFAULT_RADIX; + } + + virtual void send_envelope(NodeID relay, const MulticastEnvelopeMessage &env, + const void *payload, size_t payload_bytes) + { + // an oversized envelope is fragmented here by the ordinary ActiveMessage + // machinery and reassembled before the relay repartitions it + ActiveMessage amsg(relay, payload_bytes); + *amsg = env; + if(payload_bytes > 0) + amsg.add_payload(payload, payload_bytes); + amsg.commit(); + } + + virtual bool can_send_original(size_t hdr_size, size_t payload_size) const + { + if(payload_size == 0) + return true; + // fragmentation of the original message would need its compile-time type, and + // all we have here is a message ID - such a send goes through an envelope + // instead, which the backend can fragment + return payload_size <= Network::max_payload_size(hdr_size, nullptr); + } + + virtual void send_original(NodeID target, + ActiveMessageHandlerTable::MessageID msgid, + const void *hdr, size_t hdr_size, const void *payload, + size_t payload_size, + const MulticastCompletionToken *completion) + { + // same sequence ActiveMessage performs, except that the message ID and the + // header bytes are chosen at run time rather than by type + uint64_t storage[32]; + ActiveMessageImpl *impl = Network::create_active_message_impl( + target, msgid, hdr_size, payload_size, 0, 0, 0, &storage, sizeof(storage)); + memcpy(impl->header_base, hdr, hdr_size); + if(payload_size > 0) + memcpy(impl->payload_base, payload, payload_size); + if(completion != nullptr) { + // an ordinary remote completion already means "received AND HANDLED by the + // target", which for a single target is exactly the aggregate we want + UnicastCompletionNotifier notifier; + notifier.token = *completion; + size_t bytes = sizeof(CompletionCallback); + bytes = (((bytes - 1) / CompletionCallbackBase::ALIGNMENT) + 1) * + CompletionCallbackBase::ALIGNMENT; + void *ptr = impl->add_remote_completion(bytes); + new(ptr) CompletionCallback(notifier); + } + impl->commit(payload_size); + impl->~ActiveMessageImpl(); + } + + virtual void send_ack(NodeID from, NodeID parent, NodeID origin, + uint64_t multicast_id) + { + assert(from == Network::my_node_id); + (void)from; + ActiveMessage amsg(parent); + amsg->multicast_id = multicast_id; + amsg->origin_node = origin; + amsg.commit(); + } + + virtual void deliver_local(NodeID origin, + ActiveMessageHandlerTable::MessageID msgid, + const void *hdr, size_t hdr_size, const void *payload, + size_t payload_size, TimeLimit work_until, + const MulticastCompletionToken *completion) + { + RuntimeImpl *runtime = get_runtime(); + assert((runtime != nullptr) && (runtime->message_manager != nullptr)); + MulticastForwarder::dispatch_local(runtime->message_manager, origin, msgid, hdr, + hdr_size, payload, payload_size, work_until, + completion); + } + + virtual MulticastCompletionState &completion_state(void) { return completion; } + + protected: + // callable form of "one unicast fast-path target has handled the message" + struct UnicastCompletionNotifier { + MulticastCompletionToken token; + void operator()(void) const { MulticastForwarder::settle(token); } + }; + + mutable atomic cached_radix{0}; + MulticastCompletionState completion; + }; + + RuntimeMulticastTransport runtime_multicast_transport; + + }; // namespace + + MulticastTransport &get_runtime_multicast_transport(void) + { + return runtime_multicast_transport; + } + }; // namespace Realm diff --git a/src/realm/activemsg.h b/src/realm/activemsg.h index a605657071..d94f4c448e 100644 --- a/src/realm/activemsg.h +++ b/src/realm/activemsg.h @@ -26,6 +26,7 @@ #include "realm/mutex.h" #include "realm/serialize.h" #include "realm/nodeset.h" +#include "realm/multicast.h" #include "realm/network.h" #include "realm/atomics.h" #include "realm/threads.h" @@ -71,14 +72,17 @@ namespace Realm { // constructs an INACTIVE message object - call init(...) as needed ActiveMessage(); - // construct a new active message for either a single recipient or a mask - // of recipients + // construct a new active message for a single recipient // in addition to the header struct (T), a message can include a variable // payload which can be delivered to a particular destination address + // NOTE: there is deliberately no NodeSet ("send this to many nodes") + // constructor - a multi-target send fans out one message per target at + // the SOURCE, which does not scale. Use multicast_message(...) from + // realm/activemsg.h instead: it forwards over a bounded-radix + // tree of ordinary unicast messages (plan sections 7.1-7.6). ActiveMessage(NodeID _target, size_t _max_payload_size = 0); ActiveMessage(NodeID _target, size_t _max_payload_size, const RemoteAddress &_dest_payload_addr); - ActiveMessage(const Realm::NodeSet &_targets, size_t _max_payload_size = 0); // providing the payload (as a 1D reference, which must be PAYLOAD_KEEP) // up front can avoid a copy if the source location is directly accessible @@ -88,7 +92,6 @@ namespace Realm { ActiveMessage(NodeID _target, const void *_data, size_t _datalen); ActiveMessage(NodeID _target, const LocalAddress &_src_payload_addr, size_t _datalen, const RemoteAddress &_dest_payload_addr); - ActiveMessage(const Realm::NodeSet &_targets, const void *_data, size_t _datalen); ActiveMessage(NodeID _target, const LocalAddress &_src_payload_addr, size_t _bytes_per_line, size_t _lines, size_t _line_stride, const RemoteAddress &_dest_payload_addr); @@ -99,11 +102,9 @@ namespace Realm { void init(NodeID _target, size_t _max_payload_size = 0); void init(NodeID _target, size_t _max_payload_size, const RemoteAddress &_dest_payload_addr); - void init(const Realm::NodeSet &_targets, size_t _max_payload_size = 0); void init(NodeID _target, const void *_data, size_t _datalen); void init(NodeID _target, const LocalAddress &_src_payload_addr, size_t _datalen, const RemoteAddress &_dest_payload_addr); - void init(const Realm::NodeSet &_targets, const void *_data, size_t _datalen); void init(NodeID _target, const LocalAddress &_src_payload_addr, size_t _bytes_per_line, size_t _lines, size_t _line_stride, const RemoteAddress &_dest_payload_addr); @@ -116,16 +117,12 @@ namespace Realm { // a call that sets `with_congestion` may get a smaller value (maybe // even 0) if the path to the named target(s) is getting full static size_t recommended_max_payload(NodeID target, bool with_congestion); - static size_t recommended_max_payload(const NodeSet &targets, bool with_congestion); static size_t recommended_max_payload(NodeID target, const RemoteAddress &dest_payload_addr, bool with_congestion); static size_t recommended_max_payload(NodeID target, const void *data, size_t bytes_per_line, size_t lines, size_t line_stride, bool with_congestion); - static size_t recommended_max_payload(const NodeSet &targets, const void *data, - size_t bytes_per_line, size_t lines, - size_t line_stride, bool with_congestion); static size_t recommended_max_payload(NodeID target, const LocalAddress &src_payload_addr, size_t bytes_per_line, size_t lines, size_t line_stride, @@ -174,16 +171,14 @@ namespace Realm { // chunked-mode state: only used when the requested payload exceeds the // network's hard limit for a single message size_t network_max_payload_{0}; // 0 = normal (non-chunked) mode - NodeSet chunk_targets_; + NodeID chunk_target_{-1}; const void *chunk_src_data_{nullptr}; size_t chunk_src_datalen_{0}; std::vector chunk_alloc_; // owned buffer for network-allocated chunked mode void init_chunked(NodeID _target, size_t _max_payload_size); - void init_chunked(const NodeSet &_targets, size_t _max_payload_size); void init_chunked_data(NodeID _target, const void *_data, size_t _datalen); - void init_chunked_data(const NodeSet &_targets, const void *_data, size_t _datalen); void commit_chunked(void); static uint64_t next_chunk_message_id(NodeID node_id); }; @@ -477,6 +472,419 @@ namespace Realm { template struct is_wrapped_with_frag_info> : std::true_type {}; + //////////////////////////////////////////////////////////////////////// + // + // multicast envelope and bounded-radix forwarding (plan sections 7.3-7.5) + // + // Layered strictly above backend unicast: a multicast is an ordinary unicast + // ActiveMessage carrying an envelope, which each relay repartitions and forwards + // BEFORE delivering the original typed message locally. The target set and its wire + // encoding live in realm/multicast.h, which deliberately has no dependency on this + // file, so that codec stays unit-testable on its own. + // + + // forwarding radix used when no core module config is available (matches the + // -ll:barrier_radix / barrier_broadcast_radix default) + static const size_t MULTICAST_DEFAULT_RADIX = 4; + + //////////////////////////////////////////////////////////////////////// + // + // struct MulticastEnvelopeMessage + // + + // Flags carried in the envelope. A fire-and-forget multicast sets none of these and + // carries no completion metadata at all (plan section 7.5). + namespace MulticastEnvelopeFlags { + enum : uint32_t + { + // the envelope carries completion metadata after the original payload and the + // parent expects exactly one acknowledgement from this subtree + COMPLETION_TRACKED = 1U << 0, + + // every bit that is defined - anything else in 'flags' is a protocol violation + ALL_KNOWN = COMPLETION_TRACKED, + }; + }; // namespace MulticastEnvelopeFlags + + // The multicast envelope (plan section 7.4). The variable portion that follows is, + // in order: + // + // [target_encoding_size] encoded target slice (realm/multicast.h wire form) + // [original_header_size] original typed header bytes + // [original_payload_size] original payload bytes + // [completion_size] optional completion metadata - a single varint holding + // the node this subtree must acknowledge to, present if + // and only if COMPLETION_TRACKED is set + // + // This type deliberately does NOT declare a `handle_inline`: plan section 22 requires + // that child sends not be issued recursively from an inline handler, so an envelope + // is always queued and forwarded from the normal active-message path. It also does + // not declare a FragmentInfo member, which means an oversized envelope is handled by + // the existing WrappedWithFragInfo fragmentation machinery on every hop and is + // reassembled before the relay repartitions it (plan section 7.5). + struct MulticastEnvelopeMessage { + // (origin_node, multicast_id) is the globally unique multicast identifier + uint64_t multicast_id = 0; + // the node that ORIGINATED the multicast - this, and never the previous hop, is + // what the original handler must see as its sender + NodeID origin_node = 0; + + uint32_t original_payload_size = 0; + uint32_t target_encoding_size = 0; + uint32_t completion_size = 0; + uint32_t flags = 0; + // hops from the origin: the origin's own first-hop envelopes carry 1. This is 32 + // bits rather than 16 because with a radix of 1 the tree degenerates to a chain + // one hop long per target. + uint32_t depth = 0; + + // message ID of the ORIGINAL typed message, looked up in the same + // ActiveMessageHandlerTable at both ends + unsigned short original_message_id = 0; + uint16_t original_header_size = 0; + // redundant copy of the first payload byte, so that a receiver can report the + // claimed encoding in a fatal diagnostic even if the payload is truncated + unsigned char target_encoding_kind = 0; + + static void handle_message(NodeID sender, const MulticastEnvelopeMessage &hdr, + const void *payload, size_t payload_size, + TimeLimit work_until); + }; + + //////////////////////////////////////////////////////////////////////// + // + // struct MulticastAckMessage + // + + // The single acknowledgement a completion-tracked subtree sends to its parent once it + // has delivered locally AND collected an acknowledgement from every child (plan + // section 7.5). These exist only for the lifetime of one explicitly + // completion-tracked multicast; a fire-and-forget multicast never sends one. + // + // Like the envelope, this deliberately has no inline handler: handling an + // acknowledgement can send the next acknowledgement up the tree, and plan section 22 + // requires that not to happen recursively out of an inline handler. + struct MulticastAckMessage { + // (origin_node, multicast_id) identifies which multicast is being acknowledged; the + // child that sent it is the ordinary active-message sender + uint64_t multicast_id = 0; + NodeID origin_node = 0; + + static void handle_message(NodeID sender, const MulticastAckMessage &hdr, + const void *payload, size_t payload_size, + TimeLimit work_until); + }; + + //////////////////////////////////////////////////////////////////////// + // + // fatal diagnostics (plan section 21.1) + // + + // Everything plan section 21.1 requires of a multicast fatal error. Fields that are + // not knowable in a given failure (e.g. a header that did not decode) stay at their + // defaults. + struct MulticastFatalContext { + NodeID local_node = 0; + // the PREVIOUS HOP the envelope arrived from, which is not in general the origin + NodeID sender = 0; + NodeID origin_node = 0; + uint64_t multicast_id = 0; + unsigned original_message_id = 0; + unsigned target_encoding_kind = 0; + size_t target_encoding_size = 0; + size_t original_header_size = 0; + size_t original_payload_size = 0; + size_t received_payload_size = 0; + unsigned depth = 0; + MulticastDecodeStatus status = MulticastDecodeStatus::OK; + // plain-language statement of the protocol rule that was violated + const char *rule = ""; + + void describe(std::ostream &os) const; + std::string to_string(void) const; + }; + + // Injectable fatal-error hook, mirroring BarrierFatalReporter. The default reporter + // logs the full context and aborts; a test installs its own so it can assert on the + // diagnostic without dying. A reporter that returns normally means "this envelope + // is dropped", so every call site unwinds cleanly without delivering or forwarding. + class MulticastFatalReporter { + public: + virtual ~MulticastFatalReporter(void) {} + virtual void report(const MulticastFatalContext &ctx) = 0; + }; + + // returns the previously installed reporter (null means "the default"), so tests can + // restore it + MulticastFatalReporter *set_multicast_fatal_reporter(MulticastFatalReporter *reporter); + MulticastFatalReporter *get_multicast_fatal_reporter(void); + void report_multicast_fatal(const MulticastFatalContext &ctx); + + //////////////////////////////////////////////////////////////////////// + // + // aggregate remote completion (plan section 7.5) + // + + class MulticastTransport; + + // What the origin of a completion-tracked multicast wants run - exactly once, after + // the message has been received AND HANDLED by every target. The forwarding layer + // takes ownership: the object is deleted as soon as it has been invoked. + class MulticastCompletionCallback { + public: + virtual ~MulticastCompletionCallback(void); + virtual void invoke(void) = 0; + }; + + // (origin, multicast_id) is globally unique, and partitions are disjoint, so a node + // is part of at most one subtree per multicast and this is a sufficient key + struct MulticastCompletionKey { + NodeID origin = 0; + uint64_t multicast_id = 0; + + bool operator<(const MulticastCompletionKey &rhs) const + { + if(origin != rhs.origin) + return (origin < rhs.origin); + return (multicast_id < rhs.multicast_id); + } + bool operator==(const MulticastCompletionKey &rhs) const + { + return (origin == rhs.origin) && (multicast_id == rhs.multicast_id); + } + }; + + // The TRANSIENT acknowledgement state of plan section 7.5. There is exactly one of + // these per node (the runtime transport owns a process-global one; a unit test owns + // one per simulated node), and it is empty except while a completion-tracked + // multicast is actually in flight. Nothing here is a reusable multicast plan: no + // target set, no encoding and no routing information is retained, and every record is + // reclaimed the instant its subtree is complete (plan section 2, final bullet). + class MulticastCompletionState { + public: + MulticastCompletionState(void); + ~MulticastCompletionState(void); + + // What the caller must do once a record's outstanding count reaches zero. The + // record is already gone from the table by then - state is reclaimed before the + // acknowledgement is sent, never after. + struct Notification { + enum Action + { + NOTHING, // still waiting on other children or on local delivery + ACK_PARENT, // relay: send exactly one acknowledgement to 'parent' + INVOKE_CALLBACK, // origin: run 'callback' once, then delete it + UNKNOWN, // no such record - a protocol violation + }; + Action action = NOTHING; + NodeID parent = 0; + MulticastCompletionCallback *callback = nullptr; + }; + + // Origin side: retain the callback under (origin, multicast_id). 'outstanding' is + // the number of first-hop envelopes plus one if the origin is itself a target. + void begin_origin(NodeID origin, uint64_t multicast_id, size_t outstanding, + MulticastCompletionCallback *callback); + + // Relay side: retain the parent to acknowledge and the number of outstanding + // units, which is one per child envelope plus one for this node's own delivery. + void begin_relay(NodeID origin, uint64_t multicast_id, NodeID parent, + size_t outstanding); + + // one unit of progress - either this node's local delivery finished being HANDLED, + // or one child acknowledged its whole subtree + Notification note_completion(NodeID origin, uint64_t multicast_id); + + // number of multicasts currently being tracked here - zero except while a + // completion-tracked multicast is in flight + size_t num_pending(void) const; + + // high-water mark since construction or the last reset_peak(), so a test can prove + // that a fire-and-forget multicast created no acknowledgement state at all + size_t peak_pending(void) const; + void reset_peak(void); + + protected: + struct Record { + NodeID parent = 0; + size_t outstanding = 0; + MulticastCompletionCallback *callback = nullptr; + bool is_origin = false; + }; + + mutable Mutex mutex; + std::map pending; + size_t peak = 0; + }; + + // Everything the deferred half of a local delivery needs in order to settle one unit + // of completion. It is captured explicitly rather than rederived from the transport + // because the notification can run on a handler thread long after the forwarding call + // that created it returned. + struct MulticastCompletionToken { + MulticastTransport *transport = nullptr; + MulticastCompletionState *state = nullptr; + NodeID local = 0; // the node whose record this settles + NodeID origin = 0; + uint64_t multicast_id = 0; + }; + + //////////////////////////////////////////////////////////////////////// + // + // class MulticastTransport + // + + // Everything the forwarding algorithm needs from the rest of the runtime. The + // production implementation (get_runtime_multicast_transport()) sends ordinary + // unicast active messages and delivers through the runtime's IncomingMessageManager; + // unit tests supply an in-process implementation that captures sends so that a whole + // forwarding tree can be exercised without a real network. + class MulticastTransport { + public: + virtual ~MulticastTransport(void); + + virtual NodeID my_node_id(void) const = 0; + // configured node count - both ends of an encoding must agree on this + virtual NodeID num_nodes(void) const = 0; + // bounded forwarding radix R (plan section 7.3); must be at least 1 + virtual size_t radix(void) const = 0; + + // Sends one multicast envelope to 'relay', which is by construction the first node + // of the slice the envelope carries. 'payload' is the entire variable portion and + // must be copied before this returns. + virtual void send_envelope(NodeID relay, const MulticastEnvelopeMessage &env, + const void *payload, size_t payload_bytes) = 0; + + // May the ORIGINAL message be handed to a single target by ordinary unicast? The + // answer is no when the payload would need fragmentation, because fragmentation is + // driven by the compile-time message type and this layer only has a message ID - + // such a send goes through an envelope instead and gets fragmented there. + virtual bool can_send_original(size_t hdr_size, size_t payload_size) const = 0; + + // Sends the ORIGINAL typed message to 'target' by ordinary unicast. Only legal + // when the local node is the origin, because the receiver sees the local node as + // the sender (plan section 7.4). + // + // A non-null 'completion' must be settled once the target has received AND HANDLED + // the message - i.e. it is an ordinary active-message remote completion. + virtual void send_original(NodeID target, ActiveMessageHandlerTable::MessageID msgid, + const void *hdr, size_t hdr_size, const void *payload, + size_t payload_size, + const MulticastCompletionToken *completion) = 0; + + // Sends one acknowledgement for 'multicast_id' from 'from' to 'parent'. 'from' is + // passed explicitly because an acknowledgement can be produced by a deferred local + // delivery, i.e. from a context that is no longer "on" the sending node. + virtual void send_ack(NodeID from, NodeID parent, NodeID origin, + uint64_t multicast_id) = 0; + + // Final local delivery. 'origin' MUST be presented to the original handler as its + // sender; a relay never becomes the apparent sender just because it transmitted + // the final hop (plan section 7.4). + // + // A non-null 'completion' must be settled once the original handler has actually + // RUN, which for a message that could not be handled inline is later than this + // call returns; MulticastForwarder::dispatch_local does this correctly. + virtual void deliver_local(NodeID origin, ActiveMessageHandlerTable::MessageID msgid, + const void *hdr, size_t hdr_size, const void *payload, + size_t payload_size, TimeLimit work_until, + const MulticastCompletionToken *completion) = 0; + + // The transient acknowledgement state of this node (plan section 7.5). Empty + // except while a completion-tracked multicast is in flight. + virtual MulticastCompletionState &completion_state(void) = 0; + }; + + // The transport used by the registered envelope handler and by the typed helpers + // below. Its radix comes from the core module config value that also backs + // -ll:barrier_radix, read once and cached. + MulticastTransport &get_runtime_multicast_transport(void); + + //////////////////////////////////////////////////////////////////////// + // + // class MulticastForwarder + // + + // The bounded-radix forwarding algorithm of plan section 7.3. It is expressed purely + // in terms of MulticastTransport, so the whole tree is exercisable in one process. + class MulticastForwarder { + public: + // Origin-side entry point. 'targets' is the complete logical target set; the local + // node may or may not be a member. 'hdr'/'payload' are copied before this returns + // so a PAYLOAD_KEEP-style caller buffer may be reused immediately (plan 7.5) - + // equivalently, LOCAL completion has already happened when this returns and this + // layer no longer references any caller-owned data. + // + // An empty target set is a successful no-op, and a single remote target uses the + // ordinary unicast fast path. + // + // 'on_remote_complete', if given, is invoked EXACTLY ONCE after the message has + // been received and handled by every target, and is then deleted. Ownership + // transfers to this call. Passing null - the normal fire-and-forget case - means + // no acknowledgement metadata is put on the wire and no acknowledgement state is + // created anywhere (plan section 7.5). + static void send(MulticastTransport &transport, const MulticastTargetSet &targets, + ActiveMessageHandlerTable::MessageID msgid, const void *hdr, + size_t hdr_size, const void *payload, size_t payload_size, + TimeLimit work_until = TimeLimit(), + MulticastMetricsSink *metrics = 0, + MulticastCompletionCallback *on_remote_complete = 0); + + // Relay-side entry point, called by MulticastEnvelopeMessage::handle_message. + // 'sender' is the previous hop and is used only for diagnostics. + static void forward(MulticastTransport &transport, NodeID sender, + const MulticastEnvelopeMessage &env, const void *payload, + size_t payload_size, TimeLimit work_until, + MulticastMetricsSink *metrics = 0); + + // Acknowledgement-side entry point, called by MulticastAckMessage::handle_message. + static void handle_ack(MulticastTransport &transport, NodeID sender, + const MulticastAckMessage &ack); + + // Records one unit of progress against a retained completion record and performs + // whatever that completes: acknowledging a parent, or invoking (and deleting) the + // origin's callback. Public because it is also the deferred local-delivery + // notification and the unicast fast path's remote completion. + static void settle(const MulticastCompletionToken &token); + + // Invokes the ALREADY REGISTERED handler for 'msgid' with 'sender' as the apparent + // sender, reusing the ordinary incoming-message machinery so that inline-handler + // and TimeLimit behavior is identical to a directly received message. Handler + // signature detection is never duplicated here (plan section 7.4). + // + // If 'completion' is non-null it is settled once the handler has actually run - + // immediately if the message was handled inline, and otherwise from the ordinary + // post-handler callback so that "handled by every target" really means handled. + // The original message type must not be one that carries its own FragmentInfo, + // because an incomplete fragment never reaches a handler and so could never be + // observed as handled (checked in debug builds). + static bool dispatch_local(IncomingMessageManager *manager, NodeID sender, + ActiveMessageHandlerTable::MessageID msgid, + const void *hdr, size_t hdr_size, const void *payload, + size_t payload_size, TimeLimit work_until, + const MulticastCompletionToken *completion = 0); + + // local half of the globally unique (origin_node, counter) multicast ID + static uint64_t next_multicast_id(void); + }; + + //////////////////////////////////////////////////////////////////////// + // + // typed helpers + // + + // Multicasts one already-built typed header (and an optional copied 1-D payload) to + // 'targets' over the runtime transport. This REPLACES the old ActiveMessage(NodeSet) + // constructor, which fanned out one message per target at the source; that + // constructor, Network::create_active_message_impl(NodeSet, ...), the NodeSet + // recommended_max_payload overloads and every backend's multicast source loop are all + // gone (plan section 7.6). + // + // A caller holding a Realm::NodeSet converts explicitly with + // MulticastTargetSet(nodes) - NodeSet itself stays as a general in-memory set. + // + // The payload is always copied, so this is the PAYLOAD_COPY/PAYLOAD_KEEP equivalent; + } // namespace Realm #include "realm/activemsg.inl" diff --git a/src/realm/activemsg.inl b/src/realm/activemsg.inl index 28b01527e9..9330b15571 100644 --- a/src/realm/activemsg.inl +++ b/src/realm/activemsg.inl @@ -92,37 +92,6 @@ namespace Realm { fbs.reset(impl->payload_base, impl->payload_size); } - template - ActiveMessage::ActiveMessage(const Realm::NodeSet &_targets, - size_t _max_payload_size /*= 0*/) - : impl(0) - { - init(_targets, _max_payload_size); - } - - template - void ActiveMessage::init(const Realm::NodeSet &_targets, - size_t _max_payload_size /*= 0*/) - { - assert(impl == 0); - if constexpr(!is_wrapped_with_frag_info::value) { - if(_max_payload_size > 0) { - size_t net_max = Network::max_payload_size(sizeof(T), nullptr); - if(_max_payload_size > net_max) { - init_chunked(_targets, _max_payload_size); - return; - } - } - } - network_max_payload_ = 0; - unsigned short msgid = activemsg_handler_table.lookup_message_id(); - impl = Network::create_active_message_impl(_targets, msgid, sizeof(T), - _max_payload_size, 0, 0, 0, - &inline_capacity, sizeof(inline_capacity)); - header = new(impl->header_base) T; - fbs.reset(impl->payload_base, impl->payload_size); - } - template ActiveMessage::ActiveMessage(NodeID _target, const void *_data, size_t _datalen) @@ -178,36 +147,6 @@ namespace Realm { header = new(impl->header_base) T; } - template - ActiveMessage::ActiveMessage(const Realm::NodeSet &_targets, - const void *_data, size_t _datalen) - : impl(0) - { - init(_targets, _data, _datalen); - } - - template - void ActiveMessage::init(const Realm::NodeSet &_targets, - const void *_data, size_t _datalen) - { - assert(impl == 0); - if constexpr(!is_wrapped_with_frag_info::value) { - if(_datalen > 0) { - size_t net_max = Network::max_payload_size(sizeof(T), _data); - if(_datalen > net_max) { - init_chunked_data(_targets, _data, _datalen); - return; - } - } - } - network_max_payload_ = 0; - unsigned short msgid = activemsg_handler_table.lookup_message_id(); - impl = Network::create_active_message_impl(_targets, msgid, sizeof(T), _datalen, - _data, 0, 0, &inline_capacity, - sizeof(inline_capacity)); - header = new(impl->header_base) T; - } - template ActiveMessage::ActiveMessage(NodeID _target, const LocalAddress &_src_payload_addr, @@ -250,14 +189,6 @@ namespace Realm { return Network::recommended_max_payload(target, with_congestion, sizeof(T)); } - template - /*static*/ size_t - ActiveMessage::recommended_max_payload(const NodeSet &targets, - bool with_congestion) - { - return Network::recommended_max_payload(targets, with_congestion, sizeof(T)); - } - template /*static*/ size_t ActiveMessage::recommended_max_payload( NodeID target, const RemoteAddress &dest_payload_addr, bool with_congestion) @@ -275,15 +206,6 @@ namespace Realm { line_stride, with_congestion, sizeof(T)); } - template - /*static*/ size_t ActiveMessage::recommended_max_payload( - const NodeSet &targets, const void *data, size_t bytes_per_line, size_t lines, - size_t line_stride, bool with_congestion) - { - return Network::recommended_max_payload(targets, data, bytes_per_line, lines, - line_stride, with_congestion, sizeof(T)); - } - template /*static*/ size_t ActiveMessage::recommended_max_payload( NodeID target, const LocalAddress &src_payload_addr, size_t bytes_per_line, @@ -432,7 +354,7 @@ namespace Realm { header = 0; chunk_alloc_.clear(); chunk_alloc_.shrink_to_fit(); - chunk_targets_.clear(); + chunk_target_ = -1; chunk_src_data_ = nullptr; chunk_src_datalen_ = 0; network_max_payload_ = 0; @@ -567,8 +489,7 @@ namespace Realm { // automatically register a WrappedWithFragInfo handler so that // fragmented messages of any type can be reassembled - if constexpr(!is_wrapped_with_frag_info::value && - std::is_same::value) { + if constexpr(!is_wrapped_with_frag_info::value && std::is_same::value) { auto *wrapped_reg = new ActiveMessageHandlerReg, T>(); (void)wrapped_reg; // leaked intentionally - lives for program lifetime } @@ -597,27 +518,7 @@ namespace Realm { network_max_payload_ = Network::max_payload_size(wrapped_hdr_size, nullptr); assert(network_max_payload_ > 0); - chunk_targets_.add(_target); - chunk_alloc_.resize(_max_payload_size); - chunk_src_data_ = chunk_alloc_.data(); - chunk_src_datalen_ = _max_payload_size; - header = new(&inline_capacity) T; - fbs.reset(chunk_alloc_.data(), _max_payload_size); - } - } - - template - void ActiveMessage::init_chunked(const NodeSet &_targets, - size_t _max_payload_size) - { - if constexpr(is_wrapped_with_frag_info::value) { - assert(0 && "init_chunked called on WrappedWithFragInfo type"); - } else { - size_t wrapped_hdr_size = sizeof(WrappedWithFragInfo); - network_max_payload_ = Network::max_payload_size(wrapped_hdr_size, nullptr); - assert(network_max_payload_ > 0); - - chunk_targets_ = _targets; + chunk_target_ = _target; chunk_alloc_.resize(_max_payload_size); chunk_src_data_ = chunk_alloc_.data(); chunk_src_datalen_ = _max_payload_size; @@ -640,7 +541,7 @@ namespace Realm { network_max_payload_ = Network::max_payload_size(wrapped_hdr_size, _data); assert(network_max_payload_ > 0); - chunk_targets_.add(_target); + chunk_target_ = _target; chunk_src_data_ = _data; chunk_src_datalen_ = _datalen; // place the header in inline_capacity (it's not sent to the network yet) @@ -650,25 +551,6 @@ namespace Realm { } } - template - void ActiveMessage::init_chunked_data(const NodeSet &_targets, - const void *_data, - size_t _datalen) - { - if constexpr(is_wrapped_with_frag_info::value) { - assert(0 && "init_chunked_data called on WrappedWithFragInfo type"); - } else { - size_t wrapped_hdr_size = sizeof(WrappedWithFragInfo); - network_max_payload_ = Network::max_payload_size(wrapped_hdr_size, _data); - assert(network_max_payload_ > 0); - - chunk_targets_ = _targets; - chunk_src_data_ = _data; - chunk_src_datalen_ = _datalen; - header = new(&inline_capacity) T; - } - } - template void ActiveMessage::commit_chunked(void) { @@ -684,7 +566,6 @@ namespace Realm { size_t total_payload = chunk_src_datalen_; // msg_id only needs to be unique per-sender, so using my_node_id is fine - // for both unicast and multicast chunked sends uint64_t msg_id = next_chunk_message_id(Network::my_node_id); size_t max_chunk = network_max_payload_; uint32_t total_chunks = @@ -697,7 +578,7 @@ namespace Realm { for(uint32_t chunk_id = 0; chunk_id < total_chunks; ++chunk_id) { size_t chunk_size = std::min(max_chunk, total_payload - offset); - ActiveMessage> chunk_msg(chunk_targets_, chunk_size); + ActiveMessage> chunk_msg(chunk_target_, chunk_size); chunk_msg->frag_info = {chunk_id, total_chunks, msg_id}; chunk_msg->user = *header; if(chunk_size > 0) { @@ -713,7 +594,7 @@ namespace Realm { header = 0; chunk_alloc_.clear(); chunk_alloc_.shrink_to_fit(); - chunk_targets_.clear(); + chunk_target_ = -1; chunk_src_data_ = nullptr; chunk_src_datalen_ = 0; network_max_payload_ = 0; @@ -886,4 +767,42 @@ namespace Realm { return HandlerWrappers::template get_handler_inline(0); } + //////////////////////////////////////////////////////////////////////// + // + // multicast completion callbacks and multicast_message (plan section 7.5) + // + + template + class MulticastCompletionCallbackImpl : public MulticastCompletionCallback { + public: + MulticastCompletionCallbackImpl(const CALLABLE &_callable) + : callable(_callable) + {} + virtual void invoke(void) { callable(); } + + protected: + CALLABLE callable; + }; + + // convenience wrapper, mirroring ActiveMessage::add_remote_completion's callable + template + inline MulticastCompletionCallback *make_multicast_completion(const CALLABLE &callable) + { + return new MulticastCompletionCallbackImpl(callable); + } + + // there is deliberately no remote-address/RDMA form (see plan section 7.5 and the + // comment on MulticastForwarder::send). + template + inline void multicast_message(const MulticastTargetSet &targets, const T &header, + const void *payload = 0, size_t payload_size = 0, + MulticastMetricsSink *metrics = 0, + MulticastCompletionCallback *on_remote_complete = 0) + { + MulticastForwarder::send(get_runtime_multicast_transport(), targets, + activemsg_handler_table.lookup_message_id(), &header, + sizeof(T), payload, payload_size, TimeLimit(), metrics, + on_remote_complete); + } + }; // namespace Realm diff --git a/src/realm/barrier_impl.cc b/src/realm/barrier_impl.cc index da44f725b8..fd768cada9 100644 --- a/src/realm/barrier_impl.cc +++ b/src/realm/barrier_impl.cc @@ -60,24 +60,6 @@ namespace Realm { return b; } - Barrier Barrier::create_barrier(const Barrier::ParticipantInfo *expected_arrivals, - size_t num_participants, ReductionOpID redop_id, - const void *initial_value, size_t initial_value_size) - { - assert(0); - // TODO(apryakhin@): Implement me - return Barrier(); - } - - Barrier Barrier::set_arrival_pattern(const Barrier::ParticipantInfo *expected_arrivals, - size_t num_participants) - { - assert(0); - // TODO(apryakhin@): Implement me - BarrierImpl *impl = get_barrier_impl(*this); - return impl->current_barrier(); - } - void Barrier::destroy_barrier(void) { log_barrier.info() << "barrier destruction request: " << *this; diff --git a/src/realm/cuda/cuda_module.cc b/src/realm/cuda/cuda_module.cc index 4170b85814..e9a5e07ddf 100644 --- a/src/realm/cuda/cuda_module.cc +++ b/src/realm/cuda/cuda_module.cc @@ -39,6 +39,7 @@ #include #endif +#include "realm/activemsg.h" #include "realm/mutex.h" #include "realm/utils.h" @@ -4158,22 +4159,32 @@ namespace Realm { enumerate_ipc_entries(entries, n.memories); enumerate_ipc_entries(entries, n.ib_memories); - // Broadcast all the IPC handles to all my peers + // Broadcast all the IPC handles to all my peers over the bounded-radix + // multicast tree (plan section 7.6). The payload layout is unchanged - + // [HOST_NAME_MAX hostname bytes][entries] on POSIX, [entries] on Windows - + // but it is now assembled into one contiguous buffer, since the multicast + // layer takes a single copied 1-D payload and fragments the envelope itself + // if it is too large for one message. // TODO: this could be replaced with ipc_mailbox size_t datalen = entries.size() * sizeof(entries[0]); - ActiveMessage amsg( - ipc_peers, - ActiveMessage::recommended_max_payload( - ipc_peers, entries.data(), HOST_NAME_MAX + datalen, 1, datalen, true)); + CudaIpcImportRequest msg{}; + std::vector msg_payload; #if !defined(REALM_IS_WINDOWS) - amsg->hostid = gethostid(); + msg.hostid = gethostid(); char hostname[HOST_NAME_MAX]; gethostname(hostname, sizeof(hostname)); - amsg.add_payload(hostname, sizeof(hostname)); + msg_payload.resize(sizeof(hostname) + datalen); + memcpy(msg_payload.data(), hostname, sizeof(hostname)); + if(datalen > 0) + memcpy(msg_payload.data() + sizeof(hostname), entries.data(), datalen); +#else + msg_payload.resize(datalen); + if(datalen > 0) + memcpy(msg_payload.data(), entries.data(), datalen); #endif - amsg->count = entries.size(); - amsg.add_payload(entries.data(), datalen); - amsg.commit(); + msg.count = entries.size(); + multicast_message(MulticastTargetSet(ipc_peers), msg, msg_payload.data(), + msg_payload.size()); log_cudaipc.debug() << "Sent " << entries.size() << " IPC entries"; diff --git a/src/realm/event.h b/src/realm/event.h index 8779b35152..39ca47361a 100644 --- a/src/realm/event.h +++ b/src/realm/event.h @@ -252,34 +252,6 @@ namespace Realm { const void *initial_value = 0, size_t initial_value_size = 0); - struct ParticipantInfo { - AddressSpace address_space; - unsigned count; - }; - - /** - * Creates a barrier - * \param expected_arrivals information about the arrival pattern - * \param num_participants the size of expected arrivals - * \param redop_id ID of a reduction operator - * \param initial_value initial reduction value - * \param initial_value_size size of the initial reduction value. - * \return barrier handle - */ - static Barrier create_barrier(const Barrier::ParticipantInfo *expected_arrivals, - size_t num_participants, ReductionOpID redop_id = 0, - const void *initial_value = 0, - size_t initial_value_size = 0); - - /** - * Sets the arrival pattern - * \param expected_arrivals information about the arrival pattern - * \param num_participants the size of expected arrivals - * \return barrier handle - */ - Barrier set_arrival_pattern(const Barrier::ParticipantInfo *expected_arrivals, - size_t num_participants); - void destroy_barrier(void); static const ::realm_event_gen_t MAX_PHASES; @@ -287,20 +259,62 @@ namespace Realm { /* * Advance a barrier to the next phase, returning a new barrier * handle. Attemps to advance beyond the last phase return NO_BARRIER - * instead. + * instead. The causal timestamp of this handle (see alter_arrival_count) + * is preserved: an adjustment branch is not reset at a phase boundary. * \return the new barrier handle. */ Barrier advance_barrier(void) const; /* * Alter the arrival count of a barrier. + * + * The change is PERSISTENT: `delta` applies to this generation and to every + * subsequent generation of the barrier, not just to the phase the handle names. + * Sibling alterations accumulate, so two alterations of +2 and +3 issued on the + * same handle leave a net persistent change of +5. + * + * The returned handle carries a causal timestamp identifying this alteration and + * must be used for any subsequent work on that adjustment branch. Specifically, + * every returned handle must be used by at least one of: + * - an `arrive()` performed on that handle; + * - another `alter_arrival_count()` invoked on that handle; + * - a descendant chain of alterations that ends in such an arrival; + * - the terminal-negative exception described below. + * "At least one use" is the contract - the handle is not single-use, and multiple + * arrivals may be made on it. + * + * Sibling alterations (two or more alterations invoked on the same input handle) + * are allowed and do not have to be ordered with respect to each other. However, + * each sibling requires its OWN distinct outstanding arrival: an arrival made on + * one sibling's handle does not witness an incomparable sibling, and an arrival + * on a descendant witnesses only the ancestors on its own causal path. Before + * issuing an alteration the application must still hold at least one unissued + * arrival from the pre-alteration expected count; that reserved arrival is what + * prevents the barrier from triggering before it learns of the alteration. + * + * A negative alteration that makes the current generation's remaining arrival + * count exactly zero is the only case that needs no arrival on the returned + * handle. Such a terminal negative closes only its own causal branch - it never + * satisfies the outstanding-arrival obligation of an incomparable sibling. + * + * The expected arrival count for the current and every future generation must + * remain positive and within the supported range. + * + * Violating any of the above is a fatal application error, but because the + * runtime can only reason about alterations it has actually received, the + * violation may be detected late (or, in the worst case, show up as a barrier + * that triggered too early). + * + * This call is nonblocking and performs no round trip to the barrier's owner. + * * \param delta the amount to adjust the arrival count by - * \return the new barrier handle. + * \return the new barrier handle, carrying this alteration's causal timestamp. */ Barrier alter_arrival_count(int delta) const; /* - * Get the previous phase of a barrier. + * Get the previous phase of a barrier. As with advance_barrier(), the + * causal timestamp of this handle is preserved. * \return the previous phase of the barrier */ Barrier get_previous_phase(void) const; diff --git a/src/realm/gasnet1/gasnet1_module.cc b/src/realm/gasnet1/gasnet1_module.cc index a04ab598e8..3cfbe177ab 100644 --- a/src/realm/gasnet1/gasnet1_module.cc +++ b/src/realm/gasnet1/gasnet1_module.cc @@ -344,10 +344,6 @@ namespace Realm { size_t _max_payload_size, const void *_src_payload_addr, size_t _src_payload_lines, size_t _src_payload_line_stride, void *_dest_payload_addr); - GASNet1MessageImpl(const Realm::NodeSet &_targets, unsigned short _msgid, - size_t _header_size, size_t _max_payload_size, - const void *_src_payload_addr, size_t _src_payload_lines, - size_t _src_payload_line_stride); virtual ~GASNet1MessageImpl(); @@ -359,8 +355,6 @@ namespace Realm { protected: NodeID target; - Realm::NodeSet targets; - bool is_multicast; const void *src_payload_addr; size_t src_payload_lines; size_t src_payload_line_stride; @@ -383,7 +377,6 @@ namespace Realm { size_t _src_payload_line_stride, void *_dest_payload_addr) : target(_target) - , is_multicast(false) , src_payload_addr(_src_payload_addr) , src_payload_lines(_src_payload_lines) , src_payload_line_stride(_src_payload_line_stride) @@ -402,32 +395,6 @@ namespace Realm { assert((sizeof(BaseMedium) + 8 + header_size) <= 16 * sizeof(handlerarg_t)); } - GASNet1MessageImpl::GASNet1MessageImpl(const Realm::NodeSet &_targets, - unsigned short _msgid, size_t _header_size, - size_t _max_payload_size, - const void *_src_payload_addr, - size_t _src_payload_lines, - size_t _src_payload_line_stride) - : targets(_targets) - , is_multicast(true) - , src_payload_addr(_src_payload_addr) - , src_payload_lines(_src_payload_lines) - , src_payload_line_stride(_src_payload_line_stride) - , dest_payload_addr(0) - , comp(0) - , header_size(_header_size) - { - if(_max_payload_size && (src_payload_addr == 0)) { - payload_base = reinterpret_cast(malloc(_max_payload_size)); - } else { - payload_base = 0; - } - payload_size = _max_payload_size; - args.msgid = _msgid; - header_base = &args.msg_header; - assert((sizeof(BaseMedium) + 8 + header_size) <= 16 * sizeof(handlerarg_t)); - } - GASNet1MessageImpl::~GASNet1MessageImpl() {} void *GASNet1MessageImpl::add_local_completion(size_t size) @@ -452,48 +419,19 @@ namespace Realm { args.sender = Network::my_node_id; args.payload_len = act_payload_size; - if(is_multicast) { - assert(dest_payload_addr == 0); - assert(comp == 0); - size_t count = targets.size(); - if(count > 0) { - for(NodeSet::const_iterator it = targets.begin(); it != targets.end(); ++it) { - if(src_payload_addr != 0) { - if(src_payload_lines > 1) - enqueue_message(*it, MSGID_NEW_ACTIVEMSG, &args, header_size + 24, - src_payload_addr, act_payload_size / src_payload_lines, - src_payload_line_stride, src_payload_lines, PAYLOAD_KEEP, - 0); - else - enqueue_message(*it, MSGID_NEW_ACTIVEMSG, &args, header_size + 24, - src_payload_addr, act_payload_size, PAYLOAD_KEEP, 0); - } else - enqueue_message(*it, MSGID_NEW_ACTIVEMSG, &args, header_size + 24, - payload_base, act_payload_size, - ((count > 0) ? PAYLOAD_COPY : PAYLOAD_FREE), 0); - count--; - } - } else { - // free the (unused) payload ourselves - if((payload_size > 0) && (src_payload_addr == 0)) - free(payload_base); - } - } else { - if(src_payload_addr != 0) { - if(src_payload_lines > 1) - enqueue_message(target, MSGID_NEW_ACTIVEMSG, &args, header_size + 24, - src_payload_addr, (act_payload_size / src_payload_lines), - src_payload_line_stride, src_payload_lines, PAYLOAD_KEEP, comp, - dest_payload_addr); - else - enqueue_message(target, MSGID_NEW_ACTIVEMSG, &args, header_size + 24, - src_payload_addr, act_payload_size, PAYLOAD_KEEP, comp, - dest_payload_addr); - } else + if(src_payload_addr != 0) { + if(src_payload_lines > 1) enqueue_message(target, MSGID_NEW_ACTIVEMSG, &args, header_size + 24, - payload_base, act_payload_size, PAYLOAD_FREE, comp, + src_payload_addr, (act_payload_size / src_payload_lines), + src_payload_line_stride, src_payload_lines, PAYLOAD_KEEP, comp, dest_payload_addr); - } + else + enqueue_message(target, MSGID_NEW_ACTIVEMSG, &args, header_size + 24, + src_payload_addr, act_payload_size, PAYLOAD_KEEP, comp, + dest_payload_addr); + } else + enqueue_message(target, MSGID_NEW_ACTIVEMSG, &args, header_size + 24, payload_base, + act_payload_size, PAYLOAD_FREE, comp, dest_payload_addr); } void GASNet1MessageImpl::cancel() @@ -886,30 +824,12 @@ namespace Realm { return impl; } - ActiveMessageImpl *GASNet1Module::create_active_message_impl( - const NodeSet &targets, unsigned short msgid, size_t header_size, - size_t max_payload_size, const void *src_payload_addr, size_t src_payload_lines, - size_t src_payload_line_stride, void *storage_base, size_t storage_size) - { - assert(storage_size >= sizeof(GASNet1MessageImpl)); - GASNet1MessageImpl *impl = new(storage_base) - GASNet1MessageImpl(targets, msgid, header_size, max_payload_size, - src_payload_addr, src_payload_lines, src_payload_line_stride); - return impl; - } - size_t GASNet1Module::recommended_max_payload(NodeID target, bool with_congestion, size_t header_size) { return gasnet_AMMaxMedium(); } - size_t GASNet1Module::recommended_max_payload(const NodeSet &targets, - bool with_congestion, size_t header_size) - { - return gasnet_AMMaxMedium(); - } - size_t GASNet1Module::recommended_max_payload(NodeID target, const RemoteAddress &dest_payload_addr, bool with_congestion, size_t header_size) @@ -927,14 +847,6 @@ namespace Realm { return gasnet_AMMaxMedium(); } - size_t GASNet1Module::recommended_max_payload(const NodeSet &targets, const void *data, - size_t bytes_per_line, size_t lines, - size_t line_stride, bool with_congestion, - size_t header_size) - { - return gasnet_AMMaxMedium(); - } - size_t GASNet1Module::recommended_max_payload(NodeID target, const LocalAddress &src_payload_addr, size_t bytes_per_line, size_t lines, diff --git a/src/realm/gasnet1/gasnet1_module.h b/src/realm/gasnet1/gasnet1_module.h index 7ae802a37b..63dd005b0d 100644 --- a/src/realm/gasnet1/gasnet1_module.h +++ b/src/realm/gasnet1/gasnet1_module.h @@ -95,15 +95,8 @@ namespace Realm { NodeID target, unsigned short msgid, size_t header_size, size_t max_payload_size, const RemoteAddress &dest_payload_addr, void *storage_base, size_t storage_size); - virtual ActiveMessageImpl *create_active_message_impl( - const NodeSet &targets, unsigned short msgid, size_t header_size, - size_t max_payload_size, const void *src_payload_addr, size_t src_payload_lines, - size_t src_payload_line_stride, void *storage_base, size_t storage_size); - virtual size_t recommended_max_payload(NodeID target, bool with_congestion, size_t header_size); - virtual size_t recommended_max_payload(const NodeSet &targets, bool with_congestion, - size_t header_size); virtual size_t recommended_max_payload(NodeID target, const RemoteAddress &dest_payload_addr, bool with_congestion, size_t header_size); @@ -111,10 +104,6 @@ namespace Realm { size_t bytes_per_line, size_t lines, size_t line_stride, bool with_congestion, size_t header_size); - virtual size_t recommended_max_payload(const NodeSet &targets, const void *data, - size_t bytes_per_line, size_t lines, - size_t line_stride, bool with_congestion, - size_t header_size); virtual size_t recommended_max_payload(NodeID target, const LocalAddress &src_payload_addr, size_t bytes_per_line, size_t lines, diff --git a/src/realm/gasnetex/gasnetex_module.cc b/src/realm/gasnetex/gasnetex_module.cc index 9f8ae83b7e..82bd5d01df 100644 --- a/src/realm/gasnetex/gasnetex_module.cc +++ b/src/realm/gasnetex/gasnetex_module.cc @@ -138,11 +138,6 @@ namespace Realm { size_t _max_payload_size, const void *_src_payload_addr, size_t _src_payload_lines, size_t _src_payload_line_stride, uintptr_t _dest_payload_addr, gex_ep_index_t _dest_ep_index); - GASNetEXMessageImpl(GASNetEXInternal *_internal, const Realm::NodeSet &_targets, - unsigned short _msgid, size_t _header_size, - size_t _max_payload_size, const void *_src_payload_addr, - size_t _src_payload_lines, size_t _src_payload_line_stride); - virtual ~GASNetEXMessageImpl(); virtual void *add_local_completion(size_t size); @@ -154,8 +149,6 @@ namespace Realm { protected: GASNetEXInternal *internal; NodeID target; - Realm::NodeSet targets; - bool is_multicast; unsigned short msgid; const void *src_payload_addr; size_t src_payload_lines; @@ -184,7 +177,6 @@ namespace Realm { uintptr_t _dest_payload_addr, gex_ep_index_t _dest_ep_index) : internal(_internal) , target(_target) - , is_multicast(false) , msgid(_msgid) , src_payload_addr(_src_payload_addr) , src_payload_lines(_src_payload_lines) @@ -216,42 +208,6 @@ namespace Realm { payload_base, payload_size, _dest_payload_addr); } - GASNetEXMessageImpl::GASNetEXMessageImpl( - GASNetEXInternal *_internal, const Realm::NodeSet &_targets, unsigned short _msgid, - size_t _header_size, size_t _max_payload_size, const void *_src_payload_addr, - size_t _src_payload_lines, size_t _src_payload_line_stride) - : internal(_internal) - , targets(_targets) - , is_multicast(true) - , msgid(_msgid) - , src_payload_addr(_src_payload_addr) - , src_payload_lines(_src_payload_lines) - , src_payload_line_stride(_src_payload_line_stride) - , comp(nullptr) - { - // for multicast messages, we store the payload in a temp buffer - assert(_header_size <= INLINE_SIZE); - header_size = _header_size; - header_base = &msg_data[0]; - size_t header_padded = roundup_pow2(header_size, 8); - - payload_size = _max_payload_size; - if(payload_size > 0) { - if(src_payload_addr != nullptr) { - // use the caller's storage for now - payload_base = const_cast(src_payload_addr); - } else if((header_padded + payload_size) <= INLINE_SIZE) { - // we can use the rest of our internal storage - payload_base = &msg_data[header_padded / sizeof(gex_am_arg_t)]; - } else { - // have to dynamically allocate storage - payload_base = malloc(payload_size); - assert(payload_base != 0); - } - } else - payload_base = nullptr; - } - GASNetEXMessageImpl::~GASNetEXMessageImpl() {} void *GASNetEXMessageImpl::add_local_completion(size_t size) @@ -272,89 +228,33 @@ namespace Realm { void GASNetEXMessageImpl::commit(size_t act_payload_size) { - if(is_multicast) { - // if/when we build a tree for multicast, expected local may be - // lower than expected remote - unsigned exp_local = targets.size(); - unsigned exp_remote = exp_local; - if(comp && !comp->mark_ready(exp_local, exp_remote)) - comp = nullptr; - - for(NodeID tgt : targets) { - void *act_header = header_base; - // can't use src payload directly if it's 2d - void *act_payload = ((src_payload_lines <= 1) ? payload_base : nullptr); - - PreparedMessage *msg; - msg = internal->prepare_message(tgt, 0 /*always prim EP*/, msgid, act_header, - header_size, act_payload, act_payload_size, - 0 /* no RDMA destination*/); - - if(act_header != header_base) - memcpy(act_header, header_base, header_size); - if(act_payload != payload_base) { - if(src_payload_lines > 1) { - // copy line by line - size_t bytes_per_line = act_payload_size / src_payload_lines; - for(size_t i = 0; i < src_payload_lines; i++) - memcpy(static_cast(act_payload) + (i * bytes_per_line), - static_cast(payload_base) + - (i * src_payload_line_stride), - bytes_per_line); - } else { - // simple memcpy - memcpy(act_payload, payload_base, act_payload_size); - } - } - - internal->commit_message(msg, comp, act_header, header_size, act_payload, - act_payload_size); - } - - // if we dynamically allocated space for the payload, free that now - if((payload_size > 0) && (src_payload_addr == nullptr) && - ((header_size + payload_size) > INLINE_SIZE)) - free(payload_base); - } else { - // arm the pending completion (if present) - if(comp && !comp->mark_ready(1 /*exp_local*/, 1 /*exp_remote*/)) - comp = nullptr; - - if((src_payload_addr != 0) && (payload_base != src_payload_addr)) { - if(src_payload_lines > 1) { - // copy line by line - size_t bytes_per_line = act_payload_size / src_payload_lines; - for(size_t i = 0; i < src_payload_lines; i++) - memcpy(static_cast(payload_base) + (i * bytes_per_line), - static_cast(src_payload_addr) + - (i * src_payload_line_stride), - bytes_per_line); - } else { - // simple memcpy - memcpy(payload_base, src_payload_addr, act_payload_size); - } - - comp = internal->early_local_completion(comp); + // arm the pending completion (if present) + if(comp && !comp->mark_ready(1 /*exp_local*/, 1 /*exp_remote*/)) + comp = nullptr; + + if((src_payload_addr != 0) && (payload_base != src_payload_addr)) { + if(src_payload_lines > 1) { + // copy line by line + size_t bytes_per_line = act_payload_size / src_payload_lines; + for(size_t i = 0; i < src_payload_lines; i++) + memcpy(static_cast(payload_base) + (i * bytes_per_line), + static_cast(src_payload_addr) + + (i * src_payload_line_stride), + bytes_per_line); + } else { + // simple memcpy + memcpy(payload_base, src_payload_addr, act_payload_size); } - internal->commit_message(msg, comp, header_base, header_size, payload_base, - act_payload_size); + comp = internal->early_local_completion(comp); } - } - void GASNetEXMessageImpl::cancel() - { - if(is_multicast) { - // we never told the internal gex logic about this, so all we have to - // is free payload memory if we allocated it - if((payload_size > 0) && (src_payload_addr == nullptr) && - ((header_size + payload_size) > INLINE_SIZE)) - free(payload_base); - } else { - internal->cancel_message(msg); - } + internal->commit_message(msg, comp, header_base, header_size, payload_base, + act_payload_size); } + void GASNetEXMessageImpl::cancel() { internal->cancel_message(msg); } + //////////////////////////////////////////////////////////////////////// // // class GASNetEXModule @@ -688,34 +588,6 @@ namespace Realm { return impl; } - ActiveMessageImpl *GASNetEXModule::create_active_message_impl( - const NodeSet &targets, unsigned short msgid, size_t header_size, - size_t max_payload_size, const void *src_payload_addr, size_t src_payload_lines, - size_t src_payload_line_stride, void *storage_base, size_t storage_size) - { - // if checksums are enabled, we'll tack it on to the end of the header - // to avoid any alignment issues - if(cfg_do_checksums) - header_size = - roundup_pow2(header_size + sizeof(gex_am_arg_t), sizeof(gex_am_arg_t)); - - assert(storage_size >= sizeof(GASNetEXMessageImpl)); - if(targets.size() == 1) { - // optimization - if there's exactly 1 target, redirect to the unicast mode - NodeID target = *(targets.begin()); - GASNetEXMessageImpl *impl = new(storage_base) GASNetEXMessageImpl( - internal, target, msgid, header_size, max_payload_size, src_payload_addr, - src_payload_lines, src_payload_line_stride, 0, 0); - return impl; - } else { - // zero or 2+ targets - we'll make a temporary copy of the payload for now - GASNetEXMessageImpl *impl = new(storage_base) GASNetEXMessageImpl( - internal, targets, msgid, header_size, max_payload_size, src_payload_addr, - src_payload_lines, src_payload_line_stride); - return impl; - } - } - size_t GASNetEXModule::recommended_max_payload(NodeID target, bool with_congestion, size_t header_size) { @@ -727,24 +599,6 @@ namespace Realm { header_size, 0 /*no dest_ptr*/); } - size_t GASNetEXModule::recommended_max_payload(const NodeSet &targets, - bool with_congestion, size_t header_size) - { - if(cfg_do_checksums) - header_size = - roundup_pow2(header_size + sizeof(gex_am_arg_t), sizeof(gex_am_arg_t)); - - if(targets.size() == 1) { - // optimization - if there's exactly 1 target, redirect to the unicast mode - NodeID target = *(targets.begin()); - return internal->recommended_max_payload(target, 0 /*ep_index*/, with_congestion, - header_size, 0 /*no dest_ptr*/); - } else { - // ask without specifying a target - gets conservative answer - return internal->recommended_max_payload(with_congestion, header_size); - } - } - size_t GASNetEXModule::recommended_max_payload(NodeID target, const RemoteAddress &dest_payload_addr, bool with_congestion, size_t header_size) @@ -768,23 +622,6 @@ namespace Realm { header_size, 0 /*no dest_ptr*/); } - size_t GASNetEXModule::recommended_max_payload(const NodeSet &targets, const void *data, - size_t bytes_per_line, size_t lines, - size_t line_stride, bool with_congestion, - size_t header_size) - { - if(targets.size() == 1) { - // optimization - if there's exactly 1 target, redirect to the unicast mode - NodeID target = *(targets.begin()); - return internal->recommended_max_payload( - target, 0 /*ep_index*/, data, bytes_per_line, lines, line_stride, - with_congestion, header_size, 0 /*no dest_ptr*/); - } else { - // ask without specifying a target - gets conservative answer - return internal->recommended_max_payload(with_congestion, header_size); - } - } - size_t GASNetEXModule::recommended_max_payload(NodeID target, const LocalAddress &src_payload_addr, size_t bytes_per_line, size_t lines, diff --git a/src/realm/gasnetex/gasnetex_module.h b/src/realm/gasnetex/gasnetex_module.h index 3867429355..4931d88c81 100644 --- a/src/realm/gasnetex/gasnetex_module.h +++ b/src/realm/gasnetex/gasnetex_module.h @@ -92,15 +92,8 @@ namespace Realm { NodeID target, unsigned short msgid, size_t header_size, size_t max_payload_size, const RemoteAddress &dest_payload_addr, void *storage_base, size_t storage_size); - virtual ActiveMessageImpl *create_active_message_impl( - const NodeSet &targets, unsigned short msgid, size_t header_size, - size_t max_payload_size, const void *src_payload_addr, size_t src_payload_lines, - size_t src_payload_line_stride, void *storage_base, size_t storage_size); - virtual size_t recommended_max_payload(NodeID target, bool with_congestion, size_t header_size); - virtual size_t recommended_max_payload(const NodeSet &targets, bool with_congestion, - size_t header_size); virtual size_t recommended_max_payload(NodeID target, const RemoteAddress &dest_payload_addr, bool with_congestion, size_t header_size); @@ -108,10 +101,6 @@ namespace Realm { size_t bytes_per_line, size_t lines, size_t line_stride, bool with_congestion, size_t header_size); - virtual size_t recommended_max_payload(const NodeSet &targets, const void *data, - size_t bytes_per_line, size_t lines, - size_t line_stride, bool with_congestion, - size_t header_size); virtual size_t recommended_max_payload(NodeID target, const LocalAddress &src_payload_addr, size_t bytes_per_line, size_t lines, diff --git a/src/realm/hip/hip_module.cc b/src/realm/hip/hip_module.cc index 69fdb6eabc..967177e5d3 100644 --- a/src/realm/hip/hip_module.cc +++ b/src/realm/hip/hip_module.cc @@ -29,6 +29,7 @@ #include "realm/transfer/channel.h" #include "realm/transfer/ib_memory.h" +#include "realm/activemsg.h" #include "realm/mutex.h" #include "realm/utils.h" @@ -2945,9 +2946,9 @@ namespace Realm { hipipc_responses_needed.fetch_add(ipc_peers.size()); hipipc_releases_needed.fetch_add(ipc_peers.size()); - ActiveMessage amsg(ipc_peers); - amsg->hostid = gethostid(); - amsg.commit(); + HipIpcRequest msg{}; + msg.hostid = gethostid(); + multicast_message(MulticastTargetSet(ipc_peers), msg); // wait for responses { @@ -2995,8 +2996,8 @@ namespace Realm { } if(!ipc_peers.empty()) { - ActiveMessage amsg(ipc_peers); - amsg.commit(); + HipIpcRelease msg{}; + multicast_message(MulticastTargetSet(ipc_peers), msg); } // now wait for similar notifications from any peers we gave mappings diff --git a/src/realm/inst_impl.cc b/src/realm/inst_impl.cc index d3469ff3c9..eefc50d810 100644 --- a/src/realm/inst_impl.cc +++ b/src/realm/inst_impl.cc @@ -20,6 +20,7 @@ #include "realm/event_impl.h" #include "realm/mem_impl.h" #include "realm/logging.h" +#include "realm/activemsg.h" #include "realm/runtime_impl.h" #include "realm/deppart/inst_helper.h" @@ -967,25 +968,20 @@ namespace Realm { Serialization::DynamicBufferSerializer dbs(4096); metadata.serialize_msg(dbs); - // fragment serialized metadata if needed - size_t offset = 0; + // The whole blob goes out as a single multicast: the source-side loop that used + // to chunk it against recommended_max_payload(NodeSet, ...) is gone, because an + // oversized multicast envelope is fragmented (and reassembled before the relay + // repartitions it) by the ordinary active-message machinery on every hop - see + // plan section 7.5. The receiver's offset/total_bytes reassembly is still needed + // for the unicast response path in metadata.cc, and handles a single full-size + // message as the "complete message" case. size_t total_bytes = dbs.bytes_used(); - while(offset < total_bytes) { - size_t to_send = - std::min(total_bytes - offset, - ActiveMessage::recommended_max_payload( - early_reqs, false /*without congestion*/)); - - ActiveMessage amsg(early_reqs, to_send); - amsg->id = ID(me).id; - amsg->offset = offset; - amsg->total_bytes = total_bytes; - amsg.add_payload(static_cast(dbs.get_buffer()) + offset, to_send); - amsg.commit(); - - offset += to_send; - } + MetadataResponseMessage msg{}; + msg.id = ID(me).id; + msg.offset = 0; + msg.total_bytes = total_bytes; + multicast_message(MulticastTargetSet(early_reqs), msg, dbs.get_buffer(), total_bytes); } void RegionInstanceImpl::notify_allocation(MemoryImpl::AllocationResult result, diff --git a/src/realm/metadata.cc b/src/realm/metadata.cc index d037fbd6b2..5bc857f3b9 100644 --- a/src/realm/metadata.cc +++ b/src/realm/metadata.cc @@ -21,6 +21,7 @@ #include "realm/event_impl.h" #include "realm/inst_impl.h" +#include "realm/activemsg.h" #include "realm/runtime_impl.h" namespace Realm { @@ -180,9 +181,9 @@ namespace Realm { if(invals_to_send.empty()) return true; - ActiveMessage amsg(invals_to_send); - amsg->id = id; - amsg.commit(); + MetadataInvalidateMessage msg{}; + msg.id = id; + multicast_message(MulticastTargetSet(invals_to_send), msg); // can't free object until we receive all the acks return false; } diff --git a/src/realm/mpi/mpi_module.cc b/src/realm/mpi/mpi_module.cc index f9e752fe9d..77835c0ec4 100644 --- a/src/realm/mpi/mpi_module.cc +++ b/src/realm/mpi/mpi_module.cc @@ -352,11 +352,6 @@ namespace Realm { size_t _max_payload_size, const void *_src_payload_addr, size_t _src_payload_lines, size_t _src_payload_line_stride, const RemoteAddress &_dest_payload_addr); - MPIMessageImpl(const Realm::NodeSet &_targets, unsigned short _msgid, - size_t _header_size, size_t _max_payload_size, - const void *_src_payload_addr, size_t _src_payload_lines, - size_t _src_payload_line_stride); - virtual ~MPIMessageImpl(); // reserves space for a local/remote completion - caller will @@ -370,8 +365,6 @@ namespace Realm { protected: /* header_base, payload_base, playload_size */ NodeID target; - Realm::NodeSet targets; - bool is_multicast; const void *src_payload_addr; size_t src_payload_lines; size_t src_payload_line_stride; @@ -389,7 +382,6 @@ namespace Realm { const void *_src_payload_addr, size_t _src_payload_lines, size_t _src_payload_line_stride) : target(_target) - , is_multicast(false) , src_payload_addr(_src_payload_addr) , src_payload_lines(_src_payload_lines) , src_payload_line_stride(_src_payload_line_stride) @@ -414,7 +406,6 @@ namespace Realm { size_t _src_payload_line_stride, const RemoteAddress &_dest_payload_addr) : target(_target) - , is_multicast(false) , src_payload_addr(_src_payload_addr) , src_payload_lines(_src_payload_lines) , src_payload_line_stride(_src_payload_line_stride) @@ -433,30 +424,6 @@ namespace Realm { header_base = &msg_header; } - MPIMessageImpl::MPIMessageImpl(const Realm::NodeSet &_targets, unsigned short _msgid, - size_t _header_size, size_t _max_payload_size, - const void *_src_payload_addr, size_t _src_payload_lines, - size_t _src_payload_line_stride) - : targets(_targets) - , is_multicast(true) - , src_payload_addr(_src_payload_addr) - , src_payload_lines(_src_payload_lines) - , src_payload_line_stride(_src_payload_line_stride) - , dest_payload_offset(-1) - , header_size(_header_size) - , local_comp(0) - , remote_comp(0) - , msgid(_msgid) - { - if(_max_payload_size && (src_payload_addr == 0)) { - payload_base = reinterpret_cast(malloc(_max_payload_size)); - } else { - payload_base = 0; - } - payload_size = _max_payload_size; - header_base = &msg_header; - } - MPIMessageImpl::~MPIMessageImpl() {} void *MPIMessageImpl::add_local_completion(size_t size) @@ -485,26 +452,13 @@ namespace Realm { void MPIMessageImpl::commit(size_t act_payload_size) { - if(is_multicast) { - assert(dest_payload_offset < 0); - assert(remote_comp == 0); - for(NodeSet::const_iterator it = targets.begin(); it != targets.end(); ++it) - if(src_payload_addr != 0) - enqueue_message(*it, msgid, &msg_header, header_size, src_payload_addr, - act_payload_size, src_payload_lines, src_payload_line_stride, - -1, 0); - else - enqueue_message(*it, msgid, &msg_header, header_size, payload_base, - act_payload_size, 0, 0, -1, 0); - } else { - if(src_payload_addr != 0) - enqueue_message(target, msgid, &msg_header, header_size, src_payload_addr, - act_payload_size, src_payload_lines, src_payload_line_stride, - dest_payload_offset, remote_comp); - else - enqueue_message(target, msgid, &msg_header, header_size, payload_base, - act_payload_size, 0, 0, dest_payload_offset, remote_comp); - } + if(src_payload_addr != 0) + enqueue_message(target, msgid, &msg_header, header_size, src_payload_addr, + act_payload_size, src_payload_lines, src_payload_line_stride, + dest_payload_offset, remote_comp); + else + enqueue_message(target, msgid, &msg_header, header_size, payload_base, + act_payload_size, 0, 0, dest_payload_offset, remote_comp); if(payload_size && (src_payload_addr == 0)) free(payload_base); // we're only doing blocking transfers right now, so we can always do @@ -935,30 +889,12 @@ namespace Realm { return impl; } - ActiveMessageImpl *MPIModule::create_active_message_impl( - const NodeSet &targets, unsigned short msgid, size_t header_size, - size_t max_payload_size, const void *src_payload_addr, size_t src_payload_lines, - size_t src_payload_line_stride, void *storage_base, size_t storage_size) - { - assert(storage_size >= sizeof(MPIMessageImpl)); - MPIMessageImpl *impl = new(storage_base) - MPIMessageImpl(targets, msgid, header_size, max_payload_size, src_payload_addr, - src_payload_lines, src_payload_line_stride); - return impl; - } - size_t MPIModule::recommended_max_payload(NodeID target, bool with_congestion, size_t header_size) { return (AM_BUF_SIZE - header_size); } - size_t MPIModule::recommended_max_payload(const NodeSet &targets, bool with_congestion, - size_t header_size) - { - return (AM_BUF_SIZE - header_size); - } - size_t MPIModule::recommended_max_payload(NodeID target, const RemoteAddress &dest_payload_addr, bool with_congestion, size_t header_size) @@ -978,15 +914,6 @@ namespace Realm { return recommended_max_payload(target, with_congestion, header_size); } - size_t MPIModule::recommended_max_payload(const NodeSet &targets, const void *data, - size_t bytes_per_line, size_t lines, - size_t line_stride, bool with_congestion, - size_t header_size) - { - // we don't care about source data location - return recommended_max_payload(targets, with_congestion, header_size); - } - size_t MPIModule::recommended_max_payload(NodeID target, const LocalAddress &src_payload_addr, size_t bytes_per_line, size_t lines, diff --git a/src/realm/mpi/mpi_module.h b/src/realm/mpi/mpi_module.h index 3d701eb4d9..89b02cb415 100644 --- a/src/realm/mpi/mpi_module.h +++ b/src/realm/mpi/mpi_module.h @@ -90,15 +90,8 @@ namespace Realm { NodeID target, unsigned short msgid, size_t header_size, size_t max_payload_size, const RemoteAddress &dest_payload_addr, void *storage_base, size_t storage_size); - virtual ActiveMessageImpl *create_active_message_impl( - const NodeSet &targets, unsigned short msgid, size_t header_size, - size_t max_payload_size, const void *src_payload_addr, size_t src_payload_lines, - size_t src_payload_line_stride, void *storage_base, size_t storage_size); - virtual size_t recommended_max_payload(NodeID target, bool with_congestion, size_t header_size); - virtual size_t recommended_max_payload(const NodeSet &targets, bool with_congestion, - size_t header_size); virtual size_t recommended_max_payload(NodeID target, const RemoteAddress &dest_payload_addr, bool with_congestion, size_t header_size); @@ -106,10 +99,6 @@ namespace Realm { size_t bytes_per_line, size_t lines, size_t line_stride, bool with_congestion, size_t header_size); - virtual size_t recommended_max_payload(const NodeSet &targets, const void *data, - size_t bytes_per_line, size_t lines, - size_t line_stride, bool with_congestion, - size_t header_size); virtual size_t recommended_max_payload(NodeID target, const LocalAddress &src_payload_addr, size_t bytes_per_line, size_t lines, diff --git a/src/realm/multicast.cc b/src/realm/multicast.cc new file mode 100644 index 0000000000..71b1b58a86 --- /dev/null +++ b/src/realm/multicast.cc @@ -0,0 +1,992 @@ +/* + * Copyright 2025 Stanford University, NVIDIA Corporation + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Adaptive multicast target sets and encodings for Realm active messages, plus the +// multicast envelope and the bounded-radix forwarding algorithm built on top of them + +#include "realm/multicast.h" + +#include "realm/logging.h" + +#include +#include +#include +#include +#include + +namespace Realm { + + //////////////////////////////////////////////////////////////////////// + // + // multicast target encodings + // + + const char *multicast_target_encoding_name(MulticastTargetEncoding kind) + { + switch(kind) { + case MulticastTargetEncoding::EMPTY: + return "EMPTY"; + case MulticastTargetEncoding::SINGLE: + return "SINGLE"; + case MulticastTargetEncoding::SMALL_INLINE: + return "SMALL_INLINE"; + case MulticastTargetEncoding::RANGES: + return "RANGES"; + case MulticastTargetEncoding::DELTA_LIST: + return "DELTA_LIST"; + case MulticastTargetEncoding::BITMAP: + return "BITMAP"; + case MulticastTargetEncoding::ALL_NODES: + return "ALL_NODES"; + case MulticastTargetEncoding::ALL_EXCEPT: + return "ALL_EXCEPT"; + } + return "INVALID"; + } + + std::ostream &operator<<(std::ostream &os, MulticastTargetEncoding kind) + { + os << multicast_target_encoding_name(kind); + return os; + } + + const char *multicast_decode_status_name(MulticastDecodeStatus status) + { + switch(status) { + case MulticastDecodeStatus::OK: + return "OK"; + case MulticastDecodeStatus::UNKNOWN_KIND: + return "UNKNOWN_KIND"; + case MulticastDecodeStatus::TRUNCATED: + return "TRUNCATED"; + case MulticastDecodeStatus::TRAILING_BYTES: + return "TRAILING_BYTES"; + case MulticastDecodeStatus::BAD_CARDINALITY: + return "BAD_CARDINALITY"; + case MulticastDecodeStatus::NODE_OUT_OF_RANGE: + return "NODE_OUT_OF_RANGE"; + case MulticastDecodeStatus::RANGE_OVERFLOW: + return "RANGE_OVERFLOW"; + case MulticastDecodeStatus::NOT_CANONICAL: + return "NOT_CANONICAL"; + } + return "INVALID"; + } + + std::ostream &operator<<(std::ostream &os, MulticastDecodeStatus status) + { + os << multicast_decode_status_name(status); + return os; + } + + //////////////////////////////////////////////////////////////////////// + // + // wire primitives + // + + namespace MulticastWire { + + size_t varint_size(uint64_t value) + { + size_t bytes = 1; + while(value >= 0x80) { + value >>= 7; + bytes++; + } + return bytes; + } + + void append_varint(std::vector &buf, uint64_t value) + { + while(value >= 0x80) { + buf.push_back(static_cast((value & 0x7f) | 0x80)); + value >>= 7; + } + buf.push_back(static_cast(value)); + } + + MulticastDecodeStatus read_varint(const unsigned char *base, size_t bytes, + size_t &pos, uint64_t &value) + { + uint64_t result = 0; + unsigned shift = 0; + size_t consumed = 0; + while(true) { + if(pos + consumed >= bytes) + return MulticastDecodeStatus::TRUNCATED; + unsigned char b = base[pos + consumed]; + consumed++; + // an encoding longer than MAX_VARINT_BYTES, or one whose top byte would shift + // bits out of a 64-bit value, is not something our encoder can produce + if((consumed > MAX_VARINT_BYTES) || ((shift == 63) && ((b & 0x7f) > 1))) + return MulticastDecodeStatus::NOT_CANONICAL; + result |= (static_cast(b & 0x7f) << shift); + if((b & 0x80) == 0) { + // reject overlong encodings so that the byte string is canonical + if((consumed > 1) && (b == 0)) + return MulticastDecodeStatus::NOT_CANONICAL; + break; + } + shift += 7; + } + pos += consumed; + value = result; + return MulticastDecodeStatus::OK; + } + + }; // namespace MulticastWire + + //////////////////////////////////////////////////////////////////////// + // + // class MulticastTargetSet::const_iterator + // + + MulticastTargetSet::const_iterator::const_iterator(const MulticastTargetSet &_set, + size_t _run_idx) + : set(&_set) + , run_idx(_run_idx) + , cur_node(-1) + { + if(run_idx < set->runs.size()) + cur_node = set->runs[run_idx].first; + } + + bool MulticastTargetSet::const_iterator::operator==(const const_iterator &rhs) const + { + return (set == rhs.set) && (run_idx == rhs.run_idx) && (cur_node == rhs.cur_node); + } + + MulticastTargetSet::const_iterator &MulticastTargetSet::const_iterator::operator++(void) + { + assert((set != nullptr) && (run_idx < set->runs.size())); + if(cur_node < set->runs[run_idx].last) { + cur_node++; + } else { + run_idx++; + cur_node = (run_idx < set->runs.size()) ? set->runs[run_idx].first : -1; + } + return *this; + } + + MulticastTargetSet::const_iterator + MulticastTargetSet::const_iterator::operator++(int /*postfix*/) + { + const_iterator old = *this; + ++(*this); + return old; + } + + //////////////////////////////////////////////////////////////////////// + // + // class MulticastTargetSet + // + + MulticastTargetSet::MulticastTargetSet(const NodeSet &nodes) { add_nodeset(nodes); } + + void MulticastTargetSet::clear(void) + { + runs.clear(); + total = 0; + } + + void MulticastTargetSet::add(NodeID id) { add_range(id, id); } + + void MulticastTargetSet::add_range(NodeID first, NodeID last) + { + if(last < first) + return; // empty range - nothing to do + assert(first >= 0); + + // all of the +/-1 comparisons below are done in 64 bits so that a range touching + // the top of the NodeID space cannot overflow + const long long lo = first; + const long long hi = last; + + // first run that could possibly merge with or follow [first, last] + size_t left = 0, right = runs.size(); + while(left < right) { + size_t mid = left + ((right - left) >> 1); + if((static_cast(runs[mid].last) + 1) < lo) + left = mid + 1; + else + right = mid; + } + + // absorb every run that touches or is adjacent to the new one + size_t end = left; + NodeID merged_first = first; + NodeID merged_last = last; + size_t absorbed = 0; + while((end < runs.size()) && (static_cast(runs[end].first) <= (hi + 1))) { + merged_first = std::min(merged_first, runs[end].first); + merged_last = std::max(merged_last, runs[end].last); + absorbed += runs[end].count(); + end++; + } + + Range merged; + merged.first = merged_first; + merged.last = merged_last; + const size_t merged_count = merged.count(); + + runs.erase(runs.begin() + left, runs.begin() + end); + runs.insert(runs.begin() + left, merged); + + total += merged_count - absorbed; + } + + void MulticastTargetSet::add_nodeset(const NodeSet &nodes) + { + // NOTE: NodeSet iteration is not guaranteed to be sorted, which is fine - add() + // handles arbitrary insertion order + for(NodeSet::const_iterator it = nodes.begin(); it != nodes.end(); ++it) + add(*it); + } + + void MulticastTargetSet::add_targets(const MulticastTargetSet &other) + { + for(size_t i = 0; i < other.runs.size(); i++) + add_range(other.runs[i].first, other.runs[i].last); + } + + bool MulticastTargetSet::remove(NodeID id) + { + size_t idx = find_run(id); + if(idx >= runs.size()) + return false; + + Range &r = runs[idx]; + if((r.first == id) && (r.last == id)) { + runs.erase(runs.begin() + idx); + } else if(r.first == id) { + r.first = id + 1; + } else if(r.last == id) { + r.last = id - 1; + } else { + // split into two runs, neither of which can be empty + Range tail; + tail.first = id + 1; + tail.last = r.last; + r.last = id - 1; + runs.insert(runs.begin() + idx + 1, tail); + } + total--; + return true; + } + + bool MulticastTargetSet::append_increasing_node(NodeID id) + { + if(id < 0) + return false; + if(runs.empty()) { + Range r; + r.first = id; + r.last = id; + runs.push_back(r); + total++; + return true; + } + if(id <= runs.back().last) + return false; + if(id == (runs.back().last + 1)) + runs.back().last = id; + else { + Range r; + r.first = id; + r.last = id; + runs.push_back(r); + } + total++; + return true; + } + + bool MulticastTargetSet::append_canonical_run(NodeID first, NodeID last) + { + if((first < 0) || (last < first)) + return false; + // must be strictly after the trailing run AND not adjacent to it, otherwise the + // incoming encoding was not canonical + if(!runs.empty() && + (static_cast(first) <= (static_cast(runs.back().last) + 1))) + return false; + Range r; + r.first = first; + r.last = last; + runs.push_back(r); + total += r.count(); + return true; + } + + size_t MulticastTargetSet::find_run(NodeID id) const + { + size_t left = 0, right = runs.size(); + while(left < right) { + size_t mid = left + ((right - left) >> 1); + if(runs[mid].last < id) + left = mid + 1; + else + right = mid; + } + if((left < runs.size()) && (runs[left].first <= id) && (id <= runs[left].last)) + return left; + return runs.size(); + } + + bool MulticastTargetSet::contains(NodeID id) const + { + return (find_run(id) < runs.size()); + } + + bool MulticastTargetSet::fits_node_count(NodeID num_nodes) const + { + if(runs.empty()) + return true; + return ((runs.front().first >= 0) && (runs.back().last < num_nodes)); + } + + void MulticastTargetSet::to_nodeset(NodeSet &nodes) const + { + for(size_t i = 0; i < runs.size(); i++) + nodes.add_range(runs[i].first, runs[i].last); + } + + void MulticastTargetSet::partition(size_t max_slices, + std::vector &slices) const + { + slices.clear(); + if((total == 0) || (max_slices == 0)) + return; + + const size_t num_slices = std::min(max_slices, total); + const size_t base = total / num_slices; + const size_t extra = total % num_slices; + + slices.resize(num_slices); + + size_t run_idx = 0; + NodeID cursor = runs[0].first; + for(size_t s = 0; s < num_slices; s++) { + size_t wanted = base + ((s < extra) ? 1 : 0); + MulticastTargetSet &slice = slices[s]; + while(wanted > 0) { + assert(run_idx < runs.size()); + const Range &r = runs[run_idx]; + const size_t avail = + static_cast(static_cast(r.last) - cursor + 1); + const size_t taken = std::min(avail, wanted); + // NOTE: we cut runs here, we never expand them - a run covering thousands of + // nodes costs one entry per slice it lands in + const NodeID slice_last = + static_cast(static_cast(cursor) + taken - 1); + bool appended = slice.append_canonical_run(cursor, slice_last); + assert(appended); + (void)appended; + wanted -= taken; + if(taken == avail) { + run_idx++; + if(run_idx < runs.size()) + cursor = runs[run_idx].first; + } else { + cursor = static_cast(static_cast(cursor) + taken); + } + } + } + } + + bool MulticastTargetSet::operator==(const MulticastTargetSet &rhs) const + { + return (total == rhs.total) && (runs == rhs.runs); + } + + std::ostream &operator<<(std::ostream &os, const MulticastTargetSet &targets) + { + os << "{"; + const std::vector &runs = targets.ranges(); + for(size_t i = 0; i < runs.size(); i++) { + if(i > 0) + os << ","; + if(runs[i].first == runs[i].last) + os << runs[i].first; + else + os << runs[i].first << "-" << runs[i].last; + } + os << "}"; + return os; + } + + //////////////////////////////////////////////////////////////////////// + // + // struct MulticastEncodingTally + // + + MulticastEncodingTally::MulticastEncodingTally(void) + { + for(size_t i = 0; i < MULTICAST_ENCODING_KINDS; i++) + counts[i].store(0); + } + + void MulticastEncodingTally::record(MulticastTargetEncoding kind) + { + size_t idx = static_cast(kind); + assert(idx < MULTICAST_ENCODING_KINDS); + counts[idx].fetch_add(1); + } + + uint64_t MulticastEncodingTally::get(MulticastTargetEncoding kind) const + { + size_t idx = static_cast(kind); + assert(idx < MULTICAST_ENCODING_KINDS); + return counts[idx].load(); + } + + uint64_t MulticastEncodingTally::total(void) const + { + uint64_t sum = 0; + for(size_t i = 0; i < MULTICAST_ENCODING_KINDS; i++) + sum += counts[i].load(); + return sum; + } + + //////////////////////////////////////////////////////////////////////// + // + // class EncodedMulticastTargets + // + + namespace { + + typedef MulticastTargetSet::Range Range; + + // The complement of 'runs' within [0, num_nodes), itself canonical. This is + // O(runs.size()) - the complement of a small set of a huge node space is still + // only a couple of runs, which is what keeps the ALL_EXCEPT candidate cheap to + // evaluate. + void complement_runs(const std::vector &runs, NodeID num_nodes, + std::vector &out) + { + out.clear(); + NodeID next = 0; + for(size_t i = 0; i < runs.size(); i++) { + if(runs[i].first > next) { + Range r; + r.first = next; + r.last = runs[i].first - 1; + out.push_back(r); + } + next = runs[i].last + 1; + } + if(next < num_nodes) { + Range r; + r.first = next; + r.last = num_nodes - 1; + out.push_back(r); + } + } + + // Bytes needed for "first node, then one positive varint delta per remaining node", + // computed from the runs rather than by walking individual nodes: inside a run + // every delta is 1 and therefore exactly one byte. + uint64_t delta_list_bytes(const std::vector &runs) + { + uint64_t bytes = 0; + long long prev = -1; + for(size_t i = 0; i < runs.size(); i++) { + if(prev < 0) + bytes += MulticastWire::varint_size(static_cast(runs[i].first)); + else + bytes += MulticastWire::varint_size( + static_cast(static_cast(runs[i].first) - prev)); + bytes += runs[i].count() - 1; // interior deltas are all 1, i.e. one byte each + prev = runs[i].last; + } + return bytes; + } + + void emit_delta_list(std::vector &buf, const std::vector &runs) + { + long long prev = -1; + for(size_t i = 0; i < runs.size(); i++) { + if(prev < 0) + MulticastWire::append_varint(buf, static_cast(runs[i].first)); + else + MulticastWire::append_varint( + buf, static_cast(static_cast(runs[i].first) - prev)); + for(long long n = runs[i].first + 1; n <= runs[i].last; n++) + MulticastWire::append_varint(buf, 1); + prev = runs[i].last; + } + } + + // reads "count nodes as first-plus-positive-deltas" into 'out' + MulticastDecodeStatus read_delta_list(const unsigned char *base, size_t bytes, + size_t &pos, uint64_t count, + uint64_t node_limit, MulticastTargetSet &out) + { + uint64_t cur = 0; + MulticastDecodeStatus status = MulticastWire::read_varint(base, bytes, pos, cur); + if(status != MulticastDecodeStatus::OK) + return status; + if(cur >= node_limit) + return MulticastDecodeStatus::NODE_OUT_OF_RANGE; + if(!out.append_increasing_node(static_cast(cur))) + return MulticastDecodeStatus::NOT_CANONICAL; + + for(uint64_t i = 1; i < count; i++) { + uint64_t delta = 0; + status = MulticastWire::read_varint(base, bytes, pos, delta); + if(status != MulticastDecodeStatus::OK) + return status; + // a zero delta would repeat the previous node, which is not canonical + if(delta == 0) + return MulticastDecodeStatus::NOT_CANONICAL; + if(delta > (node_limit - 1 - cur)) + return MulticastDecodeStatus::NODE_OUT_OF_RANGE; + cur += delta; + if(!out.append_increasing_node(static_cast(cur))) + return MulticastDecodeStatus::NOT_CANONICAL; + } + return MulticastDecodeStatus::OK; + } + + }; // namespace + + EncodedMulticastTargets::EncodedMulticastTargets(void) + : chosen(MulticastTargetEncoding::EMPTY) + , count(0) + { + wire.push_back(static_cast(MulticastTargetEncoding::EMPTY)); + } + + size_t EncodedMulticastTargets::encoded_size(const MulticastTargetSet &targets, + NodeID num_nodes, + MulticastTargetEncoding kind) + { + const std::vector &runs = targets.ranges(); + const size_t n = targets.size(); + + switch(kind) { + case MulticastTargetEncoding::EMPTY: + { + return (n == 0) ? 1 : 0; + } + + case MulticastTargetEncoding::SINGLE: + { + if(n != 1) + return 0; + return 1 + MulticastWire::varint_size(static_cast(targets.first_node())); + } + + case MulticastTargetEncoding::SMALL_INLINE: + { + if((n < 1) || (n > MULTICAST_MAX_SMALL_INLINE)) + return 0; + size_t bytes = 1 + MulticastWire::varint_size(n); + // n is at most MULTICAST_MAX_SMALL_INLINE, so walking nodes here is bounded + for(MulticastTargetSet::const_iterator it = targets.begin(); it != targets.end(); + ++it) + bytes += MulticastWire::varint_size(static_cast(*it)); + return bytes; + } + + case MulticastTargetEncoding::RANGES: + { + if(n < 1) + return 0; + size_t bytes = 1 + MulticastWire::varint_size(runs.size()); + for(size_t i = 0; i < runs.size(); i++) + bytes += MulticastWire::varint_size(static_cast(runs[i].first)) + + MulticastWire::varint_size(runs[i].count()); + return bytes; + } + + case MulticastTargetEncoding::DELTA_LIST: + { + if(n < 1) + return 0; + return 1 + MulticastWire::varint_size(n) + + static_cast(delta_list_bytes(runs)); + } + + case MulticastTargetEncoding::BITMAP: + { + if(n < 1) + return 0; + const uint64_t base = static_cast(targets.first_node()); + const uint64_t bit_length = static_cast(targets.last_node()) - base + 1; + return 1 + MulticastWire::varint_size(base) + + MulticastWire::varint_size(bit_length) + + static_cast((bit_length + 7) / 8); + } + + case MulticastTargetEncoding::ALL_NODES: + { + if((num_nodes < 1) || (n != static_cast(num_nodes)) || + !targets.fits_node_count(num_nodes)) + return 0; + return 1; + } + + case MulticastTargetEncoding::ALL_EXCEPT: + { + // must exclude at least one node (otherwise ALL_NODES) and must leave at least + // one node in the set (otherwise EMPTY) + if((num_nodes < 1) || (n < 1) || (n >= static_cast(num_nodes))) + return 0; + if(!targets.fits_node_count(num_nodes)) + return 0; + const size_t excluded = static_cast(num_nodes) - n; + std::vector comp; + complement_runs(runs, num_nodes, comp); + return 1 + MulticastWire::varint_size(excluded) + + static_cast(delta_list_bytes(comp)); + } + } + return 0; + } + + void EncodedMulticastTargets::emit(std::vector &buf, + const MulticastTargetSet &targets, NodeID num_nodes, + MulticastTargetEncoding kind) + { + const std::vector &runs = targets.ranges(); + const size_t n = targets.size(); + + buf.push_back(static_cast(kind)); + + switch(kind) { + case MulticastTargetEncoding::EMPTY: + case MulticastTargetEncoding::ALL_NODES: + { + break; + } + + case MulticastTargetEncoding::SINGLE: + { + MulticastWire::append_varint(buf, static_cast(targets.first_node())); + break; + } + + case MulticastTargetEncoding::SMALL_INLINE: + { + MulticastWire::append_varint(buf, n); + for(MulticastTargetSet::const_iterator it = targets.begin(); it != targets.end(); + ++it) + MulticastWire::append_varint(buf, static_cast(*it)); + break; + } + + case MulticastTargetEncoding::RANGES: + { + MulticastWire::append_varint(buf, runs.size()); + for(size_t i = 0; i < runs.size(); i++) { + MulticastWire::append_varint(buf, static_cast(runs[i].first)); + MulticastWire::append_varint(buf, runs[i].count()); + } + break; + } + + case MulticastTargetEncoding::DELTA_LIST: + { + MulticastWire::append_varint(buf, n); + emit_delta_list(buf, runs); + break; + } + + case MulticastTargetEncoding::BITMAP: + { + const uint64_t base = static_cast(targets.first_node()); + const uint64_t bit_length = static_cast(targets.last_node()) - base + 1; + MulticastWire::append_varint(buf, base); + MulticastWire::append_varint(buf, bit_length); + const size_t map_bytes = static_cast((bit_length + 7) / 8); + const size_t map_start = buf.size(); + buf.resize(map_start + map_bytes, 0); + for(size_t i = 0; i < runs.size(); i++) + for(long long node = runs[i].first; node <= runs[i].last; node++) { + const uint64_t bit = static_cast(node) - base; + unsigned char &byte = buf[map_start + static_cast(bit >> 3)]; + byte = static_cast(byte | (1u << (bit & 7))); + } + break; + } + + case MulticastTargetEncoding::ALL_EXCEPT: + { + const size_t excluded = static_cast(num_nodes) - n; + std::vector comp; + complement_runs(runs, num_nodes, comp); + MulticastWire::append_varint(buf, excluded); + emit_delta_list(buf, comp); + break; + } + } + } + + /*static*/ EncodedMulticastTargets + EncodedMulticastTargets::encode(const MulticastTargetSet &targets, NodeID num_nodes, + MulticastEncodingTally *tally) + { + assert(targets.fits_node_count(num_nodes)); + + // evaluate the ACTUAL serialized length of every candidate and keep the smallest - + // a fixed density heuristic is explicitly not allowed here (plan section 7.2) + size_t best_size = 0; + MulticastTargetEncoding best = MulticastTargetEncoding::EMPTY; + for(size_t i = 0; i < MULTICAST_ENCODING_KINDS; i++) { + MulticastTargetEncoding kind = static_cast(i); + size_t sz = encoded_size(targets, num_nodes, kind); + if(sz == 0) + continue; // this kind cannot represent this set + // strict '<' leaves the earliest (cheapest to decode) kind winning a tie + if((best_size == 0) || (sz < best_size)) { + best_size = sz; + best = kind; + } + } + // EMPTY is always available for an empty set and RANGES for a nonempty one + assert(best_size > 0); + + EncodedMulticastTargets result; + result.wire.clear(); + result.wire.reserve(best_size); + emit(result.wire, targets, num_nodes, best); + assert(result.wire.size() == best_size); + result.chosen = best; + result.count = targets.size(); + + if(tally != nullptr) + tally->record(best); + + return result; + } + + /*static*/ MulticastDecodeStatus + EncodedMulticastTargets::decode(const void *data, size_t bytes, NodeID num_nodes, + MulticastTargetSet &targets) + { + targets.clear(); + + if(bytes < 1) + return MulticastDecodeStatus::TRUNCATED; + const unsigned char *base = static_cast(data); + const unsigned char kind_byte = base[0]; + if(kind_byte >= MULTICAST_ENCODING_KINDS) + return MulticastDecodeStatus::UNKNOWN_KIND; + const MulticastTargetEncoding kind = static_cast(kind_byte); + + // a negative or zero node count can only ever describe an empty machine + const uint64_t node_limit = + (num_nodes > 0) ? static_cast(num_nodes) : uint64_t(0); + + size_t pos = 1; + MulticastDecodeStatus status = MulticastDecodeStatus::OK; + + switch(kind) { + case MulticastTargetEncoding::EMPTY: + { + break; + } + + case MulticastTargetEncoding::SINGLE: + { + uint64_t node = 0; + status = MulticastWire::read_varint(base, bytes, pos, node); + if(status != MulticastDecodeStatus::OK) + break; + if(node >= node_limit) { + status = MulticastDecodeStatus::NODE_OUT_OF_RANGE; + break; + } + targets.append_increasing_node(static_cast(node)); + break; + } + + case MulticastTargetEncoding::SMALL_INLINE: + { + uint64_t n = 0; + status = MulticastWire::read_varint(base, bytes, pos, n); + if(status != MulticastDecodeStatus::OK) + break; + // NOTE: the transmitted cardinality is checked against both the configured node + // count and the number of bytes actually left before it is used for anything + // (plan section 22) - every element needs at least one byte + if((n < 1) || (n > MULTICAST_MAX_SMALL_INLINE) || (n > node_limit) || + (n > (bytes - pos))) { + status = MulticastDecodeStatus::BAD_CARDINALITY; + break; + } + for(uint64_t i = 0; i < n; i++) { + uint64_t node = 0; + status = MulticastWire::read_varint(base, bytes, pos, node); + if(status != MulticastDecodeStatus::OK) + break; + if(node >= node_limit) { + status = MulticastDecodeStatus::NODE_OUT_OF_RANGE; + break; + } + if(!targets.append_increasing_node(static_cast(node))) { + status = MulticastDecodeStatus::NOT_CANONICAL; + break; + } + } + break; + } + + case MulticastTargetEncoding::RANGES: + { + uint64_t num_runs = 0; + status = MulticastWire::read_varint(base, bytes, pos, num_runs); + if(status != MulticastDecodeStatus::OK) + break; + // each run needs at least two bytes, and canonical runs are separated by a gap, + // so at most ceil(num_nodes/2) of them can fit in the node space + if((num_runs < 1) || (num_runs > ((bytes - pos) / 2)) || + (num_runs > ((node_limit + 1) / 2))) { + status = MulticastDecodeStatus::BAD_CARDINALITY; + break; + } + for(uint64_t i = 0; i < num_runs; i++) { + uint64_t start = 0, length = 0; + status = MulticastWire::read_varint(base, bytes, pos, start); + if(status != MulticastDecodeStatus::OK) + break; + status = MulticastWire::read_varint(base, bytes, pos, length); + if(status != MulticastDecodeStatus::OK) + break; + if(start >= node_limit) { + status = MulticastDecodeStatus::NODE_OUT_OF_RANGE; + break; + } + // checked this way round so that a huge length cannot wrap + if((length < 1) || (length > (node_limit - start))) { + status = MulticastDecodeStatus::RANGE_OVERFLOW; + break; + } + const NodeID first = static_cast(start); + const NodeID last = static_cast(start + length - 1); + // rejects unsorted, duplicated, overlapping and adjacent-but-unmerged runs + if(!targets.append_canonical_run(first, last)) { + status = MulticastDecodeStatus::NOT_CANONICAL; + break; + } + } + break; + } + + case MulticastTargetEncoding::DELTA_LIST: + { + uint64_t n = 0; + status = MulticastWire::read_varint(base, bytes, pos, n); + if(status != MulticastDecodeStatus::OK) + break; + if((n < 1) || (n > node_limit) || (n > (bytes - pos))) { + status = MulticastDecodeStatus::BAD_CARDINALITY; + break; + } + status = read_delta_list(base, bytes, pos, n, node_limit, targets); + break; + } + + case MulticastTargetEncoding::BITMAP: + { + uint64_t bitmap_base = 0, bit_length = 0; + status = MulticastWire::read_varint(base, bytes, pos, bitmap_base); + if(status != MulticastDecodeStatus::OK) + break; + status = MulticastWire::read_varint(base, bytes, pos, bit_length); + if(status != MulticastDecodeStatus::OK) + break; + if(bitmap_base >= node_limit) { + status = MulticastDecodeStatus::NODE_OUT_OF_RANGE; + break; + } + if((bit_length < 1) || (bit_length > (node_limit - bitmap_base))) { + status = MulticastDecodeStatus::RANGE_OVERFLOW; + break; + } + const uint64_t map_bytes = (bit_length + 7) / 8; + if(map_bytes != (bytes - pos)) { + status = (map_bytes > (bytes - pos)) ? MulticastDecodeStatus::TRUNCATED + : MulticastDecodeStatus::TRAILING_BYTES; + break; + } + // canonical form pins the first and last bits and zeroes the padding + const unsigned char last_byte = base[pos + map_bytes - 1]; + const unsigned pad_bits = static_cast((map_bytes * 8) - bit_length); + if(((base[pos] & 1) == 0) || ((last_byte & (1u << ((bit_length - 1) & 7))) == 0) || + ((pad_bits > 0) && ((last_byte >> (8 - pad_bits)) != 0))) { + status = MulticastDecodeStatus::NOT_CANONICAL; + break; + } + for(uint64_t bit = 0; bit < bit_length; bit++) + if((base[pos + (bit >> 3)] & (1u << (bit & 7))) != 0) + targets.append_increasing_node(static_cast(bitmap_base + bit)); + pos += map_bytes; + break; + } + + case MulticastTargetEncoding::ALL_NODES: + { + if(node_limit < 1) { + status = MulticastDecodeStatus::BAD_CARDINALITY; + break; + } + targets.append_canonical_run(0, num_nodes - 1); + break; + } + + case MulticastTargetEncoding::ALL_EXCEPT: + { + uint64_t excluded = 0; + status = MulticastWire::read_varint(base, bytes, pos, excluded); + if(status != MulticastDecodeStatus::OK) + break; + // excluding every node would be an empty set, which is EMPTY's job + if((excluded < 1) || (excluded >= node_limit) || (excluded > (bytes - pos))) { + status = MulticastDecodeStatus::BAD_CARDINALITY; + break; + } + MulticastTargetSet exclusions; + status = read_delta_list(base, bytes, pos, excluded, node_limit, exclusions); + if(status != MulticastDecodeStatus::OK) + break; + // complement of the exclusion set within [0, num_nodes) + NodeID next = 0; + const std::vector &excl_runs = exclusions.ranges(); + for(size_t i = 0; i < excl_runs.size(); i++) { + if(excl_runs[i].first > next) + targets.append_canonical_run(next, excl_runs[i].first - 1); + next = excl_runs[i].last + 1; + } + if(next < num_nodes) + targets.append_canonical_run(next, num_nodes - 1); + break; + } + } + + if(status != MulticastDecodeStatus::OK) { + targets.clear(); + return status; + } + if(pos != bytes) { + targets.clear(); + return MulticastDecodeStatus::TRAILING_BYTES; + } + return MulticastDecodeStatus::OK; + } + +}; // namespace Realm diff --git a/src/realm/multicast.h b/src/realm/multicast.h new file mode 100644 index 0000000000..f199ab77f0 --- /dev/null +++ b/src/realm/multicast.h @@ -0,0 +1,358 @@ +/* + * Copyright 2025 Stanford University, NVIDIA Corporation + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Adaptive multicast target sets and encodings for Realm active messages - see +// sections 7.1, 7.2 and 7.3 of SCALABLE_BARRIERS_IMPLEMENTATION_PLAN.md. +// +// The logical target set and its wire encoding are deliberately separate types: +// +// MulticastTargetSet canonical logical set / builder / view +// EncodedMulticastTargets immutable wire representation +// +// This layer sits strictly above backend unicast (plan section 7.1) and deliberately +// has no dependency on activemsg.h, so it can be unit tested standalone. + +#ifndef REALM_MULTICAST_H +#define REALM_MULTICAST_H + +#include "realm/atomics.h" +#include "realm/nodeset.h" + +#include +#include +#include +#include +#include + +namespace Realm { + + //////////////////////////////////////////////////////////////////////// + // + // multicast target encodings (plan section 7.2) + // + + // The eight representations the encoder chooses between. The numeric values are the + // on-the-wire kind byte and must not be renumbered. The encoder computes the actual + // serialized size of every candidate and takes the smallest; ties are broken in + // favor of the lower value, which is also the cheaper one to decode. + enum class MulticastTargetEncoding : unsigned char + { + EMPTY = 0, // no targets at all + SINGLE = 1, // exactly one target + SMALL_INLINE = 2, // short sorted list of absolute node IDs + RANGES = 3, // sorted nonoverlapping (start, length) runs + DELTA_LIST = 4, // first node followed by positive varint deltas + BITMAP = 5, // explicit base node plus an explicit bit length + ALL_NODES = 6, // every node in the configured node count + ALL_EXCEPT = 7, // every node except a (small) explicit exclusion list + }; + + // number of distinct encodings - candidates are enumerated over [0, KINDS) + static const size_t MULTICAST_ENCODING_KINDS = 8; + + // longest sorted absolute list that SMALL_INLINE will represent ("small" in 7.2); + // larger sparse sets fall through to DELTA_LIST + static const size_t MULTICAST_MAX_SMALL_INLINE = 8; + + const char *multicast_target_encoding_name(MulticastTargetEncoding kind); + + std::ostream &operator<<(std::ostream &os, MulticastTargetEncoding kind); + + // Why a decode was rejected. Plan section 21.1 requires "malformed multicast target + // encoding" to be a fatal diagnostic, but the decoder itself never aborts: it + // returns a status so that the caller can build the full fatal context (and so that + // the decoder stays unit testable). + enum class MulticastDecodeStatus : unsigned char + { + OK = 0, + UNKNOWN_KIND, // kind byte is not one of the eight + TRUNCATED, // payload ended in the middle of a field + TRAILING_BYTES, // complete encoding did not consume the whole payload + BAD_CARDINALITY, // count is zero where >= 1 is required, or exceeds what the + // payload length / configured node count could possibly hold + NODE_OUT_OF_RANGE, // a node ID is negative or >= the configured node count + RANGE_OVERFLOW, // a run's length is zero, or start+length leaves the node space + NOT_CANONICAL, // unsorted, duplicated, overlapping, adjacent-but-unmerged, an + // overlong varint, or a bitmap with slack at either end + }; + + const char *multicast_decode_status_name(MulticastDecodeStatus status); + + std::ostream &operator<<(std::ostream &os, MulticastDecodeStatus status); + + //////////////////////////////////////////////////////////////////////// + // + // wire primitives + // + + // Exposed so that tests can build byte-exact malformed payloads, and so that the + // envelope code in the next stage can size fields without duplicating this. + namespace MulticastWire { + + // LEB128, canonical (no overlong encodings), at most 10 bytes + static const size_t MAX_VARINT_BYTES = 10; + + size_t varint_size(uint64_t value); + + void append_varint(std::vector &buf, uint64_t value); + + // advances 'pos' on success; returns TRUNCATED if the buffer ends first and + // NOT_CANONICAL for an overlong or overflowing encoding + MulticastDecodeStatus read_varint(const unsigned char *base, size_t bytes, + size_t &pos, uint64_t &value); + + }; // namespace MulticastWire + + //////////////////////////////////////////////////////////////////////// + // + // class MulticastTargetSet + // + + // The canonical logical target set (plan section 7.2). Stored as sorted, disjoint, + // nonadjacent runs so that a range covering thousands of nodes costs one entry: this + // is what lets partition() and the size estimator run without ever expanding a run + // into individual node IDs. + class MulticastTargetSet { + public: + struct Range { + NodeID first = 0; + NodeID last = 0; // inclusive + + size_t count(void) const + { + return static_cast(static_cast(last) - first + 1); + } + + bool operator==(const Range &rhs) const + { + return (first == rhs.first) && (last == rhs.last); + } + bool operator!=(const Range &rhs) const { return !(*this == rhs); } + }; + + MulticastTargetSet(void) = default; + explicit MulticastTargetSet(const NodeSet &nodes); + + // --- building ------------------------------------------------------ + + void clear(void); + + void add(NodeID id); + void add_range(NodeID first, NodeID last /*inclusive*/); + void add_nodeset(const NodeSet &nodes); + void add_targets(const MulticastTargetSet &other); + + // removal of the local relay before forwarding (plan section 7.3) - returns true if + // the node was actually present + bool remove(NodeID id); + + // Appends a single node that must be strictly greater than every node already + // present; merges into the trailing run when adjacent so the result stays + // canonical. Returns false (leaving the set unchanged) otherwise - this is the + // ordering check the decoder needs, done in O(1). + bool append_increasing_node(NodeID id); + + // Appends a run that must be strictly greater than, and not adjacent to, everything + // already present. Adjacency is rejected rather than merged so that a decoder can + // use this to prove the incoming encoding was canonical. + bool append_canonical_run(NodeID first, NodeID last /*inclusive*/); + + // --- inspection ---------------------------------------------------- + + bool empty(void) const { return runs.empty(); } + size_t size(void) const { return total; } + + bool contains(NodeID id) const; + + // both require !empty() + NodeID first_node(void) const { return runs.front().first; } + NodeID last_node(void) const { return runs.back().last; } + + size_t num_ranges(void) const { return runs.size(); } + const std::vector &ranges(void) const { return runs; } + + // every target is in [0, num_nodes) + bool fits_node_count(NodeID num_nodes) const; + + void to_nodeset(NodeSet &nodes) const; + + // --- sorted iteration ---------------------------------------------- + + class const_iterator { + public: + typedef std::input_iterator_tag iterator_category; + typedef NodeID value_type; + typedef std::ptrdiff_t difference_type; + typedef const NodeID *pointer; + typedef const NodeID &reference; + + const_iterator(void) = default; + const_iterator(const MulticastTargetSet &_set, size_t _run_idx); + + bool operator==(const const_iterator &rhs) const; + bool operator!=(const const_iterator &rhs) const { return !(*this == rhs); } + + NodeID operator*(void) const { return cur_node; } + const NodeID *operator->(void) const { return &cur_node; } + + const_iterator &operator++(/*prefix*/); + const_iterator operator++(int /*postfix*/); + + protected: + const MulticastTargetSet *set = nullptr; + size_t run_idx = 0; + NodeID cur_node = -1; + }; + + const_iterator begin(void) const { return const_iterator(*this, 0); } + const_iterator end(void) const { return const_iterator(*this, runs.size()); } + + // --- partitioning (plan section 7.3) ------------------------------- + + // Splits into at most 'max_slices' slices of nearly equal cardinality (the first + // size()%k slices get one extra node). Runs are cut, never expanded, so a run + // covering thousands of nodes costs O(1) per slice it lands in. Empty slices are + // never produced, so an empty set yields no slices and a set smaller than + // 'max_slices' yields size() singleton slices. + void partition(size_t max_slices, std::vector &slices) const; + + bool operator==(const MulticastTargetSet &rhs) const; + bool operator!=(const MulticastTargetSet &rhs) const { return !(*this == rhs); } + + protected: + // index of the run containing 'id', or runs.size() if there is none + size_t find_run(NodeID id) const; + + std::vector runs; // sorted, disjoint, nonadjacent + size_t total = 0; + }; + + std::ostream &operator<<(std::ostream &os, const MulticastTargetSet &targets); + + //////////////////////////////////////////////////////////////////////// + // + // struct MulticastEncodingTally + // + + // Optional per-choice tally (plan section 21.3, "multicast target encoding choices"). + // This layer has no barrier dependency, so the encoder records here and the barrier + // layer mirrors these into BarrierCounters::multicast_encoding_* using kind(). + struct MulticastEncodingTally { + MulticastEncodingTally(void); + + void record(MulticastTargetEncoding kind); + + uint64_t get(MulticastTargetEncoding kind) const; + uint64_t total(void) const; + + void reset(void) { *this = MulticastEncodingTally(); } + + atomic counts[MULTICAST_ENCODING_KINDS]; + }; + + //////////////////////////////////////////////////////////////////////// + // + // class MulticastMetricsSink + // + + // Optional metrics hook for one multicast (plan section 21.3: "multicast target + // encoding choices" and "multicast tree depth and first-hop count"). The forwarding + // layer in realm/activemsg.h reports into whichever sink the caller supplies + // and never touches process-global state; BarrierMulticastMetrics in + // realm/barrier_impl.h is the implementation that mirrors these into a barrier's + // BarrierCounters. + // + // It is declared here, rather than next to the forwarding code, so that a counter + // owner does not have to pull in activemsg.h. + class MulticastMetricsSink { + public: + virtual ~MulticastMetricsSink(void); + + // which of the eight encodings was selected for one outbound envelope + virtual void record_encoding_choice(MulticastTargetEncoding kind) = 0; + + // number of first-hop envelopes this node issued for one multicast - plan section + // 23 bounds this by the radix + virtual void record_first_hops(size_t num_first_hops) = 0; + + // hops from the origin of an envelope this node relayed; the origin's own first-hop + // envelopes carry depth 1 + virtual void record_tree_depth(unsigned depth) = 0; + }; + + //////////////////////////////////////////////////////////////////////// + // + // class EncodedMulticastTargets + // + + // The immutable wire form of a target slice. The byte string always starts with the + // kind byte and is entirely self-describing apart from the configured node count, + // which both sides must agree on. + class EncodedMulticastTargets { + public: + // default-constructs to a valid EMPTY encoding + EncodedMulticastTargets(void); + + // Evaluates every representation that can express 'targets', computes each + // candidate's ACTUAL serialized length (no density heuristic - plan section 7.2) + // and keeps the smallest. Every target must be in [0, num_nodes). + static EncodedMulticastTargets encode(const MulticastTargetSet &targets, + NodeID num_nodes, + MulticastEncodingTally *tally = nullptr); + + // Exact serialized length 'kind' would need for 'targets', or 0 if that kind cannot + // represent this set. Runs are never expanded, so this is O(num_ranges()) even + // for DELTA_LIST and ALL_EXCEPT. + static size_t encoded_size(const MulticastTargetSet &targets, NodeID num_nodes, + MulticastTargetEncoding kind); + + // Rebuilds the logical set. 'targets' is cleared first and left empty on failure. + // Validates the kind, every varint, the cardinality (against both the remaining + // payload length and 'num_nodes'), node bounds, run overflow and canonical + // ordering before touching 'targets', and never sizes an allocation from an + // unvalidated length (plan section 22). + static MulticastDecodeStatus decode(const void *data, size_t bytes, NodeID num_nodes, + MulticastTargetSet &targets); + + MulticastTargetEncoding kind(void) const { return chosen; } + + const void *data(void) const { return wire.data(); } + size_t bytes(void) const { return wire.size(); } + + const std::vector &wire_bytes(void) const { return wire; } + + // cardinality as known locally at encode time - a decoder must never trust a + // transmitted cardinality, so this is not on the wire for every kind + size_t num_targets(void) const { return count; } + + MulticastDecodeStatus decode_into(NodeID num_nodes, MulticastTargetSet &targets) const + { + return decode(wire.data(), wire.size(), num_nodes, targets); + } + + protected: + static void emit(std::vector &buf, const MulticastTargetSet &targets, + NodeID num_nodes, MulticastTargetEncoding kind); + + std::vector wire; + MulticastTargetEncoding chosen = MulticastTargetEncoding::EMPTY; + size_t count = 0; + }; + +}; // namespace Realm + +#endif // ifndef REALM_MULTICAST_H diff --git a/src/realm/network.cc b/src/realm/network.cc index 2c6c6ce8ce..2ffc2ab719 100644 --- a/src/realm/network.cc +++ b/src/realm/network.cc @@ -360,15 +360,8 @@ namespace Realm { NodeID target, unsigned short msgid, size_t header_size, size_t max_payload_size, const RemoteAddress &dest_payload_addr, void *storage_base, size_t storage_size); - virtual ActiveMessageImpl *create_active_message_impl( - const NodeSet &targets, unsigned short msgid, size_t header_size, - size_t max_payload_size, const void *src_payload_addr, size_t src_payload_lines, - size_t src_payload_line_stride, void *storage_base, size_t storage_size); - virtual size_t recommended_max_payload(NodeID target, bool with_congestion, size_t header_size); - virtual size_t recommended_max_payload(const NodeSet &targets, bool with_congestion, - size_t header_size); virtual size_t recommended_max_payload(NodeID target, const RemoteAddress &dest_payload_addr, bool with_congestion, size_t header_size); @@ -376,10 +369,6 @@ namespace Realm { size_t bytes_per_line, size_t lines, size_t line_stride, bool with_congestion, size_t header_size); - virtual size_t recommended_max_payload(const NodeSet &targets, const void *data, - size_t bytes_per_line, size_t lines, - size_t line_stride, bool with_congestion, - size_t header_size); virtual size_t recommended_max_payload(NodeID target, const LocalAddress &src_payload_addr, size_t bytes_per_line, size_t lines, @@ -539,15 +528,6 @@ namespace Realm { abort(); } - ActiveMessageImpl *LoopbackNetworkModule::create_active_message_impl( - const NodeSet &targets, unsigned short msgid, size_t header_size, - size_t max_payload_size, const void *src_payload_addr, size_t src_payload_lines, - size_t src_payload_line_stride, void *storage_base, size_t storage_size) - { - // should never be called - abort(); - } - size_t LoopbackNetworkModule::recommended_max_payload(NodeID target, bool with_congestion, size_t header_size) @@ -557,15 +537,6 @@ namespace Realm { return 0; } - size_t LoopbackNetworkModule::recommended_max_payload(const NodeSet &targets, - bool with_congestion, - size_t header_size) - { - // should never be called - abort(); - return 0; - } - size_t LoopbackNetworkModule::recommended_max_payload(NodeID target, const RemoteAddress &dest_payload_addr, @@ -587,15 +558,6 @@ namespace Realm { return 0; } - size_t LoopbackNetworkModule::recommended_max_payload( - const NodeSet &targets, const void *data, size_t bytes_per_line, size_t lines, - size_t line_stride, bool with_congestion, size_t header_size) - { - // should never be called - abort(); - return 0; - } - size_t LoopbackNetworkModule::recommended_max_payload( NodeID target, const LocalAddress &src_payload_addr, size_t bytes_per_line, size_t lines, size_t line_stride, const RemoteAddress &dest_payload_addr, diff --git a/src/realm/network.h b/src/realm/network.h index fa58485f36..eecf47a994 100644 --- a/src/realm/network.h +++ b/src/realm/network.h @@ -138,24 +138,17 @@ namespace Realm { NodeID target, unsigned short msgid, size_t header_size, size_t max_payload_size, const RemoteAddress &dest_payload_addr, void *storage_base, size_t storage_size); - ActiveMessageImpl *create_active_message_impl( - const NodeSet &targets, unsigned short msgid, size_t header_size, - size_t max_payload_size, const void *src_payload_addr, size_t src_payload_lines, - size_t src_payload_line_stride, void *storage_base, size_t storage_size); - + // NOTE: there is deliberately no NodeSet ("multicast") form of any of + // these - a multi-target send used to fan out one message per target at + // the source. Multicast is now built above unicast in + // realm/activemsg.h (plan sections 7.1-7.6). size_t recommended_max_payload(NodeID target, bool with_congestion, size_t header_size); - size_t recommended_max_payload(const NodeSet &targets, bool with_congestion, - size_t header_size); size_t recommended_max_payload(NodeID target, const RemoteAddress &dest_payload_addr, bool with_congestion, size_t header_size); size_t recommended_max_payload(NodeID target, const void *data, size_t bytes_per_line, size_t lines, size_t line_stride, bool with_congestion, size_t header_size); - size_t recommended_max_payload(const NodeSet &targets, const void *data, - size_t bytes_per_line, size_t lines, - size_t line_stride, bool with_congestion, - size_t header_size); size_t recommended_max_payload(NodeID target, const LocalAddress &src_payload_addr, size_t bytes_per_line, size_t lines, size_t line_stride, @@ -351,15 +344,8 @@ namespace Realm { const RemoteAddress &dest_payload_addr, void *storage_base, size_t storage_size) = 0; - virtual ActiveMessageImpl *create_active_message_impl( - const NodeSet &targets, unsigned short msgid, size_t header_size, - size_t max_payload_size, const void *src_payload_addr, size_t src_payload_lines, - size_t src_payload_line_stride, void *storage_base, size_t storage_size) = 0; - virtual size_t recommended_max_payload(NodeID target, bool with_congestion, size_t header_size) = 0; - virtual size_t recommended_max_payload(const NodeSet &targets, bool with_congestion, - size_t header_size) = 0; virtual size_t recommended_max_payload(NodeID target, const RemoteAddress &dest_payload_addr, bool with_congestion, size_t header_size) = 0; @@ -367,10 +353,6 @@ namespace Realm { size_t bytes_per_line, size_t lines, size_t line_stride, bool with_congestion, size_t header_size) = 0; - virtual size_t recommended_max_payload(const NodeSet &targets, const void *data, - size_t bytes_per_line, size_t lines, - size_t line_stride, bool with_congestion, - size_t header_size) = 0; virtual size_t recommended_max_payload(NodeID target, const LocalAddress &src_payload_addr, size_t bytes_per_line, size_t lines, diff --git a/src/realm/network.inl b/src/realm/network.inl index 463277aebb..5bad4b8ed1 100644 --- a/src/realm/network.inl +++ b/src/realm/network.inl @@ -132,21 +132,6 @@ namespace Realm { storage_size); } - inline ActiveMessageImpl *create_active_message_impl( - const NodeSet &targets, unsigned short msgid, size_t header_size, - size_t max_payload_size, const void *src_payload_addr, size_t src_payload_lines, - size_t src_payload_line_stride, void *storage_base, size_t storage_size) - { -#ifdef REALM_USE_MULTIPLE_NETWORKS - if(REALM_UNLIKELY(single_network == 0)) { - std::abort(); - } else -#endif - return single_network->create_active_message_impl( - targets, msgid, header_size, max_payload_size, src_payload_addr, - src_payload_lines, src_payload_line_stride, storage_base, storage_size); - } - inline size_t recommended_max_payload(NodeID target, bool with_congestion, size_t header_size) { @@ -159,18 +144,6 @@ namespace Realm { header_size); } - inline size_t recommended_max_payload(const NodeSet &targets, bool with_congestion, - size_t header_size) - { -#ifdef REALM_USE_MULTIPLE_NETWORKS - if(REALM_UNLIKELY(single_network == 0)) { - std::abort(); - } else -#endif - return single_network->recommended_max_payload(targets, with_congestion, - header_size); - } - inline size_t recommended_max_payload(NodeID target, const RemoteAddress &dest_payload_addr, bool with_congestion, size_t header_size) @@ -199,21 +172,6 @@ namespace Realm { with_congestion, header_size); } - inline size_t recommended_max_payload(const NodeSet &targets, const void *data, - size_t bytes_per_line, size_t lines, - size_t line_stride, bool with_congestion, - size_t header_size) - { -#ifdef REALM_USE_MULTIPLE_NETWORKS - if(REALM_UNLIKELY(single_network == 0)) { - std::abort(); - } else -#endif - return single_network->recommended_max_payload(targets, data, bytes_per_line, - lines, line_stride, - with_congestion, header_size); - } - inline size_t recommended_max_payload(NodeID target, const LocalAddress &src_payload_addr, size_t bytes_per_line, size_t lines, diff --git a/src/realm/prealm/prealm.h b/src/realm/prealm/prealm.h index 333ef1fe00..9c76fb3cc2 100644 --- a/src/realm/prealm/prealm.h +++ b/src/realm/prealm/prealm.h @@ -213,13 +213,6 @@ namespace PRealm { static Barrier create_barrier(unsigned expected_arrivals, ReductionOpID redop_id = 0, const void *initial_value = 0, size_t initial_value_size = 0); - using ParticipantInfo = Realm::Barrier::ParticipantInfo; - static Barrier create_barrier(const Barrier::ParticipantInfo *expected_arrivals, - size_t num_participants, ReductionOpID redop_id = 0, - const void *initial_value = 0, - size_t initial_value_size = 0); - Barrier set_arrival_pattern(const Barrier::ParticipantInfo *expected_arrivals, - size_t num_participants); void destroy_barrier(void); static const ::realm_event_gen_t MAX_PHASES; diff --git a/src/realm/prealm/prealm.inl b/src/realm/prealm/prealm.inl index 570d6d3a31..ecf5c17df3 100644 --- a/src/realm/prealm/prealm.inl +++ b/src/realm/prealm/prealm.inl @@ -634,23 +634,6 @@ namespace PRealm { initial_value_size); } - /*static*/ inline Barrier - Barrier::create_barrier(const Barrier::ParticipantInfo *expected_arrivals, - size_t num_participants, ReductionOpID redop, - const void *initial_value, size_t initial_value_size) - { - return Realm::Barrier::create_barrier(expected_arrivals, num_participants, redop, - initial_value, initial_value_size); - } - - inline Barrier - Barrier::set_arrival_pattern(const Barrier::ParticipantInfo *expected_arrivals, - size_t num_participants) - { - Realm::Barrier barrier = *this; - return barrier.set_arrival_pattern(expected_arrivals, num_participants); - } - inline void Barrier::destroy_barrier(void) { Realm::Barrier barrier = *this; diff --git a/src/realm/runtime_impl.cc b/src/realm/runtime_impl.cc index 69d05dcf94..af92005187 100644 --- a/src/realm/runtime_impl.cc +++ b/src/realm/runtime_impl.cc @@ -2792,14 +2792,22 @@ namespace Realm { // if we're the master, we need to notify everyone else first NodeID shutdown_master_node = 0; if((Network::my_node_id == shutdown_master_node) && (Network::max_node_id > 0)) { - NodeSet targets; - for(NodeID i = 0; i <= Network::max_node_id; i++) - if(i != Network::my_node_id) - targets.add(i); - - ActiveMessage amsg(targets); - amsg->result_code = shutdown_result_code; - amsg.commit(); + // every node except us - one run (or two, if the master is ever moved off node + // 0), so the encoder can name the whole machine in a handful of bytes no matter + // how large it is (ALL_EXCEPT in practice, RANGES for the slices) + MulticastTargetSet targets; + if(Network::my_node_id > 0) + targets.add_range(0, Network::my_node_id - 1); + if(Network::my_node_id < Network::max_node_id) + targets.add_range(Network::my_node_id + 1, Network::max_node_id); + + // Multicast (rather than one message per node) so that the master's fan-out is + // bounded by the radix. Relays forward before delivering shutdown locally, + // which is exactly why plan section 7.3 requires forward-before-deliver: a node + // that shut itself down first would strand its entire subtree. + RuntimeShutdownMessage msg{}; + msg.result_code = shutdown_result_code; + multicast_message(targets, msg); } { diff --git a/src/realm/ucx/ucp_internal.cc b/src/realm/ucx/ucp_internal.cc index c9f2ad3243..eb2f793f97 100644 --- a/src/realm/ucx/ucp_internal.cc +++ b/src/realm/ucx/ucp_internal.cc @@ -126,20 +126,6 @@ namespace Realm { char rkey[0]; } __attribute__((packed)); // gcc-specific - struct MCDesc { - enum - { - REQUEST_AM_FLAG_FAILURE = 1ul << 0 - }; - uint8_t flags; - atomic local_pending; // number of targets pending local - // completion (to support multicast) - MCDesc(size_t _local_pending) - : flags(0) - , local_pending(_local_pending) - {} - }; - struct Request { // UCPContext::Request must be the first field because // the space preceding it is used internally by ucp @@ -150,7 +136,6 @@ namespace Realm { struct { PayloadBaseType payload_base_type; CompList *local_comp; - MCDesc *mc_desc; } am_send; struct { @@ -2113,23 +2098,6 @@ namespace Realm { } } - UCPMessageImpl::UCPMessageImpl(UCPInternal *_internal, const NodeSet &_targets, - unsigned short _msgid, size_t _header_size, - size_t _max_payload_size, - const void *_src_payload_addr, - size_t _src_payload_lines, - size_t _src_payload_line_stride, size_t _storage_size) - : UCPMessageImpl(_internal, *_targets.begin(), _msgid, _header_size, - _max_payload_size, _src_payload_addr, _src_payload_lines, - _src_payload_line_stride, nullptr, nullptr, _storage_size) - { - // treat as multicast if only number of targets is greater than 1 - if(_targets.size() > 1) { - targets = _targets; - is_multicast = true; - } - } - UCPMessageImpl::~UCPMessageImpl() {} bool UCPMessageImpl::set_inline_payload_base() @@ -2164,8 +2132,7 @@ namespace Realm { void *UCPMessageImpl::add_remote_completion(size_t size) { if(remote_comp == nullptr) { - size_t remote_pending = is_multicast ? targets.size() : 1; - remote_comp = new RemoteComp(remote_pending); + remote_comp = new RemoteComp(1 /*remote_pending*/); } size_t ofs = remote_comp->comp_list->bytes; remote_comp->comp_list->bytes += size; @@ -2191,11 +2158,6 @@ namespace Realm { } } - if(req->am_send.mc_desc != nullptr) { - req->am_send.mc_desc->flags |= MCDesc::REQUEST_AM_FLAG_FAILURE; - local_pending = req->am_send.mc_desc->local_pending.fetch_sub_acqrel(1); - } - assert(local_pending != 0); if(local_pending == 1) { @@ -2212,7 +2174,6 @@ namespace Realm { UCPInternal *internal = req->internal; CompList *local_comp = req->am_send.local_comp; size_t local_pending = 1; - uint8_t comp_flags = 0; log_ucp_am.debug() << "am_local_comp_handler invoked for request " << req; @@ -2220,23 +2181,12 @@ namespace Realm { internal->notify_msg_sent(1); - if(req->am_send.mc_desc != nullptr) { - local_pending = req->am_send.mc_desc->local_pending.fetch_sub(1); - comp_flags = req->am_send.mc_desc->flags; - } - assert(local_pending != 0); if(local_pending == 1) { - if(!(comp_flags & MCDesc::REQUEST_AM_FLAG_FAILURE)) { - // all senders completed without failure - if(local_comp != nullptr) { - CompletionCallbackBase::invoke_all(local_comp->storage, local_comp->bytes); - CompletionCallbackBase::destroy_all(local_comp->storage, local_comp->bytes); - } - } else { - // TODO: should CompletionCallbackBase::destroy_all() be called here? - // TODO: should invoke some higher-level error handler + if(local_comp != nullptr) { + CompletionCallbackBase::invoke_all(local_comp->storage, local_comp->bytes); + CompletionCallbackBase::destroy_all(local_comp->storage, local_comp->bytes); } UCPMessageImpl::cleanup_request(req, internal); } @@ -2329,20 +2279,8 @@ namespace Realm { req->ucp.memtype = memtype; req->ucp.flags = 0; - if(is_multicast) { - req->am_send.mc_desc = new MCDesc(targets.size()); - CHKERR_JUMP(req->am_send.mc_desc == nullptr, "failed to new multicast desc", - log_ucp, err_rel_payload); - } else { - req->am_send.mc_desc = nullptr; - } - return req; - err_rel_payload: - if(payload_base_type == PAYLOAD_BASE_INTERNAL) { - internal->pbuf_release(worker, payload_base); - } err_rel_req: internal->request_release(req); err: @@ -2356,7 +2294,6 @@ namespace Realm { internal->pbuf_release(req->worker, req->ucp.payload); } delete req->am_send.local_comp; - delete req->am_send.mc_desc; } void UCPMessageImpl::am_put_comp_handler(void *request, ucs_status_t status, @@ -2453,53 +2390,6 @@ namespace Realm { return false; } - bool UCPMessageImpl::commit_multicast(size_t act_payload_size) - { - size_t to_submit = targets.size(); - int remote_dev_index = - dest_payload_rdma_info ? dest_payload_rdma_info->dev_index : -1; - Request *req_prim, *req; - - req_prim = make_request(act_payload_size); - CHKERR_JUMP(req_prim == nullptr, "failed to make am request", log_ucp, err); - - for(const NodeID &target : targets) { - // shallow-copy the primary request for each target - // IMPORTANT: the primary requet must not be released - // until all target copies have been created. - // Otherwise, it may be released upon completion - // which will make furhter copies invalid. - req = internal->request_get(worker); - if(req == nullptr) { - log_ucp.error() << "failed to get additional request for multicast"; - req = req_prim; - goto err_update_pending; - } - *req = *req_prim; - CHKERR_JUMP(!worker->ep_get(target, remote_dev_index, &req->ucp.ep), - "failed to get ep", log_ucp, err); - if(!UCPMessageImpl::send_request(req, AM_ID)) { - log_ucp.error() << "failed to send multicast am request"; - goto err_update_pending; - } - - to_submit--; - } - - internal->request_release(req_prim); - - return true; - - err_update_pending: - req->am_send.mc_desc->local_pending.fetch_sub(to_submit); - if(remote_comp != nullptr) { - ucp_msg_hdr.remote_comp->remote_pending.fetch_sub(to_submit); - } - UCPMessageImpl::am_local_failure_handler(req, internal); - err: - return false; - } - bool UCPMessageImpl::commit_unicast(size_t act_payload_size) { int remote_dev_index = @@ -2573,21 +2463,16 @@ namespace Realm { insert_packet_crc(&ucp_msg_hdr, header_size, act_payload_size); } - if(is_multicast) { - status = commit_multicast(act_payload_size); - } else { - status = commit_unicast(act_payload_size); - } + status = commit_unicast(act_payload_size); if(!status) { log_ucp.error() << "failed to commit am"; } log_ucp_am.info() - << "msg commit " << (is_multicast ? "multicast" : "unicast") << "context " - << context << "worker " << worker << " target " << (is_multicast ? -1 : target) - << " hsize " << header_size << " psize " << act_payload_size << " ptype " - << payload_base_type + << "msg commit unicast context " << context << "worker " << worker << " target " + << target << " hsize " << header_size << " psize " << act_payload_size + << " ptype " << payload_base_type #ifdef REALM_USE_CUDA << " src_mem_type " << (context->gpu ? "cuda" : "host") << " dst_mem_type " << (dest_payload_rdma_info diff --git a/src/realm/ucx/ucp_internal.h b/src/realm/ucx/ucp_internal.h index 53c1225f6f..945ed01f21 100644 --- a/src/realm/ucx/ucp_internal.h +++ b/src/realm/ucx/ucp_internal.h @@ -294,11 +294,6 @@ namespace Realm { size_t src_payload_line_stride, const NetworkSegment *_src_segment, const RemoteAddress *_dest_payload_addr, size_t storage_size); - UCPMessageImpl(UCPInternal *internal, const NodeSet &targets, unsigned short msgid, - size_t header_size, size_t max_payload_size, - const void *src_payload_addr, size_t src_payload_lines, - size_t src_payload_line_stride, size_t storage_size); - virtual ~UCPMessageImpl(); virtual void *add_local_completion(size_t size); @@ -311,7 +306,6 @@ namespace Realm { bool set_inline_payload_base(); bool commit_with_rma(ucp_ep_h ep); bool commit_unicast(size_t act_payload_size); - bool commit_multicast(size_t act_payload_size); bool send_fast_path(ucp_ep_h ep, size_t act_payload_size); bool send_slow_path(ucp_ep_h ep, size_t act_payload_size, uint32_t flags); Request *make_request(size_t act_payload_size); @@ -328,7 +322,6 @@ namespace Realm { UCPInternal *internal; UCPWorker *worker; NodeID target; - NodeSet targets; const void *src_payload_addr; size_t src_payload_lines; size_t src_payload_line_stride; @@ -337,7 +330,6 @@ namespace Realm { UCPRDMAInfo *dest_payload_rdma_info{nullptr}; CompList *local_comp{nullptr}; RemoteComp *remote_comp{nullptr}; - bool is_multicast{false}; ucs_memory_type_t memtype; UCPMsgHdr ucp_msg_hdr; diff --git a/src/realm/ucx/ucp_module.cc b/src/realm/ucx/ucp_module.cc index dc22ea3be2..86de92db45 100644 --- a/src/realm/ucx/ucp_module.cc +++ b/src/realm/ucx/ucp_module.cc @@ -266,17 +266,6 @@ namespace Realm { &dest_payload_addr, storage_size); } - ActiveMessageImpl *UCPModule::create_active_message_impl( - const NodeSet &targets, unsigned short msgid, size_t header_size, - size_t max_payload_size, const void *src_payload_addr, size_t src_payload_lines, - size_t src_payload_line_stride, void *storage_base, size_t storage_size) - { - assert(storage_size >= sizeof(Realm::UCP::UCPMessageImpl)); - return new(storage_base) Realm::UCP::UCPMessageImpl( - internal, targets, msgid, header_size, max_payload_size, src_payload_addr, - src_payload_lines, src_payload_line_stride, storage_size); - } - size_t UCPModule::recommended_max_payload(NodeID target, bool with_congestion, size_t header_size) { @@ -285,14 +274,6 @@ namespace Realm { header_size); } - size_t UCPModule::recommended_max_payload(const NodeSet &targets, bool with_congestion, - size_t header_size) - { - (void)targets; - return internal->recommended_max_payload(nullptr, nullptr, nullptr, with_congestion, - header_size); - } - size_t UCPModule::recommended_max_payload(NodeID target, const RemoteAddress &dest_payload_addr, bool with_congestion, size_t header_size) @@ -312,16 +293,6 @@ namespace Realm { header_size); } - size_t UCPModule::recommended_max_payload(const NodeSet &targets, const void *data, - size_t bytes_per_line, size_t lines, - size_t line_stride, bool with_congestion, - size_t header_size) - { - (void)targets; - return internal->recommended_max_payload(data, nullptr, nullptr, with_congestion, - header_size); - } - size_t UCPModule::recommended_max_payload(NodeID target, const LocalAddress &src_payload_addr, size_t bytes_per_line, size_t lines, diff --git a/src/realm/ucx/ucp_module.h b/src/realm/ucx/ucp_module.h index 8dcd0a61af..a6be1e9249 100644 --- a/src/realm/ucx/ucp_module.h +++ b/src/realm/ucx/ucp_module.h @@ -95,15 +95,8 @@ namespace Realm { NodeID target, unsigned short msgid, size_t header_size, size_t max_payload_size, const RemoteAddress &dest_payload_addr, void *storage_base, size_t storage_size); - virtual ActiveMessageImpl *create_active_message_impl( - const NodeSet &targets, unsigned short msgid, size_t header_size, - size_t max_payload_size, const void *src_payload_addr, size_t src_payload_lines, - size_t src_payload_line_stride, void *storage_base, size_t storage_size); - virtual size_t recommended_max_payload(NodeID target, bool with_congestion, size_t header_size); - virtual size_t recommended_max_payload(const NodeSet &targets, bool with_congestion, - size_t header_size); virtual size_t recommended_max_payload(NodeID target, const RemoteAddress &dest_payload_addr, bool with_congestion, size_t header_size); @@ -111,10 +104,6 @@ namespace Realm { size_t bytes_per_line, size_t lines, size_t line_stride, bool with_congestion, size_t header_size); - virtual size_t recommended_max_payload(const NodeSet &targets, const void *data, - size_t bytes_per_line, size_t lines, - size_t line_stride, bool with_congestion, - size_t header_size); virtual size_t recommended_max_payload(NodeID target, const LocalAddress &src_payload_addr, size_t bytes_per_line, size_t lines, diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index f451e976b7..edaff97e84 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -99,6 +99,7 @@ list( memcpy_channel_test.cc comp_queue_test.cc event_test.cc + multicast_test.cc path_cache_test.cc transfer_utils_test.cc ib_redistribute_test.cc @@ -273,6 +274,8 @@ add_integration_test(serializing "${REALM_TEST_DIR}/serializing.cc") set(ctxswitch_ARGS -ll:io 1 -t 30 -i 10000) add_integration_test(ctxswitch "${REALM_TEST_DIR}/ctxswitch.cc") add_integration_test(barrier_reduce "${REALM_TEST_DIR}/barrier_reduce.cc") +# barrier_arrivals.cc existed but was never registered, so it was neither built nor run. +add_integration_test(barrier_arrivals "${REALM_TEST_DIR}/barrier_arrivals.cc") add_integration_test(taskreg "${REALM_TEST_DIR}/taskreg.cc") add_integration_test(idcheck "${REALM_TEST_DIR}/idcheck.cc") add_integration_test(inst_reuse "${REALM_TEST_DIR}/inst_reuse.cc") diff --git a/tests/barrier_arrivals.cc b/tests/barrier_arrivals.cc index 7f4c1ec74d..c3a41e92d8 100644 --- a/tests/barrier_arrivals.cc +++ b/tests/barrier_arrivals.cc @@ -142,8 +142,16 @@ void waiter_task(const void *args, size_t arglen, const void *userdata, size_t u } } +// Barrier::ParticipantInfo was a dead explicit-pattern API (its only definitions were +// assert(0) stubs) and has been removed. This test only ever used it as a convenient +// data holder, so it carries its own. +struct TestParticipantInfo { + AddressSpace address_space; + unsigned count; +}; + struct BarrierArrivalInfo { - std::vector participants; + std::vector participants; std::vector waiter_address_spaces; }; diff --git a/tests/unit_tests/multicast_test.cc b/tests/unit_tests/multicast_test.cc new file mode 100644 index 0000000000..136f649959 --- /dev/null +++ b/tests/unit_tests/multicast_test.cc @@ -0,0 +1,4278 @@ +/* + * Copyright 2025 Stanford University, NVIDIA Corporation + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Stages 2a, 2b and 2c of SCALABLE_BARRIERS_IMPLEMENTATION_PLAN.md. +// +// - stage 2a: adaptive multicast target sets and encodings (plan sections 7.2, 20.1) +// - stage 2b: the multicast envelope, generic handler redispatch and bounded-radix +// forwarding over unicast (plan sections 7.1, 7.3, 7.4, 20.1) +// - stage 2c: payload, fragmentation and transient completion semantics (plan +// sections 7.5, 20.1) + +#include "realm/multicast.h" + +#include "realm/activemsg.h" +#include "realm/threads.h" +#include "realm/timers.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// the guard page under FuzzedPayloadsNeverReadPastTheEndOfTheBuffer needs the +// platform's virtual memory calls +#ifdef REALM_ON_WINDOWS +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN 1 +#endif +// windows.h defines macros called 'min' and 'max' otherwise, which the std::min and +// std::max calls below would trip over +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#else +#include +#include +#endif + +#include + +using namespace Realm; + +namespace { + + typedef std::vector ByteVec; + + void put_kind(ByteVec &buf, MulticastTargetEncoding kind) + { + buf.push_back(static_cast(kind)); + } + + void put_varint(ByteVec &buf, uint64_t value) + { + MulticastWire::append_varint(buf, value); + } + + MulticastDecodeStatus decode_bytes(const ByteVec &buf, NodeID num_nodes, + MulticastTargetSet &out) + { + return EncodedMulticastTargets::decode(buf.data(), buf.size(), num_nodes, out); + } + + MulticastDecodeStatus decode_bytes(const ByteVec &buf, NodeID num_nodes) + { + MulticastTargetSet ignored; + return decode_bytes(buf, num_nodes, ignored); + } + + size_t size_of(const MulticastTargetSet &targets, NodeID num_nodes, + MulticastTargetEncoding kind) + { + return EncodedMulticastTargets::encoded_size(targets, num_nodes, kind); + } + + // every node in the set, in iteration order + std::vector expand(const MulticastTargetSet &targets) + { + std::vector nodes; + for(MulticastTargetSet::const_iterator it = targets.begin(); it != targets.end(); + ++it) + nodes.push_back(*it); + return nodes; + } + + // sorted, disjoint, nonadjacent runs and a consistent cardinality + void check_canonical(const MulticastTargetSet &targets) + { + const std::vector &runs = targets.ranges(); + size_t total = 0; + for(size_t i = 0; i < runs.size(); i++) { + EXPECT_LE(runs[i].first, runs[i].last); + EXPECT_GE(runs[i].first, 0); + if(i > 0) + EXPECT_GT(static_cast(runs[i].first), + static_cast(runs[i - 1].last) + 1) + << "runs " << (i - 1) << " and " << i << " are unmerged or out of order"; + total += runs[i].count(); + } + EXPECT_EQ(total, targets.size()); + } + + // the encoder must pick the smallest ACTUAL serialized size (plan section 7.2) + void check_encoder_picked_minimum(const MulticastTargetSet &targets, NodeID num_nodes, + const EncodedMulticastTargets &enc) + { + size_t best = 0; + for(size_t i = 0; i < MULTICAST_ENCODING_KINDS; i++) { + size_t sz = size_of(targets, num_nodes, static_cast(i)); + if((sz != 0) && ((best == 0) || (sz < best))) + best = sz; + } + ASSERT_GT(best, 0u); + EXPECT_EQ(enc.bytes(), best) << "chose " << enc.kind() << " for " << targets; + EXPECT_EQ(size_of(targets, num_nodes, enc.kind()), best); + } + + // encode, verify minimality, verify the round trip, and hand back the encoding + EncodedMulticastTargets round_trip(const MulticastTargetSet &targets, NodeID num_nodes) + { + EncodedMulticastTargets enc = EncodedMulticastTargets::encode(targets, num_nodes); + check_encoder_picked_minimum(targets, num_nodes, enc); + EXPECT_EQ(enc.num_targets(), targets.size()); + + MulticastTargetSet decoded; + EXPECT_EQ(enc.decode_into(num_nodes, decoded), MulticastDecodeStatus::OK) + << "kind " << enc.kind(); + EXPECT_EQ(decoded, targets); + check_canonical(decoded); + return enc; + } + +}; // namespace + +//////////////////////////////////////////////////////////////////////// +// +// MulticastTargetSet: building, canonical form, iteration +// + +TEST(MulticastTargetSetTest, EmptySet) +{ + MulticastTargetSet targets; + EXPECT_TRUE(targets.empty()); + EXPECT_EQ(targets.size(), 0u); + EXPECT_EQ(targets.num_ranges(), 0u); + EXPECT_FALSE(targets.contains(0)); + EXPECT_TRUE(targets.begin() == targets.end()); + EXPECT_TRUE(expand(targets).empty()); +} + +TEST(MulticastTargetSetTest, AddMergesAdjacentAndDuplicates) +{ + MulticastTargetSet targets; + targets.add(5); + targets.add(5); // duplicate + EXPECT_EQ(targets.size(), 1u); + EXPECT_EQ(targets.num_ranges(), 1u); + + targets.add(6); // adjacent above + targets.add(4); // adjacent below + EXPECT_EQ(targets.size(), 3u); + EXPECT_EQ(targets.num_ranges(), 1u) << "adjacent nodes must merge into one run"; + EXPECT_EQ(targets.first_node(), 4); + EXPECT_EQ(targets.last_node(), 6); + + targets.add(9); // disjoint + EXPECT_EQ(targets.num_ranges(), 2u); + targets.add(8); // closes the gap partially + targets.add(7); // ...and fully + EXPECT_EQ(targets.num_ranges(), 1u); + EXPECT_EQ(targets.size(), 6u); + check_canonical(targets); +} + +TEST(MulticastTargetSetTest, ArbitraryRangeBoundaries) +{ + MulticastTargetSet targets; + targets.add_range(10, 20); + targets.add_range(40, 50); + targets.add_range(70, 80); + EXPECT_EQ(targets.num_ranges(), 3u); + EXPECT_EQ(targets.size(), 33u); + + // fully inside an existing run + targets.add_range(12, 15); + EXPECT_EQ(targets.num_ranges(), 3u); + EXPECT_EQ(targets.size(), 33u); + + // overlapping two runs and the gap between them + targets.add_range(15, 45); + EXPECT_EQ(targets.num_ranges(), 2u); + EXPECT_EQ(targets.size(), 41u + 11u); + + // exactly adjacent on both sides + targets.add_range(51, 69); + EXPECT_EQ(targets.num_ranges(), 1u); + EXPECT_EQ(targets.size(), 71u); + EXPECT_EQ(targets.first_node(), 10); + EXPECT_EQ(targets.last_node(), 80); + + // an inverted range is a no-op + targets.add_range(200, 100); + EXPECT_EQ(targets.size(), 71u); + check_canonical(targets); +} + +TEST(MulticastTargetSetTest, SortedIterationAndContains) +{ + MulticastTargetSet targets; + // deliberately unsorted insertion order + const NodeID added[] = {7, 1, 30, 31, 2, 29, 0}; + for(size_t i = 0; i < sizeof(added) / sizeof(added[0]); i++) + targets.add(added[i]); + + std::vector nodes = expand(targets); + std::vector expected = {0, 1, 2, 7, 29, 30, 31}; + EXPECT_EQ(nodes, expected); + EXPECT_EQ(nodes.size(), targets.size()); + EXPECT_TRUE(std::is_sorted(nodes.begin(), nodes.end())); + + for(NodeID id = 0; id < 40; id++) + EXPECT_EQ(targets.contains(id), + std::find(expected.begin(), expected.end(), id) != expected.end()) + << "id " << id; + check_canonical(targets); +} + +TEST(MulticastTargetSetTest, RemoveLocalRelay) +{ + MulticastTargetSet targets; + targets.add_range(0, 9); + + // from the middle: splits the run + EXPECT_TRUE(targets.remove(5)); + EXPECT_EQ(targets.num_ranges(), 2u); + EXPECT_EQ(targets.size(), 9u); + EXPECT_FALSE(targets.contains(5)); + + // from the front and back edges of a run + EXPECT_TRUE(targets.remove(0)); + EXPECT_TRUE(targets.remove(9)); + EXPECT_EQ(targets.size(), 7u); + EXPECT_EQ(targets.first_node(), 1); + EXPECT_EQ(targets.last_node(), 8); + + // a singleton run disappears entirely + targets.add(100); + EXPECT_EQ(targets.num_ranges(), 3u); + EXPECT_TRUE(targets.remove(100)); + EXPECT_EQ(targets.num_ranges(), 2u); + + // removing something that is not there changes nothing + EXPECT_FALSE(targets.remove(5)); + EXPECT_FALSE(targets.remove(1000)); + EXPECT_EQ(targets.size(), 7u); + check_canonical(targets); + + std::vector expected = {1, 2, 3, 4, 6, 7, 8}; + EXPECT_EQ(expand(targets), expected); +} + +// NodeSet's bitmask encoding needs the (process-global) bitmask allocator configured, +// exactly as nodeset_test.cc does +class MulticastNodeSetTest : public ::testing::Test { +protected: + virtual void SetUp(void) + { + NodeSetBitmask::configure_allocator(max_node_id, 1024 /*bitsets_per_chunk*/, + true /*use_twolevel*/); + } + + virtual void TearDown(void) { NodeSetBitmask::free_allocations(); } + + NodeID max_node_id = 512; +}; + +TEST_F(MulticastNodeSetTest, NodeSetConversion) +{ + NodeSet nodes; + nodes.add(17); + nodes.add(3); + nodes.add_range(100, 120); + nodes.add(4); + nodes.add(101); // already covered + + MulticastTargetSet targets(nodes); + EXPECT_EQ(targets.size(), nodes.size()); + check_canonical(targets); + for(NodeSet::const_iterator it = nodes.begin(); it != nodes.end(); ++it) + EXPECT_TRUE(targets.contains(*it)) << "missing " << *it; + EXPECT_FALSE(targets.contains(5)); + EXPECT_FALSE(targets.contains(121)); + + // ...and back again + NodeSet reconstructed; + targets.to_nodeset(reconstructed); + EXPECT_EQ(reconstructed.size(), nodes.size()); + for(NodeSet::const_iterator it = nodes.begin(); it != nodes.end(); ++it) + EXPECT_TRUE(reconstructed.contains(*it)); +} + +TEST(MulticastTargetSetTest, AppendHelpersRejectNoncanonicalInput) +{ + MulticastTargetSet targets; + EXPECT_TRUE(targets.append_canonical_run(10, 19)); + EXPECT_FALSE(targets.append_canonical_run(5, 8)); // out of order + EXPECT_FALSE(targets.append_canonical_run(15, 25)); // overlapping + EXPECT_FALSE(targets.append_canonical_run(20, 25)); // adjacent, must be merged + EXPECT_TRUE(targets.append_canonical_run(21, 25)); + EXPECT_EQ(targets.num_ranges(), 2u); + EXPECT_EQ(targets.size(), 15u); + + EXPECT_FALSE(targets.append_increasing_node(25)); // duplicate + EXPECT_FALSE(targets.append_increasing_node(3)); // out of order + EXPECT_TRUE(targets.append_increasing_node(26)); // merges with trailing run + EXPECT_EQ(targets.num_ranges(), 2u); + EXPECT_TRUE(targets.append_increasing_node(40)); + EXPECT_EQ(targets.num_ranges(), 3u); + EXPECT_EQ(targets.size(), 17u); + check_canonical(targets); +} + +TEST(MulticastTargetSetTest, RandomizedAgainstReferenceSet) +{ + std::mt19937 rng(12345); + for(int trial = 0; trial < 32; trial++) { + const NodeID num_nodes = 512; + std::set reference; + MulticastTargetSet targets; + + for(int op = 0; op < 60; op++) { + unsigned choice = rng() % 10; + if(choice < 5) { + NodeID id = static_cast(rng() % num_nodes); + targets.add(id); + reference.insert(id); + } else if(choice < 8) { + NodeID lo = static_cast(rng() % num_nodes); + NodeID hi = lo + static_cast(rng() % 40); + if(hi >= num_nodes) + hi = num_nodes - 1; + targets.add_range(lo, hi); + for(NodeID id = lo; id <= hi; id++) + reference.insert(id); + } else { + NodeID id = static_cast(rng() % num_nodes); + EXPECT_EQ(targets.remove(id), reference.erase(id) > 0); + } + } + + ASSERT_EQ(targets.size(), reference.size()); + check_canonical(targets); + std::vector nodes = expand(targets); + std::vector expected(reference.begin(), reference.end()); + ASSERT_EQ(nodes, expected); + + // and the encoding of every one of these survives a round trip + round_trip(targets, num_nodes); + } +} + +//////////////////////////////////////////////////////////////////////// +// +// EncodedMulticastTargets: every encoding is forced deterministically +// + +TEST(MulticastEncodingTest, Empty) +{ + MulticastTargetSet targets; + EncodedMulticastTargets enc = round_trip(targets, 64); + EXPECT_EQ(enc.kind(), MulticastTargetEncoding::EMPTY); + EXPECT_EQ(enc.bytes(), 1u); + EXPECT_EQ(enc.num_targets(), 0u); +} + +TEST(MulticastEncodingTest, DefaultConstructedIsEmpty) +{ + EncodedMulticastTargets enc; + EXPECT_EQ(enc.kind(), MulticastTargetEncoding::EMPTY); + EXPECT_EQ(enc.bytes(), 1u); + MulticastTargetSet decoded; + EXPECT_EQ(enc.decode_into(64, decoded), MulticastDecodeStatus::OK); + EXPECT_TRUE(decoded.empty()); +} + +TEST(MulticastEncodingTest, Single) +{ + MulticastTargetSet targets; + targets.add(5); + EncodedMulticastTargets enc = round_trip(targets, 64); + EXPECT_EQ(enc.kind(), MulticastTargetEncoding::SINGLE); + EXPECT_EQ(enc.bytes(), 2u); +} + +TEST(MulticastEncodingTest, SingleAtNodeZeroAndAtMaxNodeID) +{ + const NodeID num_nodes = 1 << 20; + + MulticastTargetSet zero; + zero.add(0); + EncodedMulticastTargets enc_zero = round_trip(zero, num_nodes); + EXPECT_EQ(enc_zero.kind(), MulticastTargetEncoding::SINGLE); + + MulticastTargetSet top; + top.add(num_nodes - 1); + EncodedMulticastTargets enc_top = round_trip(top, num_nodes); + EXPECT_EQ(enc_top.kind(), MulticastTargetEncoding::SINGLE); + + MulticastTargetSet decoded; + ASSERT_EQ(enc_top.decode_into(num_nodes, decoded), MulticastDecodeStatus::OK); + EXPECT_EQ(decoded.first_node(), num_nodes - 1); +} + +TEST(MulticastEncodingTest, SmallInline) +{ + // three widely separated nodes whose absolute IDs are single-byte varints: the + // inline list ties the delta list on size and wins the tie + MulticastTargetSet targets; + targets.add(2); + targets.add(40); + targets.add(61); + EncodedMulticastTargets enc = round_trip(targets, 64); + EXPECT_EQ(enc.kind(), MulticastTargetEncoding::SMALL_INLINE); + EXPECT_EQ(enc.bytes(), 5u); +} + +TEST(MulticastEncodingTest, SmallInlineSpanningNodeZeroAndMaxNodeID) +{ + const NodeID num_nodes = 1 << 20; + MulticastTargetSet targets; + targets.add(0); + targets.add(num_nodes - 1); + EncodedMulticastTargets enc = round_trip(targets, num_nodes); + EXPECT_EQ(enc.kind(), MulticastTargetEncoding::SMALL_INLINE); + + MulticastTargetSet decoded; + ASSERT_EQ(enc.decode_into(num_nodes, decoded), MulticastDecodeStatus::OK); + EXPECT_EQ(decoded.first_node(), 0); + EXPECT_EQ(decoded.last_node(), num_nodes - 1); + EXPECT_EQ(decoded.size(), 2u); +} + +TEST(MulticastEncodingTest, RangesForLargeContiguousSet) +{ + // plan section 23: a contiguous target set has compact range metadata + MulticastTargetSet targets; + targets.add_range(0, 999); + EncodedMulticastTargets enc = round_trip(targets, 4096); + EXPECT_EQ(enc.kind(), MulticastTargetEncoding::RANGES); + EXPECT_LE(enc.bytes(), 8u) << "1000 targets must not cost more than a few bytes"; + + MulticastTargetSet decoded; + ASSERT_EQ(enc.decode_into(4096, decoded), MulticastDecodeStatus::OK); + EXPECT_EQ(decoded.num_ranges(), 1u) << "must decode back into exactly one run"; + EXPECT_EQ(decoded.size(), 1000u); +} + +TEST(MulticastEncodingTest, RangesForSeveralLargeRuns) +{ + MulticastTargetSet targets; + targets.add_range(0, 99); + targets.add_range(500, 599); + targets.add_range(1000, 1099); + EncodedMulticastTargets enc = round_trip(targets, 4096); + EXPECT_EQ(enc.kind(), MulticastTargetEncoding::RANGES); + EXPECT_LE(enc.bytes(), 16u); + + MulticastTargetSet decoded; + ASSERT_EQ(enc.decode_into(4096, decoded), MulticastDecodeStatus::OK); + EXPECT_EQ(decoded.num_ranges(), 3u); +} + +TEST(MulticastEncodingTest, DeltaListForIrregularSparseSet) +{ + // more entries than SMALL_INLINE will carry, and far too sparse for a bitmap + MulticastTargetSet targets; + for(NodeID i = 0; i < 12; i++) + targets.add(i * 100); + EncodedMulticastTargets enc = round_trip(targets, 4096); + EXPECT_EQ(enc.kind(), MulticastTargetEncoding::DELTA_LIST); + EXPECT_EQ(enc.bytes(), 14u); +} + +TEST(MulticastEncodingTest, DeltaListWhenAbsoluteNodeIDsAreExpensive) +{ + // only four targets, but their absolute IDs need three varint bytes each while the + // deltas need one - the delta list beats the inline list on actual bytes + MulticastTargetSet targets; + targets.add(100000); + targets.add(100050); + targets.add(100100); + targets.add(100150); + EncodedMulticastTargets enc = round_trip(targets, 200000); + EXPECT_EQ(enc.kind(), MulticastTargetEncoding::DELTA_LIST); + EXPECT_LT(size_of(targets, 200000, MulticastTargetEncoding::DELTA_LIST), + size_of(targets, 200000, MulticastTargetEncoding::SMALL_INLINE)); +} + +TEST(MulticastEncodingTest, BitmapForIrregularDenseSet) +{ + MulticastTargetSet targets; + for(NodeID i = 0; i < 200; i++) + if((i % 3) != 0) + targets.add(i); + ASSERT_EQ(targets.size(), 133u); + + EncodedMulticastTargets enc = round_trip(targets, 1024); + EXPECT_EQ(enc.kind(), MulticastTargetEncoding::BITMAP); + EXPECT_EQ(enc.bytes(), 29u); +} + +TEST(MulticastEncodingTest, BitmapForIrregularDenseSetWithHighBase) +{ + MulticastTargetSet targets; + for(NodeID i = 300; i < 500; i++) + if((i % 5) < 3) + targets.add(i); + EncodedMulticastTargets enc = round_trip(targets, 4096); + EXPECT_EQ(enc.kind(), MulticastTargetEncoding::BITMAP); +} + +TEST(MulticastEncodingTest, AllNodes) +{ + const NodeID num_nodes = 1024; + MulticastTargetSet targets; + targets.add_range(0, num_nodes - 1); + EncodedMulticastTargets enc = round_trip(targets, num_nodes); + EXPECT_EQ(enc.kind(), MulticastTargetEncoding::ALL_NODES); + EXPECT_EQ(enc.bytes(), 1u); + EXPECT_EQ(enc.num_targets(), static_cast(num_nodes)); +} + +TEST(MulticastEncodingTest, AllNodesOnASingleNodeMachine) +{ + MulticastTargetSet targets; + targets.add(0); + EncodedMulticastTargets enc = round_trip(targets, 1); + EXPECT_EQ(enc.kind(), MulticastTargetEncoding::ALL_NODES); + EXPECT_EQ(enc.bytes(), 1u); +} + +TEST(MulticastEncodingTest, AllExceptForNearlyEverything) +{ + const NodeID num_nodes = 1024; + MulticastTargetSet targets; + targets.add_range(0, num_nodes - 1); + ASSERT_TRUE(targets.remove(7)); + + EncodedMulticastTargets enc = round_trip(targets, num_nodes); + EXPECT_EQ(enc.kind(), MulticastTargetEncoding::ALL_EXCEPT); + EXPECT_EQ(enc.bytes(), 3u); + EXPECT_EQ(enc.num_targets(), static_cast(num_nodes) - 1); +} + +TEST(MulticastEncodingTest, AllExceptWithSeveralExclusions) +{ + const NodeID num_nodes = 1024; + MulticastTargetSet targets; + targets.add_range(0, num_nodes - 1); + ASSERT_TRUE(targets.remove(0)); + ASSERT_TRUE(targets.remove(3)); + ASSERT_TRUE(targets.remove(900)); + ASSERT_TRUE(targets.remove(num_nodes - 1)); + + EncodedMulticastTargets enc = round_trip(targets, num_nodes); + EXPECT_EQ(enc.kind(), MulticastTargetEncoding::ALL_EXCEPT); + EXPECT_LT(enc.bytes(), size_of(targets, num_nodes, MulticastTargetEncoding::RANGES)); +} + +TEST(MulticastEncodingTest, EncodedSizeReportsUnrepresentableKindsAsZero) +{ + MulticastTargetSet empty; + EXPECT_EQ(size_of(empty, 64, MulticastTargetEncoding::EMPTY), 1u); + EXPECT_EQ(size_of(empty, 64, MulticastTargetEncoding::SINGLE), 0u); + EXPECT_EQ(size_of(empty, 64, MulticastTargetEncoding::RANGES), 0u); + EXPECT_EQ(size_of(empty, 64, MulticastTargetEncoding::ALL_EXCEPT), 0u); + + MulticastTargetSet two; + two.add(1); + two.add(9); + EXPECT_EQ(size_of(two, 64, MulticastTargetEncoding::EMPTY), 0u); + EXPECT_EQ(size_of(two, 64, MulticastTargetEncoding::SINGLE), 0u); + EXPECT_EQ(size_of(two, 64, MulticastTargetEncoding::ALL_NODES), 0u); + + // more than MULTICAST_MAX_SMALL_INLINE entries has no inline representation + MulticastTargetSet many; + for(NodeID i = 0; i <= static_cast(MULTICAST_MAX_SMALL_INLINE); i++) + many.add(i * 3); + EXPECT_EQ(size_of(many, 64, MulticastTargetEncoding::SMALL_INLINE), 0u); +} + +TEST(MulticastEncodingTest, EncoderAlwaysPicksTheSmallestRepresentation) +{ + MulticastEncodingTally tally; + std::mt19937 rng(987654321); + const NodeID node_counts[] = {1, 2, 8, 64, 1024, 65536}; + for(size_t nc = 0; nc < sizeof(node_counts) / sizeof(node_counts[0]); nc++) { + const NodeID num_nodes = node_counts[nc]; + for(int trial = 0; trial < 40; trial++) { + MulticastTargetSet targets; + // vary the density wildly so that every encoding gets its turn + const unsigned density = 1 + (rng() % 100); + for(NodeID id = 0; id < num_nodes; id++) + if((rng() % 100) < density) + targets.add(id); + // one long contiguous block occasionally, so RANGES shows up too + if((trial % 5) == 0) + targets.add_range(0, num_nodes / 2); + round_trip(targets, num_nodes); + tally.record(EncodedMulticastTargets::encode(targets, num_nodes).kind()); + } + } + + // this sweep should have exercised every one of the eight encodings + for(size_t i = 0; i < MULTICAST_ENCODING_KINDS; i++) { + MulticastTargetEncoding kind = static_cast(i); + EXPECT_GT(tally.get(kind), 0u) << "never chose " << kind; + } +} + +TEST(MulticastEncodingTest, EncodingTallyRecordsEveryChoice) +{ + MulticastEncodingTally tally; + const NodeID num_nodes = 1024; + + MulticastTargetSet empty; + EncodedMulticastTargets::encode(empty, num_nodes, &tally); + + MulticastTargetSet single; + single.add(3); + EncodedMulticastTargets::encode(single, num_nodes, &tally); + + MulticastTargetSet all; + all.add_range(0, num_nodes - 1); + EncodedMulticastTargets::encode(all, num_nodes, &tally); + EncodedMulticastTargets::encode(all, num_nodes, &tally); + + MulticastTargetSet ranges; + ranges.add_range(0, 511); + EncodedMulticastTargets::encode(ranges, num_nodes, &tally); + + EXPECT_EQ(tally.get(MulticastTargetEncoding::EMPTY), 1u); + EXPECT_EQ(tally.get(MulticastTargetEncoding::SINGLE), 1u); + EXPECT_EQ(tally.get(MulticastTargetEncoding::ALL_NODES), 2u); + EXPECT_EQ(tally.get(MulticastTargetEncoding::RANGES), 1u); + EXPECT_EQ(tally.get(MulticastTargetEncoding::BITMAP), 0u); + EXPECT_EQ(tally.total(), 5u); + + tally.reset(); + EXPECT_EQ(tally.total(), 0u); +} + +//////////////////////////////////////////////////////////////////////// +// +// malformed encodings must be rejected safely (plan sections 21.1 and 22) +// + +TEST(MulticastDecodeTest, EmptyBufferAndUnknownKind) +{ + ByteVec buf; + EXPECT_EQ(decode_bytes(buf, 64), MulticastDecodeStatus::TRUNCATED); + + buf.push_back(static_cast(MULTICAST_ENCODING_KINDS)); + EXPECT_EQ(decode_bytes(buf, 64), MulticastDecodeStatus::UNKNOWN_KIND); + + buf.clear(); + buf.push_back(0xff); + EXPECT_EQ(decode_bytes(buf, 64), MulticastDecodeStatus::UNKNOWN_KIND); +} + +TEST(MulticastDecodeTest, TrailingBytes) +{ + ByteVec buf; + put_kind(buf, MulticastTargetEncoding::EMPTY); + buf.push_back(0x00); + EXPECT_EQ(decode_bytes(buf, 64), MulticastDecodeStatus::TRAILING_BYTES); + + buf.clear(); + put_kind(buf, MulticastTargetEncoding::SINGLE); + put_varint(buf, 5); + buf.push_back(0x00); + EXPECT_EQ(decode_bytes(buf, 64), MulticastDecodeStatus::TRAILING_BYTES); + + buf.clear(); + put_kind(buf, MulticastTargetEncoding::ALL_NODES); + buf.push_back(0x00); + EXPECT_EQ(decode_bytes(buf, 64), MulticastDecodeStatus::TRAILING_BYTES); +} + +TEST(MulticastDecodeTest, TruncatedPayloads) +{ + // SINGLE with no node at all + ByteVec buf; + put_kind(buf, MulticastTargetEncoding::SINGLE); + EXPECT_EQ(decode_bytes(buf, 64), MulticastDecodeStatus::TRUNCATED); + + // SMALL_INLINE whose element varint runs off the end + buf.clear(); + put_kind(buf, MulticastTargetEncoding::SMALL_INLINE); + put_varint(buf, 2); + buf.push_back(0x80); // continuation... + buf.push_back(0x80); // ...and another, then nothing + EXPECT_EQ(decode_bytes(buf, 64), MulticastDecodeStatus::TRUNCATED); + + // RANGES with a start but no length + buf.clear(); + put_kind(buf, MulticastTargetEncoding::RANGES); + put_varint(buf, 1); + put_varint(buf, 3); + buf.push_back(0x80); + EXPECT_EQ(decode_bytes(buf, 64), MulticastDecodeStatus::TRUNCATED); + + // DELTA_LIST whose final delta is cut off + buf.clear(); + put_kind(buf, MulticastTargetEncoding::DELTA_LIST); + put_varint(buf, 3); + put_varint(buf, 1); + put_varint(buf, 1); + buf.push_back(0x80); + EXPECT_EQ(decode_bytes(buf, 64), MulticastDecodeStatus::TRUNCATED); + + // BITMAP with fewer map bytes than the bit length demands + buf.clear(); + put_kind(buf, MulticastTargetEncoding::BITMAP); + put_varint(buf, 0); + put_varint(buf, 16); + buf.push_back(0xff); + EXPECT_EQ(decode_bytes(buf, 64), MulticastDecodeStatus::TRUNCATED); + + // ALL_EXCEPT whose second exclusion is cut off + buf.clear(); + put_kind(buf, MulticastTargetEncoding::ALL_EXCEPT); + put_varint(buf, 2); + put_varint(buf, 1); + buf.push_back(0x80); + EXPECT_EQ(decode_bytes(buf, 64), MulticastDecodeStatus::TRUNCATED); + + // NOTE: a declared count with no payload at all behind it is caught by the length + // guard before any element is read, which is the point of that guard + buf.clear(); + put_kind(buf, MulticastTargetEncoding::ALL_EXCEPT); + put_varint(buf, 1); + EXPECT_EQ(decode_bytes(buf, 64), MulticastDecodeStatus::BAD_CARDINALITY); +} + +TEST(MulticastDecodeTest, AbsurdCardinalityIsRejectedWithoutAllocating) +{ + // NOTE: the point of these is that the decoder must never size an allocation from a + // remote length before checking it against the payload size and the node count + ByteVec buf; + put_kind(buf, MulticastTargetEncoding::SMALL_INLINE); + put_varint(buf, 1000000); + EXPECT_EQ(decode_bytes(buf, 64), MulticastDecodeStatus::BAD_CARDINALITY); + + buf.clear(); + put_kind(buf, MulticastTargetEncoding::SMALL_INLINE); + put_varint(buf, 0); + EXPECT_EQ(decode_bytes(buf, 64), MulticastDecodeStatus::BAD_CARDINALITY); + + // a cardinality that is fine against the node count but cannot fit in the bytes left + buf.clear(); + put_kind(buf, MulticastTargetEncoding::SMALL_INLINE); + put_varint(buf, 4); + put_varint(buf, 1); + put_varint(buf, 2); + EXPECT_EQ(decode_bytes(buf, 64), MulticastDecodeStatus::BAD_CARDINALITY); + + buf.clear(); + put_kind(buf, MulticastTargetEncoding::DELTA_LIST); + put_varint(buf, 4000000000ull); + EXPECT_EQ(decode_bytes(buf, 64), MulticastDecodeStatus::BAD_CARDINALITY); + + buf.clear(); + put_kind(buf, MulticastTargetEncoding::DELTA_LIST); + put_varint(buf, 0); + EXPECT_EQ(decode_bytes(buf, 64), MulticastDecodeStatus::BAD_CARDINALITY); + + buf.clear(); + put_kind(buf, MulticastTargetEncoding::RANGES); + put_varint(buf, 0xffffffffull); + EXPECT_EQ(decode_bytes(buf, 64), MulticastDecodeStatus::BAD_CARDINALITY); + + buf.clear(); + put_kind(buf, MulticastTargetEncoding::RANGES); + put_varint(buf, 0); + EXPECT_EQ(decode_bytes(buf, 64), MulticastDecodeStatus::BAD_CARDINALITY); + + // more runs than could possibly fit in the node space, even though the payload is + // long enough to hold them + buf.clear(); + put_kind(buf, MulticastTargetEncoding::RANGES); + put_varint(buf, 6); + for(int i = 0; i < 6; i++) { + put_varint(buf, static_cast(i)); + put_varint(buf, 1); + } + EXPECT_EQ(decode_bytes(buf, 8), MulticastDecodeStatus::BAD_CARDINALITY); + + // excluding every node would leave nothing behind + buf.clear(); + put_kind(buf, MulticastTargetEncoding::ALL_EXCEPT); + put_varint(buf, 64); + EXPECT_EQ(decode_bytes(buf, 64), MulticastDecodeStatus::BAD_CARDINALITY); + + buf.clear(); + put_kind(buf, MulticastTargetEncoding::ALL_EXCEPT); + put_varint(buf, 0); + EXPECT_EQ(decode_bytes(buf, 64), MulticastDecodeStatus::BAD_CARDINALITY); + + // ALL_NODES only means something on a machine with nodes + buf.clear(); + put_kind(buf, MulticastTargetEncoding::ALL_NODES); + EXPECT_EQ(decode_bytes(buf, 0), MulticastDecodeStatus::BAD_CARDINALITY); +} + +TEST(MulticastDecodeTest, NodesOutsideTheConfiguredNodeCount) +{ + ByteVec buf; + put_kind(buf, MulticastTargetEncoding::SINGLE); + put_varint(buf, 100); + EXPECT_EQ(decode_bytes(buf, 64), MulticastDecodeStatus::NODE_OUT_OF_RANGE); + EXPECT_EQ(decode_bytes(buf, 101), MulticastDecodeStatus::OK); + + buf.clear(); + put_kind(buf, MulticastTargetEncoding::SMALL_INLINE); + put_varint(buf, 2); + put_varint(buf, 1); + put_varint(buf, 200); + EXPECT_EQ(decode_bytes(buf, 64), MulticastDecodeStatus::NODE_OUT_OF_RANGE); + + buf.clear(); + put_kind(buf, MulticastTargetEncoding::RANGES); + put_varint(buf, 1); + put_varint(buf, 100); + put_varint(buf, 1); + EXPECT_EQ(decode_bytes(buf, 64), MulticastDecodeStatus::NODE_OUT_OF_RANGE); + + buf.clear(); + put_kind(buf, MulticastTargetEncoding::DELTA_LIST); + put_varint(buf, 2); + put_varint(buf, 1); + put_varint(buf, 100); + EXPECT_EQ(decode_bytes(buf, 64), MulticastDecodeStatus::NODE_OUT_OF_RANGE); + + buf.clear(); + put_kind(buf, MulticastTargetEncoding::BITMAP); + put_varint(buf, 100); + put_varint(buf, 8); + buf.push_back(0xff); + EXPECT_EQ(decode_bytes(buf, 64), MulticastDecodeStatus::NODE_OUT_OF_RANGE); + + buf.clear(); + put_kind(buf, MulticastTargetEncoding::ALL_EXCEPT); + put_varint(buf, 1); + put_varint(buf, 100); + EXPECT_EQ(decode_bytes(buf, 64), MulticastDecodeStatus::NODE_OUT_OF_RANGE); +} + +TEST(MulticastDecodeTest, RangeOverflow) +{ + // zero length + ByteVec buf; + put_kind(buf, MulticastTargetEncoding::RANGES); + put_varint(buf, 1); + put_varint(buf, 3); + put_varint(buf, 0); + EXPECT_EQ(decode_bytes(buf, 64), MulticastDecodeStatus::RANGE_OVERFLOW); + + // length that walks off the end of the node space + buf.clear(); + put_kind(buf, MulticastTargetEncoding::RANGES); + put_varint(buf, 1); + put_varint(buf, 60); + put_varint(buf, 1000); + EXPECT_EQ(decode_bytes(buf, 64), MulticastDecodeStatus::RANGE_OVERFLOW); + + // a length that would wrap 64-bit arithmetic if it were simply added to the start + buf.clear(); + put_kind(buf, MulticastTargetEncoding::RANGES); + put_varint(buf, 1); + put_varint(buf, 60); + put_varint(buf, 0xfffffffffffffffull); + EXPECT_EQ(decode_bytes(buf, 64), MulticastDecodeStatus::RANGE_OVERFLOW); + + // bitmap whose bit length leaves the node space + buf.clear(); + put_kind(buf, MulticastTargetEncoding::BITMAP); + put_varint(buf, 60); + put_varint(buf, 100); + EXPECT_EQ(decode_bytes(buf, 64), MulticastDecodeStatus::RANGE_OVERFLOW); + + buf.clear(); + put_kind(buf, MulticastTargetEncoding::BITMAP); + put_varint(buf, 0); + put_varint(buf, 0); + EXPECT_EQ(decode_bytes(buf, 64), MulticastDecodeStatus::RANGE_OVERFLOW); +} + +TEST(MulticastDecodeTest, NoncanonicalRanges) +{ + // out of order / overlapping + ByteVec buf; + put_kind(buf, MulticastTargetEncoding::RANGES); + put_varint(buf, 2); + put_varint(buf, 0); + put_varint(buf, 10); + put_varint(buf, 5); + put_varint(buf, 10); + EXPECT_EQ(decode_bytes(buf, 64), MulticastDecodeStatus::NOT_CANONICAL); + + // strictly descending + buf.clear(); + put_kind(buf, MulticastTargetEncoding::RANGES); + put_varint(buf, 2); + put_varint(buf, 40); + put_varint(buf, 4); + put_varint(buf, 10); + put_varint(buf, 4); + EXPECT_EQ(decode_bytes(buf, 64), MulticastDecodeStatus::NOT_CANONICAL); + + // adjacent runs that should have been merged + buf.clear(); + put_kind(buf, MulticastTargetEncoding::RANGES); + put_varint(buf, 2); + put_varint(buf, 0); + put_varint(buf, 5); + put_varint(buf, 5); + put_varint(buf, 5); + EXPECT_EQ(decode_bytes(buf, 64), MulticastDecodeStatus::NOT_CANONICAL); + + // ...but a one-node gap is fine + buf.clear(); + put_kind(buf, MulticastTargetEncoding::RANGES); + put_varint(buf, 2); + put_varint(buf, 0); + put_varint(buf, 5); + put_varint(buf, 6); + put_varint(buf, 5); + MulticastTargetSet decoded; + EXPECT_EQ(decode_bytes(buf, 64, decoded), MulticastDecodeStatus::OK); + EXPECT_EQ(decoded.size(), 10u); + EXPECT_EQ(decoded.num_ranges(), 2u); +} + +TEST(MulticastDecodeTest, NoncanonicalLists) +{ + // unsorted inline list + ByteVec buf; + put_kind(buf, MulticastTargetEncoding::SMALL_INLINE); + put_varint(buf, 3); + put_varint(buf, 5); + put_varint(buf, 4); + put_varint(buf, 9); + EXPECT_EQ(decode_bytes(buf, 64), MulticastDecodeStatus::NOT_CANONICAL); + + // duplicated inline entry + buf.clear(); + put_kind(buf, MulticastTargetEncoding::SMALL_INLINE); + put_varint(buf, 2); + put_varint(buf, 5); + put_varint(buf, 5); + EXPECT_EQ(decode_bytes(buf, 64), MulticastDecodeStatus::NOT_CANONICAL); + + // a zero delta repeats the previous node + buf.clear(); + put_kind(buf, MulticastTargetEncoding::DELTA_LIST); + put_varint(buf, 3); + put_varint(buf, 1); + put_varint(buf, 2); + put_varint(buf, 0); + EXPECT_EQ(decode_bytes(buf, 64), MulticastDecodeStatus::NOT_CANONICAL); + + // ...and so does one in the exclusion list + buf.clear(); + put_kind(buf, MulticastTargetEncoding::ALL_EXCEPT); + put_varint(buf, 2); + put_varint(buf, 1); + put_varint(buf, 0); + EXPECT_EQ(decode_bytes(buf, 64), MulticastDecodeStatus::NOT_CANONICAL); +} + +TEST(MulticastDecodeTest, NoncanonicalBitmaps) +{ + // the base node must be the first set bit + ByteVec buf; + put_kind(buf, MulticastTargetEncoding::BITMAP); + put_varint(buf, 0); + put_varint(buf, 8); + buf.push_back(0xfe); + EXPECT_EQ(decode_bytes(buf, 64), MulticastDecodeStatus::NOT_CANONICAL); + + // the last bit of the declared length must be set + buf.clear(); + put_kind(buf, MulticastTargetEncoding::BITMAP); + put_varint(buf, 0); + put_varint(buf, 8); + buf.push_back(0x01); + EXPECT_EQ(decode_bytes(buf, 64), MulticastDecodeStatus::NOT_CANONICAL); + + // padding bits past the declared length must be zero + buf.clear(); + put_kind(buf, MulticastTargetEncoding::BITMAP); + put_varint(buf, 0); + put_varint(buf, 4); + buf.push_back(0x19); + EXPECT_EQ(decode_bytes(buf, 64), MulticastDecodeStatus::NOT_CANONICAL); + + // ...whereas this one is well formed + buf.clear(); + put_kind(buf, MulticastTargetEncoding::BITMAP); + put_varint(buf, 0); + put_varint(buf, 4); + buf.push_back(0x09); + MulticastTargetSet decoded; + EXPECT_EQ(decode_bytes(buf, 64, decoded), MulticastDecodeStatus::OK); + std::vector expected = {0, 3}; + EXPECT_EQ(expand(decoded), expected); + + // extra map bytes beyond the declared bit length + buf.clear(); + put_kind(buf, MulticastTargetEncoding::BITMAP); + put_varint(buf, 0); + put_varint(buf, 8); + buf.push_back(0xff); + buf.push_back(0x00); + EXPECT_EQ(decode_bytes(buf, 64), MulticastDecodeStatus::TRAILING_BYTES); +} + +TEST(MulticastDecodeTest, NoncanonicalVarints) +{ + // overlong encoding of 5 + ByteVec buf; + put_kind(buf, MulticastTargetEncoding::SINGLE); + buf.push_back(0x85); + buf.push_back(0x00); + EXPECT_EQ(decode_bytes(buf, 64), MulticastDecodeStatus::NOT_CANONICAL); + + // a varint longer than any 64-bit value needs + buf.clear(); + put_kind(buf, MulticastTargetEncoding::SINGLE); + for(size_t i = 0; i < MulticastWire::MAX_VARINT_BYTES; i++) + buf.push_back(0x80); + buf.push_back(0x01); + EXPECT_EQ(decode_bytes(buf, 64), MulticastDecodeStatus::NOT_CANONICAL); + + // a 10-byte varint whose top byte would shift bits out of 64 + buf.clear(); + put_kind(buf, MulticastTargetEncoding::SINGLE); + for(size_t i = 0; i < MulticastWire::MAX_VARINT_BYTES - 1; i++) + buf.push_back(0xff); + buf.push_back(0x7f); + EXPECT_EQ(decode_bytes(buf, 64), MulticastDecodeStatus::NOT_CANONICAL); +} + +TEST(MulticastDecodeTest, FailureLeavesTheTargetSetEmpty) +{ + MulticastTargetSet targets; + targets.add_range(0, 100); + ASSERT_FALSE(targets.empty()); + + ByteVec buf; + put_kind(buf, MulticastTargetEncoding::RANGES); + put_varint(buf, 2); + put_varint(buf, 0); + put_varint(buf, 10); + put_varint(buf, 5); + put_varint(buf, 10); + EXPECT_EQ(decode_bytes(buf, 64, targets), MulticastDecodeStatus::NOT_CANONICAL); + EXPECT_TRUE(targets.empty()); + EXPECT_EQ(targets.size(), 0u); +} + +TEST(MulticastDecodeTest, VarintRoundTrip) +{ + const uint64_t values[] = {0, 1, 127, 128, + 255, 16383, 16384, (1ull << 31) - 1, + 1ull << 31, 1ull << 63, ~uint64_t(0)}; + for(size_t i = 0; i < sizeof(values) / sizeof(values[0]); i++) { + ByteVec buf; + put_varint(buf, values[i]); + EXPECT_EQ(buf.size(), MulticastWire::varint_size(values[i])); + EXPECT_LE(buf.size(), MulticastWire::MAX_VARINT_BYTES); + size_t pos = 0; + uint64_t got = 0; + EXPECT_EQ(MulticastWire::read_varint(buf.data(), buf.size(), pos, got), + MulticastDecodeStatus::OK); + EXPECT_EQ(got, values[i]); + EXPECT_EQ(pos, buf.size()); + } +} + +namespace { + + // A buffer whose last byte sits immediately before an inaccessible page, so that a + // decoder reading even one byte past the end of what it was given faults instead of + // quietly picking up adjacent heap. This is what makes "no out-of-bounds read" + // (plan section 22) an observed property rather than a code-reading claim. + class GuardedBuffer { + public: + GuardedBuffer(void) + { +#ifdef REALM_ON_WINDOWS + SYSTEM_INFO sysinfo; + GetSystemInfo(&sysinfo); + page_size = sysinfo.dwPageSize; + base = static_cast( + VirtualAlloc(nullptr, 2 * page_size, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); + assert(base != nullptr); + // second page is unreadable - anything past the end of the payload faults + DWORD prev_protect = 0; + BOOL ok = VirtualProtect(base + page_size, page_size, PAGE_NOACCESS, &prev_protect); + assert(ok != 0); + (void)ok; +#else + page_size = static_cast(sysconf(_SC_PAGESIZE)); + base = static_cast(mmap(nullptr, 2 * page_size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)); + assert(base != MAP_FAILED); + // second page is unreadable - anything past the end of the payload faults + int ret = mprotect(base + page_size, page_size, PROT_NONE); + assert(ret == 0); + (void)ret; +#endif + } + + ~GuardedBuffer(void) + { +#ifdef REALM_ON_WINDOWS + VirtualFree(base, 0, MEM_RELEASE); +#else + munmap(base, 2 * page_size); +#endif + } + + size_t capacity(void) const { return page_size; } + + // returns a pointer to 'bytes' writable bytes that END exactly at the guard page + unsigned char *place(size_t bytes) + { + assert(bytes <= page_size); + return base + page_size - bytes; + } + + protected: + unsigned char *base = nullptr; + size_t page_size = 0; + }; + +}; // namespace + +TEST(MulticastDecodeTest, FuzzedPayloadsNeverReadPastTheEndOfTheBuffer) +{ + GuardedBuffer guard; + std::mt19937 rng(20250729); + const NodeID node_counts[] = {1, 2, 7, 64, 1024, 1 << 20}; + + size_t num_ok = 0; + std::set statuses_seen; + + for(int trial = 0; trial < 40000; trial++) { + // deliberately favors short payloads (that is where truncation lives) but + // occasionally goes large + size_t len = ((trial % 16) == 0) ? (1 + (rng() % 400)) : (rng() % 24); + unsigned char *buf = guard.place(len); + // bias the kind byte into the valid range half the time so that the deeper + // per-kind validation actually gets exercised + for(size_t i = 0; i < len; i++) + buf[i] = static_cast(rng() & 0xff); + if((len > 0) && ((rng() & 1) != 0)) + buf[0] = static_cast(rng() % MULTICAST_ENCODING_KINDS); + + const NodeID num_nodes = + node_counts[rng() % (sizeof(node_counts) / sizeof(node_counts[0]))]; + + MulticastTargetSet targets; + MulticastDecodeStatus status = + EncodedMulticastTargets::decode(buf, len, num_nodes, targets); + statuses_seen.insert(status); + + if(status != MulticastDecodeStatus::OK) { + // a rejected encoding must leave nothing behind + EXPECT_TRUE(targets.empty()); + EXPECT_EQ(targets.size(), 0u); + continue; + } + + num_ok++; + // an accepted encoding must be canonical, in range, and must re-encode to + // something that decodes back to the same set + ASSERT_TRUE(targets.fits_node_count(num_nodes)); + const std::vector &runs = targets.ranges(); + size_t total = 0; + for(size_t i = 0; i < runs.size(); i++) { + EXPECT_LE(runs[i].first, runs[i].last); + if(i > 0) + EXPECT_GT(runs[i].first, runs[i - 1].last + 1) << "runs not canonical"; + total += runs[i].count(); + } + EXPECT_EQ(total, targets.size()); + + EncodedMulticastTargets re = EncodedMulticastTargets::encode(targets, num_nodes); + MulticastTargetSet round; + ASSERT_EQ(re.decode_into(num_nodes, round), MulticastDecodeStatus::OK); + EXPECT_TRUE(round == targets); + } + + // non-vacuity: random bytes really do get accepted sometimes, and more than one + // rejection reason is reached + EXPECT_GT(num_ok, 0u); + EXPECT_GE(statuses_seen.size(), 5u); +} + +//////////////////////////////////////////////////////////////////////// +// +// partitioning (plan sections 7.3 and 20.1) +// + +namespace { + + void check_partition(const MulticastTargetSet &targets, size_t radix) + { + std::vector slices; + targets.partition(radix, slices); + + if((targets.size() == 0) || (radix == 0)) { + EXPECT_TRUE(slices.empty()); + return; + } + + const size_t expected_slices = std::min(radix, targets.size()); + ASSERT_EQ(slices.size(), expected_slices); + + // nearly-equal cardinality: no two slices differ by more than one node + size_t smallest = targets.size(), largest = 0, sum = 0; + for(size_t i = 0; i < slices.size(); i++) { + EXPECT_GT(slices[i].size(), 0u) << "slice " << i << " is empty"; + smallest = std::min(smallest, slices[i].size()); + largest = std::max(largest, slices[i].size()); + sum += slices[i].size(); + check_canonical(slices[i]); + } + EXPECT_LE(largest - smallest, 1u) + << "radix " << radix << ": slice sizes " << smallest << ".." << largest; + EXPECT_EQ(sum, targets.size()) << "partitions must not overlap"; + + // disjoint, ordered, and their union is exactly the input + MulticastTargetSet united; + for(size_t i = 0; i < slices.size(); i++) { + if(i > 0) + EXPECT_GT(slices[i].first_node(), slices[i - 1].last_node()); + for(MulticastTargetSet::const_iterator it = slices[i].begin(); + it != slices[i].end(); ++it) { + ASSERT_FALSE(united.contains(*it)) << "node " << *it << " appears twice"; + united.add(*it); + } + } + EXPECT_EQ(united, targets); + EXPECT_EQ(united.size(), targets.size()); + } + +}; // namespace + +TEST(MulticastPartitionTest, EmptyAndDegenerateRadix) +{ + MulticastTargetSet empty; + std::vector slices; + empty.partition(4, slices); + EXPECT_TRUE(slices.empty()); + + MulticastTargetSet targets; + targets.add_range(0, 9); + targets.partition(0, slices); + EXPECT_TRUE(slices.empty()); +} + +TEST(MulticastPartitionTest, RadixOneKeepsEverythingTogether) +{ + MulticastTargetSet targets; + targets.add_range(0, 99); + targets.add(500); + std::vector slices; + targets.partition(1, slices); + ASSERT_EQ(slices.size(), 1u); + EXPECT_EQ(slices[0], targets); + check_partition(targets, 1); +} + +TEST(MulticastPartitionTest, RadixLargerThanCardinality) +{ + MulticastTargetSet targets; + targets.add(3); + targets.add(9); + targets.add(27); + std::vector slices; + targets.partition(16, slices); + ASSERT_EQ(slices.size(), 3u); + for(size_t i = 0; i < slices.size(); i++) + EXPECT_EQ(slices[i].size(), 1u); + check_partition(targets, 16); +} + +TEST(MulticastPartitionTest, BalanceAndExactUnionAcrossRadices) +{ + const size_t radices[] = {1, 2, 3, 4, 16, 64, 1000}; + + // a contiguous set, a set of several runs, an irregular sparse set and an irregular + // dense one + std::vector cases; + + MulticastTargetSet contiguous; + contiguous.add_range(0, 99); + cases.push_back(contiguous); + + MulticastTargetSet runs; + runs.add_range(0, 9); + runs.add_range(50, 52); + runs.add_range(200, 287); + cases.push_back(runs); + + MulticastTargetSet sparse; + for(NodeID i = 0; i < 37; i++) + sparse.add(i * 13 + 1); + cases.push_back(sparse); + + MulticastTargetSet dense; + for(NodeID i = 0; i < 300; i++) + if((i % 7) != 3) + dense.add(i); + cases.push_back(dense); + + MulticastTargetSet prime; + prime.add_range(0, 100); // 101 nodes, awkward for most radices + cases.push_back(prime); + + for(size_t c = 0; c < cases.size(); c++) + for(size_t r = 0; r < sizeof(radices) / sizeof(radices[0]); r++) { + SCOPED_TRACE(testing::Message() << "case " << c << " radix " << radices[r]); + check_partition(cases[c], radices[r]); + } +} + +TEST(MulticastPartitionTest, ExtraNodesGoToTheEarliestSlices) +{ + MulticastTargetSet targets; + targets.add_range(0, 99); // 100 nodes into 7 slices -> 15,15,14,14,14,14,14 + std::vector slices; + targets.partition(7, slices); + ASSERT_EQ(slices.size(), 7u); + EXPECT_EQ(slices[0].size(), 15u); + EXPECT_EQ(slices[1].size(), 15u); + for(size_t i = 2; i < 7; i++) + EXPECT_EQ(slices[i].size(), 14u); + check_partition(targets, 7); +} + +TEST(MulticastPartitionTest, LargeRangePartitionsWithoutExpanding) +{ + // plan section 7.2: partition by cardinality without requiring full expansion. One + // run of 50000 nodes cut into 8 slices must stay eight single-run slices, each of + // which still encodes as a handful of RANGES bytes. + const NodeID num_nodes = 1 << 20; + const NodeID base = 1000; + const size_t span = 50000; + MulticastTargetSet targets; + targets.add_range(base, base + static_cast(span) - 1); + ASSERT_EQ(targets.num_ranges(), 1u); + ASSERT_EQ(targets.size(), span); + + std::vector slices; + targets.partition(8, slices); + ASSERT_EQ(slices.size(), 8u); + + size_t total = 0; + NodeID expected_first = base; + for(size_t i = 0; i < slices.size(); i++) { + SCOPED_TRACE(testing::Message() << "slice " << i); + EXPECT_EQ(slices[i].num_ranges(), 1u) << "the run must be cut, never expanded"; + EXPECT_EQ(slices[i].size(), span / 8); + EXPECT_EQ(slices[i].first_node(), expected_first); + expected_first = slices[i].last_node() + 1; + total += slices[i].size(); + + EncodedMulticastTargets enc = EncodedMulticastTargets::encode(slices[i], num_nodes); + EXPECT_EQ(enc.kind(), MulticastTargetEncoding::RANGES); + EXPECT_LE(enc.bytes(), 12u) << "6250 targets must stay compact range metadata"; + + MulticastTargetSet decoded; + ASSERT_EQ(enc.decode_into(num_nodes, decoded), MulticastDecodeStatus::OK); + EXPECT_EQ(decoded, slices[i]); + } + EXPECT_EQ(total, span); + + // and recursing (as a relay would) keeps the slices range-shaped + std::vector subslices; + slices[0].partition(8, subslices); + ASSERT_EQ(subslices.size(), 8u); + for(size_t i = 0; i < subslices.size(); i++) + EXPECT_EQ(subslices[i].num_ranges(), 1u); +} + +TEST(MulticastPartitionTest, PartitionAfterRemovingTheLocalRelay) +{ + // the shape a relay actually produces: it drops itself, then splits the rest + MulticastTargetSet targets; + targets.add_range(0, 63); + const NodeID relay = targets.first_node(); + ASSERT_TRUE(targets.remove(relay)); + EXPECT_FALSE(targets.contains(relay)); + EXPECT_EQ(targets.size(), 63u); + + std::vector slices; + targets.partition(4, slices); + ASSERT_EQ(slices.size(), 4u); + for(size_t i = 0; i < slices.size(); i++) + EXPECT_FALSE(slices[i].contains(relay)); + check_partition(targets, 4); +} + +TEST(MulticastPartitionTest, RandomizedBalanceAndCoverage) +{ + std::mt19937 rng(24680); + const size_t radices[] = {1, 2, 4, 16, 64}; + for(int trial = 0; trial < 24; trial++) { + const NodeID num_nodes = 4096; + MulticastTargetSet targets; + const unsigned density = 1 + (rng() % 100); + for(NodeID id = 0; id < num_nodes; id++) + if((rng() % 100) < density) + targets.add(id); + // occasionally throw in a big contiguous block too + if((trial % 3) == 0) + targets.add_range(1000, 2500); + + for(size_t r = 0; r < sizeof(radices) / sizeof(radices[0]); r++) { + SCOPED_TRACE(testing::Message() << "trial " << trial << " radix " << radices[r]); + check_partition(targets, radices[r]); + } + } +} + +//////////////////////////////////////////////////////////////////////// +// +// Stage 2b: envelope, handler redispatch and bounded-radix forwarding +// (plan sections 7.1, 7.3, 7.4 and 20.1) +// + +namespace { + + //////////////////////////////////////////////////////////////////////// + // + // trace shared by the simulated network and the test message handlers + // + + struct TraceEvent { + enum Kind + { + SEND_ENVELOPE, + SEND_ORIGINAL, + SEND_ACK, + HANDLED, + }; + Kind kind = HANDLED; + NodeID node = 0; // node that performed the action + NodeID peer = 0; // envelope destination for a send, apparent sender for HANDLED + }; + + struct HandledMessage { + NodeID node = 0; // where the handler ran, or -1 if it ran during a drain + NodeID sender = 0; // sender as presented to the handler + int value = 0; + std::vector payload; + }; + + std::vector g_trace; + std::vector g_handled; + // the node the simulated network is currently "running on" + NodeID g_current_node = 0; + // monotonically increasing tick, so that "the completion callback ran after the last + // handler" can be asserted rather than assumed + size_t g_clock = 0; + // tick of the most recent original-message handler invocation + size_t g_last_handled_tick = 0; + + // Nodes whose "runtime shutdown" handler has already run. Such a node has stopped + // making progress, so a send attributed to it afterwards would in production be a + // message that never leaves the node - i.e. a permanently stranded subtree. This is + // exactly what plan section 7.3 step 4's forward-before-deliver rule prevents. Both + // stay empty/zero for every test that does not use McastShutdownMessage. + std::set g_stopped_nodes; + size_t g_sends_from_stopped_nodes = 0; + + size_t next_tick(void) { return ++g_clock; } + + void reset_trace(void) + { + g_trace.clear(); + g_handled.clear(); + g_current_node = 0; + g_clock = 0; + g_last_handled_tick = 0; + g_stopped_nodes.clear(); + g_sends_from_stopped_nodes = 0; + } + + void record_handled(NodeID node, NodeID sender, int value, const void *payload, + size_t payload_size) + { + HandledMessage h; + h.node = node; + h.sender = sender; + h.value = value; + if(payload_size > 0) { + const char *c = static_cast(payload); + h.payload.assign(c, c + payload_size); + } + g_handled.push_back(h); + g_last_handled_tick = next_tick(); + } + + // An ordinary message type with an inline handler, so that local delivery is + // synchronous and therefore observable in the trace relative to the child sends. + struct McastTestMessage { + int value = 0; + + static bool handle_inline(NodeID sender, const McastTestMessage &hdr, + const void *payload, size_t payload_size, + TimeLimit /*work_until*/) + { + record_handled(g_current_node, sender, hdr.value, payload, payload_size); + TraceEvent ev; + ev.kind = TraceEvent::HANDLED; + ev.node = g_current_node; + ev.peer = sender; + g_trace.push_back(ev); + return true; + } + + static void handle_message(NodeID sender, const McastTestMessage &hdr, + const void *payload, size_t payload_size) + { + handle_inline(sender, hdr, payload, payload_size, TimeLimit()); + } + }; + + // A message type with NO inline handler, so that local delivery has to go through + // IncomingMessageManager's deferred queue and its TimeLimit-aware handler path. + struct McastDeferredMessage { + int value = 0; + + static void handle_message(NodeID sender, const McastDeferredMessage &hdr, + const void *payload, size_t payload_size, + TimeLimit /*work_until*/) + { + // this runs during the drain, not at delivery time, so the receiving node is not + // attributable from here - only the apparent sender matters + record_handled(-1, sender, hdr.value, payload, payload_size); + } + }; + + // Stand-in for Realm::RuntimeShutdownMessage (plan section 7.6: "Runtime shutdown is a + // critical forwarding-order test"). Its handler does what + // RuntimeShutdownMessage::handle_message ultimately does - it stops the node - so any + // send this node makes afterwards is recorded as a violation. + // + // The real handler has no handle_inline and therefore runs off the deferred queue, + // which would make "forward before deliver" trivially true. Handling it INLINE here + // is deliberately the stronger model: the handler runs synchronously inside + // MulticastTransport::deliver_local, so the relay's child sends have to have already + // happened by then or the violation counter fires. + struct McastShutdownMessage { + int result_code = 0; + + static bool handle_inline(NodeID sender, const McastShutdownMessage &hdr, + const void *payload, size_t payload_size, + TimeLimit /*work_until*/) + { + record_handled(g_current_node, sender, hdr.result_code, payload, payload_size); + TraceEvent ev; + ev.kind = TraceEvent::HANDLED; + ev.node = g_current_node; + ev.peer = sender; + g_trace.push_back(ev); + // RuntimeShutdownMessage::handle_message asserts the request is not a duplicate, + // so a second delivery to the same node would be fatal in production + EXPECT_EQ(g_stopped_nodes.count(g_current_node), 0u) + << "node " << g_current_node << " was shut down twice"; + g_stopped_nodes.insert(g_current_node); + return true; + } + + static void handle_message(NodeID sender, const McastShutdownMessage &hdr, + const void *payload, size_t payload_size) + { + handle_inline(sender, hdr, payload, payload_size, TimeLimit()); + } + }; + + namespace { + // The multicast layer deliberately has no barrier dependency: it records through the + // abstract MulticastMetricsSink and the barrier layer supplies its own + // implementation (plan section 21.3). These tests therefore carry their OWN sink + // rather than reaching into barrier state - which is also what keeps this file + // compiling independently of whatever the barrier protocol currently looks like. + struct TestCounter { + Realm::atomic value{0}; + void bump(unsigned long long by = 1) { value.fetch_add(by); } + void bump_max(unsigned long long v) + { + unsigned long long cur = value.load(); + while((v > cur) && !value.compare_exchange(cur, v)) { + } + } + unsigned long long get(void) const { return value.load(); } + }; + + struct TestMulticastCounters { + TestCounter multicast_encoding_empty, multicast_encoding_single; + TestCounter multicast_encoding_small_inline, multicast_encoding_ranges; + TestCounter multicast_encoding_delta_list, multicast_encoding_bitmap; + TestCounter multicast_encoding_all_nodes, multicast_encoding_all_except; + TestCounter multicast_first_hops, multicast_max_depth; + }; + + class TestMulticastMetrics : public Realm::MulticastMetricsSink { + public: + TestMulticastMetrics(TestMulticastCounters *_c = nullptr) + : counters(_c) + {} + void set_counters(TestMulticastCounters *_c) { counters = _c; } + + virtual void record_encoding_choice(Realm::MulticastTargetEncoding kind) + { + if(counters == nullptr) + return; + switch(kind) { + case Realm::MulticastTargetEncoding::EMPTY: + counters->multicast_encoding_empty.bump(); + break; + case Realm::MulticastTargetEncoding::SINGLE: + counters->multicast_encoding_single.bump(); + break; + case Realm::MulticastTargetEncoding::SMALL_INLINE: + counters->multicast_encoding_small_inline.bump(); + break; + case Realm::MulticastTargetEncoding::RANGES: + counters->multicast_encoding_ranges.bump(); + break; + case Realm::MulticastTargetEncoding::DELTA_LIST: + counters->multicast_encoding_delta_list.bump(); + break; + case Realm::MulticastTargetEncoding::BITMAP: + counters->multicast_encoding_bitmap.bump(); + break; + case Realm::MulticastTargetEncoding::ALL_NODES: + counters->multicast_encoding_all_nodes.bump(); + break; + case Realm::MulticastTargetEncoding::ALL_EXCEPT: + counters->multicast_encoding_all_except.bump(); + break; + } + } + virtual void record_first_hops(size_t n) + { + if(counters != nullptr) + counters->multicast_first_hops.bump(n); + } + virtual void record_tree_depth(unsigned depth) + { + if(counters != nullptr) + counters->multicast_max_depth.bump_max(depth); + } + + protected: + TestMulticastCounters *counters; + }; + }; // namespace + + class SimMulticastNetwork; + + // The multicast envelope as the simulated network transmits it when fragmentation is + // enabled. This exists so that an oversized envelope can be pushed through the REAL + // reassembly machinery: registering this type also auto-registers + // WrappedWithFragInfo, and IncomingMessageManager reassembles + // those chunks with FragmentedMessage before the (deliberately non-inline) handler + // below hands the whole envelope to MulticastForwarder::forward(). That is exactly + // the sequence ActiveMessage's chunked mode produces on a + // real network, which is what the production transport relies on (plan section 7.5). + struct SimEnvelopeMessage { + MulticastEnvelopeMessage env; + NodeID to = 0; // the simulated node this envelope was addressed to + + static void handle_message(NodeID sender, const SimEnvelopeMessage &hdr, + const void *payload, size_t payload_size, + TimeLimit work_until); + }; + + ActiveMessageHandlerReg mcast_test_message_reg; + ActiveMessageHandlerReg mcast_deferred_message_reg; + ActiveMessageHandlerReg mcast_shutdown_message_reg; + ActiveMessageHandlerReg sim_envelope_message_reg; + + // the simulated network currently under test - the reassembly handler above has no + // other way to find it + SimMulticastNetwork *g_sim = nullptr; + + //////////////////////////////////////////////////////////////////////// + // + // class SimMulticastNetwork + // + + // In-process stand-in for a multi-node network. send_envelope() records the envelope + // in a FIFO instead of touching a backend; run() then replays each queued envelope + // into MulticastForwarder::forward() with the receiving node installed as "the local + // node". Local delivery still goes through a real IncomingMessageManager, so the + // handler-table lookup, inline-handler decision and TimeLimit behavior are the real + // ones rather than a test reimplementation. + class SimMulticastNetwork : public MulticastTransport { + public: + struct QueuedEnvelope { + NodeID from = 0; + NodeID to = 0; + MulticastEnvelopeMessage env; + std::vector payload; + }; + + struct QueuedUnicast { + NodeID from = 0; + NodeID to = 0; + ActiveMessageHandlerTable::MessageID msgid = 0; + std::vector hdr; + std::vector payload; + // an ordinary active-message remote completion requested by the origin - it fires + // once the single target has HANDLED the message + bool has_completion = false; + MulticastCompletionToken completion; + }; + + struct QueuedAck { + NodeID from = 0; + NodeID to = 0; + MulticastAckMessage ack; + }; + + // one fragment of an envelope that exceeded frag_chunk_size + struct QueuedFragment { + NodeID from = 0; + NodeID to = 0; + WrappedWithFragInfo hdr; + std::vector chunk; + }; + + SimMulticastNetwork(NodeID _nodes, size_t _radix) + : crs(nullptr) + , mgr(_nodes, 0 /*dedicated_threads*/, crs) + , metrics(&counters) + , nodes(_nodes) + , fan_radix(_radix) + { + // the transient completion state of plan section 7.5 is per node, and the + // simulation runs every node in one process + for(NodeID i = 0; i < _nodes; i++) + node_completion.emplace_back(new MulticastCompletionState); + g_sim = this; + } + + virtual ~SimMulticastNetwork(void) + { + // an IncomingMessageManager is a BackgroundWorkItem and insists on being shut + // down before it is destroyed + mgr.shutdown(); + if(g_sim == this) + g_sim = nullptr; + } + + // A node that has already run its shutdown handler has stopped making progress - + // anything it tries to transmit afterwards would never actually leave. + static void note_send(NodeID from) + { + if(g_stopped_nodes.count(from) != 0) + g_sends_from_stopped_nodes++; + } + + // --- MulticastTransport --------------------------------------------- + + virtual NodeID my_node_id(void) const { return cur; } + virtual NodeID num_nodes(void) const { return nodes; } + virtual size_t radix(void) const { return fan_radix; } + + virtual void send_envelope(NodeID relay, const MulticastEnvelopeMessage &env, + const void *payload, size_t payload_bytes) + { + note_send(cur); + TraceEvent ev; + ev.kind = TraceEvent::SEND_ENVELOPE; + ev.node = cur; + ev.peer = relay; + g_trace.push_back(ev); + + QueuedEnvelope q; + q.from = cur; + q.to = relay; + q.env = env; + const unsigned char *p = static_cast(payload); + q.payload.assign(p, p + payload_bytes); + all_envelopes.push_back(q); + sends_per_node[cur]++; + if(env.depth > max_depth_seen) + max_depth_seen = env.depth; + + if((frag_chunk_size == 0) || (q.payload.size() <= frag_chunk_size)) { + envelope_queue.push_back(q); + return; + } + + // oversized: hand it to the ordinary fragmentation machinery, exactly the way + // ActiveMessage's chunked mode does + const size_t total = q.payload.size(); + const uint32_t total_chunks = + static_cast((total + frag_chunk_size - 1) / frag_chunk_size); + const uint64_t msg_id = next_frag_msg_id++; + size_t offset = 0; + for(uint32_t chunk_id = 0; chunk_id < total_chunks; chunk_id++) { + const size_t chunk_size = std::min(frag_chunk_size, total - offset); + QueuedFragment f; + f.from = cur; + f.to = relay; + f.hdr.frag_info = {chunk_id, total_chunks, msg_id}; + f.hdr.user.env = env; + f.hdr.user.to = relay; + f.chunk.assign(q.payload.begin() + offset, + q.payload.begin() + offset + chunk_size); + fragment_queue.push_back(f); + offset += chunk_size; + } + num_fragments_sent += total_chunks; + } + + virtual bool can_send_original(size_t /*hdr_size*/, size_t /*payload_size*/) const + { + return allow_unicast_fastpath; + } + + virtual void send_original(NodeID target, ActiveMessageHandlerTable::MessageID msgid, + const void *hdr, size_t hdr_size, const void *payload, + size_t payload_size, + const MulticastCompletionToken *completion) + { + note_send(cur); + TraceEvent ev; + ev.kind = TraceEvent::SEND_ORIGINAL; + ev.node = cur; + ev.peer = target; + g_trace.push_back(ev); + + QueuedUnicast q; + q.from = cur; + q.to = target; + q.msgid = msgid; + const unsigned char *h = static_cast(hdr); + q.hdr.assign(h, h + hdr_size); + if(payload_size > 0) { + const unsigned char *p = static_cast(payload); + q.payload.assign(p, p + payload_size); + } + if(completion != nullptr) { + q.has_completion = true; + q.completion = *completion; + } + unicast_queue.push_back(q); + sends_per_node[cur]++; + num_unicasts++; + } + + virtual void send_ack(NodeID from, NodeID parent, NodeID origin, + uint64_t multicast_id) + { + note_send(from); + TraceEvent ev; + ev.kind = TraceEvent::SEND_ACK; + ev.node = from; + ev.peer = parent; + g_trace.push_back(ev); + + QueuedAck q; + q.from = from; + q.to = parent; + q.ack.multicast_id = multicast_id; + q.ack.origin_node = origin; + ack_queue.push_back(q); + acks_per_node[from]++; + num_acks++; + } + + virtual void deliver_local(NodeID origin, ActiveMessageHandlerTable::MessageID msgid, + const void *hdr, size_t hdr_size, const void *payload, + size_t payload_size, TimeLimit work_until, + const MulticastCompletionToken *completion) + { + delivered_senders[cur] = origin; + num_local_deliveries++; + bool handled = + MulticastForwarder::dispatch_local(&mgr, origin, msgid, hdr, hdr_size, payload, + payload_size, work_until, completion); + if(!handled) + deferred_queued++; + } + + virtual MulticastCompletionState &completion_state(void) + { + assert((cur >= 0) && (cur < nodes)); + return *node_completion[cur]; + } + + // --- driving the simulation ----------------------------------------- + + // issues the origin-side multicast but does NOT deliver anything yet + void start(NodeID origin, const MulticastTargetSet &targets, + ActiveMessageHandlerTable::MessageID msgid, const void *hdr, + size_t hdr_size, const void *payload = nullptr, size_t payload_size = 0, + MulticastCompletionCallback *on_remote_complete = nullptr) + { + cur = origin; + g_current_node = origin; + MulticastForwarder::send(*this, targets, msgid, hdr, hdr_size, payload, + payload_size, TimeLimit(), &metrics, on_remote_complete); + } + + // called by SimEnvelopeMessage::handle_message once a fragmented envelope has been + // reassembled by the real IncomingMessageManager + void receive_reassembled_envelope(NodeID sender, const SimEnvelopeMessage &hdr, + const void *payload, size_t payload_size, + TimeLimit work_until) + { + assert(pending_envelope_handlers > 0); + pending_envelope_handlers--; + num_reassembled++; + cur = hdr.to; + g_current_node = hdr.to; + MulticastForwarder::forward(*this, sender, hdr.env, payload, payload_size, + work_until, &metrics); + } + + bool anything_queued(void) const + { + return (!envelope_queue.empty() || !unicast_queue.empty() || + !fragment_queue.empty() || !ack_queue.empty()); + } + + void run(void) + { + while(anything_queued()) { + if(!envelope_queue.empty()) { + QueuedEnvelope q = envelope_queue.front(); + envelope_queue.pop_front(); + cur = q.to; + g_current_node = q.to; + MulticastForwarder::forward(*this, q.from, q.env, q.payload.data(), + q.payload.size(), TimeLimit(), &metrics); + } else if(!fragment_queue.empty()) { + QueuedFragment q = fragment_queue.front(); + fragment_queue.pop_front(); + cur = q.to; + g_current_node = q.to; + // the real reassembly path: only the final chunk produces a handler call, and + // that handler is not inline, so it needs the ordinary drain below + pending_envelope_handlers++; + bool handled = mgr.add_incoming_message( + q.from, sim_envelope_msgid(), &q.hdr, sizeof(q.hdr), PAYLOAD_COPY, + q.chunk.data(), q.chunk.size(), PAYLOAD_COPY, nullptr, 0, 0, TimeLimit()); + EXPECT_FALSE(handled) << "the envelope handler must not be an inline handler"; + if(q.hdr.frag_info.chunk_id + 1 < q.hdr.frag_info.total_chunks) { + // incomplete - no handler was queued for this chunk + pending_envelope_handlers--; + } else { + drain_envelope_handlers(); + } + } else if(!unicast_queue.empty()) { + QueuedUnicast q = unicast_queue.front(); + unicast_queue.pop_front(); + cur = q.to; + g_current_node = q.to; + // an ordinary unicast presents the node that sent it as the sender + delivered_senders[q.to] = q.from; + num_local_deliveries++; + bool handled = MulticastForwarder::dispatch_local( + &mgr, q.from, q.msgid, q.hdr.data(), q.hdr.size(), + q.payload.empty() ? nullptr : q.payload.data(), q.payload.size(), + TimeLimit(), q.has_completion ? &q.completion : nullptr); + if(!handled) + deferred_queued++; + } else { + QueuedAck q = ack_queue.front(); + ack_queue.pop_front(); + // sampled the moment the acknowledgement is consumed: plan section 7.5 says + // a relay reclaims its state before it acknowledges, so the acking node must + // already be holding nothing + reclaim_log.push_back( + std::make_pair(q.from, node_completion[q.from]->num_pending())); + cur = q.to; + g_current_node = q.to; + MulticastForwarder::handle_ack(*this, q.from, q.ack); + } + } + } + + void multicast(NodeID origin, const MulticastTargetSet &targets, + ActiveMessageHandlerTable::MessageID msgid, const void *hdr, + size_t hdr_size, const void *payload = nullptr, + size_t payload_size = 0, + MulticastCompletionCallback *on_remote_complete = nullptr) + { + start(origin, targets, msgid, hdr, hdr_size, payload, payload_size, + on_remote_complete); + run(); + } + + // runs whatever local deliveries could not be handled inline + void drain_deferred(void) + { + while(deferred_queued > 0) { + size_t before = g_handled.size(); + mgr.do_work(TimeLimit()); + size_t progressed = g_handled.size() - before; + if(progressed == 0) + break; + deferred_queued -= std::min(deferred_queued, progressed); + } + } + + void drain_envelope_handlers(void) + { + while(pending_envelope_handlers > 0) { + size_t before = pending_envelope_handlers; + mgr.do_work(TimeLimit()); + if(pending_envelope_handlers == before) + break; + } + } + + // alternates transmission and deferred handling until nothing is left anywhere, + // which is what a completion-tracked multicast with deferred handlers needs (an + // acknowledgement is only produced once a handler has actually run) + void run_to_quiescence(void) + { + while(true) { + if(anything_queued()) { + run(); + continue; + } + if(deferred_queued > 0) { + size_t before = deferred_queued; + drain_deferred(); + if(deferred_queued == before) + break; // no progress possible + continue; + } + break; + } + } + + size_t max_fan_out(void) const + { + size_t worst = 0; + for(std::map::const_iterator it = sends_per_node.begin(); + it != sends_per_node.end(); ++it) + worst = std::max(worst, it->second); + return worst; + } + + size_t total_sends(void) const + { + size_t total = 0; + for(std::map::const_iterator it = sends_per_node.begin(); + it != sends_per_node.end(); ++it) + total += it->second; + return total; + } + + size_t total_pending_completions(void) const + { + size_t total = 0; + for(size_t i = 0; i < node_completion.size(); i++) + total += node_completion[i]->num_pending(); + return total; + } + + size_t peak_pending_completions(void) const + { + size_t peak = 0; + for(size_t i = 0; i < node_completion.size(); i++) + peak = std::max(peak, node_completion[i]->peak_pending()); + return peak; + } + + static ActiveMessageHandlerTable::MessageID sim_envelope_msgid(void) + { + return activemsg_handler_table + .lookup_message_id>(); + } + + CoreReservationSet crs; + IncomingMessageManager mgr; + TestMulticastCounters counters; + TestMulticastMetrics metrics; + + NodeID nodes; + size_t fan_radix; + NodeID cur = 0; + bool allow_unicast_fastpath = true; + // 0 disables the simulated fragmentation path entirely + size_t frag_chunk_size = 0; + + std::deque envelope_queue; + std::deque unicast_queue; + std::deque fragment_queue; + std::deque ack_queue; + std::vector all_envelopes; + std::vector> node_completion; + std::map sends_per_node; + std::map acks_per_node; + std::map delivered_senders; + std::vector> reclaim_log; + size_t num_unicasts = 0; + size_t num_local_deliveries = 0; + size_t deferred_queued = 0; + size_t num_acks = 0; + size_t num_fragments_sent = 0; + size_t num_reassembled = 0; + size_t pending_envelope_handlers = 0; + uint64_t next_frag_msg_id = 1; + unsigned max_depth_seen = 0; + }; + + /*static*/ void SimEnvelopeMessage::handle_message(NodeID sender, + const SimEnvelopeMessage &hdr, + const void *payload, + size_t payload_size, + TimeLimit work_until) + { + ASSERT_NE(g_sim, nullptr); + g_sim->receive_reassembled_envelope(sender, hdr, payload, payload_size, work_until); + } + + class RecordingFatalReporter : public MulticastFatalReporter { + public: + virtual void report(const MulticastFatalContext &ctx) + { + contexts.push_back(ctx); + descriptions.push_back(ctx.to_string()); + } + + std::vector contexts; + std::vector descriptions; + }; + + //////////////////////////////////////////////////////////////////////// + // + // shared assertions + // + + // no node sends a child envelope after it has delivered the original message locally + // (plan section 7.3 step 4) + void expect_forward_before_deliver(void) + { + std::map handled_at; + for(size_t i = 0; i < g_trace.size(); i++) + if((g_trace[i].kind == TraceEvent::HANDLED) && + (handled_at.count(g_trace[i].node) == 0)) + handled_at[g_trace[i].node] = i; + + for(size_t i = 0; i < g_trace.size(); i++) { + if(g_trace[i].kind == TraceEvent::HANDLED) + continue; + std::map::const_iterator it = handled_at.find(g_trace[i].node); + if(it != handled_at.end()) + EXPECT_LT(i, it->second) + << "node " << g_trace[i].node << " sent to " << g_trace[i].peer + << " only after it had already delivered the message locally"; + } + } + + // the deepest envelope the forwarding recurrence can produce: the origin hands out + // slices of ceil(M/R), a relay removes itself and splits the remaining s-1 + unsigned max_possible_depth(size_t remote_targets, size_t radix) + { + unsigned depth = 0; + size_t s = remote_targets; + while(s > 0) { + s = (s + radix - 1) / radix; + depth++; + s -= 1; + } + return depth; + } + + ActiveMessageHandlerTable::MessageID test_msgid(void) + { + return activemsg_handler_table.lookup_message_id(); + } + + class MulticastForwardTest : public ::testing::Test { + protected: + void SetUp(void) + { + // the handler table is built from a static registration list, so one build is + // enough no matter which suite gets there first + static bool table_built = false; + if(!table_built) { + activemsg_handler_table.construct_handler_table(); + table_built = true; + } + reset_trace(); + } + + void TearDown(void) { set_multicast_fatal_reporter(nullptr); } + }; + + // one multicast, plus every property plan sections 7.3/7.4/23 demand of it + void check_multicast(NodeID num_nodes, size_t radix, NodeID origin, + const MulticastTargetSet &targets) + { + SCOPED_TRACE(testing::Message() << "nodes=" << num_nodes << " radix=" << radix + << " origin=" << origin << " targets=" << targets); + reset_trace(); + SimMulticastNetwork net(num_nodes, radix); + + McastTestMessage hdr; + hdr.value = 0x5EED; + net.multicast(origin, targets, test_msgid(), &hdr, sizeof(hdr)); + + // exactly one delivery per target, and every handler saw the ORIGIN as its sender + std::map per_node; + for(size_t i = 0; i < g_handled.size(); i++) { + EXPECT_EQ(g_handled[i].sender, origin); + EXPECT_EQ(g_handled[i].value, 0x5EED); + per_node[g_handled[i].node]++; + } + EXPECT_EQ(g_handled.size(), targets.size()); + EXPECT_EQ(per_node.size(), targets.size()); + for(MulticastTargetSet::const_iterator it = targets.begin(); it != targets.end(); + ++it) + EXPECT_EQ(per_node[*it], 1) << "node " << *it << " was not delivered exactly once"; + + // origin and relay fan-out are both bounded by R + EXPECT_LE(net.max_fan_out(), radix); + + // one edge per remote target: total tree edges are O(M) + const size_t remote = targets.size() - (targets.contains(origin) ? 1 : 0); + EXPECT_EQ(net.total_sends(), remote); + + EXPECT_LE(net.max_depth_seen, max_possible_depth(remote, radix)); + + expect_forward_before_deliver(); + + // a fire-and-forget multicast keeps no acknowledgement state anywhere, ever + EXPECT_EQ(net.peak_pending_completions(), 0u); + EXPECT_EQ(net.num_acks, 0u); + } + + //////////////////////////////////////////////////////////////////////// + // + // completion helpers (plan section 7.5) + // + + // Records when (and how often) the origin's remote-completion callback ran, and how + // many callback objects are still alive - the forwarding layer takes ownership, so + // 'callbacks_alive' must be back to zero on every path, including the paths where + // the callback is never invoked at all. + struct CompletionProbe { + size_t invocations = 0; + size_t tick = 0; + // handler invocations that had already happened when the callback ran + size_t handled_when_invoked = 0; + int callbacks_alive = 0; + }; + + class ProbeCallable { + public: + explicit ProbeCallable(CompletionProbe *_probe) + : probe(_probe) + { + probe->callbacks_alive++; + } + ProbeCallable(const ProbeCallable &rhs) + : probe(rhs.probe) + { + probe->callbacks_alive++; + } + ~ProbeCallable(void) { probe->callbacks_alive--; } + + void operator()(void) const + { + probe->invocations++; + probe->tick = next_tick(); + probe->handled_when_invoked = g_handled.size(); + } + + private: + ProbeCallable &operator=(const ProbeCallable &) = delete; + CompletionProbe *probe; + }; + + MulticastCompletionCallback *probe_callback(CompletionProbe *probe) + { + return make_multicast_completion(ProbeCallable(probe)); + } + + // Builds an envelope by hand, so that malformed completion metadata can be fed to a + // relay without a real origin-side record being left behind. + struct BuiltEnvelope { + MulticastEnvelopeMessage env; + std::vector payload; + }; + + BuiltEnvelope build_envelope(const MulticastTargetSet &slice, NodeID num_nodes, + NodeID origin, uint64_t multicast_id, unsigned depth, + const McastTestMessage &hdr, uint32_t flags, + const std::vector &completion_meta) + { + BuiltEnvelope built; + EncodedMulticastTargets enc = EncodedMulticastTargets::encode(slice, num_nodes); + + built.env.multicast_id = multicast_id; + built.env.origin_node = origin; + built.env.original_payload_size = 0; + built.env.target_encoding_size = static_cast(enc.bytes()); + built.env.completion_size = static_cast(completion_meta.size()); + built.env.flags = flags; + built.env.depth = depth; + built.env.original_message_id = test_msgid(); + built.env.original_header_size = sizeof(McastTestMessage); + built.env.target_encoding_kind = static_cast(enc.kind()); + + built.payload.insert(built.payload.end(), enc.wire_bytes().begin(), + enc.wire_bytes().end()); + const unsigned char *h = reinterpret_cast(&hdr); + built.payload.insert(built.payload.end(), h, h + sizeof(McastTestMessage)); + built.payload.insert(built.payload.end(), completion_meta.begin(), + completion_meta.end()); + return built; + } + +}; // namespace + +//////////////////////////////////////////////////////////////////////// +// +// empty / singleton / origin membership +// + +TEST_F(MulticastForwardTest, EmptyTargetSetIsSuccessfulNoOp) +{ + SimMulticastNetwork net(8, 4); + MulticastTargetSet targets; + McastTestMessage hdr; + net.multicast(0, targets, test_msgid(), &hdr, sizeof(hdr)); + + EXPECT_TRUE(g_trace.empty()); + EXPECT_TRUE(g_handled.empty()); + EXPECT_EQ(net.total_sends(), 0u); + EXPECT_EQ(net.counters.multicast_first_hops.get(), 0u); +} + +TEST_F(MulticastForwardTest, SingletonRemoteTargetUsesUnicastFastPath) +{ + SimMulticastNetwork net(8, 4); + MulticastTargetSet targets; + targets.add(5); + McastTestMessage hdr; + hdr.value = 77; + net.multicast(0, targets, test_msgid(), &hdr, sizeof(hdr)); + + // no envelope at all - the origin already is the sender the handler must see + EXPECT_EQ(net.all_envelopes.size(), 0u); + EXPECT_EQ(net.num_unicasts, 1u); + ASSERT_EQ(g_handled.size(), 1u); + EXPECT_EQ(g_handled[0].node, 5); + EXPECT_EQ(g_handled[0].sender, 0); + EXPECT_EQ(g_handled[0].value, 77); + EXPECT_EQ(net.counters.multicast_first_hops.get(), 1u); + // the fast path builds no target encoding at all + EXPECT_EQ(net.counters.multicast_encoding_single.get(), 0u); +} + +TEST_F(MulticastForwardTest, SingletonFallsBackToEnvelopeWhenUnicastUnavailable) +{ + // a payload that would need fragmenting cannot ride an untyped unicast, so the + // envelope path has to take over (and the origin is still the apparent sender) + SimMulticastNetwork net(8, 4); + net.allow_unicast_fastpath = false; + MulticastTargetSet targets; + targets.add(5); + McastTestMessage hdr; + hdr.value = 78; + net.multicast(0, targets, test_msgid(), &hdr, sizeof(hdr)); + + EXPECT_EQ(net.all_envelopes.size(), 1u); + EXPECT_EQ(net.num_unicasts, 0u); + ASSERT_EQ(g_handled.size(), 1u); + EXPECT_EQ(g_handled[0].node, 5); + EXPECT_EQ(g_handled[0].sender, 0); + EXPECT_EQ(net.counters.multicast_encoding_single.get(), 1u); +} + +TEST_F(MulticastForwardTest, OriginIsTheOnlyTarget) +{ + SimMulticastNetwork net(8, 4); + MulticastTargetSet targets; + targets.add(3); + McastTestMessage hdr; + hdr.value = 9; + net.multicast(3, targets, test_msgid(), &hdr, sizeof(hdr)); + + EXPECT_EQ(net.total_sends(), 0u); + ASSERT_EQ(g_handled.size(), 1u); + EXPECT_EQ(g_handled[0].node, 3); + EXPECT_EQ(g_handled[0].sender, 3); +} + +TEST_F(MulticastForwardTest, OriginIncludedAndExcluded) +{ + MulticastTargetSet with_origin; + with_origin.add_range(0, 15); + check_multicast(16, 4, 0, with_origin); + + MulticastTargetSet without_origin; + without_origin.add_range(1, 15); + check_multicast(16, 4, 0, without_origin); + + // origin in the middle of the set, and origin entirely outside it + MulticastTargetSet middle; + middle.add_range(0, 15); + check_multicast(16, 4, 7, middle); + + MulticastTargetSet elsewhere; + elsewhere.add_range(8, 15); + check_multicast(16, 4, 2, elsewhere); +} + +//////////////////////////////////////////////////////////////////////// +// +// radix sweeps +// + +TEST_F(MulticastForwardTest, RadixOneDegeneratesToAChain) +{ + MulticastTargetSet targets; + targets.add_range(1, 8); + check_multicast(16, 1, 0, targets); + + // with R == 1 every relay has exactly one child, so the tree is a chain of depth M + reset_trace(); + SimMulticastNetwork net(16, 1); + McastTestMessage hdr; + net.multicast(0, targets, test_msgid(), &hdr, sizeof(hdr)); + EXPECT_EQ(net.max_fan_out(), 1u); + EXPECT_EQ(net.max_depth_seen, 8u); +} + +TEST_F(MulticastForwardTest, RadixTwoFourAndSixteen) +{ + const size_t radices[] = {1, 2, 4, 16, 64}; + for(size_t r = 0; r < sizeof(radices) / sizeof(radices[0]); r++) { + MulticastTargetSet contiguous; + contiguous.add_range(0, 63); + check_multicast(64, radices[r], 0, contiguous); + + MulticastTargetSet sparse; + for(NodeID id = 1; id < 64; id += 3) + sparse.add(id); + check_multicast(64, radices[r], 0, sparse); + + MulticastTargetSet irregular; + irregular.add_range(2, 5); + irregular.add(11); + irregular.add_range(30, 47); + irregular.add(63); + check_multicast(64, radices[r], 17, irregular); + } +} + +TEST_F(MulticastForwardTest, LargeTargetSetKeepsOriginFanOutBounded) +{ + // plan section 19 exit criterion: a source sends at most R first-hop messages for a + // large target set + const size_t radix = 4; + reset_trace(); + SimMulticastNetwork net(1024, radix); + MulticastTargetSet targets; + targets.add_range(0, 1023); + + McastTestMessage hdr; + hdr.value = 4242; + net.multicast(0, targets, test_msgid(), &hdr, sizeof(hdr)); + + EXPECT_EQ(net.sends_per_node[0], radix); + EXPECT_EQ(net.counters.multicast_first_hops.get(), radix); + EXPECT_LE(net.max_fan_out(), radix); + EXPECT_EQ(g_handled.size(), 1024u); + EXPECT_EQ(net.total_sends(), 1023u); + // O(log_R M) depth, not O(M) + EXPECT_LE(net.max_depth_seen, max_possible_depth(1023, radix)); + EXPECT_LE(net.max_depth_seen, 6u); + + std::set seen; + for(size_t i = 0; i < g_handled.size(); i++) { + EXPECT_EQ(g_handled[i].sender, 0); + EXPECT_TRUE(seen.insert(g_handled[i].node).second) + << "node " << g_handled[i].node << " was delivered twice"; + } + EXPECT_EQ(seen.size(), 1024u); +} + +TEST_F(MulticastForwardTest, RandomizedShapesDeliverExactlyOnce) +{ + std::mt19937 rng(1357); + const size_t radices[] = {1, 2, 3, 4, 8}; + for(int trial = 0; trial < 20; trial++) { + const NodeID num_nodes = 96; + MulticastTargetSet targets; + const unsigned density = 5 + (rng() % 95); + for(NodeID id = 0; id < num_nodes; id++) + if((rng() % 100) < density) + targets.add(id); + if(targets.empty()) + targets.add(static_cast(rng() % num_nodes)); + + const NodeID origin = static_cast(rng() % num_nodes); + const size_t radix = radices[rng() % (sizeof(radices) / sizeof(radices[0]))]; + check_multicast(num_nodes, radix, origin, targets); + } +} + +//////////////////////////////////////////////////////////////////////// +// +// ordering, sender preservation, payloads +// + +TEST_F(MulticastForwardTest, RelayForwardsBeforeInvokingLocalHandler) +{ + reset_trace(); + SimMulticastNetwork net(32, 2); + MulticastTargetSet targets; + targets.add_range(0, 31); + McastTestMessage hdr; + net.multicast(0, targets, test_msgid(), &hdr, sizeof(hdr)); + + expect_forward_before_deliver(); + + // and make sure the test is not vacuous: at least one node both forwarded and + // delivered, and its forwards really are earlier in the trace + size_t nodes_that_did_both = 0; + std::map handled_at, last_send_at; + for(size_t i = 0; i < g_trace.size(); i++) { + if(g_trace[i].kind == TraceEvent::HANDLED) + handled_at[g_trace[i].node] = i; + else + last_send_at[g_trace[i].node] = i; + } + for(std::map::const_iterator it = handled_at.begin(); + it != handled_at.end(); ++it) { + std::map::const_iterator s = last_send_at.find(it->first); + if(s == last_send_at.end()) + continue; + nodes_that_did_both++; + EXPECT_LT(s->second, it->second); + } + EXPECT_GE(nodes_that_did_both, 2u); +} + +TEST_F(MulticastForwardTest, OriginalSenderSurvivesMultipleHops) +{ + reset_trace(); + SimMulticastNetwork net(64, 2); + MulticastTargetSet targets; + targets.add_range(1, 63); + const NodeID origin = 0; + McastTestMessage hdr; + hdr.value = 31337; + net.multicast(origin, targets, test_msgid(), &hdr, sizeof(hdr)); + + // the tree really is more than one hop deep + EXPECT_GE(net.max_depth_seen, 2u); + + size_t relayed_by_someone_else = 0; + for(size_t i = 0; i < net.all_envelopes.size(); i++) { + // every envelope, at every depth, still names the ORIGIN + EXPECT_EQ(net.all_envelopes[i].env.origin_node, origin); + if(net.all_envelopes[i].from != origin) { + relayed_by_someone_else++; + // ... even though the node that physically transmitted it is not the origin + EXPECT_NE(net.all_envelopes[i].from, origin); + } + } + EXPECT_GT(relayed_by_someone_else, 0u); + + ASSERT_EQ(g_handled.size(), 63u); + for(size_t i = 0; i < g_handled.size(); i++) { + EXPECT_EQ(g_handled[i].sender, origin) + << "node " << g_handled[i].node << " saw the wrong sender"; + EXPECT_EQ(g_handled[i].value, 31337); + } + + // and the transport-level record agrees for every node that is not the origin + for(std::map::const_iterator it = net.delivered_senders.begin(); + it != net.delivered_senders.end(); ++it) + EXPECT_EQ(it->second, origin); +} + +TEST_F(MulticastForwardTest, HeaderOnlyAndCopiedPayloadMessages) +{ + // header only + { + reset_trace(); + SimMulticastNetwork net(16, 3); + MulticastTargetSet targets; + targets.add_range(0, 15); + McastTestMessage hdr; + hdr.value = 11; + net.multicast(0, targets, test_msgid(), &hdr, sizeof(hdr)); + ASSERT_EQ(g_handled.size(), 16u); + for(size_t i = 0; i < g_handled.size(); i++) { + EXPECT_EQ(g_handled[i].value, 11); + EXPECT_TRUE(g_handled[i].payload.empty()); + } + } + + // simple copied 1-D payload + { + reset_trace(); + SimMulticastNetwork net(16, 3); + MulticastTargetSet targets; + targets.add_range(0, 15); + + std::vector payload(197); + for(size_t i = 0; i < payload.size(); i++) + payload[i] = static_cast((i * 7) & 0xff); + + McastTestMessage hdr; + hdr.value = 12; + net.multicast(0, targets, test_msgid(), &hdr, sizeof(hdr), payload.data(), + payload.size()); + + ASSERT_EQ(g_handled.size(), 16u); + for(size_t i = 0; i < g_handled.size(); i++) { + EXPECT_EQ(g_handled[i].value, 12); + EXPECT_EQ(g_handled[i].payload, payload) + << "node " << g_handled[i].node << " got the wrong payload"; + } + } +} + +TEST_F(MulticastForwardTest, PayloadOutlivesTheCallerBuffer) +{ + // plan section 7.5: the payload is copied into the first-hop envelopes before the + // origin-side call returns, so the caller may reuse its buffer immediately + reset_trace(); + SimMulticastNetwork net(16, 2); + MulticastTargetSet targets; + targets.add_range(1, 15); + + std::vector payload(64, 'A'); + McastTestMessage hdr; + hdr.value = 13; + net.start(0, targets, test_msgid(), &hdr, sizeof(hdr), payload.data(), payload.size()); + + // scribble over the caller's buffer and the caller's header before anything is + // actually delivered + std::fill(payload.begin(), payload.end(), 'Z'); + hdr.value = -1; + + net.run(); + + const std::vector expected(64, 'A'); + ASSERT_EQ(g_handled.size(), 15u); + for(size_t i = 0; i < g_handled.size(); i++) { + EXPECT_EQ(g_handled[i].value, 13); + EXPECT_EQ(g_handled[i].payload, expected); + } +} + +TEST_F(MulticastForwardTest, DeferredHandlerPathIsUsedWhenThereIsNoInlineHandler) +{ + reset_trace(); + SimMulticastNetwork net(16, 2); + MulticastTargetSet targets; + targets.add_range(0, 15); + + McastDeferredMessage hdr; + hdr.value = 4711; + ActiveMessageHandlerTable::MessageID msgid = + activemsg_handler_table.lookup_message_id(); + net.multicast(0, targets, msgid, &hdr, sizeof(hdr)); + + // nothing could be handled inline, so everything is sitting in the queue + EXPECT_EQ(net.num_local_deliveries, 16u); + EXPECT_EQ(net.deferred_queued, 16u); + EXPECT_TRUE(g_handled.empty()); + + net.drain_deferred(); + + EXPECT_EQ(g_handled.size(), 16u); + for(size_t i = 0; i < g_handled.size(); i++) { + EXPECT_EQ(g_handled[i].sender, 0) << "deferred delivery lost the original sender"; + EXPECT_EQ(g_handled[i].value, 4711); + } +} + +//////////////////////////////////////////////////////////////////////// +// +// envelope contents and identity +// + +TEST_F(MulticastForwardTest, EnvelopeCarriesEverythingPlan74Requires) +{ + reset_trace(); + SimMulticastNetwork net(32, 4); + MulticastTargetSet targets; + targets.add_range(1, 31); + + std::vector payload(40, 'p'); + McastTestMessage hdr; + hdr.value = 5150; + net.start(0, targets, test_msgid(), &hdr, sizeof(hdr), payload.data(), payload.size()); + + ASSERT_EQ(net.all_envelopes.size(), 4u); + const uint64_t id = net.all_envelopes[0].env.multicast_id; + EXPECT_NE(id, 0u); + for(size_t i = 0; i < net.all_envelopes.size(); i++) { + const MulticastEnvelopeMessage &env = net.all_envelopes[i].env; + EXPECT_EQ(env.origin_node, 0); + EXPECT_EQ(env.multicast_id, id) << "one multicast must use one ID"; + EXPECT_EQ(env.original_message_id, test_msgid()); + EXPECT_EQ(env.original_header_size, sizeof(McastTestMessage)); + EXPECT_EQ(env.original_payload_size, payload.size()); + EXPECT_EQ(env.completion_size, 0u); + EXPECT_EQ(env.flags, 0u); + EXPECT_EQ(env.depth, 1u); + EXPECT_GT(env.target_encoding_size, 0u); + EXPECT_EQ(net.all_envelopes[i].payload.size(), env.target_encoding_size + + env.original_header_size + + env.original_payload_size); + // the kind byte in the header agrees with the encoded slice that follows + EXPECT_EQ(net.all_envelopes[i].payload[0], env.target_encoding_kind); + // and the relay is the first node of the slice it was sent + MulticastTargetSet slice; + ASSERT_EQ(EncodedMulticastTargets::decode(net.all_envelopes[i].payload.data(), + env.target_encoding_size, 32, slice), + MulticastDecodeStatus::OK); + EXPECT_EQ(slice.first_node(), net.all_envelopes[i].to); + EXPECT_TRUE(slice.contains(net.all_envelopes[i].to)); + } + + net.run(); + + // a second multicast gets a different ID + reset_trace(); + SimMulticastNetwork net2(32, 4); + net2.start(0, targets, test_msgid(), &hdr, sizeof(hdr)); + ASSERT_FALSE(net2.all_envelopes.empty()); + EXPECT_NE(net2.all_envelopes[0].env.multicast_id, id); + net2.run(); +} + +TEST_F(MulticastForwardTest, EnvelopeHandlerIsRegisteredAndDeliberatelyNotInline) +{ + ActiveMessageHandlerTable::MessageID id = + activemsg_handler_table.lookup_message_id(); + ActiveMessageHandlerTable::HandlerEntry *entry = + activemsg_handler_table.lookup_message_handler(id); + ASSERT_NE(entry, nullptr); + EXPECT_NE(entry->handler, nullptr); + // plan section 22: child sends must not be issued recursively out of an inline + // handler, so the envelope type deliberately provides no inline handler + EXPECT_EQ(entry->handler_inline, nullptr); + + // the fragmentation wrapper is auto-registered too, so an oversized envelope is + // chunked and reassembled by the existing machinery on every hop + ActiveMessageHandlerTable::MessageID wrapped_id = + activemsg_handler_table + .lookup_message_id>(); + EXPECT_NE(wrapped_id, id); + ActiveMessageHandlerTable::HandlerEntry *wrapped = + activemsg_handler_table.lookup_message_handler(wrapped_id); + ASSERT_NE(wrapped, nullptr); + EXPECT_NE(wrapped->handler, nullptr); + + // the envelope itself carries no FragmentInfo of its own + EXPECT_FALSE(is_wrapped_with_frag_info::value); + EXPECT_FALSE(entry->extract_frag_info.has_value()); +} + +TEST_F(MulticastForwardTest, ChildSlicesAreDisjointAndCoverTheParent) +{ + reset_trace(); + SimMulticastNetwork net(64, 3); + MulticastTargetSet targets; + targets.add_range(0, 63); + McastTestMessage hdr; + net.multicast(0, targets, test_msgid(), &hdr, sizeof(hdr)); + + // union of every slice ever transmitted, counted with multiplicity: each node must + // appear in exactly one slice per tree level it is part of, and in particular each + // node is the relay of exactly one envelope + std::map relay_count; + for(size_t i = 0; i < net.all_envelopes.size(); i++) + relay_count[net.all_envelopes[i].to]++; + EXPECT_EQ(relay_count.size(), 63u); + for(std::map::const_iterator it = relay_count.begin(); + it != relay_count.end(); ++it) + EXPECT_EQ(it->second, 1) << "node " << it->first << " received two envelopes"; + + // siblings at the first hop are disjoint + std::set first_hop_union; + size_t first_hop_total = 0; + for(size_t i = 0; i < net.all_envelopes.size(); i++) { + if(net.all_envelopes[i].env.depth != 1) + continue; + MulticastTargetSet slice; + ASSERT_EQ(EncodedMulticastTargets::decode( + net.all_envelopes[i].payload.data(), + net.all_envelopes[i].env.target_encoding_size, 64, slice), + MulticastDecodeStatus::OK); + first_hop_total += slice.size(); + for(MulticastTargetSet::const_iterator it = slice.begin(); it != slice.end(); ++it) + EXPECT_TRUE(first_hop_union.insert(*it).second) + << "node " << *it << " appears in two sibling slices"; + } + EXPECT_EQ(first_hop_total, 63u); + EXPECT_EQ(first_hop_union.size(), 63u); +} + +//////////////////////////////////////////////////////////////////////// +// +// fatal diagnostics (plan section 21.1) +// + +namespace { + + // runs one multicast far enough to capture a real first-hop envelope + struct CapturedEnvelope { + MulticastEnvelopeMessage env; + std::vector payload; + NodeID from = 0; + NodeID to = 0; + }; + + CapturedEnvelope capture_first_hop(SimMulticastNetwork &net, NodeID origin, + const MulticastTargetSet &targets) + { + McastTestMessage hdr; + hdr.value = 1; + net.start(origin, targets, test_msgid(), &hdr, sizeof(hdr)); + CapturedEnvelope captured; + captured.env = net.all_envelopes.at(0).env; + captured.payload = net.all_envelopes.at(0).payload; + captured.from = net.all_envelopes.at(0).from; + captured.to = net.all_envelopes.at(0).to; + net.envelope_queue.clear(); + net.unicast_queue.clear(); + return captured; + } + +}; // namespace + +TEST_F(MulticastForwardTest, MalformedTargetEncodingIsReportedAndDropped) +{ + SimMulticastNetwork net(32, 2); + MulticastTargetSet targets; + targets.add_range(1, 31); + CapturedEnvelope captured = capture_first_hop(net, 0, targets); + + RecordingFatalReporter reporter; + set_multicast_fatal_reporter(&reporter); + + // an impossible kind byte + std::vector corrupt = captured.payload; + corrupt[0] = 200; + MulticastEnvelopeMessage env = captured.env; + env.target_encoding_kind = 200; + + reset_trace(); + net.cur = captured.to; + g_current_node = captured.to; + MulticastForwarder::forward(net, captured.from, env, corrupt.data(), corrupt.size(), + TimeLimit()); + + ASSERT_EQ(reporter.contexts.size(), 1u); + EXPECT_EQ(reporter.contexts[0].status, MulticastDecodeStatus::UNKNOWN_KIND); + EXPECT_STREQ(reporter.contexts[0].rule, "malformed multicast target encoding"); + EXPECT_EQ(reporter.contexts[0].origin_node, 0); + EXPECT_EQ(reporter.contexts[0].sender, captured.from); + EXPECT_EQ(reporter.contexts[0].local_node, captured.to); + EXPECT_NE(reporter.descriptions[0].find("malformed multicast target encoding"), + std::string::npos); + // nothing was delivered and nothing was forwarded + EXPECT_TRUE(g_handled.empty()); + EXPECT_TRUE(net.envelope_queue.empty()); +} + +TEST_F(MulticastForwardTest, TruncatedTargetEncodingIsReportedAndDropped) +{ + SimMulticastNetwork net(32, 2); + MulticastTargetSet targets; + targets.add_range(1, 31); + CapturedEnvelope captured = capture_first_hop(net, 0, targets); + + RecordingFatalReporter reporter; + set_multicast_fatal_reporter(&reporter); + + // claim one more encoded byte than the payload actually holds + MulticastEnvelopeMessage env = captured.env; + env.target_encoding_size += 1; + + reset_trace(); + net.cur = captured.to; + MulticastForwarder::forward(net, captured.from, env, captured.payload.data(), + captured.payload.size(), TimeLimit()); + + ASSERT_EQ(reporter.contexts.size(), 1u); + EXPECT_STREQ(reporter.contexts[0].rule, + "multicast envelope length fields do not match the received payload"); + EXPECT_TRUE(g_handled.empty()); +} + +TEST_F(MulticastForwardTest, EncodingKindDisagreementIsReportedAndDropped) +{ + SimMulticastNetwork net(32, 2); + MulticastTargetSet targets; + targets.add_range(1, 31); + CapturedEnvelope captured = capture_first_hop(net, 0, targets); + + RecordingFatalReporter reporter; + set_multicast_fatal_reporter(&reporter); + + MulticastEnvelopeMessage env = captured.env; + env.target_encoding_kind = + static_cast((captured.env.target_encoding_kind + 1) % 8); + + reset_trace(); + net.cur = captured.to; + MulticastForwarder::forward(net, captured.from, env, captured.payload.data(), + captured.payload.size(), TimeLimit()); + + ASSERT_EQ(reporter.contexts.size(), 1u); + EXPECT_STREQ(reporter.contexts[0].rule, + "multicast envelope encoding kind disagrees with its payload"); + EXPECT_TRUE(g_handled.empty()); +} + +TEST_F(MulticastForwardTest, RelayMissingFromItsOwnSliceIsReportedAndDropped) +{ + SimMulticastNetwork net(32, 2); + MulticastTargetSet targets; + targets.add_range(1, 15); + CapturedEnvelope captured = capture_first_hop(net, 0, targets); + + RecordingFatalReporter reporter; + set_multicast_fatal_reporter(&reporter); + + // deliver a perfectly well-formed envelope to a node that is not in its slice + reset_trace(); + net.cur = 31; + g_current_node = 31; + MulticastForwarder::forward(net, captured.from, captured.env, captured.payload.data(), + captured.payload.size(), TimeLimit()); + + ASSERT_EQ(reporter.contexts.size(), 1u); + EXPECT_STREQ(reporter.contexts[0].rule, + "multicast relay is not a member of the slice it was sent"); + EXPECT_EQ(reporter.contexts[0].local_node, 31); + EXPECT_TRUE(g_handled.empty()); + EXPECT_TRUE(net.envelope_queue.empty()); +} + +TEST_F(MulticastForwardTest, TargetOutsideTheConfiguredNodeCountIsReported) +{ + SimMulticastNetwork net(8, 2); + RecordingFatalReporter reporter; + set_multicast_fatal_reporter(&reporter); + + MulticastTargetSet targets; + targets.add(1); + targets.add(99); // >= num_nodes + + McastTestMessage hdr; + net.multicast(0, targets, test_msgid(), &hdr, sizeof(hdr)); + + ASSERT_EQ(reporter.contexts.size(), 1u); + EXPECT_EQ(reporter.contexts[0].status, MulticastDecodeStatus::NODE_OUT_OF_RANGE); + EXPECT_TRUE(g_handled.empty()); + EXPECT_EQ(net.total_sends(), 0u); +} + +//////////////////////////////////////////////////////////////////////// +// +// metrics (plan section 21.3) +// + +TEST_F(MulticastForwardTest, CountersRecordFanOutDepthAndEncodingChoices) +{ + reset_trace(); + const size_t radix = 4; + SimMulticastNetwork net(64, radix); + MulticastTargetSet targets; + targets.add_range(1, 63); + McastTestMessage hdr; + net.multicast(0, targets, test_msgid(), &hdr, sizeof(hdr)); + + const TestMulticastCounters &c = net.counters; + + // one first-hop count per origin-side send, bounded by R (plan section 23) + EXPECT_EQ(c.multicast_first_hops.get(), radix); + + // one encoding choice per envelope + const uint64_t encodings = + c.multicast_encoding_empty.get() + c.multicast_encoding_single.get() + + c.multicast_encoding_small_inline.get() + c.multicast_encoding_ranges.get() + + c.multicast_encoding_delta_list.get() + c.multicast_encoding_bitmap.get() + + c.multicast_encoding_all_nodes.get() + c.multicast_encoding_all_except.get(); + EXPECT_EQ(encodings, net.all_envelopes.size()); + EXPECT_EQ(encodings, 63u); + + // a contiguous target set uses the compact range encoding for its big slices + EXPECT_GE(c.multicast_encoding_ranges.get(), radix); + // and the leaves are single-node slices + EXPECT_GT(c.multicast_encoding_single.get(), 0u); + + // depth is a gauge, and matches what the transport observed + EXPECT_EQ(c.multicast_max_depth.get(), net.max_depth_seen); + EXPECT_GE(c.multicast_max_depth.get(), 2u); + EXPECT_LE(c.multicast_max_depth.get(), max_possible_depth(63, radix)); +} + +// NOTE: a test named BarrierMulticastMetricsAdapterWiring used to live here. It covered +// BarrierMulticastMetrics, the BARRIER-side implementation of MulticastMetricsSink, and +// went with the barrier protocol when that was reverted. The multicast layer's own side +// of that contract - that it records an encoding choice per envelope, a first-hop count +// per origin send, and a max tree depth - is covered by the counter assertions in +// AcceptanceCriteriaScalingTable and friends, using this file's TestMulticastMetrics. + +//////////////////////////////////////////////////////////////////////// +// +// Stage 2c: payload, fragmentation and transient completion semantics +// (plan sections 7.5, 20.1) +// + +namespace { + + // exact payload bytes each target's handler saw, keyed by node + std::map> payload_per_node(void) + { + std::map> result; + for(size_t i = 0; i < g_handled.size(); i++) + result[g_handled[i].node] = g_handled[i].payload; + return result; + } + + size_t count_trace(TraceEvent::Kind kind) + { + size_t n = 0; + for(size_t i = 0; i < g_trace.size(); i++) + if(g_trace[i].kind == kind) + n++; + return n; + } + +}; // namespace + +//////////////////////////////////////////////////////////////////////// +// +// payload semantics +// + +TEST_F(MulticastForwardTest, PayloadBytesRoundTripAtEveryTarget) +{ + reset_trace(); + SimMulticastNetwork net(24, 3); + MulticastTargetSet targets; + targets.add_range(0, 23); + + std::vector payload(613); + for(size_t i = 0; i < payload.size(); i++) + payload[i] = static_cast((i * 31 + 7) & 0xff); + + McastTestMessage hdr; + hdr.value = 0x2C2C; + net.multicast(0, targets, test_msgid(), &hdr, sizeof(hdr), payload.data(), + payload.size()); + + ASSERT_EQ(g_handled.size(), 24u); + std::map> seen = payload_per_node(); + ASSERT_EQ(seen.size(), 24u); + for(NodeID n = 0; n < 24; n++) { + ASSERT_EQ(seen.count(n), 1u) << "node " << n << " was never delivered"; + EXPECT_EQ(seen[n], payload) << "node " << n << " got the wrong payload bytes"; + } +} + +TEST_F(MulticastForwardTest, HeaderOnlyMulticastCarriesNoPayloadAnywhere) +{ + reset_trace(); + SimMulticastNetwork net(24, 3); + MulticastTargetSet targets; + targets.add_range(0, 23); + + McastTestMessage hdr; + hdr.value = 0x1234; + net.multicast(0, targets, test_msgid(), &hdr, sizeof(hdr)); + + ASSERT_EQ(g_handled.size(), 24u); + for(size_t i = 0; i < g_handled.size(); i++) { + EXPECT_EQ(g_handled[i].value, 0x1234); + EXPECT_TRUE(g_handled[i].payload.empty()); + } + for(size_t i = 0; i < net.all_envelopes.size(); i++) + EXPECT_EQ(net.all_envelopes[i].env.original_payload_size, 0u); +} + +TEST_F(MulticastForwardTest, PayloadOutlivesTheCallerBufferOnTheUnicastFastPath) +{ + // the singleton fast path copies too - the caller's PAYLOAD_KEEP guarantee is that + // the buffer is free the moment the origin-side call returns (plan section 7.5) + reset_trace(); + SimMulticastNetwork net(8, 4); + MulticastTargetSet targets; + targets.add(3); + + std::vector payload(37, 'k'); + McastTestMessage hdr; + hdr.value = 55; + net.start(0, targets, test_msgid(), &hdr, sizeof(hdr), payload.data(), payload.size()); + + EXPECT_EQ(net.num_unicasts, 1u); + EXPECT_TRUE(net.all_envelopes.empty()); + + std::fill(payload.begin(), payload.end(), 'X'); + hdr.value = -7; + + net.run(); + + const std::vector expected(37, 'k'); + ASSERT_EQ(g_handled.size(), 1u); + EXPECT_EQ(g_handled[0].node, 3); + EXPECT_EQ(g_handled[0].sender, 0); + EXPECT_EQ(g_handled[0].value, 55); + EXPECT_EQ(g_handled[0].payload, expected); +} + +//////////////////////////////////////////////////////////////////////// +// +// fragmentation (plan section 7.5) +// + +TEST_F(MulticastForwardTest, OversizedEnvelopesAreFragmentedAndReassembledOnEveryHop) +{ + reset_trace(); + SimMulticastNetwork net(16, 3); + // far below the envelope size, so every hop has to fragment + net.frag_chunk_size = 64; + + MulticastTargetSet targets; + targets.add_range(0, 15); + + std::vector payload(1000); + for(size_t i = 0; i < payload.size(); i++) + payload[i] = static_cast((i * 13 + 5) & 0xff); + + McastTestMessage hdr; + hdr.value = 0xF00D; + net.multicast(0, targets, test_msgid(), &hdr, sizeof(hdr), payload.data(), + payload.size()); + + // every envelope really did go through the fragmentation machinery, and the whole + // envelope was reassembled before the relay repartitioned it + ASSERT_EQ(net.all_envelopes.size(), 15u); + EXPECT_EQ(net.num_reassembled, 15u); + EXPECT_GE(net.num_fragments_sent, 15u * 16u); + + // and the payload survived intact at every one of the 16 targets + ASSERT_EQ(g_handled.size(), 16u); + std::map> seen = payload_per_node(); + ASSERT_EQ(seen.size(), 16u); + for(NodeID n = 0; n < 16; n++) { + ASSERT_EQ(seen.count(n), 1u) << "node " << n << " was never delivered"; + EXPECT_EQ(seen[n], payload) << "node " << n << " got a corrupted payload"; + } + for(size_t i = 0; i < g_handled.size(); i++) + EXPECT_EQ(g_handled[i].sender, 0) << "fragmentation lost the original sender"; + + // the shape of the tree is unchanged by fragmentation + EXPECT_LE(net.max_fan_out(), 3u); + EXPECT_EQ(net.total_sends(), 15u); + expect_forward_before_deliver(); +} + +TEST_F(MulticastForwardTest, FragmentedAndUnfragmentedDeliveriesAgree) +{ + std::vector payload(700); + for(size_t i = 0; i < payload.size(); i++) + payload[i] = static_cast((i * 17 + 3) & 0xff); + + McastTestMessage hdr; + hdr.value = 4242; + + std::map> whole; + { + reset_trace(); + SimMulticastNetwork net(12, 4); + MulticastTargetSet targets; + targets.add_range(0, 11); + net.multicast(0, targets, test_msgid(), &hdr, sizeof(hdr), payload.data(), + payload.size()); + ASSERT_EQ(g_handled.size(), 12u); + EXPECT_EQ(net.num_reassembled, 0u); + whole = payload_per_node(); + } + + std::map> fragmented; + { + reset_trace(); + SimMulticastNetwork net(12, 4); + net.frag_chunk_size = 100; + MulticastTargetSet targets; + targets.add_range(0, 11); + net.multicast(0, targets, test_msgid(), &hdr, sizeof(hdr), payload.data(), + payload.size()); + ASSERT_EQ(g_handled.size(), 12u); + EXPECT_GT(net.num_reassembled, 0u); + fragmented = payload_per_node(); + } + + EXPECT_EQ(whole, fragmented); + EXPECT_EQ(whole.size(), 12u); +} + +//////////////////////////////////////////////////////////////////////// +// +// aggregate remote completion (plan section 7.5) +// + +TEST_F(MulticastForwardTest, FireAndForgetKeepsNoAcknowledgementStateOrMetadata) +{ + reset_trace(); + SimMulticastNetwork net(32, 4); + MulticastTargetSet targets; + targets.add_range(0, 31); + McastTestMessage hdr; + hdr.value = 9; + net.multicast(0, targets, test_msgid(), &hdr, sizeof(hdr)); + + ASSERT_EQ(g_handled.size(), 32u); + + // no acknowledgement traffic, no acknowledgement records - not even transiently + EXPECT_EQ(net.num_acks, 0u); + EXPECT_EQ(count_trace(TraceEvent::SEND_ACK), 0u); + EXPECT_EQ(net.peak_pending_completions(), 0u); + EXPECT_EQ(net.total_pending_completions(), 0u); + + // and no acknowledgement metadata on the wire either + ASSERT_EQ(net.all_envelopes.size(), 31u); + for(size_t i = 0; i < net.all_envelopes.size(); i++) { + const MulticastEnvelopeMessage &env = net.all_envelopes[i].env; + EXPECT_EQ(env.flags, 0u); + EXPECT_EQ(env.completion_size, 0u); + EXPECT_EQ(net.all_envelopes[i].payload.size(), env.target_encoding_size + + env.original_header_size + + env.original_payload_size); + } +} + +TEST_F(MulticastForwardTest, RemoteCompletionFiresExactlyOnceAfterEveryTargetHandled) +{ + reset_trace(); + SimMulticastNetwork net(32, 4); + MulticastTargetSet targets; + targets.add_range(0, 31); + + CompletionProbe probe; + McastTestMessage hdr; + hdr.value = 77; + net.start(0, targets, test_msgid(), &hdr, sizeof(hdr), nullptr, 0, + probe_callback(&probe)); + + // the origin retains the callback state under the multicast ID while the tree is + // still in flight, and has not fired anything yet + EXPECT_EQ(net.total_pending_completions(), 1u); + EXPECT_EQ(probe.invocations, 0u); + + net.run_to_quiescence(); + + EXPECT_EQ(g_handled.size(), 32u); + ASSERT_EQ(probe.invocations, 1u) << "the origin callback must run exactly once"; + EXPECT_EQ(probe.callbacks_alive, 0) << "the completion callback object was leaked"; + EXPECT_EQ(probe.handled_when_invoked, 32u) + << "the callback ran before every target had handled the message"; + EXPECT_GT(probe.tick, g_last_handled_tick); + + // one acknowledgement per relay, and never two from the same node + EXPECT_EQ(net.num_acks, net.all_envelopes.size()); + EXPECT_EQ(net.num_acks, 31u); + for(std::map::const_iterator it = net.acks_per_node.begin(); + it != net.acks_per_node.end(); ++it) + EXPECT_EQ(it->second, 1u) << "node " << it->first << " acknowledged twice"; + + // every acknowledgement went to the node that had sent that relay its envelope + std::map parent_of; + for(size_t i = 0; i < net.all_envelopes.size(); i++) + parent_of[net.all_envelopes[i].to] = net.all_envelopes[i].from; + size_t acks_seen = 0; + for(size_t i = 0; i < g_trace.size(); i++) { + if(g_trace[i].kind != TraceEvent::SEND_ACK) + continue; + acks_seen++; + ASSERT_EQ(parent_of.count(g_trace[i].node), 1u); + EXPECT_EQ(g_trace[i].peer, parent_of[g_trace[i].node]) + << "node " << g_trace[i].node << " acknowledged the wrong parent"; + } + EXPECT_EQ(acks_seen, 31u); + + // every acking node had already reclaimed its record when its parent saw the ack + ASSERT_EQ(net.reclaim_log.size(), 31u); + for(size_t i = 0; i < net.reclaim_log.size(); i++) + EXPECT_EQ(net.reclaim_log[i].second, 0u) + << "node " << net.reclaim_log[i].first + << " still held state after acknowledging its parent"; + + // ... and nothing at all is left anywhere + EXPECT_GT(net.peak_pending_completions(), 0u); + EXPECT_EQ(net.total_pending_completions(), 0u); +} + +TEST_F(MulticastForwardTest, CompletionTrackedEnvelopesNameTheNodeToAcknowledge) +{ + reset_trace(); + SimMulticastNetwork net(32, 4); + MulticastTargetSet targets; + targets.add_range(1, 31); + + CompletionProbe probe; + McastTestMessage hdr; + net.start(0, targets, test_msgid(), &hdr, sizeof(hdr), nullptr, 0, + probe_callback(&probe)); + net.run_to_quiescence(); + ASSERT_EQ(probe.invocations, 1u); + + ASSERT_FALSE(net.all_envelopes.empty()); + for(size_t i = 0; i < net.all_envelopes.size(); i++) { + const SimMulticastNetwork::QueuedEnvelope &q = net.all_envelopes[i]; + EXPECT_NE(q.env.flags & MulticastEnvelopeFlags::COMPLETION_TRACKED, 0u); + ASSERT_GT(q.env.completion_size, 0u); + EXPECT_EQ(q.payload.size(), q.env.target_encoding_size + q.env.original_header_size + + q.env.original_payload_size + q.env.completion_size); + // the metadata is a single varint naming the node this subtree acknowledges to, + // which is always the node that sent the envelope + size_t pos = 0; + uint64_t parent = 0; + const unsigned char *meta = + q.payload.data() + q.payload.size() - q.env.completion_size; + ASSERT_EQ(MulticastWire::read_varint(meta, q.env.completion_size, pos, parent), + MulticastDecodeStatus::OK); + EXPECT_EQ(pos, q.env.completion_size); + EXPECT_EQ(static_cast(parent), q.from); + } +} + +TEST_F(MulticastForwardTest, NoResidualStateAfterACompletionTrackedMulticast) +{ + reset_trace(); + SimMulticastNetwork net(16, 3); + MulticastTargetSet targets; + targets.add_range(0, 15); + + for(int round = 0; round < 3; round++) { + CompletionProbe probe; + McastTestMessage hdr; + hdr.value = round; + net.multicast(0, targets, test_msgid(), &hdr, sizeof(hdr), nullptr, 0, + probe_callback(&probe)); + net.run_to_quiescence(); + EXPECT_EQ(probe.invocations, 1u); + EXPECT_EQ(probe.callbacks_alive, 0); + + // plan section 2 (final bullet) and 7.1: no per-multicast state may survive, on any + // node - not a target plan, not an encoding, not an acknowledgement record + for(size_t n = 0; n < net.node_completion.size(); n++) + EXPECT_EQ(net.node_completion[n]->num_pending(), 0u) + << "node " << n << " retained multicast state after round " << round; + } + EXPECT_EQ(net.total_pending_completions(), 0u); +} + +TEST_F(MulticastForwardTest, RemoteCompletionOnAnEmptyTargetSetFiresImmediately) +{ + reset_trace(); + SimMulticastNetwork net(8, 4); + MulticastTargetSet targets; // empty + + CompletionProbe probe; + McastTestMessage hdr; + net.start(0, targets, test_msgid(), &hdr, sizeof(hdr), nullptr, 0, + probe_callback(&probe)); + + // every one of the zero targets has trivially handled it + EXPECT_EQ(probe.invocations, 1u); + EXPECT_EQ(net.total_pending_completions(), 0u); + EXPECT_EQ(net.peak_pending_completions(), 0u); + EXPECT_TRUE(net.all_envelopes.empty()); + EXPECT_EQ(net.num_unicasts, 0u); + EXPECT_EQ(net.num_acks, 0u); + EXPECT_TRUE(g_handled.empty()); + + net.run(); + EXPECT_EQ(probe.invocations, 1u); + EXPECT_EQ(probe.callbacks_alive, 0); +} + +TEST_F(MulticastForwardTest, RemoteCompletionOnASingletonTargetUsesTheUnicastFastPath) +{ + reset_trace(); + SimMulticastNetwork net(8, 4); + MulticastTargetSet targets; + targets.add(5); + + CompletionProbe probe; + McastTestMessage hdr; + hdr.value = 31337; + net.start(0, targets, test_msgid(), &hdr, sizeof(hdr), nullptr, 0, + probe_callback(&probe)); + + // still the ordinary unicast fast path - no envelope, no acknowledgement message + EXPECT_EQ(net.num_unicasts, 1u); + EXPECT_TRUE(net.all_envelopes.empty()); + EXPECT_EQ(probe.invocations, 0u); + EXPECT_EQ(net.total_pending_completions(), 1u); + + net.run_to_quiescence(); + + ASSERT_EQ(g_handled.size(), 1u); + EXPECT_EQ(g_handled[0].node, 5); + EXPECT_EQ(g_handled[0].sender, 0); + EXPECT_EQ(probe.invocations, 1u); + EXPECT_EQ(probe.handled_when_invoked, 1u); + EXPECT_EQ(net.num_acks, 0u) << "the fast path rides the ordinary remote completion"; + EXPECT_EQ(net.total_pending_completions(), 0u); + EXPECT_EQ(probe.callbacks_alive, 0); +} + +TEST_F(MulticastForwardTest, RemoteCompletionWhenTheOriginIsTheOnlyTarget) +{ + reset_trace(); + SimMulticastNetwork net(8, 4); + MulticastTargetSet targets; + targets.add(0); + + CompletionProbe probe; + McastTestMessage hdr; + net.start(0, targets, test_msgid(), &hdr, sizeof(hdr), nullptr, 0, + probe_callback(&probe)); + + EXPECT_TRUE(net.all_envelopes.empty()); + EXPECT_EQ(net.num_unicasts, 0u); + ASSERT_EQ(g_handled.size(), 1u); + EXPECT_EQ(g_handled[0].node, 0); + // the local handler is inline, so the whole thing is already finished + EXPECT_EQ(probe.invocations, 1u); + EXPECT_EQ(probe.callbacks_alive, 0); + EXPECT_EQ(net.total_pending_completions(), 0u); +} + +TEST_F(MulticastForwardTest, RemoteCompletionWaitsForDeferredHandlers) +{ + reset_trace(); + SimMulticastNetwork net(16, 3); + MulticastTargetSet targets; + targets.add_range(0, 15); + + CompletionProbe probe; + McastDeferredMessage hdr; + hdr.value = 8888; + ActiveMessageHandlerTable::MessageID msgid = + activemsg_handler_table.lookup_message_id(); + net.start(0, targets, msgid, &hdr, sizeof(hdr), nullptr, 0, probe_callback(&probe)); + net.run(); + + // the whole tree has been transmitted, but nothing has been HANDLED yet, so no + // acknowledgement may have been produced and the callback must not have fired + EXPECT_EQ(net.num_local_deliveries, 16u); + EXPECT_EQ(net.deferred_queued, 16u); + EXPECT_TRUE(g_handled.empty()); + EXPECT_EQ(net.num_acks, 0u); + EXPECT_EQ(probe.invocations, 0u); + EXPECT_EQ(net.total_pending_completions(), 16u); + + net.run_to_quiescence(); + + EXPECT_EQ(g_handled.size(), 16u); + for(size_t i = 0; i < g_handled.size(); i++) + EXPECT_EQ(g_handled[i].sender, 0); + ASSERT_EQ(probe.invocations, 1u); + EXPECT_EQ(probe.handled_when_invoked, 16u); + EXPECT_GT(probe.tick, g_last_handled_tick); + EXPECT_EQ(net.num_acks, 15u); + EXPECT_EQ(net.total_pending_completions(), 0u); + EXPECT_EQ(probe.callbacks_alive, 0); +} + +TEST_F(MulticastForwardTest, RemoteCompletionWithAFragmentedPayload) +{ + reset_trace(); + SimMulticastNetwork net(16, 4); + net.frag_chunk_size = 96; + MulticastTargetSet targets; + targets.add_range(0, 15); + + std::vector payload(1500); + for(size_t i = 0; i < payload.size(); i++) + payload[i] = static_cast((i * 5 + 11) & 0xff); + + CompletionProbe probe; + McastTestMessage hdr; + hdr.value = 606; + net.start(0, targets, test_msgid(), &hdr, sizeof(hdr), payload.data(), payload.size(), + probe_callback(&probe)); + net.run_to_quiescence(); + + EXPECT_GT(net.num_reassembled, 0u); + ASSERT_EQ(g_handled.size(), 16u); + std::map> seen = payload_per_node(); + for(NodeID n = 0; n < 16; n++) { + ASSERT_EQ(seen.count(n), 1u); + EXPECT_EQ(seen[n], payload); + } + ASSERT_EQ(probe.invocations, 1u); + EXPECT_EQ(probe.callbacks_alive, 0); + EXPECT_EQ(probe.handled_when_invoked, 16u); + EXPECT_EQ(net.total_pending_completions(), 0u); +} + +TEST_F(MulticastForwardTest, RemoteCompletionAcrossRadicesOriginsAndShapes) +{ + const NodeID num_nodes = 24; + const size_t radices[] = {1, 2, 3, 4, 8, 64}; + const NodeID origins[] = {0, 7, 23}; + + for(size_t r = 0; r < (sizeof(radices) / sizeof(radices[0])); r++) { + for(size_t o = 0; o < (sizeof(origins) / sizeof(origins[0])); o++) { + for(int shape = 0; shape < 4; shape++) { + MulticastTargetSet targets; + switch(shape) { + case 0: + targets.add_range(0, num_nodes - 1); // everything, origin included + break; + case 1: + for(NodeID n = 0; n < num_nodes; n++) // origin excluded + if(n != origins[o]) + targets.add(n); + break; + case 2: + for(NodeID n = 1; n < num_nodes; n += 3) + targets.add(n); + break; + case 3: + targets.add(origins[o] == 11 ? 12 : 11); // singleton + break; + } + SCOPED_TRACE(testing::Message() + << "radix=" << radices[r] << " origin=" << origins[o] + << " shape=" << shape << " targets=" << targets); + reset_trace(); + SimMulticastNetwork net(num_nodes, radices[r]); + CompletionProbe probe; + McastTestMessage hdr; + hdr.value = shape; + net.multicast(origins[o], targets, test_msgid(), &hdr, sizeof(hdr), nullptr, 0, + probe_callback(&probe)); + net.run_to_quiescence(); + + EXPECT_EQ(g_handled.size(), targets.size()); + EXPECT_EQ(probe.invocations, 1u); + EXPECT_EQ(probe.callbacks_alive, 0); + EXPECT_EQ(probe.handled_when_invoked, targets.size()); + EXPECT_EQ(net.total_pending_completions(), 0u); + EXPECT_LE(net.max_fan_out(), radices[r]); + // exactly one acknowledgement per envelope, and never more + EXPECT_EQ(net.num_acks, net.all_envelopes.size()); + for(size_t i = 0; i < net.reclaim_log.size(); i++) + EXPECT_EQ(net.reclaim_log[i].second, 0u); + } + } + } +} + +TEST_F(MulticastForwardTest, RandomizedCompletionTrackedMulticasts) +{ + std::mt19937 rng(20250729); + const NodeID num_nodes = 40; + + for(int trial = 0; trial < 20; trial++) { + const size_t radix = 1 + (rng() % 6); + const NodeID origin = static_cast(rng() % num_nodes); + const int density = 1 + (rng() % 100); + + MulticastTargetSet targets; + for(NodeID n = 0; n < num_nodes; n++) + if(static_cast(rng() % 100) < density) + targets.add(n); + if(targets.empty()) + targets.add(static_cast(rng() % num_nodes)); + + SCOPED_TRACE(testing::Message() << "trial=" << trial << " radix=" << radix + << " origin=" << origin << " targets=" << targets); + reset_trace(); + SimMulticastNetwork net(num_nodes, radix); + CompletionProbe probe; + McastTestMessage hdr; + hdr.value = trial; + net.multicast(origin, targets, test_msgid(), &hdr, sizeof(hdr), nullptr, 0, + probe_callback(&probe)); + net.run_to_quiescence(); + + std::map per_node; + for(size_t i = 0; i < g_handled.size(); i++) { + EXPECT_EQ(g_handled[i].sender, origin); + per_node[g_handled[i].node]++; + } + EXPECT_EQ(g_handled.size(), targets.size()); + for(MulticastTargetSet::const_iterator it = targets.begin(); it != targets.end(); + ++it) + EXPECT_EQ(per_node[*it], 1); + + EXPECT_EQ(probe.invocations, 1u); + EXPECT_EQ(probe.callbacks_alive, 0); + EXPECT_EQ(probe.handled_when_invoked, targets.size()); + EXPECT_EQ(net.total_pending_completions(), 0u); + EXPECT_LE(net.max_fan_out(), radix); + EXPECT_EQ(net.num_acks, net.all_envelopes.size()); + } +} + +//////////////////////////////////////////////////////////////////////// +// +// completion-related fatal diagnostics (plan section 21.1) +// + +TEST_F(MulticastForwardTest, UnknownEnvelopeFlagsAreReportedAndDropped) +{ + reset_trace(); + SimMulticastNetwork net(8, 4); + RecordingFatalReporter reporter; + set_multicast_fatal_reporter(&reporter); + + MulticastTargetSet slice; + slice.add_range(2, 5); + McastTestMessage hdr; + BuiltEnvelope built = + build_envelope(slice, 8, 0, 42, 1, hdr, 0x8000u /*not a real flag*/, ByteVec()); + + net.cur = 2; + g_current_node = 2; + MulticastForwarder::forward(net, 0, built.env, built.payload.data(), + built.payload.size(), TimeLimit(), &net.metrics); + + ASSERT_EQ(reporter.contexts.size(), 1u); + EXPECT_STREQ(reporter.contexts[0].rule, "multicast envelope carries unknown flags"); + EXPECT_TRUE(g_handled.empty()); + EXPECT_TRUE(net.all_envelopes.empty()); + EXPECT_EQ(net.total_pending_completions(), 0u); +} + +TEST_F(MulticastForwardTest, CompletionMetadataWithoutTheFlagIsReportedAndDropped) +{ + reset_trace(); + SimMulticastNetwork net(8, 4); + RecordingFatalReporter reporter; + set_multicast_fatal_reporter(&reporter); + + MulticastTargetSet slice; + slice.add_range(2, 5); + McastTestMessage hdr; + ByteVec meta; + put_varint(meta, 0); + BuiltEnvelope built = build_envelope(slice, 8, 0, 42, 1, hdr, 0 /*no flag*/, meta); + + net.cur = 2; + g_current_node = 2; + MulticastForwarder::forward(net, 0, built.env, built.payload.data(), + built.payload.size(), TimeLimit(), &net.metrics); + + ASSERT_EQ(reporter.contexts.size(), 1u); + EXPECT_STREQ(reporter.contexts[0].rule, + "multicast envelope carries completion metadata without the completion " + "flag"); + EXPECT_TRUE(g_handled.empty()); + EXPECT_EQ(net.total_pending_completions(), 0u); +} + +TEST_F(MulticastForwardTest, MalformedCompletionMetadataIsReportedAndDropped) +{ + MulticastTargetSet slice; + slice.add_range(2, 5); + McastTestMessage hdr; + + struct Case { + const char *what; + ByteVec meta; + }; + std::vector cases; + { + // a node ID outside the configured node count + Case c; + c.what = "node out of range"; + put_varint(c.meta, 8); + cases.push_back(c); + } + { + // two varints where exactly one is expected + Case c; + c.what = "trailing metadata"; + put_varint(c.meta, 0); + put_varint(c.meta, 1); + cases.push_back(c); + } + { + // truncated varint + Case c; + c.what = "truncated"; + c.meta.push_back(0x81); + cases.push_back(c); + } + { + // overlong (noncanonical) varint + Case c; + c.what = "overlong"; + c.meta.push_back(0x80); + c.meta.push_back(0x00); + cases.push_back(c); + } + + for(size_t i = 0; i < cases.size(); i++) { + SCOPED_TRACE(cases[i].what); + reset_trace(); + SimMulticastNetwork net(8, 4); + RecordingFatalReporter reporter; + set_multicast_fatal_reporter(&reporter); + + BuiltEnvelope built = + build_envelope(slice, 8, 0, 42, 1, hdr, + MulticastEnvelopeFlags::COMPLETION_TRACKED, cases[i].meta); + net.cur = 2; + g_current_node = 2; + MulticastForwarder::forward(net, 0, built.env, built.payload.data(), + built.payload.size(), TimeLimit(), &net.metrics); + + ASSERT_EQ(reporter.contexts.size(), 1u); + EXPECT_STREQ(reporter.contexts[0].rule, "malformed multicast completion metadata"); + EXPECT_TRUE(g_handled.empty()); + EXPECT_TRUE(net.all_envelopes.empty()); + EXPECT_EQ(net.total_pending_completions(), 0u); + set_multicast_fatal_reporter(nullptr); + } +} + +TEST_F(MulticastForwardTest, CompletionParentThatIsNotTheSenderIsReportedAndDropped) +{ + reset_trace(); + SimMulticastNetwork net(8, 4); + RecordingFatalReporter reporter; + set_multicast_fatal_reporter(&reporter); + + MulticastTargetSet slice; + slice.add_range(2, 5); + McastTestMessage hdr; + ByteVec meta; + put_varint(meta, 1); // claims node 1 is the parent + BuiltEnvelope built = build_envelope(slice, 8, 0, 42, 1, hdr, + MulticastEnvelopeFlags::COMPLETION_TRACKED, meta); + + net.cur = 2; + g_current_node = 2; + // ... but the envelope actually arrived from node 0 + MulticastForwarder::forward(net, 0, built.env, built.payload.data(), + built.payload.size(), TimeLimit(), &net.metrics); + + ASSERT_EQ(reporter.contexts.size(), 1u); + EXPECT_STREQ(reporter.contexts[0].rule, + "multicast completion metadata names a parent that is not the sender"); + EXPECT_TRUE(g_handled.empty()); + EXPECT_TRUE(net.all_envelopes.empty()); + EXPECT_EQ(net.total_pending_completions(), 0u); +} + +TEST_F(MulticastForwardTest, ARejectedOriginSideMulticastNeitherRunsNorLeaksItsCallback) +{ + reset_trace(); + SimMulticastNetwork net(4, 2); + RecordingFatalReporter reporter; + set_multicast_fatal_reporter(&reporter); + + MulticastTargetSet targets; + targets.add(9); // outside the configured node count + + CompletionProbe probe; + McastTestMessage hdr; + net.start(0, targets, test_msgid(), &hdr, sizeof(hdr), nullptr, 0, + probe_callback(&probe)); + + ASSERT_EQ(reporter.contexts.size(), 1u); + // nothing was sent, so nothing "completed" - but the callback must still be reclaimed + EXPECT_EQ(probe.invocations, 0u); + EXPECT_EQ(probe.callbacks_alive, 0) << "a rejected multicast leaked its callback"; + EXPECT_TRUE(net.all_envelopes.empty()); + EXPECT_EQ(net.num_unicasts, 0u); + EXPECT_EQ(net.total_pending_completions(), 0u); + EXPECT_EQ(net.peak_pending_completions(), 0u); +} + +TEST_F(MulticastForwardTest, AcknowledgementForAnUnknownMulticastIsReported) +{ + reset_trace(); + SimMulticastNetwork net(8, 4); + RecordingFatalReporter reporter; + set_multicast_fatal_reporter(&reporter); + + net.cur = 0; + g_current_node = 0; + MulticastAckMessage ack; + ack.origin_node = 0; + ack.multicast_id = 999; + MulticastForwarder::handle_ack(net, 3, ack); + + ASSERT_EQ(reporter.contexts.size(), 1u); + EXPECT_STREQ(reporter.contexts[0].rule, + "multicast acknowledgement does not match any multicast in flight"); + EXPECT_EQ(reporter.contexts[0].multicast_id, 999u); + EXPECT_EQ(net.total_pending_completions(), 0u); +} + +//////////////////////////////////////////////////////////////////////// +// +// registration of the acknowledgement message +// + +TEST_F(MulticastForwardTest, AckMessageIsRegisteredAndDeliberatelyNotInline) +{ + ActiveMessageHandlerTable::MessageID id = + activemsg_handler_table.lookup_message_id(); + ActiveMessageHandlerTable::HandlerEntry *entry = + activemsg_handler_table.lookup_message_handler(id); + ASSERT_NE(entry, nullptr); + EXPECT_NE(entry->handler, nullptr); + // handling an acknowledgement can send the next one up the tree, which plan section + // 22 forbids doing recursively out of an inline handler + EXPECT_EQ(entry->handler_inline, nullptr); +} + +//////////////////////////////////////////////////////////////////////// +// +// stage 2d: migrated multicast users and the plan's stage 2 exit criteria +// + +namespace { + + ActiveMessageHandlerTable::MessageID shutdown_msgid(void) + { + return activemsg_handler_table.lookup_message_id(); + } + + // exactly the target set RuntimeImpl::initiate_shutdown() builds: every node except + // the shutdown master, which is always node 0 + MulticastTargetSet shutdown_targets(NodeID num_nodes) + { + MulticastTargetSet targets; + if(num_nodes > 1) + targets.add_range(1, num_nodes - 1); + return targets; + } + + // number of messages 'node' transmitted, of any kind + size_t sends_by(NodeID node) + { + size_t count = 0; + for(size_t i = 0; i < g_trace.size(); i++) + if((g_trace[i].kind != TraceEvent::HANDLED) && (g_trace[i].node == node)) + count++; + return count; + } + + // every slice of a contiguous target set is itself contiguous, so it must be one run + // and - once it is big enough for RANGES to beat the tied-size list encodings - must + // have been encoded as RANGES (plan section 7.2 and the stage 2 exit criteria) + void expect_contiguous_slices_use_ranges(const SimMulticastNetwork &net) + { + size_t checked = 0; + for(size_t i = 0; i < net.all_envelopes.size(); i++) { + const SimMulticastNetwork::QueuedEnvelope &q = net.all_envelopes[i]; + MulticastTargetSet slice; + ASSERT_EQ(EncodedMulticastTargets::decode( + q.payload.data(), q.env.target_encoding_size, net.nodes, slice), + MulticastDecodeStatus::OK); + ASSERT_EQ(slice.num_ranges(), 1u) + << "a slice of a contiguous target set must stay contiguous: " << slice; + // for 1 or 2 nodes SINGLE/SMALL_INLINE tie with (or beat) RANGES on size and win + // the tie-break; from 3 nodes up RANGES is strictly the smallest + if(slice.size() >= 3) { + EXPECT_EQ(static_cast(q.env.target_encoding_kind), + MulticastTargetEncoding::RANGES) + << "slice " << slice << " did not use the range encoding"; + checked++; + } + } + EXPECT_GT(checked, 0u) << "no slice was large enough to exercise RANGES"; + } + +}; // namespace + +// Plan section 7.6: "Runtime shutdown is a critical forwarding-order test. Relays must +// forward before delivering shutdown locally." This drives the REAL +// MulticastForwarder over the exact target set RuntimeImpl::initiate_shutdown() builds, +// with a handler that stops its node the instant it runs. +TEST_F(MulticastForwardTest, RuntimeShutdownRelaysForwardBeforeStoppingThemselves) +{ + const NodeID num_nodes = 64; + const size_t radix = MULTICAST_DEFAULT_RADIX; + SimMulticastNetwork net(num_nodes, radix); + + MulticastTargetSet targets = shutdown_targets(num_nodes); + ASSERT_EQ(targets.size(), 63u); + + McastShutdownMessage hdr; + hdr.result_code = 42; + net.multicast(0 /*shutdown master*/, targets, shutdown_msgid(), &hdr, sizeof(hdr)); + + // (a) nothing was transmitted by a node that had already shut itself down - had any + // relay delivered before forwarding, its whole subtree would be stranded + EXPECT_EQ(g_sends_from_stopped_nodes, 0u); + expect_forward_before_deliver(); + + // (b) every node except the master shut down exactly once, and each of them saw the + // shutdown master (not the relay that handed it the message) as the sender + ASSERT_EQ(g_handled.size(), 63u); + std::set shut_down; + for(size_t i = 0; i < g_handled.size(); i++) { + EXPECT_EQ(g_handled[i].sender, 0) + << "node " << g_handled[i].node << " did not see the shutdown master as sender"; + EXPECT_EQ(g_handled[i].value, 42); + EXPECT_TRUE(shut_down.insert(g_handled[i].node).second) + << "node " << g_handled[i].node << " was shut down twice"; + } + EXPECT_EQ(shut_down.size(), 63u); + EXPECT_EQ(shut_down.count(0), 0u) << "the shutdown master must not shut itself down " + "via the multicast"; + EXPECT_EQ(g_stopped_nodes.size(), 63u); + + // (c) the assertion in (a) is not vacuous: plenty of nodes both relayed and stopped + size_t relayed_then_stopped = 0; + for(std::set::const_iterator it = g_stopped_nodes.begin(); + it != g_stopped_nodes.end(); ++it) + if(sends_by(*it) > 0) + relayed_then_stopped++; + EXPECT_GE(relayed_then_stopped, 2u) + << "no node both forwarded and shut down, so the ordering check proved nothing"; + + // (d) the master's fan-out is the radix, not the machine size + EXPECT_EQ(sends_by(0), radix); + EXPECT_LE(net.max_fan_out(), radix); + // exactly one inbound message per target + EXPECT_EQ(net.total_sends(), 63u); +} + +// The same shutdown pattern across machine sizes and radices, including the degenerate +// two-node case that takes the unicast fast path. +TEST_F(MulticastForwardTest, RuntimeShutdownScalesAcrossMachineSizesAndRadices) +{ + const NodeID sizes[] = {2, 3, 5, 16, 64, 257, 1024}; + const size_t radices[] = {1, 2, 4, 8}; + + for(size_t si = 0; si < sizeof(sizes) / sizeof(sizes[0]); si++) { + for(size_t ri = 0; ri < sizeof(radices) / sizeof(radices[0]); ri++) { + const NodeID num_nodes = sizes[si]; + const size_t radix = radices[ri]; + SCOPED_TRACE(testing::Message() << "nodes=" << num_nodes << " radix=" << radix); + reset_trace(); + SimMulticastNetwork net(num_nodes, radix); + + MulticastTargetSet targets = shutdown_targets(num_nodes); + McastShutdownMessage hdr; + hdr.result_code = 7; + net.multicast(0, targets, shutdown_msgid(), &hdr, sizeof(hdr)); + + EXPECT_EQ(g_sends_from_stopped_nodes, 0u); + expect_forward_before_deliver(); + EXPECT_EQ(g_handled.size(), static_cast(num_nodes - 1)); + EXPECT_EQ(g_stopped_nodes.size(), static_cast(num_nodes - 1)); + for(size_t i = 0; i < g_handled.size(); i++) + EXPECT_EQ(g_handled[i].sender, 0); + // origin fan-out is bounded by the radix and total traffic is O(M) + EXPECT_LE(sends_by(0), radix); + EXPECT_LE(net.max_fan_out(), radix); + EXPECT_EQ(net.total_sends(), static_cast(num_nodes - 1)); + } + } +} + +// Plan section 19, stage 2 exit criteria, all three asserted on one large contiguous +// multicast: bounded first-hop count, exactly-once delivery with the original sender, +// and compact range metadata. +TEST_F(MulticastForwardTest, StageTwoExitCriteriaForALargeContiguousTargetSet) +{ + const NodeID num_nodes = 4096; + const size_t radix = MULTICAST_DEFAULT_RADIX; + const size_t num_targets = 2047; + SimMulticastNetwork net(num_nodes, radix); + + // the lower half of a 4096-node machine: contiguous, but not so nearly-everything + // that ALL_EXCEPT would legitimately be smaller than a range + MulticastTargetSet targets; + targets.add_range(1, static_cast(num_targets)); + ASSERT_EQ(targets.size(), num_targets); + ASSERT_EQ(targets.num_ranges(), 1u); + + // "contiguous sets use the range encoding" - the whole set first... + EncodedMulticastTargets whole = EncodedMulticastTargets::encode(targets, num_nodes); + EXPECT_EQ(whole.kind(), MulticastTargetEncoding::RANGES); + EXPECT_LE(whole.bytes(), 8u); + + McastTestMessage hdr; + hdr.value = 0x2D; + net.multicast(0, targets, test_msgid(), &hdr, sizeof(hdr)); + + // "a source sends at most R first-hop messages for a large target set" + EXPECT_EQ(sends_by(0), radix); + EXPECT_LE(net.max_fan_out(), radix); + + // "every target receives exactly one delivery with the original sender" + ASSERT_EQ(g_handled.size(), num_targets); + std::set seen; + for(size_t i = 0; i < g_handled.size(); i++) { + EXPECT_EQ(g_handled[i].sender, 0); + EXPECT_EQ(g_handled[i].value, 0x2D); + EXPECT_TRUE(seen.insert(g_handled[i].node).second) + << "node " << g_handled[i].node << " was delivered twice"; + } + EXPECT_EQ(seen.size(), num_targets); + EXPECT_EQ(net.total_sends(), num_targets); + + // ...and every slice on the wire too + expect_contiguous_slices_use_ranges(net); + + // plan section 23: no reusable multicast state remains + EXPECT_EQ(net.total_pending_completions(), 0u); + EXPECT_EQ(net.peak_pending_completions(), 0u); +} + +// "Every node except me" is the shape both the shutdown master and the IPC peer sets +// use. It is contiguous when the sender is node 0, but the encoder is free to do +// better than a range there: ALL_EXCEPT names the one excluded node in three bytes. +// Either way the metadata is compact and independent of the machine size, which is +// what the stage 2 exit criterion actually asks for. +TEST_F(MulticastForwardTest, AllButTheSenderEncodesCompactlyAtEveryMachineSize) +{ + const NodeID sizes[] = {2, 64, 4096, 1 << 20}; + for(size_t i = 0; i < sizeof(sizes) / sizeof(sizes[0]); i++) { + const NodeID num_nodes = sizes[i]; + SCOPED_TRACE(testing::Message() << "nodes=" << num_nodes); + + MulticastTargetSet targets = shutdown_targets(num_nodes); + EncodedMulticastTargets enc = EncodedMulticastTargets::encode(targets, num_nodes); + EXPECT_LE(enc.bytes(), 8u) << "encoding grew with the machine size"; + if(num_nodes > 2) + EXPECT_EQ(enc.kind(), MulticastTargetEncoding::ALL_EXCEPT); + + MulticastTargetSet round_trip; + ASSERT_EQ(enc.decode_into(num_nodes, round_trip), MulticastDecodeStatus::OK); + EXPECT_EQ(round_trip, targets); + } +} + +// The migrated header-only users (runtime shutdown, HIP IPC request/release, +// MetadataInvalidateMessage) all send a small header and no payload to a set that is +// typically NOT contiguous - Network::all_peers/shared_peers and a metadata +// remote-copy set both exclude the local node and can be sparse. +TEST_F(MulticastForwardTest, HeaderOnlyPeerSetMulticastMatchesMigratedCallSites) +{ + const NodeID num_nodes = 96; + const size_t radix = MULTICAST_DEFAULT_RADIX; + const NodeID origin = 37; + + // "all peers except me", exactly like HipModule's ipc_peers and + // MetadataBase::remote_copies + MulticastTargetSet peers; + peers.add_range(0, num_nodes - 1); + ASSERT_TRUE(peers.remove(origin)); + + SimMulticastNetwork net(num_nodes, radix); + McastTestMessage hdr; + hdr.value = 0x19C; + net.multicast(origin, peers, test_msgid(), &hdr, sizeof(hdr)); + + ASSERT_EQ(g_handled.size(), static_cast(num_nodes - 1)); + std::set seen; + for(size_t i = 0; i < g_handled.size(); i++) { + // the HIP and CUDA IPC handlers reply to 'sender', so origin preservation is what + // makes the response go back to the node that actually asked + EXPECT_EQ(g_handled[i].sender, origin); + EXPECT_TRUE(g_handled[i].payload.empty()); + EXPECT_TRUE(seen.insert(g_handled[i].node).second); + } + EXPECT_EQ(seen.count(origin), 0u); + EXPECT_LE(sends_by(origin), radix); + EXPECT_LE(net.max_fan_out(), radix); + EXPECT_EQ(net.total_sends(), static_cast(num_nodes - 1)); + expect_forward_before_deliver(); +} + +// RegionInstanceImpl::send_metadata and CudaModule's IPC broadcast used to chunk their +// payload at the source against recommended_max_payload(NodeSet, ...). They now hand +// the whole blob to the multicast layer and let the envelope be fragmented per hop, so +// a large copied 1-D payload has to arrive intact at every target either way. +TEST_F(MulticastForwardTest, WholeMetadataBlobReachesEveryEarlyRequestor) +{ + const NodeID num_nodes = 32; + const size_t radix = MULTICAST_DEFAULT_RADIX; + + std::vector blob(9001); + for(size_t i = 0; i < blob.size(); i++) + blob[i] = static_cast((i * 31) & 0xff); + + // sparse "early requestors", including node 0 and the last node + MulticastTargetSet early_reqs; + for(NodeID i = 0; i < num_nodes; i += 3) + early_reqs.add(i); + early_reqs.add(num_nodes - 1); + // the sender is never one of its own early requestors + ASSERT_FALSE(early_reqs.contains(5)); + + for(int fragmented = 0; fragmented < 2; fragmented++) { + SCOPED_TRACE(testing::Message() << "fragmented=" << fragmented); + reset_trace(); + SimMulticastNetwork net(num_nodes, radix); + net.frag_chunk_size = (fragmented ? 512 : 0); + + McastTestMessage hdr; + hdr.value = 0xB10B; + net.multicast(5, early_reqs, test_msgid(), &hdr, sizeof(hdr), blob.data(), + blob.size()); + + ASSERT_EQ(g_handled.size(), early_reqs.size()); + std::set seen; + for(size_t i = 0; i < g_handled.size(); i++) { + EXPECT_EQ(g_handled[i].sender, 5); + EXPECT_EQ(g_handled[i].payload.size(), blob.size()); + EXPECT_TRUE(std::equal(blob.begin(), blob.end(), g_handled[i].payload.begin())) + << "node " << g_handled[i].node << " got a corrupted metadata blob"; + EXPECT_TRUE(seen.insert(g_handled[i].node).second); + } + EXPECT_LE(net.max_fan_out(), radix); + if(fragmented) + EXPECT_GT(net.num_fragments_sent, 0u); + else + EXPECT_EQ(net.num_fragments_sent, 0u); + } +} + +//////////////////////////////////////////////////////////////////////// +// +// stage 9: multicast performance acceptance criteria, MEASURED (plan section 23) +// +// The multicast half of plan section 23 is five bounds on one multicast of M targets at +// radix R. Every case above asserts them for one shape; this one MEASURES them across +// a grid of (M, R) and prints the numbers, so that "origin fan-out is at most R" is +// visible as a table that stops growing rather than as a single passing assertion. +// + +TEST_F(MulticastForwardTest, AcceptanceCriteriaScalingTable) +{ + const size_t radices[] = {2, 4, 8}; + const size_t target_counts[] = {1, 2, 8, 64, 256, 1024}; + + std::cout << "\nplan section 23: active-message multicast, contiguous target set\n" + " R M | origin_out max_relay_out total_sends deliveries depth | " + "encoding bytes | pending acks\n"; + + for(size_t r = 0; r < 3; r++) { + for(size_t t = 0; t < 6; t++) { + const size_t radix = radices[r]; + const size_t num_targets = target_counts[t]; + // deliberately more nodes than targets, so that the contiguous run 1..M is a + // strict SUBSET of the machine and the encoder has to describe it as a range + // rather than collapsing it to "everyone except the origin" + const NodeID num_nodes = static_cast(2 * num_targets + 2); + SCOPED_TRACE(testing::Message() << "R=" << radix << " M=" << num_targets); + + reset_trace(); + SimMulticastNetwork net(num_nodes, radix); + MulticastTargetSet targets; + targets.add_range(1, static_cast(num_targets)); + + const EncodedMulticastTargets encoded = + EncodedMulticastTargets::encode(targets, num_nodes); + + McastTestMessage hdr; + hdr.value = 0x9009; + net.multicast(/*origin=*/0, targets, test_msgid(), &hdr, sizeof(hdr)); + + const size_t origin_out = net.sends_per_node.count(0) ? net.sends_per_node[0] : 0; + size_t relay_out = 0; + for(std::map::const_iterator it = net.sends_per_node.begin(); + it != net.sends_per_node.end(); ++it) { + if(it->first != 0) { + relay_out = std::max(relay_out, it->second); + } + } + + std::cout << " " << std::setw(2) << radix << " " << std::setw(6) << num_targets + << " | " << std::setw(10) << origin_out << " " << std::setw(13) + << relay_out << " " << std::setw(11) << net.total_sends() << " " + << std::setw(10) << g_handled.size() << " " << std::setw(5) + << net.max_depth_seen << " | " << std::setw(12) + << multicast_target_encoding_name(encoded.kind()) << " " << std::setw(5) + << encoded.bytes() << " | " << std::setw(7) + << net.peak_pending_completions() << " " << std::setw(4) << net.num_acks + << "\n"; + + // "Origin fan-out is at most R" / "relay fan-out is at most R" + EXPECT_LE(origin_out, radix); + EXPECT_LE(relay_out, radix); + EXPECT_LE(net.max_fan_out(), radix); + // "Total deliveries are exactly M", each with the ORIGIN as its sender + EXPECT_EQ(g_handled.size(), num_targets); + std::set seen; + for(size_t i = 0; i < g_handled.size(); i++) { + EXPECT_EQ(g_handled[i].sender, 0); + EXPECT_TRUE(seen.insert(g_handled[i].node).second) + << "node " << g_handled[i].node << " was delivered twice"; + } + EXPECT_EQ(seen.size(), num_targets); + // one wire message per remote target: the tree has exactly M edges + EXPECT_EQ(net.total_sends(), num_targets); + // O(log_R M) hops, not O(M) + EXPECT_LE(net.max_depth_seen, max_possible_depth(num_targets, radix)); + // "A contiguous target set has compact range metadata" - the encoded size must not + // grow with M at all, which is only true if the encoder described the run rather + // than listing its members + EXPECT_GT(encoded.bytes(), 0u); + EXPECT_LE(encoded.bytes(), size_t(16)); + if(num_targets >= 8) { + EXPECT_EQ(encoded.kind(), MulticastTargetEncoding::RANGES); + } + // "No reusable multicast state remains after completion" - a fire-and-forget + // multicast never allocates completion state and never acknowledges + EXPECT_EQ(net.peak_pending_completions(), 0u); + EXPECT_EQ(net.num_acks, 0u); + expect_forward_before_deliver(); + } + } +}