Skip to content

Commit ab7e18e

Browse files
author
Irina P. Demeshko
committed
Merge branch 'gather_scatter' into 'master'
Using Legion's copy launcher instead of implisit copy tasks See merge request laristra/flecsi!79
2 parents 7e08779 + f137c22 commit ab7e18e

File tree

6 files changed

+119
-106
lines changed

6 files changed

+119
-106
lines changed

‎flecsi/execution/legion/execution_policy.h‎

Lines changed: 82 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -42,9 +42,9 @@
4242
#include <flecsi/execution/legion/task_wrapper.h>
4343

4444
#if defined(ENABLE_CALIPER)
45-
#include <caliper/cali.h>
4645
#include <caliper/Annotation.h>
47-
#endif
46+
#include <caliper/cali.h>
47+
#endif
4848

4949
namespace flecsi {
5050
namespace execution {
@@ -163,9 +163,9 @@ struct legion_execution_policy_t {
163163

164164
#if defined(ENABLE_CALIPER)
165165
cali::Annotation et("FleCSI-Execution");
166-
std::string tname = "execute_task with TASK HASH " + std::to_string(TASK);
167-
cali::Annotation::Guard g(et.begin(tname.c_str()));
168-
#endif
166+
std::string tname = "execute_task with TASK HASH " + std::to_string(TASK);
167+
cali::Annotation::Guard g(et.begin(tname.c_str()));
168+
#endif
169169

170170
using namespace Legion;
171171

@@ -214,15 +214,16 @@ struct legion_execution_policy_t {
214214
// Execute a tuple walker that initializes the handle arguments
215215
// that are passed to the task
216216
#if defined(ENABLE_CALIPER)
217-
tname = "TASK HASH " + std::to_string(TASK)+" single-launch init_args";
218-
et.begin(tname.c_str());
219-
#endif
217+
tname =
218+
"TASK HASH " + std::to_string(TASK) + " single-launch init_args";
219+
et.begin(tname.c_str());
220+
#endif
220221
init_args_t init_args(legion_runtime, legion_context);
221222
init_args.walk(task_args);
222223

223224
#if defined(ENABLE_CALIPER)
224-
et.end();
225-
#endif
225+
et.end();
226+
#endif
226227
// Add region requirements and future dependencies to the
227228
// task launcher
228229
for(auto & req : init_args.region_reqs) {
@@ -239,9 +240,10 @@ struct legion_execution_policy_t {
239240
// Execute a tuple walker that applies the task prolog operations
240241
// on the mapped handles
241242
#if defined(ENABLE_CALIPER)
242-
tname = "TASK HASH " + std::to_string(TASK)+" single-launch task_prolog sparse false";
243-
et.begin(tname.c_str());
244-
#endif
243+
tname = "TASK HASH " + std::to_string(TASK) +
244+
" single-launch task_prolog sparse false";
245+
et.begin(tname.c_str());
246+
#endif
245247
{
246248
task_prolog_t task_prolog(
247249
legion_runtime, legion_context, launch_domain);
@@ -250,13 +252,14 @@ struct legion_execution_policy_t {
250252
task_prolog.launch_copies();
251253
} // scope
252254
#if defined(ENABLE_CALIPER)
253-
et.end();
254-
#endif
255+
et.end();
256+
#endif
255257

256258
#if defined(ENABLE_CALIPER)
257-
tname = "TASK HASH " + std::to_string(TASK)+" single-launch task_prolog sparse true";
258-
et.begin(tname.c_str());
259-
#endif
259+
tname = "TASK HASH " + std::to_string(TASK) +
260+
" single-launch task_prolog sparse true";
261+
et.begin(tname.c_str());
262+
#endif
260263
{
261264
task_prolog_t task_prolog(
262265
legion_runtime, legion_context, launch_domain);
@@ -265,8 +268,8 @@ struct legion_execution_policy_t {
265268
task_prolog.launch_copies();
266269
} // scope
267270
#if defined(ENABLE_CALIPER)
268-
et.end();
269-
#endif
271+
et.end();
272+
#endif
270273

271274
// Enqueue the task.
272275
clog(trace) << "Execute flecsi/legion task " << TASK << " on rank "
@@ -276,14 +279,15 @@ struct legion_execution_policy_t {
276279
// Execute a tuple walker that applies the task epilog operations
277280
// on the mapped handles
278281
#if defined(ENABLE_CALIPER)
279-
tname = "TASK HASH " + std::to_string(TASK)+" single-launch task_epilog";
280-
et.begin(tname.c_str());
281-
#endif
282+
tname =
283+
"TASK HASH " + std::to_string(TASK) + " single-launch task_epilog";
284+
et.begin(tname.c_str());
285+
#endif
282286
task_epilog_t task_epilog(legion_runtime, legion_context);
283287
task_epilog.walk(task_args);
284288
#if defined(ENABLE_CALIPER)
285-
et.end();
286-
#endif
289+
et.end();
290+
#endif
287291

288292
static_assert(
289293
REDUCTION == ZERO, "reductions are not supported for single tasks");
@@ -316,14 +320,15 @@ struct legion_execution_policy_t {
316320
// Execute a tuple walker that initializes the handle arguments
317321
// that are passed to the task
318322
#if defined(ENABLE_CALIPER)
319-
tname = "TASK HASH " + std::to_string(TASK)+" index-launch init_args";
320-
et.begin(tname.c_str());
321-
#endif
323+
tname =
324+
"TASK HASH " + std::to_string(TASK) + " index-launch init_args";
325+
et.begin(tname.c_str());
326+
#endif
322327
init_args_t init_args(legion_runtime, legion_context);
323328
init_args.walk(task_args);
324329
#if defined(ENABLE_CALIPER)
325-
et.end();
326-
#endif
330+
et.end();
331+
#endif
327332

328333
LegionRuntime::Arrays::Rect<1> launch_bounds(
329334
LegionRuntime::Arrays::Point<1>(0),
@@ -353,9 +358,10 @@ struct legion_execution_policy_t {
353358
// Execute a tuple walker that applies the task prolog operations
354359
// on the mapped handles
355360
#if defined(ENABLE_CALIPER)
356-
tname = "TASK HASH " + std::to_string(TASK)+" index-launch task_prolog sparse false";
357-
et.begin(tname.c_str());
358-
#endif
361+
tname = "TASK HASH " + std::to_string(TASK) +
362+
" index-launch task_prolog sparse false";
363+
et.begin(tname.c_str());
364+
#endif
359365
{
360366
task_prolog_t task_prolog(
361367
legion_runtime, legion_context, launch_domain);
@@ -364,13 +370,14 @@ struct legion_execution_policy_t {
364370
task_prolog.launch_copies();
365371
} // scope
366372
#if defined(ENABLE_CALIPER)
367-
et.end();
368-
#endif
373+
et.end();
374+
#endif
369375

370376
#if defined(ENABLE_CALIPER)
371-
tname = "TASK HASH " + std::to_string(TASK)+" index-launch task_prolog sparse true";
372-
et.begin(tname.c_str());
373-
#endif
377+
tname = "TASK HASH " + std::to_string(TASK) +
378+
" index-launch task_prolog sparse true";
379+
et.begin(tname.c_str());
380+
#endif
374381
{
375382
task_prolog_t task_prolog(
376383
legion_runtime, legion_context, launch_domain);
@@ -379,8 +386,8 @@ struct legion_execution_policy_t {
379386
task_prolog.launch_copies();
380387
} // scope
381388
#if defined(ENABLE_CALIPER)
382-
et.end();
383-
#endif
389+
et.end();
390+
#endif
384391

385392
if constexpr(REDUCTION != ZERO) {
386393
clog(info) << "executing reduction logic for " << REDUCTION
@@ -398,14 +405,15 @@ struct legion_execution_policy_t {
398405

399406
// Enqueue the epilog.
400407
#if defined(ENABLE_CALIPER)
401-
tname = "TASK HASH " + std::to_string(TASK)+" index-launch task_epilog";
402-
et.begin(tname.c_str());
403-
#endif
408+
tname =
409+
"TASK HASH " + std::to_string(TASK) + " index-launch task_epilog";
410+
et.begin(tname.c_str());
411+
#endif
404412
task_epilog_t task_epilog(legion_runtime, legion_context);
405413
task_epilog.walk(task_args);
406414
#if defined(ENABLE_CALIPER)
407-
et.end();
408-
#endif
415+
et.end();
416+
#endif
409417

410418
return legion_future_u<RETURN, launch_type_t::single>(future);
411419
}
@@ -417,14 +425,15 @@ struct legion_execution_policy_t {
417425
// Execute a tuple walker that applies the task epilog operations
418426
// on the mapped handles
419427
#if defined(ENABLE_CALIPER)
420-
tname = "TASK HASH " + std::to_string(TASK)+" index-launch task_epilog";
421-
et.begin(tname.c_str());
422-
#endif
428+
tname =
429+
"TASK HASH " + std::to_string(TASK) + " index-launch task_epilog";
430+
et.begin(tname.c_str());
431+
#endif
423432
task_epilog_t task_epilog(legion_runtime, legion_context);
424433
task_epilog.walk(task_args);
425434
#if defined(ENABLE_CALIPER)
426-
et.end();
427-
#endif
435+
et.end();
436+
#endif
428437

429438
return legion_future_u<RETURN, launch_type_t::index>(future_map);
430439
} // else
@@ -436,14 +445,14 @@ struct legion_execution_policy_t {
436445
// Execute a tuple walker that initializes the handle arguments
437446
// that are passed to the task
438447
#if defined(ENABLE_CALIPER)
439-
tname = "TASK HASH " + std::to_string(TASK)+" mpi-launch init_args";
440-
et.begin(tname.c_str());
441-
#endif
448+
tname = "TASK HASH " + std::to_string(TASK) + " mpi-launch init_args";
449+
et.begin(tname.c_str());
450+
#endif
442451
init_args_t init_args(legion_runtime, legion_context);
443452
init_args.walk(task_args);
444453
#if defined(ENABLE_CALIPER)
445-
et.end();
446-
#endif
454+
et.end();
455+
#endif
447456

448457
// FIXME: This will need to change with the new control model
449458
// if(context_.execution_state() == SPECIALIZATION_TLT_INIT) {
@@ -474,9 +483,10 @@ struct legion_execution_policy_t {
474483
// Execute a tuple walker that applies the task prolog operations
475484
// on the mapped handles
476485
#if defined(ENABLE_CALIPER)
477-
tname = "TASK HASH " + std::to_string(TASK)+" mpi-launch task_prolog sparse false";
478-
et.begin(tname.c_str());
479-
#endif
486+
tname = "TASK HASH " + std::to_string(TASK) +
487+
" mpi-launch task_prolog sparse false";
488+
et.begin(tname.c_str());
489+
#endif
480490
{
481491
task_prolog_t task_prolog(
482492
legion_runtime, legion_context, launch_domain);
@@ -485,13 +495,14 @@ struct legion_execution_policy_t {
485495
task_prolog.launch_copies();
486496
} // scope
487497
#if defined(ENABLE_CALIPER)
488-
et.end();
489-
#endif
498+
et.end();
499+
#endif
490500

491501
#if defined(ENABLE_CALIPER)
492-
tname = "TASK HASH " + std::to_string(TASK)+" mpi-launch task_prolog sparse true";
493-
et.begin(tname.c_str());
494-
#endif
502+
tname = "TASK HASH " + std::to_string(TASK) +
503+
" mpi-launch task_prolog sparse true";
504+
et.begin(tname.c_str());
505+
#endif
495506
{
496507
task_prolog_t task_prolog(
497508
legion_runtime, legion_context, launch_domain);
@@ -500,8 +511,8 @@ struct legion_execution_policy_t {
500511
task_prolog.launch_copies();
501512
} // scope
502513
#if defined(ENABLE_CALIPER)
503-
et.end();
504-
#endif
514+
et.end();
515+
#endif
505516

506517
// Launch the MPI task
507518
auto future =
@@ -521,14 +532,15 @@ struct legion_execution_policy_t {
521532
// Execute a tuple walker that applies the task epilog operations
522533
// on the mapped handles
523534
#if defined(ENABLE_CALIPER)
524-
tname = "TASK HASH " + std::to_string(TASK)+" mpi-launch task_epilog";
525-
et.begin(tname.c_str());
526-
#endif
535+
tname =
536+
"TASK HASH " + std::to_string(TASK) + " mpi-launch task_epilog";
537+
et.begin(tname.c_str());
538+
#endif
527539
task_epilog_t task_epilog(legion_runtime, legion_context);
528540
task_epilog.walk(task_args);
529541
#if defined(ENABLE_CALIPER)
530-
et.end();
531-
#endif
542+
et.end();
543+
#endif
532544

533545
if constexpr(REDUCTION != ZERO) {
534546
clog_fatal("there is no implementation for the mpi"

‎flecsi/execution/legion/runtime_main.cc‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -30,9 +30,9 @@ using namespace boost::program_options;
3030
#endif
3131

3232
#if defined(ENABLE_CALIPER)
33-
#include <caliper/cali.h>
3433
#include <caliper/cali-mpi.h>
35-
#endif
34+
#include <caliper/cali.h>
35+
#endif
3636

3737
//----------------------------------------------------------------------------//
3838
//! FleCSI runtime main function.
@@ -67,9 +67,9 @@ main(int argc, char ** argv) {
6767
MPI_Init(&argc, &argv);
6868
#endif
6969

70-
//#if defined(ENABLE_CALIPER)
71-
// cali_mpi_init();
72-
//#endif
70+
//#if defined(ENABLE_CALIPER)
71+
// cali_mpi_init();
72+
//#endif
7373

7474
// get the rank
7575
int rank{0};

‎flecsi/execution/legion/task_prolog.h‎

Lines changed: 17 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -182,19 +182,13 @@ struct task_prolog_t : public flecsi::utils::tuple_walker_u<task_prolog_t> {
182182
auto ghost_owner_pos_fid =
183183
LegionRuntime::HighLevel::FieldID(internal_field::ghost_owner_pos);
184184

185-
rr_pos.add_field(ghost_owner_pos_fid);
186-
187185
// TODO - circular dependency including internal_task.h
188186
auto constexpr key =
189187
flecsi::utils::const_string_t{EXPAND_AND_STRINGIFY(ghost_copy_task)}
190188
.hash();
191189

192190
const auto ghost_copy_tid = flecsi_context.task_id<key>();
193191

194-
Legion::IndexLauncher ghost_launcher(ghost_copy_tid, color_domain,
195-
Legion::TaskArgument(&args[first], sizeof(args[first])),
196-
Legion::ArgumentMap());
197-
198192
for(auto handle_itr = handle_groups[group].begin();
199193
handle_itr != handle_groups[group].end(); handle_itr++) {
200194
size_t handle = *handle_itr;
@@ -203,12 +197,23 @@ struct task_prolog_t : public flecsi::utils::tuple_walker_u<task_prolog_t> {
203197
rr_ghost.add_field(fids[handle]);
204198
}
205199

206-
ghost_launcher.add_region_requirement(rr_owners);
207-
ghost_launcher.add_region_requirement(rr_ghost);
208-
ghost_launcher.add_region_requirement(rr_pos);
209-
210-
ghost_launcher.tag = MAPPER_FORCE_RANK_MATCH;
211-
runtime->execute_index_space(context, ghost_launcher);
200+
if(sparse) {
201+
Legion::IndexLauncher ghost_launcher(ghost_copy_tid, color_domain,
202+
Legion::TaskArgument(&args[first], sizeof(args[first])),
203+
Legion::ArgumentMap());
204+
rr_pos.add_field(ghost_owner_pos_fid);
205+
ghost_launcher.add_region_requirement(rr_owners);
206+
ghost_launcher.add_region_requirement(rr_ghost);
207+
ghost_launcher.add_region_requirement(rr_pos);
208+
ghost_launcher.tag = MAPPER_FORCE_RANK_MATCH;
209+
runtime->execute_index_space(context, ghost_launcher);
210+
}
211+
else {
212+
Legion::IndexCopyLauncher ghost_copy_launcher(color_domain);
213+
ghost_copy_launcher.add_copy_requirements(rr_owners, rr_ghost);
214+
ghost_copy_launcher.add_src_indirect_field(rr_pos, ghost_owner_pos_fid);
215+
runtime->issue_copy_operation(context, ghost_copy_launcher);
216+
}
212217
} // for group
213218

214219
} // launch copies

0 commit comments

Comments
 (0)