diff --git a/cmake/FindZoltan.cmake b/cmake/FindZoltan.cmake index 24d8ad65..34e556da 100644 --- a/cmake/FindZoltan.cmake +++ b/cmake/FindZoltan.cmake @@ -10,14 +10,51 @@ if(ZOLTAN_PREFIX) message(STATUS "ZOLTAN_PREFIX ${ZOLTAN_PREFIX}") endif() -find_path(ZOLTAN_INCLUDE_DIR zoltan.h PATHS "${ZOLTAN_PREFIX}/include") - -find_library(ZOLTAN_LIBRARY zoltan PATHS "${ZOLTAN_PREFIX}/lib") +if(ZOLTAN_PREFIX) + find_path(ZOLTAN_INCLUDE_DIR zoltan.h + PATHS "${ZOLTAN_PREFIX}/include" + NO_DEFAULT_PATH + ) + find_library(ZOLTAN_LIBRARY zoltan + PATHS "${ZOLTAN_PREFIX}/lib" + NO_DEFAULT_PATH + ) +else() + find_path(ZOLTAN_INCLUDE_DIR zoltan.h) + find_library(ZOLTAN_LIBRARY zoltan) +endif() +if (ZOLTAN_INCLUDE_DIR) + message(DEBUG "ZOLTAN_INCLUDE_DIR ${ZOLTAN_INCLUDE_DIR}") +else() + message(FATAL_ERROR "Zoltan not found.") +endif() set(ZOLTAN_LIBRARIES ${ZOLTAN_LIBRARY} ) set(ZOLTAN_INCLUDE_DIRS ${ZOLTAN_INCLUDE_DIR} ) -find_package(Parmetis MODULE REQUIRED) +find_file(ZOLTAN_CONFIG_FILE Zoltan_config.h + "${ZOLTAN_INCLUDE_DIRS}" + NO_DEFAULT_PATH +) +message(DEBUG "ZOLTAN_CONFIG_FILE ${ZOLTAN_CONFIG_FILE}") +file(READ "${ZOLTAN_CONFIG_FILE}" ZOLTAN_CONFIG_TEXT) +string(FIND + "${ZOLTAN_CONFIG_TEXT}" "#define HAVE_PARMETIS" + ZOLTAN_FIND_PARMETIS +) +string(FIND + "${ZOLTAN_CONFIG_TEXT}" "#define HAVE_SCOTCH" + ZOLTAN_FIND_PTSCOTCH +) + +if(NOT ZOLTAN_FIND_PARMETIS EQUAL -1) + find_package(Parmetis MODULE REQUIRED) + set(PUMI_HAS_PARMETIS TRUE) +endif() +if(NOT ZOLTAN_FIND_PTSCOTCH EQUAL -1) + find_package(SCOTCH CONFIG REQUIRED) + set(PUMI_HAS_PTSCOTCH TRUE) +endif() include(FindPackageHandleStandardArgs) # handle the QUIETLY and REQUIRED arguments and set ZOLTAN_FOUND to TRUE @@ -28,4 +65,4 @@ find_package_handle_standard_args( ZOLTAN_LIBRARY ZOLTAN_INCLUDE_DIR ) -mark_as_advanced(ZOLTAN_INCLUDE_DIR ZOLTAN_LIBRARY ) +mark_as_advanced(ZOLTAN_INCLUDE_DIR ZOLTAN_LIBRARY ZOLTAN_CONFIG_FILE) diff --git a/zoltan/CMakeLists.txt b/zoltan/CMakeLists.txt index 92e9638a..92c929c4 100644 --- a/zoltan/CMakeLists.txt +++ b/zoltan/CMakeLists.txt @@ -6,7 +6,6 @@ endif() # Package options option(ENABLE_ZOLTAN "Enable Zoltan interface [ON|OFF]" OFF) xsdk_add_tpl(ZOLTAN) -xsdk_add_tpl(PARMETIS) message(STATUS "ENABLE_ZOLTAN: " ${ENABLE_ZOLTAN}) if(SCOREC_NO_MPI AND ENABLE_ZOLTAN) @@ -58,15 +57,20 @@ target_link_libraries(apf_zoltan PUBLIC pcu apf) # Do extra work if zoltan is enabled if(ENABLE_ZOLTAN) - target_include_directories(apf_zoltan PRIVATE - ${ZOLTAN_INCLUDE_DIRS} - ${PARMETIS_INCLUDE_DIRS} - ) - target_link_libraries(apf_zoltan PUBLIC - ${ZOLTAN_LIBRARIES} - ${PARMETIS_LIBRARIES} - ) + target_include_directories(apf_zoltan PRIVATE ${ZOLTAN_INCLUDE_DIRS}) + target_link_libraries(apf_zoltan PUBLIC ${ZOLTAN_LIBRARIES}) target_compile_definitions(apf_zoltan PUBLIC PUMI_HAS_ZOLTAN) + if(PUMI_HAS_PTSCOTCH) + target_link_libraries(apf_zoltan PUBLIC + SCOTCH::ptscotch SCOTCH::ptscotcherr + ) + target_compile_definitions(apf_zoltan PUBLIC PUMI_HAS_PTSCOTCH) + endif() + if(PUMI_HAS_PARMETIS) + target_include_directories(apf_zoltan PRIVATE ${PARMETIS_INCLUDE_DIRS}) + target_link_libraries(apf_zoltan PUBLIC ${PARMETIS_LIBRARIES}) + target_compile_definitions(apf_zoltan PUBLIC PUMI_HAS_PARMETIS) + endif() endif() scorec_export_library(apf_zoltan) diff --git a/zoltan/apfZoltan.h b/zoltan/apfZoltan.h index 9c50742b..56117ecb 100644 --- a/zoltan/apfZoltan.h +++ b/zoltan/apfZoltan.h @@ -42,6 +42,7 @@ enum ZoltanMethod { HYPERGRAPH, /** \brief Use ParMetis */ PARMETIS, + PTSCOTCH, /**< Use PT-Scotch */ /** \brief General graph partitionig */ GRAPH }; diff --git a/zoltan/apfZoltanCallbacks.cc b/zoltan/apfZoltanCallbacks.cc index 7fe724fc..9df98940 100644 --- a/zoltan/apfZoltanCallbacks.cc +++ b/zoltan/apfZoltanCallbacks.cc @@ -9,7 +9,9 @@ #include "apfZoltanMesh.h" #include "apfZoltan.h" #include "apfShape.h" +#ifdef PUMI_HAS_PARMETIS #include +#endif #include #include #include @@ -29,9 +31,33 @@ static int setZoltanLbMethod(struct Zoltan_Struct* ztn, ZoltanMesh* zb) case HYPERGRAPH: lbMethod = "HYPERGRAPH"; break; case PARMETIS: //fall into GRAPH settings + lbMethod = "GRAPH"; +#ifdef PUMI_HAS_PARMETIS + Zoltan_Set_Param(ztn, "GRAPH_PACKAGE", "PARMETIS"); +#else + lion_oprint(1, "WARNING: ParMETIS ZoltanMethod requested but ParMETIS" + " was not enabled at build time.\n"); +#endif + break; + case PTSCOTCH: + lbMethod = "GRAPH"; +#ifdef PUMI_HAS_PTSCOTCH + Zoltan_Set_Param(ztn, "GRAPH_PACKAGE", "Scotch"); +#else + lion_oprint(1, "WARNING: PT-Scotch ZoltanMethod requested but PT-Scotch" + " was not enabled at build time.\n"); +#endif + break; case GRAPH: lbMethod = "GRAPH"; - Zoltan_Set_Param(ztn, "GRAPH_PACKAGE", "PARMETIS"); // instead of PHG + // Prefer ParMETIS, then PT-Scotch, then Zoltan-native PHG. +#if defined(PUMI_HAS_PARMETIS) + Zoltan_Set_Param(ztn, "GRAPH_PACKAGE", "PARMETIS"); +#elif defined(PUMI_HAS_PTSCOTCH) + Zoltan_Set_Param(ztn, "GRAPH_PACKAGE", "Scotch"); +#else + // Zoltan_Set_Param(ztn, "HYPERGRAPH_PACKAGE", "PHG"); //FIXME: not required? +#endif break; default: lion_oprint(1,"ERROR %s Invalid LB_METHOD %d\n",__func__, zb->method); @@ -68,8 +94,14 @@ static int setZoltanLbApproach(struct Zoltan_Struct* ztn, ZoltanMesh* zb) return 1; } Zoltan_Set_Param(ztn, "LB_APPROACH", ptnAp.c_str()); - if ( (3 == zb->method) || (4 == zb->method) ) + if (zb->method == PARMETIS +#ifdef PUMI_HAS_PARMETIS // in this case GRAPH implies PARMETIS. + || zb->method == GRAPH +#endif + ) { Zoltan_Set_Param(ztn, "PARMETIS_METHOD", pMethod.c_str()); + } + // No zb->method == PTSCOTCH because Zoltan only supports RBISECT. return 0; } @@ -304,7 +336,9 @@ ZoltanData::~ZoltanData() void ZoltanData::run() { /* ensure Metis indices are the same size as Zoltan indices */ +#ifdef PUMI_HAS_PARMETIS PCU_ALWAYS_ASSERT(IDXTYPEWIDTH == sizeof(ZOLTAN_ID_TYPE)*8); +#endif setup(); ptn(); } @@ -331,7 +365,9 @@ void ZoltanData::setup() if ( zb->isLocal && 0 != m->getPCU()->Self() ) snprintf(paramStr, 128, "%d", 0); //if local silence all but rank 0 Zoltan_Set_Param(ztn, "debug_level", paramStr); +#ifdef PUMI_HAS_PARMETIS Zoltan_Set_Param(ztn, "PARMETIS_OUTPUT_LEVEL", paramStr); +#endif Zoltan_Set_Param(ztn, "CHECK_GRAPH", "0"); Zoltan_Set_Param(ztn, "CHECK_HYPERGRAPH", "0");