902 lines
30 KiB
C
902 lines
30 KiB
C
/*
|
|
* @HEADER
|
|
*
|
|
* ***********************************************************************
|
|
*
|
|
* Zoltan Toolkit for Load-balancing, Partitioning, Ordering and Coloring
|
|
* Copyright 2012 Sandia Corporation
|
|
*
|
|
* Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
|
|
* the U.S. Government retains certain rights in this software.
|
|
*
|
|
* Redistribution and use in source and binary forms, with or without
|
|
* modification, are permitted provided that the following conditions are
|
|
* met:
|
|
*
|
|
* 1. Redistributions of source code must retain the above copyright
|
|
* notice, this list of conditions and the following disclaimer.
|
|
*
|
|
* 2. Redistributions in binary form must reproduce the above copyright
|
|
* notice, this list of conditions and the following disclaimer in the
|
|
* documentation and/or other materials provided with the distribution.
|
|
*
|
|
* 3. Neither the name of the Corporation nor the names of the
|
|
* contributors may be used to endorse or promote products derived from
|
|
* this software without specific prior written permission.
|
|
*
|
|
* THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
|
|
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
|
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
|
|
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
|
|
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
|
|
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
|
|
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
|
|
* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
|
|
* NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
|
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
*
|
|
* Questions? Contact Karen Devine kddevin@sandia.gov
|
|
* Erik Boman egboman@sandia.gov
|
|
*
|
|
* ***********************************************************************
|
|
*
|
|
* @HEADER
|
|
*/
|
|
|
|
#ifdef __cplusplus
|
|
/* if C++, define the rest of this header file as extern C */
|
|
extern "C" {
|
|
#endif
|
|
|
|
|
|
#include <ctype.h>
|
|
#include "zz_const.h"
|
|
#include "zz_util_const.h"
|
|
#include "all_allo_const.h"
|
|
#include "params_const.h"
|
|
#include "order_const.h"
|
|
#include "third_library.h"
|
|
#include "parmetis_interface.h"
|
|
#include "parmetis_interface_params.h"
|
|
|
|
/*********** COMPATIBILITY CHECKING AT COMPILE TIME ************/
|
|
#ifdef ZOLTAN_PARMETIS
|
|
#if (PARMETIS_MAJOR_VERSION < 3)
|
|
#error "Specified version of ParMETIS is not compatible with Zoltan; upgrade to ParMETIS v3.1 or later, or build Zoltan without ParMETIS."
|
|
#endif
|
|
|
|
#if (PARMETIS_MAJOR_VERSION == 3) && (PARMETIS_MINOR_VERSION < 1)
|
|
#error "Specified version of ParMETIS is not compatible with Zoltan; upgrade to ParMETIS v3.1 or later, or build Zoltan without ParMETIS."
|
|
#endif
|
|
|
|
/********** Workaround for memory bug in ParMETIS 3.1.0 **********/
|
|
#ifndef PARMETIS_SUBMINOR_VERSION
|
|
#define PARMETIS_SUBMINOR_VERSION 0
|
|
#endif
|
|
|
|
#if (PARMETIS_MAJOR_VERSION == 3) && (PARMETIS_MINOR_VERSION == 1) && (PARMETIS_SUBMINOR_VERSION == 0)
|
|
#define PARMETIS31_ALWAYS_FREES_VSIZE
|
|
#endif
|
|
|
|
#endif
|
|
|
|
static int pmv3method(char *alg);
|
|
|
|
static int mylog2(int x)
|
|
{
|
|
int i = 0;
|
|
|
|
for (i=0 ; (1<<i) <= x ; ++i);
|
|
return (i-1);
|
|
}
|
|
|
|
static int Zoltan_Parmetis_Parse(ZZ*, indextype *, char*, realtype*, double *,
|
|
ZOLTAN_Output_Order*);
|
|
|
|
|
|
/**********************************************************/
|
|
/* Interface routine for ParMetis: Partitioning */
|
|
/**********************************************************/
|
|
|
|
int Zoltan_ParMetis(
|
|
ZZ *zz, /* Zoltan structure */
|
|
float *part_sizes, /* Input: Array of size zz->Num_Global_Parts
|
|
containing the percentage of work to be
|
|
assigned to each partition. */
|
|
int *num_imp, /* number of objects to be imported */
|
|
ZOLTAN_ID_PTR *imp_gids, /* global ids of objects to be imported */
|
|
ZOLTAN_ID_PTR *imp_lids, /* local ids of objects to be imported */
|
|
int **imp_procs, /* list of processors to import from */
|
|
int **imp_to_part, /* list of partitions to which imported objects are
|
|
assigned. */
|
|
int *num_exp, /* number of objects to be exported */
|
|
ZOLTAN_ID_PTR *exp_gids, /* global ids of objects to be exported */
|
|
ZOLTAN_ID_PTR *exp_lids, /* local ids of objects to be exported */
|
|
int **exp_procs, /* list of processors to export to */
|
|
int **exp_to_part /* list of partitions to which exported objects are
|
|
assigned. */
|
|
)
|
|
{
|
|
char *yo = "Zoltan_ParMetis";
|
|
int ierr;
|
|
ZOLTAN_Third_Graph gr;
|
|
ZOLTAN_Third_Geom *geo = NULL;
|
|
ZOLTAN_Third_Vsize vsp;
|
|
ZOLTAN_Third_Part prt;
|
|
ZOLTAN_Output_Part part;
|
|
|
|
ZOLTAN_ID_PTR global_ids = NULL;
|
|
ZOLTAN_ID_PTR local_ids = NULL;
|
|
|
|
int use_timers = 0;
|
|
int timer_p = -1;
|
|
int get_times = 0;
|
|
double times[5];
|
|
|
|
double pmv3_itr = 0.0;
|
|
realtype itr = 0.0;
|
|
indextype options[MAX_PARMETIS_OPTIONS];
|
|
char alg[MAX_PARAM_STRING_LEN];
|
|
|
|
#ifdef ZOLTAN_PARMETIS
|
|
MPI_Comm comm = zz->Communicator;/* don't risk letting external packages */
|
|
/* change our zz struct. */
|
|
#endif
|
|
|
|
indextype i;
|
|
realtype *imb_tols;
|
|
indextype ncon;
|
|
indextype edgecut;
|
|
indextype wgtflag;
|
|
indextype numflag = 0;
|
|
indextype num_part = zz->LB.Num_Global_Parts; /* passed to ParMETIS. */
|
|
|
|
ZOLTAN_TRACE_ENTER(zz, yo);
|
|
|
|
Zoltan_Third_Init(&gr, &prt, &vsp, &part,
|
|
imp_gids, imp_lids, imp_procs, imp_to_part,
|
|
exp_gids, exp_lids, exp_procs, exp_to_part);
|
|
|
|
if (sizeof(realtype) != sizeof(float)) {
|
|
int tmp = zz->LB.Num_Global_Parts * MAX(zz->Obj_Weight_Dim, 1);
|
|
prt.input_part_sizes = (realtype *) ZOLTAN_MALLOC(tmp * sizeof(realtype));
|
|
|
|
for (i = 0; i < tmp; i++)
|
|
prt.input_part_sizes[i] = (realtype) part_sizes[i];
|
|
|
|
/* KDD 2/2014: removed re-scaling part sizes so they sum to one.
|
|
* part_sizes are already scaled in Zoltan_LB_Get_Part_Sizes.
|
|
* plus, the code here was wrong for multiple object weights.
|
|
* similar scaling code did not exist in the Scotch interface.
|
|
*/
|
|
prt.part_sizes = prt.input_part_sizes;
|
|
}
|
|
else
|
|
prt.input_part_sizes = prt.part_sizes = (realtype *) part_sizes;
|
|
|
|
|
|
ierr = Zoltan_Parmetis_Parse(zz, options, alg, &itr, &pmv3_itr, NULL);
|
|
if ((ierr != ZOLTAN_OK) && (ierr != ZOLTAN_WARN)) {
|
|
Zoltan_Third_Exit(&gr, geo, &prt, &vsp, &part, NULL);
|
|
return (ierr);
|
|
}
|
|
|
|
gr.graph_type = 0;
|
|
|
|
#ifdef ZOLTAN_PARMETIS
|
|
SET_GLOBAL_GRAPH(&gr.graph_type);
|
|
/* Select type of graph, negative because we impose them */
|
|
/* TODO: add a parameter to select the type, shared with Scotch */
|
|
/* if (strcmp (graph_type, "GLOBAL") != 0) { */
|
|
/* gr.graph_type = - LOCAL_GRAPH; */
|
|
/* if (zz->Num_Proc > 1) { */
|
|
/* ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Distributed graph: cannot call METIS, switching to ParMetis"); */
|
|
/* gr.graph_type = - GLOBAL_GRAPH; */
|
|
/* retval = ZOLTAN_WARN; */
|
|
/* } */
|
|
/* } */
|
|
#else /* graph is local */
|
|
SET_LOCAL_GRAPH(&gr.graph_type);
|
|
#endif /* ZOLTAN_PARMETIS */
|
|
|
|
|
|
/* Some algorithms use geometry data */
|
|
if (strncmp(alg, "PARTGEOM", 8) == 0){ /* PARTGEOM & PARTGEOMKWAY */
|
|
geo = (ZOLTAN_Third_Geom*) ZOLTAN_MALLOC(sizeof(ZOLTAN_Third_Geom));
|
|
memset (geo, 0, sizeof(ZOLTAN_Third_Geom));
|
|
/* ParMETIS will crash if geometric method and some procs have no nodes. */
|
|
/* Avoid fatal crash by setting scatter to level 2 or higher. */
|
|
gr.scatter_min = 2;
|
|
if (geo == NULL) {
|
|
ZOLTAN_PRINT_ERROR (zz->Proc, yo, "Out of memory.");
|
|
return (ZOLTAN_MEMERR);
|
|
}
|
|
if (strcmp(alg, "PARTGEOM") == 0) {
|
|
gr.get_data = 0;
|
|
}
|
|
}
|
|
|
|
timer_p = Zoltan_Preprocess_Timer(zz, &use_timers);
|
|
|
|
/* Start timer */
|
|
get_times = (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME);
|
|
if (get_times){
|
|
MPI_Barrier(zz->Communicator);
|
|
times[0] = Zoltan_Time(zz->Timer);
|
|
}
|
|
|
|
vsp.vsize_malloc = 0;
|
|
#ifdef PARMETIS31_ALWAYS_FREES_VSIZE
|
|
if (!strcmp(alg, "ADAPTIVEREPART") && (zz->Num_Proc > 1)) {
|
|
/* ParMETIS will free this memory; use malloc to allocate so
|
|
ZOLTAN_MALLOC counters don't show an error. */
|
|
vsp.vsize_malloc = 1 ;
|
|
}
|
|
#endif /* PARMETIS31_ALWAYS_FREES_VSIZE */
|
|
|
|
|
|
ierr = Zoltan_Preprocess_Graph(zz, &global_ids, &local_ids, &gr,
|
|
geo, &prt, &vsp);
|
|
if ((ierr != ZOLTAN_OK) && (ierr != ZOLTAN_WARN)) {
|
|
Zoltan_Third_Exit(&gr, geo, &prt, &vsp, &part, NULL);
|
|
return (ierr);
|
|
}
|
|
|
|
/* Get object sizes if requested */
|
|
if (options[PMV3_OPT_USE_OBJ_SIZE] &&
|
|
(zz->Get_Obj_Size || zz->Get_Obj_Size_Multi) &&
|
|
(!strcmp(alg, "ADAPTIVEREPART") || gr.final_output))
|
|
gr.showMoveVol = 1;
|
|
|
|
|
|
/* Get a time here */
|
|
if (get_times) times[1] = Zoltan_Time(zz->Timer);
|
|
|
|
/* Get ready to call ParMETIS */
|
|
edgecut = -1;
|
|
wgtflag = 2*(gr.obj_wgt_dim>0) + (gr.edge_wgt_dim>0);
|
|
numflag = 0;
|
|
ncon = (gr.obj_wgt_dim > 0 ? gr.obj_wgt_dim : 1);
|
|
|
|
if (!prt.part_sizes){
|
|
ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL,"Input parameter part_sizes is NULL.");
|
|
}
|
|
if ((zz->Proc == 0) && (zz->Debug_Level >= ZOLTAN_DEBUG_ALL)) {
|
|
for (i=0; i<num_part; i++){
|
|
indextype j;
|
|
|
|
printf("Debug: Size(s) for part " TPL_IDX_SPEC " = ", i);
|
|
for (j=0; j<ncon; j++)
|
|
printf("%f ", prt.part_sizes[i*ncon+j]);
|
|
printf("\n");
|
|
}
|
|
}
|
|
|
|
/* if (strcmp(alg, "ADAPTIVEREPART") == 0) */
|
|
for (i = 0; i < num_part*ncon; i++)
|
|
if (prt.part_sizes[i] == 0)
|
|
ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL, "Zero-sized part(s) requested! "
|
|
"ParMETIS 3.x will likely fail. Please use a "
|
|
"different method, or remove the zero-sized "
|
|
"parts from the problem.");
|
|
|
|
|
|
/* Set Imbalance Tolerance for each weight component. */
|
|
imb_tols = (realtype *) ZOLTAN_MALLOC(ncon * sizeof(realtype));
|
|
if (!imb_tols){
|
|
/* Not enough memory */
|
|
ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR, "Out of memory.");
|
|
}
|
|
for (i=0; i<ncon; i++)
|
|
imb_tols[i] = (realtype) (zz->LB.Imbalance_Tol[i]);
|
|
|
|
/* Now we can call ParMetis */
|
|
|
|
/* Zoltan_Third_Graph_Print(zz, &gr, "Before calling parmetis"); */
|
|
|
|
|
|
#ifdef ZOLTAN_PARMETIS
|
|
if (!IS_LOCAL_GRAPH(gr.graph_type)) { /* May be GLOBAL or NO GRAPH */
|
|
|
|
/* First check for ParMetis 3 routines */
|
|
if (strcmp(alg, "PARTKWAY") == 0){
|
|
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library "
|
|
"ParMETIS_V3_PartKway");
|
|
ParMETIS_V3_PartKway(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt, gr.ewgts,
|
|
&wgtflag, &numflag, &ncon, &num_part, prt.part_sizes,
|
|
imb_tols, options, &edgecut, prt.part, &comm);
|
|
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
|
|
}
|
|
else if (strcmp(alg, "PARTGEOMKWAY") == 0){
|
|
indextype ndims = geo->ndims;
|
|
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library "
|
|
"ParMETIS_V3_PartGeomKway");
|
|
ParMETIS_V3_PartGeomKway(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt,gr.ewgts,
|
|
&wgtflag, &numflag, &ndims, geo->xyz, &ncon,
|
|
&num_part, prt.part_sizes,
|
|
imb_tols, options, &edgecut, prt.part, &comm);
|
|
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
|
|
}
|
|
else if (strcmp(alg, "PARTGEOM") == 0){
|
|
indextype ndims = geo->ndims;
|
|
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library "
|
|
"ParMETIS_V3_PartGeom");
|
|
ParMETIS_V3_PartGeom(gr.vtxdist, &ndims, geo->xyz, prt.part, &comm);
|
|
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
|
|
}
|
|
else if (strcmp(alg, "ADAPTIVEREPART") == 0){
|
|
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library "
|
|
"ParMETIS_V3_AdaptiveRepart");
|
|
ParMETIS_V3_AdaptiveRepart(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt,
|
|
vsp.vsize, gr.ewgts, &wgtflag, &numflag, &ncon,
|
|
&num_part, prt.part_sizes, imb_tols,
|
|
&itr, options, &edgecut, prt.part, &comm);
|
|
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
|
|
}
|
|
else if (strcmp(alg, "REFINEKWAY") == 0){
|
|
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library "
|
|
"ParMETIS_V3_RefineKway");
|
|
ParMETIS_V3_RefineKway(gr.vtxdist, gr.xadj, gr.adjncy, gr.vwgt, gr.ewgts,
|
|
&wgtflag, &numflag, &ncon, &num_part,
|
|
prt.part_sizes, imb_tols,
|
|
options, &edgecut, prt.part, &comm);
|
|
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
|
|
}
|
|
else {
|
|
/* Sanity check: This should never happen! */
|
|
char msg[256];
|
|
sprintf(msg, "Unknown ParMetis algorithm %s.", alg);
|
|
ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL, msg);
|
|
}
|
|
}
|
|
#endif /* ZOLTAN_PARMETIS */
|
|
#ifdef ZOLTAN_METIS
|
|
/* TODO: I don't know how to set balance ! */
|
|
if (IS_LOCAL_GRAPH(gr.graph_type)) {
|
|
/* Check for Metis routines */
|
|
if (strcmp(alg, "PARTKWAY") == 0){
|
|
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the METIS library ");
|
|
/* Use default options for METIS */
|
|
#if !defined(METIS_VER_MAJOR) || METIS_VER_MAJOR < 5
|
|
options[0] = 0;
|
|
METIS_WPartGraphKway (gr.vtxdist+1, gr.xadj, gr.adjncy,
|
|
gr.vwgt, gr.ewgts, &wgtflag,
|
|
&numflag, &num_part, prt.part_sizes,
|
|
options, &edgecut, prt.part);
|
|
#else
|
|
METIS_SetDefaultOptions(options);
|
|
METIS_PartGraphKway (gr.vtxdist+1, &ncon, gr.xadj, gr.adjncy,
|
|
gr.vwgt, vsp.vsize, gr.ewgts, &num_part,
|
|
prt.part_sizes, imb_tols, options,
|
|
&edgecut, prt.part);
|
|
#endif
|
|
|
|
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the METIS library");
|
|
}
|
|
else {
|
|
/* Sanity check: This should never happen! */
|
|
char msg[256];
|
|
sprintf(msg, "Unknown Metis algorithm %s.", alg);
|
|
ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL, msg);
|
|
}
|
|
}
|
|
#endif /* ZOLTAN_METIS */
|
|
|
|
|
|
/* Get a time here */
|
|
if (get_times) times[2] = Zoltan_Time(zz->Timer);
|
|
|
|
|
|
if (gr.final_output) {
|
|
/* Do final output now because after the data will not be coherent:
|
|
unscatter only unscatter part data, not graph */
|
|
ierr = Zoltan_Postprocess_FinalOutput (zz, &gr, &prt, &vsp, use_timers, itr);
|
|
}
|
|
/* Ignore the timings of Final Ouput */
|
|
if (get_times) times[3] = Zoltan_Time(zz->Timer);
|
|
|
|
ierr = Zoltan_Postprocess_Graph(zz, global_ids, local_ids, &gr,
|
|
geo, &prt, &vsp, NULL, &part);
|
|
|
|
Zoltan_Third_Export_User(&part,
|
|
num_imp, imp_gids, imp_lids, imp_procs, imp_to_part,
|
|
num_exp, exp_gids, exp_lids, exp_procs, exp_to_part);
|
|
|
|
/* Get a time here */
|
|
if (get_times) times[4] = Zoltan_Time(zz->Timer);
|
|
|
|
if (get_times) Zoltan_Third_DisplayTime(zz, times);
|
|
|
|
if (use_timers && timer_p >= 0)
|
|
ZOLTAN_TIMER_STOP(zz->ZTime, timer_p, zz->Communicator);
|
|
|
|
Zoltan_Third_Exit(&gr, geo, &prt, &vsp, NULL, NULL);
|
|
if (imb_tols != NULL) ZOLTAN_FREE(&imb_tols);
|
|
if (geo != NULL) ZOLTAN_FREE(&geo);
|
|
ZOLTAN_FREE(&global_ids);
|
|
ZOLTAN_FREE(&local_ids);
|
|
|
|
ZOLTAN_TRACE_EXIT(zz, yo);
|
|
|
|
return (ierr);
|
|
}
|
|
|
|
|
|
static int Zoltan_Parmetis_Parse(
|
|
ZZ* zz,
|
|
indextype *options,
|
|
char* alg,
|
|
realtype* itr,
|
|
double *pmv3_itr,
|
|
ZOLTAN_Output_Order *ord
|
|
)
|
|
{
|
|
static char * yo = "Zoltan_Parmetis_Parse";
|
|
|
|
int i;
|
|
int output_level, seed, coarse_alg, fold, use_obj_size;
|
|
|
|
/* Always use ParMetis option array because Zoltan by default
|
|
produces no output (silent mode). ParMetis requires options[0]=1
|
|
when options array is to be used. */
|
|
options[0] = 1;
|
|
for (i = 1; i < MAX_PARMETIS_OPTIONS; i++)
|
|
options[i] = 0;
|
|
|
|
/* Set the default option values. */
|
|
output_level = 0;
|
|
coarse_alg = 2;
|
|
use_obj_size = 1;
|
|
fold = 0;
|
|
seed = GLOBAL_SEED;
|
|
|
|
if(ord == NULL) {
|
|
/* Map LB_APPROACH to suitable PARMETIS_METHOD */
|
|
if (!strcasecmp(zz->LB.Approach, "partition")){
|
|
strcpy(alg, "PARTKWAY");
|
|
}
|
|
else if (!strcasecmp(zz->LB.Approach, "repartition")){
|
|
strcpy(alg, "ADAPTIVEREPART");
|
|
*pmv3_itr = 100.; /* Ratio of inter-proc comm. time to data redist. time;
|
|
100 gives similar partition quality to GDiffusion */
|
|
}
|
|
else if (!strcasecmp(zz->LB.Approach, "refine")){
|
|
strcpy(alg, "REFINEKWAY");
|
|
}
|
|
else { /* If no LB_APPROACH is set, use repartition */
|
|
strcpy(alg, "ADAPTIVEREPART");
|
|
*pmv3_itr = 100.; /* Ratio of inter-proc comm. time to data redist. time;
|
|
100 gives similar partition quality to GDiffusion */
|
|
}
|
|
}
|
|
else {
|
|
strcpy(alg, "NODEND");
|
|
}
|
|
|
|
Zoltan_Bind_Param(Parmetis_params, "PARMETIS_METHOD",
|
|
(void *) alg);
|
|
Zoltan_Bind_Param(Parmetis_params, "PARMETIS_OUTPUT_LEVEL",
|
|
(void *) &output_level);
|
|
Zoltan_Bind_Param(Parmetis_params, "PARMETIS_SEED",
|
|
(void *) &seed);
|
|
Zoltan_Bind_Param(Parmetis_params, "PARMETIS_ITR",
|
|
(void *) pmv3_itr);
|
|
Zoltan_Bind_Param(Parmetis_params, "PARMETIS_COARSE_ALG",
|
|
(void *) &coarse_alg);
|
|
Zoltan_Bind_Param(Parmetis_params, "PARMETIS_FOLD",
|
|
(void *) &fold);
|
|
|
|
Zoltan_Assign_Param_Vals(zz->Params, Parmetis_params, zz->Debug_Level,
|
|
zz->Proc, zz->Debug_Proc);
|
|
|
|
/* Copy option values to ParMetis options array */
|
|
|
|
/* In this version of Zoltan, processors and partitions are coupled. */
|
|
/* This will likely change in future releases, and then the options */
|
|
/* value should change to DISCOUPLED. */
|
|
options[PMV3_OPTION_PSR] = COUPLED;
|
|
|
|
if (pmv3method(alg)){
|
|
/* ParMetis 3.0 options */
|
|
options[PMV3_OPTION_DBGLVL] = output_level;
|
|
options[PMV3_OPTION_SEED] = seed;
|
|
options[PMV3_OPT_USE_OBJ_SIZE] = use_obj_size;
|
|
if (ord == NULL)
|
|
*itr = (realtype)*pmv3_itr;
|
|
}
|
|
|
|
/* If ordering, use ordering method instead of load-balancing method */
|
|
if (ord && ord->order_opt){
|
|
strcpy(alg, ord->order_opt->method);
|
|
}
|
|
|
|
if ((zz->Num_Proc == 1) &&
|
|
(!strcmp(alg, "ADAPTIVEREPART") ||
|
|
!strcmp(alg, "REPARTLDIFFUSION") ||
|
|
!strcmp(alg, "REPARTGDIFFUSION") ||
|
|
!strcmp(alg, "REPARTREMAP") ||
|
|
!strcmp(alg, "REPARTMLREMAP"))) {
|
|
/* These ParMETIS methods fail on one processor; an MPI command assumes
|
|
at least two processors. */
|
|
char str[256];
|
|
sprintf(str, "ParMETIS method %s fails on one processor due to a bug"
|
|
" in ParMETIS v3.x; resetting method to PartKway.", alg);
|
|
ZOLTAN_PRINT_WARN(zz->Proc, yo, str);
|
|
|
|
strcpy(alg, "PARTKWAY");
|
|
return (ZOLTAN_WARN);
|
|
}
|
|
|
|
return(ZOLTAN_OK);
|
|
}
|
|
|
|
/***************************************************************************/
|
|
|
|
static int pmv3method( char *alg)
|
|
{
|
|
/* Check if alg is a supported ParMetis 3.0 method */
|
|
return ((!strcmp(alg, "PARTKWAY"))
|
|
|| (!strcmp(alg, "PARTGEOMKWAY"))
|
|
|| (!strcmp(alg, "ADAPTIVEREPART"))
|
|
|| (!strcmp(alg, "REFINEKWAY"))
|
|
|| (!strcmp(alg, "NODEND"))
|
|
);
|
|
}
|
|
|
|
|
|
/***************************************************************************
|
|
* The ParMetis ordering routine piggy-backs on the ParMetis
|
|
* partitioning routines.
|
|
**************************************************************************/
|
|
|
|
int Zoltan_ParMetis_Order(
|
|
ZZ *zz, /* Zoltan structure */
|
|
int num_obj, /* Number of (local) objects to order. */
|
|
ZOLTAN_ID_PTR gids, /* List of global ids (local to this proc) */
|
|
/* The application must allocate enough space */
|
|
ZOLTAN_ID_PTR lids, /* List of local ids (local to this proc) */
|
|
/* The application must allocate enough space */
|
|
ZOLTAN_ID_PTR rank, /* rank[i] is the rank of gids[i] */
|
|
int *iperm,
|
|
ZOOS *order_opt /* Ordering options, parsed by Zoltan_Order */
|
|
)
|
|
{
|
|
static char *yo = "Zoltan_ParMetis_Order";
|
|
int i, n, ierr;
|
|
ZOLTAN_Output_Order ord;
|
|
ZOLTAN_Third_Graph gr;
|
|
|
|
#ifdef ZOLTAN_PARMETIS
|
|
MPI_Comm comm = zz->Communicator;/* don't want to risk letting external
|
|
packages changing our communicator */
|
|
#endif
|
|
indextype numflag = 0;
|
|
|
|
int timer_p = 0;
|
|
int get_times = 0;
|
|
int use_timers = 0;
|
|
double times[5];
|
|
|
|
ZOLTAN_ID_PTR l_gids = NULL;
|
|
ZOLTAN_ID_PTR l_lids = NULL;
|
|
|
|
indextype options[MAX_PARMETIS_OPTIONS];
|
|
char alg[MAX_PARAM_STRING_LEN];
|
|
|
|
ZOLTAN_TRACE_ENTER(zz, yo);
|
|
|
|
#ifdef ZOLTAN_PARMETIS
|
|
#if TPL_USE_DATATYPE != TPL_METIS_DATATYPES
|
|
|
|
#ifdef TPL_FLOAT_WEIGHT
|
|
i = 1;
|
|
#else
|
|
i = 0;
|
|
#endif
|
|
|
|
if ((sizeof(indextype) != sizeof(idxtype)) ||
|
|
(sizeof(weighttype) != sizeof(idxtype)) || i){
|
|
|
|
ZOLTAN_THIRD_ERROR(ZOLTAN_FATAL,
|
|
"Not supported: Multiple 3rd party libraries with incompatible "
|
|
"data types.");
|
|
return ZOLTAN_FATAL;
|
|
}
|
|
#endif
|
|
#endif
|
|
|
|
memset(&gr, 0, sizeof(ZOLTAN_Third_Graph));
|
|
memset(&ord, 0, sizeof(ZOLTAN_Output_Order));
|
|
memset(times, 0, sizeof(times));
|
|
|
|
ord.order_opt = order_opt;
|
|
|
|
if (!order_opt){
|
|
/* If for some reason order_opt is NULL, allocate a new ZOOS here. */
|
|
/* This should really never happen. */
|
|
order_opt = (ZOOS *) ZOLTAN_MALLOC(sizeof(ZOOS));
|
|
strcpy(order_opt->method,"PARMETIS");
|
|
}
|
|
|
|
ierr = Zoltan_Parmetis_Parse(zz, options, alg, NULL, NULL, &ord);
|
|
/* ParMetis only computes the rank vector */
|
|
order_opt->return_args = RETURN_RANK;
|
|
|
|
/* Check that num_obj equals the number of objects on this proc. */
|
|
/* This constraint may be removed in the future. */
|
|
n = zz->Get_Num_Obj(zz->Get_Num_Obj_Data, &ierr);
|
|
if ((ierr!= ZOLTAN_OK) && (ierr!= ZOLTAN_WARN)){
|
|
/* Return error code */
|
|
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Get_Num_Obj returned error.");
|
|
return(ZOLTAN_FATAL);
|
|
}
|
|
if (n != num_obj){
|
|
/* Currently this is a fatal error. */
|
|
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Input num_obj does not equal the "
|
|
"number of objects.");
|
|
return(ZOLTAN_FATAL);
|
|
}
|
|
|
|
/* Do not use weights for ordering */
|
|
gr.obj_wgt_dim = -1;
|
|
gr.edge_wgt_dim = -1;
|
|
gr.num_obj = num_obj;
|
|
|
|
/* Check what ordering type is requested */
|
|
if (order_opt){
|
|
SET_GLOBAL_GRAPH(&gr.graph_type); /* GLOBAL by default */
|
|
|
|
#ifdef ZOLTAN_PARMETIS
|
|
if ((strcmp(order_opt->method, "METIS") == 0))
|
|
#endif /* ZOLTAN_PARMETIS */
|
|
SET_LOCAL_GRAPH(&gr.graph_type);
|
|
}
|
|
gr.get_data = 1;
|
|
|
|
if (IS_LOCAL_GRAPH(gr.graph_type) && zz->Num_Proc > 1) {
|
|
ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Serial ordering on more than 1 process: "
|
|
"set ParMetis instead.");
|
|
return(ZOLTAN_FATAL);
|
|
}
|
|
|
|
timer_p = Zoltan_Preprocess_Timer(zz, &use_timers);
|
|
|
|
/* Start timer */
|
|
get_times = (zz->Debug_Level >= ZOLTAN_DEBUG_ATIME);
|
|
if (get_times){
|
|
MPI_Barrier(zz->Communicator);
|
|
times[0] = Zoltan_Time(zz->Timer);
|
|
}
|
|
|
|
ierr = Zoltan_Preprocess_Graph(zz, &l_gids, &l_lids, &gr, NULL, NULL, NULL);
|
|
if ((ierr != ZOLTAN_OK) && (ierr != ZOLTAN_WARN)) {
|
|
Zoltan_Third_Exit(&gr, NULL, NULL, NULL, NULL, NULL);
|
|
return (ierr);
|
|
}
|
|
|
|
/* Allocate space for separator sizes */
|
|
|
|
if (IS_GLOBAL_GRAPH(gr.graph_type)) {
|
|
if (Zoltan_TPL_Order_Init_Tree(&zz->TPL_Order, 2*zz->Num_Proc, zz->Num_Proc) != ZOLTAN_OK) {
|
|
/* Not enough memory */
|
|
Zoltan_Third_Exit(&gr, NULL, NULL, NULL, NULL, &ord);
|
|
ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR, "Out of memory.");
|
|
}
|
|
ord.sep_sizes = (indextype*)ZOLTAN_MALLOC((2*zz->Num_Proc+1)*sizeof(indextype));
|
|
if (ord.sep_sizes == NULL) {
|
|
Zoltan_Third_Exit(&gr, NULL, NULL, NULL, NULL, &ord);
|
|
ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR, "Out of memory.");
|
|
}
|
|
memset(ord.sep_sizes, 0, (2*zz->Num_Proc+1)*sizeof(int)); /* It seems parmetis don't initialize correctly */
|
|
}
|
|
|
|
/* Allocate space for direct perm */
|
|
ord.rank = (indextype *) ZOLTAN_MALLOC(gr.num_obj*sizeof(indextype));
|
|
if (!ord.rank){
|
|
/* Not enough memory */
|
|
Zoltan_Third_Exit(&gr, NULL, NULL, NULL, NULL, &ord);
|
|
ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR, "Out of memory.");
|
|
}
|
|
if (IS_LOCAL_GRAPH(gr.graph_type)){
|
|
/* Allocate space for inverse perm */
|
|
ord.iperm = (indextype *) ZOLTAN_MALLOC(gr.num_obj*sizeof(indextype));
|
|
if (!ord.iperm){
|
|
/* Not enough memory */
|
|
Zoltan_Third_Exit(&gr, NULL, NULL, NULL, NULL, &ord);
|
|
ZOLTAN_THIRD_ERROR(ZOLTAN_MEMERR, "Out of memory.");
|
|
}
|
|
}
|
|
else
|
|
ord.iperm = NULL;
|
|
|
|
/* Get a time here */
|
|
if (get_times) times[1] = Zoltan_Time(zz->Timer);
|
|
|
|
#ifdef ZOLTAN_PARMETIS
|
|
if (IS_GLOBAL_GRAPH(gr.graph_type)){
|
|
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the ParMETIS library");
|
|
|
|
ParMETIS_V3_NodeND (gr.vtxdist, gr.xadj, gr.adjncy,
|
|
&numflag, options, ord.rank, ord.sep_sizes, &comm);
|
|
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the ParMETIS library");
|
|
|
|
}
|
|
else
|
|
#endif /* ZOLTAN_PARMETIS */
|
|
#if defined(ZOLTAN_METIS) || defined(ZOLTAN_PARMETIS)
|
|
if (IS_LOCAL_GRAPH(gr.graph_type)) { /* Be careful : permutation parameters are in the opposite order */
|
|
indextype numobj = gr.num_obj;
|
|
ZOLTAN_TRACE_DETAIL(zz, yo, "Calling the METIS library");
|
|
order_opt->return_args = RETURN_RANK|RETURN_IPERM; /* We provide directly all the permutations */
|
|
#if !defined(METIS_VER_MAJOR) || METIS_VER_MAJOR < 5
|
|
options[0] = 0; /* Use default options for METIS. */
|
|
METIS_NodeND(&numobj, gr.xadj, gr.adjncy, &numflag, options,
|
|
ord.iperm, ord.rank);
|
|
#else
|
|
METIS_SetDefaultOptions(options);
|
|
METIS_NodeND(&numobj, gr.xadj, gr.adjncy, NULL, options,
|
|
ord.iperm, ord.rank); /* NULL is vwgt -- new interface in v4 */
|
|
#endif
|
|
|
|
|
|
ZOLTAN_TRACE_DETAIL(zz, yo, "Returned from the METIS library");
|
|
}
|
|
#endif /* ZOLTAN_METIS */
|
|
|
|
/* Get a time here */
|
|
if (get_times) times[2] = Zoltan_Time(zz->Timer);
|
|
|
|
if (IS_GLOBAL_GRAPH(gr.graph_type)){ /* Update Elimination tree */
|
|
int numbloc;
|
|
int start;
|
|
int leaf;
|
|
int *converttab;
|
|
int levelmax;
|
|
|
|
levelmax = mylog2(zz->Num_Proc) + 1;
|
|
converttab = (int*)ZOLTAN_MALLOC(zz->Num_Proc*2*sizeof(int));
|
|
|
|
memset(converttab, 0, zz->Num_Proc*2*sizeof(int));
|
|
/* Determine the first node in each separator, store it in zz->TPL_Order.start */
|
|
for (numbloc = 0, start=0, leaf=0; numbloc < zz->Num_Proc /2; numbloc++) {
|
|
int father;
|
|
|
|
father = zz->Num_Proc + numbloc;
|
|
converttab[start] = 2*numbloc;
|
|
zz->TPL_Order.leaves[leaf++]=start;
|
|
zz->TPL_Order.ancestor[start] = start + 2;
|
|
converttab[start+1] = 2*numbloc+1;
|
|
zz->TPL_Order.leaves[leaf++]=start+1;
|
|
zz->TPL_Order.ancestor[start+1] = start + 2;
|
|
start+=2;
|
|
do {
|
|
converttab[start] = father;
|
|
if (father %2 == 0) {
|
|
int nextoffset;
|
|
int level;
|
|
|
|
level = mylog2(2*zz->Num_Proc - 1 - father);
|
|
nextoffset = (1<<(levelmax-level));
|
|
zz->TPL_Order.ancestor[start] = start+nextoffset;
|
|
start++;
|
|
break;
|
|
}
|
|
else {
|
|
zz->TPL_Order.ancestor[start] = start+1;
|
|
start++;
|
|
father = zz->Num_Proc + father/2;
|
|
}
|
|
} while (father < 2*zz->Num_Proc - 1);
|
|
}
|
|
|
|
zz->TPL_Order.start[0] = 0;
|
|
zz->TPL_Order.ancestor [2*zz->Num_Proc - 2] = -1;
|
|
for (numbloc = 1 ; numbloc < 2*zz->Num_Proc ; numbloc++) {
|
|
int oldblock=converttab[numbloc-1];
|
|
zz->TPL_Order.start[numbloc] = zz->TPL_Order.start[numbloc-1] + ord.sep_sizes[oldblock];
|
|
}
|
|
|
|
ZOLTAN_FREE(&converttab);
|
|
ZOLTAN_FREE(&ord.sep_sizes);
|
|
|
|
zz->TPL_Order.leaves[zz->Num_Proc] = -1;
|
|
zz->TPL_Order.nbr_leaves = zz->Num_Proc;
|
|
zz->TPL_Order.nbr_blocks = 2*zz->Num_Proc-1;
|
|
}
|
|
else { /* No tree */
|
|
zz->TPL_Order.nbr_blocks = 0;
|
|
zz->TPL_Order.start = NULL;
|
|
zz->TPL_Order.ancestor = NULL;
|
|
zz->TPL_Order.leaves = NULL;
|
|
}
|
|
|
|
/* Correct because no redistribution */
|
|
memcpy(gids, l_gids, n*zz->Num_GID*sizeof(ZOLTAN_ID_TYPE));
|
|
memcpy(lids, l_lids, n*zz->Num_LID*sizeof(ZOLTAN_ID_TYPE));
|
|
|
|
ierr = Zoltan_Postprocess_Graph (zz, l_gids, l_lids, &gr, NULL, NULL, NULL, &ord, NULL);
|
|
|
|
ZOLTAN_FREE(&l_gids);
|
|
ZOLTAN_FREE(&l_lids);
|
|
|
|
/* Get a time here */
|
|
if (get_times) times[3] = Zoltan_Time(zz->Timer);
|
|
|
|
if (get_times) Zoltan_Third_DisplayTime(zz, times);
|
|
|
|
if (use_timers)
|
|
ZOLTAN_TIMER_STOP(zz->ZTime, timer_p, zz->Communicator);
|
|
|
|
if (sizeof(indextype) == sizeof(ZOLTAN_ID_TYPE)){
|
|
memcpy(rank, ord.rank, gr.num_obj*sizeof(indextype));
|
|
}
|
|
else{
|
|
for (i=0; i < gr.num_obj; i++){
|
|
rank[i] = (ZOLTAN_ID_TYPE)ord.rank[i];
|
|
}
|
|
}
|
|
|
|
if ((ord.iperm != NULL) && (iperm != NULL)){
|
|
if (sizeof(indextype) == sizeof(int)){
|
|
memcpy(iperm, ord.iperm, gr.num_obj*sizeof(indextype));
|
|
}
|
|
else{
|
|
for (i=0; i < gr.num_obj; i++){
|
|
iperm[i] = (int)ord.iperm[i];
|
|
}
|
|
}
|
|
}
|
|
|
|
if (ord.iperm != NULL) ZOLTAN_FREE(&ord.iperm);
|
|
ZOLTAN_FREE(&ord.rank);
|
|
|
|
/* Free all other "graph" stuff */
|
|
Zoltan_Third_Exit(&gr, NULL, NULL, NULL, NULL, NULL);
|
|
|
|
ZOLTAN_TRACE_EXIT(zz, yo);
|
|
|
|
return (ZOLTAN_OK);
|
|
}
|
|
|
|
/*********************************************************************/
|
|
/* ParMetis parameter routine */
|
|
/*********************************************************************/
|
|
|
|
int Zoltan_ParMetis_Set_Param(
|
|
char *name, /* name of variable */
|
|
char *val) /* value of variable */
|
|
{
|
|
int status, i;
|
|
PARAM_UTYPE result; /* value returned from Check_Param */
|
|
int index; /* index returned from Check_Param */
|
|
char *valid_methods[] = {
|
|
"PARTKWAY", "PARTGEOMKWAY", "PARTGEOM",
|
|
"REPARTLDIFFUSION", "REPARTGDIFFUSION",
|
|
"REPARTREMAP", "REPARTMLREMAP",
|
|
"REFINEKWAY", "ADAPTIVEREPART",
|
|
"NODEND", /* for nested dissection ordering */
|
|
NULL };
|
|
|
|
status = Zoltan_Check_Param(name, val, Parmetis_params, &result, &index);
|
|
|
|
if (status == 0){
|
|
/* OK so far, do sanity check of parameter values */
|
|
|
|
if (strcmp(name, "PARMETIS_METHOD") == 0){
|
|
status = 2;
|
|
for (i=0; valid_methods[i] != NULL; i++){
|
|
if (strcmp(val, valid_methods[i]) == 0){
|
|
status = 0;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return(status);
|
|
}
|
|
|
|
#ifdef __cplusplus
|
|
}
|
|
#endif
|