rheolef  7.0
geo_mpi_partition.cc
Go to the documentation of this file.
1 
2 #include "rheolef/config.h"
3 #ifdef _RHEOLEF_HAVE_MPI
4 #include "rheolef/geo_element.h"
5 #include "rheolef/hack_array.h"
6 
7 #if defined(_RHEOLEF_HAVE_SCOTCH)
8 #include "geo_partition_scotch.h"
9 #elif defined(_RHEOLEF_HAVE_PARMETIS)
10 #include <parmetis.h>
11 typedef idx_t idxtype; // change from previous parmetis versions
12 #endif // _RHEOLEF_HAVE_PARMETIS
13 
14 namespace rheolef {
15 
16 disarray<size_t>
17 geo_mpi_partition (
18  const std::array<hack_array<geo_element_hack>,reference_element::max_variant>&
19  ios_geo_element,
20  const distributor& ios_ownership, // by_dimension
21  size_t map_dim,
22  size_t dis_nv)
23 {
25  typedef hack_array<geo_element_hack>::const_iterator const_iterator_by_variant;
26 
27  communicator comm = ios_ownership.comm();
28  size_type nproc = comm.size();
29  size_type my_proc = comm.rank();
30 
31  disarray<size_type> partition (ios_ownership);
32  size_type iproc = 0;
33  if (partition.dis_size() <= nproc || dis_nv <= nproc) {
34  // mesh is very small (nelts <= nproc): parmetis does not support it !
35  for (size_type ie = 0, nie = ios_ownership.size(); ie < nie; ie++) {
36  partition [ie] = my_proc;
37  }
38  return partition;
39  }
40  disarray<idxtype> part (ios_ownership);
41  std::vector<idxtype> elmdist (nproc+1);
42  std::copy (
43  ios_ownership.begin(),
44  ios_ownership.end(),
45  elmdist.begin());
46  std::vector<idxtype> eptr (ios_ownership.dis_size()+1);
47  {
48  size_type ie = 0;
49  eptr [0] = 0;
50  for (size_type variant = reference_element::first_variant_by_dimension(map_dim);
51  variant < reference_element:: last_variant_by_dimension(map_dim); variant++) {
52  for (const_iterator_by_variant iter = ios_geo_element [variant].begin(), last = ios_geo_element [variant].end();
53  iter != last; iter++, ie++) {
54  const geo_element& K = *iter;
55  eptr [ie+1] = eptr[ie] + K.size();
56  }
57  }
58  }
59  std::vector<idxtype> eind (eptr [ios_ownership.size()]);
60  std::vector<idxtype>::iterator iter_eind = eind.begin();
61  {
62  for (size_type variant = reference_element::first_variant_by_dimension(map_dim);
63  variant < reference_element:: last_variant_by_dimension(map_dim); variant++) {
64  for (const_iterator_by_variant iter = ios_geo_element [variant].begin(), last = ios_geo_element [variant].end();
65  iter != last; iter++) {
66  const geo_element& K = *iter;
67  for (size_type iloc = 0; iloc < K.size(); iloc++, iter_eind++) {
68  *iter_eind = K[iloc];
69  }
70  }
71  }
72  }
73  idxtype wgtflag = 0;
74  idxtype numflag = 0;
75  idxtype mgcnum = map_dim;
76  idxtype ncon = 1;
77  idxtype nparts = nproc;
78  std::vector<float> tpwgts (nparts*ncon);
79  std::fill (tpwgts.begin(), tpwgts.end(), 1./nparts);
80  float ubvec [12];
81  std::fill (ubvec, ubvec+ncon, 1.05);
82  idxtype options [10];
83  options[0] = 1;
84  const int pvm3_option_dbglvl = 1;
85  const int pvm3_option_seed = 2;
86  options[pvm3_option_dbglvl] = 0; // otherwise: timming print to stdout...
87  options[pvm3_option_seed] = 0;
88  idxtype edgecut;
89  MPI_Comm raw_comm = comm;
90  idxtype *elmwgt = 0;
91 #if defined(_RHEOLEF_HAVE_SCOTCH)
93  elmdist.begin().operator->(),
94  eptr.begin().operator->(),
95  eind,
96  elmwgt,
97  &ncon,
98  &mgcnum,
99  &nparts,
100  tpwgts.begin().operator->(),
101  ubvec,
102  &edgecut,
103  part.begin().operator->(),
104  comm);
105 #elif defined(_RHEOLEF_HAVE_PARMETIS)
106  ParMETIS_V3_PartMeshKway(
107  elmdist.begin().operator->(),
108  eptr.begin().operator->(),
109  eind.begin().operator->(),
110  NULL,
111  &wgtflag,
112  &numflag,
113  &ncon,
114  &mgcnum,
115  &nparts,
116  tpwgts.begin().operator->(),
117  ubvec,
118  options,
119  &edgecut,
120  part.begin().operator->(),
121  &raw_comm);
122 #else // _RHEOLEF_HAVE_PARMETIS
123 # error either parmetis nor scotch partitioner founded
124 #endif // _RHEOLEF_HAVE_PARMETIS
125 
126  std::copy (part.begin(), part.end(), partition.begin());
127  return partition;
128 }
129 
130 } // namespace rheolef
131 #endif // _RHEOLEF_HAVE_MPI
std::vector< T, A >::const_iterator const_iterator
Definition: hack_array.h:304
static const variant_type max_variant
static variant_type last_variant_by_dimension(size_type dim)
irheostream, orheostream - large data streams
Definition: compiler.h:7
void geo_partition_scotch(my_idxtype *elmdist, my_idxtype *eptr, std::vector< my_idxtype > &eind, my_idxtype *elmwgt, int *ncon, int *ncommonnodes, int *nparts, float *tpwgts, float *ubvec, int *edgecut, my_idxtype *part, const mpi::communicator &comm)
my_idxtype idxtype
static variant_type first_variant_by_dimension(size_type dim)
std::vector< T, A >::size_type size_type
Definition: disarray.h:353