rheolef  6.6
geo_mpi_partition.cc
Go to the documentation of this file.
1 
2 #include "rheolef/config.h"
3 #ifdef _RHEOLEF_HAVE_MPI
4 #include "rheolef/geo_element.h"
5 #include "rheolef/hack_array.h"
6 
7 #if defined(_RHEOLEF_HAVE_SCOTCH)
8 #include "geo_partition_scotch.h"
9 #elif defined(_RHEOLEF_HAVE_PARMETIS)
10 #include <parmetis.h>
11 typedef idx_t idxtype; // change from previous parmetis versions
12 #endif // _RHEOLEF_HAVE_PARMETIS
13 
14 namespace rheolef {
15 
16 array<size_t>
17 geo_mpi_partition (
18  hack_array<geo_element_hack> ios_geo_element [reference_element::max_variant],
19  const distributor& ios_ownership, // by_dimension
20  size_t map_dim,
21  size_t dis_nv)
22 {
24  typedef hack_array<geo_element_hack>::const_iterator const_iterator_by_variant;
25 
26  communicator comm = ios_ownership.comm();
27  size_type nproc = comm.size();
28  size_type my_proc = comm.rank();
29 
30  array<size_type> partition (ios_ownership);
31  size_type iproc = 0;
32  if (partition.dis_size() <= nproc || dis_nv <= nproc) {
33  // mesh is very small (nelts <= nproc): parmetis does not support it !
34  for (size_type ie = 0, nie = ios_ownership.size(); ie < nie; ie++) {
35  partition [ie] = my_proc;
36  }
37  return partition;
38  }
39  array<idxtype> part (ios_ownership);
40  std::vector<idxtype> elmdist (nproc+1);
41  std::copy (
42  ios_ownership.begin(),
43  ios_ownership.end(),
44  elmdist.begin());
45  std::vector<idxtype> eptr (ios_ownership.dis_size()+1);
46  {
47  size_type ie = 0;
48  eptr [0] = 0;
49  for (size_type variant = reference_element::first_variant_by_dimension(map_dim);
50  variant < reference_element:: last_variant_by_dimension(map_dim); variant++) {
51  for (const_iterator_by_variant iter = ios_geo_element [variant].begin(), last = ios_geo_element [variant].end();
52  iter != last; iter++, ie++) {
53  const geo_element& K = *iter;
54  eptr [ie+1] = eptr[ie] + K.size();
55  }
56  }
57  }
58  std::vector<idxtype> eind (eptr [ios_ownership.size()]);
59  std::vector<idxtype>::iterator iter_eind = eind.begin();
60  {
61  for (size_type variant = reference_element::first_variant_by_dimension(map_dim);
62  variant < reference_element:: last_variant_by_dimension(map_dim); variant++) {
63  for (const_iterator_by_variant iter = ios_geo_element [variant].begin(), last = ios_geo_element [variant].end();
64  iter != last; iter++) {
65  const geo_element& K = *iter;
66  for (size_type iloc = 0; iloc < K.size(); iloc++, iter_eind++) {
67  *iter_eind = K[iloc];
68  }
69  }
70  }
71  }
72  idxtype wgtflag = 0;
73  idxtype numflag = 0;
74  idxtype mgcnum = map_dim;
75  idxtype ncon = 1;
76  idxtype nparts = nproc;
77  std::vector<float> tpwgts (nparts*ncon);
78  std::fill (tpwgts.begin(), tpwgts.end(), 1./nparts);
79  float ubvec [12];
80  std::fill (ubvec, ubvec+ncon, 1.05);
81  idxtype options [10];
82  options[0] = 1;
83  const int pvm3_option_dbglvl = 1;
84  const int pvm3_option_seed = 2;
85  options[pvm3_option_dbglvl] = 0; // otherwise: timming print to stdout...
86  options[pvm3_option_seed] = 0;
87  idxtype edgecut;
88  MPI_Comm raw_comm = comm;
89  idxtype *elmwgt = 0;
90 #if defined(_RHEOLEF_HAVE_SCOTCH)
92  elmdist.begin().operator->(),
93  eptr.begin().operator->(),
94  eind,
95  elmwgt,
96  &ncon,
97  &mgcnum,
98  &nparts,
99  tpwgts.begin().operator->(),
100  ubvec,
101  &edgecut,
102  part.begin().operator->(),
103  comm);
104 #elif defined(_RHEOLEF_HAVE_PARMETIS)
105  ParMETIS_V3_PartMeshKway(
106  elmdist.begin().operator->(),
107  eptr.begin().operator->(),
108  eind.begin().operator->(),
109  NULL,
110  &wgtflag,
111  &numflag,
112  &ncon,
113  &mgcnum,
114  &nparts,
115  tpwgts.begin().operator->(),
116  ubvec,
117  options,
118  &edgecut,
119  part.begin().operator->(),
120  &raw_comm);
121 #else // _RHEOLEF_HAVE_PARMETIS
122 # error either parmetis nor scotch partitioner founded
123 #endif // _RHEOLEF_HAVE_PARMETIS
124 
125  std::copy (part.begin(), part.end(), partition.begin());
126  return partition;
127 }
128 
129 } // namespace rheolef
130 #endif // _RHEOLEF_HAVE_MPI
static const variant_type max_variant
size_type size() const
Definition: distributed.h:54
static variant_type last_variant_by_dimension(size_type dim)
const communicator & comm() const
Definition: diststream.h:240
irheostream, orheostream - large data streams
Definition: compiler.h:7
void geo_partition_scotch(my_idxtype *elmdist, my_idxtype *eptr, std::vector< my_idxtype > &eind, my_idxtype *elmwgt, int *ncon, int *ncommonnodes, int *nparts, float *tpwgts, float *ubvec, int *edgecut, my_idxtype *part, const mpi::communicator &comm)
my_idxtype idxtype
static variant_type first_variant_by_dimension(size_type dim)
reference_element_H::size_type size_type