rheolef  6.5
geo_mpi_partition.cc
Go to the documentation of this file.
1 
2 #include "rheolef/config.h"
3 #ifdef _RHEOLEF_HAVE_MPI
4 #include "rheolef/geo_element.h"
5 #include "rheolef/hack_array.h"
6 
7 #if defined(_RHEOLEF_HAVE_SCOTCH)
8 #include "geo_partition_scotch.h"
9 #elif defined(_RHEOLEF_HAVE_PARMETIS)
10 #include <parmetis.h>
11 #endif // _RHEOLEF_HAVE_PARMETIS
12 
13 namespace rheolef {
14 
15 array<size_t>
18  const distributor& ios_ownership, // by_dimension
19  size_t map_dim,
20  size_t dis_nv)
21 {
23  typedef hack_array<geo_element_hack>::const_iterator const_iterator_by_variant;
24 
25  communicator comm = ios_ownership.comm();
26  size_type nproc = comm.size();
27  size_type my_proc = comm.rank();
28 
29  array<size_type> partition (ios_ownership);
30  size_type iproc = 0;
31  if (partition.dis_size() <= nproc || dis_nv <= nproc) {
32  // mesh is very small (nelts <= nproc): parmetis does not support it !
33  for (size_type ie = 0, nie = ios_ownership.size(); ie < nie; ie++) {
34  partition [ie] = my_proc;
35  }
36  return partition;
37  }
38  array<idxtype> part (ios_ownership);
39  std::vector<idxtype> elmdist (nproc+1);
40  std::copy (
41  ios_ownership.begin(),
42  ios_ownership.end(),
43  elmdist.begin());
44  std::vector<idxtype> eptr (ios_ownership.dis_size()+1);
45  {
46  size_type ie = 0;
47  eptr [0] = 0;
48  for (size_type variant = reference_element::first_variant_by_dimension(map_dim);
49  variant < reference_element:: last_variant_by_dimension(map_dim); variant++) {
50  for (const_iterator_by_variant iter = ios_geo_element [variant].begin(), last = ios_geo_element [variant].end();
51  iter != last; iter++, ie++) {
52  const geo_element& K = *iter;
53  eptr [ie+1] = eptr[ie] + K.size();
54  }
55  }
56  }
57  std::vector<idxtype> eind (eptr [ios_ownership.size()]);
58  std::vector<idxtype>::iterator iter_eind = eind.begin();
59  {
60  for (size_type variant = reference_element::first_variant_by_dimension(map_dim);
61  variant < reference_element:: last_variant_by_dimension(map_dim); variant++) {
62  for (const_iterator_by_variant iter = ios_geo_element [variant].begin(), last = ios_geo_element [variant].end();
63  iter != last; iter++) {
64  const geo_element& K = *iter;
65  for (size_type iloc = 0; iloc < K.size(); iloc++, iter_eind++) {
66  *iter_eind = K[iloc];
67  }
68  }
69  }
70  }
71  int wgtflag = 0;
72  int numflag = 0;
73  int mgcnum = map_dim;
74  int ncon = 1;
75  int nparts = nproc;
76  std::vector<float> tpwgts (nparts*ncon);
77  std::fill (tpwgts.begin(), tpwgts.end(), 1./nparts);
78  float ubvec [12];
79  std::fill (ubvec, ubvec+ncon, 1.05);
80  int options [10];
81  options[0] = 1;
82  const int pvm3_option_dbglvl = 1;
83  const int pvm3_option_seed = 2;
84  options[pvm3_option_dbglvl] = 0; // otherwise: timming print to stdout...
85  options[pvm3_option_seed] = 0;
86  int edgecut;
87  MPI_Comm raw_comm = comm;
88  idxtype *elmwgt = 0;
89 #if defined(_RHEOLEF_HAVE_SCOTCH)
91  elmdist.begin().operator->(),
92  eptr.begin().operator->(),
93  eind,
94  elmwgt,
95  &ncon,
96  &mgcnum,
97  &nparts,
98  tpwgts.begin().operator->(),
99  ubvec,
100  &edgecut,
101  part.begin().operator->(),
102  comm);
103 #elif defined(_RHEOLEF_HAVE_PARMETIS)
104  ParMETIS_V3_PartMeshKway(
105  elmdist.begin().operator->(),
106  eptr.begin().operator->(),
107  eind.begin().operator->(),
108  NULL,
109  &wgtflag,
110  &numflag,
111  &ncon,
112  &mgcnum,
113  &nparts,
114  tpwgts.begin().operator->(),
115  ubvec,
116  options,
117  &edgecut,
118  part.begin().operator->(),
119  &raw_comm);
120 #else // _RHEOLEF_HAVE_PARMETIS
121 # error either parmetis nor scotch partitioner founded
122 #endif // _RHEOLEF_HAVE_PARMETIS
123 
124  std::copy (part.begin(), part.end(), partition.begin());
125  return partition;
126 }
127 
128 } // namespace rheolef
129 #endif // _RHEOLEF_HAVE_MPI
130