parallel.h
Go to the documentation of this file.
1 // The libMesh Finite Element Library.
2 // Copyright (C) 2002-2014 Benjamin S. Kirk, John W. Peterson, Roy H. Stogner
3 
4 // This library is free software; you can redistribute it and/or
5 // modify it under the terms of the GNU Lesser General Public
6 // License as published by the Free Software Foundation; either
7 // version 2.1 of the License, or (at your option) any later version.
8 
9 // This library is distributed in the hope that it will be useful,
10 // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 // Lesser General Public License for more details.
13 
14 // You should have received a copy of the GNU Lesser General Public
15 // License along with this library; if not, write to the Free Software
16 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 
18 
19 #ifndef LIBMESH_PARALLEL_H
20 #define LIBMESH_PARALLEL_H
21 
22 // Local includes
23 #include "libmesh/libmesh_common.h" // libmesh_assert, libmesh_cast_int
25 
26 // C++ includes
27 #include <cstddef>
28 #include <climits>
29 #include <iterator>
30 #include <limits>
31 #include <map>
32 #include <set>
33 #include <string>
34 #include <vector>
35 
36 namespace libMesh
37 {
38 
39 
40 // Macro to identify and debug functions which should only be called in
41 // parallel on every processor at once
42 
43 #undef parallel_only
44 #ifndef NDEBUG
45  #define parallel_only() do { \
46  libmesh_deprecated(); \
47  libmesh_assert(CommWorld.verify(std::string(__FILE__).size())); \
48  libmesh_assert(CommWorld.verify(std::string(__FILE__))); \
49  libmesh_assert(CommWorld.verify(__LINE__)); } while (0)
50 #else
51  #define parallel_only() ((void) 0)
52 #endif
53 
54 #undef libmesh_parallel_only
55 #ifndef NDEBUG
56  #define libmesh_parallel_only(comm_obj) do { \
57  libmesh_assert(comm_obj.verify(std::string(__FILE__).size())); \
58  libmesh_assert(comm_obj.verify(std::string(__FILE__))); \
59  libmesh_assert(comm_obj.verify(__LINE__)); } while (0)
60 #else
61  #define libmesh_parallel_only(comm_obj) ((void) 0)
62 #endif
63 
64 // Macro to identify and debug functions which should only be called in
65 // parallel on every processor at once
66 
67 #undef parallel_only_on
68 #ifndef NDEBUG
69  #define parallel_only_on(comm_arg) do { \
70  libmesh_deprecated(); \
71  libmesh_assert(CommWorld.verify(std::string(__FILE__).size(), comm_arg)); \
72  libmesh_assert(CommWorld.verify(std::string(__FILE__), comm_arg)); \
73  libmesh_assert(CommWorld.verify(__LINE__), comm_arg); } while (0)
74 #else
75  #define parallel_only_on(comm_arg) ((void) 0)
76 #endif
77 
78 #undef libmesh_parallel_only_on
79 #ifndef NDEBUG
80  #define libmesh_parallel_only_on(comm_obj,comm_arg) do { \
81  libmesh_assert(comm_obj.verify(std::string(__FILE__).size(), comm_arg)); \
82  libmesh_assert(comm_obj.verify(std::string(__FILE__), comm_arg)); \
83  libmesh_assert(comm_obj.verify(__LINE__), comm_arg); } while (0)
84 #else
85  #define libmesh_parallel_only_on(comm_obj,comm_arg) ((void) 0)
86 #endif
87 
95 namespace Parallel
96 {
97  //-------------------------------------------------------------------
101  class Communicator;
102  class DataType;
103  class Request;
104  class Status;
105 
106 #ifdef LIBMESH_HAVE_MPI
107  //-------------------------------------------------------------------
111  typedef MPI_Datatype data_type;
112 
116  typedef MPI_Request request;
117 
121  typedef MPI_Status status;
122 
126  typedef MPI_Comm communicator;
127 
132  template <typename T>
133  inline data_type dataplusint_type();
134 
138  template <typename T>
140  {
141  public:
142  T val;
143  int rank;
144  };
145 
149  const unsigned int any_source =
150  static_cast<unsigned int>(MPI_ANY_SOURCE);
151 
152 
153 #else
154 
155  // These shouldn't actually be needed, but must be
156  // unique types for function overloading to work
157  // properly.
158  struct data_type { /* unsigned int t; */ };
159  struct request { /* unsigned int r; */ };
160  struct status { /* unsigned int s; */ };
161  typedef int communicator; // Must match petsc-nompi definition
162 
163  const unsigned int any_source=0;
164 #endif // LIBMESH_HAVE_MPI
165 
166 
167 
168  //-------------------------------------------------------------------
173  {
174  public:
175 
179  static const int invalid_tag = INT_MIN;
180 
185  explicit MessageTag(int tagvalue = invalid_tag)
186  : _tagvalue(tagvalue), _comm(NULL) {}
187 
192  MessageTag(const MessageTag& other);
193 
198  ~MessageTag();
199 
200  int value() const {
201  return _tagvalue;
202  }
203 
204  private:
207 
208  // Constructor for reference-counted unique tags
209  MessageTag(int tagvalue, const Communicator *comm)
210  : _tagvalue(tagvalue), _comm(comm) {}
211 
212  // Let Communicator handle the reference counting
213  friend class Communicator;
214  };
215 
216 
217  //-------------------------------------------------------------------
221 #ifdef LIBMESH_HAVE_MPI
222  const MessageTag any_tag = MessageTag(MPI_ANY_TAG);
223 #else
224  const MessageTag any_tag = MessageTag(-1);
225 #endif
226 
228 
229 
230  //-------------------------------------------------------------------
234  class DataType
235  {
236  public:
237  DataType () : _datatype() {}
238 
239  DataType (const DataType &other) :
240  _datatype(other._datatype)
241  {}
242 
243  DataType (const data_type &type) :
244  _datatype(type)
245  {}
246 
247 #ifdef LIBMESH_HAVE_MPI
248  DataType (const DataType &other, unsigned int count)
249  {
250  MPI_Type_contiguous(count, other._datatype, &_datatype);
251  this->commit();
252  }
253 #else
254  DataType (const DataType &, unsigned int)
255  {
256  }
257 #endif
258 
259  DataType & operator = (const DataType &other)
260  { _datatype = other._datatype; return *this; }
261 
263  { _datatype = type; return *this; }
264 
265  operator const data_type & () const
266  { return _datatype; }
267 
268  operator data_type & ()
269  { return _datatype; }
270 
271 // operator data_type const * () const
272 // { return &_datatype; }
273 
274 // operator data_type * ()
275 // { return &_datatype; }
276 
277  void commit ()
278  {
279 #ifdef LIBMESH_HAVE_MPI
280  MPI_Type_commit (&_datatype);
281 #endif
282  }
283 
284  void free ()
285  {
286 #ifdef LIBMESH_HAVE_MPI
287  MPI_Type_free (&_datatype);
288 #endif
289  }
290 
291  protected:
292 
294  };
295 
296 
297  //-------------------------------------------------------------------
305  template <typename T>
306  class StandardType : public DataType
307  {
308  /*
309  * The unspecialized class is useless, so we make its constructor
310  * private to catch mistakes at compile-time rather than link-time.
311  * Specializations should have a public constructor of the same
312  * form.
313  */
314  private:
315  StandardType(const T* example = NULL);
316  };
317 
318  /*
319  * The unspecialized class gives default, lowest-common-denominator
320  * attributes, for values which can't be used with Parallel min/max.
321  * Specialized classes can set this to true, and should define
322  * the lowest and highest values possible for the type.
323  */
324  template<typename T>
325  struct Attributes
326  {
327  static const bool has_min_max = false;
328  static void set_lowest(T&) {}
329  static void set_highest(T&) {}
330  };
331 
332 
333 
334  //-------------------------------------------------------------------
339  class Status
340  {
341  public:
342  Status ();
343 
344  explicit Status (const data_type &type);
345 
346  explicit Status (const status &status);
347 
348  Status (const status &status,
349  const data_type &type);
350 
351  Status (const Status &status);
352 
353  Status (const Status &status,
354  const data_type &type);
355 
356  status * get() { return &_status; }
357 
358  status const * get() const { return &_status; }
359 
360  int source () const;
361 
362  int tag () const;
363 
364  data_type& datatype () { return _datatype; }
365 
366  const data_type& datatype () const { return _datatype; }
367 
368  unsigned int size (const data_type &type) const;
369 
370  unsigned int size () const;
371 
372  private:
373 
374  status _status;
376  };
377 
378 
379  //-------------------------------------------------------------------
384  struct PostWaitWork {
385  virtual ~PostWaitWork() {}
386 
387  virtual void run() {}
388  };
389 
390 
391  //-------------------------------------------------------------------
395  class Request
396  {
397  public:
398  Request ();
399 
400  Request (const request &r);
401 
402  Request (const Request &other);
403 
404  void cleanup();
405 
406  Request & operator = (const Request &other);
407 
408  Request & operator = (const request &r);
409 
410  ~Request ();
411 
412  request* get() { return &_request; }
413 
414  const request* get() const { return &_request; }
415 
416  Status wait ();
417 
418  bool test ();
419 
420  bool test (status &status);
421 
422  void add_post_wait_work(PostWaitWork* work);
423 
424  private:
426 
427  // post_wait_work->first is a vector of work to do after a wait
428  // finishes; post_wait_work->second is a reference count so that
429  // Request objects will behave roughly like a shared_ptr and be
430  // usable in STL containers
431  std::pair<std::vector <PostWaitWork* >, unsigned int>* post_wait_work;
432  };
433 
437  inline Status wait (Request &r) { return r.wait(); }
438 
442  inline void wait (std::vector<Request> &r)
443  { for (unsigned int i=0; i<r.size(); i++) r[i].wait(); }
444 
445 
450  template <typename T>
451  struct BufferType;
452 
458  template <typename T>
459  struct BufferType<T*> {
461  };
462 
471  template <typename T, typename buffertype, typename Context>
472  void pack(const T* object,
473  typename std::vector<buffertype>& data,
474  const Context* context);
475 
484  template <typename T, typename Context>
485  unsigned int packable_size(const T*, const Context*);
486 
498  template <typename T, typename BufferIter>
499  unsigned int packed_size(const T*,
500  BufferIter);
501 
510  template <typename T, typename BufferIter, typename Context>
511  void unpack(BufferIter in, T** out, Context* ctx);
512 
517  template <typename Context, typename buffertype, typename OutputIter>
518  inline void unpack_range (const typename std::vector<buffertype>& buffer,
519  Context *context,
520  OutputIter out);
521 
526  template <typename Context, typename buffertype, typename Iter>
527  inline void pack_range (const Context *context,
528  Iter range_begin,
529  const Iter range_end,
530  typename std::vector<buffertype>& buffer);
531 
532  //-------------------------------------------------------------------
541  {
542  // Basic operations:
543  public:
544 
548  Communicator ();
549 
550  /*
551  * Constructor from MPI_Comm
552  */
553  explicit Communicator (const communicator &comm);
554 
555  /*
556  * NON-VIRTUAL destructor
557  */
558  ~Communicator ();
559 
560  /*
561  * Create a new communicator between some subset of \p this
562  */
563  void split(int color, int key, Communicator &target);
564 
565  /*
566  * Create a new duplicate of \p this communicator
567  */
568  void duplicate(const Communicator &comm);
569 
570  /*
571  * Create a new duplicate of an MPI communicator
572  */
573  void duplicate(const communicator &comm);
574 
575  communicator& get() { return _communicator; }
576 
577  const communicator& get() const { return _communicator; }
578 
585  MessageTag get_unique_tag(int tagvalue) const;
586 
591  void reference_unique_tag(int tagvalue) const;
592 
597  void dereference_unique_tag(int tagvalue) const;
598 
602  void clear();
603 
604  Communicator& operator= (const communicator &comm);
605 
606  unsigned int rank() const { return _rank; }
607 
608  unsigned int size() const { return _size; }
609 
614 
615  private:
616 
617  // Don't use the copy constructor, just copy by reference or
618  // pointer - it's too hard to keep a common used_tag_values if
619  // each communicator is shared by more than one Communicator
620  explicit Communicator (const Communicator &);
621 
626  void assign(const communicator &comm);
627 
629  unsigned int _rank, _size;
631 
632  // mutable used_tag_values - not thread-safe, but then Parallel::
633  // isn't thread-safe in general.
634  mutable std::map<int, unsigned int> used_tag_values;
636 
637  // Communication operations:
638  public:
639 
643  void send_mode (const SendMode sm) { _send_mode = sm; }
644 
648  SendMode send_mode() const { return _send_mode; }
649 
653  void barrier () const;
654 
659  template <typename T>
660  bool verify(const T &r) const;
661 
667  template <typename T>
668  bool semiverify(const T *r) const;
669 
674  template <typename T>
675  void min(T &r) const;
676 
682  template <typename T>
683  void minloc(T &r,
684  unsigned int &min_id) const;
685 
691  template <typename T>
692  void minloc(std::vector<T> &r,
693  std::vector<unsigned int> &min_id) const;
694 
699  template <typename T>
700  void max(T &r) const;
701 
707  template <typename T>
708  void maxloc(T &r,
709  unsigned int &max_id) const;
710 
716  template <typename T>
717  void maxloc(std::vector<T> &r,
718  std::vector<unsigned int> &max_id) const;
719 
724  template <typename T>
725  void sum(T &r) const;
726 
732  template <typename T>
733  void set_union(T &data, const unsigned int root_id) const;
734 
739  template <typename T>
740  void set_union(T &data) const;
741 
746  status probe (const unsigned int src_processor_id,
747  const MessageTag &tag=any_tag) const;
748 
752  template <typename T>
753  void send (const unsigned int dest_processor_id,
754  T &buf,
755  const MessageTag &tag=no_tag) const;
756 
760  template <typename T>
761  void send (const unsigned int dest_processor_id,
762  T &buf,
763  Request &req,
764  const MessageTag &tag=no_tag) const;
765 
769  template <typename T>
770  void send (const unsigned int dest_processor_id,
771  T &buf,
772  const DataType &type,
773  const MessageTag &tag=no_tag) const;
774 
778  template <typename T>
779  void send (const unsigned int dest_processor_id,
780  T &buf,
781  const DataType &type,
782  Request &req,
783  const MessageTag &tag=no_tag) const;
784 
788  template <typename T>
789  Status receive (const unsigned int dest_processor_id,
790  T &buf,
791  const MessageTag &tag=any_tag) const;
792 
796  template <typename T>
797  void receive (const unsigned int dest_processor_id,
798  T &buf,
799  Request &req,
800  const MessageTag &tag=any_tag) const;
801 
805  template <typename T>
806  Status receive (const unsigned int dest_processor_id,
807  T &buf,
808  const DataType &type,
809  const MessageTag &tag=any_tag) const;
810 
814  template <typename T>
815  void receive (const unsigned int dest_processor_id,
816  T &buf,
817  const DataType &type,
818  Request &req,
819  const MessageTag &tag=any_tag) const;
820 
834  template <typename Context, typename Iter>
835  void send_packed_range (const unsigned int dest_processor_id,
836  const Context *context,
837  Iter range_begin,
838  const Iter range_end,
839  const MessageTag &tag=no_tag) const;
840 
854  template <typename Context, typename Iter>
855  void send_packed_range (const unsigned int dest_processor_id,
856  const Context *context,
857  Iter range_begin,
858  const Iter range_end,
859  Request &req,
860  const MessageTag &tag=no_tag) const;
861 
888  template <typename Context, typename OutputIter>
889  void receive_packed_range (const unsigned int dest_processor_id,
890  Context *context,
891  OutputIter out,
892  const MessageTag &tag=any_tag) const;
893 
920  template <typename Context, typename OutputIter>
921  void receive_packed_range (const unsigned int dest_processor_id,
922  Context *context,
923  OutputIter out,
924  Request &req,
925  const MessageTag &tag=any_tag) const;
926 
931  template <typename T1, typename T2>
932  void send_receive(const unsigned int dest_processor_id,
933  T1 &send,
934  const unsigned int source_processor_id,
935  T2 &recv,
936  const MessageTag &send_tag = no_tag,
937  const MessageTag &recv_tag = any_tag) const;
938 
975  template <typename Context1, typename RangeIter, typename Context2, typename OutputIter>
976  void send_receive_packed_range(const unsigned int dest_processor_id,
977  const Context1* context1,
978  RangeIter send_begin,
979  const RangeIter send_end,
980  const unsigned int source_processor_id,
981  Context2* context2,
982  OutputIter out,
983  const MessageTag &send_tag = no_tag,
984  const MessageTag &recv_tag = any_tag) const;
985 
991  template <typename T1, typename T2>
992  void send_receive(const unsigned int dest_processor_id,
993  T1 &send,
994  const DataType &type1,
995  const unsigned int source_processor_id,
996  T2 &recv,
997  const DataType &type2,
998  const MessageTag &send_tag = no_tag,
999  const MessageTag &recv_tag = any_tag) const;
1000 
1005  template <typename T>
1006  inline void gather(const unsigned int root_id,
1007  T send,
1008  std::vector<T> &recv) const;
1009 
1034  template <typename T>
1035  inline void gather(const unsigned int root_id,
1036  std::vector<T> &r) const;
1037 
1042  template <typename T>
1043  inline void allgather(T send,
1044  std::vector<T> &recv) const;
1045 
1046 
1071  template <typename T>
1072  inline void allgather(std::vector<T> &r,
1073  const bool identical_buffer_sizes = false) const;
1074 
1075  //-------------------------------------------------------------------
1080  template <typename Context, typename Iter, typename OutputIter>
1081  inline void gather_packed_range (const unsigned int root_id,
1082  Context *context,
1083  Iter range_begin,
1084  const Iter range_end,
1085  OutputIter out) const;
1086 
1091  template <typename Context, typename Iter, typename OutputIter>
1092  inline void allgather_packed_range (Context *context,
1093  Iter range_begin,
1094  const Iter range_end,
1095  OutputIter out) const;
1096 
1102  template <typename T>
1103  inline void alltoall(std::vector<T> &r) const;
1104 
1113  template <typename T>
1114  inline void broadcast(T &data, const unsigned int root_id=0) const;
1115 
1133  template <typename Context, typename OutputContext, typename Iter, typename OutputIter>
1134  inline void broadcast_packed_range (const Context *context1,
1135  Iter range_begin,
1136  const Iter range_end,
1137  OutputContext *context2,
1138  OutputIter out,
1139  const unsigned int root_id = 0) const;
1140 
1148 #include "libmesh/parallel_communicator_specializations"
1149 
1150  }; // class Communicator
1151 
1152  // FakeCommunicator for debugging inappropriate CommWorld uses
1154  {
1155  operator Communicator& () {
1156  libmesh_error();
1157  static Communicator temp;
1158  return temp;
1159  }
1160  };
1161 
1162  // PostWaitWork specialization for copying from temporary to
1163  // output containers
1164  template <typename Container, typename OutputIter>
1166  PostWaitCopyBuffer(const Container& buffer, const OutputIter out)
1167  : _buf(buffer), _out(out) {}
1168 
1169  virtual void run() { std::copy(_buf.begin(), _buf.end(), _out); }
1170 
1171  private:
1172  const Container& _buf;
1173  OutputIter _out;
1174  };
1175 
1176  // PostWaitWork specialization for unpacking received buffers.
1177  template <typename Container, typename Context, typename OutputIter>
1179  PostWaitUnpackBuffer(const Container& buffer, Context *context, OutputIter out) :
1180  _buf(buffer), _context(context), _out(out) {}
1181 
1183 
1184  private:
1185  const Container& _buf;
1186  Context *_context;
1187  OutputIter _out;
1188  };
1189 
1190 
1191  // PostWaitWork specialization for freeing no-longer-needed buffers.
1192  template <typename Container>
1194  PostWaitDeleteBuffer(Container* buffer) : _buf(buffer) {}
1195 
1196  virtual void run() { delete _buf; }
1197 
1198  private:
1199  Container* _buf;
1200  };
1201 
1202 } // namespace Parallel
1203 
1217 #ifdef LIBMESH_DISABLE_COMMWORLD
1219  #define LIBMESH_CAN_DEFAULT_TO_COMMWORLD
1220 #else
1222  #define LIBMESH_CAN_DEFAULT_TO_COMMWORLD = libMesh::CommWorld
1223 #endif
1224 
1225 } // namespace libMesh
1226 
1227 // Define all the implementations separately; users might want to look
1228 // through this file for APIs, and it's long enough already.
1229 
1231 
1232 #endif // LIBMESH_PARALLEL_H

Site Created By: libMesh Developers
Last modified: February 07 2014 16:57:06 UTC

Hosted By:
SourceForge.net Logo