parallel_implementation.h
Go to the documentation of this file.
1 // The libMesh Finite Element Library.
2 // Copyright (C) 2002-2014 Benjamin S. Kirk, John W. Peterson, Roy H. Stogner
3 
4 // This library is free software; you can redistribute it and/or
5 // modify it under the terms of the GNU Lesser General Public
6 // License as published by the Free Software Foundation; either
7 // version 2.1 of the License, or (at your option) any later version.
8 
9 // This library is distributed in the hope that it will be useful,
10 // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 // Lesser General Public License for more details.
13 
14 // You should have received a copy of the GNU Lesser General Public
15 // License along with this library; if not, write to the Free Software
16 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 
18 
19 #ifndef LIBMESH_PARALLEL_IMPLEMENTATION_H
20 #define LIBMESH_PARALLEL_IMPLEMENTATION_H
21 
22 // Local includes
23 #include "parallel.h"
24 #include "libmesh_logging.h"
25 
26 // C++ includes
27 #include <iterator> // iterator_traits
28 
29 
30 // First declare StandardType specializations so we can use them in anonymous
31 // helper functions later
32 
33 namespace libMesh {
34 namespace Parallel {
35 
36 #ifdef LIBMESH_HAVE_MPI
37 
38 #define STANDARD_TYPE(cxxtype,mpitype) \
39 template<> \
40 class StandardType<cxxtype> : public DataType \
41 { \
42 public: \
43  explicit \
44  StandardType(const cxxtype* = NULL) : DataType(mpitype) {} \
45 }
46 
47 #else
48 
49 #define STANDARD_TYPE(cxxtype,mpitype) \
50 template<> \
51 class StandardType<cxxtype> : public DataType \
52 { \
53 public: \
54  explicit \
55  StandardType(const cxxtype* = NULL) : DataType() {} \
56 }
57 
58 #endif
59 
60 #define INT_TYPE(cxxtype,mpitype) \
61 STANDARD_TYPE(cxxtype,mpitype); \
62  \
63 template<> \
64 struct Attributes<cxxtype> \
65 { \
66  static const bool has_min_max = true; \
67  static void set_lowest(cxxtype& x) { x = std::numeric_limits<cxxtype>::min(); } \
68  static void set_highest(cxxtype& x) { x = std::numeric_limits<cxxtype>::max(); } \
69 }
70 
71 #define FLOAT_TYPE(cxxtype,mpitype) \
72 STANDARD_TYPE(cxxtype,mpitype); \
73  \
74 template<> \
75 struct Attributes<cxxtype> \
76 { \
77  static const bool has_min_max = true; \
78  static void set_lowest(cxxtype& x) { x = -std::numeric_limits<cxxtype>::max(); } \
79  static void set_highest(cxxtype& x) { x = std::numeric_limits<cxxtype>::max(); } \
80 }
81 
82 #define CONTAINER_TYPE(cxxtype) \
83 template<typename T> \
84 struct Attributes<cxxtype<T> > \
85 { \
86  static const bool has_min_max = Attributes<T>::has_min_max; \
87  static void set_lowest(cxxtype<T>& x) { \
88  for (typename cxxtype<T>::iterator i = x.begin(); i != x.end(); ++i) \
89  Attributes<T>::set_lowest(*i); } \
90  static void set_highest(cxxtype<T>& x) { \
91  for (typename cxxtype<T>::iterator i = x.begin(); i != x.end(); ++i) \
92  Attributes<T>::set_highest(*i); } \
93 }
94 
95 
96 INT_TYPE(char,MPI_CHAR);
97 #if MPI_VERSION > 1
98 INT_TYPE(signed char,MPI_SIGNED_CHAR);
99 #endif
100 INT_TYPE(unsigned char,MPI_UNSIGNED_CHAR);
101 INT_TYPE(short int,MPI_SHORT);
102 INT_TYPE(unsigned short int,MPI_UNSIGNED_SHORT);
103 INT_TYPE(int,MPI_INT);
104 INT_TYPE(unsigned int,MPI_UNSIGNED);
105 INT_TYPE(long,MPI_LONG);
106 INT_TYPE(unsigned long,MPI_UNSIGNED_LONG);
107 INT_TYPE(unsigned long long,MPI_LONG_LONG_INT);
108 FLOAT_TYPE(float,MPI_FLOAT);
109 FLOAT_TYPE(double,MPI_DOUBLE);
110 FLOAT_TYPE(long double,MPI_LONG_DOUBLE);
111 CONTAINER_TYPE(std::set);
112 CONTAINER_TYPE(std::vector);
113 
114 // We'd love to do a singleton pattern on derived data types, rather
115 // than commit, free, commit, free, ad infinitum... but it's a
116 // little tricky when our T1 and T2 are undefined.
117 template<typename T1, typename T2>
118 class StandardType<std::pair<T1, T2> > : public DataType
119 {
120 public:
121  explicit
122  StandardType(const std::pair<T1, T2> *example = NULL) {
123  // We need an example for MPI_Address to use
124  libmesh_assert(example);
125 
126 #ifdef LIBMESH_HAVE_MPI
127  // Get the sub-data-types, and make sure they live long enough
128  // to construct the derived type
129  StandardType<T1> d1(&example->first);
130  StandardType<T2> d2(&example->second);
131  MPI_Datatype types[] = { (data_type)d1, (data_type)d2 };
132  int blocklengths[] = {1,1};
133 
134  MPI_Aint displs[2];
135 #if MPI_VERSION > 1
136  MPI_Get_address (const_cast<T1*>(&example->first), &displs[0]);
137  MPI_Get_address (const_cast<T2*>(&example->second), &displs[1]);
138 #else
139  MPI_Address (const_cast<T1*>(&example->first), &displs[0]);
140  MPI_Address (const_cast<T2*>(&example->second), &displs[1]);
141 #endif
142  displs[1] -= displs[0];
143  displs[0] = 0;
144 
145 #if MPI_VERSION > 1
146  MPI_Type_create_struct (2, blocklengths, displs, types, &_datatype);
147 #else
148  MPI_Type_struct (2, blocklengths, displs, types, &_datatype);
149 #endif // #if MPI_VERSION > 1
150  MPI_Type_commit (&_datatype);
151 #endif // LIBMESH_HAVE_MPI
152  }
153 
154  ~StandardType() { this->free(); }
155 };
156 
157 template<typename T>
158 class StandardType<std::complex<T> > : public DataType
159 {
160 public:
161  explicit
162  StandardType(const std::complex<T> * /*example*/ = NULL) :
163  DataType(StandardType<T>(NULL), 2) {}
164 
165  ~StandardType() { this->free(); }
166 };
167 
168 } // namespace Parallel
169 
170 } // namespace libMesh
171 
172 
173 // Anonymous namespace for helper functions
174 namespace {
175 
176 // Safe to use this here since it won't "infect" anything outside the
177 // anonymous namespace
178 using namespace libMesh;
179 
180 // Internal helper function to create vector<something_useable> from
181 // vector<bool> for compatibility with MPI bitwise operations
182 template <typename T>
183 inline void pack_vector_bool(const std::vector<bool> &in,
184  std::vector<T> &out)
185 {
186  unsigned int data_bits = 8*sizeof(T);
187  std::size_t in_size = in.size();
188  std::size_t out_size = in_size/data_bits + (in_size%data_bits?1:0);
189  out.clear();
190  out.resize(out_size);
191  for (std::size_t i=0; i != in_size; ++i)
192  {
193  std::size_t index = i/data_bits;
194  std::size_t offset = i%data_bits;
195  out[index] += (in[i]?1:0) << offset;
196  }
197 }
198 
199 // Internal helper function to create vector<bool> from
200 // vector<something usable> for compatibility with MPI byte
201 // operations
202 template <typename T>
203 inline void unpack_vector_bool(const std::vector<T> &in,
204  std::vector<bool> &out)
205 {
206  unsigned int data_bits = 8*sizeof(T);
207  // We need the output vector to already be properly sized
208  std::size_t out_size = out.size();
209  libmesh_assert_equal_to (out_size/data_bits + (out_size%data_bits?1:0), in.size());
210 
211  for (std::size_t i=0; i != out_size; ++i)
212  {
213  std::size_t index = i/data_bits;
214  std::size_t offset = i%data_bits;
215  out[i] = in[index] << (data_bits-1-offset) >> (data_bits-1);
216  }
217 }
218 
219 
220 #ifdef LIBMESH_HAVE_MPI
221 // We use a helper function here to avoid ambiguity when calling
222 // send_receive of (vector<vector<T>>,vector<vector<T>>)
223 template <typename T1, typename T2>
224 inline void send_receive_vec_of_vec
225  (const unsigned int dest_processor_id,
226  std::vector<std::vector<T1> > &send,
227  const unsigned int source_processor_id,
228  std::vector<std::vector<T2> > &recv,
229  const Parallel::MessageTag &send_tag,
230  const Parallel::MessageTag &recv_tag,
232 {
233  START_LOG("send_receive()", "Parallel");
234 
235  if (dest_processor_id == comm.rank() &&
236  source_processor_id == comm.rank())
237  {
238  recv = send;
239  STOP_LOG("send_receive()", "Parallel");
240  return;
241  }
242 
243  // temporary buffers - these will be sized in bytes
244  // and manipulated with MPI_Pack and friends
245  std::vector<char> sendbuf, recvbuf;
246 
247  // figure out how many bytes we need to pack all the data
248  int packedsize=0, sendsize=0;
249 
250  // The outer buffer size
251  MPI_Pack_size (1,
253  comm.get(),
254  &packedsize);
255  sendsize += packedsize;
256 
257  for (std::size_t i=0; i<send.size(); i++)
258  {
259  // The size of the ith inner buffer
260  MPI_Pack_size (1,
262  comm.get(),
263  &packedsize);
264  sendsize += packedsize;
265 
266  // The data for each inner buffer
267  MPI_Pack_size (libmesh_cast_int<int>(send[i].size()),
269  (send[i].empty() ? NULL : &send[i][0]),
270  comm.get(),
271  &packedsize);
272  sendsize += packedsize;
273  }
274 
275  libmesh_assert (sendsize /* should at least be 1! */);
276  sendbuf.resize (sendsize);
277 
278  // Pack the send buffer
279  int pos=0;
280 
281  // ... the size of the outer buffer
282  sendsize = libmesh_cast_int<int>(send.size());
283  MPI_Pack (&sendsize, 1, Parallel::StandardType<unsigned int>(),
284  &sendbuf[0], libmesh_cast_int<int>(sendbuf.size()), &pos,
285  comm.get());
286 
287  for (std::size_t i=0; i<send.size(); i++)
288  {
289  // ... the size of the ith inner buffer
290  sendsize = libmesh_cast_int<int>(send[i].size());
291  MPI_Pack (&sendsize, 1, Parallel::StandardType<unsigned int>(),
292  &sendbuf[0], libmesh_cast_int<int>(sendbuf.size()), &pos,
293  comm.get());
294 
295  // ... the contents of the ith inner buffer
296  if (!send[i].empty())
297  MPI_Pack (&send[i][0], libmesh_cast_int<int>(send[i].size()),
299  &sendbuf[0], libmesh_cast_int<int>(sendbuf.size()), &pos,
300  comm.get());
301  }
302 
303  libmesh_assert_equal_to (static_cast<unsigned int>(pos), sendbuf.size());
304 
306 
307  comm.send (dest_processor_id, sendbuf, MPI_PACKED, request, send_tag);
308 
309  comm.receive (source_processor_id, recvbuf, MPI_PACKED, recv_tag);
310 
311  // Unpack the received buffer
312  libmesh_assert (!recvbuf.empty());
313  pos=0;
314  MPI_Unpack (&recvbuf[0], libmesh_cast_int<int>(recvbuf.size()), &pos,
315  &sendsize, 1, Parallel::StandardType<unsigned int>(),
316  comm.get());
317 
318  // ... size the outer buffer
319  recv.resize (sendsize);
320 
321  for (std::size_t i=0; i<recv.size(); i++)
322  {
323  MPI_Unpack (&recvbuf[0], libmesh_cast_int<int>(recvbuf.size()), &pos,
324  &sendsize, 1, Parallel::StandardType<unsigned int>(),
325  comm.get());
326 
327  // ... size the inner buffer
328  recv[i].resize (sendsize);
329 
330  // ... unpack the inner buffer if it is not empty
331  if (!recv[i].empty())
332  MPI_Unpack (&recvbuf[0], libmesh_cast_int<int>(recvbuf.size()), &pos,
333  &recv[i][0], libmesh_cast_int<int>(recv[i].size()),
334  Parallel::StandardType<T2>(&recv[i][0]),
335  comm.get());
336  }
337 
338  request.wait();
339 
340  STOP_LOG("send_receive()", "Parallel");
341 }
342 
343 #endif // LIBMESH_HAVE_MPI
344 
345 } // Anonymous namespace
346 
347 
348 
349 namespace libMesh
350 {
351 
352 namespace Parallel
353 {
354 
355 /*
356  * A reference to the default libMesh communicator. This is now
357  * deprecated - instead of libMesh::Parallel::Communicator_World use
358  * libMesh::CommWorld
359  */
360 #ifdef LIBMESH_DISABLE_COMMWORLD
361 extern FakeCommunicator& Communicator_World;
362 #else
364 #endif
365 
366 
370 template <typename Context, typename buffertype, typename Iter>
371 inline void pack_range (const Context *context,
372  Iter range_begin,
373  const Iter range_end,
374  std::vector<buffertype>& buffer)
375 {
376  // Count the total size of and preallocate buffer for efficiency
377  std::size_t buffer_size = 0;
378  for (Iter range_count = range_begin;
379  range_count != range_end;
380  ++range_count)
381  {
382  buffer_size += Parallel::packable_size(*range_count, context);
383  }
384  buffer.reserve(buffer.size() + buffer_size);
385 
386  // Pack the objects into the buffer
387  for (; range_begin != range_end; ++range_begin)
388  {
389 #ifndef NDEBUG
390  std::size_t old_size = buffer.size();
391 #endif
392 
393  Parallel::pack(*range_begin, buffer, context);
394 
395 #ifndef NDEBUG
396  unsigned int my_packable_size =
397  Parallel::packable_size(*range_begin, context);
398  unsigned int my_packed_size =
399  Parallel::packed_size (*range_begin, buffer.begin() +
400  old_size);
401  libmesh_assert_equal_to (my_packable_size, my_packed_size);
402  libmesh_assert_equal_to (buffer.size(), old_size + my_packable_size);
403 #endif
404  }
405 }
406 
407 
408 
412 template <typename Context, typename buffertype, typename OutputIter>
413 inline void unpack_range (const std::vector<buffertype>& buffer,
414  Context *context,
415  OutputIter out)
416 {
417  // Our objects should be of the correct type to be assigned to the
418  // output iterator
419  typedef typename std::iterator_traits<OutputIter>::value_type T;
420 
421  // Loop through the buffer and unpack each object, returning the
422  // object pointer via the output iterator
423  typename std::vector<buffertype>::const_iterator
424  next_object_start = buffer.begin();
425 
426  while (next_object_start < buffer.end())
427  {
428  T* obj;
429  Parallel::unpack(next_object_start, &obj, context);
430  libmesh_assert(obj);
431  next_object_start += Parallel::packed_size(obj, next_object_start);
432  *out++ = obj;
433  }
434 
435  // We should have used up the exact amount of data in the buffer
436  libmesh_assert (next_object_start == buffer.end());
437 }
438 
439 
441 #ifdef LIBMESH_HAVE_MPI
442  _communicator(MPI_COMM_NULL),
443 #endif
444  _rank(0),
445  _size(1),
446  _send_mode(DEFAULT),
447  used_tag_values(),
448  _I_duped_it(false) {}
449 
451 #ifdef LIBMESH_HAVE_MPI
452  _communicator(MPI_COMM_NULL),
453 #endif
454  _rank(0),
455  _size(1),
456  _send_mode(DEFAULT),
457  used_tag_values(),
458  _I_duped_it(false)
459 {
460  this->assign(comm);
461 }
462 
464  this->clear();
465 }
466 
467 #ifdef LIBMESH_HAVE_MPI
468 inline void Communicator::split(int color, int key, Communicator &target) {
469  MPI_Comm_split(this->get(), color, key, &target.get());
470  target.send_mode(this->send_mode());
471 }
472 #else
473 inline void Communicator::split(int, int, Communicator &target) {
474  target.assign(this->get());
475 }
476 #endif
477 
478 inline void Communicator::duplicate(const Communicator &comm) {
479  this->duplicate(comm._communicator);
480  this->send_mode(comm.send_mode());
481 }
482 
483 #ifdef LIBMESH_HAVE_MPI
484 inline void Communicator::duplicate(const communicator &comm) {
485  if (_communicator != MPI_COMM_NULL)
486  {
487  MPI_Comm_dup(comm, &_communicator);
488  _I_duped_it = true;
489  }
490  this->assign(_communicator);
491 }
492 #else
493 inline void Communicator::duplicate(const communicator &) { }
494 #endif
495 
496 inline void Communicator::clear() {
497 #ifdef LIBMESH_HAVE_MPI
498  if (_I_duped_it)
499  {
500  libmesh_assert (_communicator != MPI_COMM_NULL);
501  MPI_Comm_free(&_communicator);
502  _communicator = MPI_COMM_NULL;
503  }
504  _I_duped_it = false;
505 #endif
506 }
507 
509  this->clear();
510  this->assign(comm);
511  return *this;
512 }
513 
514 // Disallowed copy constructor
516 #ifdef LIBMESH_HAVE_MPI
517  _communicator(MPI_COMM_NULL),
518 #endif
519  _rank(0),
520  _size(1),
521  _send_mode(DEFAULT),
522  used_tag_values(),
523  _I_duped_it(false)
524 {
525  libmesh_error();
526 }
527 
528 inline void Communicator::assign(const communicator &comm)
529 {
531 #ifdef LIBMESH_HAVE_MPI
532  if (_communicator != MPI_COMM_NULL)
533  {
534  int i;
535  MPI_Comm_size(_communicator, &i);
536  libmesh_assert_greater_equal (i, 0);
537  _size = static_cast<unsigned int>(i);
538 
539  MPI_Comm_rank(_communicator, &i);
540  libmesh_assert_greater_equal (i, 0);
541  _rank = static_cast<unsigned int>(i);
542  }
543  else
544  {
545  _rank = 0;
546  _size = 1;
547  }
548 #endif
550 }
551 
552 
553 
554 inline Status::Status () :
555  _status(),
556  _datatype()
557 {}
558 
559 inline Status::Status (const data_type &type) :
560  _status(),
561  _datatype(type)
562 {}
563 
564 inline Status::Status (const status &stat) :
565  _status(stat),
566  _datatype()
567 {}
568 
569 inline Status::Status (const status &stat,
570  const data_type &type) :
571  _status(stat),
572  _datatype(type)
573 {}
574 
575 inline Status::Status (const Status &stat) :
576  _status(stat._status),
577  _datatype(stat._datatype)
578 {}
579 
580 inline Status::Status (const Status &stat,
581  const data_type &type) :
582  _status(stat._status),
583  _datatype(type)
584 {}
585 
586 inline int Status::source () const
587 {
588 #ifdef LIBMESH_HAVE_MPI
589  return _status.MPI_SOURCE;
590 #else
591  return 0;
592 #endif
593 }
594 
595 inline int Status::tag () const
596 {
597 #ifdef LIBMESH_HAVE_MPI
598  return _status.MPI_TAG;
599 #else
600  libmesh_error();
601  return 0;
602 #endif
603 }
604 
605 #ifdef LIBMESH_HAVE_MPI
606 inline unsigned int Status::size (const data_type &type) const
607 {
608  int msg_size;
609  MPI_Get_count (const_cast<MPI_Status*>(&_status), type, &msg_size);
610  libmesh_assert_greater_equal (msg_size, 0);
611  return msg_size;
612 }
613 #else
614 inline unsigned int Status::size (const data_type &) const
615 {
616  libmesh_error();
617  return 0;
618 }
619 #endif
620 
621 inline unsigned int Status::size () const
622 { return this->size (this->datatype()); }
623 
624 
625 
626 inline Request::Request () :
627 #ifdef LIBMESH_HAVE_MPI
628  _request(MPI_REQUEST_NULL),
629 #else
630  _request(),
631 #endif
632  post_wait_work(NULL)
633 {}
634 
635 inline Request::Request (const request &r) :
636  _request(r),
637  post_wait_work(NULL)
638 {}
639 
640 inline Request::Request (const Request &other) :
641  _request(other._request),
642  post_wait_work(other.post_wait_work)
643 {
644  // operator= should behave like a shared pointer
645  if (post_wait_work)
646  post_wait_work->second++;
647 }
648 
649 inline void Request::cleanup()
650 {
651  if (post_wait_work)
652  {
653  // Decrement the use count
654  post_wait_work->second--;
655 
656  if (!post_wait_work->second)
657  {
658 #ifdef DEBUG
659  // If we're done using this request, then we'd better have
660  // done the work we waited for
661  for (std::vector<PostWaitWork*>::iterator i =
662  post_wait_work->first.begin();
663  i != post_wait_work->first.end(); ++i)
664  libmesh_assert(!(*i));
665 #endif
666  delete post_wait_work;
667  post_wait_work = NULL;
668  }
669  }
670 }
671 
672 inline Request& Request::operator = (const Request &other)
673 {
674  this->cleanup();
675  _request = other._request;
677 
678  // operator= should behave like a shared pointer
679  if (post_wait_work)
680  post_wait_work->second++;
681 
682  return *this;
683 }
684 
685 inline Request& Request::operator = (const request &r)
686 {
687  this->cleanup();
688  _request = r;
689  post_wait_work = NULL;
690  return *this;
691 }
692 
693 inline Request::~Request () {
694  this->cleanup();
695 }
696 
698 {
699  START_LOG("wait()", "Parallel::Request");
700 
701  Status stat;
702 #ifdef LIBMESH_HAVE_MPI
703  MPI_Wait (&_request, stat.get());
704 #endif
705  if (post_wait_work)
706  for (std::vector<PostWaitWork*>::iterator i =
707  post_wait_work->first.begin();
708  i != post_wait_work->first.end(); ++i)
709  {
710  // The user should never try to give us NULL work or try
711  // to wait() twice.
712  libmesh_assert (*i);
713  (*i)->run();
714  delete (*i);
715  *i = NULL;
716  }
717 
718  STOP_LOG("wait()", "Parallel::Request");
719  return stat;
720 }
721 
722 inline bool Request::test ()
723 {
724 #ifdef LIBMESH_HAVE_MPI
725  int val=0;
726 
727  MPI_Test (&_request,
728  &val,
729  MPI_STATUS_IGNORE);
730  if (val)
731  {
732  libmesh_assert (_request == MPI_REQUEST_NULL);
733  libmesh_assert_equal_to (val, 1);
734  }
735 
736  return val;
737 #else
738  return true;
739 #endif
740 }
741 
742 #ifdef LIBMESH_HAVE_MPI
743 inline bool Request::test (status &stat)
744 {
745  int val=0;
746 
747  MPI_Test (&_request,
748  &val,
749  &stat);
750 
751  return val;
752 }
753 #else
754 inline bool Request::test (status &)
755 {
756  return true;
757 }
758 #endif
759 
761 {
762  if (!post_wait_work)
763  post_wait_work = new
764  std::pair<std::vector <PostWaitWork* >, unsigned int>
765  (std::vector <PostWaitWork* >(), 1);
766  post_wait_work->first.push_back(work);
767 }
768 
769 
770 
774 #ifdef LIBMESH_HAVE_MPI
775 inline void Communicator::barrier () const
776 {
777  if (this->size() > 1)
778  {
779  START_LOG("barrier()", "Parallel");
780 
781  MPI_Barrier (this->get());
782 
783  STOP_LOG("barrier()", "Parallel");
784  }
785 }
786 #else
787 inline void Communicator::barrier () const {}
788 #endif
789 
790 
791 // legacy e.g. Paralell::send() methods, requires
792 // Communicator_World
793 #ifndef LIBMESH_DISABLE_COMMWORLD
794 inline void barrier (const Communicator &comm = Communicator_World)
795 {
796  comm.barrier();
797 }
798 
799 template <typename T>
800 inline bool verify(const T &r,
801  const Communicator &comm = Communicator_World)
802 { return comm.verify(r); }
803 
804 template <typename T>
805 inline void min(T &r,
806  const Communicator &comm = Communicator_World)
807 { comm.min(r); }
808 
809 template <typename T, typename U>
810 inline void minloc(T &r,
811  U &min_id,
812  const Communicator &comm = Communicator_World)
813 { comm.minloc(r, min_id); }
814 
815 template <typename T>
816 inline void max(T &r,
817  const Communicator &comm = Communicator_World)
818 { comm.max(r); }
819 
820 template <typename T, typename U>
821 inline void maxloc(T &r,
822  U &max_id,
823  const Communicator &comm = Communicator_World)
824 { comm.maxloc(r, max_id); }
825 
826 template <typename T>
827 inline void sum(T &r,
828  const Communicator &comm = Communicator_World)
829 { comm.sum(r); }
830 
831 template <typename T>
832 inline void set_union(T &data, const unsigned int root_id,
833  const Communicator &comm = Communicator_World)
834 { comm.set_union(data, root_id); }
835 
836 template <typename T>
837 inline void set_union(T &data,
838  const Communicator &comm = Communicator_World)
839 { comm.set_union(data); }
840 
841 inline status probe (const unsigned int src_processor_id,
842  const MessageTag &tag=any_tag,
843  const Communicator &comm = Communicator_World)
844 { return comm.probe(src_processor_id, tag); }
845 
846 template <typename T>
847 inline void send (const unsigned int dest_processor_id,
848  T &data,
849  const MessageTag &tag=no_tag,
850  const Communicator &comm = Communicator_World)
851 { comm.send(dest_processor_id, data, tag); }
852 
853 template <typename T>
854 inline void send (const unsigned int dest_processor_id,
855  T &data,
856  Request &req,
857  const MessageTag &tag=no_tag,
858  const Communicator &comm = Communicator_World)
859 { comm.send(dest_processor_id, data, req, tag); }
860 
861 template <typename T>
862 inline void send (const unsigned int dest_processor_id,
863  T &data,
864  const DataType &type,
865  const MessageTag &tag=no_tag,
866  const Communicator &comm = Communicator_World)
867 { comm.send(dest_processor_id, data, type, tag); }
868 
869 template <typename T>
870 inline void send (const unsigned int dest_processor_id,
871  T &data,
872  const DataType &type,
873  Request &req,
874  const MessageTag &tag=no_tag,
875  const Communicator &comm = Communicator_World)
876 { comm.send(dest_processor_id, data, type, req, tag); }
877 
878 
879 template <typename Context, typename Iter>
880 inline void send_packed_range (const unsigned int dest_processor_id,
881  const Context *context,
882  Iter range_begin,
883  const Iter range_end,
884  const MessageTag &tag=no_tag,
885  const Communicator &comm = Communicator_World)
886 { comm.send_packed_range(dest_processor_id, context, range_begin, range_end, tag); }
887 
888 
889 template <typename Context, typename Iter>
890 inline void send_packed_range (const unsigned int dest_processor_id,
891  const Context *context,
892  Iter range_begin,
893  const Iter range_end,
894  Request &req,
895  const MessageTag &tag=no_tag,
896  const Communicator &comm = Communicator_World)
897 { comm.send_packed_range(dest_processor_id, context, range_begin, range_end, req, tag); }
898 
899 
900 template <typename T>
901 inline void nonblocking_send (const unsigned int dest_processor_id,
902  T &buf,
903  const DataType &type,
904  Request &r,
905  const MessageTag &tag=no_tag,
906  const Communicator &comm = Communicator_World)
907 { comm.send (dest_processor_id, buf, type, r, tag); }
908 
909 template <typename T>
910 inline void nonblocking_send (const unsigned int dest_processor_id,
911  T &buf,
912  Request &r,
913  const MessageTag &tag=no_tag,
914  const Communicator &comm = Communicator_World)
915 { comm.send (dest_processor_id, buf, r, tag); }
916 
917 template <typename T>
918 inline Status receive (const unsigned int src_processor_id,
919  T &buf,
920  const MessageTag &tag=any_tag,
921  const Communicator &comm = Communicator_World)
922 { return comm.receive (src_processor_id, buf, tag); }
923 
924 template <typename T>
925 inline void receive (const unsigned int src_processor_id,
926  T &buf,
927  Request &req,
928  const MessageTag &tag=any_tag,
929  const Communicator &comm = Communicator_World)
930 { comm.receive (src_processor_id, buf, req, tag); }
931 
932 template <typename T>
933 inline Status receive (const unsigned int src_processor_id,
934  T &buf,
935  const DataType &type,
936  const MessageTag &tag=any_tag,
937  const Communicator &comm = Communicator_World)
938 { return comm.receive (src_processor_id, buf, type, tag); }
939 
940 template <typename T>
941 inline void receive (const unsigned int src_processor_id,
942  T &buf,
943  const DataType &type,
944  Request &req,
945  const MessageTag &tag=any_tag,
946  const Communicator &comm = Communicator_World)
947 { comm.receive (src_processor_id, buf, type, req, tag); }
948 
949 template <typename Context, typename OutputIter>
950 inline void receive_packed_range (const unsigned int src_processor_id,
951  Context *context,
952  OutputIter out,
953  const MessageTag &tag=any_tag,
954  const Communicator &comm = Communicator_World)
955 { comm.receive_packed_range (src_processor_id, context, out, tag); }
956 
957 template <typename Context, typename OutputIter>
958 inline void receive_packed_range (const unsigned int src_processor_id,
959  Context *context,
960  OutputIter out,
961  Request &req,
962  const MessageTag &tag=any_tag,
963  const Communicator &comm = Communicator_World)
964 { comm.receive_packed_range (src_processor_id, context, out, req, tag); }
965 
966 template <typename T>
967 inline void nonblocking_receive (const unsigned int src_processor_id,
968  T &buf,
969  const DataType &type,
970  Request &r,
971  const MessageTag &tag=any_tag,
972  const Communicator &comm = Communicator_World)
973 { comm.receive (src_processor_id, buf, type, r, tag); }
974 
975 template <typename T>
976 inline void nonblocking_receive (const unsigned int src_processor_id,
977  T &buf,
978  Request &r,
979  const MessageTag &tag=any_tag,
980  const Communicator &comm = Communicator_World)
981 { comm.receive (src_processor_id, buf, r, tag); }
982 
983 template <typename T1, typename T2>
984 inline void send_receive(const unsigned int dest_processor_id,
985  T1 &send,
986  const unsigned int source_processor_id,
987  T2 &recv,
988  const MessageTag &send_tag = no_tag,
989  const MessageTag &recv_tag = any_tag,
990  const Communicator &comm = Communicator_World)
991 { comm.send_receive(dest_processor_id, send, source_processor_id, recv,
992  send_tag, recv_tag); }
993 
994 template <typename Context1, typename RangeIter, typename Context2, typename OutputIter>
995 inline void send_receive_packed_range(const unsigned int dest_processor_id,
996  const Context1* context1,
997  RangeIter send_begin,
998  const RangeIter send_end,
999  const unsigned int source_processor_id,
1000  Context2* context2,
1001  OutputIter out,
1002  const MessageTag &send_tag = no_tag,
1003  const MessageTag &recv_tag = any_tag,
1004  const Communicator &comm = Communicator_World)
1005 { comm.send_receive_packed_range(dest_processor_id, context1, send_begin, send_end,
1006  source_processor_id, context2, out, send_tag, recv_tag); }
1007 
1008 template <typename T1, typename T2>
1009 inline void send_receive(const unsigned int dest_processor_id,
1010  T1 &send,
1011  const DataType &type1,
1012  const unsigned int source_processor_id,
1013  T2 &recv,
1014  const DataType &type2,
1015  const MessageTag &send_tag = no_tag,
1016  const MessageTag &recv_tag = any_tag,
1017  const Communicator &comm = Communicator_World)
1018 { comm.send_receive(dest_processor_id, send, type1, source_processor_id,
1019  recv, type2, send_tag, recv_tag); }
1020 
1021 template <typename T>
1022 inline void gather(const unsigned int root_id,
1023  T send,
1024  std::vector<T> &recv,
1025  const Communicator &comm = Communicator_World)
1026 { comm.gather(root_id, send, recv); }
1027 
1028 template <typename T>
1029 inline void gather(const unsigned int root_id,
1030  std::vector<T> &r,
1031  const Communicator &comm = Communicator_World)
1032 { comm.gather(root_id, r); }
1033 
1034 template <typename T>
1035 inline void allgather(T send,
1036  std::vector<T> &recv,
1037  const Communicator &comm = Communicator_World)
1038 { comm.allgather(send, recv); }
1039 
1040 template <typename T>
1041 inline void allgather(std::vector<T> &r,
1042  const bool identical_buffer_sizes = false,
1043  const Communicator &comm = Communicator_World)
1044 { comm.allgather(r, identical_buffer_sizes); }
1045 
1046 template <typename Context, typename Iter, typename OutputIter>
1047 inline void gather_packed_range (const unsigned int root_id,
1048  Context *context,
1049  Iter range_begin,
1050  const Iter range_end,
1051  OutputIter out,
1052  const Communicator &comm = Communicator_World)
1053 { comm.gather_packed_range(root_id, context, range_begin, range_end, out); }
1054 
1055 template <typename Context, typename Iter, typename OutputIter>
1056 inline void allgather_packed_range (Context *context,
1057  Iter range_begin,
1058  const Iter range_end,
1059  OutputIter out,
1060  const Communicator &comm = Communicator_World)
1061 { comm.allgather_packed_range(context, range_begin, range_end, out); }
1062 
1063 template <typename T>
1064 inline void alltoall(std::vector<T> &r,
1065  const Communicator &comm = Communicator_World)
1066 { comm.alltoall(r); }
1067 
1068 template <typename T>
1069 inline void broadcast(T &data, const unsigned int root_id=0,
1070  const Communicator &comm = Communicator_World)
1071 { comm.broadcast(data, root_id); }
1072 
1073 template <typename Context, typename OutputContext, typename Iter, typename OutputIter>
1074 inline void broadcast_packed_range (const Context *context1,
1075  Iter range_begin,
1076  const Iter range_end,
1077  OutputContext *context2,
1078  OutputIter out,
1079  const unsigned int root_id = 0,
1080  const Communicator &comm = Communicator_World)
1081 { comm.broadcast_packed_range(context1, range_begin, range_end, context2, out, root_id); }
1082 
1083 #endif // #ifndef LIBMESH_DISABLE_COMMWORLD
1084 
1085 //-----------------------------------------------------------------------
1086 // Parallel members
1087 
1088 inline
1090 {
1091  if (_comm)
1093 }
1094 
1095 
1096 inline
1098  : _tagvalue(other._tagvalue), _comm(other._comm)
1099 {
1100  if (_comm)
1102 }
1103 
1104 
1105 inline
1107 {
1108  if (used_tag_values.count(tagvalue))
1109  {
1110  // Get the largest value in the used values, and pick one
1111  // larger
1112  tagvalue = used_tag_values.rbegin()->first+1;
1113  libmesh_assert(!used_tag_values.count(tagvalue));
1114  }
1115  used_tag_values[tagvalue] = 1;
1116 
1117 // #ifndef NDEBUG
1118 // // Make sure everyone called get_unique_tag and make sure
1119 // // everyone got the same value
1120 // int maxval = tagvalue;
1121 // this->max(maxval);
1122 // libmesh_assert_equal_to (tagvalue, maxval);
1123 // #endif
1124 
1125  return MessageTag(tagvalue, this);
1126 }
1127 
1128 
1129 inline
1130 void Communicator::reference_unique_tag(int tagvalue) const
1131 {
1132  // This has better be an already-acquired tag.
1133  libmesh_assert(used_tag_values.count(tagvalue));
1134 
1135  used_tag_values[tagvalue]++;
1136 }
1137 
1138 
1139 inline
1141 {
1142  // This has better be an already-acquired tag.
1143  libmesh_assert(used_tag_values.count(tagvalue));
1144 
1145  used_tag_values[tagvalue]--;
1146  // If we don't have any more outstanding references, we
1147  // don't even need to keep this tag in our "used" set.
1148  if (!used_tag_values[tagvalue])
1149  used_tag_values.erase(tagvalue);
1150 }
1151 
1152 
1153 #ifdef LIBMESH_HAVE_MPI
1154 template<>
1155 inline data_type dataplusint_type<short int>() { return MPI_SHORT_INT; }
1156 
1157 template<>
1158 inline data_type dataplusint_type<int>() { return MPI_2INT; }
1159 
1160 template<>
1161 inline data_type dataplusint_type<long>() { return MPI_LONG_INT; }
1162 
1163 template<>
1164 inline data_type dataplusint_type<float>() { return MPI_FLOAT_INT; }
1165 
1166 template<>
1167 inline data_type dataplusint_type<double>() { return MPI_DOUBLE_INT; }
1168 
1169 template<>
1170 inline data_type dataplusint_type<long double>() { return MPI_LONG_DOUBLE_INT; }
1171 
1172 template <typename T>
1173 inline bool Communicator::verify(const T &r) const
1174 {
1175  if (this->size() > 1 && Attributes<T>::has_min_max == true)
1176  {
1177  T tempmin = r, tempmax = r;
1178  this->min(tempmin);
1179  this->max(tempmax);
1180  bool verified = (r == tempmin) &&
1181  (r == tempmax);
1182  this->min(verified);
1183  return verified;
1184  }
1185  return true;
1186 }
1187 
1188 
1189 
1190 template <typename T>
1191 inline bool Communicator::semiverify(const T *r) const
1192 {
1193  if (this->size() > 1 && Attributes<T>::has_min_max == true)
1194  {
1195  T tempmin, tempmax;
1196  if (r)
1197  tempmin = tempmax = *r;
1198  else
1199  {
1200  Attributes<T>::set_highest(tempmin);
1201  Attributes<T>::set_lowest(tempmax);
1202  }
1203  this->min(tempmin);
1204  this->max(tempmax);
1205  bool invalid = r && ((*r != tempmin) &&
1206  (*r != tempmax));
1207  this->max(invalid);
1208  return !invalid;
1209  }
1210  return true;
1211 }
1212 
1213 
1214 
1215 template <typename T>
1216 inline bool Communicator::semiverify(const std::vector<T> *r) const
1217 {
1218  if (this->size() > 1 && Attributes<T>::has_min_max == true)
1219  {
1220  std::size_t rsize = r ? r->size() : 0;
1221  std::size_t *psize = r ? &rsize : NULL;
1222 
1223  if (!this->semiverify(psize))
1224  return false;
1225 
1226  this->max(rsize);
1227 
1228  std::vector<T> tempmin, tempmax;
1229  if (r)
1230  {
1231  tempmin = tempmax = *r;
1232  }
1233  else
1234  {
1235  tempmin.resize(rsize);
1236  tempmax.resize(rsize);
1237  Attributes<std::vector<T> >::set_highest(tempmin);
1238  Attributes<std::vector<T> >::set_lowest(tempmax);
1239  }
1240  this->min(tempmin);
1241  this->max(tempmax);
1242  bool invalid = r && ((*r != tempmin) &&
1243  (*r != tempmax));
1244  this->max(invalid);
1245  return !invalid;
1246  }
1247  return true;
1248 }
1249 
1250 
1251 
1252 inline bool Communicator::verify(const std::string & r) const
1253 {
1254  if (this->size() > 1)
1255  {
1256  // Cannot use <char> since MPI_MIN is not
1257  // strictly defined for chars!
1258  std::vector<short int> temp; temp.reserve(r.size());
1259  for (std::size_t i=0; i != r.size(); ++i)
1260  temp.push_back(r[i]);
1261  return this->verify(temp);
1262  }
1263  return true;
1264 }
1265 
1266 
1267 
1268 inline bool Communicator::semiverify(const std::string * r) const
1269 {
1270  if (this->size() > 1)
1271  {
1272  std::size_t rsize = r ? r->size() : 0;
1273  std::size_t *psize = r ? &rsize : NULL;
1274 
1275  if (!this->semiverify(psize))
1276  return false;
1277 
1278  this->max(rsize);
1279 
1280  // Cannot use <char> since MPI_MIN is not
1281  // strictly defined for chars!
1282  std::vector<short int> temp (rsize);
1283  if (r)
1284  {
1285  temp.reserve(rsize);
1286  for (std::size_t i=0; i != rsize; ++i)
1287  temp.push_back((*r)[i]);
1288  }
1289 
1290  std::vector<short int> *ptemp = r ? &temp: NULL;
1291 
1292  return this->semiverify(ptemp);
1293  }
1294  return true;
1295 }
1296 
1297 
1298 
1299 template <typename T>
1300 inline void Communicator::min(T &r) const
1301 {
1302  if (this->size() > 1)
1303  {
1304  START_LOG("min(scalar)", "Parallel");
1305 
1306  T temp = r;
1307  MPI_Allreduce (&temp,
1308  &r,
1309  1,
1310  StandardType<T>(&temp),
1311  MPI_MIN,
1312  this->get());
1313 
1314  STOP_LOG("min(scalar)", "Parallel");
1315  }
1316 }
1317 
1318 
1319 inline void Communicator::min(bool &r) const
1320 {
1321  if (this->size() > 1)
1322  {
1323  START_LOG("min(bool)", "Parallel");
1324 
1325  unsigned int tempsend = r;
1326  unsigned int temp;
1327  MPI_Allreduce (&tempsend,
1328  &temp,
1329  1,
1331  MPI_MIN,
1332  this->get());
1333  r = temp;
1334 
1335  STOP_LOG("min(bool)", "Parallel");
1336  }
1337 }
1338 
1339 
1340 template <typename T>
1341 inline void Communicator::min(std::vector<T> &r) const
1342 {
1343  if (this->size() > 1 && !r.empty())
1344  {
1345  START_LOG("min(vector)", "Parallel");
1346 
1347  libmesh_assert(this->verify(r.size()));
1348 
1349  std::vector<T> temp(r);
1350  MPI_Allreduce (&temp[0],
1351  &r[0],
1352  libmesh_cast_int<int>(r.size()),
1353  StandardType<T>(&temp[0]),
1354  MPI_MIN,
1355  this->get());
1356 
1357  STOP_LOG("min(vector)", "Parallel");
1358  }
1359 }
1360 
1361 
1362 inline void Communicator::min(std::vector<bool> &r) const
1363 {
1364  if (this->size() > 1 && !r.empty())
1365  {
1366  START_LOG("min(vector<bool>)", "Parallel");
1367 
1368  libmesh_assert(this->verify(r.size()));
1369 
1370  std::vector<unsigned int> ruint;
1371  pack_vector_bool(r, ruint);
1372  std::vector<unsigned int> temp(ruint.size());
1373  MPI_Allreduce (&ruint[0],
1374  &temp[0],
1375  libmesh_cast_int<int>(ruint.size()),
1377  MPI_BAND,
1378  this->get());
1379  unpack_vector_bool(temp, r);
1380 
1381  STOP_LOG("min(vector<bool>)", "Parallel");
1382  }
1383 }
1384 
1385 
1386 template <typename T>
1387 inline void Communicator::minloc(T &r,
1388  unsigned int &min_id) const
1389 {
1390  if (this->size() > 1)
1391  {
1392  START_LOG("minloc(scalar)", "Parallel");
1393 
1394  DataPlusInt<T> in;
1395  in.val = r;
1396  in.rank = this->rank();
1398  MPI_Allreduce (&in,
1399  &out,
1400  1,
1401  dataplusint_type<T>(),
1402  MPI_MINLOC,
1403  this->get());
1404  r = out.val;
1405  min_id = out.rank;
1406 
1407  STOP_LOG("minloc(scalar)", "Parallel");
1408  }
1409  else
1410  min_id = this->rank();
1411 }
1412 
1413 
1414 inline void Communicator::minloc(bool &r,
1415  unsigned int &min_id) const
1416 {
1417  if (this->size() > 1)
1418  {
1419  START_LOG("minloc(bool)", "Parallel");
1420 
1421  DataPlusInt<int> in;
1422  in.val = r;
1423  in.rank = this->rank();
1425  MPI_Allreduce (&in,
1426  &out,
1427  1,
1429  MPI_MINLOC,
1430  this->get());
1431  r = out.val;
1432  min_id = out.rank;
1433 
1434  STOP_LOG("minloc(bool)", "Parallel");
1435  }
1436  else
1437  min_id = this->rank();
1438 }
1439 
1440 
1441 template <typename T>
1442 inline void Communicator::minloc(std::vector<T> &r,
1443  std::vector<unsigned int> &min_id) const
1444 {
1445  if (this->size() > 1 && !r.empty())
1446  {
1447  START_LOG("minloc(vector)", "Parallel");
1448 
1449  libmesh_assert(this->verify(r.size()));
1450 
1451  std::vector<DataPlusInt<T> > in(r.size());
1452  for (std::size_t i=0; i != r.size(); ++i)
1453  {
1454  in[i].val = r[i];
1455  in[i].rank = this->rank();
1456  }
1457  std::vector<DataPlusInt<T> > out(r.size());
1458  MPI_Allreduce (&in[0],
1459  &out[0],
1460  libmesh_cast_int<int>(r.size()),
1461  dataplusint_type<T>(),
1462  MPI_MINLOC,
1463  this->get());
1464  for (std::size_t i=0; i != r.size(); ++i)
1465  {
1466  r[i] = out[i].val;
1467  min_id[i] = out[i].rank;
1468  }
1469 
1470  STOP_LOG("minloc(vector)", "Parallel");
1471  }
1472  else if (!r.empty())
1473  {
1474  for (std::size_t i=0; i != r.size(); ++i)
1475  min_id[i] = this->rank();
1476  }
1477 }
1478 
1479 
1480 inline void Communicator::minloc(std::vector<bool> &r,
1481  std::vector<unsigned int> &min_id) const
1482 {
1483  if (this->size() > 1 && !r.empty())
1484  {
1485  START_LOG("minloc(vector<bool>)", "Parallel");
1486 
1487  libmesh_assert(this->verify(r.size()));
1488 
1489  std::vector<DataPlusInt<int> > in(r.size());
1490  for (std::size_t i=0; i != r.size(); ++i)
1491  {
1492  in[i].val = r[i];
1493  in[i].rank = this->rank();
1494  }
1495  std::vector<DataPlusInt<int> > out(r.size());
1496  MPI_Allreduce (&in[0],
1497  &out[0],
1498  libmesh_cast_int<int>(r.size()),
1499  StandardType<int>(),
1500  MPI_MINLOC,
1501  this->get());
1502  for (std::size_t i=0; i != r.size(); ++i)
1503  {
1504  r[i] = out[i].val;
1505  min_id[i] = out[i].rank;
1506  }
1507 
1508  STOP_LOG("minloc(vector<bool>)", "Parallel");
1509  }
1510  else if (!r.empty())
1511  {
1512  for (std::size_t i=0; i != r.size(); ++i)
1513  min_id[i] = this->rank();
1514  }
1515 }
1516 
1517 
1518 template <typename T>
1519 inline void Communicator::max(T &r) const
1520 {
1521  if (this->size() > 1)
1522  {
1523  START_LOG("max(scalar)", "Parallel");
1524 
1525  T temp;
1526  MPI_Allreduce (&r,
1527  &temp,
1528  1,
1529  StandardType<T>(&r),
1530  MPI_MAX,
1531  this->get());
1532  r = temp;
1533 
1534  STOP_LOG("max(scalar)", "Parallel");
1535  }
1536 }
1537 
1538 
1539 inline void Communicator::max(bool &r) const
1540 {
1541  if (this->size() > 1)
1542  {
1543  START_LOG("max(bool)", "Parallel");
1544 
1545  unsigned int tempsend = r;
1546  unsigned int temp;
1547  MPI_Allreduce (&tempsend,
1548  &temp,
1549  1,
1551  MPI_MAX,
1552  this->get());
1553  r = temp;
1554 
1555  STOP_LOG("max(bool)", "Parallel");
1556  }
1557 }
1558 
1559 
1560 template <typename T>
1561 inline void Communicator::max(std::vector<T> &r) const
1562 {
1563  if (this->size() > 1 && !r.empty())
1564  {
1565  START_LOG("max(vector)", "Parallel");
1566 
1567  libmesh_assert(this->verify(r.size()));
1568 
1569  std::vector<T> temp(r);
1570  MPI_Allreduce (&temp[0],
1571  &r[0],
1572  libmesh_cast_int<int>(r.size()),
1573  StandardType<T>(&temp[0]),
1574  MPI_MAX,
1575  this->get());
1576 
1577  STOP_LOG("max(vector)", "Parallel");
1578  }
1579 }
1580 
1581 
1582 inline void Communicator::max(std::vector<bool> &r) const
1583 {
1584  if (this->size() > 1 && !r.empty())
1585  {
1586  START_LOG("max(vector<bool>)", "Parallel");
1587 
1588  libmesh_assert(this->verify(r.size()));
1589 
1590  std::vector<unsigned int> ruint;
1591  pack_vector_bool(r, ruint);
1592  std::vector<unsigned int> temp(ruint.size());
1593  MPI_Allreduce (&ruint[0],
1594  &temp[0],
1595  libmesh_cast_int<int>(ruint.size()),
1597  MPI_BOR,
1598  this->get());
1599  unpack_vector_bool(temp, r);
1600 
1601  STOP_LOG("max(vector<bool>)", "Parallel");
1602  }
1603 }
1604 
1605 
1606 template <typename T>
1607 inline void Communicator::maxloc(T &r,
1608  unsigned int &max_id) const
1609 {
1610  if (this->size() > 1)
1611  {
1612  START_LOG("maxloc(scalar)", "Parallel");
1613 
1614  DataPlusInt<T> in;
1615  in.val = r;
1616  in.rank = this->rank();
1618  MPI_Allreduce (&in,
1619  &out,
1620  1,
1621  dataplusint_type<T>(),
1622  MPI_MAXLOC,
1623  this->get());
1624  r = out.val;
1625  max_id = out.rank;
1626 
1627  STOP_LOG("maxloc(scalar)", "Parallel");
1628  }
1629  else
1630  max_id = this->rank();
1631 }
1632 
1633 
1634 inline void Communicator::maxloc(bool &r,
1635  unsigned int &max_id) const
1636 {
1637  if (this->size() > 1)
1638  {
1639  START_LOG("maxloc(bool)", "Parallel");
1640 
1641  DataPlusInt<int> in;
1642  in.val = r;
1643  in.rank = this->rank();
1645  MPI_Allreduce (&in,
1646  &out,
1647  1,
1649  MPI_MAXLOC,
1650  this->get());
1651  r = out.val;
1652  max_id = out.rank;
1653 
1654  STOP_LOG("maxloc(bool)", "Parallel");
1655  }
1656  else
1657  max_id = this->rank();
1658 }
1659 
1660 
1661 template <typename T>
1662 inline void Communicator::maxloc(std::vector<T> &r,
1663  std::vector<unsigned int> &max_id) const
1664 {
1665  if (this->size() > 1 && !r.empty())
1666  {
1667  START_LOG("maxloc(vector)", "Parallel");
1668 
1669  libmesh_assert(this->verify(r.size()));
1670 
1671  std::vector<DataPlusInt<T> > in(r.size());
1672  for (std::size_t i=0; i != r.size(); ++i)
1673  {
1674  in[i].val = r[i];
1675  in[i].rank = this->rank();
1676  }
1677  std::vector<DataPlusInt<T> > out(r.size());
1678  MPI_Allreduce (&in[0],
1679  &out[0],
1680  libmesh_cast_int<int>(r.size()),
1681  dataplusint_type<T>(),
1682  MPI_MAXLOC,
1683  this->get());
1684  for (std::size_t i=0; i != r.size(); ++i)
1685  {
1686  r[i] = out[i].val;
1687  max_id[i] = out[i].rank;
1688  }
1689 
1690  STOP_LOG("maxloc(vector)", "Parallel");
1691  }
1692  else if (!r.empty())
1693  {
1694  for (std::size_t i=0; i != r.size(); ++i)
1695  max_id[i] = this->rank();
1696  }
1697 }
1698 
1699 
1700 inline void Communicator::maxloc(std::vector<bool> &r,
1701  std::vector<unsigned int> &max_id) const
1702 {
1703  if (this->size() > 1 && !r.empty())
1704  {
1705  START_LOG("maxloc(vector<bool>)", "Parallel");
1706 
1707  libmesh_assert(this->verify(r.size()));
1708 
1709  std::vector<DataPlusInt<int> > in(r.size());
1710  for (std::size_t i=0; i != r.size(); ++i)
1711  {
1712  in[i].val = r[i];
1713  in[i].rank = this->rank();
1714  }
1715  std::vector<DataPlusInt<int> > out(r.size());
1716  MPI_Allreduce (&in[0],
1717  &out[0],
1718  libmesh_cast_int<int>(r.size()),
1719  StandardType<int>(),
1720  MPI_MAXLOC,
1721  this->get());
1722  for (std::size_t i=0; i != r.size(); ++i)
1723  {
1724  r[i] = out[i].val;
1725  max_id[i] = out[i].rank;
1726  }
1727 
1728  STOP_LOG("maxloc(vector<bool>)", "Parallel");
1729  }
1730  else if (!r.empty())
1731  {
1732  for (std::size_t i=0; i != r.size(); ++i)
1733  max_id[i] = this->rank();
1734  }
1735 }
1736 
1737 
1738 template <typename T>
1739 inline void Communicator::sum(T &r) const
1740 {
1741  if (this->size() > 1)
1742  {
1743  START_LOG("sum()", "Parallel");
1744 
1745  T temp = r;
1746  MPI_Allreduce (&temp,
1747  &r,
1748  1,
1749  StandardType<T>(&temp),
1750  MPI_SUM,
1751  this->get());
1752 
1753  STOP_LOG("sum()", "Parallel");
1754  }
1755 }
1756 
1757 
1758 template <typename T>
1759 inline void Communicator::sum(std::vector<T> &r) const
1760 {
1761  if (this->size() > 1 && !r.empty())
1762  {
1763  START_LOG("sum()", "Parallel");
1764 
1765  libmesh_assert(this->verify(r.size()));
1766 
1767  std::vector<T> temp(r);
1768  MPI_Allreduce (&temp[0],
1769  &r[0],
1770  libmesh_cast_int<int>(r.size()),
1771  StandardType<T>(&temp[0]),
1772  MPI_SUM,
1773  this->get());
1774 
1775  STOP_LOG("sum()", "Parallel");
1776  }
1777 }
1778 
1779 
1780 // We still do function overloading for complex sums - in a perfect
1781 // world we'd have a StandardSumOp to go along with StandardType...
1782 template <typename T>
1783 inline void Communicator::sum(std::complex<T> &r) const
1784 {
1785  if (this->size() > 1)
1786  {
1787  START_LOG("sum()", "Parallel");
1788 
1789  std::complex<T> temp(r);
1790  MPI_Allreduce (&temp,
1791  &r,
1792  2,
1793  StandardType<T>(),
1794  MPI_SUM,
1795  this->get());
1796 
1797  STOP_LOG("sum()", "Parallel");
1798  }
1799 }
1800 
1801 
1802 template <typename T>
1803 inline void Communicator::sum(std::vector<std::complex<T> > &r) const
1804 {
1805  if (this->size() > 1 && !r.empty())
1806  {
1807  START_LOG("sum()", "Parallel");
1808 
1809  libmesh_assert(this->verify(r.size()));
1810 
1811  std::vector<std::complex<T> > temp(r);
1812  MPI_Allreduce (&temp[0],
1813  &r[0],
1814  libmesh_cast_int<int>(r.size() * 2),
1815  StandardType<T>(NULL),
1816  MPI_SUM,
1817  this->get());
1818 
1819  STOP_LOG("sum()", "Parallel");
1820  }
1821 }
1822 
1823 
1824 template <typename T>
1825 inline void Communicator::set_union(std::set<T> &data,
1826  const unsigned int root_id) const
1827 {
1828  std::vector<T> vecdata(data.begin(), data.end());
1829  this->gather(root_id, vecdata);
1830  if (this->rank() == root_id)
1831  data.insert(vecdata.begin(), vecdata.end());
1832 }
1833 
1834 
1835 
1836 template <typename T>
1837 inline void Communicator::set_union(std::set<T> &data) const
1838 {
1839  std::vector<T> vecdata(data.begin(), data.end());
1840  this->allgather(vecdata, false);
1841  data.insert(vecdata.begin(), vecdata.end());
1842 }
1843 
1844 
1845 
1846 template <typename T1, typename T2>
1847 inline void Communicator::set_union(std::map<T1,T2> &data,
1848  const unsigned int root_id) const
1849 {
1850  std::vector<std::pair<T1,T2> > vecdata(data.begin(), data.end());
1851  this->gather(root_id, vecdata);
1852  if (this->rank() == root_id)
1853  data.insert(vecdata.begin(), vecdata.end());
1854 }
1855 
1856 
1857 
1858 template <typename T1, typename T2>
1859 inline void Communicator::set_union(std::map<T1,T2> &data) const
1860 {
1861  std::vector<std::pair<T1,T2> > vecdata(data.begin(), data.end());
1862  this->allgather(vecdata, false);
1863  data.insert(vecdata.begin(), vecdata.end());
1864 }
1865 
1866 
1867 
1868 inline status Communicator::probe (const unsigned int src_processor_id,
1869  const MessageTag &tag) const
1870 {
1871  START_LOG("probe()", "Parallel");
1872 
1873  status stat;
1874 
1875  MPI_Probe (src_processor_id,
1876  tag.value(),
1877  this->get(),
1878  &stat);
1879 
1880  STOP_LOG("probe()", "Parallel");
1881 
1882  return stat;
1883 }
1884 
1885 
1886 
1887 template<typename T>
1888 inline void Communicator::send (const unsigned int dest_processor_id,
1889  std::basic_string<T> &buf,
1890  const MessageTag &tag) const
1891 {
1892  START_LOG("send()", "Parallel");
1893 
1894  T* dataptr = buf.empty() ? NULL : const_cast<T*>(buf.data());
1895 
1896 #ifndef NDEBUG
1897  // Only catch the return value when asserts are active.
1898  const int ierr =
1899 #endif
1900  ((this->send_mode() == SYNCHRONOUS) ?
1901  MPI_Ssend : MPI_Send) (dataptr,
1902  libmesh_cast_int<int>(buf.size()),
1903  StandardType<T>(dataptr),
1904  dest_processor_id,
1905  tag.value(),
1906  this->get());
1907 
1908  libmesh_assert (ierr == MPI_SUCCESS);
1909 
1910  STOP_LOG("send()", "Parallel");
1911 }
1912 
1913 
1914 
1915 template <typename T>
1916 inline void Communicator::send (const unsigned int dest_processor_id,
1917  std::basic_string<T> &buf,
1918  Request &req,
1919  const MessageTag &tag) const
1920 {
1921  START_LOG("send()", "Parallel");
1922 
1923  T* dataptr = buf.empty() ? NULL : const_cast<T*>(buf.data());
1924 
1925 #ifndef NDEBUG
1926  // Only catch the return value when asserts are active.
1927  const int ierr =
1928 #endif
1929  ((this->send_mode() == SYNCHRONOUS) ?
1930  MPI_Issend : MPI_Isend) (dataptr,
1931  libmesh_cast_int<int>(buf.size()),
1932  StandardType<T>(dataptr),
1933  dest_processor_id,
1934  tag.value(),
1935  this->get(),
1936  req.get());
1937 
1938  libmesh_assert (ierr == MPI_SUCCESS);
1939 
1940  STOP_LOG("send()", "Parallel");
1941 }
1942 
1943 
1944 
1945 template <typename T>
1946 inline void Communicator::send (const unsigned int dest_processor_id,
1947  T &buf,
1948  const MessageTag &tag) const
1949 {
1950  START_LOG("send()", "Parallel");
1951 
1952  T* dataptr = &buf;
1953 
1954 #ifndef NDEBUG
1955  // Only catch the return value when asserts are active.
1956  const int ierr =
1957 #endif
1958  ((this->send_mode() == SYNCHRONOUS) ?
1959  MPI_Ssend : MPI_Send) (dataptr,
1960  1,
1961  StandardType<T>(dataptr),
1962  dest_processor_id,
1963  tag.value(),
1964  this->get());
1965 
1966  libmesh_assert (ierr == MPI_SUCCESS);
1967 
1968  STOP_LOG("send()", "Parallel");
1969 }
1970 
1971 
1972 
1973 template <typename T>
1974 inline void Communicator::send (const unsigned int dest_processor_id,
1975  T &buf,
1976  Request &req,
1977  const MessageTag &tag) const
1978 {
1979  START_LOG("send()", "Parallel");
1980 
1981  T* dataptr = &buf;
1982 
1983 #ifndef NDEBUG
1984  // Only catch the return value when asserts are active.
1985  const int ierr =
1986 #endif
1987  ((this->send_mode() == SYNCHRONOUS) ?
1988  MPI_Issend : MPI_Isend) (dataptr,
1989  1,
1990  StandardType<T>(dataptr),
1991  dest_processor_id,
1992  tag.value(),
1993  this->get(),
1994  req.get());
1995 
1996  libmesh_assert (ierr == MPI_SUCCESS);
1997 
1998  STOP_LOG("send()", "Parallel");
1999 }
2000 
2001 
2002 
2003 template <typename T>
2004 inline void Communicator::send (const unsigned int dest_processor_id,
2005  std::set<T> &buf,
2006  const MessageTag &tag) const
2007 {
2008  this->send(dest_processor_id,
2009  StandardType<T>(buf.empty() ? NULL : &buf.front()), tag);
2010 }
2011 
2012 
2013 
2014 template <typename T>
2015 inline void Communicator::send (const unsigned int dest_processor_id,
2016  std::set<T> &buf,
2017  Request &req,
2018  const MessageTag &tag) const
2019 {
2020  this->send(dest_processor_id,
2021  StandardType<T>(buf.empty() ? NULL : &buf.front()), req, tag);
2022 }
2023 
2024 
2025 
2026 template <typename T>
2027 inline void Communicator::send (const unsigned int dest_processor_id,
2028  std::set<T> &buf,
2029  const DataType &type,
2030  const MessageTag &tag) const
2031 {
2032  START_LOG("send()", "Parallel");
2033 
2034  std::vector<T> vecbuf(buf.begin(), buf.end());
2035  this->send(dest_processor_id, vecbuf, type, tag);
2036 
2037  STOP_LOG("send()", "Parallel");
2038 }
2039 
2040 
2041 
2042 template <typename T>
2043 inline void Communicator::send (const unsigned int dest_processor_id,
2044  std::set<T> &buf,
2045  const DataType &type,
2046  Request &req,
2047  const MessageTag &tag) const
2048 {
2049  START_LOG("send()", "Parallel");
2050 
2051  // Allocate temporary buffer on the heap so it lives until after
2052  // the non-blocking send completes
2053  std::vector<T> *vecbuf =
2054  new std::vector<T>(buf.begin(), buf.end());
2055 
2056  // Make the Request::wait() handle deleting the buffer
2057  req.add_post_wait_work
2058  (new Parallel::PostWaitDeleteBuffer<std::vector<T> >(vecbuf));
2059 
2060  this->send(dest_processor_id, *vecbuf, type, req, tag);
2061 
2062  STOP_LOG("send()", "Parallel");
2063 }
2064 
2065 
2066 
2067 template <typename T>
2068 inline void Communicator::send (const unsigned int dest_processor_id,
2069  std::vector<T> &buf,
2070  const MessageTag &tag) const
2071 {
2072  this->send(dest_processor_id, buf,
2073  StandardType<T>(buf.empty() ? NULL : &buf.front()), tag);
2074 }
2075 
2076 
2077 
2078 template <typename T>
2079 inline void Communicator::send (const unsigned int dest_processor_id,
2080  std::vector<T> &buf,
2081  Request &req,
2082  const MessageTag &tag) const
2083 {
2084  this->send(dest_processor_id, buf,
2085  StandardType<T>(buf.empty() ? NULL : &buf.front()), req, tag);
2086 }
2087 
2088 
2089 
2090 template <typename T>
2091 inline void Communicator::send (const unsigned int dest_processor_id,
2092  std::vector<T> &buf,
2093  const DataType &type,
2094  const MessageTag &tag) const
2095 {
2096  START_LOG("send()", "Parallel");
2097 
2098 #ifndef NDEBUG
2099  // Only catch the return value when asserts are active.
2100  const int ierr =
2101 #endif
2102  ((this->send_mode() == SYNCHRONOUS) ?
2103  MPI_Ssend : MPI_Send) (buf.empty() ? NULL : &buf[0],
2104  libmesh_cast_int<int>(buf.size()),
2105  type,
2106  dest_processor_id,
2107  tag.value(),
2108  this->get());
2109 
2110  libmesh_assert (ierr == MPI_SUCCESS);
2111 
2112  STOP_LOG("send()", "Parallel");
2113 }
2114 
2115 
2116 
2117 template <typename T>
2118 inline void Communicator::send (const unsigned int dest_processor_id,
2119  std::vector<T> &buf,
2120  const DataType &type,
2121  Request &req,
2122  const MessageTag &tag) const
2123 {
2124  START_LOG("send()", "Parallel");
2125 
2126 #ifndef NDEBUG
2127  // Only catch the return value when asserts are active.
2128  const int ierr =
2129 #endif
2130  ((this->send_mode() == SYNCHRONOUS) ?
2131  MPI_Issend : MPI_Isend) (buf.empty() ? NULL : &buf[0],
2132  libmesh_cast_int<int>(buf.size()),
2133  type,
2134  dest_processor_id,
2135  tag.value(),
2136  this->get(),
2137  req.get());
2138 
2139  libmesh_assert (ierr == MPI_SUCCESS);
2140 
2141  STOP_LOG("send()", "Parallel");
2142 }
2143 
2144 
2145 template <typename Context, typename Iter>
2146 inline void Communicator::send_packed_range (const unsigned int dest_processor_id,
2147  const Context *context,
2148  Iter range_begin,
2149  const Iter range_end,
2150  const MessageTag &tag) const
2151 {
2152  // We will serialize variable size objects from *range_begin to
2153  // *range_end as a sequence of plain data (e.g. ints) in this buffer
2154  typedef typename std::iterator_traits<Iter>::value_type T;
2155  std::vector<typename Parallel::BufferType<T>::type> buffer;
2156 
2157  Parallel::pack_range(context, range_begin, range_end, buffer);
2158 
2159  // Blocking send of the buffer
2160  this->send(dest_processor_id, buffer, tag);
2161 }
2162 
2163 
2164 template <typename Context, typename Iter>
2165 inline void Communicator::send_packed_range (const unsigned int dest_processor_id,
2166  const Context *context,
2167  Iter range_begin,
2168  const Iter range_end,
2169  Request &req,
2170  const MessageTag &tag) const
2171 {
2172  // Allocate a buffer on the heap so we don't have to free it until
2173  // after the Request::wait()
2174  typedef typename std::iterator_traits<Iter>::value_type T;
2175  typedef typename Parallel::BufferType<T>::type buffer_t;
2176  std::vector<buffer_t> *buffer = new std::vector<buffer_t>();
2177 
2178  Parallel::pack_range(context, range_begin, range_end, *buffer);
2179 
2180  // Make the Request::wait() handle deleting the buffer
2181  req.add_post_wait_work
2182  (new Parallel::PostWaitDeleteBuffer<std::vector<buffer_t> >
2183  (buffer));
2184 
2185  // Non-blocking send of the buffer
2186  this->send(dest_processor_id, *buffer, req, tag);
2187 }
2188 
2189 
2190 
2191 template <typename T>
2192 inline Status Communicator::receive (const unsigned int src_processor_id,
2193  std::basic_string<T> &buf,
2194  const MessageTag &tag) const
2195 {
2196  std::vector<T> tempbuf; // Officially C++ won't let us get a
2197  // modifiable array from a string
2198 
2199  Status stat = this->receive(src_processor_id, tempbuf, tag);
2200  buf.assign(tempbuf.begin(), tempbuf.end());
2201  return stat;
2202 }
2203 
2204 
2205 
2206 template <typename T>
2207 inline void Communicator::receive (const unsigned int src_processor_id,
2208  std::basic_string<T> &buf,
2209  Request &req,
2210  const MessageTag &tag) const
2211 {
2212  // Officially C++ won't let us get a modifiable array from a
2213  // string, and we can't even put one on the stack for the
2214  // non-blocking case.
2215  std::vector<T> *tempbuf = new std::vector<T>();
2216 
2217  // We can clear the string, but the Request::wait() will need to
2218  // handle copying our temporary buffer to it
2219  buf.clear();
2220 
2221  req.add_post_wait_work
2222  (new Parallel::PostWaitCopyBuffer<std::vector<T>,
2223  std::back_insert_iterator<std::basic_string<T> > >
2224  (tempbuf, std::back_inserter(buf)));
2225 
2226  // Make the Request::wait() then handle deleting the buffer
2227  req.add_post_wait_work
2228  (new Parallel::PostWaitDeleteBuffer<std::vector<T> >(tempbuf));
2229 
2230  this->receive(src_processor_id, tempbuf, req, tag);
2231 }
2232 
2233 
2234 
2235 template <typename T>
2236 inline Status Communicator::receive (const unsigned int src_processor_id,
2237  T &buf,
2238  const MessageTag &tag) const
2239 {
2240  START_LOG("receive()", "Parallel");
2241 
2242  // Get the status of the message, explicitly provide the
2243  // datatype so we can later query the size
2244  Status stat(this->probe(src_processor_id, tag), StandardType<T>(&buf));
2245 
2246 #ifndef NDEBUG
2247  // Only catch the return value when asserts are active.
2248  const int ierr =
2249 #endif
2250  MPI_Recv (&buf,
2251  1,
2252  StandardType<T>(&buf),
2253  src_processor_id,
2254  tag.value(),
2255  this->get(),
2256  stat.get());
2257  libmesh_assert (ierr == MPI_SUCCESS);
2258 
2259  STOP_LOG("receive()", "Parallel");
2260 
2261  return stat;
2262 }
2263 
2264 
2265 
2266 template <typename T>
2267 inline void Communicator::receive (const unsigned int src_processor_id,
2268  T &buf,
2269  Request &req,
2270  const MessageTag &tag) const
2271 {
2272  START_LOG("receive()", "Parallel");
2273 
2274 #ifndef NDEBUG
2275  // Only catch the return value when asserts are active.
2276  const int ierr =
2277 #endif
2278  MPI_Irecv (&buf,
2279  1,
2280  StandardType<T>(&buf),
2281  src_processor_id,
2282  tag.value(),
2283  this->get(),
2284  req.get());
2285  libmesh_assert (ierr == MPI_SUCCESS);
2286 
2287  STOP_LOG("receive()", "Parallel");
2288 }
2289 
2290 
2291 
2292 template <typename T>
2293 inline Status Communicator::receive (const unsigned int src_processor_id,
2294  std::set<T> &buf,
2295  const MessageTag &tag) const
2296 {
2297  return this->receive
2298  (src_processor_id, buf,
2299  StandardType<T>(buf.empty() ? NULL : &buf.front()), tag);
2300 }
2301 
2302 
2303 
2304 template <typename T>
2305 inline void Communicator::receive (const unsigned int src_processor_id,
2306  std::set<T> &buf,
2307  Request &req,
2308  const MessageTag &tag) const
2309 {
2310  this->receive (src_processor_id, buf,
2311  StandardType<T>(buf.empty() ? NULL : &buf.front()), req, tag);
2312 }
2313 
2314 
2315 
2316 template <typename T>
2317 inline Status Communicator::receive (const unsigned int src_processor_id,
2318  std::set<T> &buf,
2319  const DataType &type,
2320  const MessageTag &tag) const
2321 {
2322  START_LOG("receive()", "Parallel");
2323 
2324  std::vector<T> vecbuf;
2325  Status stat = this->receive(src_processor_id, vecbuf, type, tag);
2326  buf.clear();
2327  buf.insert(vecbuf.begin(), vecbuf.end());
2328 
2329  STOP_LOG("receive()", "Parallel");
2330 
2331  return stat;
2332 }
2333 
2334 
2335 
2336 template <typename T>
2337 inline void Communicator::receive (const unsigned int src_processor_id,
2338  std::set<T> &buf,
2339  const DataType &type,
2340  Request &req,
2341  const MessageTag &tag) const
2342 {
2343  START_LOG("receive()", "Parallel");
2344 
2345  // Allocate temporary buffer on the heap so it lives until after
2346  // the non-blocking send completes
2347  std::vector<T> *vecbuf = new std::vector<T>();
2348 
2349  // We can clear the set, but the Request::wait() will need to
2350  // handle copying our temporary buffer to it
2351  buf.clear();
2352 
2353  req.add_post_wait_work
2354  (new Parallel::PostWaitCopyBuffer<std::vector<T>,
2355  std::back_insert_iterator<std::set<T> > >
2356  (vecbuf, std::back_inserter(buf)));
2357 
2358  // Make the Request::wait() then handle deleting the buffer
2359  req.add_post_wait_work
2360  (new Parallel::PostWaitDeleteBuffer<std::vector<T> >(vecbuf));
2361 
2362  this->receive(src_processor_id, *vecbuf, type, req, tag);
2363 
2364  STOP_LOG("receive()", "Parallel");
2365 }
2366 
2367 
2368 
2369 template <typename T>
2370 inline Status Communicator::receive (const unsigned int src_processor_id,
2371  std::vector<T> &buf,
2372  const MessageTag &tag) const
2373 {
2374  return this->receive
2375  (src_processor_id, buf,
2376  StandardType<T>(buf.empty() ? NULL : &buf.front()), tag);
2377 }
2378 
2379 
2380 
2381 template <typename T>
2382 inline void Communicator::receive (const unsigned int src_processor_id,
2383  std::vector<T> &buf,
2384  Request &req,
2385  const MessageTag &tag) const
2386 {
2387  this->receive (src_processor_id, buf,
2388  StandardType<T>(buf.empty() ? NULL : &buf.front()), req, tag);
2389 }
2390 
2391 
2392 
2393 template <typename T>
2394 inline Status Communicator::receive (const unsigned int src_processor_id,
2395  std::vector<T> &buf,
2396  const DataType &type,
2397  const MessageTag &tag) const
2398 {
2399  START_LOG("receive()", "Parallel");
2400 
2401  // Get the status of the message, explicitly provide the
2402  // datatype so we can later query the size
2403  Status stat(this->probe(src_processor_id, tag), type);
2404 
2405  buf.resize(stat.size());
2406 
2407 #ifndef NDEBUG
2408  // Only catch the return value when asserts are active.
2409  const int ierr =
2410 #endif
2411  MPI_Recv (buf.empty() ? NULL : &buf[0],
2412  libmesh_cast_int<int>(buf.size()),
2413  type,
2414  src_processor_id,
2415  tag.value(),
2416  this->get(),
2417  stat.get());
2418  libmesh_assert (ierr == MPI_SUCCESS);
2419 
2420  STOP_LOG("receive()", "Parallel");
2421 
2422  return stat;
2423 }
2424 
2425 
2426 
2427 template <typename T>
2428 inline void Communicator::receive (const unsigned int src_processor_id,
2429  std::vector<T> &buf,
2430  const DataType &type,
2431  Request &req,
2432  const MessageTag &tag) const
2433 {
2434  START_LOG("receive()", "Parallel");
2435 
2436 #ifndef NDEBUG
2437  // Only catch the return value when asserts are active.
2438  const int ierr =
2439 #endif
2440  MPI_Irecv (buf.empty() ? NULL : &buf[0],
2441  libmesh_cast_int<int>(buf.size()),
2442  type,
2443  src_processor_id,
2444  tag.value(),
2445  this->get(),
2446  req.get());
2447  libmesh_assert (ierr == MPI_SUCCESS);
2448 
2449  STOP_LOG("receive()", "Parallel");
2450 }
2451 
2452 
2453 template <typename Context, typename OutputIter>
2454 inline void Communicator::receive_packed_range (const unsigned int src_processor_id,
2455  Context *context,
2456  OutputIter out,
2457  const MessageTag &tag) const
2458 {
2459  typedef typename std::iterator_traits<OutputIter>::value_type T;
2460  typedef typename Parallel::BufferType<T>::type buffer_t;
2461 
2462  // Receive serialized variable size objects as a sequence of ints
2463  std::vector<buffer_t> buffer;
2464  this->receive(src_processor_id, buffer, tag);
2465  Parallel::unpack_range(buffer, context, out);
2466 }
2467 
2468 
2469 
2470 template <typename Context, typename OutputIter>
2471 inline void Communicator::receive_packed_range (const unsigned int src_processor_id,
2472  Context *context,
2473  OutputIter out,
2474  Request &req,
2475  const MessageTag &tag) const
2476 {
2477  typedef typename std::iterator_traits<OutputIter>::value_type T;
2478  typedef typename Parallel::BufferType<T>::type buffer_t;
2479 
2480  // Receive serialized variable size objects as a sequence of
2481  // buffer_t.
2482  // Allocate a buffer on the heap so we don't have to free it until
2483  // after the Request::wait()
2484  std::vector<buffer_t> *buffer = new std::vector<buffer_t>();
2485  this->receive(src_processor_id, *buffer, req, tag);
2486 
2487  // Make the Request::wait() handle unpacking the buffer
2488  req.add_post_wait_work
2489  (new Parallel::PostWaitUnpackBuffer<std::vector<buffer_t>, Context, OutputIter>
2490  (buffer, context, out));
2491 
2492  // Make the Request::wait() then handle deleting the buffer
2493  req.add_post_wait_work
2494  (new Parallel::PostWaitDeleteBuffer<std::vector<buffer_t> >(buffer));
2495 }
2496 
2497 
2498 
2499 template <typename T1, typename T2>
2500 inline void Communicator::send_receive(const unsigned int dest_processor_id,
2501  std::vector<T1> &sendvec,
2502  const DataType &type1,
2503  const unsigned int source_processor_id,
2504  std::vector<T2> &recv,
2505  const DataType &type2,
2506  const MessageTag &send_tag,
2507  const MessageTag &recv_tag) const
2508 {
2509  START_LOG("send_receive()", "Parallel");
2510 
2511  if (dest_processor_id == this->rank() &&
2512  source_processor_id == this->rank())
2513  {
2514  recv = sendvec;
2515  STOP_LOG("send_receive()", "Parallel");
2516  return;
2517  }
2518 
2519  Parallel::Request req;
2520 
2521  this->send (dest_processor_id, sendvec, type1, req, send_tag);
2522 
2523  this->receive (source_processor_id, recv, type2, recv_tag);
2524 
2525  req.wait();
2526 
2527  STOP_LOG("send_receive()", "Parallel");
2528 }
2529 
2530 
2531 
2532 template <typename T1, typename T2>
2533 inline void Communicator::send_receive(const unsigned int dest_processor_id,
2534  T1 &sendvec,
2535  const unsigned int source_processor_id,
2536  T2 &recv,
2537  const MessageTag &send_tag,
2538  const MessageTag &recv_tag) const
2539 {
2540  START_LOG("send_receive()", "Parallel");
2541 
2542  if (dest_processor_id == this->rank() &&
2543  source_processor_id == this->rank())
2544  {
2545  recv = sendvec;
2546  STOP_LOG("send_receive()", "Parallel");
2547  return;
2548  }
2549 
2550  MPI_Sendrecv(&sendvec, 1, StandardType<T1>(&sendvec),
2551  dest_processor_id, send_tag.value(),
2552  &recv, 1, StandardType<T2>(&recv),
2553  source_processor_id, recv_tag.value(),
2554  this->get(),
2555  MPI_STATUS_IGNORE);
2556 
2557  STOP_LOG("send_receive()", "Parallel");
2558 }
2559 
2560 
2561 
2562 // This is both a declaration and definition for a new overloaded
2563 // function template, so we have to re-specify the default
2564 // arguments.
2565 //
2566 // We specialize on the T1==T2 case so that we can handle
2567 // send_receive-to-self with a plain copy rather than going through
2568 // MPI.
2569 template <typename T>
2570 inline void Communicator::send_receive(const unsigned int dest_processor_id,
2571  std::vector<T> &sendvec,
2572  const unsigned int source_processor_id,
2573  std::vector<T> &recv,
2574  const MessageTag &send_tag,
2575  const MessageTag &recv_tag) const
2576 {
2577  if (dest_processor_id == this->rank() &&
2578  source_processor_id == this->rank())
2579  {
2580  START_LOG("send_receive()", "Parallel");
2581  recv = sendvec;
2582  STOP_LOG("send_receive()", "Parallel");
2583  return;
2584  }
2585 
2586  // Call the user-defined type version with automatic
2587  // type conversion based on template argument:
2588  this->send_receive (dest_processor_id, sendvec,
2589  StandardType<T>(sendvec.empty() ? NULL : &sendvec[0]),
2590  source_processor_id, recv,
2591  StandardType<T>(recv.empty() ? NULL : &recv[0]),
2592  send_tag, recv_tag);
2593 }
2594 
2595 
2596 // This is both a declaration and definition for a new overloaded
2597 // function template, so we have to re-specify the default arguments
2598 template <typename T1, typename T2>
2599 inline void Communicator::send_receive(const unsigned int dest_processor_id,
2600  std::vector<T1> &sendvec,
2601  const unsigned int source_processor_id,
2602  std::vector<T2> &recv,
2603  const MessageTag &send_tag,
2604  const MessageTag &recv_tag) const
2605 {
2606  // Call the user-defined type version with automatic
2607  // type conversion based on template argument:
2608  this->send_receive (dest_processor_id, sendvec,
2609  StandardType<T1>(sendvec.empty() ? NULL : &sendvec[0]),
2610  source_processor_id, recv,
2611  StandardType<T2>(recv.empty() ? NULL : &recv[0]),
2612  send_tag, recv_tag);
2613 }
2614 
2615 
2616 
2617 
2618 template <typename T1, typename T2>
2619 inline void Communicator::send_receive(const unsigned int dest_processor_id,
2620  std::vector<std::vector<T1> > &sendvec,
2621  const unsigned int source_processor_id,
2622  std::vector<std::vector<T2> > &recv,
2623  const MessageTag & /* send_tag */,
2624  const MessageTag & /* recv_tag */) const
2625 {
2626  // FIXME - why aren't we honoring send_tag and recv_tag here?
2627  send_receive_vec_of_vec
2628  (dest_processor_id, sendvec, source_processor_id, recv,
2629  no_tag, any_tag, *this);
2630 }
2631 
2632 
2633 
2634 // This is both a declaration and definition for a new overloaded
2635 // function template, so we have to re-specify the default arguments
2636 template <typename T>
2637 inline void Communicator::send_receive(const unsigned int dest_processor_id,
2638  std::vector<std::vector<T> > &sendvec,
2639  const unsigned int source_processor_id,
2640  std::vector<std::vector<T> > &recv,
2641  const MessageTag & /* send_tag */,
2642  const MessageTag & /* recv_tag */) const
2643 {
2644  // FIXME - why aren't we honoring send_tag and recv_tag here?
2645  send_receive_vec_of_vec
2646  (dest_processor_id, sendvec, source_processor_id, recv,
2647  no_tag, any_tag, *this);
2648 }
2649 
2650 
2651 
2652 
2653 template <typename Context1, typename RangeIter, typename Context2, typename OutputIter>
2655  (const unsigned int dest_processor_id,
2656  const Context1* context1,
2657  RangeIter send_begin,
2658  const RangeIter send_end,
2659  const unsigned int source_processor_id,
2660  Context2* context2,
2661  OutputIter out,
2662  const MessageTag &send_tag,
2663  const MessageTag &recv_tag) const
2664 {
2665  START_LOG("send_receive()", "Parallel");
2666 
2667  Parallel::Request req;
2668 
2669  this->send_packed_range (dest_processor_id, context1, send_begin, send_end,
2670  req, send_tag);
2671 
2672  this->receive_packed_range (source_processor_id, context2, out, recv_tag);
2673 
2674  req.wait();
2675 
2676  STOP_LOG("send_receive()", "Parallel");
2677 
2678 }
2679 
2680 
2681 
2682 template <typename T>
2683 inline void Communicator::gather(const unsigned int root_id,
2684  T sendval,
2685  std::vector<T> &recv) const
2686 {
2687  libmesh_assert_less (root_id, this->size());
2688 
2689  if (this->rank() == root_id)
2690  recv.resize(this->size());
2691 
2692  if (this->size() > 1)
2693  {
2694  START_LOG("gather()", "Parallel");
2695 
2696  StandardType<T> send_type(&sendval);
2697 
2698  MPI_Gather(&sendval,
2699  1,
2700  send_type,
2701  recv.empty() ? NULL : &recv[0],
2702  1,
2703  send_type,
2704  root_id,
2705  this->get());
2706 
2707  STOP_LOG("gather()", "Parallel");
2708  }
2709  else
2710  recv[0] = sendval;
2711 }
2712 
2713 
2714 
2715 template <typename T>
2716 inline void Communicator::gather(const unsigned int root_id,
2717  std::vector<T> &r) const
2718 {
2719  if (this->size() == 1)
2720  {
2721  libmesh_assert (!this->rank());
2722  libmesh_assert (!root_id);
2723  return;
2724  }
2725 
2726  libmesh_assert_less (root_id, this->size());
2727 
2728  std::vector<int>
2729  sendlengths (this->size(), 0),
2730  displacements(this->size(), 0);
2731 
2732  const int mysize = static_cast<int>(r.size());
2733  this->allgather(mysize, sendlengths);
2734 
2735  START_LOG("gather()", "Parallel");
2736 
2737  // Find the total size of the final array and
2738  // set up the displacement offsets for each processor.
2739  unsigned int globalsize = 0;
2740  for (unsigned int i=0; i != this->size(); ++i)
2741  {
2742  displacements[i] = globalsize;
2743  globalsize += sendlengths[i];
2744  }
2745 
2746  // Check for quick return
2747  if (globalsize == 0)
2748  {
2749  STOP_LOG("gather()", "Parallel");
2750  return;
2751  }
2752 
2753  // copy the input buffer
2754  std::vector<T> r_src(r);
2755 
2756  // now resize it to hold the global data
2757  // on the receiving processor
2758  if (root_id == this->rank())
2759  r.resize(globalsize);
2760 
2761  // and get the data from the remote processors
2762 #ifndef NDEBUG
2763  // Only catch the return value when asserts are active.
2764  const int ierr =
2765 #endif
2766  MPI_Gatherv (r_src.empty() ? NULL : &r_src[0], mysize, StandardType<T>(),
2767  r.empty() ? NULL : &r[0], &sendlengths[0],
2768  &displacements[0], StandardType<T>(),
2769  root_id,
2770  this->get());
2771 
2772  libmesh_assert (ierr == MPI_SUCCESS);
2773 
2774  STOP_LOG("gather()", "Parallel");
2775 }
2776 
2777 
2778 template <typename T>
2779 inline void Communicator::allgather(T sendval,
2780  std::vector<T> &recv) const
2781 {
2782  START_LOG ("allgather()","Parallel");
2783 
2784  libmesh_assert(this->size());
2785  recv.resize(this->size());
2786 
2787  unsigned int comm_size = this->size();
2788  if (comm_size > 1)
2789  {
2790  StandardType<T> send_type(&sendval);
2791 
2792  MPI_Allgather (&sendval,
2793  1,
2794  send_type,
2795  &recv[0],
2796  1,
2797  send_type,
2798  this->get());
2799  }
2800  else if (comm_size > 0)
2801  recv[0] = sendval;
2802 
2803  STOP_LOG ("allgather()","Parallel");
2804 }
2805 
2806 
2807 
2808 template <typename T>
2809 inline void Communicator::allgather
2810  (std::vector<T> &r,
2811  const bool identical_buffer_sizes) const
2812 {
2813  if (this->size() < 2)
2814  return;
2815 
2816  START_LOG("allgather()", "Parallel");
2817 
2818  if (identical_buffer_sizes)
2819  {
2820  if (r.empty())
2821  return;
2822 
2823  libmesh_assert(this->verify(r.size()));
2824 
2825  std::vector<T> r_src(r.size()*this->size());
2826  r_src.swap(r);
2827  StandardType<T> send_type(&r_src[0]);
2828 
2829  MPI_Allgather (&r_src[0],
2830  libmesh_cast_int<int>(r_src.size()),
2831  send_type,
2832  &r[0],
2833  libmesh_cast_int<int>(r_src.size()),
2834  send_type,
2835  this->get());
2836  libmesh_assert(this->verify(r));
2837  STOP_LOG("allgather()", "Parallel");
2838  return;
2839  }
2840 
2841  std::vector<int>
2842  sendlengths (this->size(), 0),
2843  displacements(this->size(), 0);
2844 
2845  const int mysize = static_cast<int>(r.size());
2846  this->allgather(mysize, sendlengths);
2847 
2848  // Find the total size of the final array and
2849  // set up the displacement offsets for each processor.
2850  unsigned int globalsize = 0;
2851  for (unsigned int i=0; i != this->size(); ++i)
2852  {
2853  displacements[i] = globalsize;
2854  globalsize += sendlengths[i];
2855  }
2856 
2857  // Check for quick return
2858  if (globalsize == 0)
2859  {
2860  STOP_LOG("allgather()", "Parallel");
2861  return;
2862  }
2863 
2864  // copy the input buffer
2865  std::vector<T> r_src(globalsize);
2866  r_src.swap(r);
2867 
2868  StandardType<T> send_type(&r[0]);
2869 
2870  // and get the data from the remote processors.
2871  // Pass NULL if our vector is empty.
2872 #ifndef NDEBUG
2873  // Only catch the return value when asserts are active.
2874  const int ierr =
2875 #endif
2876  MPI_Allgatherv (r_src.empty() ? NULL : &r_src[0], mysize, send_type,
2877  &r[0], &sendlengths[0],
2878  &displacements[0], send_type, this->get());
2879 
2880  libmesh_assert (ierr == MPI_SUCCESS);
2881 
2882  STOP_LOG("allgather()", "Parallel");
2883 }
2884 
2885 
2886 template <typename Context, typename Iter, typename OutputIter>
2888  (const unsigned int root_id,
2889  Context *context,
2890  Iter range_begin,
2891  const Iter range_end,
2892  OutputIter out) const
2893 {
2894  typedef typename std::iterator_traits<Iter>::value_type T;
2895  typedef typename Parallel::BufferType<T>::type buffer_t;
2896 
2897  // We will serialize variable size objects from *range_begin to
2898  // *range_end as a sequence of ints in this buffer
2899  std::vector<buffer_t> buffer;
2900 
2901  Parallel::pack_range(context, range_begin, range_end, buffer);
2902 
2903  this->gather(root_id, buffer);
2904 
2905  Parallel::unpack_range(buffer, context, out);
2906 }
2907 
2908 
2909 template <typename Context, typename Iter, typename OutputIter>
2911  (Context *context,
2912  Iter range_begin,
2913  const Iter range_end,
2914  OutputIter out) const
2915 {
2916  typedef typename std::iterator_traits<Iter>::value_type T;
2917  typedef typename Parallel::BufferType<T>::type buffer_t;
2918 
2919  // We will serialize variable size objects from *range_begin to
2920  // *range_end as a sequence of ints in this buffer
2921  std::vector<buffer_t> buffer;
2922 
2923  Parallel::pack_range(context, range_begin, range_end, buffer);
2924 
2925  this->allgather(buffer, false);
2926 
2927  Parallel::unpack_range(buffer, context, out);
2928 }
2929 
2930 
2931 template <typename T>
2932 inline void Communicator::alltoall(std::vector<T> &buf) const
2933 {
2934  if (this->size() < 2 || buf.empty())
2935  return;
2936 
2937  START_LOG("alltoall()", "Parallel");
2938 
2939  // the per-processor size. this is the same for all
2940  // processors using MPI_Alltoall, could be variable
2941  // using MPI_Alltoallv
2942  const int size_per_proc =
2943  libmesh_cast_int<int>(buf.size()/this->size());
2944 
2945  libmesh_assert_equal_to (buf.size()%this->size(), 0);
2946 
2947  libmesh_assert(this->verify(size_per_proc));
2948 
2949  std::vector<T> tmp(buf);
2950 
2951  StandardType<T> send_type(&tmp[0]);
2952 
2953 #ifndef NDEBUG
2954  // Only catch the return value when asserts are active.
2955  const int ierr =
2956 #endif
2957  MPI_Alltoall (&tmp[0],
2958  size_per_proc,
2959  send_type,
2960  &buf[0],
2961  size_per_proc,
2962  send_type,
2963  this->get());
2964  libmesh_assert (ierr == MPI_SUCCESS);
2965 
2966  STOP_LOG("alltoall()", "Parallel");
2967 }
2968 
2969 
2970 
2971 template <typename T>
2972 inline void Communicator::broadcast (T &data, const unsigned int root_id) const
2973 {
2974  if (this->size() == 1)
2975  {
2976  libmesh_assert (!this->rank());
2977  libmesh_assert (!root_id);
2978  return;
2979  }
2980 
2981  libmesh_assert_less (root_id, this->size());
2982 
2983  START_LOG("broadcast()", "Parallel");
2984 
2985  // Spread data to remote processors.
2986 #ifndef NDEBUG
2987  // Only catch the return value when asserts are active.
2988  const int ierr =
2989 #endif
2990  MPI_Bcast (&data, 1, StandardType<T>(&data), root_id, this->get());
2991 
2992  libmesh_assert (ierr == MPI_SUCCESS);
2993 
2994  STOP_LOG("broadcast()", "Parallel");
2995 }
2996 
2997 
2998 template <typename T>
2999 inline void Communicator::broadcast (std::basic_string<T> &data,
3000  const unsigned int root_id) const
3001 {
3002  if (this->size() == 1)
3003  {
3004  libmesh_assert (!this->rank());
3005  libmesh_assert (!root_id);
3006  return;
3007  }
3008 
3009  libmesh_assert_less (root_id, this->size());
3010 
3011  START_LOG("broadcast()", "Parallel");
3012 
3013  std::size_t data_size = data.size();
3014  this->broadcast(data_size, root_id);
3015 
3016  std::vector<T> data_c(data_size);
3017 #ifndef NDEBUG
3018  std::string orig(data);
3019 #endif
3020 
3021  if (this->rank() == root_id)
3022  for(std::size_t i=0; i<data.size(); i++)
3023  data_c[i] = data[i];
3024 
3025  this->broadcast (data_c, root_id);
3026 
3027  data.assign(data_c.begin(), data_c.end());
3028 
3029 #ifndef NDEBUG
3030  if (this->rank() == root_id)
3031  libmesh_assert_equal_to (data, orig);
3032 #endif
3033 
3034  STOP_LOG("broadcast()", "Parallel");
3035 }
3036 
3037 
3038 
3039 template <typename T>
3040 inline void Communicator::broadcast (std::vector<T> &data,
3041  const unsigned int root_id) const
3042 {
3043  if (this->size() == 1)
3044  {
3045  libmesh_assert (!this->rank());
3046  libmesh_assert (!root_id);
3047  return;
3048  }
3049 
3050  libmesh_assert_less (root_id, this->size());
3051 
3052  START_LOG("broadcast()", "Parallel");
3053 
3054  // and get the data from the remote processors.
3055  // Pass NULL if our vector is empty.
3056  T *data_ptr = data.empty() ? NULL : &data[0];
3057 
3058 #ifndef NDEBUG
3059  // Only catch the return value when asserts are active.
3060  const int ierr =
3061 #endif
3062  MPI_Bcast (data_ptr, libmesh_cast_int<int>(data.size()),
3063  StandardType<T>(data_ptr), root_id, this->get());
3064 
3065  libmesh_assert (ierr == MPI_SUCCESS);
3066 
3067  STOP_LOG("broadcast()", "Parallel");
3068 }
3069 
3070 
3071 template <typename T>
3072 inline void Communicator::broadcast (std::vector<std::basic_string<T> > &data,
3073  const unsigned int root_id) const
3074 {
3075  if (this->size() == 1)
3076  {
3077  libmesh_assert (!this->rank());
3078  libmesh_assert (!root_id);
3079  return;
3080  }
3081 
3082  libmesh_assert_less (root_id, this->size());
3083 
3084  START_LOG("broadcast()", "Parallel");
3085 
3086  std::size_t bufsize=0;
3087  if (root_id == this->rank())
3088  {
3089  for (std::size_t i=0; i<data.size(); ++i)
3090  bufsize += data[i].size() + 1; // Add one for the string length word
3091  }
3092  this->broadcast(bufsize, root_id);
3093 
3094  // Here we use unsigned int to store up to 32-bit characters
3095  std::vector<unsigned int> temp; temp.reserve(bufsize);
3096  // Pack the strings
3097  if (root_id == this->rank())
3098  {
3099  for (unsigned int i=0; i<data.size(); ++i)
3100  {
3101  temp.push_back(data[i].size());
3102  for (std::size_t j=0; j != data[i].size(); ++j)
3107  temp.push_back(data[i][j]);
3108  }
3109  }
3110  else
3111  temp.resize(bufsize);
3112 
3113  // broad cast the packed strings
3114  this->broadcast(temp, root_id);
3115 
3116  // Unpack the strings
3117  if (root_id != this->rank())
3118  {
3119  data.clear();
3120  std::vector<unsigned int>::const_iterator iter = temp.begin();
3121  while (iter != temp.end())
3122  {
3123  std::size_t curr_len = *iter++;
3124  data.push_back(std::string(iter, iter+curr_len));
3125  iter += curr_len;
3126  }
3127  }
3128 
3129  STOP_LOG("broadcast()", "Parallel");
3130 }
3131 
3132 
3133 
3134 
3135 template <typename T>
3136 inline void Communicator::broadcast (std::set<T> &data,
3137  const unsigned int root_id) const
3138 {
3139  if (this->size() == 1)
3140  {
3141  libmesh_assert (!this->rank());
3142  libmesh_assert (!root_id);
3143  return;
3144  }
3145 
3146  libmesh_assert_less (root_id, this->size());
3147 
3148  START_LOG("broadcast()", "Parallel");
3149 
3150  std::vector<T> vecdata;
3151  if (this->rank() == root_id)
3152  vecdata.assign(data.begin(), data.end());
3153 
3154  std::size_t vecsize = vecdata.size();
3155  this->broadcast(vecsize, root_id);
3156  if (this->rank() != root_id)
3157  vecdata.resize(vecsize);
3158 
3159  this->broadcast(vecdata, root_id);
3160  if (this->rank() != root_id)
3161  {
3162  data.clear();
3163  data.insert(vecdata.begin(), vecdata.end());
3164  }
3165 
3166  STOP_LOG("broadcast()", "Parallel");
3167 }
3168 
3169 
3170 
3171 template <typename T1, typename T2>
3172 inline void Communicator::broadcast(std::map<T1, T2> &data,
3173  const unsigned int root_id) const
3174 {
3175  if (this->size() == 1)
3176  {
3177  libmesh_assert (!this->rank());
3178  libmesh_assert (!root_id);
3179  return;
3180  }
3181 
3182  libmesh_assert_less (root_id, this->size());
3183 
3184  START_LOG("broadcast()", "Parallel");
3185 
3186  std::size_t data_size=data.size();
3187  this->broadcast(data_size, root_id);
3188 
3189  std::vector<T1> pair_first; pair_first.reserve(data_size);
3190  std::vector<T2> pair_second; pair_first.reserve(data_size);
3191 
3192  if (root_id == this->rank())
3193  {
3194  for (typename std::map<T1, T2>::const_iterator it = data.begin();
3195  it != data.end(); ++it)
3196  {
3197  pair_first.push_back(it->first);
3198  pair_second.push_back(it->second);
3199  }
3200  }
3201  else
3202  {
3203  pair_first.resize(data_size);
3204  pair_second.resize(data_size);
3205  }
3206 
3207  this->broadcast(pair_first, root_id);
3208  this->broadcast(pair_second, root_id);
3209 
3210  libmesh_assert(pair_first.size() == pair_first.size());
3211 
3212  if (this->rank() != root_id)
3213  {
3214  data.clear();
3215  for (std::size_t i=0; i<pair_first.size(); ++i)
3216  data[pair_first[i]] = pair_second[i];
3217  }
3218  STOP_LOG("broadcast()", "Parallel");
3219 }
3220 
3221 
3222 
3223 template <typename Context, typename OutputContext,
3224  typename Iter, typename OutputIter>
3226  (const Context *context1,
3227  Iter range_begin,
3228  const Iter range_end,
3229  OutputContext *context2,
3230  OutputIter out,
3231  const unsigned int root_id) const
3232 {
3233  typedef typename std::iterator_traits<Iter>::value_type T;
3234  typedef typename Parallel::BufferType<T>::type buffer_t;
3235 
3236  // We will serialize variable size objects from *range_begin to
3237  // *range_end as a sequence of ints in this buffer
3238  std::vector<buffer_t> buffer;
3239 
3240  if (this->rank() == root_id)
3241  Parallel::pack_range(context1, range_begin, range_end, buffer);
3242 
3243  // this->broadcast(vector) requires the receiving vectors to
3244  // already be the appropriate size
3245  std::size_t buffer_size = buffer.size();
3246  this->broadcast (buffer_size);
3247  buffer.resize(buffer_size);
3248 
3249  // Broadcast the packed data
3250  this->broadcast (buffer, root_id);
3251 
3252  if (this->rank() != root_id)
3253  Parallel::unpack_range(buffer, context2, out);
3254 }
3255 
3256 
3257 #else // LIBMESH_HAVE_MPI
3258 
3259 template <typename T>
3260 inline bool Communicator::verify(const T &) const { return true; }
3261 
3262 template <typename T>
3263 inline bool Communicator::semiverify(const T *) const { return true; }
3264 
3265 template <typename T>
3266 inline void Communicator::min(T &) const {}
3267 
3268 template <typename T>
3269 inline void Communicator::minloc(T &, unsigned int &min_id) const { min_id = 0; }
3270 
3271 template <typename T>
3272 inline void Communicator::minloc
3273  (std::vector<T> &r, std::vector<unsigned int> &min_id) const
3274  { for (std::size_t i=0; i!= r.size(); ++i) min_id[i] = 0; }
3275 
3276 template <typename T>
3277 inline void Communicator::max(T &) const {}
3278 
3279 template <typename T>
3280 inline void Communicator::maxloc(T &, unsigned int &max_id) const { max_id = 0; }
3281 
3282 template <typename T>
3283 inline void Communicator::maxloc
3284  (std::vector<T> &r, std::vector<unsigned int> &max_id) const
3285  { for (std::size_t i=0; i!= r.size(); ++i) max_id[i] = 0; }
3286 
3287 template <typename T>
3288 inline void Communicator::sum(T &) const {}
3289 
3290 template <typename T>
3291 inline void Communicator::set_union(T&) const {}
3292 
3293 template <typename T>
3294 inline void Communicator::set_union(T&, const unsigned int root_id) const
3295 { libmesh_assert_equal_to(root_id, 0); }
3296 
3300 inline status Communicator::probe (const unsigned int,
3301  const MessageTag&) const
3302 { libmesh_error(); status s; return s; }
3303 
3307 template <typename T>
3308 inline void Communicator::send (const unsigned int, T&, const MessageTag &) const
3309 { libmesh_error(); }
3310 
3311 template <typename T>
3312 inline void Communicator::send (const unsigned int, T&, Request&,
3313  const MessageTag&) const
3314 { libmesh_error(); }
3315 
3316 template <typename T>
3317 inline void Communicator::send (const unsigned int, T&, const DataType&,
3318  const MessageTag &) const
3319 { libmesh_error(); }
3320 
3321 template <typename T>
3322 inline void Communicator::send (const unsigned int, T&, const DataType&, Request&,
3323  const MessageTag &) const
3324 { libmesh_error(); }
3325 
3326 template <typename Context, typename Iter>
3328  (const unsigned int, const Context*, Iter, const Iter, const MessageTag&) const
3329 { libmesh_error(); }
3330 
3331 template <typename Context, typename Iter>
3333  (const unsigned int, const Context*, Iter, const Iter, Request&,
3334  const MessageTag&) const
3335 { libmesh_error(); }
3336 
3340 template <typename T>
3341 inline Status Communicator::receive (const unsigned int, T&, const MessageTag&) const
3342 { libmesh_error(); return Status(); }
3343 
3344 template <typename T>
3345 inline void Communicator::receive
3346  (const unsigned int, T&, Request&, const MessageTag&) const
3347 { libmesh_error(); }
3348 
3349 template <typename T>
3350 inline Status Communicator::receive
3351  (const unsigned int, T&, const DataType&, const MessageTag&) const
3352 { libmesh_error(); return Status(); }
3353 
3354 template <typename T>
3355 inline void Communicator::receive
3356  (const unsigned int, T&, const DataType&, Request&, const MessageTag&) const
3357 { libmesh_error(); }
3358 
3359 template <typename Context, typename OutputIter>
3361  (const unsigned int, Context*, OutputIter, const MessageTag&) const
3362 { libmesh_error(); }
3363 
3364 template <typename Context, typename OutputIter>
3366  (const unsigned int, Context*, OutputIter, Request&, const MessageTag&) const
3367 { libmesh_error(); }
3368 
3372 template <typename T1, typename T2>
3373 inline void Communicator::send_receive (const unsigned int send_tgt,
3374  T1 &send,
3375  const unsigned int recv_source,
3376  T2 &recv,
3377  const MessageTag &,
3378  const MessageTag &) const
3379 {
3380  libmesh_assert_equal_to (send_tgt, 0);
3381  libmesh_assert_equal_to (recv_source, 0);
3382  recv = send;
3383 }
3384 
3390 template <typename Context1, typename RangeIter,
3391  typename Context2, typename OutputIter>
3393  (const unsigned int /* dest_processor_id */, const Context1*,
3394  RangeIter /* send_begin */, const RangeIter /* send_end */,
3395  const unsigned int /* source_processor_id */, Context2*,
3396  OutputIter /* out */, const MessageTag &, const MessageTag &) const
3397 { libmesh_error(); }
3398 
3402 template <typename T>
3403 inline void Communicator::gather(const unsigned int root_id,
3404  T send,
3405  std::vector<T> &recv) const
3406 {
3407  libmesh_assert_equal_to (root_id, 0);
3408  recv.resize(1);
3409  recv[0] = send;
3410 }
3411 
3412 template <typename T>
3413 inline void Communicator::gather(const unsigned int root_id, std::vector<T>&) const
3414 { libmesh_assert_equal_to(root_id, 0); }
3415 
3416 template <typename T>
3417 inline void Communicator::allgather(T send, std::vector<T> &recv) const
3418 {
3419  recv.resize(1);
3420  recv[0] = send;
3421 }
3422 
3423 template <typename T>
3424 inline void Communicator::allgather(std::vector<T> &, const bool) const {}
3425 
3426 template <typename T>
3427 inline void Communicator::alltoall(std::vector<T> &) const {}
3428 
3429 template <typename T>
3430 inline void Communicator::broadcast (T &, const unsigned int root_id) const
3431 { libmesh_assert_equal_to(root_id, 0); }
3432 
3433 #endif // LIBMESH_HAVE_MPI
3434 
3435 } // namespace Parallel
3436 
3437 } // namespace libMesh
3438 
3439 #endif // LIBMESH_PARALLEL_IMPLEMENTATION_H

Site Created By: libMesh Developers
Last modified: February 07 2014 16:57:06 UTC

Hosted By:
SourceForge.net Logo