Home
Publications
Presentations
Support
Download
Examples
Developers
Class Docs
Namespaces
Classes
Files
Mailing Lists
Gallery
Wiki

parallel.h

Go to the documentation of this file.

// The libMesh Finite Element Library.
// Copyright (C) 2002-2012 Benjamin S. Kirk, John W. Peterson, Roy H. Stogner

// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.

// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.

// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA


#ifndef LIBMESH_PARALLEL_H
#define LIBMESH_PARALLEL_H

// Local includes
#include "libmesh/libmesh_common.h" // libmesh_assert, libmesh_cast_int
#include "libmesh/libmesh_logging.h"

// C++ includes
#include <cstddef>
#include <climits>
#include <iterator>
#include <limits>
#include <map>
#include <set>
#include <string>
#include <vector>

namespace libMesh
{


// Macro to identify and debug functions which should only be called in
// parallel on every processor at once

#undef parallel_only
#ifndef NDEBUG
  #define parallel_only() do { \
    libmesh_assert(CommWorld.verify(std::string(__FILE__).size())); \
    libmesh_assert(CommWorld.verify(std::string(__FILE__))); \
    libmesh_assert(CommWorld.verify(__LINE__)); } while (0)
#else
  #define parallel_only()  ((void) 0)
#endif

// Macro to identify and debug functions which should only be called in
// parallel on every processor at once

#undef parallel_only_on
#ifndef NDEBUG
  #define parallel_only_on(comm_arg) do { \
    libmesh_assert(CommWorld.verify(std::string(__FILE__).size(), comm_arg)); \
    libmesh_assert(CommWorld.verify(std::string(__FILE__), comm_arg)); \
    libmesh_assert(CommWorld.verify(__LINE__), comm_arg); } while (0)
#else
  #define parallel_only_on(comm_arg)  ((void) 0)
#endif

namespace Parallel
{
  //-------------------------------------------------------------------
  class Communicator;
  class DataType;
  class Request;
  class Status;

#ifdef LIBMESH_HAVE_MPI
  //-------------------------------------------------------------------
  typedef MPI_Datatype data_type;

  typedef MPI_Request request;

  typedef MPI_Status status;

  typedef MPI_Comm communicator;

  template <typename T>
  inline data_type dataplusint_type();

  template <typename T>
  class DataPlusInt
  {
    public:
      T val;
      int rank;
  };

  const unsigned int any_source =
    static_cast<unsigned int>(MPI_ANY_SOURCE);


#else

  // These shouldn't actually be needed, but must be
  // unique types for function overloading to work
  // properly.
  struct data_type    { /* unsigned int t; */ };
  struct request      { /* unsigned int r; */ };
  struct status       { /* unsigned int s; */ };
  struct communicator { /* unsigned int s; */ };

  const unsigned int any_source=0;
#endif // LIBMESH_HAVE_MPI



  //-------------------------------------------------------------------
  class MessageTag
  {
  public:

    static const int invalid_tag = INT_MIN;

    explicit MessageTag(int tagvalue = invalid_tag)
      : _tagvalue(tagvalue), _comm(NULL) {}

    MessageTag(const MessageTag& other);

    ~MessageTag();

    int value() const {
      return _tagvalue;
    }

  private:
    int _tagvalue;
    const Communicator *_comm;

    // Constructor for reference-counted unique tags
    MessageTag(int tagvalue, const Communicator *comm)
      : _tagvalue(tagvalue), _comm(comm) {}

    // Let Communicator handle the reference counting
    friend class Communicator;
  };


  //-------------------------------------------------------------------
#ifdef LIBMESH_HAVE_MPI
  const MessageTag any_tag = MessageTag(MPI_ANY_TAG);
#else
  const MessageTag any_tag = MessageTag(-1);
#endif

  const MessageTag no_tag = MessageTag(0);


  //-------------------------------------------------------------------
  class DataType
  {
  public:
    DataType () : _datatype() {}

    DataType (const DataType &other) :
      _datatype(other._datatype)
    {}

    DataType (const data_type &type) :
      _datatype(type)
    {}

#ifdef LIBMESH_HAVE_MPI
    DataType (const DataType &other, unsigned int count)
    {
      MPI_Type_contiguous(count, other._datatype, &_datatype);
      this->commit();
    }
#else
    DataType (const DataType &, unsigned int)
    {
    }
#endif

    DataType & operator = (const DataType &other)
    { _datatype = other._datatype; return *this; }

    DataType & operator = (const data_type &type)
    { _datatype = type; return *this; }

    operator const data_type & () const
    { return _datatype; }

    operator data_type & ()
    { return _datatype; }

//     operator data_type const * () const
//     { return &_datatype; }

//     operator data_type * ()
//     { return &_datatype; }

    void commit ()
    {
#ifdef LIBMESH_HAVE_MPI
      MPI_Type_commit (&_datatype);
#endif
    }

    void free ()
    {
#ifdef LIBMESH_HAVE_MPI
      MPI_Type_free (&_datatype);
#endif
    }

  protected:

    data_type _datatype;
  };


  //-------------------------------------------------------------------
  template <typename T>
  class StandardType : public DataType
  {
  /*
   * The unspecialized class is useless, so we make its constructor
   * private to catch mistakes at compile-time rather than link-time.
   * Specializations should have a public constructor of the same
   * form.
   */
  private:
    StandardType(const T* example = NULL);
  };

  /*
   * The unspecialized class gives default, lowest-common-denominator
   * attributes, for values which can't be used with Parallel min/max.
   * Specialized classes can set this to true, and should define
   * the lowest and highest values possible for the type.
   */
  template<typename T>
  struct Attributes
  {
    static const bool has_min_max = false;
    static void set_lowest(T&) {}
    static void set_highest(T&) {}
  };



  //-------------------------------------------------------------------
  class Status
  {
  public:
    Status ();

    explicit Status (const data_type &type);

    explicit Status (const status &status);

    Status (const status    &status,
            const data_type &type);

    Status (const Status &status);

    Status (const Status    &status,
            const data_type &type);

    status * get() { return &_status; }

    status const * get() const { return &_status; }

    int source () const;

    int tag () const;

    data_type& datatype () { return _datatype; }

    const data_type& datatype () const { return _datatype; }

    unsigned int size (const data_type &type) const;

    unsigned int size () const;

  private:

    status    _status;
    data_type _datatype;
  };


  //-------------------------------------------------------------------
  struct PostWaitWork {
    virtual ~PostWaitWork() {};

    virtual void run() {};
  };


  //-------------------------------------------------------------------
  class Request
  {
  public:
    Request ();

    Request (const request &r);

    Request (const Request &other);

    void cleanup();

    Request & operator = (const Request &other);

    Request & operator = (const request &r);

    ~Request ();

    request* get() { return &_request; }

    const request* get() const { return &_request; }

    Status wait ();

    bool test ();

    bool test (status &status);

    void add_post_wait_work(PostWaitWork* work);

  private:
    request _request;

    // post_wait_work->first is a vector of work to do after a wait
    // finishes; post_wait_work->second is a reference count so that
    // Request objects will behave roughly like a shared_ptr and be
    // usable in STL containers
    std::pair<std::vector <PostWaitWork* >, unsigned int>* post_wait_work;
  };

  inline Status wait (Request &r) { return r.wait(); }

  inline void wait (std::vector<Request> &r)
  { for (unsigned int i=0; i<r.size(); i++) r[i].wait(); }



  template <typename T, typename Context>
  void pack(const T* object,
            std::vector<int>& data,
            const Context* context);

  template <typename T, typename Context>
  unsigned int packable_size(const T*, const Context*);

  template <typename T>
  unsigned int packed_size(const T*,
                           const std::vector<int>::const_iterator);

  template <typename T, typename Context>
  void unpack(std::vector<int>::const_iterator in, T** out, Context*);

  template <typename Context, typename OutputIter>
  inline void unpack_range (const std::vector<int>& buffer,
                            Context *context,
                            OutputIter out);

  template <typename Context, typename Iter>
  inline void pack_range (const Context *context,
                          Iter range_begin,
                          const Iter range_end,
                          std::vector<int>& buffer);

  //-------------------------------------------------------------------
  class Communicator
  {
  // Basic operations:
  public:

    Communicator ();

    /*
     * Constructor from MPI_Comm
     */
    explicit Communicator (const communicator &comm);

    /*
     * NON-VIRTUAL destructor
     */
    ~Communicator ();

    /*
     * Create a new communicator between some subset of \p this
     */
    void split(int color, int key, Communicator &target);

    /*
     * Create a new duplicate of \p this communicator
     */
    void duplicate(const Communicator &comm);

    /*
     * Create a new duplicate of an MPI communicator
     */
    void duplicate(const communicator &comm);

    communicator& get() { return _communicator; }

    const communicator& get() const { return _communicator; }

    MessageTag get_unique_tag(int tagvalue) const;

    void reference_unique_tag(int tagvalue) const;

    void dereference_unique_tag(int tagvalue) const;

    void clear();

    Communicator& operator= (const communicator &comm);

    unsigned int rank() const { return _rank; }

    unsigned int size() const { return _size; }

  private:

    // Don't use the copy constructor, just copy by reference or
    // pointer - it's too hard to keep a common used_tag_values if
    // each communicator is shared by more than one Communicator
    explicit Communicator (const Communicator &);

    void assign(const communicator &comm);

    communicator  _communicator;
    unsigned int  _rank, _size;

    // mutable used_tag_values - not thread-safe, but then Parallel::
    // isn't thread-safe in general.
    mutable std::map<int, unsigned int> used_tag_values;
    bool          _I_duped_it;

  // Communication operations:
  public:

    void barrier () const;

    template <typename T>
    bool verify(const T &r) const;

    template <typename T>
    bool semiverify(const T *r) const;

    template <typename T>
    void min(T &r) const;

    template <typename T>
    void minloc(T &r,
                unsigned int &min_id) const;

    template <typename T>
    void minloc(std::vector<T> &r,
                std::vector<unsigned int> &min_id) const;

    template <typename T>
    void max(T &r) const;

    template <typename T>
    void maxloc(T &r,
                unsigned int &max_id) const;
  
    template <typename T>
    void maxloc(std::vector<T> &r,
                std::vector<unsigned int> &max_id) const;

    template <typename T>
    void sum(T &r) const;

    template <typename T>
    void set_union(T &data, const unsigned int root_id) const;

    template <typename T>
    void set_union(T &data) const;

    status probe (const unsigned int src_processor_id,
                  const MessageTag &tag=any_tag) const;

    template <typename T>
    void send (const unsigned int dest_processor_id,
               T &buf,
               const MessageTag &tag=no_tag) const;

    template <typename T>
    void send (const unsigned int dest_processor_id,
               T &buf,
               Request &req,
               const MessageTag &tag=no_tag) const;

    template <typename T>
    void send (const unsigned int dest_processor_id,
               T &buf,
               const DataType &type,
               const MessageTag &tag=no_tag) const;

    template <typename T>
    void send (const unsigned int dest_processor_id,
               T &buf,
               const DataType &type,
               Request &req,
               const MessageTag &tag=no_tag) const;

    template <typename T>
    Status receive (const unsigned int dest_processor_id,
                    T &buf,
                    const MessageTag &tag=any_tag) const;

    template <typename T>
    void receive (const unsigned int dest_processor_id,
                  T &buf,
                  Request &req,
                  const MessageTag &tag=any_tag) const;

    template <typename T>
    Status receive (const unsigned int dest_processor_id,
                    T &buf,
                    const DataType &type,
                    const MessageTag &tag=any_tag) const;

    template <typename T>
    void receive (const unsigned int dest_processor_id,
                  T &buf,
                  const DataType &type,
                  Request &req,
                  const MessageTag &tag=any_tag) const;

    template <typename Context, typename Iter>
    void send_packed_range (const unsigned int dest_processor_id,
                            const Context *context,
                            Iter range_begin,
                            const Iter range_end,
                            const MessageTag &tag=no_tag) const;

    template <typename Context, typename Iter>
    void send_packed_range (const unsigned int dest_processor_id,
                            const Context *context,
                            Iter range_begin,
                            const Iter range_end,
                            Request &req,
                            const MessageTag &tag=no_tag) const;

    template <typename Context, typename OutputIter>
    void receive_packed_range (const unsigned int dest_processor_id,
                               Context *context,
                               OutputIter out,
                               const MessageTag &tag=any_tag) const;

    template <typename Context, typename OutputIter>
    void receive_packed_range (const unsigned int dest_processor_id,
                               Context *context,
                               OutputIter out,
                               Request &req,
                               const MessageTag &tag=any_tag) const;

    template <typename T1, typename T2>
    void send_receive(const unsigned int dest_processor_id,
                      T1 &send,
                      const unsigned int source_processor_id,
                      T2 &recv,
                      const MessageTag &send_tag = no_tag,
                      const MessageTag &recv_tag = any_tag) const;

    template <typename Context1, typename RangeIter, typename Context2, typename OutputIter>
    void send_receive_packed_range(const unsigned int dest_processor_id,
                                   const Context1* context1,
                                   RangeIter send_begin,
                                   const RangeIter send_end,
                                   const unsigned int source_processor_id,
                                   Context2* context2,
                                   OutputIter out,
                                   const MessageTag &send_tag = no_tag,
                                   const MessageTag &recv_tag = any_tag) const;

    template <typename T1, typename T2>
    void send_receive(const unsigned int dest_processor_id,
                      T1 &send,
                      const DataType &type1,
                      const unsigned int source_processor_id,
                      T2 &recv,
                      const DataType &type2,
                      const MessageTag &send_tag = no_tag,
                      const MessageTag &recv_tag = any_tag) const;

    template <typename T>
    inline void gather(const unsigned int root_id,
                       T send,
                       std::vector<T> &recv) const;

    template <typename T>
    inline void gather(const unsigned int root_id,
                       std::vector<T> &r) const;

    template <typename T>
    inline void allgather(T send,
                          std::vector<T> &recv) const;


    template <typename T>
    inline void allgather(std::vector<T> &r,
                          const bool identical_buffer_sizes = false) const;

    //-------------------------------------------------------------------
    template <typename Context, typename Iter, typename OutputIter>
    inline void allgather_packed_range (Context *context,
                                        Iter range_begin,
                                        const Iter range_end,
                                        OutputIter out) const;

    template <typename T>
    inline void alltoall(std::vector<T> &r) const;

    template <typename T>
    inline void broadcast(T &data, const unsigned int root_id=0) const;

    template <typename Context, typename OutputContext, typename Iter, typename OutputIter>
    inline void broadcast_packed_range (const Context *context1,
                                        Iter range_begin,
                                        const Iter range_end,
                                        OutputContext *context2,
                                        OutputIter out,
                                        const unsigned int root_id = 0) const;

#include "libmesh/parallel_communicator_specializations"

  }; // class Communicator

  // PostWaitWork specialization for copying from temporary to
  // output containers
  template <typename Container, typename OutputIter>
  struct PostWaitCopyBuffer : public PostWaitWork {
    PostWaitCopyBuffer(const Container& buffer, const OutputIter out) 
      : _buf(buffer), _out(out) {}

    virtual void run() { std::copy(_buf.begin(), _buf.end(), _out); }

    private:
    const Container& _buf;
    OutputIter _out;
  };

  // PostWaitWork specialization for unpacking received buffers.
  template <typename Container, typename Context, typename OutputIter>
  struct PostWaitUnpackBuffer : public PostWaitWork {
    PostWaitUnpackBuffer(const Container& buffer, Context *context, OutputIter out) :
      _buf(buffer), _context(context), _out(out) {}

    virtual void run() { Parallel::unpack_range(_buf, _context, _out); };

    private:
    const Container& _buf;
    Context *_context;
    OutputIter _out;
  };


  // PostWaitWork specialization for freeing no-longer-needed buffers.
  template <typename Container>
  struct PostWaitDeleteBuffer : public PostWaitWork {
    PostWaitDeleteBuffer(Container* buffer) : _buf(buffer) {}

    virtual void run() { delete _buf; };

    private:
    Container* _buf;
  };

} // namespace Parallel

  extern Parallel::Communicator CommWorld;

} // namespace libMesh

// Define all the implementations separately; users might want to look
// through this file for APIs, and it's long enough already.

#include "libmesh/parallel_implementation.h"

#endif // LIBMESH_PARALLEL_H

---

Site Created By: libMesh Developers
Last modified: February 05 2013 19:54:48 UTC

Hosted By: