parallel_ghost_sync.h
Go to the documentation of this file.
1 // The libMesh Finite Element Library.
2 // Copyright (C) 2002-2014 Benjamin S. Kirk, John W. Peterson, Roy H. Stogner
3 
4 // This library is free software; you can redistribute it and/or
5 // modify it under the terms of the GNU Lesser General Public
6 // License as published by the Free Software Foundation; either
7 // version 2.1 of the License, or (at your option) any later version.
8 
9 // This library is distributed in the hope that it will be useful,
10 // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 // Lesser General Public License for more details.
13 
14 // You should have received a copy of the GNU Lesser General Public
15 // License along with this library; if not, write to the Free Software
16 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 
18 
19 
20 #ifndef LIBMESH_PARALLEL_GHOST_SYNC_H
21 #define LIBMESH_PARALLEL_GHOST_SYNC_H
22 
23 // Local Includes -----------------------------------
24 #include "libmesh/auto_ptr.h"
25 #include "libmesh/elem.h"
26 #include "libmesh/location_maps.h"
27 #include "libmesh/mesh_base.h"
28 #include "libmesh/parallel.h"
29 
30 // C++ Includes -----------------------------------
31 
32 namespace libMesh
33 {
34 
35 
36 
37 //--------------------------------------------------------------------------
38 namespace Parallel {
39 
40  //------------------------------------------------------------------------
57  template <typename Iterator,
58  typename DofObjType,
59  typename SyncFunctor>
60  void sync_dofobject_data_by_xyz(const Communicator& communicator,
61  const Iterator& range_begin,
62  const Iterator& range_end,
63  LocationMap<DofObjType>* location_map,
64  SyncFunctor& sync);
65 
66  //------------------------------------------------------------------------
79  template <typename Iterator,
80  typename SyncFunctor>
81  void sync_dofobject_data_by_id(const Communicator& communicator,
82  const Iterator& range_begin,
83  const Iterator& range_end,
84  SyncFunctor& sync);
85 
86  //------------------------------------------------------------------------
100  template <typename Iterator,
101  typename SyncFunctor>
102  void sync_element_data_by_parent_id(MeshBase& mesh,
103  const Iterator& range_begin,
104  const Iterator& range_end,
105  SyncFunctor& sync);
106 
107  //------------------------------------------------------------------------
108  // Parallel members
109 
110 template <typename Iterator,
111  typename DofObjType,
112  typename SyncFunctor>
114  const Iterator& range_begin,
115  const Iterator& range_end,
116  LocationMap<DofObjType>& location_map,
117  SyncFunctor& sync)
118 {
119  // This function must be run on all processors at once
120  libmesh_parallel_only(communicator);
121 
122  // We need a valid location_map
123 #ifdef DEBUG
124  bool need_map_update = (range_begin != range_end && location_map.empty());
125  communicator.max(need_map_update);
126  libmesh_assert(!need_map_update);
127 #endif
128 
129  // Count the objectss to ask each processor about
130  std::vector<dof_id_type>
131  ghost_objects_from_proc(communicator.size(), 0);
132 
133  for (Iterator it = range_begin; it != range_end; ++it)
134  {
135  DofObjType *obj = *it;
136  libmesh_assert (obj);
137  processor_id_type obj_procid = obj->processor_id();
138  if (obj_procid != DofObject::invalid_processor_id)
139  ghost_objects_from_proc[obj_procid]++;
140  }
141 
142  // Request sets to send to each processor
143  std::vector<std::vector<Real> >
144  requested_objs_x(communicator.size()),
145  requested_objs_y(communicator.size()),
146  requested_objs_z(communicator.size());
147  // Corresponding ids to keep track of
148  std::vector<std::vector<dof_id_type> >
149  requested_objs_id(communicator.size());
150 
151  // We know how many objects live on each processor, so reserve()
152  // space for each.
153  for (processor_id_type p=0; p != communicator.size(); ++p)
154  if (p != communicator.rank())
155  {
156  requested_objs_x[p].reserve(ghost_objects_from_proc[p]);
157  requested_objs_y[p].reserve(ghost_objects_from_proc[p]);
158  requested_objs_z[p].reserve(ghost_objects_from_proc[p]);
159  requested_objs_id[p].reserve(ghost_objects_from_proc[p]);
160  }
161  for (Iterator it = range_begin; it != range_end; ++it)
162  {
163  DofObjType *obj = *it;
164  processor_id_type obj_procid = obj->processor_id();
165  if (obj_procid == communicator.rank() ||
166  obj_procid == DofObject::invalid_processor_id)
167  continue;
168 
169  Point p = location_map.point_of(*obj);
170  requested_objs_x[obj_procid].push_back(p(0));
171  requested_objs_y[obj_procid].push_back(p(1));
172  requested_objs_z[obj_procid].push_back(p(2));
173  requested_objs_id[obj_procid].push_back(obj->id());
174  }
175 
176  // Trade requests with other processors
177  for (processor_id_type p=1; p != communicator.size(); ++p)
178  {
179  // Trade my requests with processor procup and procdown
180  const processor_id_type procup =
181  libmesh_cast_int<processor_id_type>
182  ((communicator.rank() + p) % communicator.size());
183  const processor_id_type procdown =
184  libmesh_cast_int<processor_id_type>
185  ((communicator.size() + communicator.rank() - p) %
186  communicator.size());
187  std::vector<Real> request_to_fill_x,
188  request_to_fill_y,
189  request_to_fill_z;
190  communicator.send_receive(procup, requested_objs_x[procup],
191  procdown, request_to_fill_x);
192  communicator.send_receive(procup, requested_objs_y[procup],
193  procdown, request_to_fill_y);
194  communicator.send_receive(procup, requested_objs_z[procup],
195  procdown, request_to_fill_z);
196 
197  // Find the local id of each requested object
198  std::vector<dof_id_type> request_to_fill_id(request_to_fill_x.size());
199  for (std::size_t i=0; i != request_to_fill_x.size(); ++i)
200  {
201  Point pt(request_to_fill_x[i],
202  request_to_fill_y[i],
203  request_to_fill_z[i]);
204 
205  // Look for this object in the multimap
206  DofObjType *obj = location_map.find(pt);
207 
208  // We'd better find every object we're asked for
209  libmesh_assert (obj);
210 
211  // Return the object's correct processor id,
212  // and our (correct if it's local) id for it.
213  request_to_fill_id[i] = obj->id();
214  }
215 
216  // Gather whatever data the user wants
217  std::vector<typename SyncFunctor::datum> data;
218  sync.gather_data(request_to_fill_id, data);
219 
220  // Trade back the results
221  std::vector<typename SyncFunctor::datum> received_data;
222  communicator.send_receive(procdown, data,
223  procup, received_data);
224  libmesh_assert_equal_to (requested_objs_x[procup].size(),
225  received_data.size());
226 
227  // Let the user process the results
228  sync.act_on_data(requested_objs_id[procup], received_data);
229  }
230 }
231 
232 
233 
234 template <typename Iterator,
235  typename SyncFunctor>
237  const Iterator& range_begin,
238  const Iterator& range_end,
239  SyncFunctor& sync)
240 {
241  // This function must be run on all processors at once
242  libmesh_parallel_only(communicator);
243 
244  // Count the objects to ask each processor about
245  std::vector<dof_id_type>
246  ghost_objects_from_proc(communicator.size(), 0);
247 
248  for (Iterator it = range_begin; it != range_end; ++it)
249  {
250  DofObject *obj = *it;
251  libmesh_assert (obj);
252  processor_id_type obj_procid = obj->processor_id();
253  if (obj_procid != DofObject::invalid_processor_id)
254  ghost_objects_from_proc[obj_procid]++;
255  }
256 
257  // Request sets to send to each processor
258  std::vector<std::vector<dof_id_type> >
259  requested_objs_id(communicator.size());
260 
261  // We know how many objects live on each processor, so reserve()
262  // space for each.
263  for (processor_id_type p=0; p != communicator.size(); ++p)
264  if (p != communicator.rank())
265  {
266  requested_objs_id[p].reserve(ghost_objects_from_proc[p]);
267  }
268  for (Iterator it = range_begin; it != range_end; ++it)
269  {
270  DofObject *obj = *it;
271  processor_id_type obj_procid = obj->processor_id();
272  if (obj_procid == communicator.rank() ||
273  obj_procid == DofObject::invalid_processor_id)
274  continue;
275 
276  requested_objs_id[obj_procid].push_back(obj->id());
277  }
278 
279  // Trade requests with other processors
280  for (processor_id_type p=1; p != communicator.size(); ++p)
281  {
282  // Trade my requests with processor procup and procdown
283  const processor_id_type procup =
284  libmesh_cast_int<processor_id_type>
285  (communicator.rank() + p) % communicator.size();
286  const processor_id_type procdown =
287  libmesh_cast_int<processor_id_type>
288  ((communicator.size() + communicator.rank() - p) %
289  communicator.size());
290  std::vector<dof_id_type> request_to_fill_id;
291  communicator.send_receive(procup, requested_objs_id[procup],
292  procdown, request_to_fill_id);
293 
294  // Gather whatever data the user wants
295  std::vector<typename SyncFunctor::datum> data;
296  sync.gather_data(request_to_fill_id, data);
297 
298  // Trade back the results
299  std::vector<typename SyncFunctor::datum> received_data;
300  communicator.send_receive(procdown, data,
301  procup, received_data);
302  libmesh_assert_equal_to (requested_objs_id[procup].size(),
303  received_data.size());
304 
305  // Let the user process the results
306  sync.act_on_data(requested_objs_id[procup], received_data);
307  }
308 }
309 
310 
311 
312  // If there's no refined elements, there's nothing to sync
313 #ifdef LIBMESH_ENABLE_AMR
314 template <typename Iterator,
315  typename SyncFunctor>
317  const Iterator& range_begin,
318  const Iterator& range_end,
319  SyncFunctor& sync)
320 {
321  const Communicator &communicator (mesh.comm());
322 
323  // This function must be run on all processors at once
325 
326  // Count the objects to ask each processor about
327  std::vector<dof_id_type>
328  ghost_objects_from_proc(communicator.size(), 0);
329 
330  for (Iterator it = range_begin; it != range_end; ++it)
331  {
332  DofObject *obj = *it;
333  libmesh_assert (obj);
334  processor_id_type obj_procid = obj->processor_id();
335  if (obj_procid != DofObject::invalid_processor_id)
336  ghost_objects_from_proc[obj_procid]++;
337  }
338 
339  // Request sets to send to each processor
340  std::vector<std::vector<dof_id_type> >
341  requested_objs_id(communicator.size()),
342  requested_objs_parent_id(communicator.size());
343  std::vector<std::vector<unsigned char> >
344  requested_objs_child_num(communicator.size());
345 
346  // We know how many objects live on each processor, so reserve()
347  // space for each.
348  for (processor_id_type p=0; p != communicator.size(); ++p)
349  if (p != communicator.rank())
350  {
351  requested_objs_id[p].reserve(ghost_objects_from_proc[p]);
352  requested_objs_parent_id[p].reserve(ghost_objects_from_proc[p]);
353  requested_objs_child_num[p].reserve(ghost_objects_from_proc[p]);
354  }
355 
356  for (Iterator it = range_begin; it != range_end; ++it)
357  {
358  Elem *elem = *it;
359  processor_id_type obj_procid = elem->processor_id();
360  if (obj_procid == communicator.rank() ||
361  obj_procid == DofObject::invalid_processor_id)
362  continue;
363  const Elem *parent = elem->parent();
364  if (!parent || !elem->active())
365  continue;
366 
367  requested_objs_id[obj_procid].push_back(elem->id());
368  requested_objs_parent_id[obj_procid].push_back(parent->id());
369  requested_objs_child_num[obj_procid].push_back
370  (libmesh_cast_int<unsigned char>
371  (parent->which_child_am_i(elem)));
372  }
373 
374  // Trade requests with other processors
375  for (processor_id_type p=1; p != communicator.size(); ++p)
376  {
377  // Trade my requests with processor procup and procdown
378  const processor_id_type procup =
379  libmesh_cast_int<processor_id_type>
380  (communicator.rank() + p) % communicator.size();
381  const processor_id_type procdown =
382  libmesh_cast_int<processor_id_type>
383  ((communicator.size() + communicator.rank() - p) %
384  communicator.size());
385  std::vector<dof_id_type> request_to_fill_parent_id;
386  std::vector<unsigned char> request_to_fill_child_num;
387  communicator.send_receive(procup, requested_objs_parent_id[procup],
388  procdown, request_to_fill_parent_id);
389  communicator.send_receive(procup, requested_objs_child_num[procup],
390  procdown, request_to_fill_child_num);
391 
392  // Find the id of each requested element
393  std::size_t request_size = request_to_fill_parent_id.size();
394  std::vector<dof_id_type> request_to_fill_id(request_size);
395  for (std::size_t i=0; i != request_size; ++i)
396  {
397  Elem *parent = mesh.elem(request_to_fill_parent_id[i]);
398  libmesh_assert(parent);
399  libmesh_assert(parent->has_children());
400  Elem *child = parent->child(request_to_fill_child_num[i]);
401  libmesh_assert(child);
402  libmesh_assert(child->active());
403  request_to_fill_id[i] = child->id();
404  }
405 
406  // Gather whatever data the user wants
407  std::vector<typename SyncFunctor::datum> data;
408  sync.gather_data(request_to_fill_id, data);
409 
410  // Trade back the results
411  std::vector<typename SyncFunctor::datum> received_data;
412  communicator.send_receive(procdown, data,
413  procup, received_data);
414  libmesh_assert_equal_to (requested_objs_id[procup].size(),
415  received_data.size());
416 
417  // Let the user process the results
418  sync.act_on_data(requested_objs_id[procup], received_data);
419  }
420 }
421 #else
422 template <typename Iterator,
423  typename SyncFunctor>
425  const Iterator&,
426  const Iterator&,
427  SyncFunctor&)
428 {
429 }
430 #endif // LIBMESH_ENABLE_AMR
431 
432 
433 
434 
435 }
436 
437 
438 // This struct can be created and passed to the
439 // Parallel::sync_dofobject_data_by_id() function.
441 {
442  // The constructor. You need a reference to the mesh where you will
443  // be setting/getting nodal positions.
444  explicit
446 
447  // The datum typedef is required of this functor, so that the
448  // Parallel::sync_dofobject_data_by_id() function can create e.g.
449  // std::vector<datum>.
450  typedef Point datum;
451 
452  // First required interface. This function must fill up the data vector for the
453  // ids specified in the ids vector.
454  void gather_data (const std::vector<dof_id_type>& ids, std::vector<datum>& data);
455 
456  // Second required interface. This function must do something with the data in
457  // the data vector for the ids in the ids vector.
458  void act_on_data (const std::vector<dof_id_type>& ids, std::vector<datum>& data);
459 
461 };
462 
463 
464 } // namespace libMesh
465 
466 #endif // LIBMESH_PARALLEL_GHOST_SYNC_H

Site Created By: libMesh Developers
Last modified: February 07 2014 16:57:06 UTC

Hosted By:
SourceForge.net Logo