parallel_elem.C
Go to the documentation of this file.
1 // The libMesh Finite Element Library.
2 // Copyright (C) 2002-2014 Benjamin S. Kirk, John W. Peterson, Roy H. Stogner
3 
4 // This library is free software; you can redistribute it and/or
5 // modify it under the terms of the GNU Lesser General Public
6 // License as published by the Free Software Foundation; either
7 // version 2.1 of the License, or (at your option) any later version.
8 
9 // This library is distributed in the hope that it will be useful,
10 // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 // Lesser General Public License for more details.
13 
14 // You should have received a copy of the GNU Lesser General Public
15 // License along with this library; if not, write to the Free Software
16 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 
18 
19 
20 // C++ includes
21 
22 // Local includes
23 #include "libmesh/boundary_info.h"
24 #include "libmesh/elem.h"
25 #include "libmesh/mesh_base.h"
26 #include "libmesh/parallel.h"
27 #include "libmesh/parallel_mesh.h"
28 #include "libmesh/remote_elem.h"
29 
30 // Helper functions in anonymous namespace
31 
32 namespace
33 {
34 #ifdef LIBMESH_ENABLE_UNIQUE_ID
35  static const unsigned int header_size = 11;
36 #else
37  static const unsigned int header_size = 10;
38 #endif
39 
40  static const largest_id_type elem_magic_header = 987654321;
41 }
42 
43 
44 namespace libMesh
45 {
46 
47 #ifdef LIBMESH_HAVE_MPI
48 
49 namespace Parallel
50 {
51 
52 template <>
53 unsigned int packed_size (const Elem*,
54  std::vector<largest_id_type>::const_iterator in)
55 {
56 #ifndef NDEBUG
57  const largest_id_type packed_header = *in++;
58  libmesh_assert_equal_to (packed_header, elem_magic_header);
59 #endif
60 
61  // int 0: level
62  const unsigned int level =
63  static_cast<unsigned int>(*in);
64 
65  // int 4: element type
66  const int typeint = *(in+4);
67  libmesh_assert_greater_equal (typeint, 0);
68  libmesh_assert_less (typeint, INVALID_ELEM);
69  const ElemType type =
70  static_cast<ElemType>(typeint);
71 
72  const unsigned int n_nodes =
74 
75  const unsigned int n_sides =
77 
78  const unsigned int n_edges =
80 
81  const unsigned int pre_indexing_size =
82  header_size + n_nodes + n_sides;
83 
84  const unsigned int indexing_size =
85  DofObject::unpackable_indexing_size(in+pre_indexing_size);
86 
87  unsigned int total_packed_bc_data = 0;
88  if (level == 0)
89  {
90  for (unsigned int s = 0; s != n_sides; ++s)
91  {
92  const int n_bcs =
93  *(in + pre_indexing_size + indexing_size +
94  total_packed_bc_data++);
95  libmesh_assert_greater_equal (n_bcs, 0);
96  total_packed_bc_data += n_bcs;
97  }
98 
99  for (unsigned int e = 0; e != n_edges; ++e)
100  {
101  const int n_bcs =
102  *(in + pre_indexing_size + indexing_size +
103  total_packed_bc_data++);
104  libmesh_assert_greater_equal (n_bcs, 0);
105  total_packed_bc_data += n_bcs;
106  }
107  }
108 
109  return
110 #ifndef NDEBUG
111  1 + // Account for magic header
112 #endif
113  pre_indexing_size + indexing_size + total_packed_bc_data;
114 }
115 
116 
117 
118 template <>
119 unsigned int packed_size (const Elem* e,
120  std::vector<largest_id_type>::iterator in)
121 {
122  return packed_size(e, std::vector<largest_id_type>::const_iterator(in));
123 }
124 
125 
126 
127 template <>
128 unsigned int packable_size (const Elem* elem, const MeshBase* mesh)
129 {
130  unsigned int total_packed_bcs = 0;
131  if (elem->level() == 0)
132  {
133  total_packed_bcs += elem->n_sides();
134  for (unsigned int s = 0; s != elem->n_sides(); ++s)
135  total_packed_bcs += mesh->boundary_info->n_boundary_ids(elem,s);
136 
137  total_packed_bcs += elem->n_edges();
138  for (unsigned int e = 0; e != elem->n_edges(); ++e)
139  total_packed_bcs += mesh->boundary_info->n_edge_boundary_ids(elem,e);
140  }
141 
142  return
143 #ifndef NDEBUG
144  1 + // add an int for the magic header when testing
145 #endif
146  header_size + elem->n_nodes() +
147  elem->n_neighbors() +
148  elem->packed_indexing_size() + total_packed_bcs;
149 }
150 
151 
152 
153 template <>
154 unsigned int packable_size (const Elem* elem, const ParallelMesh* mesh)
155 {
156  return packable_size(elem, static_cast<const MeshBase*>(mesh));
157 }
158 
159 
160 
161 template <>
162 void pack (const Elem* elem,
163  std::vector<largest_id_type>& data,
164  const MeshBase* mesh)
165 {
166  libmesh_assert(elem);
167 
168  // This should be redundant when used with Parallel::pack_range()
169  // data.reserve (data.size() + Parallel::packable_size(elem, mesh));
170 
171 #ifndef NDEBUG
172  data.push_back (elem_magic_header);
173 #endif
174 
175 #ifdef LIBMESH_ENABLE_AMR
176  data.push_back (static_cast<largest_id_type>(elem->level()));
177  data.push_back (static_cast<largest_id_type>(elem->p_level()));
178  data.push_back (static_cast<largest_id_type>(elem->refinement_flag()));
179  data.push_back (static_cast<largest_id_type>(elem->p_refinement_flag()));
180 #else
181  data.push_back (0);
182  data.push_back (0);
183  data.push_back (0);
184  data.push_back (0);
185 #endif
186  data.push_back (static_cast<largest_id_type>(elem->type()));
187  data.push_back (elem->processor_id());
188  data.push_back (elem->subdomain_id());
189  data.push_back (elem->id());
190 
191 #ifdef LIBMESH_ENABLE_UNIQUE_ID
192  if (elem->valid_unique_id())
193  data.push_back (static_cast<largest_id_type>(elem->unique_id()));
194  else
195  // OK to send invalid unique id, we must not own this DOF
196  data.push_back (static_cast<largest_id_type>(DofObject::invalid_unique_id));
197 #endif
198 
199 #ifdef LIBMESH_ENABLE_AMR
200  // use parent_ID of -1 to indicate a level 0 element
201  if (elem->level() == 0)
202  {
203  data.push_back(-1);
204  data.push_back(-1);
205  }
206  else
207  {
208  data.push_back(elem->parent()->id());
209  data.push_back(elem->parent()->which_child_am_i(elem));
210  }
211 #else
212  data.push_back (-1);
213  data.push_back (-1);
214 #endif
215 
216  for (unsigned int n=0; n<elem->n_nodes(); n++)
217  data.push_back (elem->node(n));
218 
219  for (unsigned int n=0; n<elem->n_neighbors(); n++)
220  {
221  const Elem *neigh = elem->neighbor(n);
222  if (neigh)
223  data.push_back (neigh->id());
224  else
225  data.push_back (-1);
226  }
227 
228 #ifndef NDEBUG
229  const std::size_t start_indices = data.size();
230 #endif
231  // Add any DofObject indices
232  elem->pack_indexing(std::back_inserter(data));
233 
236  start_indices));
237 
238  libmesh_assert_equal_to (elem->packed_indexing_size(),
239  data.size() - start_indices);
240 
241 
242  // If this is a coarse element,
243  // Add any element side boundary condition ids
244  if (elem->level() == 0)
245  {
246  for (unsigned int s = 0; s != elem->n_sides(); ++s)
247  {
248  std::vector<boundary_id_type> bcs =
249  mesh->boundary_info->boundary_ids(elem, s);
250 
251  data.push_back(bcs.size());
252 
253  for(unsigned int bc_it=0; bc_it < bcs.size(); bc_it++)
254  data.push_back(bcs[bc_it]);
255  }
256 
257  for (unsigned int e = 0; e != elem->n_edges(); ++e)
258  {
259  std::vector<boundary_id_type> bcs =
260  mesh->boundary_info->edge_boundary_ids(elem, e);
261 
262  data.push_back(bcs.size());
263 
264  for(unsigned int bc_it=0; bc_it < bcs.size(); bc_it++)
265  data.push_back(bcs[bc_it]);
266  }
267  }
268 }
269 
270 
271 
272 template <>
273 void pack (const Elem* elem,
274  std::vector<largest_id_type>& data,
275  const ParallelMesh* mesh)
276 {
277  pack(elem, data, static_cast<const MeshBase*>(mesh));
278 }
279 
280 
281 
282 // FIXME - this needs serious work to be 64-bit compatible
283 template <>
284 void unpack(std::vector<largest_id_type>::const_iterator in,
285  Elem** out,
286  MeshBase* mesh)
287 {
288 #ifndef NDEBUG
289  const std::vector<largest_id_type>::const_iterator original_in = in;
290 
291  const largest_id_type incoming_header = *in++;
292  libmesh_assert_equal_to (incoming_header, elem_magic_header);
293 #endif
294 
295  // int 0: level
296  const unsigned int level =
297  static_cast<unsigned int>(*in++);
298 
299 #ifdef LIBMESH_ENABLE_AMR
300  // int 1: p level
301  const unsigned int p_level =
302  static_cast<unsigned int>(*in++);
303 
304  // int 2: refinement flag
305  const int rflag = *in++;
306  libmesh_assert_greater_equal (rflag, 0);
307  libmesh_assert_less (rflag, Elem::INVALID_REFINEMENTSTATE);
308  const Elem::RefinementState refinement_flag =
309  static_cast<Elem::RefinementState>(rflag);
310 
311  // int 3: p refinement flag
312  const int pflag = *in++;
313  libmesh_assert_greater_equal (pflag, 0);
314  libmesh_assert_less (pflag, Elem::INVALID_REFINEMENTSTATE);
315  const Elem::RefinementState p_refinement_flag =
316  static_cast<Elem::RefinementState>(pflag);
317 #else
318  in += 3;
319 #endif // LIBMESH_ENABLE_AMR
320 
321  // int 4: element type
322  const int typeint = *in++;
323  libmesh_assert_greater_equal (typeint, 0);
324  libmesh_assert_less (typeint, INVALID_ELEM);
325  const ElemType type =
326  static_cast<ElemType>(typeint);
327 
328  const unsigned int n_nodes =
330 
331  // int 5: processor id
333  static_cast<processor_id_type>(*in++);
334  libmesh_assert (processor_id < mesh->n_processors() ||
335  processor_id == DofObject::invalid_processor_id);
336 
337  // int 6: subdomain id
338  const subdomain_id_type subdomain_id =
339  static_cast<subdomain_id_type>(*in++);
340 
341  // int 7: dof object id
342  const dof_id_type id =
343  static_cast<dof_id_type>(*in++);
344  libmesh_assert_not_equal_to (id, DofObject::invalid_id);
345 
346 #ifdef LIBMESH_ENABLE_UNIQUE_ID
347  // int 8: dof object unique id
348  const unique_id_type unique_id =
349  static_cast<unique_id_type>(*in++);
350 #endif
351 
352 #ifdef LIBMESH_ENABLE_AMR
353  // int 9: parent dof object id
354  const dof_id_type parent_id =
355  static_cast<dof_id_type>(*in++);
356  libmesh_assert (level == 0 || parent_id != DofObject::invalid_id);
357  libmesh_assert (level != 0 || parent_id == DofObject::invalid_id);
358 
359  // int 10: local child id
360  const unsigned int which_child_am_i =
361  static_cast<unsigned int>(*in++);
362 #else
363  in += 2;
364 #endif // LIBMESH_ENABLE_AMR
365 
366  // Make sure we don't miscount above when adding the "magic" header
367  // plus the real data header
368  libmesh_assert_equal_to (in - original_in, header_size + 1);
369 
370  Elem *elem = mesh->query_elem(id);
371 
372  // if we already have this element, make sure its
373  // properties match, and update any missing neighbor
374  // links, but then go on
375  if (elem)
376  {
377  libmesh_assert_equal_to (elem->level(), level);
378  libmesh_assert_equal_to (elem->id(), id);
379 //#ifdef LIBMESH_ENABLE_UNIQUE_ID
380  // No check for unqiue id sanity
381 //#endif
382  libmesh_assert_equal_to (elem->processor_id(), processor_id);
383  libmesh_assert_equal_to (elem->subdomain_id(), subdomain_id);
384  libmesh_assert_equal_to (elem->type(), type);
385  libmesh_assert_equal_to (elem->n_nodes(), n_nodes);
386 
387 #ifndef NDEBUG
388  // All our nodes should be correct
389  for (unsigned int i=0; i != n_nodes; ++i)
390  libmesh_assert(elem->node(i) ==
391  static_cast<dof_id_type>(*in++));
392 #else
393  in += n_nodes;
394 #endif
395 
396 #ifdef LIBMESH_ENABLE_AMR
397  libmesh_assert_equal_to (elem->p_level(), p_level);
398  libmesh_assert_equal_to (elem->refinement_flag(), refinement_flag);
399  libmesh_assert_equal_to (elem->p_refinement_flag(), p_refinement_flag);
400 
401  libmesh_assert (!level || elem->parent() != NULL);
402  libmesh_assert (!level || elem->parent()->id() == parent_id);
403  libmesh_assert (!level || elem->parent()->child(which_child_am_i) == elem);
404 #endif
405 
406  // Our neighbor links should be "close to" correct - we may have
407  // to update them, but we can check for some inconsistencies.
408  for (unsigned int n=0; n != elem->n_neighbors(); ++n)
409  {
410  const dof_id_type neighbor_id =
411  static_cast<dof_id_type>(*in++);
412 
413  // If the sending processor sees a domain boundary here,
414  // we'd better agree.
415  if (neighbor_id == DofObject::invalid_id)
416  {
417  libmesh_assert (!(elem->neighbor(n)));
418  continue;
419  }
420 
421  // If the sending processor has a remote_elem neighbor here,
422  // then all we know is that we'd better *not* have a domain
423  // boundary.
424  if (neighbor_id == remote_elem->id())
425  {
426  libmesh_assert(elem->neighbor(n));
427  continue;
428  }
429 
430  Elem *neigh = mesh->query_elem(neighbor_id);
431 
432  // The sending processor sees a neighbor here, so if we
433  // don't have that neighboring element, then we'd better
434  // have a remote_elem signifying that fact.
435  if (!neigh)
436  {
437  libmesh_assert_equal_to (elem->neighbor(n), remote_elem);
438  continue;
439  }
440 
441  // The sending processor has a neighbor here, and we have
442  // that element, but that does *NOT* mean we're already
443  // linking to it. Perhaps we initially received both elem
444  // and neigh from processors on which their mutual link was
445  // remote?
446  libmesh_assert(elem->neighbor(n) == neigh ||
447  elem->neighbor(n) == remote_elem);
448 
449  // If the link was originally remote, we should update it,
450  // and make sure the appropriate parts of its family link
451  // back to us.
452  if (elem->neighbor(n) == remote_elem)
453  {
454  elem->set_neighbor(n, neigh);
455 
456  elem->make_links_to_me_local(n);
457  }
458  }
459 
460  // FIXME: We should add some debug mode tests to ensure that the
461  // encoded indexing and boundary conditions are consistent.
462  }
463  else
464  {
465  // We don't already have the element, so we need to create it.
466 
467  // Find the parent if necessary
468  Elem *parent = NULL;
469 #ifdef LIBMESH_ENABLE_AMR
470  // Find a child element's parent
471  if (level > 0)
472  {
473  // Note that we must be very careful to construct the send
474  // connectivity so that parents are encountered before
475  // children. If we get here and can't find the parent that
476  // is a fatal error.
477  parent = mesh->elem(parent_id);
478  }
479  // Or assert that the sending processor sees no parent
480  else
481  libmesh_assert_equal_to (parent_id, static_cast<dof_id_type>(-1));
482 #else
483  // No non-level-0 elements without AMR
484  libmesh_assert_equal_to (level, 0);
485 #endif
486 
487  elem = Elem::build(type,parent).release();
488  libmesh_assert (elem);
489 
490 #ifdef LIBMESH_ENABLE_AMR
491  if (level != 0)
492  {
493  // Since this is a newly created element, the parent must
494  // have previously thought of this child as a remote element.
495  libmesh_assert_equal_to (parent->child(which_child_am_i), remote_elem);
496 
497  parent->add_child(elem, which_child_am_i);
498  }
499 
500  // Assign the refinement flags and levels
501  elem->set_p_level(p_level);
502  elem->set_refinement_flag(refinement_flag);
503  elem->set_p_refinement_flag(p_refinement_flag);
504  libmesh_assert_equal_to (elem->level(), level);
505 
506  // If this element definitely should have children, assign
507  // remote_elem to all of them for now, for consistency. Later
508  // unpacked elements may overwrite that.
509  if (!elem->active())
510  for (unsigned int c=0; c != elem->n_children(); ++c)
511  elem->add_child(const_cast<RemoteElem*>(remote_elem), c);
512 
513 #endif // LIBMESH_ENABLE_AMR
514 
515  // Assign the IDs
516  elem->subdomain_id() = subdomain_id;
517  elem->processor_id() = processor_id;
518  elem->set_id() = id;
519 #ifdef LIBMESH_ENABLE_UNIQUE_ID
520  elem->set_unique_id() = unique_id;
521 #endif
522 
523  // Assign the connectivity
524  libmesh_assert_equal_to (elem->n_nodes(), n_nodes);
525 
526  for (unsigned int n=0; n != n_nodes; n++)
527  elem->set_node(n) =
528  mesh->node_ptr
529  (static_cast<dof_id_type>(*in++));
530 
531  for (unsigned int n=0; n<elem->n_neighbors(); n++)
532  {
533  const dof_id_type neighbor_id =
534  static_cast<dof_id_type>(*in++);
535 
536  if (neighbor_id == DofObject::invalid_id)
537  continue;
538 
539  // We may be unpacking an element that was a ghost element on the
540  // sender, in which case the element's neighbors may not all be
541  // known by the packed element. We'll have to set such
542  // neighbors to remote_elem ourselves and wait for a later
543  // packed element to give us better information.
544  if (neighbor_id == remote_elem->id())
545  {
546  elem->set_neighbor(n, const_cast<RemoteElem*>(remote_elem));
547  continue;
548  }
549 
550  // If we don't have the neighbor element, then it's a
551  // remote_elem until we get it.
552  Elem *neigh = mesh->query_elem(neighbor_id);
553  if (!neigh)
554  {
555  elem->set_neighbor(n, const_cast<RemoteElem*>(remote_elem));
556  continue;
557  }
558 
559  // If we have the neighbor element, then link to it, and
560  // make sure the appropriate parts of its family link back
561  // to us.
562  elem->set_neighbor(n, neigh);
563 
564  elem->make_links_to_me_local(n);
565  }
566 
567  elem->unpack_indexing(in);
568  }
569 
570  in += elem->packed_indexing_size();
571 
572  // If this is a coarse element,
573  // add any element side or edge boundary condition ids
574  if (level == 0)
575  {
576  for (unsigned int s = 0; s != elem->n_sides(); ++s)
577  {
578  const int num_bcs = *in++;
579  libmesh_assert_greater_equal (num_bcs, 0);
580 
581  for(int bc_it=0; bc_it < num_bcs; bc_it++)
582  mesh->boundary_info->add_side (elem, s, *in++);
583  }
584 
585  for (unsigned int e = 0; e != elem->n_edges(); ++e)
586  {
587  const int num_bcs = *in++;
588  libmesh_assert_greater_equal (num_bcs, 0);
589 
590  for(int bc_it=0; bc_it < num_bcs; bc_it++)
591  mesh->boundary_info->add_edge (elem, e, *in++);
592  }
593  }
594 
595  // Return the new element
596  *out = elem;
597 }
598 
599 
600 
601 template <>
602 void unpack(std::vector<largest_id_type>::const_iterator in,
603  Elem** out,
605 {
606  unpack(in, out, static_cast<MeshBase*>(mesh));
607 }
608 
609 } // namespace Parallel
610 
611 #endif // LIBMESH_HAVE_MPI
612 
613 } // namespace libMesh

Site Created By: libMesh Developers
Last modified: February 07 2014 16:57:06 UTC

Hosted By:
SourceForge.net Logo