parallel_mesh.C
Go to the documentation of this file.
1 // The libMesh Finite Element Library.
2 // Copyright (C) 2002-2014 Benjamin S. Kirk, John W. Peterson, Roy H. Stogner
3 
4 // This library is free software; you can redistribute it and/or
5 // modify it under the terms of the GNU Lesser General Public
6 // License as published by the Free Software Foundation; either
7 // version 2.1 of the License, or (at your option) any later version.
8 
9 // This library is distributed in the hope that it will be useful,
10 // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 // Lesser General Public License for more details.
13 
14 // You should have received a copy of the GNU Lesser General Public
15 // License along with this library; if not, write to the Free Software
16 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 
18 
19 
20 // Local includes
21 #include "libmesh/boundary_info.h"
22 #include "libmesh/elem.h"
25 #include "libmesh/parallel_mesh.h"
26 #include "libmesh/parallel.h"
28 
29 namespace libMesh
30 {
31 
32 // ------------------------------------------------------------
33 // ParallelMesh class member functions
35  unsigned int d) :
36  UnstructuredMesh (comm,d), _is_serial(true),
37  _n_nodes(0), _n_elem(0), _max_node_id(0), _max_elem_id(0),
38  _next_free_local_node_id(this->processor_id()),
39  _next_free_local_elem_id(this->processor_id()),
40  _next_free_unpartitioned_node_id(this->n_processors()),
41  _next_free_unpartitioned_elem_id(this->n_processors())
42 {
43 #ifdef LIBMESH_ENABLE_UNIQUE_ID
44  _next_unique_id = this->processor_id();
45 #endif
46 
47  // FIXME: give parmetis the communicator!
49 }
50 
51 
52 #ifndef LIBMESH_DISABLE_COMMWORLD
53 // ------------------------------------------------------------
54 // ParallelMesh class member functions
55 ParallelMesh::ParallelMesh (unsigned int d) :
56  UnstructuredMesh (d), _is_serial(true),
57  _n_nodes(0), _n_elem(0), _max_node_id(0), _max_elem_id(0),
58  _next_free_local_node_id(this->processor_id()),
59  _next_free_local_elem_id(this->processor_id()),
60  _next_free_unpartitioned_node_id(this->n_processors()),
61  _next_free_unpartitioned_elem_id(this->n_processors())
62 {
63 #ifdef LIBMESH_ENABLE_UNIQUE_ID
64  _next_unique_id = this->processor_id();
65 #endif
66 
67  // FIXME: give parmetis the communicator!
69 }
70 #endif
71 
72 
74 {
75  this->clear(); // Free nodes and elements
76 }
77 
78 
79 // This might be specialized later, but right now it's just here to
80 // make sure the compiler doesn't give us a default (non-deep) copy
81 // constructor instead.
83  UnstructuredMesh (other_mesh), _is_serial(other_mesh._is_serial),
84  _n_nodes(0), _n_elem(0), _max_node_id(0), _max_elem_id(0),
85  _next_free_local_node_id(this->processor_id()),
86  _next_free_local_elem_id(this->processor_id()),
87  _next_free_unpartitioned_node_id(this->n_processors()),
88  _next_free_unpartitioned_elem_id(this->n_processors())
89 {
90  this->copy_nodes_and_elements(other_mesh);
91  _n_nodes = other_mesh.n_nodes();
92  _n_elem = other_mesh.n_elem();
93  _max_node_id = other_mesh.max_node_id();
94  _max_elem_id = other_mesh.max_elem_id();
96  other_mesh._next_free_local_node_id;
98  other_mesh._next_free_local_elem_id;
103  *this->boundary_info = *other_mesh.boundary_info;
104 
105  // Need to copy extra_ghost_elems
106  for(std::set<Elem *>::iterator it = other_mesh._extra_ghost_elems.begin();
107  it != other_mesh._extra_ghost_elems.end();
108  ++it)
109  _extra_ghost_elems.insert(elem((*it)->id()));
110 }
111 
112 
113 
115  UnstructuredMesh (other_mesh), _is_serial(other_mesh.is_serial()),
116  _n_nodes(0), _n_elem(0), _max_node_id(0), _max_elem_id(0),
117  _next_free_local_node_id(this->processor_id()),
118  _next_free_local_elem_id(this->processor_id()),
119  _next_free_unpartitioned_node_id(this->n_processors()),
120  _next_free_unpartitioned_elem_id(this->n_processors())
121 {
122  this->copy_nodes_and_elements(other_mesh);
123  *this->boundary_info = *other_mesh.boundary_info;
124 
126 }
127 
128 
129 // We use cached values for these so they can be called
130 // from one processor without bothering the rest, but
131 // we may need to update those caches before doing a full
132 // renumbering
134 {
135  // This function must be run on all processors at once
136  parallel_object_only();
137 
138  _n_elem = this->parallel_n_elem();
139  _n_nodes = this->parallel_n_nodes();
142 
145  ((_max_elem_id-1) / (this->n_processors() + 1) + 1) *
146  (this->n_processors() + 1) + this->n_processors();
149  ((_max_elem_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
150  (this->n_processors() + 1) + this->processor_id();
151 
154  ((_max_node_id-1) / (this->n_processors() + 1) + 1) *
155  (this->n_processors() + 1) + this->n_processors();
158  ((_max_node_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
159  (this->n_processors() + 1) + this->processor_id();
160 }
161 
162 
163 // Or in debug mode we may want to test the uncached values without
164 // changing the cache
166 {
167  // This function must be run on all processors at once
168  parallel_object_only();
169 
170  dof_id_type n_local = this->n_local_elem();
171  this->comm().sum(n_local);
172  n_local += this->n_unpartitioned_elem();
173  return n_local;
174 }
175 
176 
177 
179 {
180  // This function must be run on all processors at once
181  parallel_object_only();
182 
183  dof_id_type max_local = _elements.empty() ?
184  0 : _elements.rbegin()->first + 1;
185  this->comm().max(max_local);
186  return max_local;
187 }
188 
189 
190 
192 {
193  // This function must be run on all processors at once
194  parallel_object_only();
195 
197  this->comm().sum(n_local);
198  n_local += this->n_unpartitioned_nodes();
199  return n_local;
200 }
201 
202 
203 
205 {
206  // This function must be run on all processors at once
207  parallel_object_only();
208 
209  dof_id_type max_local = _nodes.empty() ?
210  0 : _nodes.rbegin()->first + 1;
211  this->comm().max(max_local);
212  return max_local;
213 }
214 
215 
216 
217 const Point& ParallelMesh::point (const dof_id_type i) const
218 {
220  libmesh_assert_equal_to (_nodes[i]->id(), i);
221 
222  return (*_nodes[i]);
223 }
224 
225 
226 
227 
228 
229 const Node& ParallelMesh::node (const dof_id_type i) const
230 {
232  libmesh_assert_equal_to (_nodes[i]->id(), i);
233 
234  return (*_nodes[i]);
235 }
236 
237 
238 
239 
240 
242 {
244  libmesh_assert_equal_to (_nodes[i]->id(), i);
245 
246  return (*_nodes[i]);
247 }
248 
249 
250 
252 {
254  libmesh_assert_equal_to (_nodes[i]->id(), i);
255 
256  return _nodes[i];
257 }
258 
259 
260 
261 
263 {
265  libmesh_assert_equal_to (_nodes[i]->id(), i);
266 
267  return _nodes[i];
268 }
269 
270 
271 
272 
274 {
275  std::map<dof_id_type, Node*>::const_iterator it = _nodes.find(i);
276  if (it != _nodes.end().it)
277  {
278  const Node* n = it->second;
279  libmesh_assert (!n || n->id() == i);
280  return n;
281  }
282 
283  return NULL;
284 }
285 
286 
287 
288 
290 {
291  std::map<dof_id_type, Node*>::const_iterator it = _nodes.find(i);
292  if (it != _nodes.end().it)
293  {
294  Node* n = it->second;
295  libmesh_assert (!n || n->id() == i);
296  return n;
297  }
298 
299  return NULL;
300 }
301 
302 
303 
304 
305 const Elem* ParallelMesh::elem (const dof_id_type i) const
306 {
308  libmesh_assert_equal_to (_elements[i]->id(), i);
309 
310  return _elements[i];
311 }
312 
313 
314 
315 
317 {
319  libmesh_assert_equal_to (_elements[i]->id(), i);
320 
321  return _elements[i];
322 }
323 
324 
325 
326 
328 {
329  std::map<dof_id_type, Elem*>::const_iterator it = _elements.find(i);
330  if (it != _elements.end().it)
331  {
332  const Elem* e = it->second;
333  libmesh_assert (!e || e->id() == i);
334  return e;
335  }
336 
337  return NULL;
338 }
339 
340 
341 
342 
344 {
345  std::map<dof_id_type, Elem*>::const_iterator it = _elements.find(i);
346  if (it != _elements.end().it)
347  {
348  Elem* e = _elements[i];
349  libmesh_assert (!e || e->id() == i);
350  return e;
351  }
352 
353  return NULL;
354 }
355 
356 
357 
358 
360 {
361  // Don't try to add NULLs!
362  libmesh_assert(e);
363 
364  // Trying to add an existing element is a no-op
365  if (e->valid_id() && _elements[e->id()] == e)
366  return e;
367 
368  const processor_id_type elem_procid = e->processor_id();
369 
370  if (!e->valid_id())
371  {
372  // We should only be creating new ids past the end of the range
373  // of existing ids
374  libmesh_assert_greater_equal(_next_free_unpartitioned_elem_id,
375  _max_elem_id);
376  libmesh_assert_greater_equal(_next_free_local_elem_id, _max_elem_id);
377 
378  // Use the unpartitioned ids for unpartitioned elems,
379  // in serial, and temporarily for ghost elems
381  if (elem_procid == this->processor_id() &&
382  !this->is_serial())
383  next_id = &_next_free_local_elem_id;
384  e->set_id (*next_id);
385  }
386 
387  {
388  // Advance next_ids up high enough that each is pointing to an
389  // unused id and any subsequent increments will still point us
390  // to unused ids
392  static_cast<dof_id_type>(e->id()+1));
393 
396  ((_max_elem_id-1) / (this->n_processors() + 1) + 1) *
397  (this->n_processors() + 1) + this->n_processors();
400  ((_max_elem_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
401  (this->n_processors() + 1) + this->processor_id();
402 
403 #ifndef NDEBUG
404  // We need a const mapvector so we don't inadvertently create
405  // NULL entries when testing for non-NULL ones
406  const mapvector<Elem*,dof_id_type>& const_elements = _elements;
407 #endif
409  libmesh_assert(!const_elements[_next_free_local_elem_id]);
410  }
411 
412  // Don't try to overwrite existing elems
413  libmesh_assert (!_elements[e->id()]);
414 
415  _elements[e->id()] = e;
416 
417  // Try to make the cached elem data more accurate
418  if (elem_procid == this->processor_id() ||
419  elem_procid == DofObject::invalid_processor_id)
420  _n_elem++;
421 
422 #ifdef LIBMESH_ENABLE_UNIQUE_ID
423  if (!e->valid_unique_id() && libMesh::processor_id() == e->processor_id())
424  {
426  _next_unique_id += this->n_processors();
427  }
428 #endif
429 
430 // Unpartitioned elems should be added on every processor
431 // And shouldn't be added in the same batch as ghost elems
432 // But we might be just adding on processor 0 to
433 // broadcast later
434 #if 0
435 #ifdef DEBUG
436  if (elem_procid == DofObject::invalid_processor_id)
437  {
438  dof_id_type elem_id = e->id();
439  this->comm().max(elem_id);
440  libmesh_assert_equal_to (elem_id, e->id());
441  }
442 #endif
443 #endif
444 
445  return e;
446 }
447 
448 
449 
451 {
452  if (_elements[e->id()])
453  this->delete_elem(_elements[e->id()]);
454 
455  _elements[e->id()] = e;
456 
457  return e;
458 }
459 
460 
461 
463 {
464  libmesh_assert (e);
465 
466  // Delete the element from the BoundaryInfo object
467  this->boundary_info->remove(e);
468 
469  // But not yet from the container; we might invalidate
470  // an iterator that way!
471 
472  //_elements.erase(e->id());
473 
474  // Instead, we set it to NULL for now
475 
476  _elements[e->id()] = NULL;
477 
478  // delete the element
479  delete e;
480 }
481 
482 
483 
485  const dof_id_type new_id)
486 {
487  Elem *el = _elements[old_id];
488  libmesh_assert (el);
489  libmesh_assert_equal_to (el->id(), old_id);
490 
491  el->set_id(new_id);
492  libmesh_assert (!_elements[new_id]);
493  _elements[new_id] = el;
494  _elements.erase(old_id);
495 }
496 
497 
498 
500  const dof_id_type id,
501  const processor_id_type proc_id)
502 {
503  if (_nodes.count(id))
504  {
505  Node *n = _nodes[id];
506  libmesh_assert (n);
507  libmesh_assert_equal_to (n->id(), id);
508 
509  *n = p;
510  n->processor_id() = proc_id;
511 
512  return n;
513  }
514 
515  Node* n = Node::build(p, id).release();
516  n->processor_id() = proc_id;
517 
518  return ParallelMesh::add_node(n);
519 }
520 
521 
522 
524 {
525  // Don't try to add NULLs!
526  libmesh_assert(n);
527 
528  // Trying to add an existing node is a no-op
529  if (n->valid_id() && _nodes[n->id()] == n)
530  return n;
531 
532  const processor_id_type node_procid = n->processor_id();
533 
534  if (!n->valid_id())
535  {
536  // We should only be creating new ids past the end of the range
537  // of existing ids
538  libmesh_assert_greater_equal(_next_free_unpartitioned_node_id,
539  _max_node_id);
540  libmesh_assert_greater_equal(_next_free_local_node_id, _max_node_id);
541 
542  // Use the unpartitioned ids for unpartitioned nodes,
543  // in serial, and temporarily for ghost nodes
545  if (node_procid == this->processor_id() &&
546  !this->is_serial())
547  next_id = &_next_free_local_node_id;
548  n->set_id (*next_id);
549  }
550 
551  {
552  // Advance next_ids up high enough that each is pointing to an
553  // unused id and any subsequent increments will still point us
554  // to unused ids
556  static_cast<dof_id_type>(n->id()+1));
557 
560  ((_max_node_id-1) / (this->n_processors() + 1) + 1) *
561  (this->n_processors() + 1) + this->n_processors();
564  ((_max_node_id + this->n_processors() - 1) / (this->n_processors() + 1) + 1) *
565  (this->n_processors() + 1) + this->processor_id();
566 
567 #ifndef NDEBUG
568  // We need a const mapvector so we don't inadvertently create
569  // NULL entries when testing for non-NULL ones
570  const mapvector<Node*,dof_id_type>& const_nodes = _nodes;
571 #endif
574  }
575 
576  // Don't try to overwrite existing nodes
577  libmesh_assert (!_nodes[n->id()]);
578 
579  _nodes[n->id()] = n;
580 
581  // Try to make the cached node data more accurate
582  if (node_procid == this->processor_id() ||
583  node_procid == DofObject::invalid_processor_id)
584  _n_nodes++;
585 
586 #ifdef LIBMESH_ENABLE_UNIQUE_ID
587  if (!n->valid_unique_id() && libMesh::processor_id() == n->processor_id())
588  {
590  _next_unique_id += this->n_processors();
591  }
592 #endif
593 
594 
595 // Unpartitioned nodes should be added on every processor
596 // And shouldn't be added in the same batch as ghost nodes
597 // But we might be just adding on processor 0 to
598 // broadcast later
599 #if 0
600 #ifdef DEBUG
601  if (node_procid == DofObject::invalid_processor_id)
602  {
603  dof_id_type node_id = n->id();
604  this->comm().max(node_id);
605  libmesh_assert_equal_to (node_id, n->id());
606  }
607 #endif
608 #endif
609 
610  return n;
611 }
612 
613 
614 
616 {
617  return ParallelMesh::add_node(n);
618 }
619 
620 
621 
623 {
624  libmesh_assert(n);
625  libmesh_assert(_nodes[n->id()]);
626 
627  // Delete the node from the BoundaryInfo object
628  this->boundary_info->remove(n);
629 
630  // But not yet from the container; we might invalidate
631  // an iterator that way!
632 
633  //_nodes.erase(n->id());
634 
635  // Instead, we set it to NULL for now
636 
637  _nodes[n->id()] = NULL;
638 
639  // delete the node
640  delete n;
641 }
642 
643 
644 
646  const dof_id_type new_id)
647 {
648  Node *nd = _nodes[old_id];
649  libmesh_assert (nd);
650  libmesh_assert_equal_to (nd->id(), old_id);
651 
652  nd->set_id(new_id);
653  libmesh_assert (!_nodes[new_id]);
654  _nodes[new_id] = nd;
655  _nodes.erase(old_id);
656 }
657 
658 
659 
661 {
662  // Call parent clear function
663  MeshBase::clear();
664 
665 
666  // Clear our elements and nodes
667  {
668  elem_iterator_imp it = _elements.begin();
669  const elem_iterator_imp end = _elements.end();
670 
671  // There is no need to remove the elements from
672  // the BoundaryInfo data structure since we
673  // already cleared it.
674  for (; it != end; ++it)
675  delete *it;
676 
677  _elements.clear();
678  }
679 
680  // clear the nodes data structure
681  {
682  node_iterator_imp it = _nodes.begin();
683  node_iterator_imp end = _nodes.end();
684 
685  // There is no need to remove the nodes from
686  // the BoundaryInfo data structure since we
687  // already cleared it.
688  for (; it != end; ++it)
689  delete *it;
690 
691  _nodes.clear();
692  }
693 
694  // We're no longer distributed if we were before
695  _is_serial = true;
696 
697  // Correct our caches
698  _n_nodes = 0;
699  _n_elem = 0;
700  _max_node_id = 0;
701  _max_elem_id = 0;
706 }
707 
708 
709 
711 {
712  // If this is a truly parallel mesh, go through the redistribution/gather/delete remote steps
713  if (!this->is_serial())
714  {
715  // Construct a MeshCommunication object to actually redistribute the nodes
716  // and elements according to the partitioner, and then to re-gather the neighbors.
718  mc.redistribute(*this);
719 
721 
722  // Is this necessary? If we are called from prepare_for_use(), this will be called
723  // anyway... but users can always call partition directly, in which case we do need
724  // to call delete_remote_elements()...
725  //
726  // Regardless of whether it's necessary, it isn't safe. We
727  // haven't communicated new node processor_ids yet, and we can't
728  // delete nodes until we do.
729  // this->delete_remote_elements();
730  }
731 }
732 
733 
734 
736 {
737  // this->recalculate_n_partitions();
738 
739  // Partitioning changes our numbers of unpartitioned objects
741 }
742 
743 
744 
745 template <typename T>
747  (const mapvector<T*,dof_id_type> &objects) const
748 {
749  // This function must be run on all processors at once
750  parallel_object_only();
751 
752  const dof_id_type pmax_node_id = this->parallel_max_node_id();
753  const dof_id_type pmax_elem_id = this->parallel_max_elem_id();
754  const dof_id_type pmax_id = std::max(pmax_node_id, pmax_elem_id);
755 
756  for (dof_id_type i=0; i != pmax_id; ++i)
757  {
758  T* obj = objects[i]; // Returns NULL if there's no map entry
759 
760  dof_id_type dofid = obj && obj->valid_id() ?
761  obj->id() : DofObject::invalid_id;
762  // Local lookups by id should return the requested object
763  libmesh_assert(!obj || obj->id() == i);
764 
765  dof_id_type min_dofid = dofid;
766  this->comm().min(min_dofid);
767  // All processors with an object should agree on id
768  libmesh_assert (!obj || dofid == min_dofid);
769 
770  dof_id_type procid = obj && obj->valid_processor_id() ?
771  obj->processor_id() : DofObject::invalid_processor_id;
772 
773  dof_id_type min_procid = procid;
774  this->comm().min(min_procid);
775 
776  // All processors with an object should agree on processor id
777  libmesh_assert (!obj || procid == min_procid);
778 
779  // Either:
780  // 1.) I own this elem (min_procid == this->processor_id()) *and* I have a valid pointer to it (obj != NULL)
781  // or
782  // 2.) I don't own this elem (min_procid != this->processor_id()). (In this case I may or may not have a valid pointer to it.)
783 
784  // Original assert logic
785  // libmesh_assert (min_procid != this->processor_id() || obj);
786 
787  // More human-understandable logic...
789  ((min_procid == this->processor_id()) && obj)
790  ||
791  (min_procid != this->processor_id())
792  );
793  }
794 }
795 
796 
797 
799 {
802 }
803 
804 
805 
807 {
808 #ifdef LIBMESH_ENABLE_AMR
809  // This function must be run on all processors at once
810  parallel_object_only();
811 
812  dof_id_type pmax_elem_id = this->parallel_max_elem_id();
813 
814  for (dof_id_type i=0; i != pmax_elem_id; ++i)
815  {
816  Elem* el = _elements[i]; // Returns NULL if there's no map entry
817 
818  unsigned int refinement_flag = el ?
819  static_cast<unsigned int> (el->refinement_flag()) : libMesh::invalid_uint;
820 #ifndef NDEBUG
821  unsigned int p_refinement_flag = el ?
822  static_cast<unsigned int> (el->p_refinement_flag()) : libMesh::invalid_uint;
823 #endif
824 
825  unsigned int min_rflag = refinement_flag;
826  this->comm().min(min_rflag);
827  // All processors with this element should agree on flag
828  libmesh_assert (!el || min_rflag == refinement_flag);
829 
830 #ifndef NDEBUG
831  unsigned int min_pflag = p_refinement_flag;
832 #endif
833  // All processors with this element should agree on flag
834  libmesh_assert (!el || min_pflag == p_refinement_flag);
835  }
836 #endif // LIBMESH_ENABLE_AMR
837 }
838 
839 
840 
841 template <typename T>
844 {
845  // This function must be run on all processors at once
846  parallel_object_only();
847 
848  typedef typename mapvector<T*,dof_id_type>::veclike_iterator object_iterator;
849 
850  // In parallel we may not know what objects other processors have.
851  // Start by figuring out how many
852  dof_id_type unpartitioned_objects = 0;
853 
854  std::vector<dof_id_type>
855  ghost_objects_from_proc(this->n_processors(), 0);
856 
857  object_iterator it = objects.begin();
858  object_iterator end = objects.end();
859 
860  for (; it != end;)
861  {
862  T *obj = *it;
863 
864  // Remove any NULL container entries while we're here,
865  // being careful not to invalidate our iterator
866  if (!*it)
867  objects.erase(it++);
868  else
869  {
870  processor_id_type obj_procid = obj->processor_id();
871  if (obj_procid == DofObject::invalid_processor_id)
872  unpartitioned_objects++;
873  else
874  ghost_objects_from_proc[obj_procid]++;
875  ++it;
876  }
877  }
878 
879  std::vector<dof_id_type> objects_on_proc(this->n_processors(), 0);
880  this->comm().allgather(ghost_objects_from_proc[this->processor_id()],
881  objects_on_proc);
882 
883 #ifndef NDEBUG
884  libmesh_assert(this->comm().verify(unpartitioned_objects));
885  for (processor_id_type p=0; p != this->n_processors(); ++p)
886  libmesh_assert_less_equal (ghost_objects_from_proc[p], objects_on_proc[p]);
887 #endif
888 
889  // We'll renumber objects in blocks by processor id
890  std::vector<dof_id_type> first_object_on_proc(this->n_processors());
891  for (processor_id_type i=1; i != this->n_processors(); ++i)
892  first_object_on_proc[i] = first_object_on_proc[i-1] +
893  objects_on_proc[i-1];
894  dof_id_type next_id = first_object_on_proc[this->processor_id()];
895  dof_id_type first_free_id =
896  first_object_on_proc[this->n_processors()-1] +
897  objects_on_proc[this->n_processors()-1] +
898  unpartitioned_objects;
899 
900  // First set new local object ids and build request sets
901  // for non-local object ids
902 
903  // Request sets to send to each processor
904  std::vector<std::vector<dof_id_type> >
905  requested_ids(this->n_processors());
906 
907 #ifdef LIBMESH_ENABLE_UNIQUE_ID
908  std::vector<std::vector<unique_id_type> >
909  requested_unique_ids(this->n_processors());
910 #endif
911 
912  // We know how many objects live on each processor, so reseve() space for
913  // each.
914  for (processor_id_type p=0; p != this->n_processors(); ++p)
915  if (p != this->processor_id())
916  {
917  requested_ids[p].reserve(ghost_objects_from_proc[p]);
918 #ifdef LIBMESH_ENABLE_UNIQUE_ID
919  requested_unique_ids[p].reserve(ghost_objects_from_proc[p]);
920 #endif
921  }
922 
923  end = objects.end();
924  for (it = objects.begin(); it != end; ++it)
925  {
926  T *obj = *it;
927  if (obj->processor_id() == this->processor_id())
928  obj->set_id(next_id++);
929  else if (obj->processor_id() != DofObject::invalid_processor_id)
930  {
931  requested_ids[obj->processor_id()].push_back(obj->id());
932 #ifdef LIBMESH_ENABLE_UNIQUE_ID
933  // It's possible to have an invalid id for dofs not owned by this process.
934  // We'll assert that they match on the receiving end.
935  requested_unique_ids[obj->processor_id()].push_back(obj->valid_unique_id() ? obj-> unique_id() : DofObject::invalid_unique_id);
936 #endif
937  }
938  }
939 
940  // Next set ghost object ids from other processors
941  if (this->n_processors() > 1)
942  {
943  for (processor_id_type p=1; p != this->n_processors(); ++p)
944  {
945  // Trade my requests with processor procup and procdown
946  processor_id_type procup = (this->processor_id() + p) %
947  this->n_processors();
948  processor_id_type procdown = (this->n_processors() +
949  this->processor_id() - p) %
950  this->n_processors();
951  std::vector<dof_id_type> request_to_fill;
952  this->comm().send_receive(procup, requested_ids[procup],
953  procdown, request_to_fill);
954 
955 #ifdef LIBMESH_ENABLE_UNIQUE_ID
956  std::vector<unique_id_type> unique_request_to_fill;
957  this->comm().send_receive(procup, requested_unique_ids[procup],
958  procdown, unique_request_to_fill);
959  std::vector<unique_id_type> new_unique_ids(unique_request_to_fill.size());
960 #endif
961 
962  // Fill those requests
963  std::vector<dof_id_type> new_ids(request_to_fill.size());
964  for (std::size_t i=0; i != request_to_fill.size(); ++i)
965  {
966  T *obj = objects[request_to_fill[i]];
967  libmesh_assert(obj);
968  libmesh_assert_equal_to (obj->processor_id(), this->processor_id());
969  new_ids[i] = obj->id();
970 #ifdef LIBMESH_ENABLE_UNIQUE_ID
971  new_unique_ids[i] = obj->valid_unique_id() ? obj->unique_id() : DofObject::invalid_unique_id;
972 #endif
973 
974  libmesh_assert_greater_equal (new_ids[i],
975  first_object_on_proc[this->processor_id()]);
976  libmesh_assert_less (new_ids[i],
977  first_object_on_proc[this->processor_id()] +
978  objects_on_proc[this->processor_id()]);
979  }
980 
981  // Trade back the results
982  std::vector<dof_id_type> filled_request;
983  this->comm().send_receive(procdown, new_ids,
984  procup, filled_request);
985 
986 #ifdef LIBMESH_ENABLE_UNIQUE_ID
987  std::vector<unique_id_type> unique_filled_request;
988  this->comm().send_receive(procdown, new_unique_ids,
989  procup, unique_filled_request);
990 #endif
991 
992  // And copy the id changes we've now been informed of
993  for (std::size_t i=0; i != filled_request.size(); ++i)
994  {
995  T *obj = objects[requested_ids[procup][i]];
996  libmesh_assert (obj);
997  libmesh_assert_equal_to (obj->processor_id(), procup);
998  libmesh_assert_greater_equal (filled_request[i],
999  first_object_on_proc[procup]);
1000  libmesh_assert_less (filled_request[i],
1001  first_object_on_proc[procup] +
1002  objects_on_proc[procup]);
1003  obj->set_id(filled_request[i]);
1004 
1005 #ifdef LIBMESH_ENABLE_UNIQUE_ID
1006  if (!obj->valid_unique_id() && unique_filled_request[i] != DofObject::invalid_unique_id)
1007  obj->set_unique_id() = unique_filled_request[i];
1008 #endif
1009  }
1010  }
1011  }
1012 
1013  // Next set unpartitioned object ids
1014  next_id = 0;
1015  for (processor_id_type i=0; i != this->n_processors(); ++i)
1016  next_id += objects_on_proc[i];
1017  for (it = objects.begin(); it != end; ++it)
1018  {
1019  T *obj = *it;
1020  if (obj->processor_id() == DofObject::invalid_processor_id)
1021  obj->set_id(next_id++);
1022  }
1023 
1024  // Finally shuffle around objects so that container indices
1025  // match ids
1026  end = objects.end();
1027  for (it = objects.begin(); it != end;)
1028  {
1029  T *obj = *it;
1030  if (obj) // don't try shuffling already-NULL entries
1031  {
1032  T *next = objects[obj->id()];
1033  // If we have to move this object
1034  if (next != obj)
1035  {
1036  // NULL out its original position for now
1037  // (our shuffling may put another object there shortly)
1038  *it = NULL;
1039 
1040  // There may already be another object with this id that
1041  // needs to be moved itself
1042  while (next)
1043  {
1044  // We shouldn't be trying to give two objects the
1045  // same id
1046  libmesh_assert_not_equal_to (next->id(), obj->id());
1047  objects[obj->id()] = obj;
1048  obj = next;
1049  next = objects[obj->id()];
1050  }
1051  objects[obj->id()] = obj;
1052  }
1053  }
1054  // Remove any container entries that were left as NULL,
1055  // being careful not to invalidate our iterator
1056  if (!*it)
1057  objects.erase(it++);
1058  else
1059  ++it;
1060  }
1061 
1062  return first_free_id;
1063 }
1064 
1065 
1067 {
1068  parallel_object_only();
1069 
1071  {
1072  this->update_parallel_id_counts();
1073  return;
1074  }
1075 
1076  START_LOG("renumber_nodes_and_elements()", "ParallelMesh");
1077 
1078 #ifdef DEBUG
1079 // Make sure our ids and flags are consistent
1082 #endif
1083 
1084  std::set<dof_id_type> used_nodes;
1085 
1086  // flag the nodes we need
1087  {
1090 
1091  for (; it != end; ++it)
1092  {
1093  Elem *el = *it;
1094 
1095  for (unsigned int n=0; n != el->n_nodes(); ++n)
1096  used_nodes.insert(el->node(n));
1097  }
1098  }
1099 
1100  // Nodes not connected to any local elements, and NULL node entries
1101  // in our container, are deleted
1102  {
1103  node_iterator_imp it = _nodes.begin();
1104  node_iterator_imp end = _nodes.end();
1105 
1106  for (; it != end;)
1107  {
1108  Node *nd = *it;
1109  if (!nd)
1110  _nodes.erase(it++);
1111  else if (!used_nodes.count(nd->id()))
1112  {
1113  // remove any boundary information associated with
1114  // this node
1115  this->boundary_info->remove (nd);
1116 
1117  // delete the node
1118  delete nd;
1119 
1120  _nodes.erase(it++);
1121  }
1122  else
1123  ++it;
1124  }
1125  }
1126 
1127  // Finally renumber all the elements
1128  _n_elem = this->renumber_dof_objects (this->_elements);
1129 
1130  // and all the remaining nodes
1131  _n_nodes = this->renumber_dof_objects (this->_nodes);
1132 
1133  // And figure out what IDs we should use when adding new nodes and
1134  // new elements
1135  this->update_parallel_id_counts();
1136 
1137 // Make sure our caches are up to date and our
1138 // DofObjects are well packed
1139 #ifdef DEBUG
1140  libmesh_assert_equal_to (this->n_nodes(), this->parallel_n_nodes());
1141  libmesh_assert_equal_to (this->n_elem(), this->parallel_n_elem());
1142  const dof_id_type pmax_node_id = this->parallel_max_node_id();
1143  const dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1144  libmesh_assert_equal_to (this->max_node_id(), pmax_node_id);
1145  libmesh_assert_equal_to (this->max_elem_id(), pmax_elem_id);
1146  libmesh_assert_equal_to (this->n_nodes(), this->max_node_id());
1147  libmesh_assert_equal_to (this->n_elem(), this->max_elem_id());
1148 
1149  // Make sure our ids and flags are consistent
1152 
1153 // And make sure we've made our numbering monotonic
1155 #endif
1156 
1157  STOP_LOG("renumber_nodes_and_elements()", "ParallelMesh");
1158 }
1159 
1160 
1161 
1163 {
1164  // We need access to iterators for the underlying containers,
1165  // not the mapvector<> reimplementations.
1168 
1169  // Nodes first
1170  {
1172  it = nodes.begin(),
1173  end = nodes.end();
1174 
1175  for (; it != end; ++it)
1176  if (it->second != NULL)
1177  it->second->set_id() = it->first;
1178  }
1179 
1180  // Elements next
1181  {
1183  it = elems.begin(),
1184  end = elems.end();
1185 
1186  for (; it != end; ++it)
1187  if (it->second != NULL)
1188  it->second->set_id() = it->first;
1189  }
1190 }
1191 
1192 
1193 
1195 {
1196  parallel_object_only();
1197 
1198  // Get local active elements first
1199  dof_id_type active_elements =
1200  static_cast<dof_id_type>(std::distance (this->active_local_elements_begin(),
1201  this->active_local_elements_end()));
1202  this->comm().sum(active_elements);
1203 
1204  // Then add unpartitioned active elements, which should exist on
1205  // every processor
1206  active_elements +=
1207  static_cast<dof_id_type>(std::distance
1210  return active_elements;
1211 }
1212 
1213 
1214 
1216 {
1217 #ifdef DEBUG
1218 // Make sure our neighbor links are all fine
1220 
1221 // And our child/parent links, and our flags
1223 
1224 // Make sure our ids and flags are consistent
1227 
1228  libmesh_assert_equal_to (this->n_nodes(), this->parallel_n_nodes());
1229  libmesh_assert_equal_to (this->n_elem(), this->parallel_n_elem());
1230  const dof_id_type pmax_node_id = this->parallel_max_node_id();
1231  const dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1232  libmesh_assert_equal_to (this->max_node_id(), pmax_node_id);
1233  libmesh_assert_equal_to (this->max_elem_id(), pmax_elem_id);
1234 #endif
1235 
1236  _is_serial = false;
1238 
1239  libmesh_assert_equal_to (this->max_elem_id(), this->parallel_max_elem_id());
1240 
1241  // Now make sure the containers actually shrink - strip
1242  // any newly-created NULL voids out of the element array
1245  for (; e_it != e_end;)
1246  if (!*e_it)
1247  _elements.erase(e_it++);
1248  else
1249  ++e_it;
1250 
1253  for (; n_it != n_end;)
1254  if (!*n_it)
1255  _nodes.erase(n_it++);
1256  else
1257  ++n_it;
1258 
1259  // We may have deleted no-longer-connected nodes or coarsened-away
1260  // elements; let's update our caches.
1261  this->update_parallel_id_counts();
1262 
1263 #ifdef DEBUG
1264  // We might not have well-packed objects if the user didn't allow us
1265  // to renumber
1266  // libmesh_assert_equal_to (this->n_nodes(), this->max_node_id());
1267  // libmesh_assert_equal_to (this->n_elem(), this->max_elem_id());
1268 
1269 // Make sure our neighbor links are all fine
1271 
1272 // And our child/parent links, and our flags
1274 
1275 // Make sure our ids and flags are consistent
1278 #endif
1279 }
1280 
1281 
1283 {
1284  // First add the elem like normal
1285  add_elem(e);
1286 
1287  // Now add it to the set that won't be deleted when we call
1288  // delete_remote_elements()
1289  _extra_ghost_elems.insert(e);
1290 }
1291 
1292 
1294 {
1295  if (_is_serial)
1296  return;
1297  _is_serial = true;
1298  MeshCommunication().allgather(*this);
1299 
1300 // Make sure our caches are up to date and our
1301 // DofObjects are well packed
1302 #ifdef DEBUG
1303  libmesh_assert_equal_to (this->n_nodes(), this->parallel_n_nodes());
1304  libmesh_assert_equal_to (this->n_elem(), this->parallel_n_elem());
1305  const dof_id_type pmax_node_id = this->parallel_max_node_id();
1306  const dof_id_type pmax_elem_id = this->parallel_max_elem_id();
1307  libmesh_assert_equal_to (this->max_node_id(), pmax_node_id);
1308  libmesh_assert_equal_to (this->max_elem_id(), pmax_elem_id);
1309 
1310  // If we've disabled renumbering we can't be sure we're contiguous
1311  // libmesh_assert_equal_to (this->n_nodes(), this->max_node_id());
1312  // libmesh_assert_equal_to (this->n_elem(), this->max_elem_id());
1313 
1314 // Make sure our neighbor links are all fine
1316 
1317 // Make sure our ids and flags are consistent
1320 #endif
1321 }
1322 
1323 #ifdef LIBMESH_ENABLE_UNIQUE_ID
1325 {
1326  {
1327  elem_iterator_imp it = _elements.begin();
1328  const elem_iterator_imp end = _elements.end();
1329 
1330  for (; it != end; ++it)
1331  if ((*it) && ! (*it)->valid_unique_id() && libMesh::processor_id() == (*it)->processor_id())
1332  {
1333  (*it)->set_unique_id() = _next_unique_id;
1334  _next_unique_id += this->n_processors();
1335  }
1336  }
1337 
1338  {
1339  node_iterator_imp it = _nodes.begin();
1340  node_iterator_imp end = _nodes.end();
1341 
1342  for (; it != end; ++it)
1343  if ((*it) && ! (*it)->valid_unique_id() && libMesh::processor_id() == (*it)->processor_id())
1344  {
1345  (*it)->set_unique_id() = _next_unique_id;
1346  _next_unique_id += this->n_processors();
1347  }
1348  }
1349 }
1350 #endif
1351 
1352 
1353 } // namespace libMesh

Site Created By: libMesh Developers
Last modified: February 07 2014 16:57:06 UTC

Hosted By:
SourceForge.net Logo