vtkPStructuredGridConnectivity.h

Go to the documentation of this file.

/*=========================================================================
 
 Program:   Visualization Toolkit
 Module:    vtkPStructuredGridConnectivity.h
 
 Copyright (c) Ken Martin, Will Schroeder, Bill Lorensen
 All rights reserved.
 See Copyright.txt or http://www.kitware.com/Copyright.htm for details.
 
 This software is distributed WITHOUT ANY WARRANTY; without even
 the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
 PURPOSE.  See the above copyright notice for more information.
 
 =========================================================================*/
#ifndef vtkPStructuredGridConnectivity_h
#define vtkPStructuredGridConnectivity_h
 
// VTK include directives
#include "vtkFiltersParallelGeometryModule.h" // For export macro
#include "vtkMPICommunicator.h"               // Needed for vtkMPICommunicator::Request
#include "vtkStructuredGridConnectivity.h"
 
// C++ include directives
#include <vector> // For STL vector
 
// Forward declarations
class vtkMultiProcessController;
class vtkMPIController;
class vtkMultiProcessStream;
// class vtkMPICommunicator::Request;
 
class VTKFILTERSPARALLELGEOMETRY_EXPORT vtkPStructuredGridConnectivity
  : public vtkStructuredGridConnectivity
{
public:
  static vtkPStructuredGridConnectivity* New();
  vtkTypeMacro(vtkPStructuredGridConnectivity, vtkStructuredGridConnectivity);
  void PrintSelf(ostream& os, vtkIndent indent) override;
 
 
  vtkSetMacro(Controller, vtkMultiProcessController*);
  vtkGetMacro(Controller, vtkMultiProcessController*);
 
  void SetNumberOfGrids(const unsigned int N) override;
 
  void RegisterGrid(const int gridID, int extents[6], vtkUnsignedCharArray* nodesGhostArray,
    vtkUnsignedCharArray* cellGhostArray, vtkPointData* pointData, vtkCellData* cellData,
    vtkPoints* gridNodes) override;
 
  int GetNumberOfLocalGrids() { return static_cast<int>(this->GridIds.size()); }
 
  int GetGridRank(const int gridID);
 
  bool IsGridRemote(const int gridID);
 
  bool IsGridLocal(const int gridID);
 
  void Initialize();
 
  void ComputeNeighbors() override;
 
  void CreateGhostLayers(const int N = 1) override;
 
protected:
  vtkPStructuredGridConnectivity();
  ~vtkPStructuredGridConnectivity() override;
 
  vtkMultiProcessController* Controller;
  int Rank;
  bool Initialized;
 
  std::vector<int> GridRanks; // Corresponding rank for each grid
  std::vector<int> GridIds;   // List of GridIds, owned by this process
 
  // Data structures to store the remote ghost data of each grid for each one
  // of its neighbors. The first index is the global grid index. The second
  // is the neighbor index.
  std::vector<std::vector<vtkPoints*>> RemotePoints;
  std::vector<std::vector<vtkPointData*>> RemotePointData;
  std::vector<std::vector<vtkCellData*>> RemoteCellData;
 
  // Data structures to store the send/receive buffer sizes and corresponding
  // persistent buffers. The first index is the global grid index. The second
  // index is the neighbor index for the given grid.
  std::vector<std::vector<unsigned int>> SendBufferSizes;
  std::vector<std::vector<unsigned int>> RcvBufferSizes;
  std::vector<std::vector<unsigned char*>> SendBuffers;
  std::vector<std::vector<unsigned char*>> RcvBuffers;
 
  int TotalNumberOfSends;
  int TotalNumberOfRcvs;
  int TotalNumberOfMsgs;
 
  // Array of MPI requests
  vtkMPICommunicator::Request* MPIRequests;
 
  bool GridExtentsAreEqual(int rhs[6], int lhs[6]);
 
  bool HasPointData(const int gridIdx);
 
  bool HasCellData(const int gridIdx);
 
  bool HasPoints(const int gridIdx);
 
  void InitializeMessageCounters();
 
  void ClearRemoteData();
 
  void ClearRawBuffers();
 
  void RegisterRemoteGrid(const int gridID, int extents[6], int process);
 
  void TransferRemoteNeighborData(
    const int gridIdx, const int nei, const vtkStructuredNeighbor& Neighbor);
 
  void TransferGhostDataFromNeighbors(const int gridID) override;
 
  void PackGhostData();
 
  void UnpackGhostData();
 
  void DeserializeBufferSizesForProcess(int* buffersizes, vtkIdType N, const int processId);
 
  void SerializeBufferSizes(int*& sizesbuf, vtkIdType& N);
 
  void ExchangeBufferSizes();
 
  void ExchangeGhostDataInit();
 
  void PostReceives();
 
  void PostSends();
 
  void CommunicateGhostData();
 
  void ExchangeGhostDataPost();
 
  void ExchangeGhostData();
 
  void SerializeGhostPoints(const int gridIdx, int ext[6], vtkMultiProcessStream& bytestream);
 
  void SerializeDataArray(vtkDataArray* dataArray, vtkMultiProcessStream& bytestream);
 
  void SerializeFieldData(int sourceExtent[6], int targetExtent[6], vtkFieldData* fieldData,
    vtkMultiProcessStream& bytestream);
 
  void SerializeGhostPointData(const int gridIdx, int ext[6], vtkMultiProcessStream& bytestream);
 
  void SerializeGhostCellData(const int gridIdx, int ext[6], vtkMultiProcessStream& bytestream);
 
  void DeserializeGhostPoints(
    const int gridIdx, const int nei, int ext[6], vtkMultiProcessStream& bytestream);
 
  void DeserializeDataArray(vtkDataArray*& dataArray, const int dataType, const int numberOfTuples,
    const int numberOfComponents, vtkMultiProcessStream& bytestream);
 
  void DeserializeFieldData(int ext[6], vtkFieldData* fieldData, vtkMultiProcessStream& bytestream);
 
  void DeserializeGhostPointData(
    const int gridIdx, const int nei, int ext[6], vtkMultiProcessStream& bytestream);
 
  void DeserializeGhostCellData(
    const int gridIdx, const int nei, int ext[6], vtkMultiProcessStream& bytestream);
 
  void SerializeGhostData(const int sndGridID, const int rcvGrid, int sndext[6],
    unsigned char*& buffer, unsigned int& size);
 
  void DeserializeGhostData(const int gridID, const int neiListID, const int neiGridIdx,
    int rcvext[6], unsigned char* buffer, unsigned int size);
 
  void ExchangeGridExtents();
 
  void SerializeGridExtents(int*& sndbuffer, vtkIdType& N);
 
  void DeserializeGridExtentForProcess(int* rcvbuffer, vtkIdType& N, const int processId);
 
private:
  vtkPStructuredGridConnectivity(const vtkPStructuredGridConnectivity&) = delete;
  void operator=(const vtkPStructuredGridConnectivity&) = delete;
};
 
//=============================================================================
//  INLINE METHODS
//=============================================================================
 
inline bool vtkPStructuredGridConnectivity::GridExtentsAreEqual(int rhs[6], int lhs[6])
{
  for (int i = 0; i < 6; ++i)
  {
    if (rhs[i] != lhs[i])
    {
      return false;
    }
  }
  return true;
}
 
//------------------------------------------------------------------------------
inline bool vtkPStructuredGridConnectivity::HasPointData(const int gridIdx)
{
  // Sanity check
  assert("pre: grid index is out-of-bounds!" && (gridIdx >= 0) &&
    (gridIdx < static_cast<int>(this->NumberOfGrids)));
 
  if ((this->GridPointData[gridIdx] != nullptr) &&
    (this->GridPointData[gridIdx]->GetNumberOfArrays() > 0))
  {
    return true;
  }
  return false;
}
 
//------------------------------------------------------------------------------
inline bool vtkPStructuredGridConnectivity::HasCellData(const int gridIdx)
{
  // Sanity check
  assert("pre: grid index is out-of-bounds!" && (gridIdx >= 0) &&
    (gridIdx < static_cast<int>(this->NumberOfGrids)));
 
  if ((this->GridCellData[gridIdx] != nullptr) &&
    (this->GridCellData[gridIdx]->GetNumberOfArrays() > 0))
  {
    return true;
  }
  return false;
}
 
//------------------------------------------------------------------------------
inline bool vtkPStructuredGridConnectivity::HasPoints(const int gridIdx)
{
  // Sanity check
  assert("pre: grid index is out-of-bounds!" && (gridIdx >= 0) &&
    (gridIdx < static_cast<int>(this->NumberOfGrids)));
 
  if (this->GridPoints[gridIdx] != nullptr)
  {
    return true;
  }
  return false;
}
 
//------------------------------------------------------------------------------
inline void vtkPStructuredGridConnectivity::InitializeMessageCounters()
{
  this->TotalNumberOfMsgs = this->TotalNumberOfRcvs = this->TotalNumberOfSends = 0;
}
 
//------------------------------------------------------------------------------
inline void vtkPStructuredGridConnectivity::ClearRawBuffers()
{
  this->SendBufferSizes.clear();
  this->RcvBufferSizes.clear();
 
  // STEP 0: Clear send buffers
  for (unsigned int i = 0; i < this->SendBuffers.size(); ++i)
  {
    for (unsigned int j = 0; j < this->SendBuffers[i].size(); ++j)
    {
      delete[] this->SendBuffers[i][j];
    } // END for all neighbors
    this->SendBuffers[i].clear();
  } // END for all grids
  this->SendBuffers.clear();
 
  // STEP 1: Clear rcv buffers
  for (unsigned int i = 0; i < this->RcvBuffers.size(); ++i)
  {
    for (unsigned int j = 0; j < this->RcvBuffers[i].size(); ++j)
    {
      delete[] this->RcvBuffers[i][j];
    } // END for all neighbors
    this->RcvBuffers[i].clear();
  } // END for all grids
  this->RcvBuffers.clear();
}
 
//------------------------------------------------------------------------------
inline void vtkPStructuredGridConnectivity::ClearRemoteData()
{
  // STEP 0: Clear remote points
  for (unsigned int i = 0; i < this->RemotePoints.size(); ++i)
  {
    for (unsigned int j = 0; j < this->RemotePoints[i].size(); ++j)
    {
      if (this->RemotePoints[i][j] != nullptr)
      {
        this->RemotePoints[i][j]->Delete();
      }
    } // END for all j
    this->RemotePoints[i].clear();
  } // END for all i
  this->RemotePoints.clear();
 
  // STEP 1: Clear remote point data
  for (unsigned int i = 0; i < this->RemotePointData.size(); ++i)
  {
    for (unsigned int j = 0; j < this->RemotePointData[i].size(); ++j)
    {
      if (this->RemotePointData[i][j] != nullptr)
      {
        this->RemotePointData[i][j]->Delete();
      }
    } // END for all j
    this->RemotePointData[i].clear();
  } // END for all i
  this->RemotePointData.clear();
 
  // STEP 2: Clear remote cell data
  for (unsigned int i = 0; i < this->RemoteCellData.size(); ++i)
  {
    for (unsigned int j = 0; j < this->RemoteCellData[i].size(); ++j)
    {
      if (this->RemoteCellData[i][j] != nullptr)
      {
        this->RemoteCellData[i][j]->Delete();
      }
    } // END for all j
    this->RemoteCellData[i].clear();
  }
  this->RemoteCellData.clear();
}
 
//------------------------------------------------------------------------------
inline bool vtkPStructuredGridConnectivity::IsGridRemote(const int gridID)
{
  return (!this->IsGridLocal(gridID));
}
 
//------------------------------------------------------------------------------
inline bool vtkPStructuredGridConnectivity::IsGridLocal(const int gridID)
{
  assert("pre: Instance has not been initialized!" && this->Initialized);
  assert("pre: gridID is out-of-bounds" && (gridID >= 0) &&
    (gridID < static_cast<int>(this->NumberOfGrids)));
  assert(
    "pre: GridRanks is not properly allocated" && this->NumberOfGrids == this->GridRanks.size());
  return ((this->GridRanks[gridID] == this->Rank));
}
 
//------------------------------------------------------------------------------
inline int vtkPStructuredGridConnectivity::GetGridRank(const int gridID)
{
  assert("pre: Instance has not been initialized!" && this->Initialized);
  assert("pre: gridID out-of-bounds!" &&
    (gridID >= 0 && gridID < static_cast<int>(this->NumberOfGrids)));
  return (this->GridRanks[gridID]);
}
#endif /* vtkPStructuredGridConnectivity_h */