SendReceiveBuffer.h

// Copyright (C) 2015 Technische Universitaet Muenchen
// This file is part of the Mamico project. For conditions of distribution
// and use, please see the copyright notice in Mamico's main folder, or at
// www5.in.tum.de/mamico
#ifndef _MOLECULARDYNAMICS_COUPLING_SENDRECV_SENDRECEIVEBUFFER_H_
#define _MOLECULARDYNAMICS_COUPLING_SENDRECV_SENDRECEIVEBUFFER_H_
 
#include "DataExchangeFromMD2Macro.h"
#include "coupling/CouplingMDDefinitions.h"
#include "coupling/indexing/IndexingService.h"
#include "coupling/sendrecv/DataExchange.h"
#include "tarch/la/Vector.h"
#include <algorithm>
#include <map>
#include <set>
#include <string.h>
#if (COUPLING_MD_PARALLEL == COUPLING_MD_YES)
#include <mpi.h>
#endif
 
namespace coupling {
namespace sendrecv {
template <class Cell_T, unsigned int dim> class SendReceiveBuffer;
}
} // namespace coupling

template <class Cell_T, unsigned int dim> class coupling::sendrecv::SendReceiveBuffer {
public:
  SendReceiveBuffer();
  virtual ~SendReceiveBuffer();
 
protected:
  void deleteBuffers();

  template <class Container_T> void writeToSendBuffer(coupling::sendrecv::DataExchange<Cell_T, dim>& dataExchange, const Container_T& cells);

  void writeToBcastBuffer(coupling::sendrecv::DataExchange<Cell_T, dim>& dataExchange, const Cell_T& cell, I01 idx);

  void writeToReduceBuffer(coupling::sendrecv::DataExchange<Cell_T, dim>& dataExchange, const Cell_T& cell, I01 idx);

  template <class Container_T> void readFromReceiveBuffer(coupling::sendrecv::DataExchange<Cell_T, dim>& dataExchange, const Container_T& cells);
 
  void readFromCollectiveBuffer(coupling::sendrecv::DataExchange<Cell_T, dim>& dataExchange, Cell_T& couplingCell, I01 idx);

  void readFromReduceBuffer(coupling::sendrecv::DataExchangeFromMD2Macro<dim>& dataExchange, Cell_T& couplingCell, I01 idx);

  void allocateReceiveBuffers(coupling::sendrecv::DataExchange<Cell_T, dim>& dataExchange, I01 idx);

  void allocateBcastBufferForReceiving(coupling::sendrecv::DataExchange<Cell_T, dim>& dataExchange, I01 idx);

  void allocateReduceBufferForReceiving(coupling::sendrecv::DataExchange<Cell_T, dim>& dataExchange, I01 idx);

  void triggerSending(coupling::sendrecv::DataExchange<Cell_T, dim>& dataExchange);

  void triggerBcasts(unsigned int rank);

  void triggerReceiving(coupling::sendrecv::DataExchange<Cell_T, dim>& dataExchange);

  void triggerReduce(unsigned int rank);

  void waitAllOperations();

  void waitAllCollectiveOperations();

  void allocateRequests();

  void allocateBcastRequests(unsigned int thisRank);

  void allocateReduceRequests(unsigned int thisRank);
 
private:
  struct BufferWithID {
    double* buffer;
    unsigned int bufferSize;
 
    BufferWithID() : buffer(NULL), bufferSize(0) {}
  };
 
  struct BufferCollective {
    std::vector<double> buffer;
    std::set<unsigned int> nonRootRanks;
    std::set<unsigned int> cellIndices;
    unsigned int rootRank;
 
    BufferCollective() : buffer(), nonRootRanks(), cellIndices(), rootRank(-1) {}
  };

  void deleteBuffer(std::map<unsigned int, BufferWithID>& buffer);

  std::map<unsigned int, BufferWithID> _receiveBuffer;
  std::map<unsigned int, BufferWithID> _sendBuffer;

  std::map<unsigned int, BufferCollective> _bcastBuffer;
  std::map<unsigned int, BufferCollective> _reduceBuffer;
 
#if (COUPLING_MD_PARALLEL == COUPLING_MD_YES)
  bool _requestsAllocated; 
  MPI_Request* _requests;
  int _receiveSize; 
  int _sendSize;    
 
  std::vector<MPI_Comm> _subComms;
  std::vector<MPI_Group> _subGroups;
  int _bcastOrReduceSize;
 
  static void elementWiseSum(void* in, void* inout, int* len, MPI_Datatype* datatype) {
    auto* output = (double*)inout;
    auto* input = (double*)in;
    for (int i = 0; i < *len; ++i) {
      output[i] += input[i];
    }
  }
 
  MPI_Op elementWiseSumOperation;
#endif
};
 
#include "SendReceiveBuffer.cpph"
 
#endif // _MOLECULARDYNAMICS_COUPLING_SENDRECV_SENDRECEIVEBUFFER_H_