Line data    Source code

       1             : // Copyright (C) 2015 Technische Universitaet Muenchen
       2             : // This file is part of the Mamico project. For conditions of distribution
       3             : // and use, please see the copyright notice in Mamico's main folder, or at
       4             : // www5.in.tum.de/mamico
       5             : #ifndef _MOLECULARDYNAMICS_COUPLING_SENDRECV_SENDRECEIVEBUFFER_H_
       6             : #define _MOLECULARDYNAMICS_COUPLING_SENDRECV_SENDRECEIVEBUFFER_H_
       7             : 
       8             : #include "DataExchangeFromMD2Macro.h"
       9             : #include "coupling/CouplingMDDefinitions.h"
      10             : #include "coupling/indexing/IndexingService.h"
      11             : #include "coupling/sendrecv/DataExchange.h"
      12             : #include "tarch/la/Vector.h"
      13             : #include <algorithm>
      14             : #include <map>
      15             : #include <set>
      16             : #include <string.h>
      17             : #if (COUPLING_MD_PARALLEL == COUPLING_MD_YES)
      18             : #include <mpi.h>
      19             : #endif
      20             : 
      21             : namespace coupling {
      22             : namespace sendrecv {
      23             : template <class Cell_T, unsigned int dim> class SendReceiveBuffer;
      24             : }
      25             : } // namespace coupling
      26             : 
      27             : /** generic class for send-/ receive methodology including buffer
      28             :  *implementations. The access to the buffers is prescribed by the DataExchange
      29             :  *object.
      30             :  *      @brief generic class for send-/ receive methodology.
      31             :  *      @tparam Cell_T cell type
      32             :  *      @tparam dim Number of dimensions; it can be 1, 2 or 3
      33             :  *  @author Philipp Neumann
      34             :  */
      35             : template <class Cell_T, unsigned int dim> class coupling::sendrecv::SendReceiveBuffer {
      36             : public:
      37             :   /** Constructor */
      38             :   SendReceiveBuffer();
      39             :   /** Destructor */
      40             :   virtual ~SendReceiveBuffer();
      41             : 
      42             : protected:
      43             :   /** @brief deletes the buffers */
      44             :   void deleteBuffers();
      45             : 
      46             :   /** @brief fills all information that needs to be sent from a coupling cell
      47             :    * into the send-buffer.
      48             :    *    @param dataExchange
      49             :    *    @param cells
      50             :    */
      51             :   template <class Container_T> void writeToSendBuffer(coupling::sendrecv::DataExchange<Cell_T, dim>& dataExchange, const Container_T& cells);
      52             : 
      53             :   /** @brief fills all information that needs to be broadcast from a coupling cell
      54             :    * into the broadcast-buffer.
      55             :    *    @param dataExchange
      56             :    *    @param cell
      57             :    *    @param idx
      58             :    */
      59             :   void writeToBcastBuffer(coupling::sendrecv::DataExchange<Cell_T, dim>& dataExchange, const Cell_T& cell, I01 idx);
      60             : 
      61             :   /** @brief fills all information that needs to be reduced to a coupling cell
      62             :    * into the reduce-buffer.
      63             :    *    @param dataExchange
      64             :    *    @param cell
      65             :    *    @param idx
      66             :    */
      67             :   void writeToReduceBuffer(coupling::sendrecv::DataExchange<Cell_T, dim>& dataExchange, const Cell_T& cell, I01 idx);
      68             : 
      69             :   /** reads the information from the receive-buffer and fills it into a
      70             :    * coupling cell.
      71             :    *    @param dataExchange
      72             :    *    @param couplingCell
      73             :    *    @param idx
      74             :    */
      75             :   template <class Container_T> void readFromReceiveBuffer(coupling::sendrecv::DataExchange<Cell_T, dim>& dataExchange, const Container_T& cells);
      76             : 
      77             :   void readFromCollectiveBuffer(coupling::sendrecv::DataExchange<Cell_T, dim>& dataExchange, Cell_T& couplingCell, I01 idx);
      78             : 
      79             :   /** reads the information from the reduce-buffer and fills it into a
      80             :    * coupling cell.
      81             :    *    @param dataExchange
      82             :    *    @param couplingCell
      83             :    *    @param idx
      84             :    */
      85             :   void readFromReduceBuffer(coupling::sendrecv::DataExchangeFromMD2Macro<dim>& dataExchange, Cell_T& couplingCell, I01 idx);
      86             : 
      87             :   /** according to rule by dataExchange, the receive buffers are allocated. This
      88             :    * function adds a contribution for the cell at idx.
      89             :    *    @param dataExchange
      90             :    *    @param idx
      91             :    */
      92             :   void allocateReceiveBuffers(coupling::sendrecv::DataExchange<Cell_T, dim>& dataExchange, I01 idx);
      93             : 
      94             :   /** Allocates buffer for receiving in the context of the broadcast operation
      95             :    *    @param dataExchange
      96             :    *    @param idx
      97             :    */
      98             :   void allocateBcastBufferForReceiving(coupling::sendrecv::DataExchange<Cell_T, dim>& dataExchange, I01 idx);
      99             : 
     100             :   /** Allocates buffer for receiving in the context of the reduce operation
     101             :    *    @param dataExchange
     102             :    *    @param idx
     103             :    */
     104             :   void allocateReduceBufferForReceiving(coupling::sendrecv::DataExchange<Cell_T, dim>& dataExchange, I01 idx);
     105             : 
     106             :   /** triggers the MPI-sending on the respective buffers. No sending for
     107             :    * information transfer from/ to this rank.
     108             :    *    @param dataExchange
     109             :    */
     110             :   void triggerSending(coupling::sendrecv::DataExchange<Cell_T, dim>& dataExchange);
     111             : 
     112             :   /** triggers the MPI-broadcast on the respective buffers.
     113             :    *    @param rank
     114             :    */
     115             :   void triggerBcasts(unsigned int rank);
     116             : 
     117             :   /** triggers the MPI-receiving on the respective buffers. No receiving of
     118             :    * information from/to this rank.
     119             :    *    @param dataExchange
     120             :    */
     121             :   void triggerReceiving(coupling::sendrecv::DataExchange<Cell_T, dim>& dataExchange);
     122             : 
     123             :   /** triggers the MPI-reduce on the respective buffers.
     124             :    *    @param rank
     125             :    */
     126             :   void triggerReduce(unsigned int rank);
     127             : 
     128             :   /** wait for all send and receive operations to complete.
     129             :    */
     130             :   void waitAllOperations();
     131             : 
     132             :   /** wait for all broadcast or reduce operations to complete */
     133             :   void waitAllCollectiveOperations();
     134             : 
     135             :   /** allocates send and receive requests
     136             :    */
     137             :   void allocateRequests();
     138             : 
     139             :   /** allocates broadcast request
     140             :    *    @param thisRank
     141             :    */
     142             :   void allocateBcastRequests(unsigned int thisRank);
     143             : 
     144             :   /** allocates reduce request
     145             :    *    @param thisRank
     146             :    */
     147             :   void allocateReduceRequests(unsigned int thisRank);
     148             : 
     149             : private:
     150             :   /** data structure for send- and receive-buffer. */
     151             :   struct BufferWithID {
     152             :     double* buffer;
     153             :     unsigned int bufferSize;
     154             : 
     155           0 :     BufferWithID() : buffer(NULL), bufferSize(0) {}
     156             :   };
     157             : 
     158             :   struct BufferCollective {
     159             :     std::vector<double> buffer;
     160             :     std::set<unsigned int> nonRootRanks;
     161             :     std::set<unsigned int> cellIndices;
     162             :     unsigned int rootRank;
     163             : 
     164             :     BufferCollective() : buffer(), nonRootRanks(), cellIndices(), rootRank(-1) {}
     165             :   };
     166             : 
     167             :   /** deletes everything inside a given buffer
     168             :    *    @param buffer
     169             :    */
     170             :   void deleteBuffer(std::map<unsigned int, BufferWithID>& buffer);
     171             : 
     172             :   /** buffer for storing all received messages from MD. Each map entry is
     173             :    * identified by a respective rank. */
     174             :   std::map<unsigned int, BufferWithID> _receiveBuffer;
     175             :   std::map<unsigned int, BufferWithID> _sendBuffer;
     176             : 
     177             :   /** members for collective communication */
     178             :   std::map<unsigned int, BufferCollective> _bcastBuffer;
     179             :   std::map<unsigned int, BufferCollective> _reduceBuffer;
     180             : 
     181             : #if (COUPLING_MD_PARALLEL == COUPLING_MD_YES)
     182             :   bool _requestsAllocated; /** flag that will always be reset after every send
     183             :                               operation. Triggers instantiation of requests */
     184             :   MPI_Request* _requests;
     185             :   int _receiveSize; /** number of receive requests */
     186             :   int _sendSize;    /** number of send requests */
     187             : 
     188             :   std::vector<MPI_Comm> _subComms;
     189             :   std::vector<MPI_Group> _subGroups;
     190             :   int _bcastOrReduceSize;
     191             : 
     192           0 :   static void elementWiseSum(void* in, void* inout, int* len, MPI_Datatype* datatype) {
     193           0 :     auto* output = (double*)inout;
     194           0 :     auto* input = (double*)in;
     195           0 :     for (int i = 0; i < *len; ++i) {
     196           0 :       output[i] += input[i];
     197             :     }
     198           0 :   }
     199             : 
     200             :   MPI_Op elementWiseSumOperation;
     201             : #endif
     202             : };
     203             : 
     204             : #include "SendReceiveBuffer.cpph"
     205             : 
     206             : #endif // _MOLECULARDYNAMICS_COUPLING_SENDRECV_SENDRECEIVEBUFFER_H_