Line data Source code
1 : // Copyright (C) 2015 Technische Universitaet Muenchen
2 : // This file is part of the Mamico project. For conditions of distribution
3 : // and use, please see the copyright notice in Mamico's main folder, or at
4 : // www5.in.tum.de/mamico
5 : #ifndef _MOLECULARDYNAMICS_COUPLING_SENDRECV_SENDRECEIVEBUFFER_H_
6 : #define _MOLECULARDYNAMICS_COUPLING_SENDRECV_SENDRECEIVEBUFFER_H_
7 :
8 : #include "DataExchangeFromMD2Macro.h"
9 : #include "coupling/CouplingMDDefinitions.h"
10 : #include "coupling/indexing/IndexingService.h"
11 : #include "coupling/sendrecv/DataExchange.h"
12 : #include "tarch/la/Vector.h"
13 : #include <algorithm>
14 : #include <map>
15 : #include <set>
16 : #include <string.h>
17 : #if (COUPLING_MD_PARALLEL == COUPLING_MD_YES)
18 : #include <mpi.h>
19 : #endif
20 :
21 : namespace coupling {
22 : namespace sendrecv {
23 : template <class Cell_T, unsigned int dim> class SendReceiveBuffer;
24 : }
25 : } // namespace coupling
26 :
27 : /** generic class for send-/ receive methodology including buffer
28 : *implementations. The access to the buffers is prescribed by the DataExchange
29 : *object.
30 : * @brief generic class for send-/ receive methodology.
31 : * @tparam Cell_T cell type
32 : * @tparam dim Number of dimensions; it can be 1, 2 or 3
33 : * @author Philipp Neumann
34 : */
35 : template <class Cell_T, unsigned int dim> class coupling::sendrecv::SendReceiveBuffer {
36 : public:
37 : /** Constructor */
38 : SendReceiveBuffer();
39 : /** Destructor */
40 : virtual ~SendReceiveBuffer();
41 :
42 : protected:
43 : /** @brief deletes the buffers */
44 : void deleteBuffers();
45 :
46 : /** @brief fills all information that needs to be sent from a coupling cell
47 : * into the send-buffer.
48 : * @param dataExchange
49 : * @param cells
50 : */
51 : template <class Container_T> void writeToSendBuffer(coupling::sendrecv::DataExchange<Cell_T, dim>& dataExchange, const Container_T& cells);
52 :
53 : /** @brief fills all information that needs to be broadcast from a coupling cell
54 : * into the broadcast-buffer.
55 : * @param dataExchange
56 : * @param cell
57 : * @param idx
58 : */
59 : void writeToBcastBuffer(coupling::sendrecv::DataExchange<Cell_T, dim>& dataExchange, const Cell_T& cell, I01 idx);
60 :
61 : /** @brief fills all information that needs to be reduced to a coupling cell
62 : * into the reduce-buffer.
63 : * @param dataExchange
64 : * @param cell
65 : * @param idx
66 : */
67 : void writeToReduceBuffer(coupling::sendrecv::DataExchange<Cell_T, dim>& dataExchange, const Cell_T& cell, I01 idx);
68 :
69 : /** reads the information from the receive-buffer and fills it into a
70 : * coupling cell.
71 : * @param dataExchange
72 : * @param couplingCell
73 : * @param idx
74 : */
75 : template <class Container_T> void readFromReceiveBuffer(coupling::sendrecv::DataExchange<Cell_T, dim>& dataExchange, const Container_T& cells);
76 :
77 : void readFromCollectiveBuffer(coupling::sendrecv::DataExchange<Cell_T, dim>& dataExchange, Cell_T& couplingCell, I01 idx);
78 :
79 : /** reads the information from the reduce-buffer and fills it into a
80 : * coupling cell.
81 : * @param dataExchange
82 : * @param couplingCell
83 : * @param idx
84 : */
85 : void readFromReduceBuffer(coupling::sendrecv::DataExchangeFromMD2Macro<dim>& dataExchange, Cell_T& couplingCell, I01 idx);
86 :
87 : /** according to rule by dataExchange, the receive buffers are allocated. This
88 : * function adds a contribution for the cell at idx.
89 : * @param dataExchange
90 : * @param idx
91 : */
92 : void allocateReceiveBuffers(coupling::sendrecv::DataExchange<Cell_T, dim>& dataExchange, I01 idx);
93 :
94 : /** Allocates buffer for receiving in the context of the broadcast operation
95 : * @param dataExchange
96 : * @param idx
97 : */
98 : void allocateBcastBufferForReceiving(coupling::sendrecv::DataExchange<Cell_T, dim>& dataExchange, I01 idx);
99 :
100 : /** Allocates buffer for receiving in the context of the reduce operation
101 : * @param dataExchange
102 : * @param idx
103 : */
104 : void allocateReduceBufferForReceiving(coupling::sendrecv::DataExchange<Cell_T, dim>& dataExchange, I01 idx);
105 :
106 : /** triggers the MPI-sending on the respective buffers. No sending for
107 : * information transfer from/ to this rank.
108 : * @param dataExchange
109 : */
110 : void triggerSending(coupling::sendrecv::DataExchange<Cell_T, dim>& dataExchange);
111 :
112 : /** triggers the MPI-broadcast on the respective buffers.
113 : * @param rank
114 : */
115 : void triggerBcasts(unsigned int rank);
116 :
117 : /** triggers the MPI-receiving on the respective buffers. No receiving of
118 : * information from/to this rank.
119 : * @param dataExchange
120 : */
121 : void triggerReceiving(coupling::sendrecv::DataExchange<Cell_T, dim>& dataExchange);
122 :
123 : /** triggers the MPI-reduce on the respective buffers.
124 : * @param rank
125 : */
126 : void triggerReduce(unsigned int rank);
127 :
128 : /** wait for all send and receive operations to complete.
129 : */
130 : void waitAllOperations();
131 :
132 : /** wait for all broadcast or reduce operations to complete */
133 : void waitAllCollectiveOperations();
134 :
135 : /** allocates send and receive requests
136 : */
137 : void allocateRequests();
138 :
139 : /** allocates broadcast request
140 : * @param thisRank
141 : */
142 : void allocateBcastRequests(unsigned int thisRank);
143 :
144 : /** allocates reduce request
145 : * @param thisRank
146 : */
147 : void allocateReduceRequests(unsigned int thisRank);
148 :
149 : private:
150 : /** data structure for send- and receive-buffer. */
151 : struct BufferWithID {
152 : double* buffer;
153 : unsigned int bufferSize;
154 :
155 0 : BufferWithID() : buffer(NULL), bufferSize(0) {}
156 : };
157 :
158 : struct BufferCollective {
159 : std::vector<double> buffer;
160 : std::set<unsigned int> nonRootRanks;
161 : std::set<unsigned int> cellIndices;
162 : unsigned int rootRank;
163 :
164 : BufferCollective() : buffer(), nonRootRanks(), cellIndices(), rootRank(UINT_MAX) {}
165 : };
166 :
167 : /** deletes everything inside a given buffer
168 : * @param buffer
169 : */
170 : void deleteBuffer(std::map<unsigned int, BufferWithID>& buffer);
171 :
172 : /** buffer for storing all received messages from MD. Each map entry is
173 : * identified by a respective rank. */
174 : std::map<unsigned int, BufferWithID> _receiveBuffer;
175 : std::map<unsigned int, BufferWithID> _sendBuffer;
176 :
177 : /** members for collective communication */
178 : std::map<unsigned int, BufferCollective> _bcastBuffer;
179 : std::map<unsigned int, BufferCollective> _reduceBuffer;
180 :
181 : #if (COUPLING_MD_PARALLEL == COUPLING_MD_YES)
182 : bool _requestsAllocated; /** flag that will always be reset after every send
183 : operation. Triggers instantiation of requests */
184 : MPI_Request* _requests;
185 : int _receiveSize; /** number of receive requests */
186 : int _sendSize; /** number of send requests */
187 :
188 : std::vector<MPI_Comm> _subComms;
189 : std::vector<MPI_Group> _subGroups;
190 : int _bcastOrReduceSize;
191 :
192 0 : static void elementWiseSum(void* in, void* inout, int* len, MPI_Datatype* datatype) {
193 0 : auto* output = (double*)inout;
194 0 : auto* input = (double*)in;
195 0 : for (int i = 0; i < *len; ++i) {
196 0 : output[i] += input[i];
197 : }
198 0 : }
199 :
200 : MPI_Op elementWiseSumOperation;
201 : #endif
202 : };
203 :
204 : #include "SendReceiveBuffer.cpph"
205 :
206 : #endif // _MOLECULARDYNAMICS_COUPLING_SENDRECV_SENDRECEIVEBUFFER_H_
|
