Line data Source code
1 : // Copyright (C) 2015 Technische Universitaet Muenchen
2 : // This file is part of the Mamico project. For conditions of distribution
3 : // and use, please see the copyright notice in Mamico's main folder, or at
4 : // www5.in.tum.de/mamico
5 : #ifndef _MOLECULARDYNAMICS_SERVICES_PARALLELTOPOLOGYSERVICE_H_
6 : #define _MOLECULARDYNAMICS_SERVICES_PARALLELTOPOLOGYSERVICE_H_
7 :
8 : #include "simplemd/MolecularDynamicsDefinitions.h"
9 : #include "tarch/la/Vector.h"
10 : #if (MD_PARALLEL == MD_YES)
11 : #include <mpi.h>
12 : #endif
13 : #include "simplemd/LinkedCell.h"
14 : #include "simplemd/Molecule.h"
15 : #include "simplemd/services/ParallelAndLocalBufferService.h"
16 : #include "simplemd/MoleculeContainer.h"
17 : #include <cstdlib>
18 : #include <iostream>
19 : #include <vector>
20 :
21 : namespace simplemd {
22 : // forward declarations to circumvent circular includes
23 : class MoleculeContainer;
24 : namespace services {
25 : class ParallelTopologyService;
26 : } // namespace services
27 : } // namespace simplemd
28 :
29 : /** services managing the distributed memory parallelisation things.
30 : * It is based on a simple domain decomposition method for the molecules.
31 : * Therefore, if we have N=Nx x Ny x Nz processes available, we enumerate
32 : * the processes lexicographically in their ranks and so set up an easy-to-use
33 : * topology of our parallel simulation.
34 : *
35 : * @author Philipp Neumann, Nikola Tchipev
36 : */
37 : class simplemd::services::ParallelTopologyService {
38 : public:
39 : /** initialise the service with global domain size and offset,
40 : * predefined meshwidth for the linked cells (which is to be adopted such that
41 : * it fits to the local process properties), the number of processes to be used
42 : * in each spatial direction, the rank of the current process, the global
43 : * description of the outer boundaries, the number of molecules per direction
44 : * and initialises the ParallelAndLocalBufferService.
45 : *
46 : * Needs to be called before the local services are initialised.
47 : */
48 : ParallelTopologyService(const tarch::la::Vector<MD_DIM, double>& domainSize, const tarch::la::Vector<MD_DIM, double>& domainOffset,
49 : const tarch::la::Vector<MD_DIM, double>& meshWidth, const tarch::la::Vector<MD_DIM, unsigned int>& numberProcesses,
50 : const tarch::la::Vector<MD_LINKED_CELL_NEIGHBOURS, simplemd::BoundaryType>& boundary
51 : #if (MD_PARALLEL == MD_YES)
52 : ,
53 : MPI_Comm communicator
54 : #endif
55 : );
56 180 : ~ParallelTopologyService() {}
57 :
58 : /** finish initialization by allocating buffers
59 : *
60 : * Needs to be called after MoleculeContainer has been initialised.
61 : */
62 : void initBuffers(const unsigned int& localNumberOfMolecules);
63 :
64 : /** shuts down the buffer service and then the parallel service */
65 : void shutdown();
66 :
67 : /** returns the rank of the current process */
68 : int getRank() const;
69 :
70 : /** returns the process coordinates of the local process */
71 : const tarch::la::Vector<MD_DIM, unsigned int>& getProcessCoordinates() const;
72 :
73 : /** returns local information on boundary relations */
74 12 : const tarch::la::Vector<MD_LINKED_CELL_NEIGHBOURS, simplemd::BoundaryType>& getLocalBoundaryInformation() const { return _boundary; }
75 :
76 : /** returns the mesh width for the linked cells in the current simulation. Depending on the meshwidth
77 : * handed over during initialisation, this meshwidth might be different; we try to choose it as close
78 : * to the meshwidth defined by the user, but big enough so that we have the same (integer) number of
79 : * cells on each process. So, _meshWidth typicall is bigger or equal the user-defined meshwidth.
80 : * This may result in higher computation times! However, the physics remain the same as the cutoff-radius
81 : * is not affected by these changes.
82 : */
83 : const tarch::la::Vector<MD_DIM, double>& getMeshWidth() const { return _meshWidth; }
84 :
85 : /** returns the global number of cells in each spatial direction */
86 : const tarch::la::Vector<MD_DIM, unsigned int>& getGlobalNumberOfCells() const { return _globalNumberOfCells; }
87 :
88 : /** returns the local number of cells in each spatial direction (i.e. only the cells of this process) */
89 : const tarch::la::Vector<MD_DIM, unsigned int>& getLocalNumberOfCells(bool includingGhostCells = false) const {
90 : return includingGhostCells ? _localNumberOfCellsWithGhostCells : _localNumberOfCells;
91 : }
92 :
93 : /** returns the local number of cells (i.e. only the cells of this process) */
94 : size_t getLocalNumberOfCellsLinear(bool includingGhostCells = false) const {
95 : auto numCells = getLocalNumberOfCells(includingGhostCells);
96 : size_t numCellsLinear = 1;
97 : for (int d = 0; d < MD_DIM; d++) {
98 : numCellsLinear *= numCells[d];
99 : }
100 : return numCellsLinear;
101 : }
102 :
103 : /** returns the number of processes used in each spatial direction */
104 : const tarch::la::Vector<MD_DIM, unsigned int>& getNumberOfProcesses() const { return _numberProcesses; }
105 :
106 : /** returns the global domain size*/
107 4 : const tarch::la::Vector<MD_DIM, double>& getGlobalDomainSize() const { return _domainSize; }
108 :
109 : /** returns the global domain offset */
110 4 : const tarch::la::Vector<MD_DIM, double>& getGlobalDomainOffset() const { return _domainOffset; }
111 :
112 : /** returns the global index of the first (non-ghost) cell */
113 : const tarch::la::Vector<MD_DIM, unsigned int>& getGlobalIndexOfFirstCell() const { return _globalIndexOfFirstCell; }
114 :
115 : const tarch::la::Vector<MD_DIM, unsigned int>& getLocalIndexOfFirstCell() const { return _localIndexOfFirstCell; }
116 :
117 : /** returns true, if this process does not carry any work. This can be the case, if we have more ranks available
118 : * in the NodePool than specified in the xml config.
119 : */
120 : bool isIdle() const;
121 :
122 : /** broadcasts all molecules contained in the (inner) cell "cell" with index cellIndex to all
123 : * ghost cells that can be identified with this inner cell. Returns a vector containing the local
124 : * cell coordinates of all neighbored ghost cells that are a local, non-parallel boundary.
125 : * This is important for the case that we have 1 processor in one direction only. Then, we might
126 : * have a periodic boundary that needs to be handled locally!
127 : *
128 : * This method is also used for handling process-leaving particles in the optimised case when
129 : * they are sent together with molecules for ghost layers.
130 : * The local buffer of the buffer service is filled only with process-leaving particles that need
131 : * to be handled locally.
132 : * Ghost particles that need to be handled locally are handled via the indices in the returned vector.
133 : *
134 : * In the parallel case, this method makes use of the communication buffers: molecules are not
135 : * messaged to neighbour one by one, but rather pushed into buffers, which store all molecules to be sent to a process.
136 : *
137 : * The MPI send calls are executed later (from within BoundaryTreatment).
138 : */
139 : std::vector<tarch::la::Vector<MD_DIM, unsigned int>> broadcastInnerCellViaBuffer(LinkedCell& cell, const size_t cellIndex,
140 : const simplemd::MoleculeContainer& moleculeContainer);
141 :
142 : /** sends all molecules from cell cellIndex to the respective neighbouring process. The cell
143 : * cellIndex needs to be a ghost cell.
144 : * The function returns true, if the respective cell where the molecules need to be sorted in
145 : * is part of another process; in this case the molecules are pushed to the send buffers and erased.
146 : * Otherwise, the other (non-ghost) cell is part of the same process. In this case, no send-operation is triggered and false
147 : * is returned.
148 : *
149 : * The MPI send calls are executed later (from within BoundaryTreatment).
150 : */
151 : bool reduceGhostCellViaBuffer(LinkedCell& cell, const size_t cellIndex, const simplemd::MoleculeContainer& moleculeContainer);
152 :
153 : /** unpack and resort local buffer
154 : */
155 : void unpackLocalBuffer(simplemd::MoleculeContainer& moleculeContainer);
156 :
157 : /** Communication schedule:
158 : * 1. Irecv on buffers
159 : * 2. Isend on buffers
160 : * 3. Wait for all requests to be completed
161 : * 4. Unpack receive buffers and sort into linked cells
162 : *
163 : * For large messages, waiting for sender requests is also a must-have.
164 : */
165 : void communicationSteps_1_2();
166 :
167 : /** See comment of communicationSteps_1_2() */
168 : void communicationSteps_3_4(simplemd::MoleculeContainer& MoleculeContainer);
169 :
170 : /** Compute (non-overlapping) intersection of a global region of interest (ROI) with local domain.
171 : For example for purposes of profile plotter. */
172 : bool globalToLocalRegionOfInterest(const tarch::la::Vector<MD_DIM, unsigned int>& globalStartCell, const tarch::la::Vector<MD_DIM, unsigned int>& globalRange,
173 : tarch::la::Vector<MD_DIM, unsigned int>& localStartCell, tarch::la::Vector<MD_DIM, unsigned int>& localRange) const;
174 :
175 : /** Compute global cell index from local cell index in vector form */
176 : tarch::la::Vector<MD_DIM, unsigned int> localToGlobalCellIndexVector(const tarch::la::Vector<MD_DIM, unsigned int>& localCellIndexVector) const;
177 :
178 : #if (MD_PARALLEL == MD_YES)
179 : int MD_Allreduce(void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op) {
180 : return MPI_Allreduce(sendbuf, recvbuf, count, datatype, op, _communicator);
181 : }
182 : #endif
183 :
184 : tarch::la::Vector<MD_DIM, unsigned int> getGhostCellLayerThickness() const { return _ghostCellLayerThickness; }
185 :
186 : private:
187 : /** computes all neighbour ranks and stores the results in neighbourRanks.
188 : * Computes _neighbourRanksUnique, _numUniqueNeighbours and cells per buffer -
189 : * values needed by the buffer service.
190 : */
191 : void createNeighbourRanks(const tarch::la::Vector<MD_DIM, unsigned int>& processCoordinates,
192 : const tarch::la::Vector<MD_LINKED_CELL_NEIGHBOURS, simplemd::BoundaryType>& localBoundary,
193 : const tarch::la::Vector<MD_DIM, unsigned int>& numberProcesses, std::vector<int>& neighbourRanks,
194 : std::vector<int>& neighbourRanksUnique, unsigned int& numUniqueNeighbours, unsigned int numberOfCellsPerBuffer[]) const;
195 :
196 : /** if addedNeighbour is not already present in neighbourRanksUnique,
197 : * add it, also incrementing numUniqueNeighbours.
198 : */
199 : void addNeighbourToNeighbourRanksUnique(std::vector<int>& neighbourRanksUnique, unsigned int& numUniqueNeighbours, const int& addedNeighbour) const;
200 :
201 : /** computes the actual mesh size that is used in the simulation. We make the width as similar to the prescribed mesh width "prescribedWidth" as possible so
202 : * to have an integer number of grid cells.
203 : */
204 : tarch::la::Vector<MD_DIM, double> computeMeshwidth(const tarch::la::Vector<MD_DIM, double>& prescribedWidth,
205 : const tarch::la::Vector<MD_DIM, unsigned int>& numberProcesses,
206 : const tarch::la::Vector<MD_DIM, double>& domainSize) const;
207 :
208 : /** computes the number of cells in each process block.
209 : */
210 : tarch::la::Vector<MD_DIM, unsigned int> computeNumberOfCells(const tarch::la::Vector<MD_DIM, double>& meshWidth,
211 : const tarch::la::Vector<MD_DIM, unsigned int>& numberProcesses,
212 : const tarch::la::Vector<MD_DIM, double>& domainSize) const;
213 :
214 : /** computes the global number of grid cells */
215 : tarch::la::Vector<MD_DIM, unsigned int> computeGlobalNumberOfCells(const tarch::la::Vector<MD_DIM, double>& meshWidth,
216 : const tarch::la::Vector<MD_DIM, unsigned int>& numberProcesses,
217 : const tarch::la::Vector<MD_DIM, double>& domainSize) const;
218 :
219 : /** returns true if neighbourRank is present in the _neighbourRanks vector. Returns false otherwise. */
220 : bool isParallelNeighbour(const int& neighbourRank) const;
221 :
222 : /** returns the local information needed for periodic boundary treatment. This function
223 : * is called within the init() call and initialises the _periodicBoundary field.
224 : */
225 : tarch::la::Vector<MD_LINKED_CELL_NEIGHBOURS, simplemd::BoundaryType>
226 : computeLocalBoundaryInformation(const tarch::la::Vector<MD_LINKED_CELL_NEIGHBOURS, simplemd::BoundaryType>& boundary,
227 : const tarch::la::Vector<MD_DIM, unsigned int>& processCoordinates,
228 : const tarch::la::Vector<MD_DIM, unsigned int>& numberProcesses);
229 :
230 : /** returns the number of transferred cells, if the neighbour is placed in distance x,y,z
231 : * from the current cell.
232 : */
233 : unsigned int getNumberOfTransferredCells(const int& x
234 : #if (MD_DIM > 1)
235 : ,
236 : const int& y
237 : #endif
238 : #if (MD_DIM > 2)
239 : ,
240 : const int& z
241 : #endif
242 : ) const;
243 :
244 : /** corrects the position vector 'position' according to periodic boundary conditions.
245 : * If the boundaryType is PERIODIC_BOUNDARY the index neighbourIndex is used to determine
246 : * the location of the present boundary and to modify the position vector accordingly.
247 : */
248 : void adaptPositionForPeriodicBoundaries(tarch::la::Vector<MD_DIM, double>& position, const simplemd::BoundaryType& boundaryType, const int& x
249 : #if (MD_DIM > 1)
250 : ,
251 : const int& y
252 : #endif
253 : #if (MD_DIM > 2)
254 : ,
255 : const int& z
256 : #endif
257 : ) const;
258 :
259 : /** get buffer index corresponding to neighbourRank from _neighbourRanksUnique */
260 : unsigned int getCurrentBufferIndexFromNeighbourRank(const int& neighbourRank) const;
261 :
262 : /** Read off all molecules from buffer and resort them in the respective linked cells.
263 : */
264 : void unpackBuffer(ParallelAndLocalBufferService::SimpleBuffer* buf, simplemd::MoleculeContainer& MoleculeContainer);
265 :
266 : /** place position, velocity, forceOld and isFixed at the end of the respective local buffer.
267 : */
268 : void pushMoleculeToLocalBuffer(const Molecule* mol, const tarch::la::Vector<MD_DIM, double>& pos);
269 :
270 : #if (MD_PARALLEL == MD_YES)
271 : /** place position, velocity, forceOld and isFixed at the end of the respective sendBuffer.
272 : */
273 : void pushMoleculeToSendBuffer(const unsigned int& bufferIndex, const Molecule* mol, const tarch::la::Vector<MD_DIM, double>& pos);
274 :
275 : /** Issues an Isend call on buffer.
276 : * Does not wait for the request to be fulfilled, but uses up the respective request
277 : */
278 : void bufferIsend(ParallelAndLocalBufferService::SimpleBuffer* buffer, const int& neighbourRank, MPI_Request& request) const;
279 :
280 : /** Issues an Irecv call on buffer.
281 : * Does not wait for the request to be fulfilled, but uses up the respective request
282 : */
283 : void bufferIrecv(ParallelAndLocalBufferService::SimpleBuffer* buffer, const int& neighbourRank, MPI_Request& request) const;
284 : #endif
285 :
286 : /** domain size */
287 : const tarch::la::Vector<MD_DIM, double> _domainSize;
288 :
289 : /** domain offset */
290 : const tarch::la::Vector<MD_DIM, double> _domainOffset;
291 :
292 : /** The size of a linked cell along each axis */
293 : const tarch::la::Vector<MD_DIM, double> _meshWidth;
294 :
295 : /** The number of ghost cells around the local domain along each axis */
296 : static constexpr unsigned int _ghostCellLayerThickness = 1; // NOTE: This MUST be 1 (otherwise nothing will work anymore)
297 :
298 : /** index of first cell w.r.t. global grid
299 : */
300 : tarch::la::Vector<MD_DIM, unsigned int> _globalIndexOfFirstCell;
301 :
302 : /** index of first cell w.r.t. local grid (including ghost cells )*/
303 : const tarch::la::Vector<MD_DIM, unsigned int> _localIndexOfFirstCell;
304 :
305 : /** definition of the process matrix, i.e. how many processes work in each
306 : * spatial direction.
307 : */
308 : const tarch::la::Vector<MD_DIM, unsigned int> _numberProcesses;
309 :
310 : /** local number of cells (i.e. on each process) in all d directions */
311 : const tarch::la::Vector<MD_DIM, unsigned int> _localNumberOfCells;
312 :
313 : /** local number of cells (i.e. on each process) in all d directions including ghost cells */
314 : const tarch::la::Vector<MD_DIM, unsigned int> _localNumberOfCellsWithGhostCells;
315 :
316 : /** global number of cells in all d directions */
317 : const tarch::la::Vector<MD_DIM, unsigned int> _globalNumberOfCells;
318 :
319 : /** rank of this process */
320 : int _rank;
321 :
322 : /** boundary information on the local process */
323 : tarch::la::Vector<MD_LINKED_CELL_NEIGHBOURS, simplemd::BoundaryType> _boundary;
324 :
325 : /** coordinates of this process in the process matrix. This vector is the same as
326 : * _rank, but written in vector form.
327 : */
328 : tarch::la::Vector<MD_DIM, unsigned int> _processCoordinates;
329 :
330 : /** vector containing all ranks of the neighbouring processes. This includes
331 : * periodic neighbour relations.
332 : */
333 : std::vector<int> _neighbourRanks;
334 :
335 : /** Buffer service exported in a separate class
336 : * @see ParallelAndLocalBufferService
337 : */
338 : ParallelAndLocalBufferService _bufferService;
339 :
340 : /** like _neighbourRanks, but without possible repetitions
341 : * @see _neighbourRanks
342 : */
343 : std::vector<int> _neighbourRanksUnique;
344 :
345 : #if (MD_PARALLEL == MD_YES)
346 : /** tag for communicating buffers */
347 : int _bufferTag;
348 : #endif
349 :
350 : /** number of processes current process will communicate with
351 : * also equal to number of used buffers and MPI requests
352 : */
353 : unsigned int _numUniqueNeighbours;
354 :
355 : /** how many cells a buffer is responsible for */
356 : unsigned int _numberOfCellsPerBuffer[MD_LINKED_CELL_NEIGHBOURS];
357 :
358 : #if (MD_PARALLEL == MD_YES)
359 : MPI_Comm _communicator;
360 : /** requests for sending and receiving buffers */
361 : MPI_Request _receiveRequests[MD_LINKED_CELL_NEIGHBOURS];
362 : MPI_Request _sendRequests[MD_LINKED_CELL_NEIGHBOURS];
363 : #endif
364 : };
365 : #endif // _MOLECULARDYNAMICS_SERVICES_PARALLELTOPOLOGYSERVICE_H_
|
