escript  Revision_4925
blocktools.h
Go to the documentation of this file.
1 /*****************************************************************************
2 *
3 * Copyright (c) 2014 by University of Queensland
4 * http://www.uq.edu.au
5 *
6 * Primary Business: Queensland, Australia
7 * Licensed under the Open Software License version 3.0
8 * http://www.opensource.org/licenses/osl-3.0.php
9 *
10 * Development until 2012 by Earth Systems Science Computational Center (ESSCC)
11 * Development 2012-2013 by School of Earth Sciences
12 * Development from 2014 by Centre for Geoscience Computing (GeoComp)
13 *
14 *****************************************************************************/
15 
16 
17 #include <vector>
18 
19 
20 /* This file contains two main classes for dealing with a large 3D region which has been divided
21  into a 3D Grid of Blocks (usually to be distributed). Each block is divided into 27 subblocks.
22  The first and last subblocks in each dimension are cubes.
23 
24  class Block mangages a single block. It has methods for copying between a flat array (storing
25  all the values in the 3D block) and buffers storing individual subblocks.
26  These buffers can be used to transfer individual subblocks to other blocks (using some external
27  means).
28 
29  class BlockGrid deals with the position of a given block in relation to the rest of the blocks
30  in the grid. (In an MPI setting, there would be one block per rank.)
31  It also required transfers of subblocks into and out of the block in order to make the whole
32  (global) array consistant.
33 
34  Each block has a region "inset" wide in from each edge which is shared with nieghbouring blocks.
35  Where these regions overlap with another block, the block closest to block 0,0,0 are used.
36  Or more precisely, values move left->right then lowy->highy and finally lowz -> highz.
37 
38  Please don't mix external calls into this file, it may be useful to separate it for
39  debugging purposes.
40 
41  Types required:
42  neighbourID_t - Stores the label of a neighbouring block.
43  In an MPI setting, this will be the type used refer to ranks
44  coord_t - Stores a position of a block in the grid of blocks (it could be
45  within one dimension or overall in a flat structure.
46  It is not (necessarily) the same as neighbourID_t because
47  coord_t should be _unsigned_ and there is no guarantee
48  that neighbourID_t will even be an integral type.
49 */
50 
51 #ifdef ESYS_MPI
52 #include <esysUtils/Esys_MPI.h>
53 typedef Esys_MPI_rank neighbourID_t; // This should be the MPI_rank type
54 typedef unsigned coord_t; // if we ever get more than 2^32 ranks, we have other problems
55 
56 
57 #else
58 
59 typedef int neighbourID_t; // This should be the MPI_rank type
60 typedef unsigned coord_t; // if we ever get more than 2^32 ranks, we have other problems
61 
62 #endif
63 
64 
65 typedef std::pair<neighbourID_t, int> neighpair;
66 typedef std::vector<neighpair> neighbourvector;
67 
68 
69 
70 typedef struct
71 {
72 public:
73  neighbourID_t sourceID; // ranks involved in communication
75  int tag;
76  unsigned char srcbuffid; // number of buffer to use for coms
77  unsigned char destbuffid;
78 } message;
79 
80 typedef std::vector<message> messvec;
81 
82 
83 
84 
85 class BlockGrid
86 {
87 public:
88  BlockGrid(coord_t maxx, coord_t maxy, coord_t maxz);
89 
91 
92 
93 // generate all incoming com messages for this block.
94 // for each subblock (27 of them), there may be an x, y, z direction to search in
95 void generateInNeighbours(coord_t blockx, coord_t blocky, coord_t blockz, messvec& v);
96 
97 
98 // generate all outgoing com messages for this block
99 void generateOutNeighbours(coord_t blockx, coord_t blocky, coord_t blockz, messvec& v);
100 private:
104 };
105 
106 
107 
108 
109 /* Do not ask about buffers for sub-block 1,1,1 (also known as #13)
110  They do not exist, such buffers woudl be:
111  1) big
112  2) unnecessary since the centre sub-block is not sent anywhere
113 
114 Note that this class does not deal with data transfer between blocks
115 Sub-blocks are copied to and from buffers. Other code is required to
116 actually move the data.
117 
118 "dpp" == doubles per point and gives the number of doubles that make up each "point"
119 This is required to calculate offsets and buffer sizes.
120 
121 */
122 class Block
123 {
124 public:
125 
126  // s? specifiy the [local] size (in points) of each dimension
127  Block(size_t sx, size_t sy, size_t sz, size_t inset, size_t xmidlen,
128  size_t ymidlen, size_t zmidlen, unsigned int dpp=1);
129 
130  ~Block();
131 
132  // Out buffers are loaded with the contents of the flat array and are
133  // to be sent to other blocks
134  double* getOutBuffer(unsigned char subx, unsigned char suby, unsigned char subz);
135  double* getOutBuffer(unsigned char bid);
136 
137  // In buffers are populated from external communications
138  // and copied back to the flat array
139  double* getInBuffer(unsigned char subx, unsigned char suby, unsigned char subz);
140  double* getInBuffer(unsigned char bid);
141 
142  // return number of doubles in the given block
143  size_t getBuffSize(unsigned char subx, unsigned char suby, unsigned char subz);
144  size_t getBuffSize(unsigned char bid);
145 
146  // where does the subblock specified start in a source array
147  size_t startOffset(unsigned char subx, unsigned char suby, unsigned char subz);
148 
149  // debug only
150  void displayBlock(unsigned char subx, unsigned char suby, unsigned char subz, bool out);
151 
152  // Copy a 3d region from a flat array into a buffer
153  void copyToBuffer(unsigned char buffid, double* src);
154 
155  // Copy a 3d region from a buffer into a flat array
156  void copyFromBuffer(unsigned char buffid, double* dest);
157 
158 
159  void copyAllToBuffer(double* src);
160 
161  void copyUsedFromBuffer(double* dest);
162 
163  void setUsed(unsigned char buffid);
164 
165 private:
166 
167  // determines the dimensions of each subblock
168  void populateDimsTable();
169  void populateOffsetTable(size_t inset, size_t xmidlen, size_t ymidlen, size_t zmidlen);
170  void createBuffArrays(double* startaddress, double* buffptr[27], size_t inset, size_t xmidlen, size_t ymidlen, size_t zmidlen);
171 
172 
173 
174  double* inbuff;
175  double* outbuff;
176  size_t buffoffsets[27]; // offsets of the various blocks within the buffer arrays
177  size_t flatoffsets[27]; // starting point of each block within a flat array
178  bool used[27];
179  size_t dims[27][3]; // dimension of each subblock
180  size_t sx;
181  size_t sy;
182  size_t sz;
183  size_t inset;
184  size_t xmidlen;
185  size_t ymidlen;
186  size_t zmidlen;
187  double* inbuffptr[27];
188  double* outbuffptr[27];
189  const unsigned int dpsize; // number of doubles which make up a point
190 
191 
192 
193 };
194 
195 // Returns the MPI message tag to use for a transfer between the two subblocks
196 int getTag(unsigned char sourcex, unsigned char sourcey, unsigned char sourcez, unsigned char targetx, unsigned char targety, unsigned char targetz);
197 
198 // computes the tag based on the destination and the direction it comes from
199 // the booleans indicate whether a negative shift in that direction is required
200 int getTag(unsigned char destx, unsigned char desty, unsigned char destz, bool deltax, bool deltay, bool deltaz);
201 
202 
203 // the booleans indicate whether a negative shift in that direction is required
204 unsigned char getSrcBuffID(unsigned char destx, unsigned char desty, unsigned char destz, bool deltax, bool deltay, bool deltaz);
205 
206 
207 /* Now the 2D versions */
208 
209 // 2D version
211 {
212 public:
213  BlockGrid2(coord_t maxx, coord_t maxy);
214 
215  neighbourID_t getNID(coord_t x, coord_t y) const;
216 
217 
218 // generate all incoming com messages for this block.
219 // for each subblock (9 of them), there may be an x, y direction to search in
220 void generateInNeighbours(coord_t blockx, coord_t blocky, messvec& v);
221 
222 
223 // generate all outgoing com messages for this block
224 void generateOutNeighbours(coord_t blockx, coord_t blocky, messvec& v);
225 private:
228 };
229 
230 // The 2D version - there is no block 4
231 class Block2
232 {
233 public:
234 
235  // s? specifiy the [local] size (in points) of each dimension
236  Block2(size_t sx, size_t sy, size_t inset, size_t xmidlen,
237  size_t ymidlen, unsigned int dpp=1);
238 
239  ~Block2();
240 
241  // Out buffers are loaded with the contents of the flat array and are
242  // to be sent to other blocks
243  double* getOutBuffer(unsigned char subx, unsigned char suby);
244  double* getOutBuffer(unsigned char bid);
245 
246  // In buffers are populated from external communications
247  // and copied back to the flat array
248  double* getInBuffer(unsigned char subx, unsigned char suby);
249  double* getInBuffer(unsigned char bid);
250 
251  // return number of doubles in the given block
252  size_t getBuffSize(unsigned char subx, unsigned char suby);
253  size_t getBuffSize(unsigned char bid);
254 
255  // where does the subblock specified start in a source array
256  size_t startOffset(unsigned char subx, unsigned char suby);
257 
258  // debug only
259  void displayBlock(unsigned char subx, unsigned char suby, bool out);
260 
261  // Copy a 3d region from a flat array into a buffer
262  void copyToBuffer(unsigned char buffid, double* src);
263 
264  // Copy a 3d region from a buffer into a flat array
265  void copyFromBuffer(unsigned char buffid, double* dest);
266 
267 
268  void copyAllToBuffer(double* src);
269 
270  void copyUsedFromBuffer(double* dest);
271 
272  void setUsed(unsigned char buffid);
273 
274 private:
275 
276  // determines the dimensions of each subblock
277  void populateDimsTable();
278  void populateOffsetTable(size_t inset, size_t xmidlen, size_t ymidlen);
279  void createBuffArrays(double* startaddress, double* buffptr[27], size_t inset, size_t xmidlen, size_t ymidlen);
280 
281 
282 
283  double* inbuff;
284  double* outbuff;
285  size_t buffoffsets[9]; // offsets of the various blocks within the buffer arrays
286  size_t flatoffsets[9]; // starting point of each block within a flat array
287  bool used[9];
288  size_t dims[9][2]; // dimension of each subblock
289  size_t sx;
290  size_t sy;
291  size_t inset;
292  size_t xmidlen;
293  size_t ymidlen;
294  double* inbuffptr[9];
295  double* outbuffptr[9];
296  const unsigned int dpsize; // number of doubles which make up a point
297 };
298 
299 
300 // Returns the MPI message tag to use for a transfer between the two subblocks
301 int getTag2(unsigned char sourcex, unsigned char sourcey, unsigned char targetx, unsigned char targety);
302 
303 // computes the tag based on the destination and the direction it comes from
304 // the booleans indicate whether a negative shift in that direction is required
305 int getTag2(unsigned char destx, unsigned char desty, bool deltax, bool deltay);
306 
307 
308 // the booleans indicate whether a negative shift in that direction is required
309 unsigned char getSrcBuffID2(unsigned char destx, unsigned char desty, bool deltax, bool deltay);
310 
double * getOutBuffer(unsigned char subx, unsigned char suby)
Definition: blocktools2.cpp:99
void generateOutNeighbours(coord_t blockx, coord_t blocky, messvec &v)
Definition: blocktools2.cpp:66
void createBuffArrays(double *startaddress, double *buffptr[27], size_t inset, size_t xmidlen, size_t ymidlen)
Definition: blocktools2.cpp:204
void generateInNeighbours(coord_t blockx, coord_t blocky, coord_t blockz, messvec &v)
Definition: blocktools.cpp:42
Definition: blocktools.h:70
int tag
Definition: blocktools.h:75
void copyFromBuffer(unsigned char buffid, double *dest)
Definition: blocktools2.cpp:344
size_t buffoffsets[27]
Definition: blocktools.h:176
void setUsed(unsigned char buffid)
Definition: blocktools2.cpp:214
double * getInBuffer(unsigned char subx, unsigned char suby, unsigned char subz)
Definition: blocktools.cpp:131
unsigned coord_t
Definition: blocktools.h:60
Block(size_t sx, size_t sy, size_t sz, size_t inset, size_t xmidlen, size_t ymidlen, size_t zmidlen, unsigned int dpp=1)
Definition: blocktools.cpp:276
void copyFromBuffer(unsigned char buffid, double *dest)
Definition: blocktools.cpp:385
size_t sx
Definition: blocktools.h:180
double * inbuff
Definition: blocktools.h:174
unsigned char destbuffid
Definition: blocktools.h:77
std::pair< neighbourID_t, int > neighpair
Definition: blocktools.h:65
size_t sx
Definition: blocktools.h:289
void displayBlock(unsigned char subx, unsigned char suby, bool out)
Definition: blocktools2.cpp:287
const unsigned int dpsize
Definition: blocktools.h:189
size_t startOffset(unsigned char subx, unsigned char suby, unsigned char subz)
Definition: blocktools.cpp:308
size_t inset
Definition: blocktools.h:183
size_t getBuffSize(unsigned char subx, unsigned char suby)
Definition: blocktools2.cpp:119
Definition: blocktools.h:231
double * outbuff
Definition: blocktools.h:284
size_t dims[27][3]
Definition: blocktools.h:179
size_t ymidlen
Definition: blocktools.h:293
Definition: blocktools.h:210
double * getInBuffer(unsigned char subx, unsigned char suby)
Definition: blocktools2.cpp:109
void createBuffArrays(double *startaddress, double *buffptr[27], size_t inset, size_t xmidlen, size_t ymidlen, size_t zmidlen)
Definition: blocktools.cpp:235
~Block2()
Definition: blocktools2.cpp:158
std::vector< neighpair > neighbourvector
Definition: blocktools.h:66
double * outbuffptr[27]
Definition: blocktools.h:188
unsigned char getSrcBuffID(unsigned char destx, unsigned char desty, unsigned char destz, bool deltax, bool deltay, bool deltaz)
Definition: blocktools.cpp:429
size_t dims[9][2]
Definition: blocktools.h:288
int getTag(unsigned char sourcex, unsigned char sourcey, unsigned char sourcez, unsigned char targetx, unsigned char targety, unsigned char targetz)
Definition: blocktools.cpp:411
void generateOutNeighbours(coord_t blockx, coord_t blocky, coord_t blockz, messvec &v)
Definition: blocktools.cpp:81
Definition: blocktools.h:85
double * outbuff
Definition: blocktools.h:175
void populateOffsetTable(size_t inset, size_t xmidlen, size_t ymidlen, size_t zmidlen)
Definition: blocktools.cpp:213
size_t buffoffsets[9]
Definition: blocktools.h:285
int getTag2(unsigned char sourcex, unsigned char sourcey, unsigned char targetx, unsigned char targety)
Definition: blocktools2.cpp:363
coord_t zmax
Definition: blocktools.h:103
neighbourID_t sourceID
Definition: blocktools.h:73
const unsigned int dpsize
Definition: blocktools.h:296
double * inbuff
Definition: blocktools.h:283
void populateDimsTable()
Definition: blocktools2.cpp:164
size_t ymidlen
Definition: blocktools.h:185
neighbourID_t destID
Definition: blocktools.h:74
size_t flatoffsets[27]
Definition: blocktools.h:177
BlockGrid2(coord_t maxx, coord_t maxy)
Definition: blocktools2.cpp:22
BlockGrid(coord_t maxx, coord_t maxy, coord_t maxz)
Definition: blocktools.cpp:23
size_t startOffset(unsigned char subx, unsigned char suby)
Definition: blocktools2.cpp:275
coord_t xmax
Definition: blocktools.h:226
void copyAllToBuffer(double *src)
Definition: blocktools.cpp:250
void copyUsedFromBuffer(double *dest)
Definition: blocktools2.cpp:232
unsigned char srcbuffid
Definition: blocktools.h:76
std::vector< message > messvec
Definition: blocktools.h:80
void generateInNeighbours(coord_t blockx, coord_t blocky, messvec &v)
Definition: blocktools2.cpp:33
void copyToBuffer(unsigned char buffid, double *src)
Definition: blocktools.cpp:356
neighbourID_t getNID(coord_t x, coord_t y, coord_t z) const
Definition: blocktools.cpp:27
size_t flatoffsets[9]
Definition: blocktools.h:286
size_t xmidlen
Definition: blocktools.h:184
bool used[9]
Definition: blocktools.h:287
void populateOffsetTable(size_t inset, size_t xmidlen, size_t ymidlen)
Definition: blocktools2.cpp:182
size_t sy
Definition: blocktools.h:290
size_t getBuffSize(unsigned char subx, unsigned char suby, unsigned char subz)
Definition: blocktools.cpp:141
Block2(size_t sx, size_t sy, size_t inset, size_t xmidlen, size_t ymidlen, unsigned int dpp=1)
Definition: blocktools2.cpp:245
size_t xmidlen
Definition: blocktools.h:292
int Esys_MPI_rank
Definition: Esys_MPI.h:37
unsigned char getSrcBuffID2(unsigned char destx, unsigned char desty, bool deltax, bool deltay)
Definition: blocktools2.cpp:379
double * outbuffptr[9]
Definition: blocktools.h:295
void copyUsedFromBuffer(double *dest)
Definition: blocktools.cpp:263
size_t sy
Definition: blocktools.h:181
~Block()
Definition: blocktools.cpp:180
size_t inset
Definition: blocktools.h:291
void populateDimsTable()
Definition: blocktools.cpp:186
double * inbuffptr[27]
Definition: blocktools.h:187
void setUsed(unsigned char buffid)
Definition: blocktools.cpp:245
void copyAllToBuffer(double *src)
Definition: blocktools2.cpp:219
void copyToBuffer(unsigned char buffid, double *src)
Definition: blocktools2.cpp:316
coord_t ymax
Definition: blocktools.h:102
coord_t xmax
Definition: blocktools.h:101
neighbourID_t getNID(coord_t x, coord_t y) const
Definition: blocktools2.cpp:26
double * getOutBuffer(unsigned char subx, unsigned char suby, unsigned char subz)
Definition: blocktools.cpp:121
size_t zmidlen
Definition: blocktools.h:186
bool used[27]
Definition: blocktools.h:178
Definition: blocktools.h:122
void displayBlock(unsigned char subx, unsigned char suby, unsigned char subz, bool out)
Definition: blocktools.cpp:322
double * inbuffptr[9]
Definition: blocktools.h:294
coord_t ymax
Definition: blocktools.h:227
size_t sz
Definition: blocktools.h:182
int neighbourID_t
Definition: blocktools.h:59