casacore
TSMCube.h
Go to the documentation of this file.
1 //# TSMCube.h: Tiled hypercube in a table
2 //# Copyright (C) 1995,1996,1997,1999,2000,2001,2002
3 //# Associated Universities, Inc. Washington DC, USA.
4 //#
5 //# This library is free software; you can redistribute it and/or modify it
6 //# under the terms of the GNU Library General Public License as published by
7 //# the Free Software Foundation; either version 2 of the License, or (at your
8 //# option) any later version.
9 //#
10 //# This library is distributed in the hope that it will be useful, but WITHOUT
11 //# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 //# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public
13 //# License for more details.
14 //#
15 //# You should have received a copy of the GNU Library General Public License
16 //# along with this library; if not, write to the Free Software Foundation,
17 //# Inc., 675 Massachusetts Ave, Cambridge, MA 02139, USA.
18 //#
19 //# Correspondence concerning AIPS++ should be addressed as follows:
20 //# Internet email: aips2-request@nrao.edu.
21 //# Postal address: AIPS++ Project Office
22 //# National Radio Astronomy Observatory
23 //# 520 Edgemont Road
24 //# Charlottesville, VA 22903-2475 USA
25 //#
26 //# $Id: TSMCube.h 21521 2014-12-10 08:06:42Z gervandiepen $
27 
28 #ifndef TABLES_TSMCUBE_H
29 #define TABLES_TSMCUBE_H
30 
31 
32 //# Includes
33 #include <casacore/casa/aips.h>
34 #include <casacore/tables/DataMan/TSMShape.h>
35 #include <casacore/casa/Containers/Record.h>
36 #include <casacore/casa/Arrays/IPosition.h>
37 #include <casacore/casa/OS/Conversion.h>
38 #include <casacore/casa/iosfwd.h>
39 
40 namespace casacore { //# NAMESPACE CASACORE - BEGIN
41 
42 //# Forward declarations
43 class TiledStMan;
44 class TSMFile;
45 class TSMColumn;
46 class BucketCache;
47 template<class T> class Block;
48 
49 // <summary>
50 // Tiled hypercube in a table
51 // </summary>
52 
53 // <use visibility=local>
54 
55 // <reviewed reviewer="UNKNOWN" date="before2004/08/25" tests="">
56 // </reviewed>
57 
58 // <prerequisite>
59 //# Classes you should understand before using this one.
60 // <li> <linkto class=TiledStMan>TiledStMan</linkto>
61 // <li> <linkto class=ROTiledStManAccessor>ROTiledStManAccessor</linkto>
62 // for a discussion of the maximum cache size
63 // <li> <linkto class=TSMFile>TSMFile</linkto>
64 // <li> <linkto class=BucketCache>BucketCache</linkto>
65 // </prerequisite>
66 
67 // <etymology>
68 // TSMCube represents a hypercube in the Tiled Storage Manager.
69 // </etymology>
70 
71 // <synopsis>
72 // TSMCube defines a tiled hypercube. The data is stored in a TSMFile
73 // object and accessed using a BucketCache object. The hypercube can
74 // be extensible in its last dimension to support tables with a size
75 // which is not known in advance.
76 // <br>
77 // Normally hypercubes share the same TSMFile object, but extensible
78 // hypercubes have their own TSMFile object (to be extensible).
79 // If the hypercolumn has multiple data columns, their cells share the same
80 // tiles. Per tile data column A appears first, thereafter B, etc..
81 // <br>
82 // The data in the cache is held in external format and is converted
83 // when accessed. The alternative would be to hold it in the cache in
84 // local format and convert it when read/written from the file. It was
85 // felt that the latter approach would generate more needless conversions.
86 // <p>
87 // The possible id and coordinate values are stored in a Record
88 // object. They are written in the main hypercube AipsIO file.
89 // <p>
90 // TSMCube uses the maximum cache size set for a Tiled Storage manager.
91 // The description of class
92 // <linkto class=ROTiledStManAccessor>ROTiledStManAccessor</linkto>
93 // contains a discussion about the effect of setting the maximum cache size.
94 // </synopsis>
95 
96 // <motivation>
97 // TSMCube encapsulates all operations on a hypercube.
98 // </motivation>
99 
100 //# <todo asof="$DATE:$">
101 //# A List of bugs, limitations, extensions or planned refinements.
102 //# </todo>
103 
104 
105 class TSMCube
106 {
107 public:
108  // Define the possible access types for TSMDataColumn.
109  enum AccessType {
115  };
116 
117  // Construct the hypercube using the given file with the given shape.
118  // The record contains the id and possible coordinate values.
119  // <br>If the cubeshape is empty, the hypercube is still undefined and
120  // can be added later with setShape. That is only used by TiledCellStMan.
121  // <br> The fileOffset argument is meant for class TiledFileAccess.
122  TSMCube (TiledStMan* stman, TSMFile* file,
123  const IPosition& cubeShape,
124  const IPosition& tileShape,
125  const Record& values,
126  Int64 fileOffset,
127  Bool useDerived = False);
128 
129  // Reconstruct the hypercube by reading its data from the AipsIO stream.
130  // It will link itself to the correct TSMFile. The TSMFile objects
131  // must have been reconstructed in advance.
132  TSMCube (TiledStMan* stman, AipsIO& ios,
133  Bool useDerived = False);
134 
135  virtual ~TSMCube();
136 
137  // Flush the data in the cache.
138  virtual void flushCache();
139 
140  // Clear the cache, so data will be reread.
141  // If wanted, the data is flushed before the cache is cleared.
142  void clearCache (Bool doFlush = True);
143 
144  // Empty the cache.
145  // It will flush the cache as needed and remove all buckets from it
146  // resulting in a possibly large drop in memory used.
147  // It'll also clear the <src>userSetCache_p</src> flag.
148  void emptyCache();
149 
150  // Show the cache statistics.
151  virtual void showCacheStatistics (ostream& os) const;
152 
153  // Put the data of the object into the AipsIO stream.
154  void putObject (AipsIO& ios);
155 
156  // Get the data of the object from the AipsIO stream.
157  // It returns the data manager sequence number, which is -1 if
158  // no file is attached to the cube (for cells without a value).
160 
161  // Resync the object with the data file.
162  // It reads the object, and adjusts the cache.
163  virtual void resync (AipsIO& ios);
164 
165  // Is the hypercube extensible?
167 
168  // Get the bucket size (bytes).
169  // It is the length of a tile in external format.
170  uInt bucketSize() const;
171 
172  // Get the length of a tile (in bytes) in local format.
173  uInt localTileLength() const;
174 
175  // Set the hypercube shape.
176  // This is only possible if the shape was not defined yet.
177  virtual void setShape (const IPosition& cubeShape,
178  const IPosition& tileShape);
179 
180  // Get the shape of the hypercube.
181  const IPosition& cubeShape() const;
182 
183  // Get the shape of the tiles.
184  const IPosition& tileShape() const;
185 
186  // Get the shape of the data cells in the cube.
188 
189  // Get the size of a coordinate (i.e. the number of values in it).
190  // If not defined, it returns zero.
191  uInt coordinateSize (const String& coordinateName) const;
192 
193  // Get the record containing the id and coordinate values.
194  // It is used by TSMIdColumn and TSMCoordColumn.
195  // <group>
196  const Record& valueRecord() const;
198  // </group>
199 
200  // Test if the id values match.
202  const Record& idValues);
203 
204  // Extend the last dimension of the cube with the given number.
205  // The record can contain the coordinates of the elements added.
206  virtual void extend (uInt64 nr, const Record& coordValues,
207  const TSMColumn* lastCoordColumn);
208 
209  // Extend the coordinates vector for the given coordinate
210  // to the given length with the given coordValues.
211  // It will be initialized to zero if no coordValues are given.
212  // If the coordinate vector does not exist yet, it will be created.
213  void extendCoordinates (const Record& coordValues,
214  const String& coordName, uInt length);
215 
216  // Read or write a section in the cube.
217  // It is assumed that the section buffer is long enough.
218  virtual void accessSection (const IPosition& start, const IPosition& end,
219  char* section, uInt colnr,
220  uInt localPixelSize, uInt externalPixelSize,
221  Bool writeFlag);
222 
223  // Read or write a section in a strided way.
224  // It is assumed that the section buffer is long enough.
225  virtual void accessStrided (const IPosition& start, const IPosition& end,
226  const IPosition& stride,
227  char* section, uInt colnr,
228  uInt localPixelSize, uInt externalPixelSize,
229  Bool writeFlag);
230 
231  // Get the current cache size (in buckets).
232  uInt cacheSize() const;
233 
234  // Calculate the cache size (in buckets) for the given slice
235  // and access path.
236  // <group>
237  uInt calcCacheSize (const IPosition& sliceShape,
238  const IPosition& windowStart,
239  const IPosition& windowLength,
240  const IPosition& axisPath) const;
242  const IPosition& tileShape,
243  Bool extensible,
244  const IPosition& sliceShape,
245  const IPosition& windowStart,
246  const IPosition& windowLength,
247  const IPosition& axisPath,
248  uInt maxCacheSizeMiB, uInt bucketSize);
249  // </group>
250 
251  // Set the cache size for the given slice and access path.
252  virtual void setCacheSize (const IPosition& sliceShape,
253  const IPosition& windowStart,
254  const IPosition& windowLength,
255  const IPosition& axisPath,
256  Bool forceSmaller, Bool userSet);
257 
258  // Resize the cache object.
259  // If forceSmaller is False, the cache will only be resized when it grows.
260  // If the given size exceeds the maximum size with more
261  // than 10%, the maximum size will be used.
262  // The cacheSize has to be given in buckets.
263  // <br>The flag <src>userSet</src> inidicates if the cache size is set by
264  // the user (by an Accessor object) or automatically (by TSMDataColumn).
265  virtual void setCacheSize (uInt cacheSize, Bool forceSmaller, Bool userSet);
266 
267  // Validate the cache size (in buckets).
268  // This means it will return the given cache size (in buckets) if
269  // smaller than the maximum cache size (given in MiB).
270  // Otherwise the maximum is returned.
271  // <group>
273  static uInt validateCacheSize (uInt cacheSize, uInt maxSizeMiB,
274  uInt bucketSize);
275  // </group>
276 
277  // Determine if the user set the cache size (using setCacheSize).
278  Bool userSetCache() const;
279 
280  // Functions for TSMDataColumn to keep track of the last type of
281  // access to a hypercube. It uses it to determine if the cache
282  // has to be reset.
283  // <group>
285  const IPosition& getLastColSlice() const;
286  void setLastColAccess (AccessType type);
287  void setLastColSlice (const IPosition& slice);
288  // </group>
289 
290 protected:
291  // Initialize the various variables.
292  // <group>
293  void setup();
294  void setupNrTiles();
295  // </group>
296 
297  // Adjust the tile shape to the hypercube shape.
298  // A size of 0 gets set to 1.
299  // A tile size > cube size gets set to the cube size.
301  const IPosition& tileShape) const;
302 
303  // Resize the IPosition member variables used in accessSection()
304  // if nrdim_p changes value.
306 
307 private:
308  // Forbid copy constructor.
309  TSMCube (const TSMCube&);
310 
311  // Forbid assignment.
313 
314  // Get the cache object.
315  // This will construct the cache object if not present yet.
317 
318  // Construct the cache object (if not constructed yet).
319  virtual void makeCache();
320 
321  // Resync the cache object.
322  virtual void resyncCache();
323 
324  // Delete the cache object.
325  virtual void deleteCache();
326 
327  // Access a line in a more optimized way.
328  void accessLine (char* section, uInt pixelOffset,
329  uInt localPixelSize,
330  Bool writeFlag, BucketCache* cachePtr,
331  const IPosition& startTile, uInt endTile,
332  const IPosition& startPixelInFirstTile,
333  uInt endPixelInLastTile,
334  uInt lineIndex);
335 
336  // Define the callback functions for the BucketCache.
337  // <group>
338  static char* readCallBack (void* owner, const char* external);
339  static void writeCallBack (void* owner, char* external,
340  const char* local);
341  static char* initCallBack (void* owner);
342  static void deleteCallBack (void* owner, char* buffer);
343  // </group>
344 
345  // Define the functions doing the actual read and write of the
346  // data in the tile and converting it to/from local format.
347  // <group>
348  char* readTile (const char* external);
349  void writeTile (char* external, const char* local);
350  // </group>
351 
352 protected:
353  //# Declare member variables.
354 
355  char * cachedTile_p; // optimization to hold one tile chunk
356 
357  // Pointer to the parent storage manager.
359  // Is the class used directly or only by a derived class only?
361  // The values of the possible id and coordinate columns.
363  // Is the hypercube extensible?
365  // Dimensionality of the hypercube.
367  // Number of tiles in the hypercube.
369  // The shape of the hypercube.
371  // The shape of the tiles in the hypercube.
373  // The number of tiles in each hypercube dimension.
375  // Precomputed tileShape information.
377  // Precomputed tilesPerDim information.
379  // Number of tiles in all but last dimension (used when extending).
381  // The tilesize in bytes.
383  // Pointer to the TSMFile object holding the data.
385  // Offset in the TSMFile object where the data of this hypercube starts.
387  // Offset for each data column in a tile (in external format).
389  // Offset for each data column in a tile (in local format).
391  // The bucket size in bytes (is equal to tile size in bytes).
393  // The tile size in bytes in local format.
395  // The bucket cache.
397  // Did the user set the cache size?
399  // Was the last column access to a cell, slice, or column?
401  // The slice shape of the last column access to a slice.
403 
404  // IPosition variables used in accessSection(); declared here
405  // as member variables to avoid significant construction and
406  // desctruction overhead if they are local to accessSection()
407  // #tiles needed for the section
409  // First tile needed
411  // Last tile needed
413  // First pixel in first tile
415  // Last pixel in first tile
417  // Last pixel in last tile
419 };
420 
421 
422 
424 {
425  if (cache_p == 0) {
426  makeCache();
427  }
428  return cache_p;
429 }
430 inline uInt TSMCube::bucketSize() const
431 {
432  return bucketSize_p;
433 }
435 {
436  return localTileLength_p;
437 }
438 inline const IPosition& TSMCube::cubeShape() const
439 {
440  return cubeShape_p;
441 }
442 inline const IPosition& TSMCube::tileShape() const
443 {
444  return tileShape_p;
445 }
446 inline const Record& TSMCube::valueRecord() const
447 {
448  return values_p;
449 }
451 {
452  return values_p;
453 }
455 {
456  return userSetCache_p;
457 }
459 {
460  return lastColAccess_p;
461 }
462 inline const IPosition& TSMCube::getLastColSlice() const
463 {
464  return lastColSlice_p;
465 }
467 {
468  lastColAccess_p = type;
469 }
470 inline void TSMCube::setLastColSlice (const IPosition& slice)
471 {
472  lastColSlice_p.resize (slice.nelements());
473  lastColSlice_p = slice;
474 }
475 
476 
477 
478 
479 } //# NAMESPACE CASACORE - END
480 
481 #endif
Cache for buckets in a part of a file.
Definition: BucketCache.h:218
void resize(size_t newSize, bool copy=true)
Old values are copied on resize if copy==true.
size_t nelements() const
The number of elements in this IPosition.
Definition: IPosition.h:568
A drop-in replacement for Block<T*>.
Definition: Block.h:814
String: the storage and methods of handling collections of characters.
Definition: String.h:225
virtual ~TSMCube()
void extendCoordinates(const Record &coordValues, const String &coordName, uInt length)
Extend the coordinates vector for the given coordinate to the given length with the given coordValues...
uInt cacheSize() const
Get the current cache size (in buckets).
TSMFile * filePtr_p
Pointer to the TSMFile object holding the data.
Definition: TSMCube.h:384
void emptyCache()
Empty the cache.
const Record & valueRecord() const
Get the record containing the id and coordinate values.
Definition: TSMCube.h:446
void putObject(AipsIO &ios)
Put the data of the object into the AipsIO stream.
virtual void showCacheStatistics(ostream &os) const
Show the cache statistics.
IPosition lastColSlice_p
The slice shape of the last column access to a slice.
Definition: TSMCube.h:402
uInt validateCacheSize(uInt cacheSize) const
Validate the cache size (in buckets).
virtual void setCacheSize(const IPosition &sliceShape, const IPosition &windowStart, const IPosition &windowLength, const IPosition &axisPath, Bool forceSmaller, Bool userSet)
Set the cache size for the given slice and access path.
virtual void resyncCache()
Resync the cache object.
IPosition tilesPerDim_p
The number of tiles in each hypercube dimension.
Definition: TSMCube.h:374
void resizeTileSections()
Resize the IPosition member variables used in accessSection() if nrdim_p changes value.
virtual void makeCache()
Construct the cache object (if not constructed yet).
static void deleteCallBack(void *owner, char *buffer)
IPosition tileShape_p
The shape of the tiles in the hypercube.
Definition: TSMCube.h:372
IPosition endPixelInFirstTile_p
Last pixel in first tile.
Definition: TSMCube.h:416
Bool userSetCache() const
Determine if the user set the cache size (using setCacheSize).
Definition: TSMCube.h:454
Block< uInt > localOffset_p
Offset for each data column in a tile (in local format).
Definition: TSMCube.h:390
TSMCube(TiledStMan *stman, AipsIO &ios, Bool useDerived=False)
Reconstruct the hypercube by reading its data from the AipsIO stream.
uInt localTileLength() const
Get the length of a tile (in bytes) in local format.
Definition: TSMCube.h:434
void setLastColAccess(AccessType type)
Definition: TSMCube.h:466
virtual void resync(AipsIO &ios)
Resync the object with the data file.
const IPosition & tileShape() const
Get the shape of the tiles.
Definition: TSMCube.h:442
uInt tileSize_p
The tilesize in bytes.
Definition: TSMCube.h:382
IPosition adjustTileShape(const IPosition &cubeShape, const IPosition &tileShape) const
Adjust the tile shape to the hypercube shape.
TSMCube(const TSMCube &)
Forbid copy constructor.
TSMCube & operator=(const TSMCube &)
Forbid assignment.
uInt coordinateSize(const String &coordinateName) const
Get the size of a coordinate (i.e.
virtual void setShape(const IPosition &cubeShape, const IPosition &tileShape)
Set the hypercube shape.
Block< uInt > externalOffset_p
Offset for each data column in a tile (in external format).
Definition: TSMCube.h:388
virtual void extend(uInt64 nr, const Record &coordValues, const TSMColumn *lastCoordColumn)
Extend the last dimension of the cube with the given number.
static char * readCallBack(void *owner, const char *external)
Define the callback functions for the BucketCache.
void accessLine(char *section, uInt pixelOffset, uInt localPixelSize, Bool writeFlag, BucketCache *cachePtr, const IPosition &startTile, uInt endTile, const IPosition &startPixelInFirstTile, uInt endPixelInLastTile, uInt lineIndex)
Access a line in a more optimized way.
IPosition cubeShape_p
The shape of the hypercube.
Definition: TSMCube.h:370
uInt nrTiles_p
Number of tiles in the hypercube.
Definition: TSMCube.h:368
void clearCache(Bool doFlush=True)
Clear the cache, so data will be reread.
uInt nrTilesSubCube_p
Number of tiles in all but last dimension (used when extending).
Definition: TSMCube.h:380
static uInt calcCacheSize(const IPosition &cubeShape, const IPosition &tileShape, Bool extensible, const IPosition &sliceShape, const IPosition &windowStart, const IPosition &windowLength, const IPosition &axisPath, uInt maxCacheSizeMiB, uInt bucketSize)
char * readTile(const char *external)
Define the functions doing the actual read and write of the data in the tile and converting it to/fro...
TSMShape expandedTilesPerDim_p
Precomputed tilesPerDim information.
Definition: TSMCube.h:378
IPosition startPixelInFirstTile_p
First pixel in first tile.
Definition: TSMCube.h:414
void setLastColSlice(const IPosition &slice)
Definition: TSMCube.h:470
TSMCube(TiledStMan *stman, TSMFile *file, const IPosition &cubeShape, const IPosition &tileShape, const Record &values, Int64 fileOffset, Bool useDerived=False)
Construct the hypercube using the given file with the given shape.
virtual void accessStrided(const IPosition &start, const IPosition &end, const IPosition &stride, char *section, uInt colnr, uInt localPixelSize, uInt externalPixelSize, Bool writeFlag)
Read or write a section in a strided way.
AccessType lastColAccess_p
Was the last column access to a cell, slice, or column?
Definition: TSMCube.h:400
static char * initCallBack(void *owner)
IPosition endPixelInLastTile_p
Last pixel in last tile.
Definition: TSMCube.h:418
Bool matches(const PtrBlock< TSMColumn * > &idColSet, const Record &idValues)
Test if the id values match.
Bool useDerived_p
Is the class used directly or only by a derived class only?
Definition: TSMCube.h:360
BucketCache * getCache()
Get the cache object.
Definition: TSMCube.h:423
uInt nrdim_p
Dimensionality of the hypercube.
Definition: TSMCube.h:366
Int64 fileOffset_p
Offset in the TSMFile object where the data of this hypercube starts.
Definition: TSMCube.h:386
IPosition endTile_p
Last tile needed.
Definition: TSMCube.h:412
const IPosition & cubeShape() const
Get the shape of the hypercube.
Definition: TSMCube.h:438
Bool extensible_p
Is the hypercube extensible?
Definition: TSMCube.h:364
virtual void flushCache()
Flush the data in the cache.
static uInt validateCacheSize(uInt cacheSize, uInt maxSizeMiB, uInt bucketSize)
void writeTile(char *external, const char *local)
uInt bucketSize_p
The bucket size in bytes (is equal to tile size in bytes).
Definition: TSMCube.h:392
Record values_p
The values of the possible id and coordinate columns.
Definition: TSMCube.h:362
Int getObject(AipsIO &ios)
Get the data of the object from the AipsIO stream.
void setup()
Initialize the various variables.
uInt calcCacheSize(const IPosition &sliceShape, const IPosition &windowStart, const IPosition &windowLength, const IPosition &axisPath) const
Calculate the cache size (in buckets) for the given slice and access path.
virtual void deleteCache()
Delete the cache object.
TSMShape expandedTileShape_p
Precomputed tileShape information.
Definition: TSMCube.h:376
IPosition cellShape() const
Get the shape of the data cells in the cube.
static void writeCallBack(void *owner, char *external, const char *local)
AccessType
Define the possible access types for TSMDataColumn.
Definition: TSMCube.h:109
TiledStMan * stmanPtr_p
Pointer to the parent storage manager.
Definition: TSMCube.h:358
Bool userSetCache_p
Did the user set the cache size?
Definition: TSMCube.h:398
virtual void accessSection(const IPosition &start, const IPosition &end, char *section, uInt colnr, uInt localPixelSize, uInt externalPixelSize, Bool writeFlag)
Read or write a section in the cube.
Record & rwValueRecord()
Definition: TSMCube.h:450
IPosition startTile_p
First tile needed.
Definition: TSMCube.h:410
BucketCache * cache_p
The bucket cache.
Definition: TSMCube.h:396
AccessType getLastColAccess() const
Functions for TSMDataColumn to keep track of the last type of access to a hypercube.
Definition: TSMCube.h:458
virtual void setCacheSize(uInt cacheSize, Bool forceSmaller, Bool userSet)
Resize the cache object.
Bool isExtensible() const
Is the hypercube extensible?
uInt localTileLength_p
The tile size in bytes in local format.
Definition: TSMCube.h:394
const IPosition & getLastColSlice() const
Definition: TSMCube.h:462
char * cachedTile_p
Definition: TSMCube.h:355
IPosition nrTileSection_p
IPosition variables used in accessSection(); declared here as member variables to avoid significant c...
Definition: TSMCube.h:408
uInt bucketSize() const
Get the bucket size (bytes).
Definition: TSMCube.h:430
this file contains all the compiler specific defines
Definition: mainpage.dox:28
const Bool False
Definition: aipstype.h:44
unsigned int uInt
Definition: aipstype.h:51
long long Int64
Define the extra non-standard types used by Casacore (like proposed uSize, Size)
Definition: aipsxtype.h:38
LatticeExprNode length(const LatticeExprNode &expr, const LatticeExprNode &axis)
2-argument function to get the length of an axis.
int Int
Definition: aipstype.h:50
bool Bool
Define the standard types used by Casacore.
Definition: aipstype.h:42
const Bool True
Definition: aipstype.h:43
unsigned long long uInt64
Definition: aipsxtype.h:39