My Project
programmer's documentation
Typedefs | Enumerations | Functions | Variables
cs_file.h File Reference
#include <mpi.h>
#include "cs_defs.h"
Include dependency graph for cs_file.h:

Go to the source code of this file.

Typedefs

typedef struct _cs_file_t cs_file_t
 
typedef struct _cs_file_serializer_t cs_file_serializer_t
 
typedef long long cs_file_off_t
 

Enumerations

enum  cs_file_mode_t { CS_FILE_MODE_READ, CS_FILE_MODE_WRITE, CS_FILE_MODE_APPEND }
 File acces modes. More...
 
enum  cs_file_seek_t { CS_FILE_SEEK_SET, CS_FILE_SEEK_CUR, CS_FILE_SEEK_END }
 seek semantics (third argument of cs_file_seek) More...
 
enum  cs_file_access_t {
  CS_FILE_DEFAULT, CS_FILE_STDIO_SERIAL, CS_FILE_STDIO_PARALLEL, CS_FILE_MPI_INDEPENDENT,
  CS_FILE_MPI_NON_COLLECTIVE, CS_FILE_MPI_COLLECTIVE
}
 Shared file access methods. More...
 
enum  cs_file_mpi_positionning_t { CS_FILE_MPI_EXPLICIT_OFFSETS, CS_FILE_MPI_INDIVIDUAL_POINTERS }
 MPI-IO positionning methods. More...
 

Functions

cs_file_tcs_file_open (const char *name, cs_file_mode_t mode, cs_file_access_t method, MPI_Info hints, MPI_Comm block_comm, MPI_Comm comm)
 Create a file descriptor and open the associated file. More...
 
cs_file_tcs_file_open_default (const char *name, cs_file_mode_t mode)
 Create a file descriptor and open the associated file, using the default file communicator and access method. More...
 
cs_file_tcs_file_open_serial (const char *name, cs_file_mode_t mode)
 Create a file descriptor and open the associated file, using the serial IO on the root rank. More...
 
cs_file_tcs_file_free (cs_file_t *f)
 Destroy a file descriptor and close the associated file. More...
 
const char * cs_file_get_name (const cs_file_t *f)
 Return a file's name. More...
 
void cs_file_set_big_endian (cs_file_t *f)
 Ensure that data is read or written in big-endian (network standard) format. More...
 
int cs_file_get_swap_endian (const cs_file_t *f)
 Return a file's byte-swapping behavior. More...
 
void cs_file_set_swap_endian (cs_file_t *f, int swap)
 Set a file's byte-swapping behavior. More...
 
size_t cs_file_read_global (cs_file_t *f, void *buf, size_t size, size_t ni)
 Read global data from a file, distributing it to all processes associated with that file. More...
 
size_t cs_file_write_global (cs_file_t *f, const void *buf, size_t size, size_t ni)
 Write global data to a file. More...
 
size_t cs_file_read_block (cs_file_t *f, void *buf, size_t size, size_t stride, cs_gnum_t global_num_start, cs_gnum_t global_num_end)
 Read data to a buffer, distributing a contiguous part of it to each process associated with a file. More...
 
size_t cs_file_write_block (cs_file_t *f, const void *buf, size_t size, size_t stride, cs_gnum_t global_num_start, cs_gnum_t global_num_end)
 Write data to a file, each associated process providing a contiguous part of this data. More...
 
size_t cs_file_write_block_buffer (cs_file_t *f, void *buf, size_t size, size_t stride, cs_gnum_t global_num_start, cs_gnum_t global_num_end)
 Write data to a file, each associated process providing a contiguous part of this data. More...
 
int cs_file_seek (cs_file_t *f, cs_file_off_t offset, cs_file_seek_t whence)
 Update the file pointer according to whence. More...
 
cs_file_off_t cs_file_tell (cs_file_t *f)
 Return the position of the file pointer. More...
 
void cs_file_dump (const cs_file_t *f)
 Dump the metadata of a file structure in human readable form. More...
 
void cs_file_free_defaults (void)
 Free the default options for file access. More...
 
void cs_file_get_default_access (cs_file_mode_t mode, cs_file_access_t *method, MPI_Info *hints)
 Get the default options for file access. More...
 
void cs_file_set_default_access (cs_file_mode_t mode, cs_file_access_t method, MPI_Info hints)
 Set the default options for file access. More...
 
void cs_file_get_default_comm (int *block_rank_step, int *block_min_size, MPI_Comm *block_comm, MPI_Comm *comm)
 Get default MPI communicator values for file access. More...
 
void cs_file_set_default_comm (int block_rank_step, int block_min_size, MPI_Comm comm)
 Set default MPI communicator values for file access. More...
 
MPI_Comm cs_file_block_comm (int block_rank_step, MPI_Comm comm)
 Create an MPI communicator for distributed block parallel IO. More...
 
cs_file_mpi_positionning_t cs_file_get_mpi_io_positionning (void)
 Get the positionning method for MPI-IO. More...
 
void cs_file_set_mpi_io_positionning (cs_file_mpi_positionning_t positionning)
 Set the positionning method for MPI-IO. More...
 
void cs_file_defaults_info (void)
 Print information on default options for file access. More...
 
cs_file_serializer_tcs_file_serializer_create (size_t size, size_t stride, cs_gnum_t global_num_start, cs_gnum_t global_num_end, size_t buf_block_size, void *buf, MPI_Comm comm)
 Create a cs_file_serializer_t structure. More...
 
void cs_file_serializer_destroy (cs_file_serializer_t **s)
 Destroy a cs_file_serializer_t structure. More...
 
void * cs_file_serializer_advance (cs_file_serializer_t *s, cs_gnum_t cur_range[2])
 Advance a cs_file_serializer_t structure. More...
 
int cs_file_mkdir_default (const char *path)
 Create a new directory using default permissions. More...
 
int cs_file_isreg (const char *path)
 Check if a file exists and is a regular file. More...
 
int cs_file_isdir (const char *path)
 Check if a directory exists. More...
 
char ** cs_file_listdir (const char *path)
 List files inside a directory. More...
 
cs_file_off_t cs_file_size (const char *path)
 Return the size of a file. More...
 
int cs_file_remove (const char *path)
 Remove a file if it exists and is a regular file. More...
 

Variables

const char * cs_file_access_name []
 
const char * cs_file_mpi_positionning_name []
 

Typedef Documentation

◆ cs_file_off_t

Offset for file position indicator

◆ cs_file_serializer_t

typedef struct _cs_file_serializer_t cs_file_serializer_t

◆ cs_file_t

File descriptor (opaque object)

Enumeration Type Documentation

◆ cs_file_access_t

Shared file access methods.

Enumerator
CS_FILE_DEFAULT 
CS_FILE_STDIO_SERIAL 

Default IO option

Serial standard C IO (funnelled through rank 0 in parallel)

CS_FILE_STDIO_PARALLEL 

Per-process standard C IO (for reading only)

CS_FILE_MPI_INDEPENDENT 

Non-collective MPI-IO with independent file open and close (for reading only)

CS_FILE_MPI_NON_COLLECTIVE 

Non-collective MPI-IO with collective file open and close

CS_FILE_MPI_COLLECTIVE 

Collective MPI-IO

◆ cs_file_mode_t

File acces modes.

Enumerator
CS_FILE_MODE_READ 

Read mode

CS_FILE_MODE_WRITE 

Write mode

CS_FILE_MODE_APPEND 

Append

Abort immediately

◆ cs_file_mpi_positionning_t

MPI-IO positionning methods.

It is not always known whether a performance or robustness difference is to be expected using explicit file offsets or individual file pointers. Perusal of a sampling of ROMIO code would seem to indicate that no difference is to be expected, but this might change with MPI IO variants or file systems, so an advanced setting is made possible.

Enumerator
CS_FILE_MPI_EXPLICIT_OFFSETS 

Use explicit offsets positionning with MPI-IO

CS_FILE_MPI_INDIVIDUAL_POINTERS 

Use individual file pointer positionning with MPI-IO

◆ cs_file_seek_t

seek semantics (third argument of cs_file_seek)

Enumerator
CS_FILE_SEEK_SET 

Seek from beginning of file

CS_FILE_SEEK_CUR 

Seek from current position

CS_FILE_SEEK_END 

Seek from end of file

Function Documentation

◆ cs_file_block_comm()

MPI_Comm cs_file_block_comm ( int  block_rank_step,
MPI_Comm  comm 
)

Create an MPI communicator for distributed block parallel IO.

Parameters
[in]block_rank_stepMPI rank stepping between non-empty blocks
[in]commHandle to main MPI communicator
Returns
communicator associated with IO, MPI_COMM_NULL for ranks not participating in parallel IO (including ranks participating in IO where communicator size would be 1)

◆ cs_file_defaults_info()

void cs_file_defaults_info ( void  )

Print information on default options for file access.

◆ cs_file_dump()

void cs_file_dump ( const cs_file_t f)

Dump the metadata of a file structure in human readable form.

Parameters
[in]fcs_file_t descriptor

◆ cs_file_free()

cs_file_t* cs_file_free ( cs_file_t f)

Destroy a file descriptor and close the associated file.

Parameters
[in,out]ffile descriptor to destroy

◆ cs_file_free_defaults()

void cs_file_free_defaults ( void  )

Free the default options for file access.

◆ cs_file_get_default_access()

void cs_file_get_default_access ( cs_file_mode_t  mode,
cs_file_access_t method,
MPI_Info hints 
)

Get the default options for file access.

Parameters
[in]modefile mode for which the default is queried (write and append use the same method, and are interchangeable here)
[out]methoddefault file access method, or NULL
[out]hintsMPI-IO hints, or NULL

◆ cs_file_get_default_comm()

void cs_file_get_default_comm ( int *  block_rank_step,
int *  block_min_size,
MPI_Comm *  block_comm,
MPI_Comm *  comm 
)

Get default MPI communicator values for file access.

A block rank stepping value may be used, allowing the use of a reduced communicator for distributed block reads and writes. If this value is greater than 1, ranks not a multiple of this step must be guaranteed to be empty for block reads and writes with files opened using this default.

A minimum block size target may also be used, so as to limit the number of active blocks to a value proportional to the data size (limiting latency issues for small data sets, while not requiring too much local memory).

Parameters
[out]block_rank_stepMPI rank stepping between non-empty distributed blocks, or NULL
[out]block_min_sizeminimum block size target for non-empty distributed blocks, or NULL
[out]block_commHandle to MPI communicator used for distributed file block access, or NULL
[out]commHandle to main MPI communicator, or NULL

◆ cs_file_get_mpi_io_positionning()

cs_file_mpi_positionning_t cs_file_get_mpi_io_positionning ( void  )

Get the positionning method for MPI-IO.

For details, see cs_file_set_mpi_io_positionning.

Returns
positionning method for MPI-IO

◆ cs_file_get_name()

const char* cs_file_get_name ( const cs_file_t f)

Return a file's name.

Parameters
[in]fcs_file_t descriptor
Returns
pointer to the file's name.

◆ cs_file_get_swap_endian()

int cs_file_get_swap_endian ( const cs_file_t f)

Return a file's byte-swapping behavior.

Parameters
[in]fcs_file_t descriptor
Returns
0 if file's endianness is the same as the system's, 1 otherwise.

◆ cs_file_isdir()

int cs_file_isdir ( const char *  path)

Check if a directory exists.

Parameters
[in]pathdirectory path.
Returns
1 if directory exists, 0 otherwise.

◆ cs_file_isreg()

int cs_file_isreg ( const char *  path)

Check if a file exists and is a regular file.

Parameters
[in]pathfile path.
Returns
1 if file exists and is a regular file, 0 otherwise.

◆ cs_file_listdir()

char** cs_file_listdir ( const char *  path)

List files inside a directory.

The array returned must be freed by the caller using BFT_FREE, as well as the individual entries in the array.

Parameters
[in]pathname of directory.
Returns
an array of file names in a directory. The last entry is set to NULL. If no means to list the directory or an error occured, the return value is simply NULL.

◆ cs_file_mkdir_default()

int cs_file_mkdir_default ( const char *  path)

Create a new directory using default permissions.

This function is similar to the POSIX function mkdir(), except that it has no "mode" argument: by default, on a POSIX type system, permissions include read, write, and execute access for the user, group and others, modified by the users umask value (so with a typical configuration, the user will have read, write, and execute pemission, the group and others will only have read and execute permission, but this behavior may be modified).

Also, contrary to the usual mkdir(), if the directory already exists (and is truly a directory), this is considered a success and not a failure, and 0 is returned: the aim of this function is to make a directory available, so if it already exists, this is considered acceptable.

Parameters
[in]pathname of new directory.
Returns
0 on success, -1 if an error occured (in which case errno contains the appropriate error code). If the underlying system has no mkdir() function or it was not detected upon BFT configuration, 1 is returned.

◆ cs_file_open()

cs_file_t* cs_file_open ( const char *  name,
cs_file_mode_t  mode,
cs_file_access_t  method,
MPI_Info  hints,
MPI_Comm  block_comm,
MPI_Comm  comm 
)

Create a file descriptor and open the associated file.

By default, data is written or read as native data. This behavior may be modified by cs_file_set_swap_endian().

Parameters
[in]namefile name
[in]modefile acces mode: read, write, or append
[in]methodfile access method
[in]hintsassociated hints for MPI-IO, or MPI_INFO_NULL
[in]block_commhandle to MPI communicator used for distributed file block access (may be a subset of comm if some ranks do not directly access distributed data blocks)
[in]commhandle to main MPI communicator
Returns
pointer to cs_file_t file descriptor (NULL in case of failure); currently, errors are fatal.

◆ cs_file_open_default()

cs_file_t* cs_file_open_default ( const char *  name,
cs_file_mode_t  mode 
)

Create a file descriptor and open the associated file, using the default file communicator and access method.

By default, data is written or read as native data. This behavior may be modified by cs_file_set_swap_endian().

Parameters
[in]namefile name
[in]modefile acces mode: read, write, or append
Returns
pointer to cs_file_t file descriptor (NULL in case of failure); currently, errors are fatal.

◆ cs_file_open_serial()

cs_file_t* cs_file_open_serial ( const char *  name,
cs_file_mode_t  mode 
)

Create a file descriptor and open the associated file, using the serial IO on the root rank.

By default, data is written or read as native data. This behavior may be modified by cs_file_set_swap_endian().

Parameters
[in]namefile name
[in]modefile acces mode: read, write, or append
Returns
pointer to cs_file_t file descriptor (NULL in case of failure); currently, errors are fatal.

◆ cs_file_read_block()

size_t cs_file_read_block ( cs_file_t f,
void *  buf,
size_t  size,
size_t  stride,
cs_gnum_t  global_num_start,
cs_gnum_t  global_num_end 
)

Read data to a buffer, distributing a contiguous part of it to each process associated with a file.

Each process should receive a (possibly empty) block of the data, and we should have: global_num_start at rank 0 = 1 global_num_start at rank i+1 = global_num_end at rank i. Otherwise, behavior (especially positioning for future reads) is undefined.

Parameters
[in]fcs_file_t descriptor
[out]bufpointer to location receiving data
[in]sizesize of each item of data in bytes
[in]stridenumber of (interlaced) values per block item
[in]global_num_startglobal number of first block item (1 to n numbering)
[in]global_num_endglobal number of past-the end block item (1 to n numbering)
Returns
the (local) number of items (not bytes) sucessfully read; currently, errors are fatal.

◆ cs_file_read_global()

size_t cs_file_read_global ( cs_file_t f,
void *  buf,
size_t  size,
size_t  ni 
)

Read global data from a file, distributing it to all processes associated with that file.

Parameters
[in]fcs_file_t descriptor
[out]bufpointer to location receiving data
[in]sizesize of each item of data in bytes
[in]ninumber of items to read
Returns
the number of items (not bytes) sucessfully read; currently, errors are fatal.

◆ cs_file_remove()

int cs_file_remove ( const char *  path)

Remove a file if it exists and is a regular file.

Parameters
[in]pathfile path.
Returns
0 in case of success or if file does not exist, 0 otherwise.

◆ cs_file_seek()

int cs_file_seek ( cs_file_t f,
cs_file_off_t  offset,
cs_file_seek_t  whence 
)

Update the file pointer according to whence.

Parameters
[in,out]fcs_file_t descriptor
[in]offsetadd to position specified to whence to obtain new position, measured in characters from the beginning of the file
[in]whencebeginning if CS_FILE_SEEK_SET, current if CS_FILE_SEEK_CUR, or end-of-file if CS_FILE_SEEK_END
Returns
0 upon success, nonzero otherwise; currently, errors are fatal.

◆ cs_file_serializer_advance()

void* cs_file_serializer_advance ( cs_file_serializer_t s,
cs_gnum_t  cur_range[2] 
)

Advance a cs_file_serializer_t structure.

Data from the buffer of the next communicating rank is copied to rank 0 (this is a no-op the first time this function is called, as rank 0 already has its data).

On rank 0, the return value may point to the buffer defined when initializing the serializer, or to an aditional buffer if the former is too small to receive data from all ranks.

Note also that for ranks > 0, this function always returns NULL, as only one call is needed for those ranks.

Parameters
[in]spointer to serializer structure
[out]cur_rangeoptional start and past-the end global numbers for the current block (size: 2), or NULL; only on rank 0
Returns
a pointer to the buffer containing new data (first call counts as new), or NULL if we are finished; always NULL on ranks > 0.

◆ cs_file_serializer_create()

cs_file_serializer_t* cs_file_serializer_create ( size_t  size,
size_t  stride,
cs_gnum_t  global_num_start,
cs_gnum_t  global_num_end,
size_t  buf_block_size,
void *  buf,
MPI_Comm  comm 
)

Create a cs_file_serializer_t structure.

The buf_block_size argument is optional, and may be used when the buffer on rank 0 is larger than (global_num_end - global_num_start)*size*stride bytes. If zero, a block size of (global_num_end - global_num_start) on rank 0 is assumed; a buffer may not be smaller than this, as it must initially contain all data on rank 0's block.

Parameters
[in]sizesize of each item of data in bytes
[in]stridenumber of (interlaced) values per block item
[in]global_num_startglobal number of first block item (1 to n numbering)
[in]global_num_endglobal number of past-the end block item (1 to n numbering)
[in]buf_block_sizeLocal data buffer block size, or 0 for default global_num_end - global_num_start (only useful on rank 0)
[in]bufpointer to local block data buffer
[in]commassociated MPI communicator
Returns
pointer to new serializer structure.

◆ cs_file_serializer_destroy()

void cs_file_serializer_destroy ( cs_file_serializer_t **  s)

Destroy a cs_file_serializer_t structure.

Parameters
[in,out]spointer to pointer structure that should be destroyed

◆ cs_file_set_big_endian()

void cs_file_set_big_endian ( cs_file_t f)

Ensure that data is read or written in big-endian (network standard) format.

Parameters
[in,out]fcs_file_t descriptor

◆ cs_file_set_default_access()

void cs_file_set_default_access ( cs_file_mode_t  mode,
cs_file_access_t  method,
MPI_Info  hints 
)

Set the default options for file access.

If the method given contains incompatible values, such as when setting MPI-IO methods when MPI-IO is not available, a "reasonable" default is used instead.

Parameters
[in]modefile mode for which the default is being set (write and append use the same method, and are interchangeable here)
[in]methoddefault access method to set
[in]hintsMPI-IO hints, or MPI_INFO_NULL

◆ cs_file_set_default_comm()

void cs_file_set_default_comm ( int  block_rank_step,
int  block_min_size,
MPI_Comm  comm 
)

Set default MPI communicator values for file access.

A block rank stepping value may be used, allowing the use of a reduced communicator for distributed block reads and writes. If this value is greater than 1, ranks not a multiple of this step must be guaranteed to be empty for block reads and writes with files opened using this default.

A minimum block size target may also be used, so as to limit the number of active blocks to a value proportional to the data size (limiting latency issues for small data sets, while not requiring too much local memory).

For each argument, an "out of range" value may be used to avoid modifying the previous default for that argument.

Parameters
[in]block_rank_stepMPI rank stepping between non-empty blocks for file block reads and writes (not set if <= 0)
[in]block_min_sizeminimum block size target for non-empty distributed blocks (not set if < 1)
[in]commHandle to main MPI communicator (not set if MPI_COMM_SELF)

◆ cs_file_set_mpi_io_positionning()

void cs_file_set_mpi_io_positionning ( cs_file_mpi_positionning_t  positionning)

Set the positionning method for MPI-IO.

It is not always known whether a performance or robustness difference is to be expected using explicit file offsets or individual file pointers. Perusal of a sampling of ROMIO code would seem to indicate that no difference is to be expected, but this might change with MPI IO variants or file systems, so this advanced setting is made possible.

This setting is not available on a per-file basis, though this could be done in the future in the unexpected case of performance results showing this would be useful.

Parameters
[in]positionningchosen positionning method for MPI-IO

◆ cs_file_set_swap_endian()

void cs_file_set_swap_endian ( cs_file_t f,
int  swap 
)

Set a file's byte-swapping behavior.

Parameters
[in,out]fcs_file_t descriptor
[in]swap1 if bytes must be swapped, 0 otherwise

◆ cs_file_size()

cs_file_off_t cs_file_size ( const char *  path)

Return the size of a file.

If the file does not exist, 0 is returned.

Note also that for some special files, such as files in the Linux /proc directory, this may return 0.

Parameters
[in]pathfile path.
Returns
size of file.

◆ cs_file_tell()

cs_file_off_t cs_file_tell ( cs_file_t f)

Return the position of the file pointer.

In parallel, we consider the file pointer to be equal to the highest value of the individual file pointers.

Parameters
[in]fcs_file_t descriptor
Returns
current position of the file pointer.

◆ cs_file_write_block()

size_t cs_file_write_block ( cs_file_t f,
const void *  buf,
size_t  size,
size_t  stride,
cs_gnum_t  global_num_start,
cs_gnum_t  global_num_end 
)

Write data to a file, each associated process providing a contiguous part of this data.

Each process should provide a (possibly empty) block of the data, and we should have: global_num_start at rank 0 = 1 global_num_start at rank i+1 = global_num_end at rank i. Otherwise, behavior (especially positioning for future reads) is undefined.

This function may require an internal copy of the data to ensure that the buffer contents are not modified, so if the buffer contents are temporary values, to be deleted after writing, using cs_file_write_block_buffer() instead may be used to avoid an unneeded memory allocation and copy.

Parameters
[in]fcs_file_t descriptor
[in]bufpointer to location containing data
[in]sizesize of each item of data in bytes
[in]stridenumber of (interlaced) values per block item
[in]global_num_startglobal number of first block item (1 to n numbering)
[in]global_num_endglobal number of past-the end block item (1 to n numbering)
Returns
the (local) number of items (not bytes) sucessfully written; currently, errors are fatal.

◆ cs_file_write_block_buffer()

size_t cs_file_write_block_buffer ( cs_file_t f,
void *  buf,
size_t  size,
size_t  stride,
cs_gnum_t  global_num_start,
cs_gnum_t  global_num_end 
)

Write data to a file, each associated process providing a contiguous part of this data.

Each process should provide a (possibly empty) block of the data, and we should have: global_num_start at rank 0 = 1 global_num_start at rank i+1 = global_num_end at rank i. Otherwise, behavior (especially positioning for future reads) is undefined.

This function is intended to be used mainly data that is already a copy of original data (such as data that has been redistributed across processors just for the sake of output), or that is to be deleted after writing, so it may modify the values in its input buffer (notably to convert from little-endian to big-endian of vice-versa if necessary).

Parameters
[in]fcs_file_t descriptor
[in,out]bufpointer to location containing data
[in]sizesize of each item of data in bytes
[in]stridenumber of (interlaced) values per block item
[in]global_num_startglobal number of first block item (1 to n numbering)
[in]global_num_endglobal number of past-the end block item (1 to n numbering)
Returns
the (local) number of items (not bytes) sucessfully written; currently, errors are fatal.

◆ cs_file_write_global()

size_t cs_file_write_global ( cs_file_t f,
const void *  buf,
size_t  size,
size_t  ni 
)

Write global data to a file.

Under MPI, data is only written by the associated communicator's root rank. The buffers on other ranks are ignored, though the file offset is updated (i.e. the call to this function is collective).

Parameters
[in]fcs_file_t descriptor
[in]bufpointer to location containing data
[in]sizesize of each item of data in bytes
[in]ninumber of items to write
Returns
the number of items (not bytes) sucessfully written; currently, errors are fatal.

Variable Documentation

◆ cs_file_access_name

const char* cs_file_access_name[]

◆ cs_file_mpi_positionning_name

const char* cs_file_mpi_positionning_name[]