GlobalMpiInformation Class Reference

Class to store global MPI information and to prevent accidental changes. More...

#include <globalmpiinfo.h>

Public Member Functions
void	init (const MInt domainId, const MInt noDomains)

Private Member Functions
void	initMPIInformation ()

Private Attributes
MInt	m_globalDomainId = 0
MInt	m_globalNoDomains = 1
MPI_Info	m_mpiInfo = MPI_INFO_NULL

Friends
MInt	globalDomainId ()
	Return global domain id. More...
MInt	globalNoDomains ()
	Return global number of domains. More...
const MPI_Info &	globalMpiInfo ()
	Return global MPI information. More...

Detailed Description

Definition at line 22 of file globalmpiinfo.h.

Member Function Documentation

init()

void GlobalMpiInformation::init ( const MInt  domainId, const MInt  noDomains  )

inline

Definition at line 24 of file globalmpiinfo.h.

                                                       {
    m_globalDomainId = domainId;
    m_globalNoDomains = noDomains;
 
    initMPIInformation();
  }

initMPIInformation()

void GlobalMpiInformation::initMPIInformation ( )

inlineprivate

Definition at line 32 of file globalmpiinfo.h.

                            {
    MPI_Info_create(&m_mpiInfo, AT_);
 
    // Set header align size to 10KB for netCDF files. Allows to append header data without the need
    // to move all variable data if the header size is exceeded (which may cause MPI I/O errors).
    // Source: https://trac.mcs.anl.gov/projects/parallel-netcdf/wiki/VariableAlignment
    MPI_Info_set(m_mpiInfo, "nc_header_align_size", "10240");
    // Note: possibility to set variable align size
    /* MPI_Info_set(m_mpiInfo, "nc_var_align_size", "4194304"); */
 
#if !defined(WITH_HDF5) && defined(MPI_IO_OPT) && defined(HOST_HAZELHEN)
    // taken from Cray Wiki: https://wickie.hlrs.de/platforms/index.php/MPI-IO,
    // see also PNetcdf documentation: http://trac.mcs.anl.gov/projects/parallel-netcdf/wiki/HintsForPnetcdf
    if(m_globalNoDomains > 256) {
      MPI_Info_set(m_mpiInfo, (char*)"cb_align", (char*)"2");             /* Default: OMPI: none, CrayMPT: 2 */
      MPI_Info_set(m_mpiInfo, (char*)"cb_nodes_list", (char*)"*:*");      /* Default: OMPI: *:1, CrayMPT: *:* */
      MPI_Info_set(m_mpiInfo, (char*)"direct_io", (char*)"false");        /* Default: OMPI: none, CrayMPT: false */
      MPI_Info_set(m_mpiInfo, (char*)"romio_ds_read", (char*)"disable");  /* Default: OMPI: none, CrayMPT: disable */
      MPI_Info_set(m_mpiInfo, (char*)"romio_ds_write", (char*)"disable"); /* Default: OMPI: none, CrayMPT: disable */
      /* Let's reduce the number of aggregators, should be roughly 2 to 4 times the stripe-factor */
      // MPI_Info_set (m_mpiInfo, (char*)"cb_nodes", (char*)"8");
      /* Default: OMPI: set automatically to the number of distinct nodes; However TOO High */
 
      MPI_Info_set(m_mpiInfo, (char*)"ind_wr_buffer_size", (char*)"16777216");
      /* proposed by PNetcdf documentation */
      MPI_Info_set(m_mpiInfo, (char*)"striping_factor", (char*)"64");
      /* no. of I/O devices across which the file should be striped */
      MPI_Info_set(m_mpiInfo, (char*)"cb_nodes", (char*)"128");
    }
#endif
 
#if defined(MPI_IO_OPT) && defined(HOST_Hawk)
    if(m_globalNoDomains > 10000) { // TODO labels:HAWK,IO
      // NOTE: PNetcdf memory issue for large scale simulations. During the pnetcdf write call a significant amount of
      // memory is allocated (at least on Hawk; scales linear with noDomains), which is not freed thereafter.
      // Setting these romio hints solves the memory allocation problem, however it is not clear if this was responsible
      // for some incomplete written data files.
      // To be able to check for erroneous files you can enabled the fill mode for PNetcdf in config.h with
      // MAIA_NCMPI_FILL_VARIABLES = true
      // and check your files for fill values in the data (which should have been overwritten).
      MPI_Info_set(m_mpiInfo, (char*)"romio_cb_read", (char*)"disable");
      MPI_Info_set(m_mpiInfo, (char*)"romio_cb_write", (char*)"disable");
      if(m_globalDomainId == 0) {
        std::cerr << std::endl
                  << std::endl
                  << "NOTE: disabling ROMIO hints romio_cb_read/write to avoid PNetcdf/Hdf5 memory allocation issues "
                     "on HAWK... "
                  << std::endl
                  << "NOTE: see comment at " << AT_ << std::endl
                  << "NOTE: undefine MPI_IO_OPT to turn off the ROMIO hint changes." << std::endl
                  << std::endl
                  << std::endl;
      }
 
      // TODO labels:HAWK,IO check if it makes sense to disable these
      // MPI_Info_set(m_mpiInfo, (char*)"romio_ds_read", (char*)"disable");
      // MPI_Info_set(m_mpiInfo, (char*)"romio_ds_write", (char*)"disable");
    }
#endif
 
#ifdef MPI_IO_PRINT_INFO
    // Print MPI information on global rank 0
    if(m_globalDomainId == 0) {
      std::cerr << std::endl << "Global MPI information" << std::endl;
      printMpiInfo(m_mpiInfo);
    }
#endif
  }

Friends And Related Function Documentation

globalDomainId

MInt globalDomainId ( )

friend

Definition at line 113 of file globalmpiinfo.h.

{ return g_mpiInformation.m_globalDomainId; }

globalMpiInfo

const MPI_Info & globalMpiInfo ( )

friend

Definition at line 117 of file globalmpiinfo.h.

{ return g_mpiInformation.m_mpiInfo; }

globalNoDomains

MInt globalNoDomains ( )

friend

Definition at line 115 of file globalmpiinfo.h.

{ return g_mpiInformation.m_globalNoDomains; }

Member Data Documentation

m_globalDomainId

MInt GlobalMpiInformation::m_globalDomainId = 0

private

Definition at line 105 of file globalmpiinfo.h.

m_globalNoDomains

MInt GlobalMpiInformation::m_globalNoDomains = 1

private

Definition at line 106 of file globalmpiinfo.h.

m_mpiInfo

MPI_Info GlobalMpiInformation::m_mpiInfo = MPI_INFO_NULL

private

Definition at line 107 of file globalmpiinfo.h.

---

The documentation for this class was generated from the following file:

• /home/gitlab-runner/scratch/builds/oxpnswJ6/1/aia/m-AIA/m-AIA/src/COMM/globalmpiinfo.h