Unidata

2.2 Parallel Access for NetCDF Files

To use parallel access, open or create the file with nc_open_par (see nc_open_par) or nc_create_par (see nc_create_par).

The mode flag NC_PNETCDF will be automatically turned on for classic or 64-bit offset files opened or created with the parallel access functions.

The following example shows the creation of a file using parallel access with a netCDF-4/HDF5 file, and how a program might write data to such a file.

     #include "netcdf.h"
     #include <mpi.h>
     #include <assert.h>
     #include "hdf5.h"
     #include <string.h>
     #include <stdlib.h>
     
     #define BAIL(e) do { \
     printf("Bailing out in file %s, line %d, error:%s.\n", __FILE__, __LINE__, nc_strerror(e)); \
     return e; \
     } while (0)
     
     #define FILE "test_par.nc"
     #define NDIMS 2
     #define DIMSIZE 24
     #define QTR_DATA (DIMSIZE*DIMSIZE/4)
     #define NUM_PROC 4
     
     int
     main(int argc, char **argv)
     {
         /* MPI stuff. */
         int mpi_namelen;
         char mpi_name[MPI_MAX_PROCESSOR_NAME];
         int mpi_size, mpi_rank;
         MPI_Comm comm = MPI_COMM_WORLD;
         MPI_Info info = MPI_INFO_NULL;
     
         /* Netcdf-4 stuff. */
         int ncid, v1id, dimids[NDIMS];
         size_t start[NDIMS], count[NDIMS];
     
         int data[DIMSIZE*DIMSIZE], j, i, res;
     
         /* Initialize MPI. */
         MPI_Init(&argc,&argv);
         MPI_Comm_size(MPI_COMM_WORLD, &mpi_size);
         MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);
         MPI_Get_processor_name(mpi_name, &mpi_namelen);
         printf("mpi_name: %s size: %d rank: %d\n", mpi_name,
                mpi_size, mpi_rank);
     
         /* Create a parallel netcdf-4 file. */
         if ((res = nc_create_par(FILE, NC_NETCDF4|NC_MPIIO, comm,
                                  info, &ncid)))
             BAIL(res);
     
         /* Create two dimensions. */
         if ((res = nc_def_dim(ncid, "d1", DIMSIZE, dimids)))
            BAIL(res);
         if ((res = nc_def_dim(ncid, "d2", DIMSIZE, &dimids[1])))
            BAIL(res);
     
         /* Create one var. */
         if ((res = nc_def_var(ncid, "v1", NC_INT, NDIMS, dimids, &v1id)))
            BAIL(res);
     
         if ((res = nc_enddef(ncid)))
            BAIL(res);
     
         /* Set up slab for this process. */
         start[0] = mpi_rank * DIMSIZE/mpi_size;
         start[1] = 0;
         count[0] = DIMSIZE/mpi_size;
         count[1] = DIMSIZE;
         printf("mpi_rank=%d start[0]=%d start[1]=%d count[0]=%d count[1]=%d\n",
                mpi_rank, start[0], start[1], count[0], count[1]);
     
         /* Create phony data. We're going to write a 24x24 array of ints,
            in 4 sets of 144. */
         printf("mpi_rank*QTR_DATA=%d (mpi_rank+1)*QTR_DATA-1=%d\n",
                mpi_rank*QTR_DATA, (mpi_rank+1)*QTR_DATA);
         for (i=mpi_rank*QTR_DATA; i<(mpi_rank+1)*QTR_DATA; i++)
            data[i] = mpi_rank;
     
         /*if ((res = nc_var_par_access(ncid, v1id, NC_COLLECTIVE)))
           BAIL(res);*/
         if ((res = nc_var_par_access(ncid, v1id, NC_INDEPENDENT)))
            BAIL(res);
     
         /* Write slabs of phony data. */
         if ((res = nc_put_vara_int(ncid, v1id, start, count,
                                    &data[mpi_rank*QTR_DATA])))
            BAIL(res);
     
         /* Close the netcdf file. */
         if ((res = nc_close(ncid)))
             BAIL(res);
     
         /* Shut down MPI. */
         MPI_Finalize();
     
         return 0;
     }