[netcdf-java] Variable shape problem & EOFException

[Nick Bower]

I've found a problem in which cache/memory and disk shape information 
about variables will disagree with v2.2.22 of Java Netcdf library.

When you add a new value to a variable, automatically increasing the 
length of a dimension, subsequent reads can throw EOFException because 
RandomAccessFile is instructed to read more values than the file 
contains - the cached and actual shapes disagree.

I've created a runnable test case below to explain and demonstrate 
success and failure conditions.

I am getting around this now by not interleaving read/write operations 
on variables, but instead reading all variables' data to memory, then 
performing any writes I need to after.

TestInsertRecord.java:


package com.metoceanengineers.datafeeds.netcdf.test;

import java.io.File;
import java.io.IOException;
import java.text.DateFormat;
import java.text.SimpleDateFormat;

import junit.framework.TestCase;
import ucar.ma2.Array;
import ucar.ma2.ArrayInt;
import ucar.ma2.DataType;
import ucar.nc2.Dimension;
import ucar.nc2.NetcdfFileWriteable;

public class TestInsertRecord extends TestCase {
    
    DateFormat dateFormat = new SimpleDateFormat("yyyyMMdd HHMM");
    
    protected NetcdfFileWriteable createNc(String prefix) throws 
IOException {
        
        File mainline = File.createTempFile(prefix+"-", ".nc");
        NetcdfFileWriteable mainlineNc = 
NetcdfFileWriteable.createNew(mainline.getAbsolutePath(), false);

        Dimension recordsDim = mainlineNc.addUnlimitedDimension("records");
        
        Dimension timeDims[] = {recordsDim};
        Dimension var1Dims[] = {recordsDim}; // 1D
        
        mainlineNc.addVariable("time", DataType.INT, timeDims);
        mainlineNc.addVariable("var1", DataType.INT, var1Dims);

        mainlineNc.create();
        return mainlineNc;
    }


    protected String getNcInstance() throws Exception {

        NetcdfFileWriteable mainlineNc = createNc("testfile");
        
        int[] origin = {0};
        
        ArrayInt.D1 timeArr = new ArrayInt.D1(2);
        timeArr.set(0, (int)dateFormat.parse("20071130 0924").getTime());
        timeArr.set(1, (int)dateFormat.parse("20071130 0926").getTime());
        mainlineNc.write("time", origin, timeArr);
        
        ArrayInt.D1 var1Arr = new ArrayInt.D1(2);
        var1Arr.set(0, 10);
        var1Arr.set(1, 12);
        mainlineNc.write("var1", origin, var1Arr);

        mainlineNc.close();

        return mainlineNc.getLocation();
    }
    

    /**
     * Append new data to end of existing variables.
     *
     * @throws Exception
     */
    public void testAppendWorksOk() throws Exception {
        
        String ncFilename = getNcInstance();
        NetcdfFileWriteable ncFile = 
NetcdfFileWriteable.openExisting(ncFilename, false);

        /*
         * Append value (20071130 0924, 11) into (time, var1)
         */
        
        ArrayInt.D1 newTimeValue = new ArrayInt.D1(1);
        newTimeValue.set(0, (int)dateFormat.parse("20071130 
0925").getTime());

        ArrayInt.D1 newVarValue = new ArrayInt.D1(1);
        newVarValue.set(0, 11);
        
        int[] origin = {2};

        /* The first write will expand the variables,
         * but second write ok as we're just writing
         * and not reading */
        
        ncFile.write("time", origin, newTimeValue);
        ncFile.write("var1", origin, newVarValue);
        
        assertEquals(3, ncFile.findDimension("records").getLength());
        
    }

    /**
     * Test insertion of a record in between the 2 existing
     * records by reading the existing tail, inserting new data
     * and re-appending.
     *
     * Triggers EOFException through interleaved read/writes
     *
     * @throws Exception
     */
    public void testInsertFails() throws Exception {
        
        String ncFilename = getNcInstance();
        NetcdfFileWriteable ncFile = 
NetcdfFileWriteable.openExisting(ncFilename, false);

        ArrayInt.D1 newTimeValue = new ArrayInt.D1(1);
        newTimeValue.set(0, (int)dateFormat.parse("20071130 
0925").getTime());

        ArrayInt.D1 newVarValue = new ArrayInt.D1(1);
        newVarValue.set(0, 11);
        
        /* Going to insert at 1, so read existing value,
         * write down new one, and re-append old tail.
         */
        
        int[] insertPointOrigin = {1};
        int[] appendOrigin = {2};
        int[] shape = {1};
        
        Array tailTime = 
ncFile.findVariable("time").read(insertPointOrigin, shape);
        ncFile.write("time", insertPointOrigin, newTimeValue);
        ncFile.write("time", appendOrigin, tailTime);
        
        /* Next line excepts - why?  Because the last write above at
         * records index 2 triggers an increase in the CACHED/MEMORY
         * length of all variables to 3, but on disk it's still the
         * original length 2.
         *
         *  Therefore we get EOFException.
         */
        
        Array tailVar1 = 
ncFile.findVariable("var1").read(insertPointOrigin, shape);
        ncFile.write("var1", insertPointOrigin, newVarValue);
        ncFile.write("var1", appendOrigin, tailVar1);
        
        assertEquals(3, ncFile.findDimension("records").getLength());
        
    }

}