[netcdf-java] Variable shape problem & EOFException

[Nick Bower]

Hello.  Is the lack of response here because this is not considered a 
bug?  Is there a another recommended way perhaps to safely increase a 
dimension and invalidate the memory cache in the process?

Nick


Nick Bower wrote:
> I've found a problem in which cache/memory and disk shape information 
> about variables will disagree with v2.2.22 of Java Netcdf library.
>
> When you add a new value to a variable, automatically increasing the 
> length of a dimension, subsequent reads can throw EOFException because 
> RandomAccessFile is instructed to read more values than the file 
> contains - the cached and actual shapes disagree.
>
> I've created a runnable test case below to explain and demonstrate 
> success and failure conditions.
>
> I am getting around this now by not interleaving read/write operations 
> on variables, but instead reading all variables' data to memory, then 
> performing any writes I need to after.
>
> TestInsertRecord.java:
>
>
> package com.metoceanengineers.datafeeds.netcdf.test;
>
> import java.io.File;
> import java.io.IOException;
> import java.text.DateFormat;
> import java.text.SimpleDateFormat;
>
> import junit.framework.TestCase;
> import ucar.ma2.Array;
> import ucar.ma2.ArrayInt;
> import ucar.ma2.DataType;
> import ucar.nc2.Dimension;
> import ucar.nc2.NetcdfFileWriteable;
>
> public class TestInsertRecord extends TestCase {
>       DateFormat dateFormat = new SimpleDateFormat("yyyyMMdd HHMM");
>       protected NetcdfFileWriteable createNc(String prefix) throws 
> IOException {
>               File mainline = File.createTempFile(prefix+"-", ".nc");
>        NetcdfFileWriteable mainlineNc = 
> NetcdfFileWriteable.createNew(mainline.getAbsolutePath(), false);
>
>        Dimension recordsDim = 
> mainlineNc.addUnlimitedDimension("records");
>               Dimension timeDims[] = {recordsDim};
>        Dimension var1Dims[] = {recordsDim}; // 1D
>               mainlineNc.addVariable("time", DataType.INT, timeDims);
>        mainlineNc.addVariable("var1", DataType.INT, var1Dims);
>
>        mainlineNc.create();
>        return mainlineNc;
>    }
>
>
>    protected String getNcInstance() throws Exception {
>
>        NetcdfFileWriteable mainlineNc = createNc("testfile");
>               int[] origin = {0};
>               ArrayInt.D1 timeArr = new ArrayInt.D1(2);
>        timeArr.set(0, (int)dateFormat.parse("20071130 0924").getTime());
>        timeArr.set(1, (int)dateFormat.parse("20071130 0926").getTime());
>        mainlineNc.write("time", origin, timeArr);
>               ArrayInt.D1 var1Arr = new ArrayInt.D1(2);
>        var1Arr.set(0, 10);
>        var1Arr.set(1, 12);
>        mainlineNc.write("var1", origin, var1Arr);
>
>        mainlineNc.close();
>
>        return mainlineNc.getLocation();
>    }
>   
>    /**
>     * Append new data to end of existing variables.
>     *
>     * @throws Exception
>     */
>    public void testAppendWorksOk() throws Exception {
>               String ncFilename = getNcInstance();
>        NetcdfFileWriteable ncFile = 
> NetcdfFileWriteable.openExisting(ncFilename, false);
>
>        /*
>         * Append value (20071130 0924, 11) into (time, var1)
>         */
>               ArrayInt.D1 newTimeValue = new ArrayInt.D1(1);
>        newTimeValue.set(0, (int)dateFormat.parse("20071130 
> 0925").getTime());
>
>        ArrayInt.D1 newVarValue = new ArrayInt.D1(1);
>        newVarValue.set(0, 11);
>               int[] origin = {2};
>
>        /* The first write will expand the variables,
>         * but second write ok as we're just writing
>         * and not reading */
>               ncFile.write("time", origin, newTimeValue);
>        ncFile.write("var1", origin, newVarValue);
>               assertEquals(3, 
> ncFile.findDimension("records").getLength());
>           }
>
>    /**
>     * Test insertion of a record in between the 2 existing
>     * records by reading the existing tail, inserting new data
>     * and re-appending.
>     *
>     * Triggers EOFException through interleaved read/writes
>     *
>     * @throws Exception
>     */
>    public void testInsertFails() throws Exception {
>               String ncFilename = getNcInstance();
>        NetcdfFileWriteable ncFile = 
> NetcdfFileWriteable.openExisting(ncFilename, false);
>
>        ArrayInt.D1 newTimeValue = new ArrayInt.D1(1);
>        newTimeValue.set(0, (int)dateFormat.parse("20071130 
> 0925").getTime());
>
>        ArrayInt.D1 newVarValue = new ArrayInt.D1(1);
>        newVarValue.set(0, 11);
>               /* Going to insert at 1, so read existing value,
>         * write down new one, and re-append old tail.
>         */
>               int[] insertPointOrigin = {1};
>        int[] appendOrigin = {2};
>        int[] shape = {1};
>               Array tailTime = 
> ncFile.findVariable("time").read(insertPointOrigin, shape);
>        ncFile.write("time", insertPointOrigin, newTimeValue);
>        ncFile.write("time", appendOrigin, tailTime);
>               /* Next line excepts - why?  Because the last write 
> above at
>         * records index 2 triggers an increase in the CACHED/MEMORY
>         * length of all variables to 3, but on disk it's still the
>         * original length 2.
>         *
>         *  Therefore we get EOFException.
>         */
>               Array tailVar1 = 
> ncFile.findVariable("var1").read(insertPointOrigin, shape);
>        ncFile.write("var1", insertPointOrigin, newVarValue);
>        ncFile.write("var1", appendOrigin, tailVar1);
>               assertEquals(3, 
> ncFile.findDimension("records").getLength());
>           }
>
> }
>
>