Bug#177956: length greater than 1000000 fails
Package: libhdf4g
Version: 4.1r4-13
There is a bug in VSread(). The following code will result in the
last 9 doubles being overwritten where the first 9 locations in the
buffer. I attempted to fix it but cannot be sure that my fix is
completely general. I have included a test case and my fix to
VSread() in this email.
I also sent a copy of this email to hdfhelp@ncsa.uiuc.edu. The bug
occurs in version 4.1r5 as well.
Sincerely,
Nathan Salwen
-------------- command to compile test program foo.cxx --------
g++ -g -Wall -DINLINES -I/lin1/home/nks/download/HDF4.1r5/NewHDF/include -L/lin1/home/nks/download/HDF4.1r5/NewHDF/lib -o foo foo.cxx -ldf -ljpeg -lz
--------------- start of test program (foo.cxx) ------------------
#include <stdio.h>
#include <stdlib.h>
#include <math.h>
#include <string.h>
#include <stdarg.h>
#include "hdf.h"
void hdfstore(char *filename, double *buffer, int size);
void hdfread(char *filename, double *buffer, int size);
void hdfstore(char *filename, double *buffer, int size)
{
int hdfile = Hopen(filename, DFACC_CREATE, 0);
int status = Vstart (hdfile);
int32 vdata1_ref = VHstoredata (hdfile, "Single-component Field",
(uint8 *) buffer, size,
DFNT_FLOAT64, "First Vdata", "Test");
Hclose(hdfile);
}
void hdfread(char *filename, double *buffer, int size)
{
int hdfile = Hopen (filename, DFACC_READ, 0);
int status = Vstart (hdfile);
int vdata_ref = VSfind (hdfile, "First Vdata");
if (vdata_ref == 0)
printf("VSfind failed");
int vdata_id = VSattach (hdfile, vdata_ref, "r");
int num_of_records = VSread (vdata_id, (uint8 *)buffer, size,
FULL_INTERLACE);
status = Hclose (hdfile);
printf("num_of_records = %d\n", num_of_records);
printf("hdfread returning\n");
}
double *sendbuf;
double *rcvbuf;
int size=125010;
int main(int argc, char **argv)
{
sendbuf = (double *) malloc(size * sizeof(double));
rcvbuf = (double *) malloc(size * sizeof(double));
for (int i = 0; i < size; i++)
sendbuf[i] = rand()/(double) RAND_MAX;
hdfstore("temp", sendbuf, size);
hdfread("temp", rcvbuf, size);
for (int i = 0; i < size; i++)
{
printf("diff[%d] = %g, data = %g\n",
i, rcvbuf[i] - sendbuf[i], sendbuf[i]);
}
}
------ new version of VSread() from HDF4.1r5/hdf/src/vrw.c --------
/* Look for "BEGINNING OF BUGFIX" for change to file */
int32
VSread(int32 vkey, /* IN: vdata key */
uint8 buf[], /* IN/OUT: space to put elements in */
int32 nelt, /* IN: number of elements to read */
int32 interlace /* IN: interlace to return elements in 'buf' */)
{
intn isize = 0;
intn order = 0;
intn index = 0;
intn esize = 0;
intn hsize = 0;
uint8 *Src;
uint8 *b1 = NULL;
uint8 *b2 = NULL;
int32 i, j;
int32 nv;
int32 offset;
int32 type;
int32 uvsize; /* size of "element" as NEEDED by user */
int32 total_bytes; /* total number of bytes that need to be read in */
int32 bytes; /* number of elements / bytes to read next time */
int32 chunk; /* number of records in a buffer */
int32 done; /* number of records to do / done */
DYN_VWRITELIST *w = NULL;
DYN_VREADLIST *r = NULL;
vsinstance_t *wi = NULL;
VDATA *vs = NULL;
int32 ret_value = SUCCEED;
CONSTR(FUNC, "VSread");
#ifdef HAVE_PABLO
TRACE_ON(PABLO_mask,ID_VSread);
#endif /* HAVE_PABLO */
/* clear error stack */
HEclear();
/* check if vdata is part of vdata group */
if (HAatom_group(vkey) != VSIDGROUP)
HGOTO_ERROR(DFE_ARGS, FAIL);
/* get vdata instance */
if (NULL == (wi = (vsinstance_t *) HAatom_object(vkey)))
HGOTO_ERROR(DFE_NOVS, FAIL);
/* get vdata itself and check it */
vs = wi->vs;
if (vs == NULL)
HGOTO_ERROR(DFE_ARGS, FAIL);
/* check access id and number of vertices in vdata */
if ((vs->aid == 0) || (vs->nvertices == 0))
HGOTO_ERROR(DFE_ARGS, FAIL);
/* Don't allow reads in 0-field vdatas */
if (vs->wlist.n<=0)
HGOTO_ERROR(DFE_BADFIELDS, FAIL);
/* check if vdata exists in file */
if (vexistvs(vs->f, vs->oref) == FAIL)
HGOTO_ERROR(DFE_NOVS, FAIL);
/* check interlace parameter */
if (interlace != FULL_INTERLACE && interlace != NO_INTERLACE)
HGOTO_ERROR(DFE_ARGS, FAIL);
/* read/write lists */
w = &(vs->wlist);
r = &(vs->rlist);
hsize = (intn)vs->wlist.ivsize; /* size as stored in HDF */
total_bytes = hsize * nelt;
/*
Now, convert and repack field(s) from Vtbuf into buf.
This section of the code deals with interlacing. In all cases
the items for each of the fields are converted and shuffled
around from the internal buffer "Vtbuf" to the user's buffer
"buf".
There are 5 cases :
(A) user=NO_INTERLACE & vdata=FULL_INTERLACE)
(B) user=NO_INTERLACE & vdata=NO_INTERLACE)
(C) user=FULL_INTERLACE & vdata=FULL_INTERLACE)
(D) user=FULL_INTERLACE & vadat=NO_INTERLACE)
(E) SPECIAL CASE when only one field.
Cases (A)-(D) handles multiple fields.
Case (E) handles reading from a Vdata with a single field.
Cases (E) and (C) are the most frequently used. Limit buffer
allocations to VDATA_BUFFER_MAX size so that we conserve
memory. Doing this involves a certain degree of added code
complexity so don't bother doing it for the less frequent
cases. Cases E and C have been rolled together since they are
very similar and both need the incremental writing.
*/
/* ----------------------------------------------------------------- */
/* CASE (E + C): Easy to unroll case */
if ((w->n == 1) || (interlace == FULL_INTERLACE && vs->interlace == FULL_INTERLACE))
{
/*
* figure out how many elements we can move at a time and
* make sure our buffer is big enough
*/
if ((uint32) total_bytes < Vtbufsize)
{
chunk = nelt;
}
else
{
int32 buf_size;
/* we are bounded above by VDATA_BUFFER_MAX */
buf_size = MIN(total_bytes, VDATA_BUFFER_MAX);
/* make sure there is at least room for one record in our buffer */
chunk = buf_size / hsize + 1;
/* get a buffer big enough to hold the values */
Vtbufsize = (size_t)chunk * (size_t)hsize;
if (Vtbuf)
HDfree(Vtbuf);
if ((Vtbuf = (uint8 *) HDmalloc(Vtbufsize)) == NULL)
HGOTO_ERROR(DFE_NOSPACE, FAIL);
}
done = 0;
/* set loop invariant parameters */
Src = buf;
bytes = hsize * chunk;
for (uvsize = 0, j = 0; j < r->n; j++)
uvsize += w->esize[r->item[j]];
/********************* BEGINNING OF BUGFIX *************************/
/* DFKconvert will be called with 0 for dest stride */
/* and will adjust to size of objects */
if (w->n == 1)
uvsize = w->esize[0];
/********************* END OF BUGFIX *******************************/
while (done < nelt)
{
/* chunk has changed so update the byte counts */
if (nelt - done < chunk)
{
chunk = nelt - done;
bytes = hsize * chunk;
}
/* ================ start reading ============================== */
if ((nv = Hread(vs->aid, bytes, (uint8 *) Vtbuf)) != bytes)
{
HERROR(DFE_READERROR);
HEreport("Tried to read %d, only read %d", bytes, nv);
HGOTO_DONE(FAIL);
}
/* CASE (E): Only a single field in the Vdata */
if (w->n == 1)
{
DFKconvert(Vtbuf,Src,w->type[0], (uint32) w->order[0] * (uint32)chunk, DFACC_READ, 0, 0);
} /* case (e) */
/* ----------------------------------------------------------------- */
/* CASE (C): iu=full, iv=full */
else
{
offset = 0;
for (j = 0; j < r->n; j++)
{
i = r->item[j];
b1 = Src + offset;
b2 = Vtbuf + (size_t)w->off[i];
type = (int32)w->type[r->item[j]];
esize = (intn)w->esize[i];
isize = (intn)w->isize[i];
order = (intn)w->order[i];
for (index = 0; index < order; index++)
{
DFKconvert(b2, b1, type, (uint32) chunk, DFACC_READ, (uint32) hsize, (uint32) uvsize);
b1 += (int) esize / order;
b2 += (int) isize / order;
}
offset += esize;
}
} /* case (E) */
/* record what we've done and move to next group */
done += chunk;
Src += chunk * uvsize;
} /* end while */
} /* case (C + E) */
else {
/*
* Handle the other cases now.
* These cases are less frequent so don't bother unrolling
* the loops for now. As a result, we may get into memory
* problems since we may end up allocating a huge buffer
*/
/* alloc space (Vtbuf) for reading in the raw data from vdata */
if (Vtbufsize < (size_t)nelt * (size_t) hsize)
{
Vtbufsize = (size_t)nelt * (size_t) hsize;
if (Vtbuf)
HDfree(Vtbuf);
if ((Vtbuf = (uint8 *) HDmalloc(Vtbufsize)) == NULL)
HGOTO_ERROR(DFE_NOSPACE, FAIL);
}
/* ================ start reading ============================== */
nv = Hread(vs->aid, nelt * hsize, (uint8 *) Vtbuf);
if (nv != nelt * hsize)
{
HERROR(DFE_READERROR);
HEreport("Tried to read %d, only read %d", nelt * hsize, nv);
HGOTO_DONE(FAIL);
}
/* ----------------------------------------------------------------- */
/* CASE (A): user=none, vdata=full */
if (interlace == NO_INTERLACE && vs->interlace == FULL_INTERLACE)
{
b1 = buf;
for (j = 0; j < r->n; j++)
{
i = r->item[j];
b2 = Vtbuf + (size_t)w->off[i];
type = (int32)w->type[i];
isize = (intn)w->isize[i];
esize = (intn)w->esize[i];
order = (intn)w->order[i];
for (index = 0; index < order; index++)
{
DFKconvert(b2, b1, type, (uint32) nelt, DFACC_READ, (uint32) hsize, (uint32) esize);
b2 += isize / order;
b1 += esize / order;
}
b1 += ((nelt - 1) * esize);
}
} /* case (a) */
/* ----------------------------------------------------------------- */
/* CASE (B): user=none, vdata=none */
else if (interlace == NO_INTERLACE && vs->interlace == NO_INTERLACE)
{
b1 = buf;
for (j = 0; j < r->n; j++)
{
i = r->item[j];
b2 = Vtbuf + (size_t)w->off[i] * (size_t)nelt;
type = (int32)w->type[i];
esize = (intn)w->esize[i];
isize = (intn)w->isize[i];
order = (intn)w->order[i];
for (index = 0; index < order; index++)
{
DFKconvert(b2, b1, type, (uint32) nelt, DFACC_READ, (uint32) isize, (uint32) esize);
b1 += esize / order;
b2 += isize / order;
}
b1 += ((nelt - 1) * esize);
}
} /* case (b) */
/* ----------------------------------------------------------------- */
/* CASE (D): user=full, vdata=none */
else if (interlace == FULL_INTERLACE && vs->interlace == NO_INTERLACE)
{
for (uvsize = 0, j = 0; j < r->n; j++)
uvsize += w->esize[r->item[j]];
offset = 0;
for (j = 0; j < r->n; j++)
{
i = r->item[j];
b1 = buf + offset;
b2 = Vtbuf + (size_t)w->off[i] * (size_t)nelt;
type = (int32)w->type[i];
isize = (intn)w->isize[i];
esize = (intn)w->esize[i];
order = (intn)w->order[i];
for (index = 0; index < order; index++)
{
DFKconvert(b2, b1, type, (uint32) nelt, DFACC_READ, (uint32) isize, (uint32) uvsize);
b1 += esize / order;
b2 += isize / order;
}
offset += isize;
}
} /* case (d) */
} /* end else, cases a, b, and d */
ret_value = (nelt);
done:
if(ret_value == FAIL)
{ /* Error condition cleanup */
} /* end if */
/* Normal function cleanup */
#ifdef HAVE_PABLO
TRACE_OFF( PABLO_mask,ID_VSread);
#endif /* HAVE_PABLO */
return ret_value;
} /* VSread */
Reply to: