bes  Updated for version 3.20.13
h5get.cc
Go to the documentation of this file.
1 // This file is part of hdf5_handler: an HDF5 file handler for the OPeNDAP
2 // data server.
3 
4 // Copyright (c) 2007-2016 The HDF Group, Inc. and OPeNDAP, Inc.
5 //
6 // This is free software; you can redistribute it and/or modify it under the
7 // terms of the GNU Lesser General Public License as published by the Free
8 // Software Foundation; either version 2.1 of the License, or (at your
9 // option) any later version.
10 //
11 // This software is distributed in the hope that it will be useful, but
12 // WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13 // or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
14 // License for more details.
15 //
16 // You should have received a copy of the GNU Lesser General Public
17 // License along with this library; if not, write to the Free Software
18 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 //
20 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
21 // You can contact The HDF Group, Inc. at 1800 South Oak Street,
22 // Suite 203, Champaign, IL 61820
23 
36 
37 #include "h5get.h"
38 #include "HDF5Int32.h"
39 #include "HDF5UInt32.h"
40 #include "HDF5UInt16.h"
41 #include "HDF5Int16.h"
42 #include "HDF5Byte.h"
43 #include "HDF5Int8.h"
44 #include "HDF5Int64.h"
45 #include "HDF5UInt64.h"
46 #include "HDF5Array.h"
47 #include "HDF5Str.h"
48 #include "HDF5Float32.h"
49 #include "HDF5Float64.h"
50 #include "HDF5Url.h"
51 #include "HDF5Structure.h"
52 #include "HDF5RequestHandler.h"
53 
54 #include <BESDebug.h>
55 #include <math.h>
56 #include <sstream>
57 
58 using namespace libdap;
59 
60 
61 // H5Lvisit call back function. After finding all the hard links, return 1.
62 static int visit_link_cb(hid_t group_id, const char *name, const H5L_info_t *oinfo,
63  void *_op_data);
64 
65 // H5OVISIT call back function. When finding the dimension scale attributes, return 1.
66 static int
67 visit_obj_cb(hid_t o_id, const char *name, const H5O_info_t *oinfo,
68  void *_op_data);
69 
70 // H5Aiterate2 call back function, check if having the dimension scale attributes.
71 static herr_t attr_info_dimscale(hid_t loc_id, const char *name, const H5A_info_t *ainfo, void *opdata);
72 
73 
90 hid_t get_attr_info(hid_t dset, int index, bool is_dap4, DSattr_t * attr_inst_ptr,
91  bool *ignore_attr_ptr)
92 {
93 
94  hid_t attrid = -1;
95 
96  // Always assume that we don't ignore any attributes.
97  *ignore_attr_ptr = false;
98 
99  if ((attrid = H5Aopen_by_idx(dset, ".", H5_INDEX_CRT_ORDER, H5_ITER_INC,(hsize_t)index, H5P_DEFAULT, H5P_DEFAULT)) < 0) {
100  string msg = "unable to open attribute by index ";
101  throw InternalErr(__FILE__, __LINE__, msg);
102  }
103 
104  // Obtain the size of attribute name.
105  ssize_t name_size = H5Aget_name(attrid, 0, nullptr);
106  if (name_size < 0) {
107  H5Aclose(attrid);
108  string msg = "unable to obtain the size of the hdf5 attribute name ";
109  throw InternalErr(__FILE__, __LINE__, msg);
110  }
111 
112  vector<char> attr_name;
113  attr_name.resize(name_size+1);
114  // Obtain the attribute name.
115  if ((H5Aget_name(attrid, name_size+1, attr_name.data())) < 0) {
116  H5Aclose(attrid);
117  string msg = "unable to obtain the hdf5 attribute name ";
118  throw InternalErr(__FILE__, __LINE__, msg);
119  }
120 
121  // Obtain the type of the attribute.
122  hid_t ty_id = -1;
123  if ((ty_id = H5Aget_type(attrid)) < 0) {
124  string msg = "unable to obtain hdf5 datatype for the attribute ";
125  string attrnamestr(attr_name.begin(),attr_name.end());
126  msg += attrnamestr;
127  H5Aclose(attrid);
128  throw InternalErr(__FILE__, __LINE__, msg);
129  }
130 
131  H5T_class_t ty_class = H5Tget_class(ty_id);
132  if (ty_class < 0) {
133  string msg = "cannot get hdf5 attribute datatype class for the attribute ";
134  string attrnamestr(attr_name.begin(),attr_name.end());
135  msg += attrnamestr;
136  H5Aclose(attrid);
137  throw InternalErr(__FILE__, __LINE__, msg);
138  }
139 
140  // The following datatype will not be supported for mapping to DAS for both DAP2 and DAP4.
141  // Note:DAP4 explicitly states that the data should be defined atomic datatype(int, string).
142  // 1-D variable length of string can also be mapped to both DAS and DDS.
143  // The variable length string class is H5T_STRING rather than H5T_VLEN,
144  // So safe here.
145  // We also ignore the mapping of integer 64 bit for DAP2 since DAP2 doesn't
146  // support 64-bit integer. In theory, DAP2 doesn't support long double
147  // (128-bit or 92-bit floating point type), since this rarely happens
148  // in DAP application, we simply don't consider here.
149  // However, DAP4 supports 64-bit integer.
150  if ((ty_class == H5T_TIME) || (ty_class == H5T_BITFIELD)
151  || (ty_class == H5T_OPAQUE) || (ty_class == H5T_ENUM)
152  || (ty_class == H5T_REFERENCE) ||(ty_class == H5T_COMPOUND)
153  || (ty_class == H5T_VLEN) || (ty_class == H5T_ARRAY)){
154 
155  *ignore_attr_ptr = true;
156  H5Tclose(ty_id);
157  return attrid;
158  }
159 
160  // Ignore 64-bit integer for DAP2.
161  // The nested if is better to understand the code. Don't combine
162  if (false == is_dap4) {
163  if((ty_class == H5T_INTEGER) && (H5Tget_size(ty_id)== 8)) {//64-bit int
164  *ignore_attr_ptr = true;
165  H5Tclose(ty_id);
166  return attrid;
167  }
168  }
169 
170  // Here we ignore netCDF-4 specific attributes for DAP4 to make filenetCDF-4 work
171  if (true == is_dap4 && HDF5RequestHandler::get_default_handle_dimension() == true) {
172  // Remove the nullptrTERM etc.
173  string attr_name_str(attr_name.begin(),attr_name.end()-1);
174  if(attr_name_str == "CLASS" || attr_name_str == "NAME" || attr_name_str == "_Netcdf4Dimid"
175  || attr_name_str == "_nc3_strict" || attr_name_str=="_NCProperties" || attr_name_str=="_Netcdf4Coordinates") {
176  *ignore_attr_ptr = true;
177  H5Tclose(ty_id);
178  return attrid;
179  }
180  }
181 
182  hid_t aspace_id = -1;
183  if ((aspace_id = H5Aget_space(attrid)) < 0) {
184  string msg = "cannot get hdf5 dataspace id for the attribute ";
185  string attrnamestr(attr_name.begin(),attr_name.end());
186  msg += attrnamestr;
187  H5Aclose(attrid);
188  throw InternalErr(__FILE__, __LINE__, msg);
189  }
190 
191  // It is better to use the dynamic allocation of the array.
192  // However, since the DODS_MAX_RANK is not big and it is also
193  // used in other location, we still keep the original code.
194  // KY 2011-11-16
195 
196  int ndims = H5Sget_simple_extent_ndims(aspace_id);
197  if (ndims < 0) {
198  string msg = "cannot get hdf5 dataspace number of dimension for attribute ";
199  string attrnamestr(attr_name.begin(),attr_name.end());
200  msg += attrnamestr;
201  H5Sclose(aspace_id);
202  H5Aclose(attrid);
203  throw InternalErr(__FILE__, __LINE__, msg);
204  }
205 
206  // Check if the dimension size exceeds the maximum number of dimension DAP supports
207  if (ndims > DODS_MAX_RANK) {
208  string msg = "number of dimensions exceeds allowed for attribute ";
209  string attrnamestr(attr_name.begin(),attr_name.end());
210  msg += attrnamestr;
211  H5Sclose(aspace_id);
212  H5Aclose(attrid);
213  throw InternalErr(__FILE__, __LINE__, msg);
214  }
215 
216  vector<hsize_t> size(ndims);
217  vector<hsize_t> maxsize(ndims);
218 
219  //The HDF5 attribute should not have unlimited dimension,
220  // maxsize is only a place holder.
221  if (H5Sget_simple_extent_dims(aspace_id, size.data(), maxsize.data())<0){
222  string msg = "cannot obtain the dim. info for the attribute ";
223  string attrnamestr(attr_name.begin(),attr_name.end());
224  msg += attrnamestr;
225  H5Sclose(aspace_id);
226  H5Aclose(attrid);
227  throw InternalErr(__FILE__, __LINE__, msg);
228  }
229 
230  // Return ndims and size[ndims].
231  hsize_t nelmts = 1;
232  if (ndims) {
233  for (int j = 0; j < ndims; j++)
234  nelmts *= size[j];
235  }
236 
237  size_t ty_size = H5Tget_size(ty_id);
238  if (ty_size == 0) {
239  string msg = "cannot obtain the dtype size for the attribute ";
240  string attrnamestr(attr_name.begin(),attr_name.end());
241  msg += attrnamestr;
242  H5Sclose(aspace_id);
243  H5Aclose(attrid);
244  throw InternalErr(__FILE__, __LINE__, msg);
245  }
246 
247  size_t need = nelmts * H5Tget_size(ty_id);
248 
249  // We want to save memory type in the struct.
250  hid_t memtype = H5Tget_native_type(ty_id, H5T_DIR_ASCEND);
251  if (memtype < 0){
252  string msg = "cannot obtain the memory dtype for the attribute ";
253  string attrnamestr(attr_name.begin(),attr_name.end());
254  msg += attrnamestr;
255  H5Sclose(aspace_id);
256  H5Aclose(attrid);
257  throw InternalErr(__FILE__, __LINE__, msg);
258  }
259 
260  // Save the information to the struct
261  (*attr_inst_ptr).type = memtype;
262  (*attr_inst_ptr).ndims = ndims;
263  (*attr_inst_ptr).nelmts = nelmts;
264  (*attr_inst_ptr).need = need;
265  strncpy((*attr_inst_ptr).name, attr_name.data(), name_size+1);
266 
267  // The handler assumes the size of an attribute is limited to 32-bit int
268  for (int j = 0; j < ndims; j++) {
269  (*attr_inst_ptr).size[j] = (int)(size[j]);
270  }
271 
272  if(H5Sclose(aspace_id)<0) {
273  H5Aclose(attrid);
274  throw InternalErr(__FILE__,__LINE__,"Cannot close HDF5 dataspace ");
275  }
276 
277  return attrid;
278 }
279 
292 string get_dap_type(hid_t type, bool is_dap4)
293 {
294  size_t size = 0;
295  H5T_sign_t sign;
296  BESDEBUG("h5", ">get_dap_type(): type=" << type << endl);
297  H5T_class_t class_t = H5Tget_class(type);
298  if (H5T_NO_CLASS == class_t)
299  throw InternalErr(__FILE__, __LINE__,
300  "The HDF5 datatype doesn't belong to any Class.");
301  switch (class_t) {
302 
303  case H5T_INTEGER:
304 
305  size = H5Tget_size(type);
306  if (size == 0){
307  throw InternalErr(__FILE__, __LINE__,
308  "size of datatype is invalid");
309  }
310 
311  sign = H5Tget_sign(type);
312  if (sign < 0){
313  throw InternalErr(__FILE__, __LINE__,
314  "sign of datatype is invalid");
315  }
316 
317  BESDEBUG("h5", "=get_dap_type(): H5T_INTEGER" <<
318  " sign = " << sign <<
319  " size = " << size <<
320  endl);
321  if (size == 1){
322  // DAP2 doesn't have signed 8-bit integer, so we need map it to INT16.
323  if(true == is_dap4) {
324  if (sign == H5T_SGN_NONE)
325  return BYTE;
326  else
327  return INT8;
328  }
329  else {
330  if (sign == H5T_SGN_NONE)
331  return BYTE;
332  else
333  return INT16;
334  }
335  }
336 
337  if (size == 2) {
338  if (sign == H5T_SGN_NONE)
339  return UINT16;
340  else
341  return INT16;
342  }
343 
344  if (size == 4) {
345  if (sign == H5T_SGN_NONE)
346  return UINT32;
347  else
348  return INT32;
349  }
350 
351  if(size == 8) {
352  // DAP4 supports 64-bit integer.
353  if (true == is_dap4) {
354  if (sign == H5T_SGN_NONE)
355  return UINT64;
356  else
357  return INT64;
358  }
359  else
360  return INT_ELSE;
361  }
362 
363  return INT_ELSE;
364 
365  case H5T_FLOAT:
366  size = H5Tget_size(type);
367  if (size == 0){
368  throw InternalErr(__FILE__, __LINE__,
369  "size of the datatype is invalid");
370  }
371 
372  BESDEBUG("h5", "=get_dap_type(): FLOAT size = " << size << endl);
373  if (size == 4)
374  return FLOAT32;
375  if (size == 8)
376  return FLOAT64;
377 
378  return FLOAT_ELSE;
379 
380  case H5T_STRING:
381  BESDEBUG("h5", "<get_dap_type(): H5T_STRING" << endl);
382  return STRING;
383 
384  case H5T_REFERENCE:
385  BESDEBUG("h5", "<get_dap_type(): H5T_REFERENCE" << endl);
386  return URL;
387  // Note: Currently DAP2 and DAP4 only support defined atomic types.
388  // So the H5T_COMPOUND and H5_ARRAY cases should never be reached for attribute handling.
389  // However, this function may be used for variable handling.
390  case H5T_COMPOUND:
391  BESDEBUG("h5", "<get_dap_type(): COMPOUND" << endl);
392  return COMPOUND;
393 
394  case H5T_ARRAY:
395  return ARRAY;
396 
397  default:
398  BESDEBUG("h5", "<get_dap_type(): Unmappable Type" << endl);
399  return "Unmappable Type";
400  }
401 }
402 
412 hid_t get_fileid(const char *filename)
413 {
414  hid_t fileid = H5Fopen(filename, H5F_ACC_RDONLY, H5P_DEFAULT);
415  if (fileid < 0){
416  string msg = "cannot open the HDF5 file ";
417  string filenamestr(filename);
418  msg += filenamestr;
419  throw InternalErr(__FILE__, __LINE__, msg);
420  }
421 
422  return fileid;
423 }
424 
434 void close_fileid(hid_t fid)
435 {
436  if (H5Fclose(fid) < 0)
437  throw Error(unknown_error,
438  string("Could not close the HDF5 file."));
439 
440 }
441 
452 
453 void get_dataset(hid_t pid, const string &dname, DS_t * dt_inst_ptr)
454 {
455  BESDEBUG("h5", ">get_dataset()" << endl);
456 
457  // Obtain the dataset ID
458  hid_t dset = -1;
459  if ((dset = H5Dopen(pid, dname.c_str(),H5P_DEFAULT)) < 0) {
460  string msg = "cannot open the HDF5 dataset ";
461  msg += dname;
462  throw InternalErr(__FILE__, __LINE__, msg);
463  }
464 
465  // Obtain the datatype ID
466  hid_t dtype = -1;
467  if ((dtype = H5Dget_type(dset)) < 0) {
468  H5Dclose(dset);
469  string msg = "cannot get the the datatype of HDF5 dataset ";
470  msg += dname;
471  throw InternalErr(__FILE__, __LINE__, msg);
472  }
473 
474  // Obtain the datatype class
475  H5T_class_t ty_class = H5Tget_class(dtype);
476  if (ty_class < 0) {
477  H5Tclose(dtype);
478  H5Dclose(dset);
479  string msg = "cannot get the datatype class of HDF5 dataset ";
480  msg += dname;
481  throw InternalErr(__FILE__, __LINE__, msg);
482  }
483 
484  // These datatype classes are unsupported. Note we do support
485  // variable length string and the variable length string class is
486  // H5T_STRING rather than H5T_VLEN.
487  if ((ty_class == H5T_TIME) || (ty_class == H5T_BITFIELD)
488  || (ty_class == H5T_OPAQUE) || (ty_class == H5T_ENUM) || (ty_class == H5T_VLEN)) {
489  string msg = "unexpected datatype of HDF5 dataset ";
490  msg += dname;
491  throw InternalErr(__FILE__, __LINE__, msg);
492  }
493 
494  hid_t dspace = -1;
495  if ((dspace = H5Dget_space(dset)) < 0) {
496  H5Tclose(dtype);
497  H5Dclose(dset);
498  string msg = "cannot get the the dataspace of HDF5 dataset ";
499  msg += dname;
500  throw InternalErr(__FILE__, __LINE__, msg);
501  }
502 
503  // It is better to use the dynamic allocation of the array.
504  // However, since the DODS_MAX_RANK is not big and it is also
505  // used in other location, we still keep the original code.
506  // KY 2011-11-17
507 
508  int ndims = H5Sget_simple_extent_ndims(dspace);
509  if (ndims < 0) {
510  H5Tclose(dtype);
511  H5Sclose(dspace);
512  H5Dclose(dset);
513  string msg = "cannot get hdf5 dataspace number of dimension for dataset ";
514  msg += dname;
515  throw InternalErr(__FILE__, __LINE__, msg);
516  }
517 
518  // Check if the dimension size exceeds the maximum number of dimension DAP supports
519  if (ndims > DODS_MAX_RANK) {
520  string msg = "number of dimensions exceeds allowed for dataset ";
521  msg += dname;
522  H5Tclose(dtype);
523  H5Sclose(dspace);
524  H5Dclose(dset);
525  throw InternalErr(__FILE__, __LINE__, msg);
526  }
527 
528  vector<hsize_t>size(ndims);
529  vector<hsize_t>maxsize(ndims);
530 
531  // Retrieve size. DAP4 doesn't have a convention to support multi-unlimited dimension yet.
532  if (H5Sget_simple_extent_dims(dspace, size.data(), maxsize.data())<0){
533  string msg = "cannot obtain the dim. info for the dataset ";
534  msg += dname;
535  H5Tclose(dtype);
536  H5Sclose(dspace);
537  H5Dclose(dset);
538  throw InternalErr(__FILE__, __LINE__, msg);
539  }
540 
541  // return ndims and size[ndims].
542  hsize_t nelmts = 1;
543  if (ndims !=0) {
544  for (int j = 0; j < ndims; j++)
545  nelmts *= size[j];
546  }
547 
548  size_t dtype_size = H5Tget_size(dtype);
549  if (dtype_size == 0) {
550  string msg = "cannot obtain the data type size for the dataset ";
551  msg += dname;
552  H5Tclose(dtype);
553  H5Sclose(dspace);
554  H5Dclose(dset);
555  throw InternalErr(__FILE__, __LINE__, msg);
556  }
557 
558  size_t need = nelmts * dtype_size;
559 
560  hid_t memtype = H5Tget_native_type(dtype, H5T_DIR_ASCEND);
561  if (memtype < 0){
562  string msg = "cannot obtain the memory data type for the dataset ";
563  msg += dname;
564  H5Tclose(dtype);
565  H5Sclose(dspace);
566  H5Dclose(dset);
567  throw InternalErr(__FILE__, __LINE__, msg);
568  }
569 
570  (*dt_inst_ptr).type = memtype;
571  (*dt_inst_ptr).ndims = ndims;
572  (*dt_inst_ptr).nelmts = nelmts;
573  (*dt_inst_ptr).need = need;
574  strncpy((*dt_inst_ptr).name, dname.c_str(), dname.length());
575  (*dt_inst_ptr).name[dname.length()] = '\0';
576  for (int j = 0; j < ndims; j++)
577  (*dt_inst_ptr).size[j] = (int)(size[j]);
578 
579  if(H5Tclose(dtype)<0) {
580  H5Sclose(dspace);
581  H5Dclose(dset);
582  throw InternalErr(__FILE__, __LINE__, "Cannot close the HDF5 datatype.");
583  }
584 
585  if(H5Sclose(dspace)<0) {
586  H5Dclose(dset);
587  throw InternalErr(__FILE__, __LINE__, "Cannot close the HDF5 dataspace.");
588  }
589 
590  if(H5Dclose(dset)<0) {
591  throw InternalErr(__FILE__, __LINE__, "Cannot close the HDF5 dataset.");
592  }
593 
594 }
595 
610 void get_dataset_dmr(const hid_t file_id, hid_t pid, const string &dname, DS_t * dt_inst_ptr,bool use_dimscale, bool &is_pure_dim, vector<link_info_t> &hdf5_hls)
611 {
612 
613  BESDEBUG("h5", ">get_dataset()" << endl);
614 
615  // Obtain the dataset ID
616  hid_t dset = -1;
617  if ((dset = H5Dopen(pid, dname.c_str(),H5P_DEFAULT)) < 0) {
618  string msg = "cannot open the HDF5 dataset ";
619  msg += dname;
620  throw InternalErr(__FILE__, __LINE__, msg);
621  }
622 
623  // Obtain the datatype ID
624  hid_t dtype = -1;
625  if ((dtype = H5Dget_type(dset)) < 0) {
626  H5Dclose(dset);
627  string msg = "cannot get the the datatype of HDF5 dataset ";
628  msg += dname;
629  throw InternalErr(__FILE__, __LINE__, msg);
630  }
631 
632  // Obtain the datatype class
633  H5T_class_t ty_class = H5Tget_class(dtype);
634  if (ty_class < 0) {
635  H5Tclose(dtype);
636  H5Dclose(dset);
637  string msg = "cannot get the datatype class of HDF5 dataset ";
638  msg += dname;
639  throw InternalErr(__FILE__, __LINE__, msg);
640  }
641 
642  // These datatype classes are unsupported. Note we do support
643  // variable length string and the variable length string class is
644  // H5T_STRING rather than H5T_VLEN.
645  if ((ty_class == H5T_TIME) || (ty_class == H5T_BITFIELD)
646  || (ty_class == H5T_OPAQUE) || (ty_class == H5T_ENUM) || (ty_class == H5T_VLEN)) {
647  string msg = "unexpected datatype of HDF5 dataset ";
648  msg += dname;
649  throw InternalErr(__FILE__, __LINE__, msg);
650  }
651 
652  hid_t dspace = -1;
653  if ((dspace = H5Dget_space(dset)) < 0) {
654  H5Tclose(dtype);
655  H5Dclose(dset);
656  string msg = "cannot get the the dataspace of HDF5 dataset ";
657  msg += dname;
658  throw InternalErr(__FILE__, __LINE__, msg);
659  }
660 
661  // It is better to use the dynamic allocation of the array.
662  // However, since the DODS_MAX_RANK is not big and it is also
663  // used in other location, we still keep the original code.
664  // KY 2011-11-17
665 
666  int ndims = H5Sget_simple_extent_ndims(dspace);
667  if (ndims < 0) {
668  H5Tclose(dtype);
669  H5Sclose(dspace);
670  H5Dclose(dset);
671  string msg = "cannot get hdf5 dataspace number of dimension for dataset ";
672  msg += dname;
673  throw InternalErr(__FILE__, __LINE__, msg);
674  }
675 
676  // Check if the dimension size exceeds the maximum number of dimension DAP supports
677  if (ndims > DODS_MAX_RANK) {
678  string msg = "number of dimensions exceeds allowed for dataset ";
679  msg += dname;
680  H5Tclose(dtype);
681  H5Sclose(dspace);
682  H5Dclose(dset);
683  throw InternalErr(__FILE__, __LINE__, msg);
684  }
685 
686  vector<hsize_t>size(ndims);
687  vector<hsize_t>maxsize(ndims);
688 
689  // Retrieve size. DAP4 doesn't have a convention to support multi-unlimited dimension yet.
690  if (H5Sget_simple_extent_dims(dspace, size.data(), maxsize.data())<0){
691  string msg = "cannot obtain the dim. info for the dataset ";
692  msg += dname;
693  H5Tclose(dtype);
694  H5Sclose(dspace);
695  H5Dclose(dset);
696  throw InternalErr(__FILE__, __LINE__, msg);
697  }
698 
699  // return ndims and size[ndims].
700  hsize_t nelmts = 1;
701  if (ndims !=0) {
702  for (int j = 0; j < ndims; j++)
703  nelmts *= size[j];
704  }
705 
706  size_t dtype_size = H5Tget_size(dtype);
707  if (dtype_size == 0) {
708  string msg = "cannot obtain the data type size for the dataset ";
709  msg += dname;
710  H5Tclose(dtype);
711  H5Sclose(dspace);
712  H5Dclose(dset);
713  throw InternalErr(__FILE__, __LINE__, msg);
714  }
715 
716  size_t need = nelmts * dtype_size;
717 
718  hid_t memtype = H5Tget_native_type(dtype, H5T_DIR_ASCEND);
719  if (memtype < 0){
720  string msg = "cannot obtain the memory data type for the dataset ";
721  msg += dname;
722  H5Tclose(dtype);
723  H5Sclose(dspace);
724  H5Dclose(dset);
725  throw InternalErr(__FILE__, __LINE__, msg);
726  }
727 
728  (*dt_inst_ptr).type = memtype;
729  (*dt_inst_ptr).ndims = ndims;
730  (*dt_inst_ptr).nelmts = nelmts;
731  (*dt_inst_ptr).need = need;
732  strncpy((*dt_inst_ptr).name, dname.c_str(), dname.length());
733  (*dt_inst_ptr).name[dname.length()] = '\0';
734  for (int j = 0; j < ndims; j++)
735  (*dt_inst_ptr).size[j] = (int)(size[j]);
736 
737  // For DAP4 when dimension scales are used.
738  if(true == use_dimscale) {
739  BESDEBUG("h5", "<h5get.cc: get_dataset() use dim scale is true." << endl);
740 
741  // Some HDF5 datasets are dimension scale datasets; some are not. We need to distinguish.
742  bool is_dimscale = false;
743 
744  // Dimension scales must be 1-D.
745  if(1 == ndims) {
746 
747  bool has_ds_attr = false;
748 
749  try{
750  has_ds_attr = has_dimscale_attr(dset);
751  }
752  catch(...) {
753  H5Tclose(dtype);
754  H5Sclose(dspace);
755  H5Dclose(dset);
756  throw InternalErr(__FILE__, __LINE__, "Fail to check dim. scale.");
757  }
758 
759  if(true == has_ds_attr) {
760 
761 #if 0
762  //int count = 0;
763  //vector<bool>dim_attr_mark;
764  //dim_attr_mark.resize(4);
765  //bool dim_attr_mark[4];
766 #endif
767  int dim_attr_mark[3];
768  for(int i = 0;i<3;i++)
769  dim_attr_mark[i] = 0;
770 
771  // This will check if "NAME" and "REFERENCE_LIST" exists.
772  //herr_t ret = H5Aiterate2(dset, H5_INDEX_NAME, H5_ITER_INC, nullptr, attr_info, dim_attr_mark.data());
773  herr_t ret = H5Aiterate2(dset, H5_INDEX_NAME, H5_ITER_INC, nullptr, attr_info_dimscale, dim_attr_mark);
774  if(ret < 0) {
775  string msg = "cannot interate the attributes of the dataset ";
776  msg += dname;
777  H5Tclose(dtype);
778  H5Sclose(dspace);
779  H5Dclose(dset);
780  throw InternalErr(__FILE__, __LINE__, msg);
781  }
782 
783  for (int i = 0; i<3;i++)
784  BESDEBUG("h5","dim_attr_mark is "<<dim_attr_mark[i] <<endl);
785  // Find the dimension scale. DIM*SCALE is a must. Then NAME=VARIABLE or (REFERENCE_LIST and not PURE DIM)
786  // Here a little bias towards files created by the netCDF-4 APIs.
787  // If we don't have RERERENCE_LIST in a dataset that has CLASS=DIMENSION_SCALE attribute,
788  // we will ignore this orphage dimension sale since it is not associated with other datasets.
789  // However, it is an orphage dimension scale created by the netCDF-4 APIs, we think
790  // it must have a purpose to do this way by data creator. So keep this as a dimension scale.
791  //
792  if ((dim_attr_mark[0] && !dim_attr_mark[1]) || dim_attr_mark[2])
793  is_dimscale =true;
794  else if(dim_attr_mark[1])
795  is_pure_dim = true;
796  }
797  }
798 
799  if (true == is_dimscale) {
800  BESDEBUG("h5", "<h5get.cc: dname is " << dname << endl);
801  BESDEBUG("h5", "<h5get.cc: get_dataset() this is dim scale." << endl);
802  BESDEBUG("h5", "<h5get.cc: dataset storage size is: " <<H5Dget_storage_size(dset)<< endl);
803 #if 0
804  //if(H5Dget_storage_size(dset)!=0) {
805 #endif
806  // Save the dimension names.We Only need to provide the dimension name(not the full path).
807  // We still need the dimension name fullpath for distinguishing the different dimension that
808  // has the same dimension name but in the different path
809  // TODO; pure dimension doesn't work for all cases. See https://jira.hdfgroup.org/browse/HFVHANDLER-340
810  (*dt_inst_ptr).dimnames.push_back(dname.substr(dname.find_last_of("/")+1));
811  (*dt_inst_ptr).dimnames_path.push_back(dname);
812 #if 0
813  //}
814  //else
815  //is_pure_dim = true;
816 #endif
817  is_pure_dim = false;
818  }
819 
820  else if(false == is_pure_dim) // Except pure dimension,we need to save all dimension names in this dimension.
821  obtain_dimnames(file_id,dset,ndims,dt_inst_ptr,hdf5_hls);
822  }
823 
824  if(H5Tclose(dtype)<0) {
825  H5Sclose(dspace);
826  H5Dclose(dset);
827  throw InternalErr(__FILE__, __LINE__, "Cannot close the HDF5 datatype.");
828  }
829 
830  if(H5Sclose(dspace)<0) {
831  H5Dclose(dset);
832  throw InternalErr(__FILE__, __LINE__, "Cannot close the HDF5 dataspace.");
833  }
834 
835  if(H5Dclose(dset)<0) {
836  throw InternalErr(__FILE__, __LINE__, "Cannot close the HDF5 dataset.");
837  }
838 
839 }
840 
849 bool check_h5str(hid_t h5type)
850 {
851  if (H5Tget_class(h5type) == H5T_STRING)
852  return true;
853  else
854  return false;
855 }
856 
857 
868 string print_attr(hid_t type, int loc, void *sm_buf) {
869  union {
870  unsigned char* ucp;
871  char *tcp;
872  short *tsp;
873  unsigned short *tusp;
874  int *tip;
875  unsigned int*tuip;
876  long *tlp;
877  unsigned long*tulp;
878  float *tfp;
879  double *tdp;
880  } gp;
881 
882  vector<char> rep;
883 
884  switch (H5Tget_class(type)) {
885 
886  case H5T_INTEGER: {
887 
888  size_t size = H5Tget_size(type);
889  if (size == 0){
890  throw InternalErr(__FILE__, __LINE__,
891  "size of datatype is invalid");
892  }
893 
894  H5T_sign_t sign = H5Tget_sign(type);
895  if (sign < 0){
896  throw InternalErr(__FILE__, __LINE__,
897  "sign of datatype is invalid");
898  }
899 
900  BESDEBUG("h5", "=get_dap_type(): H5T_INTEGER" <<
901  " sign = " << sign <<
902  " size = " << size <<
903  endl);
904 
905  // change void pointer into the corresponding integer datatype.
906  // 32 should be long enough to hold one integer and one
907  // floating point number.
908  rep.resize(32);
909 
910  if (size == 1){
911 
912  if(sign == H5T_SGN_NONE) {
913  gp.ucp = (unsigned char *) sm_buf;
914  unsigned char tuchar = *(gp.ucp + loc);
915  snprintf(rep.data(), 32, "%u", tuchar);
916  }
917 
918  else {
919  gp.tcp = (char *) sm_buf;
920  snprintf(rep.data(), 32, "%d", *(gp.tcp + loc));
921  }
922  }
923 
924  else if (size == 2) {
925 
926  if(sign == H5T_SGN_NONE) {
927  gp.tusp = (unsigned short *) sm_buf;
928  snprintf(rep.data(), 32, "%hu", *(gp.tusp + loc));
929 
930  }
931  else {
932  gp.tsp = (short *) sm_buf;
933  snprintf(rep.data(), 32, "%hd", *(gp.tsp + loc));
934 
935  }
936  }
937 
938  else if (size == 4) {
939 
940  if(sign == H5T_SGN_NONE) {
941  gp.tuip = (unsigned int *) sm_buf;
942  snprintf(rep.data(), 32, "%u", *(gp.tuip + loc));
943 
944  }
945  else {
946  gp.tip = (int *) sm_buf;
947  snprintf(rep.data(), 32, "%d", *(gp.tip + loc));
948  }
949  }
950  else if (size == 8) {
951 
952  if(sign == H5T_SGN_NONE) {
953  gp.tulp = (unsigned long *) sm_buf;
954  snprintf(rep.data(), 32, "%lu", *(gp.tulp + loc));
955  }
956  else {
957  gp.tlp = (long *) sm_buf;
958  snprintf(rep.data(), 32, "%ld", *(gp.tlp + loc));
959  }
960  }
961  else
962  throw InternalErr(__FILE__, __LINE__,"Unsupported integer type, check the size of datatype.");
963 
964  break;
965  }
966 
967  case H5T_FLOAT: {
968  rep.resize(32);
969  char gps[32];
970 
971  if (H5Tget_size(type) == 4) {
972 
973  float attr_val = *(float*)sm_buf;
974  bool is_a_fin = isfinite(attr_val);
975  // Represent the float number.
976  // Some space may be wasted. But it is okay.
977  gp.tfp = (float *) sm_buf;
978  int ll = snprintf(gps, 30, "%.10g", *(gp.tfp + loc));
979 #if 0
980  //int ll = strlen(gps);
981 #endif
982  // Add the dot to assure this is a floating number
983  if (!strchr(gps, '.') && !strchr(gps, 'e') && !strchr(gps,'E')
984  && (true == is_a_fin)){
985  gps[ll++] = '.';
986  }
987 
988  gps[ll] = '\0';
989  snprintf(rep.data(), 32, "%s", gps);
990  }
991  else if (H5Tget_size(type) == 8) {
992 
993  double attr_val = *(double*)sm_buf;
994  bool is_a_fin = isfinite(attr_val);
995  gp.tdp = (double *) sm_buf;
996  int ll = snprintf(gps, 30, "%.17g", *(gp.tdp + loc));
997 #if 0
998  //int ll = strlen(gps);
999 #endif
1000  if (!strchr(gps, '.') && !strchr(gps, 'e')&& !strchr(gps,'E')
1001  && (true == is_a_fin)) {
1002  gps[ll++] = '.';
1003  }
1004  gps[ll] = '\0';
1005  snprintf(rep.data(), 32, "%s", gps);
1006  }
1007  else if (H5Tget_size(type) == 0){
1008  throw InternalErr(__FILE__, __LINE__, "H5Tget_size() failed.");
1009  }
1010  break;
1011  }
1012 
1013  case H5T_STRING: {
1014  size_t str_size = H5Tget_size(type);
1015  if(H5Tis_variable_str(type)>0) {
1016  throw InternalErr(__FILE__, __LINE__,
1017  "print_attr function doesn't handle variable length string, variable length string should be handled separately.");
1018  }
1019  if (str_size == 0){
1020  throw InternalErr(__FILE__, __LINE__, "H5Tget_size() failed.");
1021  }
1022  BESDEBUG("h5", "=print_attr(): H5T_STRING sm_buf=" << (char *) sm_buf
1023  << " size=" << str_size << endl);
1024  // Not sure why the original code add 1 byte to the buffer, perhaps to keep the c-style? KY 2021-04-12
1025 #if 0
1026  //rep.resize(str_size+1);
1027 #endif
1028  rep.resize(str_size);
1029  strncpy(rep.data(), (char *) sm_buf, str_size);
1030 
1031  //Also should add the NULL term at the end. We just need the data in C++.
1032 #if 0
1033  //rep[str_size] = '\0';
1034 #endif
1035 #if 0
1036  char *buf = nullptr;
1037  // This try/catch block is here to protect the allocation of buf.
1038  try {
1039 
1040 
1041  buf = new char[str_size + 1];
1042  strncpy(buf.data(), (char *) sm_buf, str_size);
1043  buf[str_size] = '\0';
1044  // Not necessarily allocate 3 more bytes.
1045  rep.resize(str_size+3);
1046  snprintf(rep.data(), str_size + 3, "%s", buf);
1047  rep[str_size + 2] = '\0';
1048  delete[] buf; buf = 0;
1049  }
1050  catch (...) {
1051  if( buf ) delete[] buf;
1052  throw;
1053  }
1054 #endif
1055  break;
1056  }
1057 
1058  default:
1059  break;
1060  } // switch(H5Tget_class(type))
1061 
1062  string rep_str(rep.begin(),rep.end());
1063  return rep_str;
1064 }
1065 
1066 D4AttributeType daptype_strrep_to_dap4_attrtype(const string & s){
1067 
1068  if (s == "Byte")
1069  return attr_byte_c;
1070  else if (s == "Int8")
1071  return attr_int8_c;
1072  else if (s == "UInt8") // This may never be used.
1073  return attr_uint8_c;
1074  else if (s == "Int16")
1075  return attr_int16_c;
1076  else if (s == "UInt16")
1077  return attr_uint16_c;
1078  else if (s == "Int32")
1079  return attr_int32_c;
1080  else if (s == "UInt32")
1081  return attr_uint32_c;
1082  else if (s == "Int64")
1083  return attr_int64_c;
1084  else if (s == "UInt64")
1085  return attr_uint64_c;
1086  else if (s == "Float32")
1087  return attr_float32_c;
1088  else if (s == "Float64")
1089  return attr_float64_c;
1090  else if (s == "String")
1091  return attr_str_c;
1092  else if (s == "Url")
1093  return attr_url_c;
1094  else
1095  return attr_null_c;
1096 
1097 
1098 }
1099 
1100 
1114 //static BaseType *Get_bt(const string &vname,
1115 BaseType *Get_bt(const string &vname,
1116  const string &vpath,
1117  const string &dataset,
1118  hid_t datatype, bool is_dap4)
1119 {
1120  BaseType *btp = nullptr;
1121 
1122  try {
1123 
1124  BESDEBUG("h5", ">Get_bt varname=" << vname << " datatype=" << datatype
1125  << endl);
1126 
1127  size_t size = 0;
1128  H5T_sign_t sign = H5T_SGN_ERROR;
1129  switch (H5Tget_class(datatype)) {
1130 
1131  case H5T_INTEGER:
1132  {
1133  size = H5Tget_size(datatype);
1134  sign = H5Tget_sign(datatype);
1135  BESDEBUG("h5", "=Get_bt() H5T_INTEGER size = " << size << " sign = "
1136  << sign << endl);
1137 
1138  if (sign == H5T_SGN_ERROR) {
1139  throw InternalErr(__FILE__, __LINE__, "cannot retrieve the sign type of the integer");
1140  }
1141  if (size == 0) {
1142  throw InternalErr(__FILE__, __LINE__, "cannot return the size of the datatype");
1143  }
1144  else if (size == 1) { // Either signed char or unsigned char
1145  // DAP2 doesn't support signed char, it maps to DAP int16.
1146  if (sign == H5T_SGN_2) {
1147  if (false == is_dap4) // signed char to DAP2 int16
1148  btp = new HDF5Int16(vname, vpath, dataset);
1149  else
1150  btp = new HDF5Int8(vname,vpath,dataset);
1151  }
1152  else
1153  btp = new HDF5Byte(vname, vpath,dataset);
1154  }
1155  else if (size == 2) {
1156  if (sign == H5T_SGN_2)
1157  btp = new HDF5Int16(vname, vpath,dataset);
1158  else
1159  btp = new HDF5UInt16(vname,vpath, dataset);
1160  }
1161  else if (size == 4) {
1162  if (sign == H5T_SGN_2){
1163  btp = new HDF5Int32(vname, vpath,dataset);
1164  }
1165  else
1166  btp = new HDF5UInt32(vname,vpath, dataset);
1167  }
1168  else if (size == 8) {
1169  if(true == is_dap4) {
1170  if(sign == H5T_SGN_2)
1171  btp = new HDF5Int64(vname,vpath, dataset);
1172  else
1173  btp = new HDF5UInt64(vname,vpath, dataset);
1174  }
1175  else {
1176  throw
1177  InternalErr(__FILE__, __LINE__,
1178  string("Unsupported HDF5 64-bit Integer type:")
1179  + vname);
1180  }
1181  }
1182  }
1183  break;
1184 
1185  case H5T_FLOAT:
1186  {
1187  size = H5Tget_size(datatype);
1188  BESDEBUG("h5", "=Get_bt() H5T_FLOAT size = " << size << endl);
1189 
1190  if (size == 0) {
1191  throw InternalErr(__FILE__, __LINE__, "cannot return the size of the datatype");
1192  }
1193  else if (size == 4) {
1194  btp = new HDF5Float32(vname,vpath, dataset);
1195  }
1196  else if (size == 8) {
1197  btp = new HDF5Float64(vname,vpath, dataset);
1198  }
1199  }
1200  break;
1201 
1202  case H5T_STRING:
1203  btp = new HDF5Str(vname, vpath,dataset);
1204  break;
1205 
1206  // The array datatype is rarely,rarely used. So this
1207  // part of code is not reviewed.
1208 #if 0
1209  case H5T_ARRAY: {
1210  BaseType *ar_bt = 0;
1211  try {
1212  BESDEBUG("h5",
1213  "=Get_bt() H5T_ARRAY datatype = " << datatype
1214  << endl);
1215 
1216  // Get the base datatype of the array
1217  hid_t dtype_base = H5Tget_super(datatype);
1218  ar_bt = Get_bt(vname, dataset, dtype_base);
1219  btp = new HDF5Array(vname, dataset, ar_bt);
1220  delete ar_bt; ar_bt = 0;
1221 
1222  // Set the size of the array.
1223  int ndim = H5Tget_array_ndims(datatype);
1224  size = H5Tget_size(datatype);
1225  int nelement = 1;
1226 
1227  if (dtype_base < 0) {
1228  throw InternalErr(__FILE__, __LINE__, "cannot return the base datatype");
1229  }
1230  if (ndim < 0) {
1231  throw InternalErr(__FILE__, __LINE__, "cannot return the rank of the array datatype");
1232  }
1233  if (size == 0) {
1234  throw InternalErr(__FILE__, __LINE__, "cannot return the size of the datatype");
1235  }
1236  BESDEBUG(cerr
1237  << "=Get_bt()" << " Dim = " << ndim
1238  << " Size = " << size
1239  << endl);
1240 
1241  hsize_t size2[DODS_MAX_RANK];
1242  if(H5Tget_array_dims(datatype, size2) < 0){
1243  throw
1244  InternalErr(__FILE__, __LINE__,
1245  string("Could not get array dims for: ")
1246  + vname);
1247  }
1248 
1249 
1250  HDF5Array &h5_ar = static_cast < HDF5Array & >(*btp);
1251  for (int dim_index = 0; dim_index < ndim; dim_index++) {
1252  h5_ar.append_dim(size2[dim_index]);
1253  BESDEBUG("h5", "=Get_bt() " << size2[dim_index] << endl);
1254  nelement = nelement * size2[dim_index];
1255  }
1256 
1257  h5_ar.set_did(dt_inst.dset);
1258  // Assign the array datatype id.
1259  h5_ar.set_tid(datatype);
1260  h5_ar.set_memneed(size);
1261  h5_ar.set_numdim(ndim);
1262  h5_ar.set_numelm(nelement);
1263  h5_ar.set_length(nelement);
1264  h5_ar.d_type = H5Tget_class(dtype_base);
1265  if (h5_ar.d_type == H5T_NO_CLASS){
1266  throw InternalErr(__FILE__, __LINE__, "cannot return the datatype class identifier");
1267  }
1268  }
1269  catch (...) {
1270  if( ar_bt ) delete ar_bt;
1271  if( btp ) delete btp;
1272  throw;
1273  }
1274  break;
1275  }
1276 #endif
1277 
1278  // Reference map to DAP URL, check the technical note.
1279  case H5T_REFERENCE:
1280  btp = new HDF5Url(vname, vpath,dataset);
1281  break;
1282 
1283  default:
1284  throw InternalErr(__FILE__, __LINE__,
1285  string("Unsupported HDF5 type: ") + vname);
1286  }
1287  }
1288  catch (...) {
1289  if( btp ) delete btp;
1290  throw;
1291  }
1292 
1293  if (!btp)
1294  throw InternalErr(__FILE__, __LINE__,
1295  string("Could not make a DAP variable for: ")
1296  + vname);
1297 
1298  BESDEBUG("h5", "<Get_bt()" << endl);
1299  return btp;
1300 }
1301 
1302 
1319 //static Structure *Get_structure(const string &varname,
1320 Structure *Get_structure(const string &varname,const string &vpath,
1321  const string &dataset,
1322  hid_t datatype,bool is_dap4)
1323 {
1324  HDF5Structure *structure_ptr = nullptr;
1325  char* memb_name = nullptr;
1326  hid_t memb_type = -1;
1327 
1328  BESDEBUG("h5", ">Get_structure()" << datatype << endl);
1329 
1330  if (H5Tget_class(datatype) != H5T_COMPOUND)
1331  throw InternalErr(__FILE__, __LINE__,
1332  string("Compound-to-structure mapping error for ")
1333  + varname);
1334 
1335  try {
1336  structure_ptr = new HDF5Structure(varname, vpath, dataset);
1337 
1338  // Retrieve member types
1339  int nmembs = H5Tget_nmembers(datatype);
1340  BESDEBUG("h5", "=Get_structure() has " << nmembs << endl);
1341  if (nmembs < 0){
1342  throw InternalErr(__FILE__, __LINE__, "cannot retrieve the number of elements");
1343  }
1344  for (int i = 0; i < nmembs; i++) {
1345  memb_name = H5Tget_member_name(datatype, i);
1346  H5T_class_t memb_cls = H5Tget_member_class(datatype, i);
1347  memb_type = H5Tget_member_type(datatype, i);
1348  if (memb_name == nullptr){
1349  throw InternalErr(__FILE__, __LINE__, "cannot retrieve the name of the member");
1350  }
1351  if ((memb_cls < 0) || (memb_type < 0)) {
1352  throw InternalErr(__FILE__, __LINE__,
1353  string("Type mapping error for ")
1354  + string(memb_name) );
1355  }
1356 
1357  if (memb_cls == H5T_COMPOUND) {
1358  Structure *s = Get_structure(memb_name, memb_name, dataset, memb_type,is_dap4);
1359  structure_ptr->add_var(s);
1360  delete s; s = 0;
1361  }
1362  else if(memb_cls == H5T_ARRAY) {
1363 
1364  BaseType *ar_bt = 0;
1365  BaseType *btp = 0;
1366  Structure *s = 0;
1367  hid_t dtype_base = 0;
1368 
1369  try {
1370 
1371  // Get the base memb_type of the array
1372  dtype_base = H5Tget_super(memb_type);
1373 
1374  // Set the size of the array.
1375  int ndim = H5Tget_array_ndims(memb_type);
1376  size_t size = H5Tget_size(memb_type);
1377  int nelement = 1;
1378 
1379  if (dtype_base < 0) {
1380  throw InternalErr(__FILE__, __LINE__, "cannot return the base memb_type");
1381  }
1382  if (ndim < 0) {
1383  throw InternalErr(__FILE__, __LINE__, "cannot return the rank of the array memb_type");
1384  }
1385  if (size == 0) {
1386  throw InternalErr(__FILE__, __LINE__, "cannot return the size of the memb_type");
1387  }
1388 
1389  hsize_t size2[DODS_MAX_RANK];
1390  if(H5Tget_array_dims(memb_type, size2) < 0){
1391  throw
1392  InternalErr(__FILE__, __LINE__,
1393  string("Could not get array dims for: ")
1394  + string(memb_name));
1395  }
1396 
1397  H5T_class_t array_memb_cls = H5Tget_class(dtype_base);
1398  if(array_memb_cls == H5T_NO_CLASS) {
1399  throw InternalErr(__FILE__, __LINE__,
1400  string("cannot get the correct class for compound type member")
1401  + string(memb_name));
1402  }
1403  if(H5T_COMPOUND == array_memb_cls) {
1404 
1405  s = Get_structure(memb_name, memb_name,dataset, dtype_base,is_dap4);
1406  HDF5Array *h5_ar = new HDF5Array(memb_name, dataset, s);
1407 
1408  for (int dim_index = 0; dim_index < ndim; dim_index++) {
1409  h5_ar->append_dim(size2[dim_index]);
1410  nelement = nelement * size2[dim_index];
1411  }
1412 
1413  h5_ar->set_memneed(size);
1414  h5_ar->set_numdim(ndim);
1415  h5_ar->set_numelm(nelement);
1416  h5_ar->set_length(nelement);
1417 
1418  structure_ptr->add_var(h5_ar);
1419  delete h5_ar;
1420 
1421  }
1422  else if (H5T_INTEGER == array_memb_cls || H5T_FLOAT == array_memb_cls || H5T_STRING == array_memb_cls) {
1423  ar_bt = Get_bt(memb_name, memb_name,dataset, dtype_base,is_dap4);
1424  HDF5Array *h5_ar = new HDF5Array(memb_name,dataset,ar_bt);
1425 
1426  for (int dim_index = 0; dim_index < ndim; dim_index++) {
1427  h5_ar->append_dim(size2[dim_index]);
1428  nelement = nelement * size2[dim_index];
1429  }
1430 
1431  h5_ar->set_memneed(size);
1432  h5_ar->set_numdim(ndim);
1433  h5_ar->set_numelm(nelement);
1434  h5_ar->set_length(nelement);
1435 
1436  structure_ptr->add_var(h5_ar);
1437  delete h5_ar;
1438  }
1439  if( ar_bt ) delete ar_bt;
1440  if( btp ) delete btp;
1441  if(s) delete s;
1442  H5Tclose(dtype_base);
1443 
1444  }
1445  catch (...) {
1446  if( ar_bt ) delete ar_bt;
1447  if( btp ) delete btp;
1448  if(s) delete s;
1449  H5Tclose(dtype_base);
1450  throw;
1451  }
1452 
1453  }
1454  else if (memb_cls == H5T_INTEGER || memb_cls == H5T_FLOAT || memb_cls == H5T_STRING) {
1455  BaseType *bt = Get_bt(memb_name, memb_name,dataset, memb_type,is_dap4);
1456  structure_ptr->add_var(bt);
1457  delete bt; bt = 0;
1458  }
1459  else {
1460  free(memb_name);
1461  memb_name = nullptr;
1462  throw InternalErr(__FILE__, __LINE__, "unsupported field datatype inside a compound datatype");
1463  }
1464  // Caller needs to free the memory allocated by the library for memb_name.
1465  if(memb_name != nullptr)
1466  free(memb_name);
1467  }
1468  }
1469  catch (...) {
1470  if( structure_ptr )
1471  delete structure_ptr;
1472  if(memb_name!= nullptr)
1473  free(memb_name);
1474  if(memb_type != -1)
1475  H5Tclose(memb_type);
1476  throw;
1477  }
1478 
1479  BESDEBUG("h5", "<Get_structure()" << endl);
1480 
1481  return structure_ptr;
1482 }
1483 
1500 
1501 // Function to use H5OVISIT to check if dimension scale attributes exist.
1502 bool check_dimscale(hid_t fileid) {
1503 
1504  bool ret_value = false;
1505  herr_t ret_o= H5OVISIT(fileid, H5_INDEX_NAME, H5_ITER_INC, visit_obj_cb, nullptr);
1506  if(ret_o < 0)
1507  throw InternalErr(__FILE__, __LINE__, "H5OVISIT fails");
1508  else
1509  ret_value =(ret_o >0)?true:false;
1510 
1511  return ret_value;
1512 }
1513 
1514 static int
1515 visit_obj_cb(hid_t group_id, const char *name, const H5O_info_t *oinfo,
1516  void *_op_data)
1517 {
1518 
1519  int ret_value = 0;
1520  if(oinfo->type == H5O_TYPE_DATASET) {
1521 
1522  hid_t dataset = -1;
1523  dataset = H5Dopen2(group_id,name,H5P_DEFAULT);
1524  if(dataset <0)
1525  throw InternalErr(__FILE__, __LINE__, "H5Dopen2 fails in the H5OVISIT call back function.");
1526 
1527  hid_t dspace = -1;
1528  dspace = H5Dget_space(dataset);
1529  if(dspace <0) {
1530  H5Dclose(dataset);
1531  throw InternalErr(__FILE__, __LINE__, "H5Dget_space fails in the H5OVISIT call back function.");
1532  }
1533 
1534  // We only support netCDF-4 like dimension scales, that is the dimension scale dataset is 1-D dimension.
1535  if(H5Sget_simple_extent_ndims(dspace) == 1) {
1536  try {
1537  if(true == has_dimscale_attr(dataset))
1538  ret_value = 1;
1539  }
1540  catch(...) {
1541  H5Sclose(dspace);
1542  H5Dclose(dataset);
1543  throw;
1544  }
1545 
1546 #if 0
1547  //vector<bool>dim_attr_mark;
1548  //dim_attr_mark.resize(4);
1549  //bool dim_attr_mark[4];
1550  int dim_attr_mark[4];
1551  for(int i =0;i<4;i++)
1552  dim_attr_mark[i] = 0;
1553  //int count = 0;
1554  // Check if having "class = DIMENSION_SCALE" and REFERENCE_LIST attributes.
1555  //herr_t ret = H5Aiterate2(dataset, H5_INDEX_NAME, H5_ITER_INC, nullptr, attr_info, &count);
1556  //herr_t ret = H5Aiterate2(dataset, H5_INDEX_NAME, H5_ITER_INC, nullptr, attr_info, dim_attr_mark.data());
1557  herr_t ret = H5Aiterate2(dataset, H5_INDEX_NAME, H5_ITER_INC, nullptr, attr_info, dim_attr_mark);
1558  if(ret < 0) {
1559  H5Sclose(dspace);
1560  H5Dclose(dataset);
1561  throw InternalErr(__FILE__, __LINE__, "H5Aiterate2 fails in the H5OVISIT call back function.");
1562  }
1563 
1564  BESDEBUG("h5", "<dset name is " << name <<endl);
1565  //BESDEBUG("h5", "<count is " << count <<endl);
1566  // Find it.
1567  if (dim_attr_mark[0] && dim_attr_mark[1]){
1568  //if (2==count || 3==count) {
1569  if(dspace != -1)
1570  H5Sclose(dspace);
1571  if(dataset != -1)
1572  H5Dclose(dataset);
1573  return 1;
1574  //}
1575  }
1576 #endif
1577  }
1578  if(dspace != -1)
1579  H5Sclose(dspace);
1580  if(dataset != -1)
1581  H5Dclose(dataset);
1582  }
1583  return ret_value;
1584 }
1585 
1586 bool has_dimscale_attr(hid_t dataset) {
1587 
1588  bool ret_value = false;
1589  string dimscale_attr_name="CLASS";
1590  string dimscale_attr_value="DIMENSION_SCALE";
1591  htri_t dimscale_attr_exist = H5Aexists_by_name(dataset,".",dimscale_attr_name.c_str(),H5P_DEFAULT);
1592  if(dimscale_attr_exist <0)
1593  throw InternalErr(__FILE__, __LINE__, "H5Aexists_by_name fails when checking the CLASS attribute.");
1594  else if(dimscale_attr_exist > 0) {//Attribute CLASS exists
1595 
1596  hid_t attr_id = -1;
1597  hid_t atype_id = -1;
1598 
1599  // Open the attribute
1600  attr_id = H5Aopen(dataset,dimscale_attr_name.c_str(), H5P_DEFAULT);
1601  if(attr_id < 0)
1602  throw InternalErr(__FILE__, __LINE__, "H5Aopen fails in the attr_info call back function.");
1603 
1604  // Get attribute datatype and dataspace
1605  atype_id = H5Aget_type(attr_id);
1606  if(atype_id < 0) {
1607  H5Aclose(attr_id);
1608  throw InternalErr(__FILE__, __LINE__, "H5Aget_type fails in the attr_info call back function.");
1609  }
1610 
1611  try {
1612  // Check if finding the attribute CLASS and the value is "DIMENSION_SCALE".
1613  if (H5T_STRING == H5Tget_class(atype_id))
1614  ret_value = check_str_attr_value(attr_id,atype_id,dimscale_attr_value,false);
1615  }
1616  catch(...) {
1617  if(atype_id != -1)
1618  H5Tclose(atype_id);
1619  if(attr_id != -1)
1620  H5Aclose(attr_id);
1621  throw;
1622  }
1623  // Close IDs.
1624  if(atype_id != -1)
1625  H5Tclose(atype_id);
1626  if(attr_id != -1)
1627  H5Aclose(attr_id);
1628  }
1629  return ret_value;
1630 
1631 }
1632 #if 0
1645 
1646 static herr_t
1647 attr_info_dimscale(hid_t loc_id, const char *name, const H5A_info_t *ainfo, void *opdata)
1648 {
1649  int *countp = (int*)opdata;
1650 
1651  hid_t attr_id = -1;
1652  hid_t atype_id = -1;
1653 
1654  // Open the attribute
1655  attr_id = H5Aopen(loc_id, name, H5P_DEFAULT);
1656  if(attr_id < 0)
1657  throw InternalErr(__FILE__, __LINE__, "H5Aopen fails in the attr_info call back function.");
1658 
1659  // Get attribute datatype and dataspace
1660  atype_id = H5Aget_type(attr_id);
1661  if(atype_id < 0) {
1662  H5Aclose(attr_id);
1663  throw InternalErr(__FILE__, __LINE__, "H5Aget_type fails in the attr_info call back function.");
1664  }
1665 
1666  try {
1667 
1668 #if 0
1669  // If finding the "REFERENCE_LIST", increases the countp.
1670  if ((H5T_COMPOUND == H5Tget_class(atype_id)) && (strcmp(name,"REFERENCE_LIST")==0)) {
1671  //(*countp)++;
1672  *dimattr_p = 1;
1673  //*dimattr_p = true;
1674  }
1675 #endif
1676  // Check if finding the attribute CLASS and the value is "DIMENSION_SCALE".
1677  if (H5T_STRING == H5Tget_class(atype_id)) {
1678  if (strcmp(name,"CLASS") == 0) {
1679  string dim_scale_mark = "DIMENSION_SCALE";
1680  bool is_dim_scale = check_str_attr_value(attr_id,atype_id,dim_scale_mark,false);
1681  if(true == is_dim_scale)
1682  (*countp)++;
1683  }
1684  }
1685  }
1686  catch(...) {
1687  if(atype_id != -1)
1688  H5Tclose(atype_id);
1689  if(attr_id != -1)
1690  H5Aclose(attr_id);
1691  throw;
1692  }
1693 
1694  // Close IDs.
1695  if(atype_id != -1)
1696  H5Tclose(atype_id);
1697  if(attr_id != -1)
1698  H5Aclose(attr_id);
1699 
1700  return 0;
1701 }
1702 
1703 #endif
1716 
1717 static herr_t
1718 attr_info_dimscale(hid_t loc_id, const char *name, const H5A_info_t *ainfo, void *opdata)
1719 {
1720  //bool *countp = (bool*)opdata;
1721  //bool *dimattr_p = (bool*)opdata;
1722  int *dimattr_p = (int*)opdata;
1723 
1724 #if 0
1725  bool has_reference_list = false;
1726  bool has_dimscale = false;
1727  bool has_name_as_var = false;
1728  bool has_name_as_nc4_purdim = false;
1729 #endif
1730 
1731 
1732  hid_t attr_id = -1;
1733  hid_t atype_id = -1;
1734 
1735  // Open the attribute
1736  attr_id = H5Aopen(loc_id, name, H5P_DEFAULT);
1737  if(attr_id < 0)
1738  throw InternalErr(__FILE__, __LINE__, "H5Aopen fails in the attr_info call back function.");
1739 
1740  // Get attribute datatype and dataspace
1741  atype_id = H5Aget_type(attr_id);
1742  if(atype_id < 0) {
1743  H5Aclose(attr_id);
1744  throw InternalErr(__FILE__, __LINE__, "H5Aget_type fails in the attr_info call back function.");
1745  }
1746 
1747  try {
1748 
1749 //#if 0
1750  // If finding the "REFERENCE_LIST", increases the countp.
1751  if ((H5T_COMPOUND == H5Tget_class(atype_id)) && (strcmp(name,"REFERENCE_LIST")==0)) {
1752  //(*countp)++;
1753  *dimattr_p = 1;
1754  //*dimattr_p = true;
1755  }
1756 //#endif
1757  // Check if finding the CLASS attribute.
1758  if (H5T_STRING == H5Tget_class(atype_id)) {
1759  if (strcmp(name,"NAME") == 0) {
1760 
1761  string pure_dimname_mark = "This is a netCDF dimension but not a netCDF variable";
1762  bool is_pure_dim = check_str_attr_value(attr_id,atype_id,pure_dimname_mark,true);
1763 
1764  BESDEBUG("h5","pure dimension name yes" << is_pure_dim <<endl);
1765  if(true == is_pure_dim)
1766  *(dimattr_p+1) =1;
1767  //*(dimattr_p+2) =true;
1768  else {
1769  // netCDF save the variable name in the "NAME" attribute.
1770  // We need to retrieve the variable name first.
1771  ssize_t objnamelen = -1;
1772  if ((objnamelen= H5Iget_name(loc_id,nullptr,0))<=0) {
1773  string msg = "Cannot obtain the variable name length." ;
1774  throw InternalErr(__FILE__,__LINE__,msg);
1775  }
1776  vector<char> objname;
1777  objname.resize(objnamelen+1);
1778  if ((objnamelen= H5Iget_name(loc_id,objname.data(),objnamelen+1))<=0) {
1779  string msg = "Cannot obtain the variable name." ;
1780  throw InternalErr(__FILE__,__LINE__,msg);
1781  }
1782 
1783  string objname_str = string(objname.begin(),objname.end());
1784 
1785  // Must trim the string delimter.
1786  objname_str = objname_str.substr(0,objnamelen);
1787  // Remove the path
1788  string normal_dimname_mark = objname_str.substr(objname_str.find_last_of("/")+1);
1789  bool is_normal_dim = check_str_attr_value(attr_id,atype_id,normal_dimname_mark,false);
1790  if(true == is_normal_dim)
1791  *(dimattr_p+2) = 1;
1792  //*(dimattr_p+3) = true;
1793  }
1794  }
1795  }
1796 
1797 #if 0
1798  H5T_str_t str_pad = H5Tget_strpad(atype_id);
1799 
1800  hid_t aspace_id = -1;
1801  aspace_id = H5Aget_space(attr_id);
1802  if(aspace_id < 0)
1803  throw InternalErr(__FILE__, __LINE__, "H5Aget_space fails in the attr_info call back function.");
1804 
1805  // CLASS is a variable-length string
1806  int ndims = H5Sget_simple_extent_ndims(aspace_id);
1807  hsize_t nelmts = 1;
1808 
1809  // if it is a scalar attribute, just define number of elements to be 1.
1810  if (ndims != 0) {
1811 
1812  vector<hsize_t> asize;
1813  vector<hsize_t> maxsize;
1814  asize.resize(ndims);
1815  maxsize.resize(ndims);
1816 
1817  // DAP applications don't care about the unlimited dimensions
1818  // since the applications only care about retrieving the data.
1819  // So we don't check the maxsize to see if it is the unlimited dimension
1820  // attribute.
1821  if (H5Sget_simple_extent_dims(aspace_id, asize.data(), maxsize.data())<0) {
1822  H5Sclose(aspace_id);
1823  throw InternalErr(__FILE__, __LINE__, "Cannot obtain the dim. info in the H5Aiterate2 call back function.");
1824  }
1825 
1826  // Return ndims and size[ndims].
1827  for (int j = 0; j < ndims; j++)
1828  nelmts *= asize[j];
1829  } // if(ndims != 0)
1830 
1831  size_t ty_size = H5Tget_size(atype_id);
1832  if (0 == ty_size) {
1833  H5Sclose(aspace_id);
1834  throw InternalErr(__FILE__, __LINE__, "Cannot obtain the type size in the H5Aiterate2 call back function.");
1835  }
1836 
1837  size_t total_bytes = nelmts * ty_size;
1838  string total_vstring ="";
1839  if(H5Tis_variable_str(atype_id) > 0) {
1840 
1841  // Variable length string attribute values only store pointers of the actual string value.
1842  vector<char> temp_buf;
1843  temp_buf.resize(total_bytes);
1844 
1845  if (H5Aread(attr_id, atype_id, temp_buf.data()) < 0){
1846  H5Sclose(aspace_id);
1847  throw InternalErr(__FILE__,__LINE__,"Cannot read the attribute in the H5Aiterate2 call back function");
1848  }
1849 
1850  char *temp_bp = nullptr;
1851  temp_bp = temp_buf.data();
1852  char* onestring = nullptr;
1853 
1854  for (unsigned int temp_i = 0; temp_i <nelmts; temp_i++) {
1855 
1856  // This line will assure that we get the real variable length string value.
1857  onestring =*(char **)temp_bp;
1858 
1859  if(onestring!= nullptr)
1860  total_vstring +=string(onestring);
1861 
1862  // going to the next value.
1863  temp_bp +=ty_size;
1864  }
1865 
1866  if ((temp_buf.data()) != nullptr) {
1867  // Reclaim any VL memory if necessary.
1868  if (H5Dvlen_reclaim(atype_id,aspace_id,H5P_DEFAULT,temp_buf.data()) < 0) {
1869  H5Sclose(aspace_id);
1870  throw InternalErr(__FILE__,__LINE__,"Cannot reclaim VL memory in the H5Aiterate2 call back function.");
1871  }
1872  }
1873 
1874  }
1875  else {// Fixed-size string, need to retrieve the string value.
1876 
1877  // string attribute values
1878  vector<char> temp_buf;
1879  temp_buf.resize(total_bytes);
1880  if (H5Aread(attr_id, atype_id, temp_buf.data()) < 0){
1881  H5Sclose(aspace_id);
1882  throw InternalErr(__FILE__,__LINE__,"Cannot read the attribute in the H5Aiterate2 call back function");
1883  }
1884  string temp_buf_string(temp_buf.begin(),temp_buf.end());
1885  total_vstring = temp_buf_string.substr(0,total_bytes);
1886 
1887  // Note: we need to remove the string pad or term to find DIMENSION_SCALE.
1888  if(str_pad != H5T_STR_ERROR)
1889  total_vstring = total_vstring.substr(0,total_vstring.size()-1);
1890  }
1891 
1892  // Close attribute data space ID.
1893  if(aspace_id != -1)
1894  H5Sclose(aspace_id);
1895  if(total_vstring == "DIMENSION_SCALE"){
1896  (*countp)++;
1897  }
1898 #endif
1899 
1900  }
1901  catch(...) {
1902  if(atype_id != -1)
1903  H5Tclose(atype_id);
1904  if(attr_id != -1)
1905  H5Aclose(attr_id);
1906  throw;
1907  }
1908 
1909  // Close IDs.
1910  if(atype_id != -1)
1911  H5Tclose(atype_id);
1912  if(attr_id != -1)
1913  H5Aclose(attr_id);
1914 
1915  return 0;
1916 }
1917 
1918 
1926 
1928 void obtain_dimnames(const hid_t file_id,hid_t dset,int ndims, DS_t *dt_inst_ptr,vector<link_info_t> & hdf5_hls) {
1929 
1930  htri_t has_dimension_list = -1;
1931 
1932  string dimlist_name = "DIMENSION_LIST";
1933  has_dimension_list = H5Aexists(dset,dimlist_name.c_str());
1934 
1935  if(has_dimension_list > 0 && ndims > 0) {
1936 
1937  hobj_ref_t rbuf;
1938  vector<hvl_t> vlbuf;
1939  vlbuf.resize(ndims);
1940 
1941  hid_t attr_id = -1;
1942  hid_t atype_id = -1;
1943  hid_t amemtype_id = -1;
1944  hid_t aspace_id = -1;
1945  hid_t ref_dset = -1;
1946 
1947  try {
1948  attr_id = H5Aopen(dset,dimlist_name.c_str(),H5P_DEFAULT);
1949  if (attr_id <0 ) {
1950  string msg = "Cannot open the attribute " + dimlist_name + " of HDF5 dataset "+ string(dt_inst_ptr->name);
1951  throw InternalErr(__FILE__, __LINE__, msg);
1952  }
1953 
1954  atype_id = H5Aget_type(attr_id);
1955  if (atype_id <0) {
1956  string msg = "Cannot get the datatype of the attribute " + dimlist_name + " of HDF5 dataset "+ string(dt_inst_ptr->name);
1957  throw InternalErr(__FILE__, __LINE__, msg);
1958  }
1959 
1960  amemtype_id = H5Tget_native_type(atype_id, H5T_DIR_ASCEND);
1961  if (amemtype_id < 0) {
1962  string msg = "Cannot get the memory datatype of the attribute " + dimlist_name + " of HDF5 dataset "+ string(dt_inst_ptr->name);
1963  throw InternalErr(__FILE__, __LINE__, msg);
1964 
1965  }
1966 
1967  if (H5Aread(attr_id,amemtype_id,vlbuf.data()) <0) {
1968  string msg = "Cannot obtain the referenced object for the variable " + string(dt_inst_ptr->name);
1969  throw InternalErr(__FILE__, __LINE__, msg);
1970  }
1971 
1972  vector<char> objname;
1973 
1974  // The dimension names of variables will be the HDF5 dataset names dereferenced from the DIMENSION_LIST attribute.
1975  for (int i = 0; i < ndims; i++) {
1976 
1977  if(vlbuf[i].p == nullptr) {
1978  stringstream sindex ;
1979  sindex <<i;
1980  string msg = "For variable " + string(dt_inst_ptr->name) + "; ";
1981  msg = msg + "the dimension of which the index is "+ sindex.str() + " doesn't exist. ";
1982  throw InternalErr(__FILE__, __LINE__, msg);
1983  }
1984 
1985  rbuf =((hobj_ref_t*)vlbuf[i].p)[0];
1986 
1987  if ((ref_dset = H5RDEREFERENCE(attr_id, H5R_OBJECT, &rbuf)) < 0) {
1988  string msg = "Cannot dereference from the DIMENSION_LIST attribute for the variable " + string(dt_inst_ptr->name);
1989  throw InternalErr(__FILE__, __LINE__, msg);
1990  }
1991 
1992  ssize_t objnamelen = -1;
1993  if ((objnamelen= H5Iget_name(ref_dset,nullptr,0))<=0) {
1994  string msg = "Cannot obtain the dimension name length for the variable " + string(dt_inst_ptr->name);
1995  throw InternalErr(__FILE__,__LINE__,msg);
1996  }
1997 
1998  objname.resize(objnamelen+1);
1999  if ((objnamelen= H5Iget_name(ref_dset,objname.data(),objnamelen+1))<=0) {
2000  H5Dclose(ref_dset);
2001  string msg = "Cannot obtain the dimension name for the variable " + string(dt_inst_ptr->name);
2002  throw InternalErr(__FILE__,__LINE__,msg);
2003  }
2004 
2005  auto objname_str = string(objname.begin(),objname.end());
2006 
2007  // Must trim the string delimter.
2008  string trim_objname = objname_str.substr(0,objnamelen);
2009 
2010  // We need to check if there are hardlinks for this variable.
2011  // If yes, we need to find the hardlink that has the shortest path and at the ancestor group
2012  // of all links.
2013  H5O_info_t obj_info;
2014  if(H5OGET_INFO(ref_dset,&obj_info)<0) {
2015  H5Dclose(ref_dset);
2016  string msg = "Cannot obtain the object info for the dimension variable " + objname_str;
2017  throw InternalErr(__FILE__,__LINE__,msg);
2018  }
2019 
2020  // This dimension indeed has hard links.
2021  if(obj_info.rc > 1) {
2022 
2023  // 1. Search the hdf5_hls to see if the address is inside
2024  // if yes,
2025  // obtain the hard link which has the shortest path, use this as the dimension name.
2026  // else
2027  // search all the hardlinks with the callback.
2028  // obtain the shortest path, add this to hdf5_hls.
2029 
2030  bool link_find = false;
2031 
2032  // If finding the object in the hdf5_hls, obtain the hardlink and make it the dimension name(trim_objname).
2033  for (unsigned int i = 0; i <hdf5_hls.size();i++) {
2034 #if (H5_VERS_MAJOR == 1 && ((H5_VERS_MINOR == 12) || (H5_VERS_MINOR == 13)))
2035  int token_cmp = -1;
2036  if(H5Otoken_cmp(ref_dset,&(obj_info.token),&(hdf5_hls[i].link_addr),&token_cmp) <0)
2037  throw InternalErr(__FILE__,__LINE__,"H5Otoken_cmp failed");
2038  if(!token_cmp) {
2039 #else
2040  if(obj_info.addr == hdf5_hls[i].link_addr) {
2041 #endif
2042  trim_objname = '/'+hdf5_hls[i].slink_path;
2043  link_find = true;
2044  break;
2045  }
2046  }
2047 
2048  // The hard link is not in the hdf5_hls, need to iterate all objects and find those hardlinks.
2049  if(link_find == false) {
2050 
2051 #if (H5_VERS_MAJOR == 1 && ((H5_VERS_MINOR == 12) || (H5_VERS_MINOR == 13)))
2052  typedef struct {
2053  unsigned link_unvisited;
2054  H5O_token_t link_addr;
2055  vector<string> hl_names;
2056  } t_link_info_t;
2057 #else
2058  typedef struct {
2059  unsigned link_unvisited;
2060  haddr_t link_addr;
2061  vector<string> hl_names;
2062  } t_link_info_t;
2063 #endif
2064 
2065  t_link_info_t t_li_info;
2066  t_li_info.link_unvisited = obj_info.rc;
2067 
2068 #if (H5_VERS_MAJOR == 1 && ((H5_VERS_MINOR == 12) || (H5_VERS_MINOR == 13)))
2069  memcpy(&t_li_info.link_addr,&obj_info.token,sizeof(H5O_token_t));
2070 #else
2071  t_li_info.link_addr = obj_info.addr;
2072 #endif
2073 
2074  if(H5Lvisit(file_id, H5_INDEX_NAME, H5_ITER_NATIVE, visit_link_cb, (void*)&t_li_info) < 0) {
2075  H5Dclose(ref_dset);
2076  string err_msg;
2077  err_msg = "Find all hardlinks: H5Lvisit failed to iterate all the objects";
2078  throw InternalErr(__FILE__,__LINE__,err_msg);
2079  }
2080 #if 0
2081 for(int i = 0; i<t_li_info.hl_names.size();i++)
2082  cerr<<"hl name is "<<t_li_info.hl_names[i] <<endl;
2083 #endif
2084 
2085  string shortest_hl = obtain_shortest_ancestor_path(t_li_info.hl_names);
2086 //cerr<<"shortest_hl is "<<shortest_hl <<endl;
2087  if(shortest_hl =="") {
2088  H5Dclose(ref_dset);
2089  string err_msg;
2090  err_msg = "The shortest hardlink is not located under an ancestor group of all links.";
2091  err_msg +="This is not supported by netCDF4 data model and the current Hyrax DAP4 implementation.";
2092  throw InternalErr(__FILE__,__LINE__,err_msg);
2093  }
2094 
2095  // Save this link that holds the shortest path for future use.
2096  link_info_t new_hdf5_hl;
2097 #if (H5_VERS_MAJOR == 1 && ((H5_VERS_MINOR == 12) || (H5_VERS_MINOR == 13)))
2098  memcpy(&new_hdf5_hl.link_addr,&obj_info.token,sizeof(H5O_token_t));
2099 #else
2100  new_hdf5_hl.link_addr = obj_info.addr;
2101 #endif
2102  new_hdf5_hl.slink_path = shortest_hl;
2103  hdf5_hls.push_back(new_hdf5_hl);
2104  trim_objname = '/'+shortest_hl;
2105 
2106  }
2107  }
2108  // Need to save the dimension names without the path
2109 
2110  dt_inst_ptr->dimnames.push_back(trim_objname.substr(trim_objname.find_last_of("/")+1));
2111  dt_inst_ptr->dimnames_path.push_back(trim_objname);
2112 
2113  if(H5Dclose(ref_dset)<0) {
2114  throw InternalErr(__FILE__,__LINE__,"Cannot close the HDF5 dataset in the function obtain_dimnames().");
2115  }
2116  objname.clear();
2117  }// for (vector<Dimension *>::iterator ird is var->dims.begin()
2118  if(vlbuf.empty()== false) {
2119 
2120  if ((aspace_id = H5Aget_space(attr_id)) < 0) {
2121  string msg = "Cannot close the HDF5 attribute space successfully for <DIMENSION_LIST> of the variable "+string(dt_inst_ptr->name);
2122  throw InternalErr(__FILE__,__LINE__,msg);
2123  }
2124 
2125  if (H5Dvlen_reclaim(amemtype_id,aspace_id,H5P_DEFAULT,(void*)vlbuf.data())<0) {
2126  throw InternalErr(__FILE__,__LINE__,"Cannot reclaim the variable length memory in the function obtain_dimnames()");
2127  }
2128 
2129  H5Sclose(aspace_id);
2130 
2131  }
2132 
2133  H5Tclose(atype_id);
2134  H5Tclose(amemtype_id);
2135  H5Aclose(attr_id);
2136 
2137  }
2138 
2139  catch(...) {
2140 
2141  if(atype_id != -1)
2142  H5Tclose(atype_id);
2143 
2144  if(amemtype_id != -1)
2145  H5Tclose(amemtype_id);
2146 
2147  if(aspace_id != -1)
2148  H5Sclose(aspace_id);
2149 
2150  if(attr_id != -1)
2151  H5Aclose(attr_id);
2152 
2153  throw;
2154  }
2155 
2156  }
2157  return ;
2158 }
2159 
2160 void write_vlen_str_attrs(hid_t attr_id,hid_t ty_id, const DSattr_t * attr_inst_ptr,D4Attribute *d4_attr, AttrTable* d2_attr,bool is_dap4){
2161 
2162  BESDEBUG("h5","attribute name " << attr_inst_ptr->name <<endl);
2163  BESDEBUG("h5","attribute size " <<attr_inst_ptr->need <<endl);
2164  BESDEBUG("h5","attribute type size " <<(int)(H5Tget_size(ty_id))<<endl);
2165 
2166  hid_t temp_space_id = H5Aget_space(attr_id);
2167  BESDEBUG("h5","attribute calculated size "<<(int)(H5Tget_size(ty_id)) *(int)(H5Sget_simple_extent_npoints(temp_space_id)) <<endl);
2168  if(temp_space_id <0) {
2169  H5Tclose(ty_id);
2170  H5Aclose(attr_id);
2171  if(d4_attr)
2172  delete d4_attr;
2173  throw InternalErr(__FILE__, __LINE__, "unable to read HDF5 attribute data");
2174  }
2175 
2176  vector<char> temp_buf;
2177  // Variable length string attribute values only store pointers of the actual string value.
2178  temp_buf.resize((size_t)attr_inst_ptr->need);
2179 
2180  if (H5Aread(attr_id, ty_id, temp_buf.data()) < 0) {
2181  H5Tclose(ty_id);
2182  H5Aclose(attr_id);
2183  H5Sclose(temp_space_id);
2184  if(d4_attr)
2185  delete d4_attr;
2186  throw InternalErr(__FILE__, __LINE__, "unable to read HDF5 attribute data");
2187  }
2188 
2189  char *temp_bp;
2190  temp_bp = temp_buf.data();
2191  for (unsigned int temp_i = 0; temp_i <attr_inst_ptr->nelmts; temp_i++) {
2192 
2193  // This line will assure that we get the real variable length string value.
2194  char* onestring =*(char **)temp_bp;
2195 
2196  // Change the C-style string to C++ STD string just for easy appending the attributes in DAP.
2197  if (onestring !=nullptr) {
2198  string tempstring(onestring);
2199  if(true == is_dap4)
2200  d4_attr->add_value(tempstring);
2201  else
2202  d2_attr->append_attr(attr_inst_ptr->name,"String",tempstring);
2203  }
2204 
2205  temp_bp +=H5Tget_size(ty_id);
2206  }
2207  if (temp_buf.empty() != true) {
2208 
2209  // Reclaim any VL memory if necessary.
2210  herr_t ret_vlen_claim;
2211  ret_vlen_claim = H5Dvlen_reclaim(ty_id,temp_space_id,H5P_DEFAULT,temp_buf.data());
2212  if(ret_vlen_claim < 0){
2213  H5Tclose(ty_id);
2214  H5Aclose(attr_id);
2215  H5Sclose(temp_space_id);
2216  if(d4_attr)
2217  delete d4_attr;
2218  throw InternalErr(__FILE__, __LINE__, "Cannot reclaim the memory buffer of the HDF5 variable length string.");
2219  }
2220 
2221  temp_buf.clear();
2222  }
2223  H5Sclose(temp_space_id);
2224 }
2225 
2226 bool check_str_attr_value(hid_t attr_id,hid_t atype_id,const string & value_to_compare,bool check_substr) {
2227 
2228  bool ret_value = false;
2229 
2230  H5T_str_t str_pad = H5Tget_strpad(atype_id);
2231  if(str_pad == H5T_STR_ERROR)
2232  throw InternalErr(__FILE__, __LINE__, "Fail to obtain string pad.");
2233 
2234  hid_t aspace_id = -1;
2235  aspace_id = H5Aget_space(attr_id);
2236  if(aspace_id < 0)
2237  throw InternalErr(__FILE__, __LINE__, "Fail to obtain attribute space.");
2238 
2239  int ndims = H5Sget_simple_extent_ndims(aspace_id);
2240  if(ndims < 0) {
2241  H5Sclose(aspace_id);
2242  throw InternalErr(__FILE__, __LINE__, "Fail to obtain number of dimensions.");
2243  }
2244 
2245  hsize_t nelmts = 1;
2246 
2247  // if it is a scalar attribute, just define number of elements to be 1.
2248  if (ndims != 0) {
2249 
2250  vector<hsize_t> asize;
2251  asize.resize(ndims);
2252  if (H5Sget_simple_extent_dims(aspace_id, asize.data(), nullptr)<0) {
2253  H5Sclose(aspace_id);
2254  throw InternalErr(__FILE__, __LINE__, "Fail to obtain the dimension info.");
2255  }
2256 
2257  // Return ndims and size[ndims].
2258  for (int j = 0; j < ndims; j++)
2259  nelmts *= asize[j];
2260  } // if(ndims != 0)
2261 
2262  size_t ty_size = H5Tget_size(atype_id);
2263  if (0 == ty_size) {
2264  H5Sclose(aspace_id);
2265  throw InternalErr(__FILE__, __LINE__, "Fail to obtain the type size.");
2266  }
2267 
2268  size_t total_bytes = nelmts * ty_size;
2269  string total_vstring ="";
2270  if(H5Tis_variable_str(atype_id) > 0) {
2271 
2272  // Variable length string attribute values only store pointers of the actual string value.
2273  vector<char> temp_buf;
2274  temp_buf.resize(total_bytes);
2275 
2276  if (H5Aread(attr_id, atype_id, temp_buf.data()) < 0){
2277  H5Sclose(aspace_id);
2278  throw InternalErr(__FILE__,__LINE__,"Fail to read the attribute.");
2279  }
2280 
2281  char *temp_bp = nullptr;
2282  temp_bp = temp_buf.data();
2283 
2284  for (unsigned int temp_i = 0; temp_i <nelmts; temp_i++) {
2285 
2286  // This line will assure that we get the real variable length string value.
2287  char* onestring =*(char **)temp_bp;
2288 
2289  if(onestring!= nullptr)
2290  total_vstring +=string(onestring);
2291 
2292  // going to the next value.
2293  temp_bp +=ty_size;
2294  }
2295 
2296  if ((temp_buf.data()) != nullptr) {
2297  // Reclaim any VL memory if necessary.
2298  if (H5Dvlen_reclaim(atype_id,aspace_id,H5P_DEFAULT,temp_buf.data()) < 0) {
2299  H5Sclose(aspace_id);
2300  throw InternalErr(__FILE__,__LINE__,"Fail to reclaim VL memory.");
2301  }
2302  }
2303 
2304  }
2305  else {// Fixed-size string, need to retrieve the string value.
2306 
2307  // string attribute values
2308  vector<char> temp_buf;
2309  temp_buf.resize(total_bytes);
2310  if (H5Aread(attr_id, atype_id, temp_buf.data()) < 0){
2311  H5Sclose(aspace_id);
2312  throw InternalErr(__FILE__,__LINE__,"Fail to read the attribute.");
2313  }
2314  string temp_buf_string(temp_buf.begin(),temp_buf.end());
2315  total_vstring = temp_buf_string.substr(0,total_bytes);
2316 
2317  // Note: we need to remove the string pad or term to find DIMENSION_SCALE.
2318  if(str_pad != H5T_STR_ERROR)
2319  total_vstring = total_vstring.substr(0,total_vstring.size()-1);
2320  }
2321 
2322  // Close attribute data space ID.
2323  if(aspace_id != -1)
2324  H5Sclose(aspace_id);
2325 
2326  if(false == check_substr) {
2327  if(total_vstring == value_to_compare)
2328  ret_value = true;
2329  }
2330  else {
2331  if(total_vstring.size()>=value_to_compare.size()) {
2332  if( 0 == total_vstring.compare(0,value_to_compare.size(),value_to_compare))
2333  ret_value = true;
2334  }
2335  }
2336  return ret_value;
2337 }
2338 
2339 // Call back function used by H5Lvisit that iterates all HDF5 objects.
2340 static int
2341 visit_link_cb(hid_t group_id, const char *name, const H5L_info_t *linfo,
2342  void *_op_data)
2343 {
2344 #if (H5_VERS_MAJOR == 1 && ((H5_VERS_MINOR == 12) || (H5_VERS_MINOR == 13)))
2345  typedef struct {
2346  unsigned link_unvisited;
2347  H5O_token_t link_addr;
2348  vector<string> hl_names;
2349  } t_link_info_t;
2350 #else
2351  typedef struct {
2352  unsigned link_unvisited;
2353  haddr_t link_addr;
2354  vector<string> hl_names;
2355  } t_link_info_t;
2356 #endif
2357 
2358  t_link_info_t *op_data = (t_link_info_t *)_op_data;
2359  int ret = 0;
2360 
2361  // We only need the hard link info.
2362  if(linfo->type == H5L_TYPE_HARD) {
2363 #if (H5_VERS_MAJOR == 1 && ((H5_VERS_MINOR == 12) || (H5_VERS_MINOR == 13)))
2364  int token_cmp = -1;
2365  if(H5Otoken_cmp(group_id,&(op_data->link_addr),&(linfo->u.token),&token_cmp) <0)
2366  throw InternalErr(__FILE__,__LINE__,"H5Otoken_cmp failed");
2367  if(!token_cmp) {
2368 #else
2369  if(op_data->link_addr == linfo->u.address) {
2370 #endif
2371  op_data->link_unvisited = op_data->link_unvisited -1;
2372  string tmp_str(name,name+strlen(name));
2373  op_data->hl_names.push_back(tmp_str);
2374  // Once visiting all hard links, stop.
2375  if(op_data->link_unvisited == 0)
2376  ret = 1;
2377  }
2378 
2379  }
2380  return ret;
2381 
2382 }
2383 
2384 // Obtain the shortest path of all hard links of the object.
2385 std::string obtain_shortest_ancestor_path(const std::vector<std::string> & hls) {
2386 
2387  vector<string> hls_path;
2388  char slash = '/';
2389  bool hl_under_root = false;
2390  string ret_str ="";
2391  unsigned i = 0;
2392 
2393  for (i= 0; i<hls.size(); i++) {
2394 
2395  size_t path_pos = hls[i].find_last_of(slash);
2396 
2397  // The hard link may be under the root group,
2398  // This is the shortest path, we will return this path.
2399  if(path_pos == std::string::npos) {
2400  //Found
2401  hl_under_root = true;
2402  break;
2403  }
2404  else {
2405  string tmp_str = hls[i].substr(0,path_pos+1);
2406  hls_path.push_back(tmp_str);
2407  }
2408  }
2409 
2410  if(hl_under_root)
2411  ret_str = hls[i];
2412 
2413  else {
2414  // We just need to find the minimum size.
2415  unsigned short_path_index = 0;
2416  unsigned min_path_size = hls_path[0].size();
2417 
2418  // Find the shortest path index
2419  for(unsigned j = 1; j <hls_path.size();j++) {
2420  if(min_path_size>hls_path[j].size()) {
2421  min_path_size = hls_path[j].size();
2422  short_path_index = j;
2423  }
2424  }
2425  string tmp_sp = hls_path[short_path_index];
2426  ret_str = hls[short_path_index];
2427 
2428  //check if all hardlinks have a common ancestor link
2429  // If not, set the return value be the empty string.
2430  for(unsigned j = 0; j <hls_path.size();j++) {
2431  if(hls_path[j].find(tmp_sp)!=0) {
2432  ret_str ="";
2433  break;
2434  }
2435  }
2436  }
2437  return ret_str;
2438 
2439 
2440 }
HDF5Array.h
A class for handling all types of array in HDF5 for the default option.
HDF5Byte.h
This class provides a way to map HDF5 byte to DAP Byte for the default option.
HDF5Float32.h
A class for mapping HDF5 32-bit float to DAP for the default option.
HDF5Float64.h
A class for mapping HDF5 64-bit float to DAP for the default option.
HDF5Int16.h
A class for HDF5 signed 16 bit integer type.
HDF5Int32.h
This class provides a way to map HDF5 32 bit integer to DAP Int32 for the default option.
HDF5Int64.h
This class provides a way to map HDF5 Int64 to DAP Int64 for the default option.
HDF5Int8.h
This class provides a way to map HDF5 int8 to DAP Int8 for the default option.
HDF5RequestHandler.h
include the entry functions to execute the handlers
HDF5Str.h
This class that translates HDF5 string into DAP string for the default option.
HDF5Structure.h
This class converts HDF5 compound type into DAP structure for the default option.
HDF5UInt16.h
This class provides a way to map unsigned HDF5 16 bit integer to DAP UInt16 for the default option.
HDF5UInt32.h
This class provides a way to map unsigned HDF5 32 bit integer to DAP UInt32.
HDF5UInt64.h
This class provides a way to map HDF5 uint64 to DAP UInt64 for the default option.
HDF5Url.h
This class generates DAP URL type for the default option.
HDF5Array::set_numdim
void set_numdim(int ndims)
remembers number of dimensions of this array.
Definition: HDF5Array.cc:1732
HDF5Array::set_numelm
void set_numelm(int nelms)
remembers number of elements in this array.
Definition: HDF5Array.cc:1736
HDF5Array::set_memneed
void set_memneed(size_t need)
remembers memory size needed.
Definition: HDF5Array.cc:1728
print_attr
string print_attr(hid_t type, int loc, void *sm_buf)
Definition: h5get.cc:868
check_h5str
bool check_h5str(hid_t h5type)
Definition: h5get.cc:849
get_fileid
hid_t get_fileid(const char *filename)
Definition: h5get.cc:412
get_dap_type
string get_dap_type(hid_t type, bool is_dap4)
Definition: h5get.cc:292
close_fileid
void close_fileid(hid_t fid)
Definition: h5get.cc:434
get_dataset
void get_dataset(hid_t pid, const string &dname, DS_t *dt_inst_ptr)
Definition: h5get.cc:453
get_attr_info
hid_t get_attr_info(hid_t dset, int index, bool is_dap4, DSattr_t *attr_inst_ptr, bool *ignore_attr_ptr)
Definition: h5get.cc:90
DODS_MAX_RANK
const int DODS_MAX_RANK
Maximum number of dimensions in an array(default option only).
Definition: hdf5_handler.h:63
DS
A structure for DDS generation.
Definition: hdf5_handler.h:71
DS::name
char name[DODS_NAMELEN]
Name of HDF5 group or dataset.
Definition: hdf5_handler.h:73
DSattr
A structure for DAS generation.
Definition: hdf5_handler.h:96
DSattr::name
char name[DODS_NAMELEN]
Name of HDF5 group or dataset.
Definition: hdf5_handler.h:98
DSattr::nelmts
hsize_t nelmts
Number of elements.
Definition: hdf5_handler.h:106
DSattr::need
hsize_t need
Memory space needed to hold nelmts type.
Definition: hdf5_handler.h:108
link_info_t
Definition: h5dmr.h:69