bes  Updated for version 3.20.13
HDF5CF.cc
Go to the documentation of this file.
1 // This file is part of the hdf5_handler implementing for the CF-compliant
2 // Copyright (c) 2011-2016 The HDF Group, Inc. and OPeNDAP, Inc.
3 //
4 // This is free software; you can redistribute it and/or modify it under the
5 // terms of the GNU Lesser General Public License as published by the Free
6 // Software Foundation; either version 2.1 of the License, or (at your
7 // option) any later version.
8 //
9 // This software is distributed in the hope that it will be useful, but
10 // WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
11 // or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
12 // License for more details.
13 //
14 // You should have received a copy of the GNU Lesser General Public
15 // License along with this library; if not, write to the Free Software
16 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 //
18 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
19 // You can contact The HDF Group, Inc. at 1800 South Oak Street,
20 // Suite 203, Champaign, IL 61820
21 
36 
37 #include <sstream>
38 #include <algorithm>
39 #include <functional>
40 #include <climits>
41 #include "HDF5CF.h"
42 #include "h5cfdaputil.h"
43 #include "HDF5RequestHandler.h"
44 #include "h5apicompatible.h"
45 #include "BESDebug.h"
46 
47 using namespace HDF5CF;
48 
49 Var::Var(Var *var)
50 {
51 
52  newname = var->newname;
53  name = var->name;
54  fullpath = var->fullpath;
55  rank = var->rank;
56  total_elems = var->total_elems;
57  zero_storage_size = var->zero_storage_size;
58  dtype = var->dtype;
59  comp_ratio = var->comp_ratio;
60  unsupported_attr_dtype = var->unsupported_attr_dtype;
61  unsupported_attr_dspace = var->unsupported_attr_dspace;
62  unsupported_dspace = var->unsupported_dspace;
63  unsupported_attr_dspace = var->unsupported_attr_dspace;
64  dimnameflag = var->dimnameflag;
65  coord_attr_add_path = var->coord_attr_add_path;
66 
67  for (auto ira = var->attrs.begin(); ira != var->attrs.end(); ++ira) {
68  Attribute* attr = new Attribute();
69  attr->name = (*ira)->name;
70  attr->newname = (*ira)->newname;
71  attr->dtype = (*ira)->dtype;
72  attr->count = (*ira)->count;
73  attr->strsize = (*ira)->strsize;
74  attr->fstrsize = (*ira)->fstrsize;
75  attr->value = (*ira)->value;
76  attrs.push_back(attr);
77  }
78 
79  for (auto ird = var->dims.begin(); ird != var->dims.end(); ++ird) {
80  Dimension *dim = new Dimension((*ird)->size);
81  dim->name = (*ird)->name;
82  dim->newname = (*ird)->newname;
83  dim->unlimited_dim = (*ird)->unlimited_dim;
84  dims.push_back(dim);
85  }
86 
87 }
88 
89 bool CVar::isLatLon() const
90 {
91 
92  bool ret_value = false;
93  if (CV_EXIST == this->cvartype || CV_MODIFY == this->cvartype || CV_SPECIAL == this->cvartype) {
94  string attr_name = "units";
95  string lat_unit_value = "degrees_north";
96  string lon_unit_value = "degrees_east";
97 
98  for (auto ira = this->attrs.begin(); ira != this->attrs.end(); ira++) {
99 
100  if ((H5FSTRING == (*ira)->getType()) || (H5VSTRING == (*ira)->getType())) {
101  if (attr_name == (*ira)->newname) {
102  string attr_value1((*ira)->getValue().begin(), (*ira)->getValue().end());
103 
104  if ((*ira)->getCount() == 1) {
105  string attr_value((*ira)->getValue().begin(), (*ira)->getValue().end());
106  if (attr_value.compare(0, lat_unit_value.size(), lat_unit_value) == 0) {
107  if (attr_value.size() == lat_unit_value.size()) {
108  ret_value = true;
109  break;
110  }
111  else if (attr_value.size() == (lat_unit_value.size() + 1)) {
112  if (attr_value[attr_value.size() - 1] == '\0'
113  || attr_value[attr_value.size() - 1] == ' ') {
114  ret_value = true;
115  break;
116  }
117  }
118  }
119  else if (attr_value.compare(0, lon_unit_value.size(), lon_unit_value) == 0) {
120  if (attr_value.size() == lon_unit_value.size()) {
121  ret_value = true;
122  break;
123  }
124  else if (attr_value.size() == (lon_unit_value.size() + 1)) {
125  if (attr_value[attr_value.size() - 1] == '\0'
126  || attr_value[attr_value.size() - 1] == ' ') {
127  ret_value = true;
128  break;
129  }
130  }
131 
132  }
133 
134  }
135  }
136  }
137  }
138  }
139  else if (this->cvartype == CV_LAT_MISS || this->cvartype == CV_LON_MISS) ret_value = true;
140  return ret_value;
141 
142 }
143 File::~File()
144 {
145 
146  if (this->fileid >= 0) {
147  if (this->rootid >= 0) {
148  for_each(this->groups.begin(), this->groups.end(), delete_elem());
149  for_each(this->vars.begin(), this->vars.end(), delete_elem());
150  for_each(this->root_attrs.begin(), this->root_attrs.end(), delete_elem());
151  H5Gclose(rootid);
152  }
153  }
154 }
155 
156 Group::~Group()
157 {
158  for_each(this->attrs.begin(), this->attrs.end(), delete_elem());
159 }
160 
161 Var::~Var()
162 {
163  for_each(this->dims.begin(), this->dims.end(), delete_elem());
164  for_each(this->attrs.begin(), this->attrs.end(), delete_elem());
165 }
166 
167 
168 void File::Retrieve_H5_Info(const char * /*path*/, hid_t file_id, bool include_attr)
169 {
170 
171  BESDEBUG("h5", "coming to Retrieve_H5_Info" <<endl);
172 
173  if (true == include_attr) {
174 
175  // Obtain the BES key to check the ignored objects
176  // We will only use DAS to output these information.
177  this->check_ignored = HDF5RequestHandler::get_check_ignore_obj();
178  if (true == this->check_ignored) this->add_ignored_info_page_header();
179 
180  }
181 
182  hid_t root_id;
183  if ((root_id = H5Gopen(file_id, "/", H5P_DEFAULT)) < 0) {
184  throw1("Cannot open the HDF5 root group ");
185  }
186  this->rootid = root_id;
187  try {
188  this->Retrieve_H5_Obj(root_id, "/", include_attr);
189  }
190  catch (...) {
191  throw;
192  }
193 
194  // Obtain attributes only necessary
195  if (true == include_attr) {
196 
197  // Find the file(root group) attribute
198 
199  // Obtain the object type, such as group or dataset.
200  H5O_info_t oinfo;
201  int num_attrs = 0;
202 
203  if (H5OGET_INFO(root_id, &oinfo) < 0)
204  throw1("Error obtaining the info for the root group");
205 
206  num_attrs = (int)(oinfo.num_attrs);
207  bool temp_unsup_attr_atype = false;
208  bool temp_unsup_attr_dspace = false;
209 
210  for (int j = 0; j < num_attrs; j++) {
211  auto attr = new Attribute();
212  try {
213  this->Retrieve_H5_Attr_Info(attr, root_id, j, temp_unsup_attr_atype, temp_unsup_attr_dspace);
214  }
215  catch (...) {
216  delete attr;
217  throw;
218 
219  }
220  this->root_attrs.push_back(attr);
221  }
222 
223  this->unsupported_attr_dtype = temp_unsup_attr_atype;
224  this->unsupported_attr_dspace = temp_unsup_attr_dspace;
225  }
226 }
227 
228 void File::Retrieve_H5_Obj(hid_t grp_id, const char*gname, bool include_attr)
229 {
230 
231  // Iterate through the file to see the members of the group from the root.
232  H5G_info_t g_info;
233  hsize_t nelems = 0;
234 
235  if (H5Gget_info(grp_id, &g_info) < 0)
236  throw2("Counting hdf5 group elements error for ", gname);
237  nelems = g_info.nlinks;
238 
239  ssize_t oname_size = 0;
240  for (hsize_t i = 0; i < nelems; i++) {
241 
242  hid_t cgroup = -1;
243  hid_t cdset = -1;
244  Group *group = nullptr;
245  Var *var = nullptr;
246  Attribute *attr = nullptr;
247 
248  try {
249 
250  size_t dummy_name_len = 1;
251 
252  // Query the length of object name.
253  oname_size = H5Lget_name_by_idx(grp_id, ".", H5_INDEX_NAME, H5_ITER_NATIVE, i, nullptr, dummy_name_len,
254  H5P_DEFAULT);
255  if (oname_size <= 0)
256  throw2("Error getting the size of the hdf5 object from the group: ", gname);
257 
258  // Obtain the name of the object
259  vector<char> oname;
260  oname.resize((size_t) oname_size + 1);
261 
262  if (H5Lget_name_by_idx(grp_id, ".", H5_INDEX_NAME, H5_ITER_NATIVE, i, oname.data(), (size_t) (oname_size + 1),
263  H5P_DEFAULT) < 0)
264  throw2("Error getting the hdf5 object name from the group: ", gname);
265 
266  // Check if it is a hard link or a soft link
267  H5L_info_t linfo;
268  if (H5Lget_info(grp_id, oname.data(), &linfo, H5P_DEFAULT) < 0)
269  throw2("HDF5 link name error from ", gname);
270 
271  // We ignore soft links and external links for the CF options
272  if (H5L_TYPE_SOFT == linfo.type || H5L_TYPE_EXTERNAL == linfo.type) {
273  if (true == include_attr && true == check_ignored) {
274  this->add_ignored_info_links_header();
275  string full_path_name;
276  string temp_oname(oname.begin(), oname.end());
277  full_path_name = (
278  (string(gname) != "/") ?
279  (string(gname) + "/" + temp_oname.substr(0, temp_oname.size() - 1)) :
280  ("/" + temp_oname.substr(0, temp_oname.size() - 1)));
281  this->add_ignored_info_links(full_path_name);
282 
283  }
284  continue;
285  }
286 
287  // Obtain the object type, such as group or dataset.
288  H5O_info_t oinfo;
289 
290  if (H5OGET_INFO_BY_IDX(grp_id, ".", H5_INDEX_NAME, H5_ITER_NATIVE, i, &oinfo, H5P_DEFAULT) < 0)
291  throw2("Error obtaining the info for the object ", string(oname.begin(), oname.end()));
292 
293  H5O_type_t obj_type = oinfo.type;
294 
295  switch (obj_type) {
296 
297  case H5O_TYPE_GROUP: {
298 
299  // Obtain the full path name
300  string full_path_name;
301  string temp_oname(oname.begin(), oname.end());
302 
303  full_path_name = (
304  (string(gname) != "/") ?
305  (string(gname) + "/" + temp_oname.substr(0, temp_oname.size() - 1)) :
306  ("/" + temp_oname.substr(0, temp_oname.size() - 1)));
307 
308  cgroup = H5Gopen(grp_id, full_path_name.c_str(), H5P_DEFAULT);
309  if (cgroup < 0)
310  throw2("Error opening the group ", full_path_name);
311 
312  group = new Group();
313  group->path = full_path_name;
314  group->newname = full_path_name;
315 
316  // Retrieve group attribute if the attribute flag is true
317  if (true == include_attr) {
318 
319  int num_attrs = (int)(oinfo.num_attrs);
320  bool temp_unsup_attr_dtype = false;
321  bool temp_unsup_attr_dspace = false;
322 
323  for (int j = 0; j < num_attrs; j++) {
324 
325  attr = new Attribute();
326  Retrieve_H5_Attr_Info(attr, cgroup, j, temp_unsup_attr_dtype, temp_unsup_attr_dspace);
327  group->attrs.push_back(attr);
328  attr = nullptr;
329  }
330 
331  group->unsupported_attr_dtype = temp_unsup_attr_dtype;
332  group->unsupported_attr_dspace = temp_unsup_attr_dspace;
333  }
334  this->groups.push_back(group);
335  Retrieve_H5_Obj(cgroup, full_path_name.c_str(), include_attr);
336  if (H5Gclose(cgroup) < 0)
337  throw2("Error closing the group ", full_path_name);
338  }
339  break;
340  case H5O_TYPE_DATASET: {
341 
342  // Obtain the absolute path of the HDF5 dataset
343  string temp_oname(oname.begin(), oname.end());
344  string full_path_name = (
345  (string(gname) != "/") ?
346  (string(gname) + "/" + temp_oname.substr(0, temp_oname.size() - 1)) :
347  ("/" + temp_oname.substr(0, temp_oname.size() - 1)));
348 
349  var = new Var();
350  var->name = temp_oname.substr(0, temp_oname.size() - 1);
351  var->fullpath = full_path_name;
352 
353  // newname is for the final CF name
354  var->newname = full_path_name;
355 
356  cdset = H5Dopen(grp_id, full_path_name.c_str(), H5P_DEFAULT);
357  if (cdset < 0)
358  throw2("Error opening the HDF5 dataset ", full_path_name);
359 
360  // Retrieve the HDF5 dataset datatype, return the flag for unsupported types.
361  bool temp_unsup_var_dtype = false;
362  Retrieve_H5_VarType(var, cdset, full_path_name, temp_unsup_var_dtype);
363 
364  // Update the unsupported datatype flag
365  if (!this->unsupported_var_dtype && temp_unsup_var_dtype) this->unsupported_var_dtype = true;
366 
367  // Retrieve the HDF5 dataset data space, return the flag for unsupported dataspaces.
368  bool temp_unsup_var_dspace = false;
369  Retrieve_H5_VarDim(var, cdset, full_path_name, temp_unsup_var_dspace);
370 
371  // Update the unsupported data space flag
372  if (!this->unsupported_var_dspace && temp_unsup_var_dspace) this->unsupported_var_dspace = true;
373 
374  hsize_t d_storage_size = H5Dget_storage_size(cdset);
375  var->zero_storage_size =(d_storage_size ==0);
376  var->comp_ratio = Retrieve_H5_VarCompRatio(var, cdset);
377 
378  // Retrieve the attribute info. if asked
379  if (true == include_attr) {
380 
381  int num_attrs = (int)(oinfo.num_attrs);
382  bool temp_unsup_attr_dtype = false;
383  bool temp_unsup_attr_dspace = false;
384 
385  for (int j = 0; j < num_attrs; j++) {
386 
387  attr = new Attribute();
388 
389  Retrieve_H5_Attr_Info(attr, cdset, j, temp_unsup_attr_dtype, temp_unsup_attr_dspace);
390  var->attrs.push_back(attr);
391  attr = nullptr;
392  }
393 
394  var->unsupported_attr_dtype = temp_unsup_attr_dtype;
395  var->unsupported_attr_dspace = temp_unsup_attr_dspace;
396 
397  if (!this->unsupported_var_attr_dspace && temp_unsup_attr_dspace)
398  this->unsupported_var_attr_dspace = true;
399  }
400 
401  this->vars.push_back(var);
402  if (H5Dclose(cdset) < 0)
403  throw2("Error closing the HDF5 dataset ", full_path_name);
404  }
405  break;
406 
407  case H5O_TYPE_NAMED_DATATYPE: {
408  // ignore the named datatype
409  if (true == include_attr && true == check_ignored) {
410  this->add_ignored_info_namedtypes(string(gname), string(oname.begin(), oname.end()));
411  }
412  }
413  break;
414  default:
415  break;
416  } // "switch (obj_type)"
417  } // try
418  catch (...) {
419 
420  if (attr != nullptr) {
421  delete attr;
422  attr = nullptr;
423  }
424 
425  if (var != nullptr) {
426  delete var;
427  var = nullptr;
428  }
429 
430  if (group != nullptr) {
431  delete group;
432  group = nullptr;
433  }
434 
435  if (cgroup != -1) H5Gclose(cgroup);
436 
437  if (cdset != -1) H5Dclose(cdset);
438  throw;
439 
440  } // catch
441  } // "for (hsize_t i = 0; i < nelems; i++)"
442 
443 }
444 
445 // Retrieve HDF5 dataset compression ratio
446 float File::Retrieve_H5_VarCompRatio(const Var *var, const hid_t dset_id) const
447 {
448 
449  float comp_ratio = 1.0;
450  // Obtain the data type of the variable.
451  hid_t dset_create_plist = H5Dget_create_plist(dset_id);
452  if (dset_create_plist < 0)
453  throw1("unable to obtain hdf5 dataset creation property list ");
454  H5D_layout_t dset_layout = H5Pget_layout(dset_create_plist);
455  if (dset_layout < 0) {
456  H5Pclose(dset_create_plist);
457  throw1("unable to obtain hdf5 dataset creation property list storage layout");
458  }
459 
460  if (dset_layout == H5D_CHUNKED) {
461 
462  hsize_t dstorage_size = H5Dget_storage_size(dset_id);
463  if (dstorage_size > 0 && var->total_elems > 0) {
464  hid_t ty_id = -1;
465 
466  // Obtain the data type of the variable.
467  if ((ty_id = H5Dget_type(dset_id)) < 0)
468  throw1("unable to obtain hdf5 datatype for the dataset ");
469  size_t type_size = H5Tget_size(ty_id);
470  comp_ratio = ((float)((var->total_elems) * type_size))/dstorage_size;
471  H5Tclose(ty_id);
472  }
473 
474  }
475  H5Pclose(dset_create_plist);
476  return comp_ratio;
477 
478 }
479 // Retrieve HDF5 dataset datatype
480 void File::Retrieve_H5_VarType(Var *var, hid_t dset_id, const string & varname, bool &unsup_var_dtype)
481 {
482 
483  hid_t ty_id = -1;
484 
485  // Obtain the data type of the variable.
486  if ((ty_id = H5Dget_type(dset_id)) < 0)
487  throw2("unable to obtain hdf5 datatype for the dataset ", varname);
488 
489  // The following datatype class and datatype will not be supported for the CF option.
490  // H5T_TIME, H5T_BITFIELD
491  // H5T_OPAQUE, H5T_ENUM
492  // H5T_REFERENCE, H5T_COMPOUND
493  // H5T_VLEN,H5T_ARRAY
494  // 64-bit integer
495 
496  // Note: H5T_REFERENCE H5T_COMPOUND and H5T_ARRAY can be mapped to DAP2 DDS for the default option.
497  // H5T_COMPOUND, H5T_ARRAY can be mapped to DAP2 DAS for the default option.
498  // 1-D variable length of string can also be mapped for the CF option..
499  // The variable length string class is H5T_STRING rather than H5T_VLEN,
500  // We also ignore the mapping of integer 64 bit since DAP2 doesn't
501  // support 64-bit integer. In theory, DAP2 doesn't support long double
502  // (128-bit or 92-bit floating point type).
503  //
504 
505  var->dtype = HDF5CFUtil::H5type_to_H5DAPtype(ty_id);
506  if (false == HDF5CFUtil::cf_strict_support_type(var->dtype,_is_dap4))
507  unsup_var_dtype = true;
508 
509  if (H5Tclose(ty_id) < 0)
510  throw1("Unable to close the HDF5 datatype ");;
511 }
512 
513 // Retrieve the HDF5 dataset dimension information
514 void File::Retrieve_H5_VarDim(Var *var, hid_t dset_id, const string & varname, bool &unsup_var_dspace)
515 {
516 
517  vector<hsize_t> dsize;
518  vector<hsize_t> maxsize;
519 
520  hid_t dspace_id = -1;
521  hid_t ty_id = -1;
522 
523  try {
524  if ((dspace_id = H5Dget_space(dset_id)) < 0)
525  throw2("Cannot get hdf5 dataspace id for the variable ", varname);
526 
527  H5S_class_t space_class = H5S_NO_CLASS;
528  if ((space_class = H5Sget_simple_extent_type(dspace_id)) < 0)
529  throw2("Cannot obtain the HDF5 dataspace class for the variable ", varname);
530 
531  if (H5S_NULL == space_class)
532  unsup_var_dspace = true;
533  else {
534  if (false == unsup_var_dspace) {
535 
536  hssize_t h5_total_elms = H5Sget_simple_extent_npoints(dspace_id);
537  if (h5_total_elms < 0)
538  throw2("Cannot get the total number of elements of HDF5 dataset ", varname);
539  else
540  var->total_elems = (size_t) h5_total_elms;
541  int ndims = H5Sget_simple_extent_ndims(dspace_id);
542  if (ndims < 0)
543  throw2("Cannot get the hdf5 dataspace number of dimension for the variable ", varname);
544 
545  var->rank = ndims;
546  if (ndims != 0) {
547  dsize.resize(ndims);
548  maxsize.resize(ndims);
549  }
550 
551  // The netcdf DAP client supports the representation of the unlimited dimension.
552  // So we need to check.
553  if (H5Sget_simple_extent_dims(dspace_id, dsize.data(), maxsize.data()) < 0)
554  throw2("Cannot obtain the dim. info for the variable ", varname);
555 
556  for (int i = 0; i < ndims; i++) {
557  auto dim = new Dimension(dsize[i]);
558  if (maxsize[i] == H5S_UNLIMITED) {
559  dim->unlimited_dim = true;
560  if (false == have_udim)
561  have_udim = true;
562  }
563  var->dims.push_back(dim);
564  }
565  }
566  }
567 
568  var->unsupported_dspace = unsup_var_dspace;
569 
570  if (H5Sclose(dspace_id) < 0)
571  throw1("Cannot close the HDF5 dataspace .");
572 
573  }
574 
575  catch (...) {
576 
577  if (dspace_id != -1) H5Sclose(dspace_id);
578 
579  if (ty_id != -1) H5Tclose(ty_id);
580  throw;
581  }
582 
583 }
584 
585 // Retrieve the HDF5 attribute information.
586 void File::Retrieve_H5_Attr_Info(Attribute * attr, hid_t obj_id, const int j, bool &unsup_attr_dtype,
587  bool &unsup_attr_dspace)
588 
589 {
590 
591  hid_t attrid = -1;
592  hid_t ty_id = -1;
593  hid_t aspace_id = -1;
594  hid_t memtype = -1;
595 
596  try {
597 
598  // Obtain the attribute ID.
599  if ((attrid = H5Aopen_by_idx(obj_id, ".", H5_INDEX_CRT_ORDER, H5_ITER_INC, (hsize_t) j, H5P_DEFAULT,
600  H5P_DEFAULT)) < 0)
601  throw1("Unable to open attribute by index ");
602 
603  // Obtain the size of attribute name.
604  ssize_t name_size = H5Aget_name(attrid, 0, nullptr);
605  if (name_size < 0)
606  throw1("Unable to obtain the size of the hdf5 attribute name ");
607 
608  string attr_name;
609  attr_name.resize(name_size + 1);
610 
611  // Obtain the attribute name.
612  if ((H5Aget_name(attrid, name_size + 1, &attr_name[0])) < 0)
613  throw1("unable to obtain the hdf5 attribute name ");
614 
615  // Obtain the type of the attribute.
616  if ((ty_id = H5Aget_type(attrid)) < 0)
617  throw2("unable to obtain hdf5 datatype for the attribute ", attr_name);
618 
619  // The following datatype class and datatype will not be supported for the CF option.
620  // H5T_TIME, H5T_BITFIELD
621  // H5T_OPAQUE, H5T_ENUM
622  // H5T_REFERENCE, H5T_COMPOUND
623  // H5T_VLEN,H5T_ARRAY
624  // 64-bit integer
625 
626  // Note: H5T_REFERENCE H5T_COMPOUND and H5T_ARRAY can be mapped to DAP2 DDS for the default option.
627  // H5T_COMPOUND, H5T_ARRAY can be mapped to DAP2 DAS for the default option.
628  // 1-D variable length of string can also be mapped for the CF option..
629  // The variable length string class is H5T_STRING rather than H5T_VLEN,
630  // We also ignore the mapping of integer 64 bit since DAP2 doesn't
631  // support 64-bit integer. In theory, DAP2 doesn't support long double
632  // (128-bit or 92-bit floating point type).
633  //
634  attr->dtype = HDF5CFUtil::H5type_to_H5DAPtype(ty_id);
635  if (false == HDF5CFUtil::cf_strict_support_type(attr->dtype,_is_dap4))
636  unsup_attr_dtype = true;
637 
638  if(H5VSTRING == attr->dtype || H5FSTRING == attr->dtype) {
639  H5T_cset_t c_set_type = H5Tget_cset(ty_id);
640  if(c_set_type <0)
641  throw2("Cannot get hdf5 character set type for the attribute ", attr_name);
642  // This is a UTF-8 string
643  if(c_set_type == 1)
644  attr->is_cset_ascii = false;
645  }
646 
647  if ((aspace_id = H5Aget_space(attrid)) < 0)
648  throw2("Cannot get hdf5 dataspace id for the attribute ", attr_name);
649 
650  int ndims = H5Sget_simple_extent_ndims(aspace_id);
651  if (ndims < 0)
652  throw2("Cannot get the hdf5 dataspace number of dimension for attribute ", attr_name);
653 
654  hsize_t nelmts = 1;
655 
656  // if it is a scalar attribute, just define number of elements to be 1.
657  if (ndims != 0) {
658 
659  vector<hsize_t> asize;
660  vector<hsize_t> maxsize;
661  asize.resize(ndims);
662  maxsize.resize(ndims);
663 
664  // Obtain the attribute data space information.
665  if (H5Sget_simple_extent_dims(aspace_id, asize.data(), maxsize.data()) < 0)
666  throw2("Cannot obtain the dim. info for the attribute ", attr_name);
667 
668  // Here we need to take care of 0-length attribute. This is legal in HDF5.
669  for (int dim_count = 0;dim_count < ndims; dim_count ++) {
670  // STOP adding unsupported_attr_dspace!
671  if (asize[dim_count] == 0) {
672  unsup_attr_dspace = true;
673  break;
674  }
675  }
676 
677  if (false == unsup_attr_dspace) {
678  // Return ndims and size[ndims].
679  for (int dim_count = 0; dim_count< ndims; dim_count++)
680  nelmts *= asize[dim_count];
681  }
682  else
683  nelmts = 0;
684  } // "if(ndims != 0)"
685 
686  size_t ty_size = H5Tget_size(ty_id);
687  if (0 == ty_size)
688  throw2("Cannot obtain the dtype size for the attribute ", attr_name);
689 
690  memtype = H5Tget_native_type(ty_id, H5T_DIR_ASCEND);
691  if (memtype < 0)
692  throw2("Cannot obtain the memory datatype for the attribute ", attr_name);
693 
694  // Store the name and the count
695  string temp_aname(attr_name.begin(), attr_name.end());
696  attr->name = temp_aname.substr(0, temp_aname.size() - 1);
697  attr->newname = attr->name;
698  attr->count = nelmts;
699 
700  // Release HDF5 resources.
701  if (H5Tclose(ty_id) < 0)
702  throw1("Cannot successfully close the attribute datatype.");
703  if (H5Tclose(memtype) < 0)
704  throw1("Cannot successfully close the attribute memory datatype.");
705  if (H5Sclose(aspace_id) < 0)
706  throw1("Cannot successfully close the HDF5 dataspace.");
707  if (H5Aclose(attrid) < 0)
708  throw1("Cannot successfully close the HDF5 attribute.");
709 
710  } // try
711  catch (...) {
712 
713  if (ty_id != -1) H5Tclose(ty_id);
714 
715  if (memtype != -1) H5Tclose(memtype);
716 
717  if (aspace_id != -1) H5Sclose(aspace_id);
718 
719  if (attrid != -1) H5Aclose(attrid);
720 
721  throw;
722  }
723 
724 }
725 
726 // Retrieve all HDF5 supported attribute values.
727 void File::Retrieve_H5_Supported_Attr_Values()
728 {
729 
730  for (auto ira = this->root_attrs.begin(); ira != this->root_attrs.end(); ++ira)
731  Retrieve_H5_Attr_Value(*ira, "/");
732 
733  for (auto irg = this->groups.begin(); irg != this->groups.end(); ++irg) {
734  for (auto ira = (*irg)->attrs.begin(); ira != (*irg)->attrs.end(); ++ira) {
735  Retrieve_H5_Attr_Value(*ira, (*irg)->path);
736  }
737  }
738 
739  for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
740  for (auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira) {
741  Retrieve_H5_Attr_Value(*ira, (*irv)->fullpath);
742  }
743  }
744 }
745 
746 void File::Retrieve_H5_Var_Attr_Values(Var *var)
747 {
748  for (auto ira = var->attrs.begin(); ira != var->attrs.end(); ++ira) {
749  Retrieve_H5_Attr_Value(*ira, var->fullpath);
750  }
751 }
752 
753 // Retrieve the values of a specific HDF5 attribute.
754 void File::Retrieve_H5_Attr_Value(Attribute *attr, const string & obj_name)
755 {
756 
757  // Define HDF5 object Ids.
758  hid_t obj_id = -1;
759  hid_t attr_id = -1;
760  hid_t ty_id = -1;
761  hid_t memtype_id = -1;
762  hid_t aspace_id = -1;
763 
764  try {
765 
766  // Open the object that hold this attribute
767  obj_id = H5Oopen(this->fileid, obj_name.c_str(), H5P_DEFAULT);
768  if (obj_id < 0)
769  throw2("Cannot open the object ", obj_name);
770 
771  attr_id = H5Aopen(obj_id, (attr->name).c_str(), H5P_DEFAULT);
772  if (attr_id < 0)
773  throw4("Cannot open the attribute ", attr->name, " of object ", obj_name);
774 
775  ty_id = H5Aget_type(attr_id);
776  if (ty_id < 0)
777  throw4("Cannot obtain the datatype of the attribute ", attr->name, " of object ", obj_name);
778 
779  memtype_id = H5Tget_native_type(ty_id, H5T_DIR_ASCEND);
780  if (memtype_id < 0)
781  throw2("Cannot obtain the memory datatype for the attribute ", attr->name);
782 
783  size_t ty_size = H5Tget_size(memtype_id);
784  if (0 == ty_size)
785  throw4("Cannot obtain the dtype size for the attribute ", attr->name, " of object ", obj_name);
786 
787  size_t total_bytes = attr->count * ty_size;
788 
789  // We have to handle variable length string differently.
790  if (H5VSTRING == attr->dtype) {
791 
792  // Variable length string attribute values only store pointers of the actual string value.
793  vector<char> temp_buf;
794  temp_buf.resize(total_bytes);
795 
796  if (H5Aread(attr_id, memtype_id, temp_buf.data()) < 0)
797  throw4("Cannot obtain the value of the attribute ", attr->name, " of object ", obj_name);
798 
799  char *temp_bp = nullptr;
800  char *ptr_1stvlen_ptr = temp_buf.data();
801  temp_bp = temp_buf.data();
802  char* onestring = nullptr;
803  string total_vstring = "";
804 
805  attr->strsize.resize(attr->count);
806 
807  for (unsigned int temp_i = 0; temp_i < attr->count; temp_i++) {
808 
809  // This line will assure that we get the real variable length string value.
810  onestring = *(char **) temp_bp;
811  if (onestring != nullptr) {
812  total_vstring += string(onestring);
813  attr->strsize[temp_i] = (string(onestring)).size();
814  }
815  else
816  attr->strsize[temp_i] = 0;
817 
818  // going to the next value.
819  temp_bp += ty_size;
820  }
821 
822  if (ptr_1stvlen_ptr != nullptr) {
823  aspace_id = H5Aget_space(attr_id);
824  if (aspace_id < 0)
825  throw4("Cannot obtain space id for ", attr->name, " of object ", obj_name);
826 
827  // Reclaim any VL memory if necessary.
828  if (H5Dvlen_reclaim(memtype_id, aspace_id, H5P_DEFAULT, temp_buf.data()) < 0)
829  throw4("Cannot reclaim VL memory for ", attr->name, " of object ", obj_name);
830 
831  H5Sclose(aspace_id);
832  }
833 
834  if (HDF5CFUtil::H5type_to_H5DAPtype(ty_id) != H5VSTRING)
835  throw4("Error to obtain the VL string type for attribute ", attr->name, " of object ", obj_name);
836 
837  attr->value.resize(total_vstring.size());
838 
839  copy(total_vstring.begin(), total_vstring.end(), attr->value.begin());
840 
841  }
842  else {
843 
844  if (attr->dtype == H5FSTRING) {
845  attr->fstrsize = ty_size;
846  }
847 
848  attr->value.resize(total_bytes);
849 
850  // Read HDF5 attribute data.
851  if (H5Aread(attr_id, memtype_id, (void *) &attr->value[0]) < 0)
852  throw4("Cannot obtain the dtype size for the attribute ", attr->name, " of object ", obj_name);
853 
854  if (attr->dtype == H5FSTRING) {
855 
856  size_t sect_size = ty_size;
857  int num_sect = 1;
858  if (sect_size > 0)
859  num_sect =
860  (total_bytes % sect_size == 0) ? (total_bytes / sect_size) : (total_bytes / sect_size + 1);
861  else
862  throw4("The attribute datatype size is not a positive integer ", attr->name, " of object ",
863  obj_name);
864 
865  vector<size_t> sect_newsize;
866  sect_newsize.resize(num_sect);
867 
868  auto total_fstring = string(attr->value.begin(), attr->value.end());
869 
870  string new_total_fstring = HDF5CFUtil::trim_string(memtype_id, total_fstring, num_sect, sect_size,
871  sect_newsize);
872  attr->value.resize(new_total_fstring.size());
873  copy(new_total_fstring.begin(), new_total_fstring.end(), attr->value.begin());
874  attr->strsize.resize(num_sect);
875  for (int temp_i = 0; temp_i < num_sect; temp_i++)
876  attr->strsize[temp_i] = sect_newsize[temp_i];
877 
878 #if 0
879  // "h5","new string value " <<string(attr->value.begin(), attr->value.end()) <<endl;
880  cerr<<"Attr name is "<<attr->name <<endl;
881  for (int temp_i = 0; temp_i <num_sect; temp_i ++)
882  cerr<<"string new section size = " << attr->strsize[temp_i] <<endl;
883 #endif
884  }
885  }
886 
887  if (H5Tclose(memtype_id) < 0)
888  throw1("Fail to close the HDF5 memory datatype ID.");
889  if (H5Tclose(ty_id) < 0)
890  throw1("Fail to close the HDF5 datatype ID.");
891  if (H5Aclose(attr_id) < 0)
892  throw1("Fail to close the HDF5 attribute ID.");
893  if (H5Oclose(obj_id) < 0)
894  throw1("Fail to close the HDF5 object ID.");
895 
896  }
897 
898  catch (...) {
899 
900  if (memtype_id != -1) H5Tclose(memtype_id);
901 
902  if (ty_id != -1) H5Tclose(ty_id);
903 
904  if (aspace_id != -1) H5Sclose(aspace_id);
905 
906  if (attr_id != -1) H5Aclose(attr_id);
907 
908  if (obj_id != -1) H5Oclose(obj_id);
909 
910  throw;
911  }
912 
913 }
914 
915 // Handle the unsupported datatype
916 void File::Handle_Unsupported_Dtype(bool include_attr)
917 {
918 
919  if (true == include_attr) {
920  Handle_Group_Unsupported_Dtype();
921  Handle_VarAttr_Unsupported_Dtype();
922  }
923 
924  Handle_Var_Unsupported_Dtype();
925 }
926 
927 //Leave this code here.
928 #if 0
929 // First the root attributes
930 if (true == include_attr) {
931  if (true == this->unsupported_attr_dtype) {
932  for (vector<Attribute *>::iterator ira = this->root_attrs.begin();
933  ira != this->root_attrs.end(); ) {
934  H5DataType temp_dtype = (*ira)->getType();
935  if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype)) {
936  delete (*ira);
937  ira = this->root_attrs.erase(ira);
938 
939  }
940  else {
941  ++ira;
942  }
943  }
944  }
945 }
946 
947 // Then the group attributes
948 if (false == this->groups.empty()) {
949  for (vector<Group *>::iterator irg = this->groups.begin();
950  irg != this->groups.end(); ++irg) {
951  if (false == (*irg)->attrs.empty()) {
952  if (true == (*irg)->unsupported_attr_dtype) {
953  for (vector<Attribute *>::iterator ira = (*irg)->attrs.begin();
954  ira != (*irg)->attrs.end(); ) {
955  H5DataType temp_dtype = (*ira)->getType();
956  if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype)) {
957  delete (*ira);
958  ira = (*irg)->attrs.erase(ira);
959  }
960  else {
961  ++ira;
962  }
963  }
964  }
965  }
966  }
967 }
968 }
969 
970  // Then the variable(HDF5 dataset) and the correponding attributes.
971 if (false == this->vars.empty()) {
972 if (true == include_attr) {
973  for (vector<Var *>::iterator irv = this->vars.begin();
974  irv != this->vars.end();++irv ) {
975  if (false == (*irv)->attrs.empty()) {
976  if (true == (*irv)->unsupported_attr_dtype) {
977  for (vector<Attribute *>::iterator ira = (*irv)->attrs.begin();
978  ira != (*irv)->attrs.end(); ) {
979  H5DataType temp_dtype = (*ira)->getType();
980  if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype)) {
981  delete (*ira);
982  ira = (*irv)->attrs.erase(ira);
983  //ira--;
984  }
985  else {
986  ++ira;
987  }
988  }
989  }
990  }
991  }
992 }
993 if (true == this->unsupported_var_dtype) {
994  // "h5","having unsupported variable datatype" <<endl;
995  for (vector<Var *>::iterator irv = this->vars.begin();
996  irv != this->vars.end(); ) {
997  H5DataType temp_dtype = (*irv)->getType();
998  if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype)) {
999  delete (*irv);
1000  irv = this->vars.erase(irv);
1001  //irv--;
1002  }
1003  else
1004  ++irv;
1005  }
1006 }
1007 }
1008 #endif
1009 
1010 void File::Handle_Group_Unsupported_Dtype()
1011 {
1012 
1013  // First root
1014  if (false == this->root_attrs.empty()) {
1015  if (true == this->unsupported_attr_dtype) {
1016  for (vector<Attribute *>::iterator ira = this->root_attrs.begin(); ira != this->root_attrs.end();) {
1017  H5DataType temp_dtype = (*ira)->getType();
1018  if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4)) {
1019  delete (*ira);
1020  ira = this->root_attrs.erase(ira);
1021  }
1022  else {
1023  ++ira;
1024  }
1025  }
1026  }
1027  }
1028 
1029  // Then the group attributes
1030  if (false == this->groups.empty()) {
1031  for (auto irg = this->groups.begin(); irg != this->groups.end(); ++irg) {
1032  if (false == (*irg)->attrs.empty()) {
1033  if (true == (*irg)->unsupported_attr_dtype) {
1034  for (auto ira = (*irg)->attrs.begin(); ira != (*irg)->attrs.end();) {
1035  H5DataType temp_dtype = (*ira)->getType();
1036  if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4)) {
1037  delete (*ira);
1038  ira = (*irg)->attrs.erase(ira);
1039  }
1040  else {
1041  ++ira;
1042  }
1043  }
1044  }
1045  }
1046  }
1047  }
1048 }
1049 
1050 // Generate group unsupported datatype Information, this is for the BES ignored object key
1051 void File::Gen_Group_Unsupported_Dtype_Info()
1052 {
1053 
1054  // First root
1055  if (false == this->root_attrs.empty()) {
1056  //if (true == this->unsupported_attr_dtype) {
1057  for (auto ira = this->root_attrs.begin(); ira != this->root_attrs.end(); ++ira) {
1058  H5DataType temp_dtype = (*ira)->getType();
1059  // TODO: Don't know why we still include 64-bit integer here.
1060  //if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype) || temp_dtype == H5INT64 || temp_dtype == H5UINT64) {
1061  if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4)
1062  || temp_dtype == H5INT64 || temp_dtype == H5UINT64) {
1063  this->add_ignored_info_attrs(true, "/", (*ira)->name);
1064  }
1065  }
1066  //}
1067  }
1068 
1069  // Then the group attributes
1070  if (false == this->groups.empty()) {
1071  for (auto irg = this->groups.begin(); irg != this->groups.end(); ++irg) {
1072  if (false == (*irg)->attrs.empty()) {
1073  //if (true == (*irg)->unsupported_attr_dtype) {
1074  for (auto ira = (*irg)->attrs.begin(); ira != (*irg)->attrs.end(); ++ira) {
1075  H5DataType temp_dtype = (*ira)->getType();
1076  // TODO: Don't know why we still include 64-bit integer here.
1077  //if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype) || temp_dtype == H5INT64 || temp_dtype==H5UINT64 ) {
1078  if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4)
1079  || temp_dtype == H5INT64 || temp_dtype==H5UINT64 ) {
1080  this->add_ignored_info_attrs(true, (*irg)->path, (*ira)->name);
1081  }
1082  }
1083  //}
1084  }
1085  }
1086  }
1087 }
1088 
1089 // Handler unsupported variable datatype
1090 void File::Handle_Var_Unsupported_Dtype()
1091 {
1092  if (false == this->vars.empty()) {
1093  if (true == this->unsupported_var_dtype) {
1094  for (auto irv = this->vars.begin(); irv != this->vars.end();) {
1095  H5DataType temp_dtype = (*irv)->getType();
1096  if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4)) {
1097  delete (*irv);
1098  irv = this->vars.erase(irv);
1099  }
1100  else {
1101  ++irv;
1102 
1103  }
1104  }
1105  }
1106  }
1107 }
1108 
1109 // Generate unsupported variable type info. This is for the ignored objects.
1110 void File::Gen_Var_Unsupported_Dtype_Info()
1111 {
1112 
1113  if (false == this->vars.empty()) {
1114  //if (true == this->unsupported_var_dtype) {
1115  for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1116  H5DataType temp_dtype = (*irv)->getType();
1117  //TODO: don't know why 64-bit integer is still listed here.
1118  //if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype)||(H5INT64 == temp_dtype) ||(H5UINT64 == temp_dtype)) {
1119  if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4)
1120  ||(H5INT64 == temp_dtype) ||(H5UINT64 == temp_dtype)) {
1121  this->add_ignored_info_objs(false, (*irv)->fullpath);
1122  }
1123  }
1124  //}
1125  }
1126 
1127 }
1128 
1129 // Handling unsupported datatypes for variable(HDF5 dataset) and the correponding attributes.
1130 void File::Handle_VarAttr_Unsupported_Dtype()
1131 {
1132  if (false == this->vars.empty()) {
1133  for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1134  if (false == (*irv)->attrs.empty()) {
1135  if (true == (*irv)->unsupported_attr_dtype) {
1136  for (auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end();) {
1137  H5DataType temp_dtype = (*ira)->getType();
1138  if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4)) {
1139  delete (*ira);
1140  ira = (*irv)->attrs.erase(ira);
1141  }
1142  else {
1143  ++ira;
1144  }
1145  }
1146  }
1147  }
1148  }
1149  }
1150 }
1151 
1152 // Generated unsupported var/attribute unsupported datatype Info when the BES ignored object key is on.
1153 void File::Gen_VarAttr_Unsupported_Dtype_Info()
1154 {
1155 
1156  if (false == this->vars.empty()) {
1157  for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1158  if (false == (*irv)->attrs.empty()) {
1159  //if (true == (*irv)->unsupported_attr_dtype) {
1160  for (auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira) {
1161  H5DataType temp_dtype = (*ira)->getType();
1162  // TODO: check why 64-bit integer is still listed here.
1163  //if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype) || (temp_dtype==H5INT64) || (temp_dtype == H5UINT64)) {
1164  if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4)
1165  || (temp_dtype==H5INT64) || (temp_dtype == H5UINT64)) {
1166  this->add_ignored_info_attrs(false, (*irv)->fullpath, (*ira)->name);
1167  }
1168  }
1169  //}
1170  }
1171  }
1172  }
1173 }
1174 
1175 // Generated unsupported datatype information for HDF5 dimension scales.
1176 // The datatypes of HDF5 dimension scales("DIMENSION_LIST" and "REFERENCE_LIST")
1177 // are not supported. However, the information
1178 // are retrieved by the handlers so we don't want to report them as ignored objects.
1179 void File::Gen_DimScale_VarAttr_Unsupported_Dtype_Info()
1180 {
1181 
1182  for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1183 
1184  // If the attribute REFERENCE_LIST comes with the attribut CLASS, the
1185  // attribute REFERENCE_LIST is okay to ignore. No need to report.
1186  bool is_ignored = ignored_dimscale_ref_list((*irv));
1187  if (false == (*irv)->attrs.empty()) {
1188  //if (true == (*irv)->unsupported_attr_dtype) {
1189  for (auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira) {
1190  H5DataType temp_dtype = (*ira)->getType();
1191  // TODO: check why 64-bit is still listed here.
1192  if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4)
1193  || (temp_dtype == H5INT64) || (temp_dtype == H5UINT64)) {
1194  // "DIMENSION_LIST" is okay to ignore and "REFERENCE_LIST"
1195  // is okay to ignore if the variable has another attribute
1196  // CLASS="DIMENSION_SCALE"
1197  if (("DIMENSION_LIST" != (*ira)->name)
1198  && ("REFERENCE_LIST" != (*ira)->name || true == is_ignored))
1199  this->add_ignored_info_attrs(false, (*irv)->fullpath, (*ira)->name);
1200  }
1201  }
1202  //}
1203  }
1204  }
1205 }
1206 
1207 // Handle unsupported dataspace for group attributes.
1208 void File::Handle_GroupAttr_Unsupported_Dspace()
1209 {
1210 
1211  // First root
1212  if (false == this->root_attrs.empty()) {
1213  if (true == this->unsupported_attr_dspace) {
1214  for (auto ira = this->root_attrs.begin(); ira != this->root_attrs.end();) {
1215  // Remove 0-size attribute
1216  if ((*ira)->count == 0) {
1217  delete (*ira);
1218  ira = this->root_attrs.erase(ira);
1219  }
1220  else {
1221  ++ira;
1222  }
1223  }
1224  }
1225  }
1226 
1227  // Then the group attributes
1228  if (false == this->groups.empty()) {
1229  for (auto irg = this->groups.begin(); irg != this->groups.end(); ++irg) {
1230  if (false == (*irg)->attrs.empty()) {
1231  if (true == (*irg)->unsupported_attr_dspace) {
1232  for (auto ira = (*irg)->attrs.begin(); ira != (*irg)->attrs.end();) {
1233  if ((*ira)->count == 0) {
1234  delete (*ira);
1235  ira = (*irg)->attrs.erase(ira);
1236  }
1237  else {
1238  ++ira;
1239  }
1240  }
1241  }
1242  }
1243  }
1244  }
1245 }
1246 
1247 // Handle unsupported data space information for variable and attribute
1248 void File::Handle_VarAttr_Unsupported_Dspace()
1249 {
1250 
1251  if (false == this->vars.empty()) {
1252  if (true == this->unsupported_var_attr_dspace) {
1253  for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1254  if (false == (*irv)->attrs.empty()) {
1255  if (true == (*irv)->unsupported_attr_dspace) {
1256  for (auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end();) {
1257  if (0 == (*ira)->count) {
1258  delete (*ira);
1259  ira = (*irv)->attrs.erase(ira);
1260  }
1261  else {
1262  ++ira;
1263  }
1264  }
1265  }
1266  }
1267  }
1268  }
1269  }
1270 }
1271 
1272 // Handle unsupported data space.
1273 void File::Handle_Unsupported_Dspace(bool include_attr)
1274 {
1275 
1276  // The unsupported data space
1277  if (false == this->vars.empty()) {
1278  if (true == this->unsupported_var_dspace) {
1279  for (auto irv = this->vars.begin(); irv != this->vars.end();) {
1280  if (true == (*irv)->unsupported_dspace) {
1281  delete (*irv);
1282  irv = this->vars.erase(irv);
1283  }
1284  else {
1285  ++irv;
1286 
1287  }
1288  }
1289  }
1290  }
1291 
1292  if (true == include_attr) {
1293  Handle_GroupAttr_Unsupported_Dspace();
1294  Handle_VarAttr_Unsupported_Dspace();
1295  }
1296 }
1297 
1298 // Generated unsupported dataspace Info when the BES ignored object key is on.
1299 void File::Gen_Unsupported_Dspace_Info()
1300 {
1301 
1302  // Notice in this function, we deliberately don't put the case when an attribute dimension has 0 length.
1303  // Since doing this requires non-trivial change of the source code and the 0-size attribute case is really, really rare,
1304  // so we just "ignore" this case in the "ignored" information.
1305  // In fact, the zero size variable is allowed in both HDF5 and DAP2. So we don't ignore 0-size HDF5 dataset. So
1306  // the only case this function checks is the H5S_NULL case.
1307  if (false == this->vars.empty()) {
1308  if (true == this->unsupported_var_dspace) {
1309  for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1310  if (true == (*irv)->unsupported_dspace) {
1311  this->add_ignored_info_objs(true, (*irv)->fullpath);
1312  }
1313  }
1314  }
1315  }
1316 
1317 }
1318 
1319 // Handle other unsupported information.
1320 void File::Handle_Unsupported_Others(bool include_attr)
1321 {
1322 
1323  if (true == this->check_ignored && true == include_attr) {
1324 
1325  if (true == HDF5RequestHandler::get_drop_long_string()) {
1326 
1327  // netCDF java doesn't have limitation for attributes
1328 #if 0
1329  for (vector<Attribute *>::iterator ira = this->root_attrs.begin(); ira != this->root_attrs.end(); ++ira) {
1330  if (H5FSTRING == (*ira)->dtype || H5VSTRING == (*ira)->dtype) {
1331  if ((*ira)->getBufSize() > NC_JAVA_STR_SIZE_LIMIT) {
1332  this->add_ignored_droplongstr_hdr();
1333  this->add_ignored_grp_longstr_info("/", (*ira)->name);
1334  }
1335  }
1336  }
1337 
1338  for (vector<Group *>::iterator irg = this->groups.begin(); irg != this->groups.end(); ++irg) {
1339  for (vector<Attribute *>::iterator ira = (*irg)->attrs.begin(); ira != (*irg)->attrs.end(); ++ira) {
1340  if (H5FSTRING == (*ira)->dtype || H5VSTRING == (*ira)->dtype) {
1341  if ((*ira)->getBufSize() > NC_JAVA_STR_SIZE_LIMIT) {
1342  this->add_ignored_droplongstr_hdr();
1343  this->add_ignored_grp_longstr_info((*irg)->path, (*ira)->name);
1344  }
1345  }
1346  }
1347  }
1348 #endif
1349  for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1350  if (true == Check_DropLongStr((*irv), nullptr)) {
1351  this->add_ignored_droplongstr_hdr();
1352  this->add_ignored_var_longstr_info((*irv), nullptr);
1353  }
1354  // netCDF java doesn't have limitation for attributes
1355 #if 0
1356  for (auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira) {
1357  if (true == Check_DropLongStr((*irv), (*ira))) {
1358  this->add_ignored_droplongstr_hdr();
1359  this->add_ignored_var_longstr_info((*irv), (*ira));
1360  }
1361  }
1362 #endif
1363  }
1364  }
1365  }
1366 
1367 }
1368 
1369 // Flatten the object name, mainly call get_CF_string.
1370 void File::Flatten_Obj_Name(bool include_attr)
1371 {
1372 
1373  for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1374  (*irv)->newname = get_CF_string((*irv)->newname);
1375 
1376  for (auto ird = (*irv)->dims.begin(); ird != (*irv)->dims.end(); ++ird) {
1377  (*ird)->newname = get_CF_string((*ird)->newname);
1378  }
1379  }
1380 
1381  if (true == include_attr) {
1382 
1383  for (auto ira = this->root_attrs.begin(); ira != this->root_attrs.end(); ++ira) {
1384  (*ira)->newname = get_CF_string((*ira)->newname);
1385  }
1386 
1387  for (auto irg = this->groups.begin(); irg != this->groups.end(); ++irg) {
1388  (*irg)->newname = get_CF_string((*irg)->newname);
1389  for (auto ira = (*irg)->attrs.begin(); ira != (*irg)->attrs.end(); ++ira) {
1390  (*ira)->newname = get_CF_string((*ira)->newname);
1391  }
1392  }
1393 
1394  for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1395  for (auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira) {
1396  (*ira)->newname = get_CF_string((*ira)->newname);
1397  }
1398  }
1399  } // "if (true == include_attr)"
1400 }
1401 
1402 // Variable name clashing
1403 void File::Handle_Var_NameClashing(set<string>&objnameset)
1404 {
1405 
1406  Handle_General_NameClashing(objnameset, this->vars);
1407 }
1408 
1409 // Group name clashing
1410 void File::Handle_Group_NameClashing(set<string> &objnameset)
1411 {
1412 
1413  pair<set<string>::iterator, bool> setret;
1414 
1415  // Now for DAS, we need to handle name clashings for
1416  // DAS tables. Namely we need to make sure the global attribute
1417  // table(HDF5_GLOBAL) and the attribute tables mapped from
1418  // HDF5 groups will not have name clashings with the variable name
1419  // lists. If having the name clashings, the global attribute table and the
1420  // the attribute tables generated from the groups will be changed.
1421  // The file attribute name clashing
1422 
1423  setret = objnameset.insert(FILE_ATTR_TABLE_NAME);
1424  if (false == setret.second) {
1425 
1426  int clash_index = 1;
1427  string fa_clash_name = FILE_ATTR_TABLE_NAME;
1428  HDF5CFUtil::gen_unique_name(fa_clash_name, objnameset, clash_index);
1429  FILE_ATTR_TABLE_NAME = fa_clash_name;
1430  }
1431 
1432  // The group attribute name clashing
1433  Handle_General_NameClashing(objnameset, this->groups);
1434 
1435 }
1436 
1437 //Object attribute name clashing
1438 void File::Handle_Obj_AttrNameClashing()
1439 {
1440 
1441  // Now handling the possible name clashings for attributes
1442  // For attribute clashings, we only need to resolve the name clashings
1443  // for attributes within each variable, file attributes and attributes
1444  // within each group. The name clashings for attributes should be very rare.
1445  // Potentially the checking and the correcting may be costly.
1446  // This is another reason for special products, we may not even need to check
1447  // the name clashings. KY 2011-12-24
1448 
1449  set<string> objnameset;
1450 
1451  // For root attributes
1452  Handle_General_NameClashing(objnameset, this->root_attrs);
1453 
1454  // For group attributes
1455  for (auto irg = this->groups.begin(); irg != this->groups.end(); ++irg) {
1456  objnameset.clear();
1457  Handle_General_NameClashing(objnameset, (*irg)->attrs);
1458  }
1459 
1460  // For variable attributes
1461  for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1462  objnameset.clear();
1463  Handle_General_NameClashing(objnameset, (*irv)->attrs);
1464  }
1465 }
1466 
1467 // Handle General name clashing
1468 //class T must have member string newname. In our case, T is either groups, attributes or vars.
1469 template<class T> void File::Handle_General_NameClashing(set<string>&objnameset, vector<T*>& objvec)
1470 {
1471 
1472 #if 0
1473 // set<string> objnameset;
1474 #endif
1475 
1476  pair<set<string>::iterator, bool> setret;
1477  set<string>::iterator iss;
1478 
1479  vector<string> clashnamelist;
1480  vector<string>::iterator ivs;
1481 
1482  map<int, int> cl_to_ol;
1483  int ol_index = 0;
1484  int cl_index = 0;
1485 
1486  /*class*/
1487  typename vector<T*>::iterator irv;
1488 
1489  for (irv = objvec.begin(); irv != objvec.end(); ++irv) {
1490  setret = objnameset.insert((*irv)->newname);
1491  if (false == setret.second) {
1492  clashnamelist.insert(clashnamelist.end(), (*irv)->newname);
1493  cl_to_ol[cl_index] = ol_index;
1494  cl_index++;
1495  }
1496  ol_index++;
1497  }
1498 
1499  // Now change the clashed elements to unique elements,
1500  // Generate the set which has the same size as the original vector.
1501  for (ivs = clashnamelist.begin(); ivs != clashnamelist.end(); ivs++) {
1502  int clash_index = 1;
1503  string temp_clashname = *ivs + '_';
1504  HDF5CFUtil::gen_unique_name(temp_clashname, objnameset, clash_index);
1505  *ivs = temp_clashname;
1506  }
1507 
1508  // Now go back to the original vector, make it unique.
1509  for (unsigned int i = 0; i < clashnamelist.size(); i++)
1510  objvec[cl_to_ol[i]]->newname = clashnamelist[i];
1511 
1512 }
1513 
1514 // Handle General object name clashing
1515 void File::Handle_GeneralObj_NameClashing(bool include_attr, set<string>& objnameset)
1516 {
1517 
1518  Handle_Var_NameClashing(objnameset);
1519  if (true == include_attr) {
1520  Handle_Group_NameClashing(objnameset);
1521  Handle_Obj_AttrNameClashing();
1522  }
1523 }
1524 
1525 // Get CF name, flatten the path, change the non-alphanumeric letters to underscore.
1526 string File::get_CF_string(string s)
1527 {
1528 
1529  if ("" == s) return s;
1530  string insertString(1, '_');
1531 
1532  // Always start with _ if the first character is not a letter
1533  if (true == isdigit(s[0])) s.insert(0, insertString);
1534 
1535  for (unsigned int i = 0; i < s.length(); i++)
1536  if ((false == isalnum(s[i])) && (s[i] != '_')) s[i] = '_';
1537 
1538  return s;
1539 
1540 }
1541 
1542 // For the connection of variable dimensions. Build dimname to dimsize(unlimited) maps
1543 void File::Insert_One_NameSizeMap_Element(string name, hsize_t size, bool unlimited)
1544 {
1545  pair<map<string, hsize_t>::iterator, bool> mapret;
1546  mapret = dimname_to_dimsize.insert(pair<string, hsize_t>(name, size));
1547  if (false == mapret.second)
1548  throw4("The dimension name ", name, " should map to ", size);
1549 
1550  pair<map<string, bool>::iterator, bool> mapret2;
1551  mapret2 = dimname_to_unlimited.insert(pair<string, bool>(name, unlimited));
1552  if (false == mapret2.second)
1553  throw3("The dimension name ", name, " unlimited dimension info. should be provided.");
1554 
1555 }
1556 
1557 // Similar to Inset_One_NameSizeMap_Element but the maps are provided as parameters.
1558 void File::Insert_One_NameSizeMap_Element2(map<string, hsize_t>& name_to_size, map<string, bool>& name_to_unlimited,
1559  string name, hsize_t size, bool unlimited)
1560 {
1561  pair<map<string, hsize_t>::iterator, bool> mapret;
1562  mapret = name_to_size.insert(pair<string, hsize_t>(name, size));
1563  if (false == mapret.second)
1564  throw4("The dimension name ", name, " should map to ", size);
1565 
1566  pair<map<string, bool>::iterator, bool> mapret2;
1567  mapret2 = name_to_unlimited.insert(pair<string, bool>(name, unlimited));
1568  if (false == mapret2.second)
1569  throw3("The dimension name ", name, " unlimited dimension info. should be provided.");
1570 
1571 }
1572 
1573 // For dimension names added by the handlers, by default,
1574 // Each dimension will have a unique dimension name. For example,
1575 // Int foo[100][200] will be Int foo[Fakedim1][Fakedim2]
1576 // If you have many variables, the dimension names may be too many.
1577 // To reduce numbers, we ASSUME that the dimension having the same
1578 // size shares the same dimension. In this way, the number of dimension names
1579 // will be reduced.
1580 // For example, Int foo2[100][300] will be Int foo2[Fakedim1][Fakedim3]
1581 // instead of foo2[Fakedim3][Fakedim4]. However, that may impose
1582 // another problem. Suppose Int Foosame[100][100] becomes
1583 // Int Foosame[FakeDim1][FakeDim1]. This doesn't make sense for some
1584 // applications. The fuction Adjust_Duplicate_FakeDim_Name will make sure
1585 // this case will not happen.
1586 void File::Add_One_FakeDim_Name(Dimension *dim)
1587 {
1588 
1589  stringstream sfakedimindex;
1590  string fakedimstr = "FakeDim";
1591  pair<set<string>::iterator, bool> setret;
1592  map<hsize_t, string>::iterator im;
1593  pair<map<hsize_t, string>::iterator, bool> mapret;
1594 
1595  sfakedimindex << addeddimindex;
1596  string added_dimname = fakedimstr + sfakedimindex.str();
1597 
1598  // Build up the size to fakedim map.
1599  mapret = dimsize_to_fakedimname.insert(pair<hsize_t, string>(dim->size, added_dimname));
1600  if (false == mapret.second) { //The dim size exists, use the corresponding name.
1601  dim->name = dimsize_to_fakedimname[dim->size];
1602  dim->newname = dim->name;
1603  }
1604  else { // Insert this (dimsize,dimname) pair to dimsize_to_fakedimname map successfully.
1605  //First make sure this new dim name doesn't have name clashing
1606  // with previous dim names, after the checking, inserting to the
1607  // dimname list set.
1608  // dimnamelist is a private memeber of File.
1609  setret = dimnamelist.insert(added_dimname);
1610  if (false == setret.second) {
1611  int clash_index = 1;
1612  string temp_clashname = added_dimname + '_';
1613  HDF5CFUtil::gen_unique_name(temp_clashname, dimnamelist, clash_index);
1614  dim->name = temp_clashname;
1615  dim->newname = dim->name;
1616  setret = dimnamelist.insert(dim->name);
1617  if (false == setret.second)
1618  throw2("Fail to insert the unique dimsizede name ", dim->name);
1619 
1620  // We have to adjust the dim. name of the dimsize_to_fakedimname map, since the
1621  // dimname has been updated for this size.
1622  dimsize_to_fakedimname.erase(dim->size);
1623  mapret = dimsize_to_fakedimname.insert(pair<hsize_t, string>(dim->size, dim->name));
1624  if (false == mapret.second)
1625  throw4("The dimension size ", dim->size, " should map to ", dim->name);
1626  } // "if(false == setret.second)"
1627 
1628  // New dim name is inserted successfully, update the dimname_to_dimsize map.
1629  dim->name = added_dimname;
1630  dim->newname = dim->name;
1631  Insert_One_NameSizeMap_Element(dim->name, dim->size, dim->unlimited_dim);
1632 
1633  // Increase the dimindex since the new dimname has been inserted.
1634  addeddimindex++;
1635  } // else
1636 }
1637 
1638 // See the function comments of Add_One_FakeDim_Name
1639 void File::Adjust_Duplicate_FakeDim_Name(Dimension * dim)
1640 {
1641 
1642  // No need to adjust the dimsize_to_fakedimname map, only create a new Fakedim
1643  // The simplest way is to increase the dim index and resolve any name clashings with other dim names.
1644  // Note: No need to update the dimsize_to_dimname map since the original "FakeDim??" of this size
1645  // can be used as a dimension name of other variables. But we need to update the dimname_to_dimsize map
1646  // since this is a new dim name.
1647  stringstream sfakedimindex;
1648  pair<set<string>::iterator, bool> setret;
1649 
1650  addeddimindex++;
1651  sfakedimindex << addeddimindex;
1652  string added_dimname = "FakeDim" + sfakedimindex.str();
1653  setret = dimnamelist.insert(added_dimname);
1654  if (false == setret.second) {
1655  int clash_index = 1;
1656  string temp_clashname = added_dimname + '_';
1657  HDF5CFUtil::gen_unique_name(temp_clashname, dimnamelist, clash_index);
1658  dim->name = temp_clashname;
1659  dim->newname = dim->name;
1660  setret = dimnamelist.insert(dim->name);
1661  if (false == setret.second)
1662  throw2("Fail to insert the unique dimsizede name ", dim->name);
1663  }
1664  dim->name = added_dimname;
1665  dim->newname = dim->name;
1666  Insert_One_NameSizeMap_Element(dim->name, dim->size, dim->unlimited_dim);
1667 
1668  // Need to prepare for the next unique FakeDim.
1669  addeddimindex++;
1670 }
1671 // 1. See the function comments of Add_One_FakeDim_Name
1672 // 2. This is for the case when we have foo[100][100],foo2[100][100].
1673 // The Adjust_Duplicate_FakeDim_Name() assures the uniqueness of Dimension
1674 // names for each var,like foo[FakeDim0][FakeDim1],foo2[FakeDim0][FakeDim3]..
1675 // but the dimension names are not desirable for products like SMAP level 3.
1676 // For similar vars, the dimension names like foo[FakeDim0][FakeDim1] and
1677 // foo2[FakeDim0][FakeDim1] ,foo3[FakeDim0][FakeDim1][FakeDim2] are the way to go.
1678 
1679 void File::Adjust_Duplicate_FakeDim_Name2(Dimension * dim, int dup_dim_size_index)
1680 {
1681 
1682  // Search if vector of dup (dimsize dimname) pair has the current size
1683  // If yes, insert the dim name from the vector.
1684  // Note: in case we have foo[100][100][100] etc., we need to remember the
1685  // vector index, to obtain the correct dup_dim_size, dup_dim_name pair.
1686  // if no, build up the new FakeDim.
1687  bool dup_dim_size_exist = false;
1688  int temp_dup_dim_size_index = 0;
1689  for (unsigned i = 0; i <dup_dimsize_dimname.size();i++) {
1690  // The dup vector may include different size, so we need to check
1691  // if having the same size.
1692  if(dim->size == dup_dimsize_dimname[i].first) {
1693  temp_dup_dim_size_index++;
1694  // Make sure we obtain the correct index in the vector
1695  if(dup_dim_size_index == temp_dup_dim_size_index) {
1696  dup_dim_size_exist = true;
1697  dim->name = dup_dimsize_dimname[i].second;
1698  dim->newname = dim->name;
1699  break;
1700  }
1701  }
1702  }
1703 
1704  // If we cannot find this dimension in the existing (dup dimsize dimname) vector,
1705  // create the FakeDim by increasing the index and update the dup vector etc.
1706  if(dup_dim_size_exist == false) {
1707 
1708  stringstream sfakedimindex;
1709  pair<set<string>::iterator, bool> setret;
1710 
1711  sfakedimindex << addeddimindex;
1712  string added_dimname = "FakeDim" + sfakedimindex.str();
1713  setret = dimnamelist.insert(added_dimname);
1714  if (false == setret.second) {
1715  throw2("Inside Adjust_Duplicate_FakeDim_Name2(), Fail to insert the unique dim name ", dim->name);
1716  }
1717  dim->name = added_dimname;
1718  dim->newname = dim->name;
1719  Insert_One_NameSizeMap_Element(dim->name, dim->size, dim->unlimited_dim);
1720  // push back to the duplicate size, name vector.
1721  dup_dimsize_dimname.push_back(make_pair(dim->size,dim->name));
1722  addeddimindex++;
1723 
1724  }
1725 
1726 }
1727 
1728 // Replace all dimension names, this function is currently not used. So comment out. May delete it in the future.
1729 #if 0
1730 void File::Replace_Dim_Name_All(const string orig_dim_name, const string new_dim_name) {
1731 
1732  // The newname of the original dimension should also be replaced by new_dim_name
1733  for (vector<Var *>::iterator irv = this->vars.begin();
1734  irv != this->vars.end(); ++irv) {
1735  for (vector<Dimension *>::iterator ird= (*irv)->dims.begin();
1736  ird != (*irv)->dims.end(); ++ird) {
1737  if((*ird)->name == orig_dim_name) {
1738  (*ird)->name = new_dim_name;
1739  (*ird)->newname = new_dim_name;
1740  }
1741 
1742  }
1743  }
1744 }
1745 #endif
1746 
1747 #if 0
1748 void File::Use_Dim_Name_With_Size_All(const string dim_name, const size_t dim_size) {
1749 
1750  // The newname of the original dimension should also be replaced by new_dim_name
1751  for (vector<Var *>::iterator irv = this->vars.begin();
1752  irv != this->vars.end(); ++irv) {
1753  for (vector<Dimension *>::iterator ird= (*irv)->dims.begin();
1754  ird != (*irv)->dims.end(); ++ird) {
1755  if((*ird)->size == orig_dim_name) {
1756  (*ird)->name = new_dim_name;
1757  (*ird)->newname = new_dim_name;
1758  }
1759 
1760  }
1761  }
1762 }
1763 #endif
1764 
1765 // Often times we need to add a CF attribute with string datatype because some products don't provide them
1766 // Examples are units, comment etc.
1767 void File::Add_Str_Attr(Attribute* attr, const string &attrname, const string& strvalue)
1768 {
1769 
1770  attr->name = attrname;
1771  attr->newname = attr->name;
1772  attr->dtype = H5FSTRING;
1773  attr->count = 1;
1774  attr->fstrsize = strvalue.size();
1775  attr->strsize.resize(1);
1776  attr->strsize[0] = attr->fstrsize;
1777  attr->value.resize(strvalue.size());
1778  copy(strvalue.begin(), strvalue.end(), attr->value.begin());
1779 }
1780 
1781 #if 0
1782 bool
1783 File:: Var_Has_Attr(Var*var,const string &attrname) {
1784 
1785  for (vector<Attribute *>:: iterator ira =var->attrs.begin(); ira !=var->attrs.end(); ++ira) {
1786 
1787  // We only check the original attribute name
1788  // Remove the original "coordinates" attribute.
1789  if((*ira)->name == attrname || (*ira)->newname == attrname) {
1790  return true;
1791  }
1792  }
1793  return false;
1794 }
1795 #endif
1796 
1797 // Rretrieve the variable attribute in string.var_path is the variable path.
1798 string File::Retrieve_Str_Attr_Value(Attribute *attr, const string & var_path)
1799 {
1800 
1801  if (attr != nullptr && var_path != "") {
1802  Retrieve_H5_Attr_Value(attr, var_path);
1803  string orig_attr_value(attr->value.begin(), attr->value.end());
1804  return orig_attr_value;
1805  }
1806  return "";
1807 
1808 }
1809 
1810 //Check if the attribute value of this variable is the input value.
1811 bool File::Is_Str_Attr(Attribute* attr, const string& varfullpath, const string &attrname, const string& strvalue)
1812 {
1813  bool ret_value = false;
1814  if (attrname == get_CF_string(attr->newname)) {
1815  Retrieve_H5_Attr_Value(attr, varfullpath);
1816  string attr_value(attr->value.begin(), attr->value.end());
1817  if (attr_value == strvalue) ret_value = true;
1818  }
1819  return ret_value;
1820 }
1821 
1822 // If this latitude or longitude units follows the CF
1823 bool File::has_latlon_cf_units(Attribute *attr, const string &varfullpath, bool is_lat)
1824 {
1825  string attr_name = "units";
1826  if (true == is_lat) {
1827  string lat_unit_value = "degrees_north";
1828  return Is_Str_Attr(attr, varfullpath, attr_name, lat_unit_value);
1829  }
1830  else {
1831  string lon_unit_value = "degrees_east";
1832  return Is_Str_Attr(attr, varfullpath, attr_name, lon_unit_value);
1833  }
1834 }
1835 
1836 // This function is mainly to add _FillValue.
1837 void File::Add_One_Float_Attr(Attribute* attr, const string &attrname, float float_value)
1838 {
1839  attr->name = attrname;
1840  attr->newname = attr->name;
1841  attr->dtype = H5FLOAT32;
1842  attr->count = 1;
1843  attr->value.resize(sizeof(float));
1844  memcpy(&(attr->value[0]), (void*) (&float_value), sizeof(float));
1845 }
1846 
1847 // Products like GPM use string type for MissingValue, we need to change them to the corresponding variable datatype and
1848 // get the value corrected.
1849 void File::Change_Attr_One_Str_to_Others(Attribute* attr, const Var*var)
1850 {
1851 
1852  char *pEnd;
1853  // string to long int number.
1854  long int num_sli = 0;
1855  if (attr->dtype != H5FSTRING)
1856  throw2("Currently we only convert fixed-size string to other datatypes. ", attr->name);
1857  if (attr->count != 1)
1858  throw4("The fixed-size string count must be 1 and the current count is ", attr->count, " for the attribute ",
1859  attr->name);
1860 
1861  Retrieve_H5_Attr_Value(attr, var->fullpath);
1862  string attr_value;
1863  attr_value.resize(attr->value.size());
1864  copy(attr->value.begin(), attr->value.end(), attr_value.begin());
1865 
1866  switch (var->dtype) {
1867 
1868  case H5UCHAR: {
1869  num_sli = strtol(&(attr->value[0]), &pEnd, 10);
1870  if (num_sli < 0 || num_sli > UCHAR_MAX)
1871  throw5("Attribute type is unsigned char, the current attribute ", attr->name, " has the value ", num_sli,
1872  ". It is overflowed. ");
1873  else {
1874  unsigned char num_suc = (unsigned char) num_sli;
1875  attr->dtype = H5UCHAR;
1876  attr->value.resize(sizeof(unsigned char));
1877  memcpy(&(attr->value[0]), (void*) (&num_suc), sizeof(unsigned char));
1878  }
1879 
1880  }
1881  break;
1882  case H5CHAR: {
1883  num_sli = strtol(&(attr->value[0]), &pEnd, 10);
1884  if (num_sli < SCHAR_MIN || num_sli > SCHAR_MAX)
1885  throw5("Attribute type is signed char, the current attribute ", attr->name, " has the value ", num_sli,
1886  ". It is overflowed. ");
1887  else {
1888  char num_sc = (char) num_sli;
1889  attr->dtype = H5CHAR;
1890  attr->value.resize(sizeof(char));
1891  memcpy(&(attr->value[0]), (void*) (&num_sc), sizeof(char));
1892  }
1893 
1894  }
1895  break;
1896  case H5INT16: {
1897  num_sli = strtol(&(attr->value[0]), &pEnd, 10);
1898  if (num_sli < SHRT_MIN || num_sli > SHRT_MAX)
1899  throw5("Attribute type is 16-bit integer, the current attribute ", attr->name, " has the value ", num_sli,
1900  ". It is overflowed. ");
1901  else {
1902  short num_ss = (short) num_sli;
1903  attr->dtype = H5INT16;
1904  attr->value.resize(sizeof(short));
1905  memcpy(&(attr->value[0]), (void*) (&num_ss), sizeof(short));
1906  }
1907 
1908  }
1909  break;
1910  case H5UINT16: {
1911  num_sli = strtol(&(attr->value[0]), &pEnd, 10);
1912  if (num_sli < 0 || num_sli > USHRT_MAX)
1913  throw5("Attribute type is unsigned 16-bit integer, the current attribute ", attr->name, " has the value ",
1914  num_sli, ". It is overflowed. ");
1915  else {
1916  unsigned short num_uss = (unsigned short) num_sli;
1917  attr->dtype = H5UINT16;
1918  attr->value.resize(sizeof(unsigned short));
1919  memcpy(&(attr->value[0]), (void*) (&num_uss), sizeof(unsigned short));
1920  }
1921  }
1922  break;
1923  case H5INT32: {
1924  num_sli = strtol(&(attr->value[0]), &pEnd, 10);
1925  //No need to check overflow, the number will always be in the range.
1926 #if 0
1927  //if(num_sli <LONG_MIN || num_sli >LONG_MAX)
1928  // throw5("Attribute type is 32-bit integer, the current attribute ",attr->name, " has the value ",num_sli, ". It is overflowed. ");
1929 #endif
1930  attr->dtype = H5INT32;
1931  attr->value.resize(sizeof(long int));
1932  memcpy(&(attr->value[0]), (void*) (&num_sli), sizeof(long int));
1933 
1934  }
1935  break;
1936  case H5UINT32: {
1937  unsigned long int num_suli = strtoul(&(attr->value[0]), &pEnd, 10);
1938  // No need to check since num_suli will not be bigger than ULONG_MAX.
1939  attr->dtype = H5UINT32;
1940  attr->value.resize(sizeof(unsigned long int));
1941  memcpy(&(attr->value[0]), (void*) (&num_suli), sizeof(unsigned long int));
1942  }
1943  break;
1944  case H5FLOAT32: {
1945  float num_sf = strtof(&(attr->value[0]), nullptr);
1946  // Don't think it is necessary to check if floating-point is oveflowed for this routine. ignore it now. KY 2014-09-22
1947  attr->dtype = H5FLOAT32;
1948  attr->value.resize(sizeof(float));
1949  memcpy(&(attr->value[0]), (void*) (&num_sf), sizeof(float));
1950  }
1951  break;
1952  case H5FLOAT64: {
1953  double num_sd = strtod(&(attr->value[0]), nullptr);
1954  // Don't think it is necessary to check if floating-point is oveflowed for this routine. ignore it now. KY 2014-09-22
1955  attr->dtype = H5FLOAT64;
1956  attr->value.resize(sizeof(double));
1957  memcpy(&(attr->value[0]), (void*) (&num_sd), sizeof(double));
1958  }
1959  break;
1960 
1961  default:
1962  throw4("Unsupported HDF5 datatype that the string is converted to for the attribute ", attr->name,
1963  " of the variable ", var->fullpath);
1964  } // "switch(var->dtype)"
1965 
1966 }
1967 
1968 // Change a string type attribute to the input value
1969 void File::Replace_Var_Str_Attr(Var* var, const string &attr_name, const string& strvalue)
1970 {
1971 
1972  bool rep_attr = true;
1973  bool rem_attr = false;
1974  for (auto ira = var->attrs.begin(); ira != var->attrs.end(); ira++) {
1975  if ((*ira)->name == attr_name) {
1976  if (true == Is_Str_Attr(*ira, var->fullpath, attr_name, strvalue))
1977  rep_attr = false;
1978  else
1979  rem_attr = true;
1980  break;
1981  }
1982  }
1983 
1984  // Remove the attribute if the attribute value is not strvalue
1985  if (true == rem_attr) {
1986  for (auto ira = var->attrs.begin(); ira != var->attrs.end(); ira++) {
1987  if ((*ira)->name == attr_name) {
1988  delete (*ira);
1989  var->attrs.erase(ira);
1990  break;
1991  }
1992  }
1993  }
1994 
1995  // Add the attribute with strvalue
1996  if (true == rep_attr) {
1997  auto attr = new Attribute();
1998  Add_Str_Attr(attr, attr_name, strvalue);
1999  var->attrs.push_back(attr);
2000  }
2001 }
2002 
2003 // Check if this variable if latitude,longitude.We check the three name pairs(lat,lon),(latitude,longitude),(Latitude,Longitude)
2004 bool File::Is_geolatlon(const string & var_name, bool is_lat)
2005 {
2006 
2007  bool ret_value = false;
2008  if (true == is_lat) {
2009  string lat1 = "lat";
2010  string lat2 = "latitude";
2011  string lat3 = "Latitude";
2012 
2013  if (var_name.compare(lat1) == 0 || var_name.compare(lat2) == 0 || var_name.compare(lat3) == 0) ret_value = true;
2014  }
2015 
2016  else {
2017  string lon1 = "lon";
2018  string lon2 = "longitude";
2019  string lon3 = "Longitude";
2020  if (var_name.compare(lon1) == 0 || var_name.compare(lon2) == 0 || var_name.compare(lon3) == 0) ret_value = true;
2021 
2022  }
2023  return ret_value;
2024 }
2025 
2026 // Add supplementary attributes.
2027 void File::Add_Supplement_Attrs(bool add_path)
2028 {
2029 
2030  if (false == add_path) return;
2031 
2032  // Adding variable original name(origname) and full path(fullpath)
2033  for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
2034  auto attr = new Attribute();
2035  const string varname = (*irv)->name;
2036  const string attrname = "origname";
2037  Add_Str_Attr(attr, attrname, varname);
2038  (*irv)->attrs.push_back(attr);
2039  }
2040 
2041  for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
2042  // Turn off the fullnamepath attribute when zero_storage_size is 0.
2043  // Use the BES key since quite a few testing cases will be affected.
2044  // KY 2020-03-23
2045  if((*irv)->zero_storage_size==false
2046  || HDF5RequestHandler::get_no_zero_size_fullnameattr() == false) {
2047  auto attr = new Attribute();
2048  const string varname = (*irv)->fullpath;
2049  const string attrname = "fullnamepath";
2050  Add_Str_Attr(attr, attrname, varname);
2051  (*irv)->attrs.push_back(attr);
2052  }
2053  }
2054 
2055  // Adding group path
2056  for (auto irg = this->groups.begin(); irg != this->groups.end(); ++irg) {
2057  // Only when this group has attributes, the original path of the group has some values. So add it.
2058  if (false == (*irg)->attrs.empty()) {
2059 
2060  auto attr = new Attribute();
2061  const string varname = (*irg)->path;
2062  const string attrname = "fullnamepath";
2063  Add_Str_Attr(attr, attrname, varname);
2064  (*irg)->attrs.push_back(attr);
2065  }
2066  }
2067 
2068 }
2069 
2070 // Variable target will not be deleted, but rather its contents are replaced.
2071 // We may make this as an operator = in the future.
2072 // Note: the attributes can not be replaced.
2073 void File::Replace_Var_Info(Var *src, Var *target)
2074 {
2075 
2076 #if 0
2077  for_each (target->dims.begin (), target->dims.end (),
2078  delete_elem ());
2079  for_each (target->attrs.begin (), target->attrs.end (),
2080  delete_elem ());
2081 #endif
2082 
2083  target->newname = src->newname;
2084  target->name = src->name;
2085  target->fullpath = src->fullpath;
2086  target->rank = src->rank;
2087  target->dtype = src->dtype;
2088  target->unsupported_attr_dtype = src->unsupported_attr_dtype;
2089  target->unsupported_dspace = src->unsupported_dspace;
2090 #if 0
2091  for (auto ira = target->attrs.begin();
2092  ira!=target->attrs.end(); ++ira) {
2093  delete (*ira);
2094  target->attrs.erase(ira);
2095  ira--;
2096  }
2097 #endif
2098  for (auto ird = target->dims.begin(); ird != target->dims.end();) {
2099  delete (*ird);
2100  ird = target->dims.erase(ird);
2101  }
2102 
2103  // Somehow attributes cannot be replaced.
2104 #if 0
2105  for (vector<Attribute*>::iterator ira = src->attrs.begin();
2106  ira!=src->attrs.end(); ++ira) {
2107  Attribute* attr= new Attribute();
2108  attr->name = (*ira)->name;
2109  attr->newname = (*ira)->newname;
2110  attr->dtype =(*ira)->dtype;
2111  attr->count =(*ira)->count;
2112  attr->strsize = (*ira)->strsize;
2113  attr->fstrsize = (*ira)->fstrsize;
2114  attr->value =(*ira)->value;
2115  target->attrs.push_back(attr);
2116  }
2117 #endif
2118 
2119  for (auto ird = src->dims.begin(); ird != src->dims.end(); ++ird) {
2120  auto dim = new Dimension((*ird)->size);
2121  dim->name = (*ird)->name;
2122  dim->newname = (*ird)->newname;
2123  target->dims.push_back(dim);
2124  }
2125 
2126 }
2127 
2128 // Replace the attributes of target with src.
2129 void File::Replace_Var_Attrs(Var *src, Var *target)
2130 {
2131 
2132 #if 0
2133  for_each (target->dims.begin (), target->dims.end (),
2134  delete_elem ());
2135  for_each (target->attrs.begin (), target->attrs.end (),
2136  delete_elem ());
2137 #endif
2138 
2139  for (auto ira = target->attrs.begin(); ira != target->attrs.end();) {
2140  delete (*ira);
2141  ira = target->attrs.erase(ira);
2142  }
2143  for (auto ira = src->attrs.begin(); ira != src->attrs.end(); ++ira) {
2144  auto attr = new Attribute();
2145  attr->name = (*ira)->name;
2146  attr->newname = (*ira)->newname;
2147  attr->dtype = (*ira)->dtype;
2148  attr->count = (*ira)->count;
2149  attr->strsize = (*ira)->strsize;
2150  attr->fstrsize = (*ira)->fstrsize;
2151  attr->value = (*ira)->value;
2152  target->attrs.push_back(attr);
2153  }
2154 
2155 }
2156 
2157 // Check if a variable with a var name is under a specific group with groupname
2158 // note: the variable's size at each dimension is also returned. The user must allocate the
2159 // memory for the dimension sizes(an array(vector is perferred).
2160 bool File::is_var_under_group(const string &varname, const string &grpname, const int var_rank,
2161  vector<size_t> & var_size)
2162 {
2163 
2164  bool ret_value = false;
2165  for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
2166 
2167  if ((*irv)->rank == var_rank) {
2168  if ((*irv)->name == varname) {
2169 
2170  // Obtain the variable path
2171  string var_path = HDF5CFUtil::obtain_string_before_lastslash((*irv)->fullpath);
2172 
2173  // Check if we find the variable under this group
2174  if (grpname == var_path) {
2175  ret_value = true;
2176  for (int i = 0; i < var_rank; i++)
2177  var_size[i] = (*irv)->getDimensions()[i]->size;
2178  break;
2179  }
2180  }
2181  } // "if((*irv)->rank == var_rank)"
2182  } // "for (vector<Var *>::iterator irv = this->vars.begin()"
2183 
2184 
2185  return ret_value;
2186 
2187 }
2188 bool File::Have_Grid_Mapping_Attrs(){
2189 
2190  bool ret_value = false;
2191  for (auto irv = this->vars.begin();
2192  irv != this->vars.end(); ++irv) {
2193  for (auto ira = (*irv)->attrs.begin();
2194  ira != (*irv)->attrs.end(); ++ira) {
2195  if((*ira)->name =="grid_mapping") {
2196  ret_value = true;
2197  break;
2198  }
2199  }
2200  if(true == ret_value)
2201  break;
2202  }
2203 
2204  return ret_value;
2205 
2206 }
2207 
2208 void File::Handle_Grid_Mapping_Vars(){
2209 
2210  for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
2211  string attr_value;
2212  for (auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira) {
2213  if((*ira)->name =="grid_mapping") {
2214  Retrieve_H5_Attr_Value(*ira, (*irv)->fullpath);
2215  attr_value.resize((*ira)->value.size());
2216  copy((*ira)->value.begin(), (*ira)->value.end(), attr_value.begin());
2217  break;
2218  }
2219 
2220  }
2221  if(attr_value.find('/') ==string::npos){
2222  string new_name = Check_Grid_Mapping_VarName(attr_value,(*irv)->fullpath);
2223  if(new_name != "")
2224  Replace_Var_Str_Attr((*irv),"grid_mapping",new_name);
2225 
2226  }
2227  else {
2228  string new_name = Check_Grid_Mapping_FullPath(attr_value);
2229  //Using new_name as the attribute value
2230  if(new_name != "")
2231  Replace_Var_Str_Attr((*irv),"grid_mapping",new_name);
2232  }
2233  }
2234 
2235 }
2236 
2237 string File::Check_Grid_Mapping_VarName(const string & a_value,const string & var_fpath) {
2238 
2239  string var_path = HDF5CFUtil::obtain_string_before_lastslash(var_fpath);
2240  string gmap_new_name;
2241  for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
2242  if((*irv)->name == a_value){
2243  if(var_path == HDF5CFUtil::obtain_string_before_lastslash((*irv)->fullpath)) {
2244  gmap_new_name = (*irv)->newname;
2245  break;
2246  }
2247  }
2248  }
2249  return gmap_new_name;
2250 }
2251 
2252 
2253 string File::Check_Grid_Mapping_FullPath(const string & a_value) {
2254 
2255  string gmap_new_name;
2256  for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
2257  if((*irv)->fullpath == a_value){
2258  gmap_new_name = (*irv)->newname;
2259  break;
2260  }
2261  }
2262 
2263  return gmap_new_name;
2264 }
2265 
2266 void File::remove_netCDF_internal_attributes(bool include_attr) {
2267 
2268  if(true == include_attr) {
2269  for (auto irv = this->vars.begin();
2270  irv != this->vars.end(); ++irv) {
2271  bool var_has_dimscale = false;
2272 
2273  for(auto ira = (*irv)->attrs.begin();
2274  ira != (*irv)->attrs.end();) {
2275  if((*ira)->name == "CLASS") {
2276  string class_value = Retrieve_Str_Attr_Value(*ira,(*irv)->fullpath);
2277 
2278  // Compare the attribute "CLASS" value with "DIMENSION_SCALE". We only compare the string with the size of
2279  // "DIMENSION_SCALE", which is 15.
2280  if (0 == class_value.compare(0,15,"DIMENSION_SCALE")) {
2281  delete(*ira);
2282  ira = (*irv)->attrs.erase(ira);
2283  var_has_dimscale = true;
2284 
2285  }
2286 #if 0
2287  else if(1) {// Add a BES key,also delete
2288 
2289  }
2290 #endif
2291  else {
2292  ++ira;
2293  }
2294  }
2295 #if 0
2296  else if((*ira)->name == "NAME") {// Add a BES Key
2297  string name_value = Retrieve_Str_Attr_Value(*ira,(*irv)->fullpath);
2298  if( 0 == name_value.compare(0,(*irv)->name.size(),(*irv)->name)) {
2299  delete(*ira);
2300  ira =(*irv)->attrs.erase(ira);
2301  }
2302  else {
2303  string netcdf_dim_mark= "This is a netCDF dimension but not a netCDF variable";
2304  if( 0 == name_value.compare(0,netcdf_dim_mark.size(),netcdf_dim_mark)) {
2305  delete(*ira);
2306  ira =(*irv)->attrs.erase(ira);
2307  }
2308  else {
2309  ++ira;
2310  }
2311  }
2312 
2313  }
2314 #endif
2315  else if((*ira)->name == "_Netcdf4Dimid") {
2316  delete(*ira);
2317  ira =(*irv)->attrs.erase(ira);
2318  }
2319  else if((*ira)->name == "_Netcdf4Coordinates") {
2320  delete(*ira);
2321  ira =(*irv)->attrs.erase(ira);
2322  }
2323 #if 0
2324  else if((*ira)->name == "_nc3_strict") {
2325  delete((*ira));
2326  ira =(*irv)->attrs.erase(ira);
2327  }
2328 #endif
2329  else {
2330  ++ira;
2331  }
2332  }
2333 
2334  if(true == var_has_dimscale) {
2335  for(auto ira = (*irv)->attrs.begin();
2336  ira != (*irv)->attrs.end();++ira) {
2337  if((*ira)->name == "NAME") {// Add a BES Key
2338  delete(*ira);
2339  ira =(*irv)->attrs.erase(ira);
2340  break;
2341  }
2342  }
2343  }
2344  }
2345  }
2346 
2347 }
2348 // Add ignored page header info. Mainly a helper message.
2349 void File::add_ignored_info_page_header()
2350 {
2351  ignored_msg =
2352  " \n This page is for HDF5 CF hyrax data providers or distributors to check if any HDF5 object or attribute information are ignored during the mapping. \n\n";
2353 }
2354 
2355 // Add ignored object header info. Mainly a helper message.
2356 void File::add_ignored_info_obj_header()
2357 {
2358 
2359  ignored_msg += " Some HDF5 objects or the object information are ignored when mapping to DAP2 by the HDF5 OPeNDAP";
2360  ignored_msg += " handler due to the restrictions of DAP2, CF conventions or CF tools.";
2361  ignored_msg += " Please use HDF5 tools(h5dump or HDFView) to check carefully and make sure that these objects";
2362  ignored_msg +=
2363  " are OK to ignore for your service. For questions or requests to find a way to handle the ignored objects, please";
2364  ignored_msg += " contact the HDF5 OPeNDAP handler developer or send an email to help@hdfgroup.org.\n";
2365 
2366  ignored_msg += " \n In general, ignored HDF5 objects include HDF5 soft links, external links and named datatype.\n";
2367  ignored_msg +=
2368  " \n The HDF5 datasets(variables in the CF term) and attributes that have the following datatypes are ignored: \n";
2369  ignored_msg +=
2370  " Signed and unsigned 64-bit integers, HDF5 compound, HDF5 variable length(excluding variable length string),";
2371  ignored_msg += " HDF5 reference, HDF5 enum, HDF5 opaque , HDF5 bitfield, HDF5 Array and HDF5 Time datatypes.\n";
2372 
2373  ignored_msg +=
2374  " \n The HDF5 datasets(variables in the CF term) and attributes associated with the following dimensions are ignored: \n";
2375  ignored_msg += " 1) variables that have HDF5 NULL dataspace(H5S_NULL)(rarely occurred)\n";
2376  ignored_msg += " 2) attributes that have any zero size dimensions(not reported due to extreme rarity and non-trivial coding)\n\n";
2377 
2378 }
2379 
2380 // Add the ignored links information.Mainly a helper message.
2381 void File::add_ignored_info_links_header()
2382 {
2383 
2384  if (false == this->have_ignored) {
2385  add_ignored_info_obj_header();
2386  have_ignored = true;
2387  }
2388  // Add ignored datatype header.
2389  string lh_msg = "******WARNING******\n";
2390  lh_msg += "IGNORED soft links or external links are: ";
2391  if (ignored_msg.rfind(lh_msg) == string::npos) ignored_msg += lh_msg + "\n";
2392 
2393 }
2394 
2395 // Leave the code for the time being.
2396 #if 0
2397 void
2398 File:: add_ignored_info_obj_dtype_header() {
2399 
2400  // Add ignored datatype header.
2401  ignored_msg += " \n Variables and attributes ignored due to the unsupported datatypes. \n";
2402  ignored_msg += " In general, the unsupported datatypes include: \n";
2403  ignored_msg += " Signed and unsigned 64-bit integers, HDF5 compound, HDF5 variable length(excluding variable length string),";
2404  ignored_msg += " HDF5 reference, HDF5 enum, HDF5 opaque , HDF5 bitfield, HDF5 Array and HDF5 Time datatypes.\n";
2405 
2406 }
2407 
2408 void
2409 File:: add_ignored_info_obj_dspace_header() {
2410 
2411  // Add ignored dataspace header.
2412  ignored_msg += " \n Variables and attributes ignored due to the unsupported dimensions. \n";
2413  ignored_msg += " In general, the unsupported dimensions include: \n";
2414  ignored_msg += " 1) variables that have HDF5 NULL dataspace(H5S_NULL)(rarely occurred)\n";
2415  ignored_msg += " 2) variables that have any zero size dimensions\n";
2416 
2417 }
2418 #endif
2419 
2420 // Add the ignored link info.
2421 void File::add_ignored_info_links(const string & link_path)
2422 {
2423  if (ignored_msg.find("Link paths: ") == string::npos)
2424  ignored_msg += " Link paths: " + link_path;
2425  else
2426  ignored_msg += " " + link_path;
2427 }
2428 
2429 // Add the ignored name datatype info.
2430 void File::add_ignored_info_namedtypes(const string& grp_name, const string& named_dtype_name)
2431 {
2432 
2433  if (false == this->have_ignored) {
2434  add_ignored_info_obj_header();
2435  have_ignored = true;
2436  }
2437 
2438  string ignored_HDF5_named_dtype_hdr = "\n******WARNING******";
2439  ignored_HDF5_named_dtype_hdr += "\n IGNORED HDF5 named datatype objects:\n";
2440  string ignored_HDF5_named_dtype_msg = " Group name: " + grp_name + " HDF5 named datatype name: " + named_dtype_name.substr(0,named_dtype_name.size()-1)
2441  + "\n";
2442  if (ignored_msg.find(ignored_HDF5_named_dtype_hdr) == string::npos)
2443  ignored_msg += ignored_HDF5_named_dtype_hdr + ignored_HDF5_named_dtype_msg;
2444  else
2445  ignored_msg += ignored_HDF5_named_dtype_msg;
2446 
2447 }
2448 
2449 // Add the ignored attribute information. When is_grp is true, the ignored group attribute names are added.
2450 // Otherwise, the ignored dataset attribute names are added.
2451 void File::add_ignored_info_attrs(bool is_grp, const string & obj_path, const string & attr_name)
2452 {
2453 
2454  if (false == this->have_ignored) {
2455  add_ignored_info_obj_header();
2456  have_ignored = true;
2457  }
2458 
2459 
2460  string ignored_warning_str = "\n******WARNING******";
2461  string ignored_HDF5_grp_hdr = ignored_warning_str + "\n Ignored attributes under root and groups:\n";
2462  string ignored_HDF5_grp_msg = " Group path: " + obj_path + " Attribute names: " + attr_name + "\n";
2463  string ignored_HDF5_var_hdr = ignored_warning_str + "\n Ignored attributes for variables:\n";
2464  string ignored_HDF5_var_msg = " Variable path: " + obj_path + " Attribute names: " + attr_name + "\n";
2465 
2466 
2467  if (true == is_grp) {
2468  if (ignored_msg.find(ignored_HDF5_grp_hdr) == string::npos)
2469  ignored_msg += ignored_HDF5_grp_hdr + ignored_HDF5_grp_msg;
2470  else
2471  ignored_msg += ignored_HDF5_grp_msg;
2472  }
2473  else {
2474  if (ignored_msg.find(ignored_HDF5_var_hdr) == string::npos)
2475  ignored_msg += ignored_HDF5_var_hdr + ignored_HDF5_var_msg;
2476  else
2477  ignored_msg += ignored_HDF5_var_msg;
2478  }
2479 
2480 }
2481 
2482 //Ignored object information. When is_dim_related is true, ignored data space info. is present.
2483 //When is_dim_related is false, ignored data type info. is present.
2484 void File::add_ignored_info_objs(bool is_dim_related, const string & obj_path)
2485 {
2486 
2487  if (false == this->have_ignored) {
2488  add_ignored_info_obj_header();
2489  have_ignored = true;
2490  }
2491 
2492  string ignored_warning_str = "\n******WARNING******";
2493  string ignored_HDF5_dtype_var_hdr = ignored_warning_str + "\n IGNORED variables due to unsupported datatypes:\n";
2494  string ignored_HDF5_dspace_var_hdr = ignored_warning_str + "\n IGNORED variables due to unsupported dimensions:\n";
2495  string ignored_HDF5_var_msg = " Variable path: " + obj_path + "\n";
2496 
2497  if (true == is_dim_related) {
2498  if (ignored_msg.find(ignored_HDF5_dspace_var_hdr) == string::npos)
2499  ignored_msg += ignored_HDF5_dspace_var_hdr + ignored_HDF5_var_msg;
2500  else
2501  ignored_msg += ignored_HDF5_var_msg;
2502 
2503  }
2504  else {
2505  if (ignored_msg.find(ignored_HDF5_dtype_var_hdr) == string::npos)
2506  ignored_msg += ignored_HDF5_dtype_var_hdr + ignored_HDF5_var_msg;
2507  else
2508  ignored_msg += ignored_HDF5_var_msg;
2509  }
2510 
2511 }
2512 
2513 // No ignored info.
2514 void File::add_no_ignored_info()
2515 {
2516 
2517  ignored_msg += "There are no ignored HDF5 objects or attributes.";
2518 
2519 }
2520 
2521 // This function should only be used when the HDF5 file is following the netCDF data model.
2522 // Check if we should not report the Dimension scale related attributes as ignored.
2523 bool File::ignored_dimscale_ref_list(Var *var)
2524 {
2525 
2526  bool ignored_dimscale = true;
2527  // Only when "General_Product == this->product_type && GENERAL_DIMSCALE== this->gproduct_pattern)"
2528 
2529  bool has_dimscale = false;
2530  bool has_reference_list = false;
2531  for (auto ira = var->attrs.begin(); ira != var->attrs.end(); ira++) {
2532  if ((*ira)->name == "REFERENCE_LIST" && false == HDF5CFUtil::cf_strict_support_type((*ira)->getType(),_is_dap4))
2533  has_reference_list = true;
2534  if ((*ira)->name == "CLASS") {
2535  Retrieve_H5_Attr_Value(*ira, var->fullpath);
2536  string class_value;
2537  class_value.resize((*ira)->value.size());
2538  copy((*ira)->value.begin(), (*ira)->value.end(), class_value.begin());
2539 
2540  // Compare the attribute "CLASS" value with "DIMENSION_SCALE". We only compare the string with the size of
2541  // "DIMENSION_SCALE", which is 15.
2542  if (0 == class_value.compare(0, 15, "DIMENSION_SCALE")) {
2543  has_dimscale = true;
2544  }
2545  }
2546 
2547  if (true == has_dimscale && true == has_reference_list) {
2548  ignored_dimscale = false;
2549  break;
2550  }
2551 
2552  }
2553  //}
2554  return ignored_dimscale;
2555 }
2556 
2557 // Check if the long string can should be dropped from a dataset or an attribute. Users can set up a BES key to turn it off or on.
2558 bool File::Check_DropLongStr(const Var *var, const Attribute * attr)
2559 {
2560 
2561  bool drop_longstr = false;
2562  if (nullptr == attr) {
2563  if (H5FSTRING == var->dtype || H5VSTRING == var->dtype) {
2564  try {
2565  drop_longstr = Check_VarDropLongStr(var->fullpath, var->dims, var->dtype);
2566  }
2567  catch (...) {
2568  throw1("Check_VarDropLongStr fails ");
2569  }
2570  }
2571  }
2572  // No limitation for the attributes. KY 2018-02-26
2573 #if 0
2574  else {
2575  if (H5FSTRING == attr->dtype || H5VSTRING == attr->dtype) {
2576  if (attr->getBufSize() > NC_JAVA_STR_SIZE_LIMIT) {
2577  drop_longstr = true;
2578  }
2579  }
2580 
2581  }
2582 #endif
2583  return drop_longstr;
2584 }
2585 
2586 // Check if a long string dataset should be dropped. Users can turn on a BES key not to drop the long string.
2587 // However, the Java clients may not access.
2588 //
2589 bool File::Check_VarDropLongStr(const string & varpath, const vector<Dimension *>& dims, H5DataType dtype) const
2590 
2591 {
2592 
2593  bool drop_longstr = false;
2594 
2595  hid_t dset_id = H5Dopen2(this->fileid, varpath.c_str(), H5P_DEFAULT);
2596  if (dset_id < 0)
2597  throw2("Cannot open the dataset ", varpath);
2598 
2599  hid_t dtype_id = -1;
2600  if ((dtype_id = H5Dget_type(dset_id)) < 0) {
2601  H5Dclose(dset_id);
2602  throw2("Cannot obtain the datatype of the dataset ", varpath);
2603  }
2604 
2605  size_t ty_size = H5Tget_size(dtype_id);
2606  if (ty_size == 0) {
2607  H5Tclose(dtype_id);
2608  H5Dclose(dset_id);
2609  throw2("Cannot obtain the datatype size of the dataset ", varpath);
2610  }
2611 
2612  if (H5FSTRING == dtype) { // Fixed-size, just check the number of elements.
2613  if (ty_size > NC_JAVA_STR_SIZE_LIMIT) drop_longstr = true;
2614  }
2615  else if (H5VSTRING == dtype) {
2616 
2617  unsigned long long total_elms = 1;
2618  if (dims.size() != 0) {
2619  for (unsigned int i = 0; i < dims.size(); i++)
2620  total_elms = total_elms * ((dims[i])->size);
2621  }
2622  vector<char> strval;
2623  strval.resize(total_elms * ty_size);
2624  hid_t read_ret = H5Dread(dset_id, dtype_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, (void*) strval.data());
2625  if (read_ret < 0) {
2626  H5Tclose(dtype_id);
2627  H5Dclose(dset_id);
2628  throw2("Cannot read the data of the dataset ", varpath);
2629  }
2630 
2631  vector<string> finstrval;
2632  finstrval.resize(total_elms);
2633  char*temp_bp = strval.data();
2634  char*onestring = nullptr;
2635  for (unsigned long long i = 0; i < total_elms; i++) {
2636  onestring = *(char**) temp_bp;
2637  if (onestring != nullptr) {
2638  finstrval[i] = string(onestring);
2639  if(finstrval[i].size()>NC_JAVA_STR_SIZE_LIMIT) {
2640  drop_longstr = true;
2641  break;
2642  }
2643  }
2644  temp_bp += ty_size;
2645  }
2646 
2647  if (false == strval.empty()) {
2648  herr_t ret_vlen_claim;
2649  hid_t dspace_id = H5Dget_space(dset_id);
2650  if (dspace_id < 0) {
2651  H5Tclose(dtype_id);
2652  H5Dclose(dset_id);
2653  throw2("Cannot obtain the dataspace id.", varpath);
2654  }
2655  ret_vlen_claim = H5Dvlen_reclaim(dtype_id, dspace_id, H5P_DEFAULT, (void*) strval.data());
2656  if (ret_vlen_claim < 0) {
2657  H5Tclose(dtype_id);
2658  H5Sclose(dspace_id);
2659  H5Dclose(dset_id);
2660  throw2("Cannot reclaim the vlen space ", varpath);
2661  }
2662  if (H5Sclose(dspace_id) < 0) {
2663  H5Tclose(dtype_id);
2664  H5Dclose(dset_id);
2665  throw2("Cannot close the HDF5 data space.", varpath);
2666  }
2667  }
2668  }
2669  if (H5Tclose(dtype_id) < 0) {
2670  H5Dclose(dset_id);
2671  throw2("Cannot close the HDF5 data type.", varpath);
2672  }
2673  if (H5Dclose(dset_id) < 0)
2674  throw2("Cannot close the HDF5 data type.", varpath);
2675 
2676  return drop_longstr;
2677 }
2678 #if 0
2679 bool File::Check_VarDropLongStr(const string & varpath, const vector<Dimension *>& dims, H5DataType dtype)
2680 
2681 {
2682 
2683  bool drop_longstr = false;
2684 
2685  unsigned long long total_elms = 1;
2686  if (dims.size() != 0) {
2687  for (unsigned int i = 0; i < dims.size(); i++)
2688  total_elms = total_elms * ((dims[i])->size);
2689  }
2690 
2691  if (total_elms > NC_JAVA_STR_SIZE_LIMIT)
2692  drop_longstr = true;
2693 
2694  else { // We need to check both fixed-size and variable-length strings.
2695 
2696  hid_t dset_id = H5Dopen2(this->fileid, varpath.c_str(), H5P_DEFAULT);
2697  if (dset_id < 0)
2698  throw2("Cannot open the dataset ", varpath);
2699 
2700  hid_t dtype_id = -1;
2701  if ((dtype_id = H5Dget_type(dset_id)) < 0) {
2702  H5Dclose(dset_id);
2703  throw2("Cannot obtain the datatype of the dataset ", varpath);
2704  }
2705 
2706  size_t ty_size = H5Tget_size(dtype_id);
2707  if (ty_size == 0) {
2708  H5Tclose(dtype_id);
2709  H5Dclose(dset_id);
2710  throw2("Cannot obtain the datatype size of the dataset ", varpath);
2711  }
2712 
2713  if (H5FSTRING == dtype) { // Fixed-size, just check the number of elements.
2714  if ((ty_size * total_elms) > NC_JAVA_STR_SIZE_LIMIT) drop_longstr = true;
2715  }
2716  else if (H5VSTRING == dtype) {
2717 
2718  vector<char> strval;
2719  strval.resize(total_elms * ty_size);
2720  hid_t read_ret = H5Dread(dset_id, dtype_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, (void*) strval.data());
2721  if (read_ret < 0) {
2722  H5Tclose(dtype_id);
2723  H5Dclose(dset_id);
2724  throw2("Cannot read the data of the dataset ", varpath);
2725  }
2726 
2727  vector<string> finstrval;
2728  finstrval.resize(total_elms);
2729  char*temp_bp = strval.data();
2730  char*onestring = nullptr;
2731  for (unsigned long long i = 0; i < total_elms; i++) {
2732  onestring = *(char**) temp_bp;
2733  if (onestring != nullptr)
2734  finstrval[i] = string(onestring);
2735  else
2736  // We will add a NULL if onestring is NULL.
2737  finstrval[i] = "";
2738  temp_bp += ty_size;
2739  }
2740 
2741  if (false == strval.empty()) {
2742  herr_t ret_vlen_claim;
2743  hid_t dspace_id = H5Dget_space(dset_id);
2744  if (dspace_id < 0) {
2745  H5Tclose(dtype_id);
2746  H5Dclose(dset_id);
2747  throw2("Cannot obtain the dataspace id.", varpath);
2748  }
2749  ret_vlen_claim = H5Dvlen_reclaim(dtype_id, dspace_id, H5P_DEFAULT, (void*) strval.data());
2750  if (ret_vlen_claim < 0) {
2751  H5Tclose(dtype_id);
2752  H5Sclose(dspace_id);
2753  H5Dclose(dset_id);
2754  throw2("Cannot reclaim the vlen space ", varpath);
2755  }
2756  if (H5Sclose(dspace_id) < 0) {
2757  H5Tclose(dtype_id);
2758  H5Dclose(dset_id);
2759  throw2("Cannot close the HDF5 data space.", varpath);
2760  }
2761  }
2762  unsigned long long total_str_size = 0;
2763  for (unsigned long long i = 0; i < total_elms; i++) {
2764  total_str_size += finstrval[i].size();
2765  if (total_str_size > NC_JAVA_STR_SIZE_LIMIT) {
2766  drop_longstr = true;
2767  break;
2768  }
2769  }
2770  }
2771  if (H5Tclose(dtype_id) < 0) {
2772  H5Dclose(dset_id);
2773  throw2("Cannot close the HDF5 data type.", varpath);
2774  }
2775  if (H5Dclose(dset_id) < 0)
2776  throw2("Cannot close the HDF5 data type.", varpath);
2777  }
2778  return drop_longstr;
2779 }
2780 #endif
2781 
2782 
2783 // Provide if the long string is dropped.
2784 void File::add_ignored_grp_longstr_info(const string& grp_path, const string & attr_name)
2785 {
2786 
2787  ignored_msg += "The HDF5 group: " + grp_path + " has an empty-set string attribute: " + attr_name + "\n";
2788 
2789  return;
2790 }
2791 
2792 // Provide if the long variable string is dropped.
2793 void File::add_ignored_var_longstr_info(const Var *var, const Attribute *attr)
2794 {
2795 
2796  if (nullptr == attr)
2797  ignored_msg += "String variable: " + var->fullpath + " value is set to empty.\n";
2798  else {
2799  ignored_msg += "The variable: " + var->fullpath + " has an empty-set string attribute: " + attr->name + "\n";
2800 
2801  }
2802  return;
2803 }
2804 
2805 // The warnings of the drop of the long string header
2806 void File::add_ignored_droplongstr_hdr()
2807 {
2808 
2809  if (false == this->have_ignored) this->have_ignored = true;
2810  string hdr = "\n\n The values of the following string variables ";
2811  hdr += " are set to empty because at least one string size in this variable exceeds netCDF Java string limit(32767 bytes).\n";
2812  hdr += "To obtain the values, change the BES key H5.EnableDropLongString=true at the handler BES";
2813  hdr += " configuration file(h5.conf)\nto H5.EnableDropLongString=false.\n\n";
2814 
2815  if (ignored_msg.rfind(hdr) == string::npos) ignored_msg += hdr;
2816 
2817 }
2818 
2819 // Sometimes, we need to release the temporary added resources.
2820 void File::release_standalone_var_vector(vector<Var*>&temp_vars)
2821 {
2822 
2823  for (auto i = temp_vars.begin(); i != temp_vars.end();) {
2824  delete (*i);
2825  i = temp_vars.erase(i);
2826  }
2827 
2828 }
HDF5CF.h
This class specifies the core engineering of mapping HDF5 to DAP by following CF.
throw1
#define throw1(a1)
The followings are convenient functions to throw exceptions with different.
Definition: HDF5CF.h:128
HDF5RequestHandler.h
include the entry functions to execute the handlers
HDF5CF::Attribute
This class represents one attribute.
Definition: HDF5CF.h:189
HDF5CF::Dimension
This class repersents one dimension of an HDF5 dataset(variable).
Definition: HDF5CF.h:145
HDF5CF::File::groups
std::vector< Group * > groups
Non-root group vectors.
Definition: HDF5CF.h:795
HDF5CF::File::Retrieve_H5_Var_Attr_Values
virtual void Retrieve_H5_Var_Attr_Values(Var *var)
Retrieve attribute values for a variable.
Definition: HDF5CF.cc:746
HDF5CF::File::Handle_Unsupported_Dspace
virtual void Handle_Unsupported_Dspace(bool)
Handle unsupported HDF5 dataspaces for datasets.
Definition: HDF5CF.cc:1273
HDF5CF::File::dimsize_to_fakedimname
std::map< hsize_t, std::string > dimsize_to_fakedimname
Handle added dimension names.
Definition: HDF5CF.h:812
HDF5CF::File::Handle_Grid_Mapping_Vars
virtual void Handle_Grid_Mapping_Vars()
Handle Grid Mapping Vars.
Definition: HDF5CF.cc:2208
HDF5CF::File::Handle_Unsupported_Others
virtual void Handle_Unsupported_Others(bool)
Handle other unmapped objects/attributes.
Definition: HDF5CF.cc:1320
HDF5CF::File::Retrieve_H5_Supported_Attr_Values
virtual void Retrieve_H5_Supported_Attr_Values()
Retrieve attribute values for the supported HDF5 datatypes.
Definition: HDF5CF.cc:727
HDF5CF::File::vars
std::vector< Var * > vars
Var vectors.
Definition: HDF5CF.h:789
HDF5CF::File::Add_Supplement_Attrs
virtual void Add_Supplement_Attrs(bool)
Add supplemental attributes such as fullpath and original name.
Definition: HDF5CF.cc:2027
HDF5CF::File::Retrieve_H5_Info
virtual void Retrieve_H5_Info(const char *path, hid_t file_id, bool)
Definition: HDF5CF.cc:168
HDF5CF::File::root_attrs
std::vector< Attribute * > root_attrs
Root attribute vectors.
Definition: HDF5CF.h:792
HDF5CF::File::Handle_Unsupported_Dtype
virtual void Handle_Unsupported_Dtype(bool)
Handle unsupported HDF5 datatypes.
Definition: HDF5CF.cc:916
HDF5CF::File::Flatten_Obj_Name
virtual void Flatten_Obj_Name(bool)
Flatten the object name.
Definition: HDF5CF.cc:1370
HDF5CF::File::Have_Grid_Mapping_Attrs
virtual bool Have_Grid_Mapping_Attrs()
Check if having Grid Mapping Attrs.
Definition: HDF5CF.cc:2188
HDF5CF::Group
This class represents an HDF5 group. The group will be flattened according to the CF conventions.
Definition: HDF5CF.h:525
HDF5CF::Var
This class represents one HDF5 dataset(CF variable)
Definition: HDF5CF.h:256
h5cfdaputil.h
Helper functions for generating DAS attributes and a function to check BES Key.
HDF5CFUtil::H5type_to_H5DAPtype
static H5DataType H5type_to_H5DAPtype(hid_t h5_type_id)
Map HDF5 Datatype to the intermediate H5DAPtype for the future use.
Definition: HDF5CFUtil.cc:54
HDF5CFUtil::trim_string
static std::string trim_string(hid_t dtypeid, const std::string &s, int num_sect, size_t section_size, std::vector< size_t > &sect_newsize)
Definition: HDF5CFUtil.cc:235
HDF5CF::delete_elem
Definition: HDF5CF.h:134