bes  Updated for version 3.20.13
HDF5Array.cc
Go to the documentation of this file.
1 // This file is part of hdf5_handler a HDF5 file handler for the OPeNDAP
2 // data server.
3 
4 // Author: Hyo-Kyung Lee <hyoklee@hdfgroup.org> and Muqun Yang
5 // <myang6@hdfgroup.org>
6 
7 // Copyright (c) 2009-2016 The HDF Group, Inc. and OPeNDAP, Inc.
8 //
9 // This is H5free_memory software; you can redistribute it and/or modify it under the
10 // terms of the GNU Lesser General Public License as published by the Free
11 // Software Foundation; either version 2.1 of the License, or (at your
12 // option) any later version.
13 //
14 // This software is distributed in the hope that it will be useful, but
15 // WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
16 // or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
17 // License for more details.
18 //
19 // You should have received a copy of the GNU Lesser General Public
20 // License along with this library; if not, write to the Free Software
21 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
22 //
23 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
24 // You can contact The HDF Group, Inc. at 1800 South Oak Street,
25 // Suite 203, Champaign, IL 61820
33 
34 
35 #include "config_hdf5.h"
36 
37 #include <iostream>
38 #include <memory>
39 #include <sstream>
40 #include <algorithm>
41 #include <ctype.h>
42 
43 #include <BESDebug.h>
44 #include <libdap/Error.h>
45 #include <libdap/InternalErr.h>
46 
47 #include "HDF5Array.h"
48 #include "HDF5Structure.h"
49 #include "HDF5Str.h"
50 
51 using namespace std;
52 using namespace libdap;
53 
54 BaseType *HDF5Array::ptr_duplicate() {
55  return new HDF5Array(*this);
56 }
57 
58 HDF5Array::HDF5Array(const string & n, const string &d, BaseType * v) :
59  Array(n, d, v) {
60 }
61 
62 
63 int HDF5Array::format_constraint(int *offset, int *step, int *count) {
64 
65  // For the 0-length array case, just return 0.
66  if(length() == 0)
67  return 0;
68 
69  int nels = 1;
70  int id = 0;
71 
72  Dim_iter p = dim_begin();
73 
74  while (p != dim_end()) {
75 
76  int start = dimension_start(p, true);
77  int stride = dimension_stride(p, true);
78  int stop = dimension_stop(p, true);
79 
80  // Check for empty constraint
81  if (start > stop) {
82  ostringstream oss;
83 
84  oss << "Array/Grid hyperslab start point "<< start <<
85  " is greater than stop point " << stop <<".";
86  throw Error(malformed_expr, oss.str());
87  }
88 
89  offset[id] = start;
90  step[id] = stride;
91  count[id] = ((stop - start) / stride) + 1; // count of elements
92  nels *= count[id]; // total number of values for variable
93 
94  BESDEBUG("h5",
95  "=format_constraint():"
96  << "id=" << id << " offset=" << offset[id]
97  << " step=" << step[id]
98  << " count=" << count[id]
99  << endl);
100 
101  id++;
102  p++;
103  }
104 
105  return nels;
106 }
107 
108 
109 bool HDF5Array::read()
110 {
111  BESDEBUG("h5",
112  ">read() dataset=" << dataset()
113  << " dimension=" << d_num_dim
114  << " data_size=" << d_memneed << " length=" << length()
115  << endl);
116 
117  hid_t file_id = H5Fopen(dataset().c_str(),H5F_ACC_RDONLY,H5P_DEFAULT);
118 
119  BESDEBUG("h5","after H5Fopen "<<endl);
120  BESDEBUG("h5","variable name is "<<name() <<endl);
121  BESDEBUG("h5","variable path is "<<var_path <<endl);
122 
123  hid_t dset_id = -1;
124 
125  if(true == is_dap4())
126  dset_id = H5Dopen2(file_id,var_path.c_str(),H5P_DEFAULT);
127  else
128  dset_id = H5Dopen2(file_id,name().c_str(),H5P_DEFAULT);
129 
130  BESDEBUG("h5","after H5Dopen2 "<<endl);
131  // Leave the following code. We may replace the struct DS and DSattr(see hdf5_handler.h)
132 #if 0
133  hid_t dspace_id = H5Dget_space(dset_id);
134  if(dspace_id < 0) {
135  H5Dclose(dset_id);
136  H5Fclose(file_id);
137  throw InternalErr(__FILE__,__LINE__, "Fail to obtain the dataspace .");
138  }
139 
140  int num_dim = H5Sget_simple_extent_ndims(dspace_id);
141  if(num_dim < 0) {
142  H5Sclose(dspace_id);
143  H5Dclose(dset_id);
144  H5Fclose(file_id);
145  throw InternalErr(__FILE__,__LINE__, "Fail to obtain the datatype .");
146  }
147 
148  H5Sclose(dspace_id);
149 #endif
150 
151  hid_t dtype_id = H5Dget_type(dset_id);
152  if(dtype_id < 0) {
153  H5Dclose(dset_id);
154  H5Fclose(file_id);
155  throw InternalErr(__FILE__,__LINE__, "Fail to obtain the datatype .");
156  }
157 
158 
159  vector<int> offset(d_num_dim);
160  vector<int> count(d_num_dim);
161  vector<int> step(d_num_dim);
162  int nelms = format_constraint(offset.data(), step.data(), count.data()); // Throws Error.
163  vector<char>values;
164 
165 
166  // We only map the reference to URL when the dataset is an array of reference.
167  if (get_dap_type(dtype_id,is_dap4()) == "Url") {
168  bool ret_ref = false;
169  try {
170  ret_ref = m_array_of_reference(dset_id,dtype_id);
171  H5Tclose(dtype_id);
172  H5Dclose(dset_id);
173  H5Fclose(file_id);
174 
175  }
176  catch(...) {
177  H5Tclose(dtype_id);
178  H5Dclose(dset_id);
179  H5Fclose(file_id);
180  throw;
181 
182  }
183  return ret_ref;
184  }
185 
186  try {
187  do_array_read(dset_id,dtype_id,values,false,0,nelms,offset.data(),count.data(),step.data());
188  }
189  catch(...) {
190  H5Tclose(dtype_id);
191  H5Dclose(dset_id);
192  H5Fclose(file_id);
193  throw;
194  }
195 
196  H5Tclose(dtype_id);
197  H5Dclose(dset_id);
198  H5Fclose(file_id);
199 
200  return true;
201 }
202 
203 void HDF5Array::do_array_read(hid_t dset_id,hid_t dtype_id,vector<char>&values,bool has_values,int values_offset,
204  int nelms,int* offset,int* count, int* step)
205 {
206 
207  H5T_class_t tcls = H5Tget_class(dtype_id);
208 
209  if(H5T_COMPOUND == tcls)
210  m_array_of_structure(dset_id,values,has_values,values_offset,nelms,offset,count,step);
211  else if(H5T_INTEGER == tcls || H5T_FLOAT == tcls || H5T_STRING == tcls)
212  m_array_of_atomic(dset_id,dtype_id,nelms,offset,count,step);
213  else {
214  throw InternalErr(__FILE__,__LINE__,"Fail to read the data for Unsupported datatype.");
215  }
216 
217 }
218 
219 void HDF5Array:: m_array_of_atomic(hid_t dset_id, hid_t dtype_id,
220  int nelms,int* offset,int* count, int* step)
221 {
222 
223  hid_t memtype = -1;
224  if((memtype = H5Tget_native_type(dtype_id, H5T_DIR_ASCEND))<0) {
225  throw InternalErr (__FILE__, __LINE__, "Fail to obtain memory datatype.");
226  }
227 
228  // First handle variable-length string
229  if (H5Tis_variable_str(memtype) && H5Tget_class(memtype) == H5T_STRING) {
230 
231  vector<hsize_t> hoffset;
232  vector<hsize_t>hcount;
233  vector<hsize_t>hstep;
234  hoffset.resize(d_num_dim);
235  hcount.resize(d_num_dim);
236  hstep.resize(d_num_dim);
237  for (int i = 0; i <d_num_dim; i++) {
238  hoffset[i] = (hsize_t) offset[i];
239  hcount[i] = (hsize_t) count[i];
240  hstep[i] = (hsize_t) step[i];
241  }
242 
243  vector<string>finstrval;
244  finstrval.resize(nelms);
245  try {
246  read_vlen_string(dset_id, nelms, hoffset.data(), hstep.data(), hcount.data(),finstrval);
247  }
248  catch(...) {
249  H5Tclose(memtype);
250  throw InternalErr(__FILE__,__LINE__,"Fail to read variable-length string.");
251  }
252  set_value(finstrval,nelms);
253  H5Tclose(memtype);
254  return ;
255  }
256 
257  try {
258  if (nelms == d_num_elm) {
259 
260  vector<char> convbuf(d_memneed);
261  get_data(dset_id, (void *) convbuf.data());
262 
263  // Check if a Signed Byte to Int16 conversion is necessary, this is only valid for DAP2.
264  if(false == is_dap4()) {
265  if (1 == H5Tget_size(memtype) && H5T_SGN_2 == H5Tget_sign(memtype))
266  {
267  vector<short> convbuf2(nelms);
268  for (int i = 0; i < nelms; i++) {
269  convbuf2[i] = (signed char) (convbuf[i]);
270  BESDEBUG("h5", "convbuf[" << i << "]="
271  << (signed char)convbuf[i] << endl);
272  BESDEBUG("h5", "convbuf2[" << i << "]="
273  << convbuf2[i] << endl)
274  ;
275  }
276  // Libdap will generate the wrong output.
277  m_intern_plain_array_data((char*) convbuf2.data(),memtype);
278  }
279  else
280  m_intern_plain_array_data(convbuf.data(),memtype);
281  }
282  else
283  m_intern_plain_array_data(convbuf.data(),memtype);
284  } // end of "if (nelms == d_num_elm)"
285  else {
286  size_t data_size = nelms * H5Tget_size(memtype);
287  if (data_size == 0) {
288  throw InternalErr(__FILE__, __LINE__, "get_size failed");
289  }
290  vector<char> convbuf(data_size);
291  get_slabdata(dset_id, offset, step, count, d_num_dim, convbuf.data());
292 
293  // Check if a Signed Byte to Int16 conversion is necessary.
294  if(false == is_dap4()){
295  if (1 == H5Tget_size(memtype) && H5T_SGN_2 == H5Tget_sign(memtype)) {
296  vector<short> convbuf2(data_size);
297  for (int i = 0; i < (int)data_size; i++) {
298  convbuf2[i] = static_cast<signed char> (convbuf[i]);
299  }
300  m_intern_plain_array_data((char*) convbuf2.data(),memtype);
301  }
302  else {
303  m_intern_plain_array_data(convbuf.data(),memtype);
304  }
305  }
306  else
307  m_intern_plain_array_data(convbuf.data(),memtype);
308 
309  }
310  H5Tclose(memtype);
311  }
312  catch (...) {
313  H5Tclose(memtype);
314  throw;
315  }
316 
317 }
318 
319 bool HDF5Array::m_array_of_structure(hid_t dsetid, vector<char>&values,bool has_values,int values_offset,
320  int nelms,const int* offset,const int* count, const int* step) {
321 
322  BESDEBUG("h5", "=read() Array of Structure length=" << length() << endl);
323 
324  hid_t mspace = -1;
325  hid_t memtype = -1;
326  hid_t dtypeid = -1;
327  size_t ty_size = -1;
328 
329  if((dtypeid = H5Dget_type(dsetid)) < 0)
330  throw InternalErr (__FILE__, __LINE__, "Cannot obtain the datatype.");
331 
332  if((memtype = H5Tget_native_type(dtypeid, H5T_DIR_ASCEND))<0) {
333  H5Tclose(dtypeid);
334  throw InternalErr (__FILE__, __LINE__, "Fail to obtain memory datatype.");
335  }
336 
337  ty_size = H5Tget_size(memtype);
338  if (ty_size == 0) {
339  H5Tclose(memtype);
340  H5Tclose(dtypeid);
341  throw InternalErr (__FILE__, __LINE__,"Fail to obtain the size of HDF5 compound datatype.");
342  }
343 
344  if(false == has_values) {
345 
346  hid_t dspace = -1;
347 
348  if ((dspace = H5Dget_space(dsetid))<0) {
349  H5Tclose(memtype);
350  H5Tclose(dtypeid);
351  throw InternalErr (__FILE__, __LINE__, "Cannot obtain data space.");
352  }
353 
354  d_num_dim = H5Sget_simple_extent_ndims(dspace);
355  if(d_num_dim < 0) {
356  H5Tclose(memtype);
357  H5Tclose(dtypeid);
358  H5Sclose(dspace);
359  throw InternalErr (__FILE__, __LINE__, "Cannot obtain the number of dimensions of the data space.");
360  }
361 
362  vector<hsize_t> hoffset;
363  vector<hsize_t>hcount;
364  vector<hsize_t>hstep;
365  hoffset.resize(d_num_dim);
366  hcount.resize(d_num_dim);
367  hstep.resize(d_num_dim);
368  for (int i = 0; i <d_num_dim; i++) {
369  hoffset[i] = (hsize_t) offset[i];
370  hcount[i] = (hsize_t) count[i];
371  hstep[i] = (hsize_t) step[i];
372  }
373 
374  if (H5Sselect_hyperslab(dspace, H5S_SELECT_SET,
375  hoffset.data(), hstep.data(),
376  hcount.data(), nullptr) < 0) {
377  H5Tclose(memtype);
378  H5Tclose(dtypeid);
379  H5Sclose(dspace);
380  throw InternalErr (__FILE__, __LINE__, "Cannot generate the hyperslab of the HDF5 dataset.");
381  }
382 
383  mspace = H5Screate_simple(d_num_dim, hcount.data(),nullptr);
384  if (mspace < 0) {
385  H5Sclose(dspace);
386  H5Tclose(memtype);
387  H5Tclose(dtypeid);
388  throw InternalErr (__FILE__, __LINE__, "Cannot create the memory space.");
389  }
390 
391  values.resize(nelms*ty_size);
392  hid_t read_ret = -1;
393  read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,(void*)values.data());
394  if (read_ret < 0) {
395  H5Tclose(memtype);
396  H5Tclose(dtypeid);
397  H5Sclose(dspace);
398  throw InternalErr (__FILE__, __LINE__, "Fail to read the HDF5 compound datatype dataset.");
399  }
400 
401  H5Sclose(dspace);
402  has_values = true;
403  } // end of "if(false == has_values)" block
404 
405  HDF5Structure *h5s = nullptr;
406  hid_t memb_id = -1;
407  char* memb_name = nullptr;
408 
409  try {
410 
411  // Loop through all the elements in this compound datatype array.
412  for (int element = 0; element < nelms; ++element) {
413 
414  h5s = dynamic_cast<HDF5Structure*>(var()->ptr_duplicate());
415  H5T_class_t memb_cls = H5T_NO_CLASS;
416  int nmembs = 0;
417  size_t memb_offset = 0;
418 
419  if((nmembs = H5Tget_nmembers(memtype)) < 0)
420  throw InternalErr (__FILE__, __LINE__, "Fail to obtain number of HDF5 compound datatype.");
421 
422  for(unsigned int u = 0; u < (unsigned)nmembs; u++) {
423 
424  // Get member type ID
425  if((memb_id = H5Tget_member_type(memtype, u)) < 0)
426  throw InternalErr (__FILE__, __LINE__, "Fail to obtain the datatype of an HDF5 compound datatype member.");
427 
428  // Get member type class
429  if((memb_cls = H5Tget_member_class (memtype, u)) < 0)
430  throw InternalErr (__FILE__, __LINE__, "Fail to obtain the datatype class of an HDF5 compound datatype member.");
431 
432  // Get member offset,H5Tget_member_offset only fails
433  // when H5Tget_memeber_class fails. Sinc H5Tget_member_class
434  // is checked above. So no need to check the return value.
435  memb_offset= H5Tget_member_offset(memtype,u);
436 
437  // Get member name
438  memb_name = H5Tget_member_name(memtype,u);
439  if(memb_name == nullptr)
440  throw InternalErr (__FILE__, __LINE__, "Fail to obtain the name of an HDF5 compound datatype member.");
441 
442  BaseType *field = h5s->var(memb_name);
443  if (memb_cls == H5T_COMPOUND) {
444  HDF5Structure &memb_h5s = dynamic_cast<HDF5Structure&>(*field);
445  memb_h5s.do_structure_read(dsetid, memb_id,values,has_values,memb_offset+values_offset+ty_size*element);
446  }
447 
448  else if(memb_cls == H5T_ARRAY) {
449 
450  // memb_id, obtain the number of dimensions
451  int at_ndims = H5Tget_array_ndims(memb_id);
452  if(at_ndims <= 0)
453  throw InternalErr (__FILE__, __LINE__, "Fail to obtain number of dimensions of the array datatype.");
454 
455 #if 0
456  //HDF5Array &h5_array_type = dynamic_cast<HDF5Array&>(*h5s->var(memb_name));
457 #endif
458  HDF5Array &h5_array_type = dynamic_cast<HDF5Array&>(*field);
459  vector<int> at_offset(at_ndims,0);
460  vector<int> at_count(at_ndims,0);
461  vector<int> at_step(at_ndims,0);
462 
463  int at_nelms = h5_array_type.format_constraint(at_offset.data(),at_step.data(),at_count.data());
464 
465  // Read the array data
466  h5_array_type.do_h5_array_type_read(dsetid,memb_id,values,has_values,memb_offset+values_offset+ty_size*element,
467  at_nelms,at_offset.data(),at_count.data(),at_step.data());
468 
469  }
470  else if(memb_cls == H5T_INTEGER || memb_cls == H5T_FLOAT) {
471 
472  if(true == promote_char_to_short(memb_cls,memb_id)) {
473  void *src = (void*)(values.data() + (element*ty_size) + values_offset +memb_offset);
474  char val_int8;
475  memcpy(&val_int8,src,1);
476  short val_short=(short)val_int8;
477  field->val2buf(&val_short);
478  }
479  else {
480  field->val2buf(values.data() + (element*ty_size) + values_offset +memb_offset);
481  }
482 
483  }
484  else if(memb_cls == H5T_STRING) {
485 
486  // distinguish between variable length and fixed length
487  if(true == H5Tis_variable_str(memb_id)) {
488  void *src = (void*)(values.data()+(element*ty_size)+values_offset + memb_offset);
489  string final_str;
490  get_vlen_str_data((char*)src,final_str);
491  field->val2buf(&final_str);
492  }
493  else {// Obtain fixed-size string value
494  void *src = (void*)(values.data()+(element*ty_size)+values_offset + memb_offset);
495  vector<char> str_val;
496  size_t memb_size = H5Tget_size(memb_id);
497  if (memb_size == 0) {
498  H5Tclose(memb_id);
499  H5free_memory(memb_name);
500  delete h5s;
501  throw InternalErr (__FILE__, __LINE__,"Fail to obtain the size of HDF5 compound datatype.");
502  }
503  str_val.resize(memb_size);
504  memcpy(str_val.data(),src,memb_size);
505  string temp_string(str_val.begin(),str_val.end());
506  field->val2buf(&temp_string);
507  }
508  }
509  else {
510  H5free_memory(memb_name);
511  H5Tclose(memb_id);
512  delete h5s;
513  throw InternalErr (__FILE__, __LINE__,
514  "Only support the field of compound datatype when the field type class is integer, float, string, array or compound..");
515 
516  }
517 
518  // Close member type ID
519  H5Tclose(memb_id);
520  H5free_memory(memb_name);
521  field->set_read_p(true);
522  } // end "for(unsigned u = 0)"
523  h5s->set_read_p(true);
524  set_vec(element,h5s);
525  delete h5s;
526  } // end "for (int element=0"
527 
528  if(true == has_values) {
529  if(-1 == mspace)
530  throw InternalErr(__FILE__, __LINE__, "memory type and memory space for this compound datatype should be valid.");
531 
532  if(H5Dvlen_reclaim(memtype,mspace,H5P_DEFAULT,(void*)values.data())<0)
533  throw InternalErr(__FILE__, __LINE__, "Unable to reclaim the compound datatype array.");
534  H5Sclose(mspace);
535 
536  }
537 
538  H5Tclose(dtypeid);
539  H5Tclose(memtype);
540 
541  }
542  catch(...) {
543 
544  if(memb_id != -1)
545  H5Tclose(memb_id);
546  if(memb_name != nullptr)
547  H5free_memory(memb_name);
548  if(h5s != nullptr)
549  delete h5s;
550  if(true == has_values) {
551  if(H5Dvlen_reclaim(memtype,mspace,H5P_DEFAULT,(void*)(values.data()))<0) {
552  H5Tclose(memtype);
553  H5Sclose(mspace);
554  }
555  H5Sclose(mspace);
556  }
557  H5Tclose(memtype);
558  H5Tclose(dtypeid);
559  throw;
560  }
561 
562  set_read_p(true);
563 
564  return false;
565 }
566 
567 // Haven't checked the codes and comments
568 // Haven't added the close handles routines for error handlings yet. KY 2011-11-18
569 bool HDF5Array::m_array_of_reference(hid_t dset_id,hid_t dtype_id)
570 {
571 
572 #if (H5_VERS_MAJOR == 1 && (H5_VERS_MINOR == 10 || H5_VERS_MINOR == 8 || H5_VERS_MINOR == 6))
573  hid_t d_dset_id = dset_id;
574  hdset_reg_ref_t *rbuf = nullptr;
575 
576  try {
577 
578  vector<int> offset(d_num_dim);
579  vector<int> count(d_num_dim);
580  vector<int> step(d_num_dim);
581 
582 
583  int nelms = format_constraint(offset.data(), step.data(), count.data()); // Throws Error.
584  vector<string> v_str(nelms);
585 
586  BESDEBUG("h5", "=read() URL type is detected. "
587  << "nelms=" << nelms << " full_size=" << d_num_elm << endl);
588 
589 
590  // Handle regional reference.
591  if (H5Tequal(dtype_id, H5T_STD_REF_DSETREG) < 0) {
592  throw InternalErr(__FILE__, __LINE__, "H5Tequal() failed");
593  }
594 
595  if (H5Tequal(dtype_id, H5T_STD_REF_DSETREG) > 0) {
596  BESDEBUG("h5", "=read() Got regional reference. " << endl);
597  // Vector doesn't work for this case. somehow it doesn't support the type.
598  rbuf = new hdset_reg_ref_t[d_num_elm];
599  if(rbuf == nullptr){
600  throw InternalErr(__FILE__, __LINE__, "new() failed.");
601  }
602  if (H5Dread(d_dset_id, H5T_STD_REF_DSETREG, H5S_ALL, H5S_ALL, H5P_DEFAULT, rbuf) < 0) {
603  throw InternalErr(__FILE__, __LINE__, "H5Dread() failed.");
604  }
605 
606  for (int i = 0; i < nelms; i++) {
607  // Let's assume that URL array is always 1 dimension.
608  BESDEBUG("h5", "=read() rbuf[" << i << "]" <<
609  rbuf[offset[0] + i * step[0]] << endl);
610 
611  if (rbuf[offset[0] + i * step[0]][0] != '\0') {
612  char r_name[DODS_NAMELEN];
613 
614  hid_t did_r = H5RDEREFERENCE(d_dset_id, H5R_DATASET_REGION, rbuf[offset[0] + i * step[0]]);
615  if (did_r < 0) {
616  throw InternalErr(__FILE__, __LINE__, "H5RDEREFERENCE() failed.");
617 
618  }
619 
620  if (H5Iget_name(did_r, (char *) r_name, DODS_NAMELEN) < 0) {
621  throw InternalErr(__FILE__, __LINE__, "H5Iget_name() failed.");
622  }
623  BESDEBUG("h5", "=read() dereferenced name is " << r_name
624  << endl);
625 
626  string varname(r_name);
627  hid_t space_id = H5Rget_region(did_r, H5R_DATASET_REGION, rbuf[offset[0] + i * step[0]]);
628  if (space_id < 0) {
629  throw InternalErr(__FILE__, __LINE__, "H5Rget_region() failed.");
630 
631  }
632 
633  int ndim = H5Sget_simple_extent_ndims(space_id);
634  if (ndim < 0) {
635  throw InternalErr(__FILE__, __LINE__, "H5Sget_simple_extent_ndims() failed.");
636  }
637 
638  BESDEBUG("h5", "=read() dim is " << ndim << endl);
639 
640  string expression;
641  switch (H5Sget_select_type(space_id)) {
642 
643  case H5S_SEL_NONE:
644  BESDEBUG("h5", "=read() None selected." << endl);
645  break;
646 
647  case H5S_SEL_POINTS: {
648  BESDEBUG("h5", "=read() Points selected." << endl);
649  hssize_t npoints = H5Sget_select_npoints(space_id);
650  if (npoints < 0) {
651  throw InternalErr(__FILE__, __LINE__,
652  "Cannot determine number of elements in the dataspace selection");
653  }
654 
655  BESDEBUG("h5", "=read() npoints are " << npoints
656  << endl);
657  vector<hsize_t> buf(npoints * ndim);
658  if (H5Sget_select_elem_pointlist(space_id, 0, npoints, buf.data()) < 0) {
659  throw InternalErr(__FILE__, __LINE__, "H5Sget_select_elem_pointlist() failed.");
660  }
661 
662 #if 0
663  for (int j = 0; j < npoints * ndim; j++) {
664  "h5", "=read() npoints buf[0] =" << buf[j] <<endl;
665  }
666 #endif
667 
668  for (int j = 0; j < (int) npoints; j++) {
669  // Name of the dataset.
670  expression.append(varname);
671  for (int k = 0; k < ndim; k++) {
672  ostringstream oss;
673  oss << "[" << (int) buf[j * ndim + k] << "]";
674  expression.append(oss.str());
675  }
676  if (j != (int) (npoints - 1)) {
677  expression.append(",");
678  }
679  }
680  v_str[i].append(expression);
681 
682  break;
683  }
684  case H5S_SEL_HYPERSLABS: {
685  vector<hsize_t> start(ndim);
686  vector<hsize_t> end(ndim);
687  vector<hsize_t>stride(ndim);
688  vector<hsize_t>s_count(ndim);
689  vector<hsize_t>block(ndim);
690 
691  BESDEBUG("h5", "=read() Slabs selected." << endl);
692  BESDEBUG("h5", "=read() nblock is " <<
693  H5Sget_select_hyper_nblocks(space_id) << endl);
694 
695 #if (H5_VERS_MAJOR == 1 && H5_VERS_MINOR == 8)
696  if (H5Sget_select_bounds(space_id, start.data(), end.data()) < 0) {
697  throw InternalErr(__FILE__, __LINE__, "H5Sget_select_bounds() failed.");
698  }
699 #else
700  if (H5Sget_regular_hyperslab(space_id, start.data(), stride.data(), s_count.data(), block.data()) < 0) {
701  throw InternalErr(__FILE__, __LINE__, "H5Sget_regular_hyperslab() failed.");
702  }
703 #endif
704 
705  for (int j = 0; j < ndim; j++) {
706  ostringstream oss;
707  BESDEBUG("h5", "start " << start[j]
708  << "stride "<<stride[j]
709  << "count "<< s_count[j]
710  << "block "<< block[j]
711  <<endl);
712 
713  // Map from HDF5's start,stride,count,block to DAP's start,stride,end.
714  end[j] = start[j] + stride[j]*(s_count[j]-1)+(block[j]-1);
715  BESDEBUG("h5", "=read() start is " << start[j]
716  << "=read() end is " << end[j] << endl);
717  oss << "[" << start[j] << ":" << stride[j] << ":" << end[j] << "]";
718  expression.append(oss.str());
719  BESDEBUG("h5", "=read() expression is "
720  << expression << endl)
721  ;
722  }
723  v_str[i] = varname;
724  if (!expression.empty()) {
725  v_str[i].append(expression);
726  }
727  // Constraint expression. [start:1:end]
728  break;
729  }
730  case H5S_SEL_ALL:
731  BESDEBUG("h5", "=read() All selected." << endl);
732  break;
733 
734  default:
735  BESDEBUG("h5", "Unknown space type." << endl);
736  break;
737  }
738 
739  }
740  else {
741  v_str[i] = "";
742  }
743  }
744  delete[] rbuf;
745  }
746 
747  // Handle object reference.
748  if (H5Tequal(dtype_id, H5T_STD_REF_OBJ) < 0) {
749  throw InternalErr(__FILE__, __LINE__, "H5Tequal() failed.");
750  }
751 
752  if (H5Tequal(dtype_id, H5T_STD_REF_OBJ) > 0) {
753  BESDEBUG("h5", "=read() Got object reference. " << endl);
754  vector<hobj_ref_t> orbuf;
755  orbuf.resize(d_num_elm);
756  if (H5Dread(d_dset_id, H5T_STD_REF_OBJ, H5S_ALL, H5S_ALL, H5P_DEFAULT, orbuf.data()) < 0) {
757  throw InternalErr(__FILE__, __LINE__, "H5Dread failed()");
758  }
759 
760  for (int i = 0; i < nelms; i++) {
761  // Let's assume that URL array is always 1 dimension.
762  hid_t did_r = H5RDEREFERENCE(d_dset_id, H5R_OBJECT, &orbuf[offset[0] + i * step[0]]);
763  if (did_r < 0) {
764  throw InternalErr(__FILE__, __LINE__, "H5RDEREFERENCE() failed.");
765  }
766  char r_name[DODS_NAMELEN];
767  if (H5Iget_name(did_r, (char *) r_name, DODS_NAMELEN) < 0) {
768  throw InternalErr(__FILE__, __LINE__, "H5Iget_name() failed.");
769  }
770 
771  // Shorten the dataset name
772  string varname(r_name);
773 
774  BESDEBUG("h5", "=read() dereferenced name is " << r_name <<endl);
775  v_str[i] = varname;
776  }
777  }
778  set_value(v_str.data(), nelms);
779  return false;
780  }
781  catch (...) {
782  if(rbuf!= nullptr)
783  delete[] rbuf;
784  throw;
785  }
786 #else
787  return m_array_of_reference_new_h5_apis(dset_id,dtype_id);
788 
789 #endif
790 }
791 
792 bool HDF5Array::m_array_of_reference_new_h5_apis(hid_t dset_id,hid_t dtype_id) {
793 
794 #if (H5_VERS_MAJOR == 1 && (H5_VERS_MINOR == 10 || H5_VERS_MINOR == 8 || H5_VERS_MINOR == 6))
795  throw InternalErr(__FILE__, __LINE__,
796  "The HDF5 handler compiled with earlier version (<=110)of the HDF5 library should not call method that uses new reference APIs");
797  return false;
798 #else
799 
800  H5R_ref_t *rbuf = nullptr;
801  hid_t mem_space_id;
802  hid_t file_space_id;
803 
804  try {
805 
806  // First we need to read the reference data from DAP's hyperslab selection.
807  vector<int> offset(d_num_dim);
808  vector<int> count(d_num_dim);
809  vector<int> step(d_num_dim);
810  vector<hsize_t> hoffset(d_num_dim);
811  vector<hsize_t>hcount(d_num_dim);
812  vector<hsize_t>hstep(d_num_dim);
813 
814  int nelms = format_constraint(offset.data(), step.data(), count.data());
815  for (int i = 0; i <d_num_dim; i++) {
816  hoffset[i] = (hsize_t) offset[i];
817  hcount[i] = (hsize_t) count[i];
818  hstep[i] = (hsize_t) step[i];
819  }
820 
821  BESDEBUG("h5", "=read() URL type is detected. "
822  << "nelms=" << nelms << endl);
823 
824  rbuf = new H5R_ref_t[nelms];
825 
826  file_space_id = H5Dget_space(dset_id);
827  if(file_space_id < 0)
828  throw InternalErr(__FILE__, __LINE__, "Fail to obtain reference dataset file space.");
829 
830  if (H5Sselect_hyperslab(file_space_id, H5S_SELECT_SET,
831  hoffset.data(), hstep.data(),
832  hcount.data(), nullptr) < 0)
833  throw InternalErr (__FILE__, __LINE__, "Fail to select the hyperslab for reference dataset.");
834 
835 
836  mem_space_id = H5Screate_simple(d_num_dim,hcount.data(),nullptr);
837  if(mem_space_id < 0)
838  throw InternalErr(__FILE__, __LINE__, "Fail to obtain reference dataset memory space.");
839 
840  if(H5Dread(dset_id,H5T_STD_REF,mem_space_id,file_space_id,H5P_DEFAULT,&rbuf[0])<0)
841  throw InternalErr(__FILE__, __LINE__, "Fail to read hyperslab reference dataset.");
842 
843  H5Sclose(mem_space_id);
844  H5Sclose(file_space_id);
845 
846  // Now we need to retrieve the reference info. fot the nelms elements.
847  vector<string> v_str;
848 
849  H5R_type_t ref_type = H5Rget_type((const H5R_ref_t *)&rbuf[0]);
850 
851  // The referenced objects can only be either objects or dataset regions.
852  if(ref_type != H5R_OBJECT2 && ref_type !=H5R_DATASET_REGION2)
853  throw InternalErr(__FILE__, __LINE__, "Unsupported reference: neither object nor region references");
854 
855  for (int i = 0; i < nelms; i++) {
856 
857  hid_t obj_id = H5Ropen_object((H5R_ref_t *)&rbuf[i], H5P_DEFAULT, H5P_DEFAULT);
858  if(obj_id < 0)
859  throw InternalErr(__FILE__, __LINE__, "Cannot open the object the reference points to");
860 
861  vector<char> objname;
862  ssize_t objnamelen = -1;
863  if ((objnamelen= H5Iget_name(obj_id,nullptr,0))<=0) {
864  H5Oclose(obj_id);
865  throw InternalErr(__FILE__, __LINE__, "Cannot obtain the name length of the object the reference points to");
866  }
867  objname.resize(objnamelen+1);
868  if ((objnamelen= H5Iget_name(obj_id,objname.data(),objnamelen+1))<=0) {
869  H5Oclose(obj_id);
870  throw InternalErr(__FILE__, __LINE__, "Cannot obtain the name length of the object the reference points to");
871  }
872 
873  string objname_str = string(objname.begin(),objname.end());
874  string trim_objname = objname_str.substr(0,objnamelen);
875 
876  // For object references, we just need to save the object full path.
877  if(ref_type == H5R_OBJECT2)
878  v_str.push_back(trim_objname);
879  else {// Must be region reference.
880  H5O_type_t obj_type;
881  if(H5Rget_obj_type3((H5R_ref_t *)&rbuf[i], H5P_DEFAULT, &obj_type) < 0){
882  H5Oclose(obj_id);
883  throw InternalErr(__FILE__, __LINE__, "H5Rget_obj_type3() failed.");
884  }
885  if(obj_type != H5O_TYPE_DATASET) {
886  H5Oclose(obj_id);
887  throw InternalErr(__FILE__, __LINE__, "Region reference must point to a dataset.");
888  }
889  hid_t region_space_id = H5Ropen_region(&rbuf[i],H5P_DEFAULT,H5P_DEFAULT);
890  if (region_space_id < 0) {
891  H5Oclose(obj_id);
892  throw InternalErr(__FILE__, __LINE__, "Cannot obtain the space ID the reference points to");
893  }
894 
895  int ndim = H5Sget_simple_extent_ndims(region_space_id);
896  if (ndim < 0) {
897  H5Sclose(region_space_id);
898  H5Oclose(obj_id);
899  throw InternalErr(__FILE__, __LINE__, "H5Sget_simple_extent_ndims() failed.");
900  }
901 
902  string expression;
903  switch (H5Sget_select_type(region_space_id)) {
904 
905  case H5S_SEL_NONE:
906  BESDEBUG("h5", "=read() None selected." << endl);
907  break;
908 
909  case H5S_SEL_POINTS: {
910  BESDEBUG("h5", "=read() Points selected." << endl);
911  hssize_t npoints = H5Sget_select_npoints(region_space_id);
912  if (npoints < 0) {
913  H5Sclose(region_space_id);
914  H5Oclose(obj_id);
915  throw InternalErr(__FILE__, __LINE__,
916  "Cannot determine number of elements in the dataspace selection");
917  }
918 
919  BESDEBUG("h5", "=read() npoints are " << npoints
920  << endl);
921  vector<hsize_t> buf(npoints * ndim);
922  if (H5Sget_select_elem_pointlist(region_space_id, 0, npoints, buf.data()) < 0) {
923  H5Sclose(region_space_id);
924  H5Oclose(obj_id);
925  throw InternalErr(__FILE__, __LINE__, "H5Sget_select_elem_pointlist() failed.");
926  }
927 
928 #if 0
929  for (int j = 0; j < npoints * ndim; j++) {
930  "h5", "=read() npoints buf[0] =" << buf[j] <<endl;
931  }
932 #endif
933 
934  for (int j = 0; j < (int) npoints; j++) {
935  // Name of the dataset.
936  expression.append(trim_objname);
937  for (int k = 0; k < ndim; k++) {
938  ostringstream oss;
939  oss << "[" << (int) buf[j * ndim + k] << "]";
940  expression.append(oss.str());
941  }
942  if (j != (int) (npoints - 1)) {
943  expression.append(",");
944  }
945  }
946  v_str.push_back(expression);
947 
948  break;
949  }
950  case H5S_SEL_HYPERSLABS: {
951  vector<hsize_t> start(ndim);
952  vector<hsize_t> end(ndim);
953  vector<hsize_t>stride(ndim);
954  vector<hsize_t>s_count(ndim);
955  vector<hsize_t>block(ndim);
956 
957  BESDEBUG("h5", "=read() Slabs selected." << endl);
958  BESDEBUG("h5", "=read() nblock is " <<
959  H5Sget_select_hyper_nblocks(region_space_id) << endl);
960 
961  if (H5Sget_regular_hyperslab(region_space_id, start.data(), stride.data(), s_count.data(), block.data()) < 0) {
962  H5Sclose(region_space_id);
963  H5Oclose(obj_id);
964  throw InternalErr(__FILE__, __LINE__, "H5Sget_regular_hyperslab() failed.");
965  }
966 
967  expression.append(trim_objname);
968  for (int j = 0; j < ndim; j++) {
969  ostringstream oss;
970  BESDEBUG("h5", "start " << start[j]
971  << "stride "<<stride[j]
972  << "count "<< s_count[j]
973  << "block "<< block[j]
974  <<endl);
975 
976  // Map from HDF5's start,stride,count,block to DAP's start,stride,end.
977  end[j] = start[j] + stride[j]*(s_count[j]-1)+(block[j]-1);
978  BESDEBUG("h5", "=read() start is " << start[j]
979  << "=read() end is " << end[j] << endl);
980  oss << "[" << start[j] << ":" << stride[j] << ":" << end[j] << "]";
981  expression.append(oss.str());
982  BESDEBUG("h5", "=read() expression is "
983  << expression << endl)
984  ;
985  }
986  v_str.push_back(expression);
987  // Constraint expression. [start:stride:end]
988  break;
989  }
990  case H5S_SEL_ALL:
991  BESDEBUG("h5", "=read() All selected." << endl);
992  break;
993 
994  default:
995  BESDEBUG("h5", "Unknown space type." << endl);
996  break;
997  }
998  H5Sclose(region_space_id);
999  }
1000  H5Oclose(obj_id);
1001  }
1002  for (int i = 0; i<nelms; i++)
1003  H5Rdestroy(&rbuf[i]);
1004  delete[] rbuf;
1005  H5Sclose(mem_space_id);
1006  H5Sclose(file_space_id);
1007  set_value(v_str.data(), nelms);
1008  return false;
1009  }
1010  catch (...) {
1011  if(rbuf!= nullptr)
1012  delete[] rbuf;
1013  H5Sclose(mem_space_id);
1014  H5Sclose(file_space_id);
1015  throw;
1016  }
1017 #endif
1018 }
1019 
1020  // Unused code, keep it for a while.
1021  // Decide not check if the datatype is reference or not. This should be done by URL.
1022 #if 0
1023  bool is_new_ref = false;
1024  bool is_obj_ref = false;
1025  bool is_reg_ref = false;
1026 
1027  htri_t ref_ret = H5Tequal(dtype_id, H5T_STD_REF);
1028 
1029  // Check if this is a new reference(>= HDF5 1.12).
1030  if(ref_ret < 0)
1031  throw InternalErr(__FILE__, __LINE__, "H5Tequal() failed to compare H5T_STD_REF");
1032  else if (ref_ret > 0)
1033  is_new_ref = true;
1034 
1035  if(false == is_new_ref) {
1036 
1037  ref_ret = H5Tequal(dtype_id,H5T_STD_REF_OBJ);
1038  // Check if this is the old object reference(< HDF5 1.12).
1039  if(ref_ret < 0)
1040  throw InternalErr(__FILE__, __LINE__, "H5Tequal() failed to compare H5T_STD_REF_OBJ");
1041  else if (ref_ret > 0)
1042  is_obj_ref = true;
1043 
1044  if(ref_ret == 0) {
1045  // Check if this is the old region reference(< HDF5 1.12)
1046  ref_ret = H5Tequal(dtype_id,H5T_STD_REF_DSETREG);
1047  if(ref_ret < 0)
1048  throw InternalErr(__FILE__, __LINE__, "H5Tequal() failed to compare H5T_STD_REF_DSETREG");
1049  else if (ref_ret > 0)
1050  is_reg_ref = true;
1051  }
1052  }
1053  else {
1054  H5R_ref_t ref_type;
1055  ......
1056  }
1057  if(is_obj_ref == false && is_reg_ref == false)
1058  throw InternalErr(__FILE__, __LINE__, "datatype must be either object ref. or region ref.");
1059 #endif
1060 
1061 
1062 void HDF5Array::m_intern_plain_array_data(char *convbuf,hid_t memtype)
1063 {
1064  if (check_h5str(memtype)) {
1065  vector<string> v_str(d_num_elm);
1066  size_t elesize = H5Tget_size(memtype);
1067  if (elesize == 0) {
1068  throw InternalErr(__FILE__, __LINE__, "H5Tget_size() failed.");
1069  }
1070  vector<char> strbuf(elesize + 1);
1071  BESDEBUG("h5", "=read()<check_h5str() element size=" << elesize
1072  << " d_num_elm=" << d_num_elm << endl);
1073 
1074  for (int strindex = 0; strindex < d_num_elm; strindex++) {
1075  get_strdata(strindex, convbuf, strbuf.data(), (int)elesize);
1076  BESDEBUG("h5", "=read()<get_strdata() strbuf=" << strbuf.data() << endl);
1077  v_str[strindex] = strbuf.data();
1078  }
1079  set_read_p(true);
1080  val2buf((void *) v_str.data());
1081  }
1082  else {
1083  set_read_p(true);
1084  val2buf((void *) convbuf);
1085  }
1086 }
1087 
1088 
1089 bool HDF5Array::do_h5_array_type_read(hid_t dsetid, hid_t memb_id,vector<char>&values,bool has_values,int values_offset,
1090  int at_nelms,int* at_offset,int* at_count, int* at_step){
1091  //1. Call do array first(datatype must be derived) and the value must be set. We don't support Array datatype
1092  // unless it is inside a compound datatype
1093  if(has_values != true)
1094  throw InternalErr (__FILE__, __LINE__, "Only support the retrieval of HDF5 Array datatype values from the parent compound datatype read.");
1095 
1096  hid_t at_base_type = H5Tget_super(memb_id);
1097  if(at_base_type < 0) {
1098  throw InternalErr (__FILE__, __LINE__, "Fail to obtain the basetype of the array datatype.");
1099  }
1100 
1101  // memb_id, obtain the number of dimensions
1102  int at_ndims = H5Tget_array_ndims(memb_id);
1103  if(at_ndims <= 0) {
1104  H5Tclose(at_base_type);
1105  throw InternalErr (__FILE__, __LINE__, "Fail to obtain number of dimensions of the array datatype.");
1106  }
1107 
1108  vector<hsize_t>at_dims_h(at_ndims,0);
1109 
1110  // Obtain the number of elements for each dims
1111  if(H5Tget_array_dims(memb_id,at_dims_h.data())<0) {
1112  H5Tclose(at_base_type);
1113  throw InternalErr (__FILE__, __LINE__, "Fail to obtain dimensions of the array datatype.");
1114  }
1115  vector<int>at_dims(at_ndims,0);
1116  for(int i = 0;i<at_ndims;i++) {
1117  at_dims[i] = (int)at_dims_h[i];
1118  }
1119  int at_total_nelms = 1;
1120  for (int i = 0; i <at_ndims; i++)
1121  at_total_nelms = at_total_nelms*at_dims[i];
1122 
1123  H5T_class_t array_cls = H5Tget_class(at_base_type);
1124  if(H5T_NO_CLASS == array_cls) {
1125  H5Tclose(at_base_type);
1126  throw InternalErr (__FILE__, __LINE__, "Fail to obtain the datatype class of the array base type.");
1127  }
1128 
1129  size_t at_base_type_size = H5Tget_size(at_base_type);
1130  if(0 == at_base_type_size){
1131  H5Tclose(at_base_type);
1132  throw InternalErr (__FILE__, __LINE__, "Fail to obtain the size of the array base type.");
1133  }
1134 
1135  // H5 Array type, the basetype is COMPOUND.
1136  if(H5T_COMPOUND == array_cls) {
1137 
1138  // These vectors are used to handle subset of array datatype
1139  vector<int>at_end(at_ndims,0);
1140  vector<int>at_pos(at_ndims,0);
1141  for (int i = 0; i< at_ndims; i++){
1142  at_pos[i] = at_offset[i];
1143  at_end[i] = at_offset[i] + (at_count[i] -1)*at_step[i];
1144  }
1145 
1146  int at_orig_index = INDEX_nD_TO_1D(at_dims,at_pos);
1147 
1148  // To read the array of compound (structure) in DAP, one must read one element each. set_vec is used afterwards.
1149  for (int array_index = 0; array_index <at_nelms; array_index++) {
1150 
1151  // The basetype of the array datatype is compound,-- check if the following line is valid.
1152  HDF5Structure *h5s = dynamic_cast<HDF5Structure*>(var()->ptr_duplicate());
1153  hid_t child_memb_id;
1154  H5T_class_t child_memb_cls;
1155  int child_nmembs;
1156  size_t child_memb_offset;
1157 
1158  if((child_nmembs = H5Tget_nmembers(at_base_type)) < 0) {
1159  H5Tclose(at_base_type);
1160  delete h5s;
1161  throw InternalErr (__FILE__, __LINE__, "Fail to obtain number of HDF5 compound datatype.");
1162  }
1163 
1164  for(unsigned child_u = 0; child_u < (unsigned)child_nmembs; child_u++) {
1165 
1166  // Get member type ID
1167  if((child_memb_id = H5Tget_member_type(at_base_type, child_u)) < 0) {
1168  H5Tclose(at_base_type);
1169  delete h5s;
1170  throw InternalErr (__FILE__, __LINE__, "Fail to obtain the datatype of an HDF5 compound datatype member.");
1171  }
1172 
1173  // Get member type class
1174  if((child_memb_cls = H5Tget_member_class (at_base_type, child_u)) < 0) {
1175  H5Tclose(child_memb_id);
1176  H5Tclose(at_base_type);
1177  delete h5s;
1178  throw InternalErr (__FILE__, __LINE__, "Fail to obtain the datatype class of an HDF5 compound datatype member.");
1179  }
1180 
1181  // Get member offset
1182  child_memb_offset= H5Tget_member_offset(at_base_type,child_u);
1183 
1184  // Get member name
1185  char *child_memb_name = H5Tget_member_name(at_base_type,child_u);
1186  if(child_memb_name == nullptr) {
1187  H5Tclose(child_memb_id);
1188  H5Tclose(at_base_type);
1189  delete h5s;
1190  throw InternalErr (__FILE__, __LINE__, "Fail to obtain the name of an HDF5 compound datatype member.");
1191  }
1192 
1193  BaseType *field = h5s->var(child_memb_name);
1194  if (child_memb_cls == H5T_COMPOUND) {
1195 
1196  HDF5Structure &memb_h5s = dynamic_cast<HDF5Structure&>(*field);
1197  //
1198  // Call structure read when reading the whole array. sa1{sa2[100]}
1199  // sa2[100] is an array datatype.
1200  // If reading the whole buffer, just add the buffer.
1201  if(at_total_nelms == at_nelms) {
1202  memb_h5s.do_structure_read(dsetid,child_memb_id, values,has_values,values_offset+at_base_type_size*array_index+child_memb_offset);
1203  }
1204  // Subset of sa2, sa2[10:100:2]; sa2[100] is an array datatype. The whole array sa2[100] is to be read into somewhere in buffer values.
1205  else {// The subset should be considered. adjust memb_offset+values_offset+???,make sure only the subset is selected.
1206  // at_total_nelms is 100 but at_nelms is (100-10)/2+1=46. The starting point of the whole array is values+memb_offset_values_offset
1207  // When the datatype is structure, we have to obtain the index one by one.
1208 
1209  memb_h5s.do_structure_read(dsetid, child_memb_id, values,has_values,values_offset+at_base_type_size*at_orig_index+child_memb_offset);
1210 
1211  }
1212 
1213  }
1214  else if(child_memb_cls == H5T_ARRAY) {
1215 
1216  // memb_id, obtain the number of dimensions
1217  int child_at_ndims = H5Tget_array_ndims(child_memb_id);
1218  if(child_at_ndims <= 0) {
1219  H5Tclose(at_base_type);
1220  H5Tclose(child_memb_id);
1221  H5free_memory(child_memb_name);
1222  delete h5s;
1223  throw InternalErr (__FILE__, __LINE__, "Fail to obtain number of dimensions of the array datatype.");
1224 
1225  }
1226 
1227  HDF5Array &h5_array_type = dynamic_cast<HDF5Array&>(*field);
1228  vector<int> child_at_offset(child_at_ndims,0);
1229  vector<int> child_at_count(child_at_ndims,0);
1230  vector<int> child_at_step(child_at_ndims,0);
1231 
1232  int child_at_nelms = h5_array_type.format_constraint(child_at_offset.data(),child_at_step.data(),child_at_count.data());
1233 
1234  if(at_total_nelms == at_nelms) {
1235  h5_array_type.do_h5_array_type_read(dsetid,child_memb_id,values,has_values,child_memb_offset+values_offset+at_base_type_size*array_index,
1236  child_at_nelms,child_at_offset.data(),child_at_count.data(),child_at_step.data());
1237  }
1238  else {// Adjust memb_offset+values_offset, basically change at_base_type_size*array_index
1239  h5_array_type.do_h5_array_type_read(dsetid,child_memb_id,values,has_values,child_memb_offset+values_offset+at_base_type_size*at_orig_index,
1240  child_at_nelms,child_at_offset.data(),child_at_count.data(),child_at_step.data());
1241 
1242  }
1243  }
1244 
1245  else if(H5T_INTEGER == child_memb_cls || H5T_FLOAT == child_memb_cls){
1246 
1247  int number_index =((at_total_nelms == at_nelms)?array_index:at_orig_index);
1248  if(true == promote_char_to_short(child_memb_cls,child_memb_id)) {
1249  void *src = (void*)(values.data() + (number_index*at_base_type_size) + values_offset +child_memb_offset);
1250  char val_int8;
1251  memcpy(&val_int8,src,1);
1252  short val_short=(short)val_int8;
1253  field->val2buf(&val_short);
1254  }
1255  else
1256  field->val2buf(values.data() + (number_index * at_base_type_size) + values_offset+child_memb_offset);
1257 
1258 
1259  }
1260  else if(H5T_STRING == child_memb_cls){
1261 
1262  int string_index =((at_total_nelms == at_nelms)?array_index:at_orig_index);
1263 
1264  // distinguish between variable length and fixed length
1265  if(true == H5Tis_variable_str(child_memb_id)) {
1266 
1267  // Need to check if the size of variable length array type is right in HDF5 lib.
1268  void *src = (void*)(values.data()+(string_index *at_base_type_size)+values_offset+child_memb_offset);
1269  string final_str;
1270  char*temp_bp =(char*)src;
1271  get_vlen_str_data(temp_bp,final_str);
1272  field->val2buf(&final_str[0]);
1273 #if 0
1274  field->set_value(final_str);
1275 #endif
1276  }
1277  else {// Obtain string
1278  void *src = (void*)(values.data()+(string_index *at_base_type_size)+values_offset+child_memb_offset);
1279  vector<char> str_val;
1280  size_t memb_size = H5Tget_size(child_memb_id);
1281  if (memb_size == 0) {
1282  H5Tclose(child_memb_id);
1283  H5Tclose(at_base_type);
1284  H5free_memory(child_memb_name);
1285  delete h5s;
1286  throw InternalErr (__FILE__, __LINE__,"Fail to obtain the size of HDF5 compound datatype.");
1287  }
1288  str_val.resize(memb_size);
1289  memcpy(str_val.data(),src,memb_size);
1290  field->val2buf(str_val.data());
1291 
1292  }
1293  }
1294  else {
1295  H5Tclose(child_memb_id);
1296  H5Tclose(at_base_type);
1297  H5free_memory(child_memb_name);
1298  delete h5s;
1299  throw InternalErr (__FILE__, __LINE__, "Unsupported datatype class for the array base type.");
1300 
1301 
1302  }
1303  field->set_read_p(true);
1304  H5free_memory(child_memb_name);
1305  H5Tclose(child_memb_id);
1306 
1307  } // end "for ( child_u = 0)"
1308  h5s->set_read_p(true);
1309 
1310  // Save the value of this element to DAP structure.
1311  set_vec(array_index,h5s);
1312  delete h5s;
1313 
1314  vector<int>at_offsetv(at_pos.size(),0);
1315  vector<int>at_stepv(at_pos.size(),0);
1316  for (unsigned int at_index = 0; at_index<at_pos.size();at_index++){
1317  at_offsetv[at_index] = at_offset[at_index];
1318  at_stepv[at_index] = at_step[at_index];
1319  }
1320  //obtain the next position of the selected point based on the offset,end and step.
1321  obtain_next_pos(at_pos,at_offsetv,at_end,at_stepv,(int)(at_pos.size()));
1322  at_orig_index = INDEX_nD_TO_1D(at_dims,at_pos);
1323  }// end for "(array_index = 0) for array (compound)datatype"
1324 
1325  // Need to check the usage of set_read_p(true);
1326 #if 0
1327  //set_read_p(true);
1328 #endif
1329  }
1330  else if(H5T_INTEGER == array_cls|| H5T_FLOAT == array_cls) {
1331 
1332  // If no subset for the array datatype, just read the whole buffer.
1333  if(at_total_nelms == at_nelms) {
1334 
1335  // For DAP2 char should be mapped to short
1336  if( true == promote_char_to_short(array_cls ,at_base_type)) {
1337  vector<char> val_int8;
1338  val_int8.resize(at_nelms);
1339  void*src = (void*)(values.data() +values_offset);
1340  memcpy(val_int8.data(),src,at_nelms);
1341 
1342  vector<short> val_short;
1343  for (int i = 0; i<at_nelms; i++)
1344  val_short[i] = (short)val_int8[i];
1345 
1346  val2buf(val_short.data());
1347 
1348  }
1349  else // short cut for others
1350  val2buf(values.data() + values_offset);
1351 
1352  }
1353  else { // Adjust the value for the subset of the array datatype
1354 
1355  // Obtain the correponding DAP type of the HDF5 data type
1356  string dap_type = get_dap_type(at_base_type,is_dap4());
1357 
1358  // The total array type data is read.
1359  void*src = (void*)(values.data() + values_offset);
1360 
1361  // set the original position to the starting point
1362  vector<int>at_pos(at_ndims,0);
1363  for (int i = 0; i< at_ndims; i++)
1364  at_pos[i] = at_offset[i];
1365 
1366  if( BYTE == dap_type) {
1367 
1368  vector<unsigned char>total_val;
1369  total_val.resize(at_total_nelms);
1370  memcpy(total_val.data(),src,at_total_nelms*at_base_type_size);
1371 
1372  vector<unsigned char>final_val;
1373  subset<unsigned char>(
1374  total_val.data(),
1375  at_ndims,
1376  at_dims,
1377  at_offset,
1378  at_step,
1379  at_count,
1380  &final_val,
1381  at_pos,
1382  0
1383  );
1384 
1385  set_value(final_val.data(),at_nelms);
1386 
1387 
1388  }
1389  else if( INT16 == dap_type) {
1390 
1391  // promote char to short,DAP2 doesn't have "char" type
1392  if(true == promote_char_to_short(array_cls,at_base_type)) {
1393  vector<char>total_val;
1394  total_val.resize(at_total_nelms);
1395  memcpy(total_val.data(),src,at_total_nelms*at_base_type_size);
1396 
1397  vector<char>final_val;
1398  subset<char>(
1399  total_val.data(),
1400  at_ndims,
1401  at_dims,
1402  at_offset,
1403  at_step,
1404  at_count,
1405  &final_val,
1406  at_pos,
1407  0
1408  );
1409 
1410  vector<short> final_val_short;
1411  final_val_short.resize(at_nelms);
1412  for(int i = 0; i<at_nelms; i++)
1413  final_val_short[i] = final_val[i];
1414 
1415  val2buf(final_val_short.data());
1416 
1417  }
1418  else {// short
1419 
1420  vector<short>total_val;
1421  total_val.resize(at_total_nelms);
1422  memcpy(total_val.data(),src,at_total_nelms*at_base_type_size);
1423 
1424  vector<short>final_val;
1425  subset<short>(
1426  total_val.data(),
1427  at_ndims,
1428  at_dims,
1429  at_offset,
1430  at_step,
1431  at_count,
1432  &final_val,
1433  at_pos,
1434  0
1435  );
1436 
1437  val2buf(final_val.data());
1438 
1439  }
1440  }
1441  else if( UINT16 == dap_type) {
1442  vector<unsigned short>total_val;
1443  total_val.resize(at_total_nelms);
1444  memcpy(total_val.data(),src,at_total_nelms*at_base_type_size);
1445 
1446  vector<unsigned short>final_val;
1447  subset<unsigned short>(
1448  total_val.data(),
1449  at_ndims,
1450  at_dims,
1451  at_offset,
1452  at_step,
1453  at_count,
1454  &final_val,
1455  at_pos,
1456  0
1457  );
1458 
1459  val2buf(final_val.data());
1460 
1461  }
1462  else if(UINT32 == dap_type) {
1463  vector<unsigned int>total_val;
1464  total_val.resize(at_total_nelms);
1465  memcpy(total_val.data(),src,at_total_nelms*at_base_type_size);
1466 
1467  vector<unsigned int>final_val;
1468  subset<unsigned int>(
1469  total_val.data(),
1470  at_ndims,
1471  at_dims,
1472  at_offset,
1473  at_step,
1474  at_count,
1475  &final_val,
1476  at_pos,
1477  0
1478  );
1479  val2buf(final_val.data());
1480 
1481 
1482  }
1483  else if(INT32 == dap_type) {
1484  vector<int>total_val;
1485  total_val.resize(at_total_nelms);
1486  memcpy(total_val.data(),src,at_total_nelms*at_base_type_size);
1487 
1488  vector<int>final_val;
1489  subset<int>(
1490  total_val.data(),
1491  at_ndims,
1492  at_dims,
1493  at_offset,
1494  at_step,
1495  at_count,
1496  &final_val,
1497  at_pos,
1498  0
1499  );
1500 
1501  val2buf(final_val.data());
1502 
1503  }
1504  else if(FLOAT32 == dap_type) {
1505  vector<float>total_val;
1506  total_val.resize(at_total_nelms);
1507  memcpy(total_val.data(),src,at_total_nelms*at_base_type_size);
1508 
1509  vector<float>final_val;
1510  subset<float>(
1511  total_val.data(),
1512  at_ndims,
1513  at_dims,
1514  at_offset,
1515  at_step,
1516  at_count,
1517  &final_val,
1518  at_pos,
1519  0
1520  );
1521 
1522  val2buf(final_val.data());
1523 
1524  }
1525  else if(FLOAT64 == dap_type) {
1526  vector<double>total_val;
1527  total_val.resize(at_total_nelms);
1528  memcpy(total_val.data(),src,at_total_nelms*at_base_type_size);
1529 
1530  vector<double>final_val;
1531  subset<double>(
1532  total_val.data(),
1533  at_ndims,
1534  at_dims,
1535  at_offset,
1536  at_step,
1537  at_count,
1538  &final_val,
1539  at_pos,
1540  0
1541  );
1542 #if 0
1543  //field->val2buf(final_val.data());
1544 #endif
1545  val2buf(final_val.data());
1546 
1547  }
1548  else {
1549  H5Tclose(at_base_type);
1550  throw InternalErr (__FILE__, __LINE__,
1551  "Non-supported integer or float datatypes");
1552  }
1553 
1554  }
1555  }
1556  else if(H5T_STRING == array_cls) {
1557 
1558  // set the original position to the starting point
1559  vector<int>at_pos(at_ndims,0);
1560  for (int i = 0; i< at_ndims; i++)
1561  at_pos[i] = at_offset[i];
1562 
1563  vector<string>total_strval;
1564  total_strval.resize(at_total_nelms);
1565 
1566  if(true == H5Tis_variable_str(at_base_type)) {
1567  void *src = (void*)(values.data()+values_offset);
1568  char*temp_bp =(char*)src;
1569  for(int i = 0;i <at_total_nelms; i++){
1570  string tempstrval;
1571  get_vlen_str_data(temp_bp,tempstrval);
1572  total_strval[i] = tempstrval;
1573  temp_bp += at_base_type_size;
1574  }
1575  if(at_total_nelms == at_nelms) {
1576 #if 0
1577  //field->set_value(total_strval,at_total_nelms);
1578 #endif
1579  set_value(total_strval,at_total_nelms);
1580  }
1581  else {// obtain subset for variable-length string.
1582 #if 0
1583  //field->val2buf(values.data() + values_offset);
1584 #endif
1585  vector<string>final_val;
1586  subset<string>(
1587  total_strval.data(),
1588  at_ndims,
1589  at_dims,
1590  at_offset,
1591  at_step,
1592  at_count,
1593  &final_val,
1594  at_pos,
1595  0
1596  );
1597 
1598  set_value(final_val,at_nelms);
1599 
1600  }
1601 
1602  }
1603  else {// For fixed-size string.
1604  void *src = (void*)(values.data()+values_offset);
1605  for(int i = 0; i <at_total_nelms; i++)
1606  total_strval[i].resize(at_base_type_size);
1607 
1608  vector<char> str_val;
1609  str_val.resize(at_total_nelms*at_base_type_size);
1610  memcpy((void*)str_val.data(),src,at_total_nelms*at_base_type_size);
1611  string total_in_one_string(str_val.begin(),str_val.end());
1612  for(int i = 0; i<at_total_nelms;i++)
1613  total_strval[i] = total_in_one_string.substr(i*at_base_type_size,at_base_type_size);
1614 
1615  if(at_total_nelms == at_nelms)
1616  set_value(total_strval,at_total_nelms);
1617  else {
1618  vector<string>final_val;
1619  subset<string>(
1620  total_strval.data(),
1621  at_ndims,
1622  at_dims,
1623  at_offset,
1624  at_step,
1625  at_count,
1626  &final_val,
1627  at_pos,
1628  0
1629  );
1630  set_value(final_val,at_nelms);
1631 
1632  }
1633  }
1634 
1635  }
1636  else {
1637  H5Tclose(at_base_type);
1638  throw InternalErr (__FILE__, __LINE__,
1639  "Only support the field of compound datatype when the field type class is integer, float, string, array or compound..");
1640 
1641  }
1642 
1643  H5Tclose(at_base_type);
1644 
1645  return true;
1646 }
1647 
1649 inline int
1650 HDF5Array::INDEX_nD_TO_1D (const std::vector < int > &dims,
1651  const std::vector < int > &pos)
1652 {
1653  //
1654  // "int a[10][20][30] // & a[1][2][3] == a + (20*30+1 + 30*2 + 1 *3)"
1655  // "int b[10][2]; // &b[1][2] == b + (20*1 + 2);"
1656  //
1657  assert (dims.size () == pos.size ());
1658  int sum = 0;
1659  int start = 1;
1660 
1661  for (unsigned int p = 0; p < pos.size (); p++) {
1662  int m = 1;
1663 
1664  for (unsigned int j = start; j < dims.size (); j++)
1665  m *= dims[j];
1666  sum += m * pos[p];
1667  start++;
1668  }
1669  return sum;
1670 }
1671 
1672 // Obtain the dimension index of the next pos. of the point based on the offset, step and end
1673 bool HDF5Array::obtain_next_pos(vector<int>& pos, vector<int>&start,vector<int>&end,vector<int>&step,int rank_change) {
1674 
1675  if((pos[rank_change-1] + step[rank_change-1])<=end[rank_change-1]) {
1676  pos[rank_change-1] = pos[rank_change-1] + step[rank_change-1];
1677  return true;
1678  }
1679  else {
1680  if( 1 == rank_change)
1681  return false;
1682  pos[rank_change-1] = start[rank_change-1];
1683  obtain_next_pos(pos,start,end,step,rank_change-1);
1684  }
1685  return true;
1686 }
1687 
1689 //
1690 // \param input Input variable
1691 // \param dim dimension info of the input
1692 // \param start start indexes of each dim
1693 // \param stride stride of each dim
1694 // \param edge count of each dim
1695 // \param poutput output variable
1696 // \parrm index dimension index
1697 // \return 0 if successful. -1 otherwise.
1698 //
1699 template<typename T>
1700 int HDF5Array::subset(
1701  const T input[],
1702  int rank,
1703  vector<int> & dim,
1704  int start[],
1705  int stride[],
1706  int edge[],
1707  std::vector<T> *poutput,
1708  vector<int>& pos,
1709  int index)
1710 {
1711  for(int k=0; k<edge[index]; k++)
1712  {
1713  pos[index] = start[index] + k*stride[index];
1714  if(index+1<rank)
1715  subset(input, rank, dim, start, stride, edge, poutput,pos,index+1);
1716  if(index==rank-1)
1717  {
1718  poutput->push_back(input[INDEX_nD_TO_1D( dim, pos)]);
1719  }
1720  } // end of for
1721  return 0;
1722 } // end of template<typename T> static int subset
1723 
1724 
1725 // public functions to set all parameters needed in read function.
1726 
1727 
1728 void HDF5Array::set_memneed(size_t need) {
1729  d_memneed = need;
1730 }
1731 
1732 void HDF5Array::set_numdim(int ndims) {
1733  d_num_dim = ndims;
1734 }
1735 
1736 void HDF5Array::set_numelm(int nelms) {
1737  d_num_elm = nelms;
1738 }
1739 
1740 // TODO: This routine is not used anymore. May delete it in the future.
1741 #if 0
1742 hid_t HDF5Array::mkstr(int size, H5T_str_t pad)
1743 {
1744 
1745  hid_t str_type;
1746 
1747  if ((str_type = H5Tcopy(H5T_C_S1)) < 0)
1748  return -1;
1749  if (H5Tset_size(str_type, (size_t) size) < 0)
1750  return -1;
1751  if (H5Tset_strpad(str_type, pad) < 0)
1752  return -1;
1753 
1754  return str_type;
1755 }
1756 #endif
1757 
1758 // We don't inherit libdap Array Class's transform_to_dap4 method since CF option is still using it.
1759 BaseType* HDF5Array::h5dims_transform_to_dap4(D4Group *grp,const vector<string> &dimpath) {
1760 
1761  BESDEBUG("h5", "<h5dims_transform_to_dap4" << endl);
1762 
1763  if(grp == nullptr)
1764  return nullptr;
1765 
1766  Array *dest = static_cast<HDF5Array*>(ptr_duplicate());
1767 
1768  // If there is just a size, don't make
1769  // a D4Dimension (In DAP4 you cannot share a dimension unless it has
1770  // a name). jhrg 3/18/14
1771 
1772  int k = 0;
1773  for (Array::Dim_iter d = dest->dim_begin(), e = dest->dim_end(); d != e; ++d) {
1774 
1775  if (false == (*d).name.empty()) {
1776  BESDEBUG("h5", "<coming to the dimension loop, has name " << (*d).name<<endl);
1777  BESDEBUG("h5", "<coming to the dimension loop, has dimpath " << dimpath[k] <<endl);
1778  BESDEBUG("h5", "<coming to the dimension loop, has dimpath group " << dimpath[k].substr(0,dimpath[k].find_last_of("/")+1) <<endl);
1779 
1780  D4Group *temp_grp = grp;
1781  D4Dimension *d4_dim = nullptr;
1782  bool is_dim_nonc4_grp = false;
1783 
1784  while(temp_grp) {
1785 
1786  BESDEBUG("h5", "<coming to the group has name " << temp_grp->name()<<endl);
1787  BESDEBUG("h5", "<coming to the group has fullpath " << temp_grp->FQN()<<endl);
1788 
1789  //Obtain all the dimensions of this group.
1790  D4Dimensions *temp_dims = temp_grp->dims();
1791 
1792  // Check if this dimension is defined in this group
1793  d4_dim = temp_dims->find_dim((*d).name);
1794 
1795  // Need the full path of the dimension name
1796  string d4_dim_path = dimpath[k].substr(0,dimpath[k].find_last_of("/")+1);
1797  BESDEBUG("h5", "d4_dim_path is " << d4_dim_path<<endl);
1798 
1799  bool ancestor_grp = false;
1800 
1801  // If the dim_path is within this group or its ancestor, this is valid.
1802  if(d4_dim_path.find(temp_grp->FQN())==0 || temp_grp->FQN().find(d4_dim_path)==0)
1803  ancestor_grp = true;
1804 
1805  // If we find this dimension and the dimension is on the ancestral path,
1806  // this follows the netCDF-4/DAP4 dimension model, break.
1807  if(d4_dim && (temp_grp->FQN() == d4_dim_path)) {
1808  BESDEBUG("h5", "<FInd dimension name " << (*d).name<<endl);
1809  (*d).dim = d4_dim;
1810  is_dim_nonc4_grp = false;
1811  break;
1812  }
1813  // If the dimension name is not on the ancestral path, this
1814  // dimension must be on another path, mark it.
1815 #if 0
1816  //else if( ancestor_grp == false && is_dim_nonc4_grp == false) {
1817 #endif
1818  else if( ancestor_grp == false) {
1819  is_dim_nonc4_grp = true;
1820  break;
1821  }
1822  else
1823  d4_dim = nullptr;
1824 
1825  if(temp_grp->get_parent())
1826  temp_grp = static_cast<D4Group*>(temp_grp->get_parent());
1827  else
1828  temp_grp = nullptr;
1829 
1830  }
1831 
1832  // Not find this dimension in any of the ancestor groups, add it to this group.
1833  // The following block is fine, but to avoid the complaint from sonarcloud.
1834  // Use a bool.
1835  if(true == is_dim_nonc4_grp) {
1836  string err= "The variable " + var_path +" has dimension ";
1837  err += dimpath[k] + ". This dimension is not under its ancestor or the current group.";
1838  err += " This is not supported.";
1839  delete dest;
1840  throw InternalErr(__FILE__,__LINE__,err);
1841  }
1842 
1843  bool d4_dim_null = ((d4_dim==nullptr)?true:false);
1844  if(d4_dim_null == true) {
1845  d4_dim = new D4Dimension((*d).name, (*d).size);
1846  D4Dimensions * dims = grp->dims();
1847  BESDEBUG("h5", "<Just before adding D4 dimension to group" << endl);
1848  dims->add_dim_nocopy(d4_dim);
1849  (*d).dim = d4_dim;
1850  }
1851  }
1852  k++;
1853  }
1854 
1855  dest->set_is_dap4(true);
1856 
1857  return dest;
1858 
1859 }
HDF5Array.h
A class for handling all types of array in HDF5 for the default option.
HDF5Str.h
This class that translates HDF5 string into DAP string for the default option.
HDF5Structure.h
This class converts HDF5 compound type into DAP structure for the default option.
HDF5Array::ptr_duplicate
libdap::BaseType * ptr_duplicate() override
Definition: HDF5Array.cc:54
HDF5Array::read
bool read() override
Reads HDF5 array data into local buffer.
Definition: HDF5Array.cc:109
HDF5Array::set_numdim
void set_numdim(int ndims)
remembers number of dimensions of this array.
Definition: HDF5Array.cc:1732
HDF5Array::HDF5Array
HDF5Array(const std::string &n, const std::string &d, libdap::BaseType *v)
Constructor.
Definition: HDF5Array.cc:58
HDF5Array::set_numelm
void set_numelm(int nelms)
remembers number of elements in this array.
Definition: HDF5Array.cc:1736
HDF5Array::set_memneed
void set_memneed(size_t need)
remembers memory size needed.
Definition: HDF5Array.cc:1728
get_data
void get_data(hid_t dset, void *buf)
Definition: h5common.cc:50
get_strdata
void get_strdata(int strindex, char *allbuf, char *buf, int elesize)
Definition: h5common.cc:115
check_h5str
bool check_h5str(hid_t h5type)
Definition: h5get.cc:849
get_dap_type
string get_dap_type(hid_t type, bool is_dap4)
Definition: h5get.cc:292
DODS_NAMELEN
const int DODS_NAMELEN
Maximum length of variable or attribute name(default option only).
Definition: hdf5_handler.h:65