bes  Updated for version 3.20.13
HDF5RequestHandler.cc
Go to the documentation of this file.
1 
2 // -*- mode: c++; c-basic-offset:4 -*-
3 
4 // This file is part of hdf5_handler, a data handler for the OPeNDAP data
5 // server.
6 
7 // Copyright (c) 2002,2003 OPeNDAP, Inc.
8 // Copyright (c) 2007-2016 The HDF Group, Inc. and OPeNDAP, Inc.
9 // Author: James Gallagher <jgallagher@opendap.org>
10 //
11 // This is free software; you can redistribute it and/or modify it under the
12 // terms of the GNU Lesser General Public License as published by the Free
13 // Software Foundation; either version 2.1 of the License, or (at your
14 // option) any later version.
15 //
16 // This software is distributed in the hope that it will be useful, but
17 // WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
18 // or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
19 // License for more details.
20 //
21 // You should have received a copy of the GNU Lesser General Public
22 // License along with this library; if not, write to the Free Software
23 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
24 //
25 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
26 
27 // HDF5RequestHandler.cc
33 
34 
35 #include <iostream>
36 #include <fstream>
37 #include <sstream>
38 #include <string>
39 #include <sys/types.h>
40 #include <sys/stat.h>
41 #include <fcntl.h>
42 #include <unistd.h>
43 
44 #include <libdap/DMR.h>
45 #include <libdap/D4BaseTypeFactory.h>
46 #include <BESDMRResponse.h>
47 #include <ObjMemCache.h>
48 #include "HDF5_DMR.h"
49 
50 #include <libdap/mime_util.h>
51 #include "hdf5_handler.h"
52 #include "HDF5RequestHandler.h"
53 #include "HDF5_DDS.h"
54 
55 #include <BESDASResponse.h>
56 #include <BESDDSResponse.h>
57 #include <BESDataDDSResponse.h>
58 #include <libdap/Ancillary.h>
59 #include <BESInfo.h>
60 #include <BESDapNames.h>
61 #include <BESResponseNames.h>
62 #include <BESContainer.h>
63 #include <BESResponseHandler.h>
64 #include <BESVersionInfo.h>
65 #include <BESServiceRegistry.h>
66 #include <BESUtil.h>
67 #include <BESDapError.h>
68 #include <BESInternalFatalError.h>
69 #include <TheBESKeys.h>
70 #include <BESDebug.h>
71 #include <BESStopWatch.h>
72 #include "h5get.h"
73 #include "config_hdf5.h"
74 
75 #define HDF5_NAME "h5"
76 #include "h5cfdaputil.h"
77 
78 using namespace std;
79 using namespace libdap;
80 
81 // DYNAMIC_CONFIG_ENABLED is defined by TheBESKeys.h. As of 3/11/22, this
82 // is 0 so the dynamic keys feature is not used. jhrg 3/8/22
83 
84 #define prolog std::string("HDF5RequestHandler::").append(__func__).append("() - ")
85 
86 // The debug function to dump all the contents of a DAS table.
87 void get_attr_contents(AttrTable* temp_table);
88 
89 // Five functions to generate a DAS cache file.
90 // 1. The wrapper function to call write_das_table_to_file to generate the cache.
91 void write_das_to_file(DAS*das_ptr,FILE* das_file);
92 
93 // 2. The main function to generate the DAS cache.
94 void write_das_table_to_file(AttrTable*temp_table,FILE* das_file);
95 
96 // 3. The function to write the DAS container/table name to the cache
97 void write_container_name_to_file(const string&,FILE* das_file);
98 
99 // 4. The function to write the DAS attribute name,type and values to the cache
100 void write_das_attr_info(AttrTable* dtp,const string&,const string&,FILE * das_file);
101 
102 // 5. The function to copy a string to a memory buffer.
103 char* copy_str(char *temp_ptr,const string & str);
104 
105 // These two functions are for reading the DAS from the DAS cache file
106 // 1. Obtain the string from a memory buffer.
107 char* obtain_str(char*temp_ptr,string & str);
108 
109 // 2. The main function to obtain DAS info from the cache.
110 char* get_attr_info_from_dc(char*temp_pointer,DAS *das,AttrTable *at);
111 
112 #if 0
113 // Check if the BES key is set.
114 //bool check_and_set_beskeys(const string);
115 #endif
116 
117 // Obtain the BES key as an integer
118 static unsigned int get_uint_key(const string &key,unsigned int def_val);
119 static unsigned long get_ulong_key(const string &key,unsigned long def_val);
120 
121 // Obtain the BES key as a floating-pointer number.
122 static float get_float_key(const string &key, float def_val);
123 
124 // Obtain the BES key as a string.
125 static string get_beskeys(const string&);
126 
127 #if 0
128 //static bool is_beskeys_set_true(const string &);
129 #endif
130 
131 bool obtain_beskeys_info(const string &, bool &);
132 
133 // For the CF option
134 extern void read_cfdas(DAS &das, const string & filename,hid_t fileid);
135 extern void read_cfdds(DDS &dds, const string & filename,hid_t fileid);
136 extern void read_cfdmr(DMR *dmr, const string & filename,hid_t fileid);
137 
138 
139 
140 // Check the description of cache_entries and cache_purge_level at h5.conf.in.
141 unsigned int HDF5RequestHandler::_mdcache_entries = 500;
142 unsigned int HDF5RequestHandler::_lrdcache_entries = 0;
143 unsigned int HDF5RequestHandler::_srdcache_entries = 0;
144 float HDF5RequestHandler::_cache_purge_level = 0.2F;
145 
146 // Metadata object cache at DAS,DDS and DMR.
147 ObjMemCache *HDF5RequestHandler::das_cache = nullptr;
148 ObjMemCache *HDF5RequestHandler::dds_cache = nullptr;
149 ObjMemCache *HDF5RequestHandler::datadds_cache = nullptr;
150 ObjMemCache *HDF5RequestHandler::dmr_cache = nullptr;
151 
152 ObjMemCache *HDF5RequestHandler::lrdata_mem_cache = nullptr;
153 ObjMemCache *HDF5RequestHandler::srdata_mem_cache = nullptr;
154 
155 // Set default values of all BES keys according to h5.conf.in.
156 // This will help the generation of DMRPP. No need to
157 // set multiple keys if the user's setting is the same as
158 // the h5.conf.in. KY 2021-08-23
159 bool HDF5RequestHandler::_usecf = true;
160 bool HDF5RequestHandler::_pass_fileid = false;
161 bool HDF5RequestHandler::_disable_structmeta = true;
162 bool HDF5RequestHandler::_disable_ecsmeta = false;
163 bool HDF5RequestHandler::_keep_var_leading_underscore = false;
164 bool HDF5RequestHandler::_check_name_clashing = false;
165 bool HDF5RequestHandler::_add_path_attrs = true;
166 bool HDF5RequestHandler::_drop_long_string = true;
167 bool HDF5RequestHandler::_fillvalue_check = true;
168 bool HDF5RequestHandler::_check_ignore_obj = false;
169 bool HDF5RequestHandler::_flatten_coor_attr = true;
170 bool HDF5RequestHandler::_default_handle_dimension = true; //Ignored when _usecf=true.
171 bool HDF5RequestHandler::_eos5_rm_convention_attr_path = true;
172 bool HDF5RequestHandler::_dmr_long_int = true;
173 bool HDF5RequestHandler::_no_zero_size_fullnameattr = false;
174 bool HDF5RequestHandler::_enable_coord_attr_add_path = true;
175 
176 bool HDF5RequestHandler::_usecfdmr = true;
177 bool HDF5RequestHandler::_add_dap4_coverage = true;
178 
179 bool HDF5RequestHandler::_common_cache_dirs = false;
180 
181 bool HDF5RequestHandler::_use_disk_cache =false;
182 bool HDF5RequestHandler::_use_disk_dds_cache =false;
183 string HDF5RequestHandler::_disk_cache_dir ="";
184 string HDF5RequestHandler::_disk_cachefile_prefix ="";
185 unsigned long long HDF5RequestHandler::_disk_cache_size =0;
186 
187 
188 bool HDF5RequestHandler::_disk_cache_comp_data =false;
189 bool HDF5RequestHandler::_disk_cache_float_only_comp_data =false;
190 float HDF5RequestHandler::_disk_cache_comp_threshold =1.0;
191 unsigned long HDF5RequestHandler::_disk_cache_var_size =0;
192 
193 bool HDF5RequestHandler::_use_disk_meta_cache = false;
194 string HDF5RequestHandler::_disk_meta_cache_path ="";
195 
196 bool HDF5RequestHandler::_use_latlon_disk_cache = false;
197 long HDF5RequestHandler::_latlon_disk_cache_size =0;
198 string HDF5RequestHandler::_latlon_disk_cache_dir ="";
199 string HDF5RequestHandler::_latlon_disk_cachefile_prefix="";
200 
201 // H5.EscapeUTF8Attr jhrg 3/9/22
202 bool HDF5RequestHandler::_escape_utf8_attr = true;
203 
204 DMR* HDF5RequestHandler::dmr_int64 = nullptr;
205 
206 
207 #if 0
208 //BaseTypeFactory factory;
209 //libdap::DDS HDF5RequestHandler::hd_dds(&factory,"");
210 #endif
211 string HDF5RequestHandler::_stp_east_filename;
212 string HDF5RequestHandler::_stp_north_filename;
213 vector<string> HDF5RequestHandler::lrd_cache_dir_list;
214 vector<string> HDF5RequestHandler::lrd_non_cache_dir_list;
215 vector<string> HDF5RequestHandler::lrd_var_cache_file_list;
216 
217 #if 0
218 //libdap::DDS*cache_dds;
219 #endif
220 
221 HDF5RequestHandler::HDF5RequestHandler(const string & name)
222  :BESRequestHandler(name)
223 {
224 
225  BESDEBUG(HDF5_NAME, prolog << "BEGIN" << endl);
226 
227  add_handler(DAS_RESPONSE, HDF5RequestHandler::hdf5_build_das);
228  add_handler(DDS_RESPONSE, HDF5RequestHandler::hdf5_build_dds);
229  add_handler(DATA_RESPONSE, HDF5RequestHandler::hdf5_build_data);
230  add_handler(DMR_RESPONSE, HDF5RequestHandler::hdf5_build_dmr);
231  add_handler(DAP4DATA_RESPONSE, HDF5RequestHandler::hdf5_build_dmr);
232 
233  add_handler(HELP_RESPONSE, HDF5RequestHandler::hdf5_build_help);
234  add_handler(VERS_RESPONSE, HDF5RequestHandler::hdf5_build_version);
235 
236 #if !(DYNAMIC_CONFIG_ENABLED)
237  load_config();
238 #endif
239 
240  BESDEBUG(HDF5_NAME, prolog << "END" << endl);
241 }
242 
243 HDF5RequestHandler::~HDF5RequestHandler()
244 {
245  // delete the cache.
246  delete das_cache;
247  delete dds_cache;
248  delete datadds_cache;
249  delete dmr_cache;
250  delete lrdata_mem_cache;
251  delete srdata_mem_cache;
252 
253 }
254 
258 void HDF5RequestHandler::load_config()
259 {
260  BESDEBUG(HDF5_NAME, prolog << "BEGIN" << endl);
261  BESStopWatch sw;
262  if (BESISDEBUG(HDF5_NAME)){
263  sw.start(prolog,"ClockTheBESKeys");
264  }
265 
266  // Obtain the metadata cache entries and purge level.
267  HDF5RequestHandler::_mdcache_entries = get_uint_key("H5.MetaDataMemCacheEntries", 0);
268  HDF5RequestHandler::_lrdcache_entries = get_uint_key("H5.LargeDataMemCacheEntries", 0);
269  HDF5RequestHandler::_srdcache_entries = get_uint_key("H5.SmallDataMemCacheEntries", 0);
270  HDF5RequestHandler::_cache_purge_level = get_float_key("H5.CachePurgeLevel", 0.2F);
271 
272  if (get_mdcache_entries()) { // else it stays at its default of null
273  das_cache = new ObjMemCache(get_mdcache_entries(), get_cache_purge_level());
274  dds_cache = new ObjMemCache(get_mdcache_entries(), get_cache_purge_level());
275  datadds_cache = new ObjMemCache(get_mdcache_entries(), get_cache_purge_level());
276  dmr_cache = new ObjMemCache(get_mdcache_entries(), get_cache_purge_level());
277  }
278 
279  // Starting from hyrax 1.16.5, users don't need to explicitly set the BES keys if
280  // they are happy with the default settings in the h5.conf.in.
281  // In the previous releases, we required users to set BES key values. Otherwise,
282  // the BES key values of true/false will always be false if the key is not found.
283  // Hopefully this change will make it convenient for the general users.
284  // In the mean time, the explicit BES key settings will not be affected.
285  // KY 2021-10-13
286  //
287  // Check if the EnableCF key is set.
288  bool has_key = false;
289  bool key_value = obtain_beskeys_info("H5.EnableCF",has_key);
290  if(has_key)
291  _usecf = key_value;
292  BESDEBUG(HDF5_NAME, prolog << "H5.EnableCF: " << (_usecf?"true":"false") << endl);
293 
294  // The DefaultHandleDimension is effective only when EnableCF=false
295  key_value = obtain_beskeys_info("H5.DefaultHandleDimension",has_key);
296  if(has_key)
297  _default_handle_dimension = key_value;
298  BESDEBUG(HDF5_NAME, prolog << "H5.DefaultHandleDimension: " << (_default_handle_dimension?"true":"false") << endl);
299 
300  // The following keys are only effective when EnableCF is true or unset(EnableCF is true if users don't set the key).
301  key_value = obtain_beskeys_info("H5.EnablePassFileID",has_key);
302  if(has_key)
303  _pass_fileid = key_value;
304  BESDEBUG(HDF5_NAME, prolog << "H5.EnablePassFileID: " << (_pass_fileid?"true":"false") << endl);
305 
306  key_value = obtain_beskeys_info("H5.DisableStructMetaAttr",has_key);
307  if(has_key)
308  _disable_structmeta = key_value;
309  BESDEBUG(HDF5_NAME, prolog << "H5.DisableStructMetaAttr: " << (_disable_structmeta?"true":"false") << endl);
310 
311  key_value = obtain_beskeys_info("H5.DisableECSMetaAttr",has_key);
312  if(has_key)
313  _disable_ecsmeta = key_value;
314  BESDEBUG(HDF5_NAME, prolog << "H5.DisableECSMetaAttr: " << (_disable_ecsmeta?"true":"false") << endl);
315 
316  key_value = obtain_beskeys_info("H5.KeepVarLeadingUnderscore",has_key);
317  if(has_key)
318  _keep_var_leading_underscore = key_value;
319  BESDEBUG(HDF5_NAME, prolog << "H5.KeepVarLeadingUnderscore: " << (_keep_var_leading_underscore?"true":"false") << endl);
320 
321 #if 0
322  //_check_name_clashing = check_and_set_beskeys("H5.EnableCheckNameClashing");
323 #endif
324  key_value = obtain_beskeys_info("H5.EnableCheckNameClashing",has_key);
325  if(has_key)
326  _check_name_clashing = key_value;
327  BESDEBUG(HDF5_NAME, prolog << "H5.EnableCheckNameClashing: " << (_check_name_clashing?"true":"false") << endl);
328 
329  key_value = obtain_beskeys_info("H5.EnableAddPathAttrs",has_key);
330  if(has_key)
331  _add_path_attrs = key_value;
332  BESDEBUG(HDF5_NAME, prolog << "H5.EnableAddPathAttrs: " << (_add_path_attrs?"true":"false") << endl);
333 
334  key_value = obtain_beskeys_info("H5.EnableDropLongString",has_key);
335  if(has_key)
336  _drop_long_string = key_value;
337  BESDEBUG(HDF5_NAME, prolog << "H5.EnableDropLongString: " << (_drop_long_string?"true":"false") << endl);
338 
339  key_value = obtain_beskeys_info("H5.EnableFillValueCheck",has_key);
340  if(has_key)
341  _fillvalue_check = key_value;
342  BESDEBUG(HDF5_NAME, prolog << "H5.EnableFillValueCheck: " << (_fillvalue_check?"true":"false") << endl);
343 
344 
345  key_value = obtain_beskeys_info("H5.CheckIgnoreObj",has_key);
346  if(has_key)
347  _check_ignore_obj = key_value;
348  BESDEBUG(HDF5_NAME, prolog << "H5.CheckIgnoreObj: " << (_check_ignore_obj?"true":"false") << endl);
349 
350  key_value = obtain_beskeys_info("H5.ForceFlattenNDCoorAttr",has_key);
351  if(has_key)
352  _flatten_coor_attr = key_value;
353  BESDEBUG(HDF5_NAME, prolog << "H5.ForceFlattenNDCoorAttr: " << (_flatten_coor_attr?"true":"false") << endl);
354 
355 
356  key_value = obtain_beskeys_info("H5.RmConventionAttrPath",has_key);
357  if(has_key)
358  _eos5_rm_convention_attr_path = key_value;
359  BESDEBUG(HDF5_NAME, prolog << "H5.RmConventionAttrPath: " << (_eos5_rm_convention_attr_path?"true":"false") << endl);
360 
361  key_value = obtain_beskeys_info("H5.EnableDMR64bitInt",has_key);
362  if(has_key)
363  _dmr_long_int = key_value;
364  BESDEBUG(HDF5_NAME, prolog << "H5.EnableDMR64bitInt: " << (_dmr_long_int?"true":"false") << endl);
365 
366  key_value = obtain_beskeys_info("H5.NoZeroSizeFullnameAttr",has_key);
367  if(has_key)
368  _no_zero_size_fullnameattr = key_value;
369  BESDEBUG(HDF5_NAME, prolog << "H5.NoZeroSizeFullnameAttr: " << (_no_zero_size_fullnameattr?"true":"false") << endl);
370 
371  key_value = obtain_beskeys_info("H5.EnableCoorattrAddPath",has_key);
372  if(has_key)
373  _enable_coord_attr_add_path = key_value;
374  BESDEBUG(HDF5_NAME, prolog << "H5.EnableCoorattrAddPath: " << (_enable_coord_attr_add_path?"true":"false") << endl);
375 
376  key_value = obtain_beskeys_info("H5.EnableCFDMR",has_key);
377  if(has_key)
378  _usecfdmr = key_value;
379  BESDEBUG(HDF5_NAME, prolog << "H5.EnableCFDMR: " << (_usecfdmr?"true":"false") << endl);
380 
381  key_value = obtain_beskeys_info("H5.EnableDAP4Coverage",has_key);
382  if(has_key)
383  _add_dap4_coverage = key_value;
384  BESDEBUG(HDF5_NAME, prolog << "H5.EnableDAP4Coverage: " << (_add_dap4_coverage?"true":"false") << endl);
385 
386  key_value = obtain_beskeys_info("H5.EnableDiskDataCache",has_key);
387  if(has_key)
388  _use_disk_cache = key_value;
389  BESDEBUG(HDF5_NAME, prolog << "H5.EnableDiskDataCache: " << (_use_disk_cache?"true":"false") << endl);
390  _disk_cache_dir = get_beskeys("H5.DiskCacheDataPath");
391  _disk_cachefile_prefix = get_beskeys("H5.DiskCacheFilePrefix");
392  _disk_cache_size = get_ulong_key("H5.DiskCacheSize",0);
393 
394  key_value = obtain_beskeys_info("H5.DiskCacheComp",has_key);
395  if(has_key)
396  _disk_cache_comp_data = key_value;
397  BESDEBUG(HDF5_NAME, prolog << "H5.DiskCacheComp: " << (_disk_cache_comp_data?"true":"false") << endl);
398 
399  key_value = obtain_beskeys_info("H5.DiskCacheFloatOnlyComp",has_key);
400  if(has_key)
401  _disk_cache_float_only_comp_data = key_value;
402  BESDEBUG(HDF5_NAME, prolog << "H5.DiskCacheFloatOnlyComp: " << (_disk_cache_float_only_comp_data?"true":"false") << endl);
403  _disk_cache_comp_threshold = get_float_key("H5.DiskCacheCompThreshold",1.0);
404  _disk_cache_var_size = 1024*get_uint_key("H5.DiskCacheCompVarSize",0);
405 
406  key_value = obtain_beskeys_info("H5.EnableDiskMetaDataCache",has_key);
407  if(has_key)
408  _use_disk_meta_cache = key_value;
409  BESDEBUG(HDF5_NAME, prolog << "H5.EnableDiskMetaDataCache: " << (_use_disk_meta_cache?"true":"false") << endl);
410 
411  key_value = obtain_beskeys_info("H5.EnableDiskDDSCache",has_key);
412  if(has_key)
413  _use_disk_dds_cache = key_value;
414  BESDEBUG(HDF5_NAME, prolog << "H5.EnableDiskDDSCache: " << (_use_disk_dds_cache?"true":"false") << endl);
415  _disk_meta_cache_path = get_beskeys("H5.DiskMetaDataCachePath");
416 
417  key_value = obtain_beskeys_info("H5.EnableEOSGeoCacheFile",has_key);
418  if(has_key)
419  _use_latlon_disk_cache = key_value;
420  BESDEBUG(HDF5_NAME, prolog << "H5.EnableEOSGeoCacheFile: " << (_use_latlon_disk_cache?"true":"false") << endl);
421  _latlon_disk_cache_size = get_uint_key("H5.Cache.latlon.size",0);
422  _latlon_disk_cache_dir = get_beskeys("H5.Cache.latlon.path");
423  _latlon_disk_cachefile_prefix= get_beskeys("H5.Cache.latlon.prefix");
424 
425  key_value = obtain_beskeys_info("H5.EscapeUTF8Attr", has_key);
426  if (has_key)
427  _escape_utf8_attr = key_value;
428 
429 #if 0
430  //_default_handle_dimension = check_and_set_beskeys("H5.DefaultHandleDimension");
431  key_value = obtain_beskeys_info("H5.DefaultHandleDimension",has_key);
432  if(has_key)
433  _default_handle_dimension = key_value;
434  BESDEBUG(HDF5_NAME, prolog << "H5.DefaultHandleDimension: " << (_default_handle_dimension?"true":"false") << endl);
435 #endif
436 
437  if(get_usecf()) {
438  if(get_lrdcache_entries()) {
439  lrdata_mem_cache = new ObjMemCache(get_lrdcache_entries(), get_cache_purge_level());
440  bool has_LFMC_config = false;
441  key_value = obtain_beskeys_info("H5.LargeDataMemCacheConfig",has_key);
442  if(has_key)
443  has_LFMC_config = key_value;
444  BESDEBUG(HDF5_NAME, prolog << "H5.LargeDataMemCacheConfig: " << (has_LFMC_config?"true":"false") << endl);
445  if(true == has_LFMC_config) {
446  _common_cache_dirs =obtain_lrd_common_cache_dirs();
447 #if 0
448  if(false == _common_cache_dirs)
449 cerr<<"No specific cache info"<<endl;
450 #endif
451  }
452  }
453  if(get_srdcache_entries()) {
454 
455  BESDEBUG(HDF5_NAME, prolog << "Generate memory cache for smaller coordinate variables" << endl);
456  srdata_mem_cache = new ObjMemCache(get_srdcache_entries(),get_cache_purge_level());
457 
458  }
459 
460  if(_disk_cache_comp_data == true && _use_disk_cache == true) {
461  if(_disk_cache_comp_threshold < 1.0) {
462  ostringstream ss;
463  ss<< _disk_cache_comp_threshold;
464  string _comp_threshold_str(ss.str());
465  string invalid_comp_threshold ="The Compression Threshold is the total size of the variable array";
466  invalid_comp_threshold+=" divided by the storage size of compressed array. It should always be >1";
467  invalid_comp_threshold+=" The current threhold set at h5.conf is ";
468  invalid_comp_threshold+=_comp_threshold_str;
469  invalid_comp_threshold+=" . Go back to h5.conf and change the H5.DiskCacheCompThreshold to a >1.0 number.";
470  throw BESInternalError(invalid_comp_threshold,__FILE__,__LINE__);
471  }
472  }
473  _stp_east_filename = get_beskeys("H5.STPEastFileName");
474  _stp_north_filename = get_beskeys("H5.STPNorthFileName");
475  }
476  BESDEBUG(HDF5_NAME, prolog << "END" << endl);
477 }
478 
479 
480 // Build DAS
481 bool HDF5RequestHandler::hdf5_build_das(BESDataHandlerInterface & dhi)
482 {
483  BESDEBUG(HDF5_NAME, prolog << "BEGIN" << endl);
484 #if DYNAMIC_CONFIG_ENABLED
485  load_config();
486 #endif
487 
488  // For the time being, separate CF file ID from the default file ID(mainly for debugging)
489  hid_t cf_fileid = -1;
490 
491  // Obtain the HDF5 file name.
492  string filename = dhi.container->access();
493 
494  // Obtain the BES object from the client
495  BESResponseObject *response = dhi.response_handler->get_response_object() ;
496 
497  // Convert to the BES DAS response
498  BESDASResponse *bdas = dynamic_cast < BESDASResponse * >(response) ;
499  if( !bdas )
500  throw BESInternalError( "cast error", __FILE__, __LINE__ ) ;
501 
502  try {
503  bdas->set_container( dhi.container->get_symbolic_name() ) ;
504  DAS *das = bdas->get_das();
505 
506  // Look inside the memory cache to see if it's initialized
507  DAS *cached_das_ptr = nullptr;
508  bool use_das_cache = false;
509  if (das_cache)
510  cached_das_ptr = static_cast<DAS*>(das_cache->get(filename));
511  if (cached_das_ptr)
512  use_das_cache = true;
513 
514  if (true == use_das_cache) {
515 
516  // copy the cached DAS into the BES response object
517  BESDEBUG(HDF5_NAME, prolog << "DAS Cached hit for : " << filename << endl);
518  *das = *cached_das_ptr;
519  }
520  else {
521 
522  bool das_from_dc = false;
523  string das_cache_fname;
524 
525  // If the use_disk_meta_cache is set, check if the cache file exists and sets the flag.
526  if(_use_disk_meta_cache == true) {
527 
528  string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
529  das_cache_fname = _disk_meta_cache_path+"/" +base_filename+"_das";
530 
531  if(access(das_cache_fname.c_str(),F_OK) !=-1)
532  das_from_dc = true;
533 
534  }
535 
536  // If reading DAS from the disk cache, call the corresponding function
537  if(true == das_from_dc) {
538  read_das_from_disk_cache(das_cache_fname,das);
539 
540  // If the memory cache is set, adding the DAS copy to the memory cache
541  if (das_cache) {
542  // add a copy
543  BESDEBUG(HDF5_NAME, prolog << "HDF5 DAS reading DAS from the disk cache. For memory cache, DAS added to the cache for : " << filename << endl);
544  das_cache->add(new DAS(*das), filename);
545  }
546  }
547 
548  else {// Need to build from the HDF5 file
549  H5Eset_auto2(H5E_DEFAULT,nullptr,nullptr);
550  if (true == _usecf) {//CF option
551 
552  cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
553  if (cf_fileid < 0){
554  string invalid_file_msg="Could not open this HDF5 file ";
555  invalid_file_msg +=filename;
556  invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
557  invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
558  invalid_file_msg +=" distributor.";
559  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
560  }
561  // Need to check if DAP4 DMR CF 64-bit integer mapping is on.
562  if(HDF5RequestHandler::get_dmr_64bit_int()!=nullptr)
563  HDF5RequestHandler::set_dmr_64bit_int(nullptr);
564  read_cfdas( *das,filename,cf_fileid);
565  H5Fclose(cf_fileid);
566  }
567  else {// Default option
568  hid_t fileid = get_fileid(filename.c_str());
569  if (fileid < 0) {
570  string invalid_file_msg="Could not open this HDF5 file ";
571  invalid_file_msg +=filename;
572  invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
573  invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
574  invalid_file_msg +=" distributor.";
575  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
576 
577  }
578 
579  find_gloattr(fileid, *das);
580  depth_first(fileid, "/", *das);
581  close_fileid(fileid);
582  }
583 
584  Ancillary::read_ancillary_das( *das, filename ) ;
585 
586 #if 0
587 // Dump all attribute contents
588 AttrTable* top_table = das->get_top_level_attributes();
589 get_attr_contents(top_table);
590 
591 // Dump all variable contents
592 AttrTable::Attr_iter start_aiter=das->var_begin();
593 AttrTable::Attr_iter it = start_aiter;
594 AttrTable::Attr_iter end_aiter = das->var_end();
595 while(it != end_aiter) {
596 AttrTable* temp_table = das->get_table(it);
597 if(temp_table!=0){
598 cerr<<"var_begin"<<endl;
599 temp_table->print(cerr);
600 }
601 ++it;
602 }
603 #endif
604  // If the memory cache is turned on
605  if(das_cache) {
606  // add a copy
607  BESDEBUG(HDF5_NAME, prolog << "DAS added to the cache for : " << filename << endl);
608  das_cache->add(new DAS(*das), filename);
609  }
610 
611  // DAS disk cache fname will be set only when the metadata disk cache is turned on
612  // So if it comes here, the das cache should be generated.
613  if(das_cache_fname!="") {
614  BESDEBUG(HDF5_NAME, prolog << "HDF5 Build DAS: Write DAS to disk cache " << das_cache_fname << endl);
615  write_das_to_disk_cache(das_cache_fname,das);
616  }
617  }
618  }
619 
620  bdas->clear_container() ;
621  }
622  catch(BESError & e) {
623  if(cf_fileid !=-1)
624  H5Fclose(cf_fileid);
625  BESDEBUG(HDF5_NAME, prolog << "Caught BESError! Message: " << e.get_message() << endl);
626  throw;
627  }
628  catch(InternalErr & e) {
629 
630  if(cf_fileid !=-1)
631  H5Fclose(cf_fileid);
632  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
633  __FILE__, __LINE__);
634  }
635  catch(Error & e) {
636 
637  if(cf_fileid !=-1)
638  H5Fclose(cf_fileid);
639  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
640  __FILE__, __LINE__);
641  }
642  catch(...) {
643 
644  if(cf_fileid !=-1)
645  H5Fclose(cf_fileid);
646  string s = "unknown exception caught building HDF5 DAS";
647  throw BESInternalFatalError(s, __FILE__, __LINE__);
648  }
649 
650  BESDEBUG(HDF5_NAME, prolog << "END" << endl);
651  return true;
652 }
653 
654 // Convenient function that helps build DDS and Data
655 // Since this function will be used by both the DDS and Data services, we need to pass both BESDDSResponse and BESDataDDSResponse.
656 // This two parameters are necessary for the future DDS disk cache feature.
657 void HDF5RequestHandler::get_dds_with_attributes( BESDDSResponse*bdds,BESDataDDSResponse*data_bdds,const string &container_name, const string& filename,const string &dds_cache_fname, const string &das_cache_fname,bool dds_from_dc,bool das_from_dc, bool build_data)
658 {
659  DDS *dds;
660  if(true == build_data)
661  dds = data_bdds->get_dds();
662  else dds = bdds->get_dds();
663 
664  // For the time being, separate CF file ID from the default file ID(mainly for debugging)
665  hid_t fileid = -1;
666  hid_t cf_fileid = -1;
667 
668  try {
669 
670  // Look in memory cache to see if it's initialized
671  DDS* cached_dds_ptr = nullptr;
672  bool use_dds_cache = false;
673  if (dds_cache)
674  cached_dds_ptr = static_cast<DDS*>(dds_cache->get(filename));
675  if (cached_dds_ptr)
676  use_dds_cache = true;
677  if (true == use_dds_cache) {
678  // copy the cached DDS into the BES response object. Assume that any cached DDS
679  // includes the DAS information.
680  BESDEBUG(HDF5_NAME, prolog << "DDS Metadata Cached hit for : " << filename << endl);
681  *dds = *cached_dds_ptr; // Copy the referenced object
682  }
683  else if (true ==dds_from_dc) {//Currently the dds_from_ds is always false by default.
684  read_dds_from_disk_cache(bdds,data_bdds,build_data,container_name,filename,dds_cache_fname,das_cache_fname,-1,das_from_dc);
685  }
686  else {
687  BESDEBUG(HDF5_NAME, prolog << "Build DDS from the HDF5 file. " << filename << endl);
688  H5Eset_auto2(H5E_DEFAULT,nullptr,nullptr);
689  dds->filename(filename);
690 
691  // For the time being, not mess up CF's fileID with Default's fileID
692  if(true == _usecf) {
693 
694  cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
695  if (cf_fileid < 0){
696  string invalid_file_msg="Could not open this HDF5 file ";
697  invalid_file_msg +=filename;
698  invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
699  invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
700  invalid_file_msg +=" distributor.";
701  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
702  }
703  // The following is for DAP4 CF(DMR) 64-bit mapping, we need to set the flag
704  // to let the handler map the 64-bit integer.
705  if(HDF5RequestHandler::get_dmr_64bit_int() != nullptr)
706  HDF5RequestHandler::set_dmr_64bit_int(nullptr);
707  read_cfdds(*dds,filename,cf_fileid);
708  }
709  else {
710 
711  fileid = get_fileid(filename.c_str());
712  if (fileid < 0) {
713  string invalid_file_msg="Could not open this HDF5 file ";
714  invalid_file_msg +=filename;
715  invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
716  invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
717  invalid_file_msg +=" distributor.";
718  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
719  }
720 
721  depth_first(fileid, (char*)"/", *dds, filename.c_str());
722 
723  }
724  // Check semantics
725  if (!dds->check_semantics()) { // DDS didn't comply with the semantics
726  dds->print(cerr);
727  throw InternalErr(__FILE__, __LINE__,
728  "DDS check_semantics() failed. This can happen when duplicate variable names are defined. ");
729  }
730 
731  Ancillary::read_ancillary_dds( *dds, filename ) ;
732 
733  // Generate the DDS cached file if needed,currently this is always false by default
734  if(dds_cache_fname!="" && dds_from_dc == false)
735  write_dds_to_disk_cache(dds_cache_fname,dds);
736 
737  // Add attributes
738  {
739  hid_t h5_fd = -1;
740  if(_usecf == true)
741  h5_fd = cf_fileid;
742  else
743  h5_fd = fileid;
744  add_das_to_dds(dds,container_name,filename,das_cache_fname,h5_fd,das_from_dc);
745  }
746 
747  // Add memory cache if possible
748  if (dds_cache) {
749  // add a copy
750  BESDEBUG(HDF5_NAME, prolog << "DDS added to the cache for : " << filename << endl);
751  dds_cache->add(new DDS(*dds), filename);
752  }
753 
754  if(cf_fileid != -1)
755  H5Fclose(cf_fileid);
756  if(fileid != -1)
757  H5Fclose(fileid);
758 
759  }
760 
761  }
762  catch(InternalErr & e) {
763 
764  if(cf_fileid !=-1)
765  H5Fclose(cf_fileid);
766 
767  if(fileid != -1)
768  H5Fclose(fileid);
769 
770  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
771  __FILE__, __LINE__);
772  }
773  catch(Error & e) {
774 
775  if(cf_fileid !=-1)
776  H5Fclose(cf_fileid);
777  if(fileid !=-1)
778  H5Fclose(fileid);
779 
780  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
781  __FILE__, __LINE__);
782  }
783  catch(...) {
784 
785  if(cf_fileid !=-1)
786  H5Fclose(cf_fileid);
787  if(fileid !=-1)
788  H5Fclose(fileid);
789 
790  string s = "unknown exception caught building HDF5 DDS";
791  throw BESInternalFatalError(s, __FILE__, __LINE__);
792  }
793 
794 }
795 
796 void HDF5RequestHandler::get_dds_without_attributes_datadds(BESDataDDSResponse*data_bdds,const string &container_name, const string& filename)
797 {
798  DDS *dds = data_bdds->get_dds();
799 
800  // For the time being, separate CF file ID from the default file ID(mainly for debugging)
801  hid_t fileid = -1;
802  hid_t cf_fileid = -1;
803 
804  try {
805 
806  // Look in memory cache to see if it's initialized
807  DDS* cached_dds_ptr = nullptr;
808  bool use_datadds_cache = false;
809  if (datadds_cache)
810  cached_dds_ptr = static_cast<DDS*>(datadds_cache->get(filename));
811  if (cached_dds_ptr)
812  use_datadds_cache = true;
813  if (true == use_datadds_cache) {
814  // copy the cached DDS into the BES response object.
815  // The DAS information is not included.
816  BESDEBUG(HDF5_NAME, prolog << "DataDDS Metadata Cached hit for : " << filename << endl);
817  *dds = *cached_dds_ptr; // Copy the referenced object
818  }
819  else {
820  BESDEBUG(HDF5_NAME, prolog << "Build DDS from the HDF5 file. " << filename << endl);
821  H5Eset_auto2(H5E_DEFAULT,nullptr,nullptr);
822  dds->filename(filename);
823 
824  // For the time being, not mess up CF's fileID with Default's fileID
825  if(true == _usecf) {
826 
827  cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
828  if (cf_fileid < 0){
829  string invalid_file_msg="Could not open this HDF5 file ";
830  invalid_file_msg +=filename;
831  invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
832  invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
833  invalid_file_msg +=" distributor.";
834  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
835  }
836  // The following is for DAP4 CF(DMR) 64-bit mapping, we need to set the flag
837  // to let the handler map the 64-bit integer.
838  if(HDF5RequestHandler::get_dmr_64bit_int() != nullptr)
839  HDF5RequestHandler::set_dmr_64bit_int(nullptr);
840  read_cfdds(*dds,filename,cf_fileid);
841  }
842  else {
843 
844  fileid = get_fileid(filename.c_str());
845  if (fileid < 0) {
846  string invalid_file_msg="Could not open this HDF5 file ";
847  invalid_file_msg +=filename;
848  invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
849  invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
850  invalid_file_msg +=" distributor.";
851  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
852  }
853 
854  depth_first(fileid, (char*)"/", *dds, filename.c_str());
855 
856  }
857  // Check semantics
858  if (!dds->check_semantics()) { // DDS didn't comply with the semantics
859  dds->print(cerr);
860  throw InternalErr(__FILE__, __LINE__,
861  "DDS check_semantics() failed. This can happen when duplicate variable names are defined. ");
862  }
863 
864  Ancillary::read_ancillary_dds( *dds, filename ) ;
865 
866 #if 0
867  // Add attributes
868  {
869  hid_t h5_fd = -1;
870  if(_usecf == true)
871  h5_fd = cf_fileid;
872  else
873  h5_fd = fileid;
874  add_das_to_dds(dds,container_name,filename,das_cache_fname,h5_fd,das_from_dc);
875  }
876 #endif
877 
878  // Add memory cache if possible
879  if (datadds_cache) {
880  // add a copy
881  BESDEBUG(HDF5_NAME, prolog << "DataDDS added to the cache for : " << filename << endl);
882  datadds_cache->add(new DDS(*dds), filename);
883  }
884 
885  if(cf_fileid != -1)
886  H5Fclose(cf_fileid);
887  if(fileid != -1)
888  H5Fclose(fileid);
889 
890  }
891  BESDEBUG(HDF5_NAME, prolog << "Data ACCESS build_data(): set the including attribute flag to false: "<<filename << endl);
892  data_bdds->set_ia_flag(false);
893 
894  }
895  catch(InternalErr & e) {
896 
897  if(cf_fileid !=-1)
898  H5Fclose(cf_fileid);
899 
900  if(fileid != -1)
901  H5Fclose(fileid);
902 
903  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
904  __FILE__, __LINE__);
905  }
906  catch(Error & e) {
907 
908  if(cf_fileid !=-1)
909  H5Fclose(cf_fileid);
910  if(fileid !=-1)
911  H5Fclose(fileid);
912 
913  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
914  __FILE__, __LINE__);
915  }
916  catch(...) {
917 
918  if(cf_fileid !=-1)
919  H5Fclose(cf_fileid);
920  if(fileid !=-1)
921  H5Fclose(fileid);
922 
923  string s = "unknown exception caught building HDF5 DDS";
924  throw BESInternalFatalError(s, __FILE__, __LINE__);
925  }
926 
927 }
928 
929 
930 #if 0
931 // OLD function: Keep it for a while
932 // Convenient function that helps build DDS and Data
933 void HDF5RequestHandler::get_dds_with_attributes(const string &filename, const string &container_name, DDS*dds) {
934 
935  // For the time being, separate CF file ID from the default file ID(mainly for debugging)
936  hid_t fileid = -1;
937  hid_t cf_fileid = -1;
938 
939  try {
940 
941  // Look in memory cache to see if it's initialized
942  DDS* cached_dds_ptr = 0;
943  if (dds_cache && (cached_dds_ptr = static_cast<DDS*>(dds_cache->get(filename)))) {
944  // copy the cached DDS into the BES response object. Assume that any cached DDS
945  // includes the DAS information.
946  BESDEBUG(HDF5_NAME, prolog << "DDS Cached hit for : " << filename << endl);
947  *dds = *cached_dds_ptr; // Copy the referenced object
948  }
949 
950  else {
951 
952  H5Eset_auto2(H5E_DEFAULT,nullptr,nullptr);
953  if (!container_name.empty())
954  dds->container_name(container_name);
955  dds->filename(filename);
956 
957  // For the time being, not mess up CF's fileID with Default's fileID
958  if(true == _usecf) {
959 // This block cannot be used to cache the data
960 #if 0
961  string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
962  string dds_filename = "/tmp/"+base_filename+"_dds";
963  FILE *dds_file = fopen(dds_filename.c_str(),"r");
964 cerr<<"before parsing "<<endl;
965 BaseTypeFactory tf;
966 DDS tdds(&tf,name_path(filename),"3.2");
967 tdds.filename(filename);
968  //dds->parse(dds_file);
969  tdds.parse(dds_file);
970  //DDS *cache_dds = new DDS(tdds);
971  cache_dds = new DDS(tdds);
972 if(dds!=nullptr)
973  delete dds;
974 dds = cache_dds;
975 tdds.print(cout);
976 dds->print(cout);
977 cerr<<"after parsing "<<endl;
978 //dds->print(cout);
979  // fclose(dds_file);
980 //#endif
981 
982 #endif
983 // end of this block
984 
985 
986  cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
987  if (cf_fileid < 0){
988  string invalid_file_msg="Could not open this HDF5 file ";
989  invalid_file_msg +=filename;
990  invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
991  invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
992  invalid_file_msg +=" distributor.";
993  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
994  }
995 //#if 0
996  read_cfdds(*dds,filename,cf_fileid);
997 //#endif
998  // Generate the DDS cached file
999  string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
1000  string dds_filename = "/tmp/"+base_filename+"_dds";
1001  FILE *dds_file = fopen(dds_filename.c_str(),"w");
1002  dds->print(dds_file);
1003  fclose(dds_file);
1004 
1005  }
1006  else {
1007 
1008  fileid = get_fileid(filename.c_str());
1009  if (fileid < 0) {
1010  string invalid_file_msg="Could not open this HDF5 file ";
1011  invalid_file_msg +=filename;
1012  invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
1013  invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
1014  invalid_file_msg +=" distributor.";
1015  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1016  }
1017 
1018  depth_first(fileid, (char*)"/", *dds, filename.c_str());
1019 
1020  }
1021 
1022 
1023  // Check semantics
1024  if (!dds->check_semantics()) { // DDS didn't comply with the semantics
1025  dds->print(cerr);
1026  throw InternalErr(__FILE__, __LINE__,
1027  "DDS check_semantics() failed. This can happen when duplicate variable names are defined. ");
1028  }
1029 
1030  Ancillary::read_ancillary_dds( *dds, filename ) ;
1031 
1032 
1033  // Check DAS cache
1034  DAS *das = 0 ;
1035 
1036  if (das_cache && (das = static_cast<DAS*>(das_cache->get(filename)))) {
1037  BESDEBUG(HDF5_NAME, prolog << "DAS Cached hit for : " << filename << endl);
1038  dds->transfer_attributes(das); // no need to copy the cached DAS
1039  }
1040 
1041  else {
1042 
1043  das = new DAS ;
1044 
1045  if (!container_name.empty())
1046  das->container_name(container_name);
1047 
1048  if (true == _usecf) {
1049 
1050  // go to the CF option
1051  read_cfdas( *das,filename,cf_fileid);
1052 
1053  }
1054  else {
1055 
1056  find_gloattr(fileid, *das);
1057  depth_first(fileid, "/", *das);
1058  close_fileid(fileid);
1059  }
1060 
1061  if(cf_fileid != -1)
1062  H5Fclose(cf_fileid);
1063 
1064  Ancillary::read_ancillary_das( *das, filename ) ;
1065 
1066  dds->transfer_attributes(das);
1067 
1068 
1069  // Only free the DAS if it's not added to the cache
1070  if (das_cache) {
1071  // add a copy
1072  BESDEBUG(HDF5_NAME, prolog << "DAS added to the cache for : " << filename << endl);
1073  //das_cache->add(new DAS(*das), filename);
1074  das_cache->add(das, filename);
1075  }
1076  else {
1077  delete das;
1078  }
1079  }
1080 
1081  if (dds_cache) {
1082  // add a copy
1083  BESDEBUG(HDF5_NAME, prolog << "DDS added to the cache for : " << filename << endl);
1084  dds_cache->add(new DDS(*dds), filename);
1085  }
1086 
1087  }
1088 
1089 //dds->print(cout);
1090 
1091  }
1092  catch(InternalErr & e) {
1093 
1094  if(cf_fileid !=-1)
1095  H5Fclose(cf_fileid);
1096 
1097  if(fileid != -1)
1098  H5Fclose(fileid);
1099 
1100  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
1101  __FILE__, __LINE__);
1102  }
1103  catch(Error & e) {
1104 
1105  if(cf_fileid !=-1)
1106  H5Fclose(cf_fileid);
1107  if(fileid !=-1)
1108  H5Fclose(fileid);
1109 
1110  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
1111  __FILE__, __LINE__);
1112  }
1113  catch(...) {
1114 
1115  if(cf_fileid !=-1)
1116  H5Fclose(cf_fileid);
1117  if(fileid !=-1)
1118  H5Fclose(fileid);
1119 
1120  string s = "unknown exception caught building HDF5 DDS";
1121  throw BESInternalFatalError(s, __FILE__, __LINE__);
1122  }
1123 
1124 }
1125 #endif
1126 
1127 // Build DDS
1128 bool HDF5RequestHandler::hdf5_build_dds(BESDataHandlerInterface & dhi)
1129 {
1130  BESDEBUG(HDF5_NAME, prolog << "BEGIN" << endl);
1131 #if DYNAMIC_CONFIG_ENABLED
1132  load_config();
1133 #endif
1134 
1135  // Obtain the HDF5 file name.
1136  string filename = dhi.container->access();
1137 
1138  string container_name = dhi.container->get_symbolic_name();
1139  BESResponseObject *response = dhi.response_handler->get_response_object();
1140  BESDDSResponse *bdds = dynamic_cast < BESDDSResponse * >(response);
1141  if( !bdds )
1142  throw BESInternalError( "cast error", __FILE__, __LINE__ ) ;
1143  bdds->set_container(container_name);
1144 
1145  try {
1146 
1147  bool dds_from_dc = false;
1148  bool das_from_dc = false;
1149  bool build_data = false;
1150  string dds_cache_fname;
1151  string das_cache_fname;
1152 
1153  if(_use_disk_meta_cache) {
1154 
1155  string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
1156 
1157  // The _use_disk_dds_cache is always set to false by default
1158  if(_use_disk_dds_cache) {
1159  dds_cache_fname = _disk_meta_cache_path+"/" +base_filename+"_dds";
1160  if(access(dds_cache_fname.c_str(),F_OK) !=-1)
1161  dds_from_dc = true;
1162  }
1163 
1164  das_cache_fname = _disk_meta_cache_path+"/" +base_filename+"_das";
1165  // Check if das files exist
1166  if(access(das_cache_fname.c_str(),F_OK) !=-1)
1167  das_from_dc = true;
1168 
1169  }
1170 
1171  get_dds_with_attributes(bdds, nullptr,container_name,filename, dds_cache_fname,das_cache_fname,dds_from_dc,das_from_dc,build_data);
1172 
1173  // The following block reads dds from a dds cache file.
1174 #if 0
1175  string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
1176  string dds_filename = "/tmp/"+base_filename+"_dds";
1177 
1178  BaseTypeFactory tf;
1179  DDS tdds(&tf,name_path(filename),"3.2");
1180  tdds.filename(filename);
1181 
1182 
1183  FILE *dds_file = fopen(dds_filename.c_str(),"r");
1184  tdds.parse(dds_file);
1185 //cerr<<"before parsing "<<endl;
1186  DDS* cache_dds = new DDS(tdds);
1187  if(dds != nullptr)
1188  delete dds;
1189  bdds->set_dds(cache_dds);
1190  fclose(dds_file);
1191 #endif
1192 
1193  bdds->set_constraint( dhi ) ;
1194  bdds->clear_container() ;
1195 
1196  }
1197  catch(BESError & e) {
1198  BESDEBUG(HDF5_NAME, prolog << "Caught BESError! Message: " << e.get_message() << endl);
1199  throw;
1200  }
1201  catch(InternalErr & e) {
1202 
1203  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
1204  __FILE__, __LINE__);
1205  }
1206  catch(Error & e) {
1207 
1208  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
1209  __FILE__, __LINE__);
1210  }
1211  catch(...) {
1212 
1213  string s = "unknown exception caught building HDF5 DDS";
1214  throw BESInternalFatalError(s, __FILE__, __LINE__);
1215  }
1216 
1217  BESDEBUG(HDF5_NAME, prolog << "END" << endl);
1218  return true;
1219 }
1220 
1221 bool HDF5RequestHandler::hdf5_build_data(BESDataHandlerInterface & dhi)
1222 {
1223  BESDEBUG(HDF5_NAME, prolog << "BEGIN" << endl);
1224 #if DYNAMIC_CONFIG_ENABLED
1225  load_config();
1226 #endif
1227 
1228  if(true ==_usecf) {
1229 
1230  if(true == _pass_fileid)
1231  return hdf5_build_data_with_IDs(dhi);
1232  }
1233 
1234  string filename = dhi.container->access();
1235 
1236  string container_name = dhi.container->get_symbolic_name();
1237  BESResponseObject *response = dhi.response_handler->get_response_object();
1238  BESDataDDSResponse *bdds = dynamic_cast < BESDataDDSResponse * >(response);
1239  if( !bdds )
1240  throw BESInternalError( "cast error", __FILE__, __LINE__ ) ;
1241  bdds->set_container(container_name);
1242 
1243  try {
1244 
1245  // DDS from disk cache is not currently supported. It may be supported in the future.
1246  // We will leave the code as commented.
1247 #if 0
1248  bool dds_from_dc = false;
1249  bool build_data = true;
1250 #endif
1251 
1252  bool das_from_dc = false;
1253  string dds_cache_fname;
1254  string das_cache_fname;
1255 
1256 
1257  // Only DAS is read from the cache. dds_from_dc is always false.
1258  if(_use_disk_meta_cache == true) {
1259 
1260  string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
1261  das_cache_fname = _disk_meta_cache_path+"/" +base_filename+"_das";
1262 
1263  if(access(das_cache_fname.c_str(),F_OK) !=-1)
1264  das_from_dc = true;
1265 
1266  }
1267 
1268 #if 0
1269  get_dds_with_attributes(nullptr,bdds, container_name,filename, dds_cache_fname,das_cache_fname,dds_from_dc,das_from_dc,build_data);
1270 #endif
1271  get_dds_without_attributes_datadds(bdds,container_name,filename);
1272 
1273  bdds->set_constraint( dhi ) ;
1274  bdds->clear_container() ;
1275 
1276  }
1277  catch(BESError & e) {
1278  BESDEBUG(HDF5_NAME, prolog << "Caught BESError! Message: " << e.get_message() << endl);
1279  throw;
1280  }
1281  catch(InternalErr & e) {
1282 
1283  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
1284  __FILE__, __LINE__);
1285  }
1286  catch(Error & e) {
1287 
1288  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
1289  __FILE__, __LINE__);
1290  }
1291  catch(...) {
1292 
1293  string s = "unknown exception caught building HDF5 DDS";
1294  throw BESInternalFatalError(s, __FILE__, __LINE__);
1295  }
1296 
1297  BESDEBUG(HDF5_NAME, prolog << "END" << endl);
1298  return true;
1299 }
1300 
1301 // Obtain data when turning on the pass fileID key.The memory cache is not used.
1302 bool HDF5RequestHandler::hdf5_build_data_with_IDs(BESDataHandlerInterface & dhi)
1303 {
1304  BESDEBUG(HDF5_NAME, prolog << "BEGIN" << endl);
1305 #if DYNAMIC_CONFIG_ENABLED
1306  load_config();
1307 #endif
1308 
1309  BESDEBUG(HDF5_NAME,prolog << "Building DataDDS by passing file IDs. "<<endl);
1310  hid_t cf_fileid = -1;
1311 
1312  string filename = dhi.container->access();
1313 
1314  H5Eset_auto2(H5E_DEFAULT,nullptr,nullptr);
1315  cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
1316  if (cf_fileid < 0){
1317  string invalid_file_msg="Could not open this HDF5 file ";
1318  invalid_file_msg +=filename;
1319  invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
1320  invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
1321  invalid_file_msg +=" distributor.";
1322  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1323  }
1324 
1325  BESResponseObject *response = dhi.response_handler->get_response_object();
1326  BESDataDDSResponse *bdds = dynamic_cast < BESDataDDSResponse * >(response);
1327  if( !bdds )
1328  throw BESInternalError( "cast error", __FILE__, __LINE__ ) ;
1329 
1330  try {
1331 
1332  bdds->set_container( dhi.container->get_symbolic_name() ) ;
1333 
1334  auto hdds = new HDF5DDS(bdds->get_dds());
1335  delete bdds->get_dds();
1336 
1337  bdds->set_dds(hdds);
1338 
1339  hdds->setHDF5Dataset(cf_fileid);
1340 
1341  read_cfdds( *hdds,filename,cf_fileid);
1342 
1343  if (!hdds->check_semantics()) { // DDS didn't comply with the DAP semantics
1344  hdds->print(cerr);
1345  throw InternalErr(__FILE__, __LINE__,
1346  "DDS check_semantics() failed. This can happen when duplicate variable names are defined.");
1347  }
1348 
1349  Ancillary::read_ancillary_dds( *hdds, filename ) ;
1350 
1351  DAS *das = new DAS ;
1352  BESDASResponse bdas( das ) ;
1353  bdas.set_container( dhi.container->get_symbolic_name() ) ;
1354  read_cfdas( *das,filename,cf_fileid);
1355  Ancillary::read_ancillary_das( *das, filename ) ;
1356 
1357  hdds->transfer_attributes(das);
1358  bdds->set_constraint( dhi ) ;
1359  bdds->clear_container() ;
1360 
1361  }
1362 
1363  catch(BESError & e) {
1364  if(cf_fileid !=-1)
1365  H5Fclose(cf_fileid);
1366  BESDEBUG(HDF5_NAME, prolog << "Caught BESError! Message: " << e.get_message() << endl);
1367  throw;
1368  }
1369  catch(InternalErr & e) {
1370  if(cf_fileid !=-1)
1371  H5Fclose(cf_fileid);
1372  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
1373  __FILE__, __LINE__);
1374  }
1375  catch(Error & e) {
1376  if(cf_fileid !=-1)
1377  H5Fclose(cf_fileid);
1378  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
1379  __FILE__, __LINE__);
1380  }
1381  catch(...) {
1382  if(cf_fileid !=-1)
1383  H5Fclose(cf_fileid);
1384  string s = "unknown exception caught building HDF5 DataDDS";
1385  throw BESInternalFatalError(s, __FILE__, __LINE__);
1386  }
1387 
1388  BESDEBUG(HDF5_NAME, prolog << "END" << endl);
1389  return true;
1390 }
1391 
1392 bool HDF5RequestHandler::hdf5_build_dmr(BESDataHandlerInterface & dhi)
1393 {
1394  BESDEBUG(HDF5_NAME, prolog << "BEGIN" << endl);
1395 #if DYNAMIC_CONFIG_ENABLED
1396  load_config();
1397 #endif
1398 
1399  // Extract the DMR Response object - this holds the DMR used by the
1400  // other parts of the framework.
1401  BESResponseObject *response = dhi.response_handler->get_response_object();
1402  BESDMRResponse &bes_dmr_response = dynamic_cast<BESDMRResponse &>(*response);
1403 
1404  string filename = dhi.container->access();
1405 
1406  DMR *dmr = bes_dmr_response.get_dmr();
1407 
1408  // For the time being, separate CF file ID from the default file ID(mainly for debugging)
1409  hid_t fileid = -1;
1410  hid_t cf_fileid = -1;
1411 
1412  try {
1413 
1414  DMR* cached_dmr_ptr = nullptr;
1415  if (dmr_cache){
1416  BESDEBUG(HDF5_NAME, prolog << "Checking DMR cache for : " << filename << endl);
1417  cached_dmr_ptr = static_cast<DMR*>(dmr_cache->get(filename));
1418  }
1419 
1420  if (cached_dmr_ptr) {
1421  // copy the cached DMR into the BES response object
1422  BESDEBUG(HDF5_NAME, prolog << "DMR cache hit for : " << filename << endl);
1423  *dmr = *cached_dmr_ptr; // Copy the referenced object
1424  dmr->set_request_xml_base(bes_dmr_response.get_request_xml_base());
1425  }
1426  else {// No cache
1427 
1428  H5Eset_auto2(H5E_DEFAULT,nullptr,nullptr);
1429  D4BaseTypeFactory MyD4TypeFactory;
1430  dmr->set_factory(&MyD4TypeFactory);
1431 
1432  if(true ==_usecf) {// CF option
1433 
1434  if(true == _usecfdmr) {
1435  cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
1436  if (cf_fileid < 0){
1437  string invalid_file_msg="Could not open this HDF5 file ";
1438  invalid_file_msg +=filename;
1439  invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
1440  invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
1441  invalid_file_msg +=" distributor.";
1442  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1443  }
1444  read_cfdmr(dmr,filename,cf_fileid);
1445  H5Fclose(cf_fileid);
1446  bes_dmr_response.set_dap4_constraint(dhi);
1447  bes_dmr_response.set_dap4_function(dhi);
1448  dmr->set_factory(nullptr);
1449 
1450  BESDEBUG(HDF5_NAME, prolog << "END" << endl);
1451 
1452  return true;
1453  }
1454 
1455  if(true == _pass_fileid)
1456  return hdf5_build_dmr_with_IDs(dhi);
1457 
1458  cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
1459  if (cf_fileid < 0){
1460  string invalid_file_msg="Could not open this HDF5 file ";
1461  invalid_file_msg +=filename;
1462  invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
1463  invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
1464  invalid_file_msg +=" distributor.";
1465  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1466  }
1467 
1468 
1469  BaseTypeFactory factory;
1470  DDS dds(&factory, name_path(filename), "3.2");
1471  dds.filename(filename);
1472 
1473  DAS das;
1474 
1475  // For the CF option dmr response, we need to map 64-bit integer separately
1476  // So set the flag to map 64-bit integer.
1477  HDF5RequestHandler::set_dmr_64bit_int(dmr);
1478  read_cfdds( dds,filename,cf_fileid);
1479  if (!dds.check_semantics()) { // DDS didn't comply with the DAP semantics
1480  dds.print(cerr);
1481  throw InternalErr(__FILE__, __LINE__,
1482  "DDS check_semantics() failed. This can happen when duplicate variable names are defined.");
1483  }
1484 
1485  read_cfdas(das,filename,cf_fileid);
1486  Ancillary::read_ancillary_das( das, filename ) ;
1487 
1488  dds.transfer_attributes(&das);
1489 
1491  if(cf_fileid !=-1)
1492  H5Fclose(cf_fileid);
1493 
1494  dmr->build_using_dds(dds);
1495 
1496  }// "if(true == _usecf)"
1497  else {// default option
1498 
1499  // Obtain the HDF5 file ID.
1500  fileid = get_fileid(filename.c_str());
1501  if (fileid < 0) {
1502  string invalid_file_msg="Could not open this HDF5 file ";
1503  invalid_file_msg +=filename;
1504  invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
1505  invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
1506  invalid_file_msg +=" distributor.";
1507  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1508  }
1509 
1510  bool use_dimscale = false;
1511  if(true == _default_handle_dimension)
1512  use_dimscale = check_dimscale(fileid);
1513  dmr->set_name(name_path(filename));
1514  dmr->set_filename(name_path(filename));
1515 
1516  // The breadth_first() function builds the variables and attributes and
1517  // loads them into the root group (building child groups as needed).
1518  // jhrg 4/30/20
1519  D4Group* root_grp = dmr->root();
1520  BESDEBUG("h5", "use_dimscale is "<< use_dimscale <<endl);
1521 
1522  // It is possible that a dimension variable has hardlinks. To make it
1523  // right for the netCDF-4 data model and the current DAP4 implementation,
1524  // we need to choose the shortest path of all hardlinks as the dimension path.
1525  // So to avoid iterate all HDF5 objects multiple times, save the found
1526  // hardlinks and search them when necessary. Note we have to search hardlinks from the root.
1527  // KY 2021-11-15
1528  vector<link_info_t> hdf5_hls;
1529  breadth_first(fileid, fileid,(char*)"/",root_grp,filename.c_str(),use_dimscale,hdf5_hls);
1530 #if 0
1531  BESDEBUG("h5", "build_dmr - before obtain dimensions"<< endl);
1532  D4Dimensions *root_dims = root_grp->dims();
1533  for(D4Dimensions::D4DimensionsIter di = root_dims->dim_begin(), de = root_dims->dim_end(); di != de; ++di) {
1534  BESDEBUG("fonc", "transform_dap4() - check dimensions"<< endl);
1535  BESDEBUG("fonc", "transform_dap4() - dim name is: "<<(*di)->name()<<endl);
1536  BESDEBUG("fonc", "transform_dap4() - dim size is: "<<(*di)->size()<<endl);
1537  BESDEBUG("fonc", "transform_dap4() - fully_qualfied_dim name is: "<<(*di)->fully_qualified_name()<<endl);
1538  //cout <<"dim size is: "<<(*di)->size()<<endl;
1539  //cout <<"dim fully_qualified_name is: "<<(*di)->fully_qualified_name()<<endl;
1540  }
1541  BESDEBUG("h5", "build_dmr - after obtain dimensions"<< endl);
1542 #endif
1543 
1544 #if 0
1545  if(true == use_dimscale)
1546  //breadth_first(fileid,(char*)"/",*dmr,root_grp,filename.c_str(),true);
1547  breadth_first(fileid,(char*)"/",root_grp,filename.c_str(),true);
1548  else
1549  depth_first(fileid,(char*)"/",root_grp,filename.c_str());
1550  //depth_first(fileid,(char*)"/",*dmr,root_grp,filename.c_str());
1551 #endif
1552 
1553  close_fileid(fileid);
1554 
1555  }// else (default option)
1556 
1557  // If the cache is turned on, add the memory cache.
1558  if (dmr_cache) {
1559  // add a copy
1560  BESDEBUG(HDF5_NAME, prolog << "DMR added to the cache for : " << filename << endl);
1561  dmr_cache->add(new DMR(*dmr), filename);
1562  }
1563  }// else no cache
1564  }// try
1565  catch(BESError & e) {
1566  if(cf_fileid !=-1)
1567  H5Fclose(cf_fileid);
1568  if(fileid !=-1)
1569  H5Fclose(fileid);
1570  BESDEBUG(HDF5_NAME, prolog << "Caught BESError! Message: " << e.get_message() << endl);
1571  throw;
1572  }
1573  catch(InternalErr & e) {
1574 
1575  if(cf_fileid !=-1)
1576  H5Fclose(cf_fileid);
1577  if(fileid !=-1)
1578  H5Fclose(fileid);
1579 
1580  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
1581  __FILE__, __LINE__);
1582  }
1583  catch(Error & e) {
1584 
1585  if(cf_fileid !=-1)
1586  H5Fclose(cf_fileid);
1587  if(fileid !=-1)
1588  H5Fclose(fileid);
1589  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
1590  __FILE__, __LINE__);
1591  }
1592  catch(...) {
1593 
1594  if(cf_fileid !=-1)
1595  H5Fclose(cf_fileid);
1596  if(fileid !=-1)
1597  H5Fclose(fileid);
1598  string s = "unknown exception caught building HDF5 DMR";
1599  throw BESInternalFatalError(s, __FILE__, __LINE__);
1600  }
1601 
1602 #if 0
1603  //dmr->print(cout);
1604 #endif
1605 
1606  // Instead of fiddling with the internal storage of the DHI object,
1607  // (by setting dhi.data[DAP4_CONSTRAINT], etc., directly) use these
1608  // methods to set the constraints. But, why? Ans: from Patrick is that
1609  // in the 'container' mode of BES each container can have a different
1610  // CE.
1611  bes_dmr_response.set_dap4_constraint(dhi);
1612  bes_dmr_response.set_dap4_function(dhi);
1613  dmr->set_factory(nullptr);
1614 
1615  BESDEBUG(HDF5_NAME, prolog << "END" << endl);
1616  return true;
1617 }
1618 
1619 // This function is only used when EnableCF is true.
1620 bool HDF5RequestHandler::hdf5_build_dmr_with_IDs(BESDataHandlerInterface & dhi)
1621 {
1622  BESDEBUG(HDF5_NAME, prolog << "BEGIN" << endl);
1623 #if DYNAMIC_CONFIG_ENABLED
1624  load_config();
1625 #endif
1626 
1627  BESDEBUG("h5","Building DMR with passing file IDs. "<<endl);
1628  string filename = dhi.container->access();
1629  hid_t cf_fileid = -1;
1630 
1631  H5Eset_auto2(H5E_DEFAULT,nullptr,nullptr);
1632  cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
1633  if (cf_fileid < 0){
1634  string invalid_file_msg="Could not open this HDF5 file ";
1635  invalid_file_msg +=filename;
1636  invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
1637  invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
1638  invalid_file_msg +=" distributor.";
1639  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1640  }
1641 
1642  BaseTypeFactory factory;
1643  DDS dds(&factory, name_path(filename), "3.2");
1644  dds.filename(filename);
1645 
1646  DAS das;
1647 
1648  try {
1649 
1650 
1651  // This is the CF option
1652  read_cfdds( dds,filename,cf_fileid);
1653 
1654  if (!dds.check_semantics()) { // DDS didn't comply with the DAP semantics
1655  dds.print(cerr);
1656  throw InternalErr(__FILE__, __LINE__,
1657  "DDS check_semantics() failed. This can happen when duplicate variable names are defined.");
1658  }
1659 
1660  Ancillary::read_ancillary_dds( dds, filename ) ;
1661 
1662 
1663  read_cfdas(das,filename,cf_fileid);
1664 
1665  Ancillary::read_ancillary_das( das, filename ) ;
1666 
1667  dds.transfer_attributes(&das);
1668 
1670 #if 0
1671  //if(cf_fileid !=-1)
1672  // H5Fclose(cf_fileid);
1673 #endif
1674 
1675  }
1676  catch(BESError & e) {
1677  if(cf_fileid !=-1)
1678  H5Fclose(cf_fileid);
1679  BESDEBUG(HDF5_NAME, prolog << "Caught BESError! Message: " << e.get_message() << endl);
1680  throw;
1681  }
1682  catch(InternalErr & e) {
1683 
1684  if(cf_fileid !=-1)
1685  H5Fclose(cf_fileid);
1686 
1687  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
1688  __FILE__, __LINE__);
1689  }
1690  catch(Error & e) {
1691 
1692  if(cf_fileid !=-1)
1693  H5Fclose(cf_fileid);
1694 
1695  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
1696  __FILE__, __LINE__);
1697  }
1698  catch(...) {
1699 
1700  if(cf_fileid !=-1)
1701  H5Fclose(cf_fileid);
1702 
1703  string s = "unknown exception caught building HDF5 DataDDS";
1704  throw BESInternalFatalError(s, __FILE__, __LINE__);
1705  }
1706 
1707  // Extract the DMR Response object - this holds the DMR used by the
1708  // other parts of the framework.
1709  BESResponseObject *response = dhi.response_handler->get_response_object();
1710  BESDMRResponse &bes_dmr = dynamic_cast<BESDMRResponse &>(*response);
1711 
1712  // In this handler we use a different pattern since the handler specializes the DDS/DMR.
1713  // First, build the DMR adding the open handle to the HDF4 dataset, then free the DMR
1714  // the BES built and add this one. The HDF4DMR object will close the open dataset when
1715  // the BES runs the DMR's destructor.
1716 
1717  DMR *dmr = bes_dmr.get_dmr();
1718  D4BaseTypeFactory MyD4TypeFactory;
1719  dmr->set_factory(&MyD4TypeFactory);
1720  dmr->build_using_dds(dds);
1721 
1722  auto hdf5_dmr = new HDF5DMR(dmr);
1723  hdf5_dmr->setHDF5Dataset(cf_fileid);
1724  delete dmr; // The call below will make 'dmr' unreachable; delete it now to avoid a leak.
1725  bes_dmr.set_dmr(hdf5_dmr); // BESDMRResponse will delete hdf5_dmr
1726 
1727  // Instead of fiddling with the internal storage of the DHI object,
1728  // (by setting dhi.data[DAP4_CONSTRAINT], etc., directly) use these
1729  // methods to set the constraints. But, why? Ans: from Patrick is that
1730  // in the 'container' mode of BES each container can have a different
1731  // CE.
1732  bes_dmr.set_dap4_constraint(dhi);
1733  bes_dmr.set_dap4_function(dhi);
1734  hdf5_dmr->set_factory(nullptr);
1735 
1736  BESDEBUG(HDF5_NAME, prolog << "END" << endl);
1737  return true;
1738 }
1739 
1740 bool HDF5RequestHandler::hdf5_build_help(BESDataHandlerInterface & dhi)
1741 {
1742  BESResponseObject *response = dhi.response_handler->get_response_object();
1743  BESInfo *info = dynamic_cast<BESInfo *>(response);
1744  if( !info )
1745  throw BESInternalError( "cast error", __FILE__, __LINE__ ) ;
1746 
1747  string add_info="Just for Test";
1748 
1749  map<string,string> attrs ;
1750  attrs["name"] = MODULE_NAME ;
1751  attrs["version"] = MODULE_VERSION ;
1752  list<string> services ;
1753  BESServiceRegistry::TheRegistry()->services_handled( HDF5_NAME, services );
1754  if( services.empty()==false )
1755  {
1756  string handles = BESUtil::implode( services, ',' ) ;
1757  attrs["handles"] = handles ;
1758  }
1759  info->begin_tag( "module", &attrs ) ;
1760  info->end_tag( "module" ) ;
1761  info->add_data(add_info);
1762 
1763  return true;
1764 }
1765 
1766 bool HDF5RequestHandler::hdf5_build_version(BESDataHandlerInterface & dhi)
1767 {
1768  BESResponseObject *response = dhi.response_handler->get_response_object();
1769  BESVersionInfo *info = dynamic_cast < BESVersionInfo * >(response);
1770  if( !info )
1771  throw BESInternalError( "cast error", __FILE__, __LINE__ ) ;
1772 
1773  info->add_module( MODULE_NAME, MODULE_VERSION ) ;
1774 
1775  return true;
1776 }
1777 
1778 
1779 bool HDF5RequestHandler::obtain_lrd_common_cache_dirs()
1780 {
1781  string lrd_config_fpath;
1782  string lrd_config_fname;
1783 
1784  // Obtain DataCache path
1785  lrd_config_fpath = get_beskeys("H5.DataCachePath");
1786 
1787  // Obtain the configure file name that specifics the large file configuration
1788  lrd_config_fname = get_beskeys("H5.LargeDataMemCacheFileName");
1789 
1790  // If either the configure file path or fname is missing, won't add specific mem. cache dirs.
1791  if(lrd_config_fpath=="" || lrd_config_fname=="")
1792  return false;
1793 
1794  // temp_line for storing info of one line in the config. file
1795  string temp_line;
1796 
1797  // The full path of the configure file
1798  string mcache_config_fname = lrd_config_fpath+"/"+lrd_config_fname;
1799 
1800 #if 0
1801  //ifstream mcache_config_file("example.txt");
1802 #endif
1803  // Open the configure file
1804  ifstream mcache_config_file(mcache_config_fname.c_str());
1805 
1806  // If the configuration file is not open, return false.
1807  if(mcache_config_file.is_open()==false){
1808  BESDEBUG(HDF5_NAME, prolog << "The large data memory cache configure file "<<mcache_config_fname );
1809  BESDEBUG(HDF5_NAME, prolog << " cannot be opened."<<endl);
1810  return false;
1811  }
1812 
1813  // Read the configuration file line by line
1814  while(getline(mcache_config_file,temp_line)) {
1815 
1816  // Only consider lines that is no less than 2 characters and the 2nd character is space.
1817  if(temp_line.size()>1 && temp_line.at(1)==' ') {
1818  char sep=' ';
1819  string subline = temp_line.substr(2);
1820  vector<string> temp_name_list;
1821 
1822  // Include directories to store common latitude and longitude values
1823  if(temp_line.at(0)=='1') {
1824  HDF5CFUtil::Split_helper(temp_name_list,subline,sep);
1825 #if 0
1826  //lrd_cache_dir_list +=temp_name_list;
1827 #endif
1828  lrd_cache_dir_list.insert(lrd_cache_dir_list.end(),temp_name_list.begin(),temp_name_list.end());
1829  }
1830  // Include directories not to store common latitude and longitude values
1831  else if(temp_line.at(0)=='0'){
1832  HDF5CFUtil::Split_helper(temp_name_list,subline,sep);
1833 #if 0
1834  //lrd_non_cache_dir_list +=temp_name_list;
1835 #endif
1836  lrd_non_cache_dir_list.insert(lrd_non_cache_dir_list.end(),temp_name_list.begin(),temp_name_list.end());
1837  }
1838  // Include variable names that the server would like to store in the memory cache
1839  else if(temp_line.at(0)=='2') {
1840 
1841  // We need to handle the space case inside a variable path
1842  // either "" or '' needs to be used to identify a var path
1843  vector<int>dq_pos;
1844  vector<int>sq_pos;
1845  for(unsigned int i = 0; i<subline.size();i++){
1846  if(subline[i]=='"') {
1847  dq_pos.push_back(i);
1848  }
1849  else if(subline[i]=='\'')
1850  sq_pos.push_back(i);
1851  }
1852  if(dq_pos.empty() && sq_pos.empty())
1853  HDF5CFUtil::Split_helper(temp_name_list,subline,sep);
1854  else if((dq_pos.size()!=0) &&(dq_pos.size()%2==0)&& sq_pos.size()==0) {
1855  unsigned int dq_index= 0;
1856  while(dq_index < dq_pos.size()){
1857  if(dq_pos[dq_index+1]>(dq_pos[dq_index]+1)) {
1858  temp_name_list.push_back
1859  (subline.substr(dq_pos[dq_index]+1,dq_pos[dq_index+1]-dq_pos[dq_index]-1));
1860  }
1861  dq_index = dq_index + 2;
1862  }
1863  }
1864  else if((sq_pos.size()!=0) &&(sq_pos.size()%2==0)&& dq_pos.size()==0) {
1865  unsigned int sq_index= 0;
1866  while(sq_index < sq_pos.size()){
1867  if(sq_pos[sq_index+1]>(sq_pos[sq_index]+1)) {
1868  temp_name_list.push_back
1869  (subline.substr(sq_pos[sq_index]+1,sq_pos[sq_index+1]-sq_pos[sq_index]-1));
1870  }
1871  sq_index = sq_index+2;
1872  }
1873  }
1874 
1875  lrd_var_cache_file_list.insert(lrd_var_cache_file_list.end(),temp_name_list.begin(),temp_name_list.end());
1876  }
1877  }
1878  }
1879 
1880 
1881 #if 0
1882 
1883 for(int i =0; i<lrd_cache_dir_list.size();i++)
1884 cerr<<"lrd cache list is "<<lrd_cache_dir_list[i] <<endl;
1885 for(int i =0; i<lrd_non_cache_dir_list.size();i++)
1886 cerr<<"lrd non cache list is "<<lrd_non_cache_dir_list[i] <<endl;
1887 for(int i =0; i<lrd_var_cache_file_list.size();i++)
1888 cerr<<"lrd var cache file list is "<<lrd_var_cache_file_list[i] <<endl;
1889 #endif
1890 
1891 
1892  mcache_config_file.close();
1893  if(lrd_cache_dir_list.empty() && lrd_non_cache_dir_list.empty() && lrd_var_cache_file_list.empty())
1894  return false;
1895  else
1896  return true;
1897 }
1898 
1899 
1900 bool HDF5RequestHandler::read_das_from_disk_cache(const string & cache_filename,DAS *das_ptr) {
1901 
1902  BESDEBUG(HDF5_NAME, prolog << "Coming to read_das_from_disk_cache() " << cache_filename << endl);
1903  bool ret_value = true;
1904  FILE *md_file = nullptr;
1905  md_file = fopen(cache_filename.c_str(),"rb");
1906 
1907  if(nullptr == md_file) {
1908  string bes_error = "An error occurred trying to open a metadata cache file " + cache_filename;
1909  throw BESInternalError( bes_error, __FILE__, __LINE__);
1910  }
1911  else {
1912 
1913  int fd_md = fileno(md_file);
1914  struct flock *l_md;
1915  l_md = lock(F_RDLCK);
1916 
1917  // hold a read(shared) lock to read metadata from a file.
1918  if(fcntl(fd_md,F_SETLKW,l_md) == -1) {
1919  fclose(md_file);
1920  ostringstream oss;
1921  oss << "cache process: " << l_md->l_pid << " triggered a locking error: " << get_errno();
1922  throw BESInternalError( oss.str(), __FILE__, __LINE__);
1923  }
1924 
1925  try {
1926 
1927  struct stat sb;
1928  if(stat(cache_filename.c_str(),&sb) != 0) {
1929  string bes_error = "An error occurred trying to stat a metadata cache file size " + cache_filename;
1930  throw BESInternalError( bes_error, __FILE__, __LINE__);
1931 
1932  }
1933 
1934 
1935  auto bytes_expected_read=(size_t)sb.st_size;
1936  BESDEBUG(HDF5_NAME, prolog << "DAS Disk cache file size is " << bytes_expected_read << endl);
1937 
1938  vector<char> buf;
1939  buf.resize(bytes_expected_read);
1940  size_t bytes_to_read =fread((void*)buf.data(),1,bytes_expected_read,md_file);
1941  if(bytes_to_read != bytes_expected_read)
1942  throw InternalErr(__FILE__,__LINE__,"Fail to read the data from the das cache file.");
1943 
1944  char* temp_pointer =buf.data();
1945 
1946  AttrTable*at = nullptr;
1947  // recursively build DAS
1948 //#if 0
1949  temp_pointer = get_attr_info_from_dc(temp_pointer,das_ptr,at);
1950 //#endif
1951 
1952 
1953  }
1954  catch(...) {
1955  if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
1956  fclose(md_file);
1957  throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1958  }
1959 
1960  fclose(md_file);
1961  throw InternalErr(__FILE__,__LINE__,"Fail to parse a das cache file.");
1962  }
1963 
1964  // Unlock the cache file
1965  if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
1966  fclose(md_file);
1967  throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1968  }
1969  fclose(md_file);
1970  }
1971  return ret_value;
1972 
1973 }
1974 
1975 // This fucntion will NOT be used by default. Leave here for future improvement.
1976 bool HDF5RequestHandler::write_dds_to_disk_cache(const string& dds_cache_fname,DDS *dds_ptr) {
1977 
1978  BESDEBUG(HDF5_NAME, prolog << "Write DDS to disk cache " << dds_cache_fname << endl);
1979  FILE *dds_file = fopen(dds_cache_fname.c_str(),"w");
1980 
1981  if(nullptr == dds_file) {
1982  string bes_error = "An error occurred trying to open a metadata cache file " + dds_cache_fname;
1983  throw BESInternalError( bes_error, __FILE__, __LINE__);
1984  }
1985  else {
1986 
1987  int fd_md = fileno(dds_file);
1988  struct flock *l_md;
1989  l_md = lock(F_WRLCK);
1990 
1991  // hold a read(shared) lock to read metadata from a file.
1992  if(fcntl(fd_md,F_SETLKW,l_md) == -1) {
1993  fclose(dds_file);
1994  ostringstream oss;
1995  oss << "cache process: " << l_md->l_pid << " triggered a locking error: " << get_errno();
1996  throw BESInternalError( oss.str(), __FILE__, __LINE__);
1997  }
1998 
1999  try {
2000  dds_ptr->print(dds_file);
2001  }
2002  catch(...) {
2003  if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
2004  fclose(dds_file);
2005  throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
2006  }
2007 
2008  fclose(dds_file);
2009  throw InternalErr(__FILE__,__LINE__,"Fail to parse a dds cache file.");
2010  }
2011 
2012  if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
2013  fclose(dds_file);
2014  throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
2015  }
2016 
2017  fclose(dds_file);
2018  }
2019  return true;
2020 
2021 }
2022 
2023 // Write DAS to a binary cached file on disk.
2024 bool HDF5RequestHandler::write_das_to_disk_cache(const string & das_cache_fname, DAS *das_ptr) {
2025 
2026  BESDEBUG(HDF5_NAME, prolog << "Write DAS to disk cache " << das_cache_fname << endl);
2027  FILE *das_file = fopen(das_cache_fname.c_str(),"wb");
2028  if(nullptr == das_file) {
2029  string bes_error = "An error occurred trying to open a metadata cache file " + das_cache_fname;
2030  throw BESInternalError( bes_error, __FILE__, __LINE__);
2031  }
2032  else {
2033  int fd_md = fileno(das_file);
2034  struct flock *l_md;
2035  l_md = lock(F_WRLCK);
2036 
2037  // hold a write(exclusive) lock to write metadata to a file.
2038  if(fcntl(fd_md,F_SETLKW,l_md) == -1) {
2039  fclose(das_file);
2040  ostringstream oss;
2041  oss << "cache process: " << l_md->l_pid << " triggered a locking error: " << get_errno();
2042  throw BESInternalError( oss.str(), __FILE__, __LINE__);
2043  }
2044 
2045  try {
2046  write_das_to_file(das_ptr,das_file);
2047  }
2048  catch(...) {
2049  if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
2050  fclose(das_file);
2051  throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
2052  }
2053 
2054  fclose(das_file);
2055  throw InternalErr(__FILE__,__LINE__,"Fail to parse a dds cache file.");
2056  }
2057 
2058  if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
2059  fclose(das_file);
2060  throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
2061  }
2062 
2063  fclose(das_file);
2064 
2065  }
2066 
2067  return true;
2068 
2069 }
2070 
2071 // The wrapper function to call write_das_table_to_file to generate the cache.
2072 void write_das_to_file(DAS*das_ptr,FILE* das_file) {
2073 
2074  // When category_flag is 2, it marks the end of the file.
2075  uint8_t category_flag = 2;
2076  AttrTable* top_table = das_ptr->get_top_level_attributes();
2077  write_das_table_to_file(top_table,das_file);
2078 
2079  // Add the final ending flag for retrieving the info.
2080  fwrite((const void*)&category_flag,1,1,das_file);
2081  return;
2082 
2083 }
2084 
2085 // The main function to write DAS to a file
2086 void write_das_table_to_file(AttrTable*temp_table,FILE* das_file) {
2087 
2088  if(temp_table !=nullptr) {
2089 
2090  // 2 is the end mark of an attribute table
2091  uint8_t category_flag = 2;
2092 
2093  // Loop through the whole DAS top table
2094  AttrTable::Attr_iter top_startit = temp_table->attr_begin();
2095  AttrTable::Attr_iter top_endit = temp_table->attr_end();
2096  AttrTable::Attr_iter top_it = top_startit;
2097  while(top_it !=top_endit) {
2098  AttrType atype = temp_table->get_attr_type(top_it);
2099  if(atype == Attr_unknown)
2100  throw InternalErr(__FILE__,__LINE__,"Unsupported DAS Attribute type");
2101  else if(atype!=Attr_container) {
2102  BESDEBUG(HDF5_NAME, prolog << "DAS to the disk cache, attr name is: "
2103  << temp_table->get_name(top_it) << endl);
2104  BESDEBUG(HDF5_NAME, prolog << "DAS to the disk cache, attr type is: "
2105  << temp_table->get_type(top_it) << endl);
2106  // For the debugging purpose
2107 #if 0
2108  //unsigned int num_attrs = temp_table->get_attr_num(temp_table->get_name(top_it));
2109  //cerr<<"Attribute values are "<<endl;
2110  //for (int i = 0; i <num_attrs;i++)
2111  // cerr<<(*(temp_table->get_attr_vector(temp_table->get_name(top_it))))[i]<<" ";
2112  //cerr<<endl;
2113  //write_das_attr_info(temp_table,top_it,das_file);
2114 #endif
2115  // Write DAS attribute info to the file
2116  write_das_attr_info(temp_table,temp_table->get_name(top_it),temp_table->get_type(top_it),das_file);
2117  }
2118  else {
2119  BESDEBUG(HDF5_NAME, prolog << "DAS to the disk cache, attr container name is: "
2120  << (*top_it)->name << endl);
2121  // Write the container and then write the info. in this container
2122  AttrTable* sub_table = temp_table->get_attr_table(top_it);
2123  write_container_name_to_file(sub_table->get_name(),das_file);
2124  write_das_table_to_file(sub_table,das_file);
2125 
2126  // Write the end flag
2127  fwrite((const void*)&category_flag,1,1,das_file);
2128 
2129  }
2130  ++top_it;
2131  }
2132 
2133  }
2134 }
2135 
2136 // Write container name to the disk file
2137 void write_container_name_to_file(const string& cont_name,FILE *das_file) {
2138 
2139  // 1 marks the starting of a container
2140  uint8_t category_flag = 1;
2141  vector<char> buf;
2142  size_t bytes_to_write = cont_name.size()+sizeof(size_t)+1;
2143  buf.resize(bytes_to_write);
2144  char*temp_pointer =buf.data();
2145  memcpy((void*)temp_pointer,(void*)&category_flag,1);
2146  temp_pointer++;
2147  temp_pointer=copy_str(temp_pointer,cont_name);
2148 
2149  size_t bytes_to_be_written = fwrite((const void*)buf.data(),1,bytes_to_write,das_file);
2150  if(bytes_to_be_written != bytes_to_write)
2151  throw InternalErr(__FILE__, __LINE__,"Failed to write a DAS container name to a cache");
2152  return;
2153 }
2154 
2155 
2156 // Write DAS attribute info. to the disk cache file
2157 void write_das_attr_info(AttrTable* dtp,const string& attr_name, const string & attr_type,FILE * das_file) {
2158 
2159  // 0 marks the starting of a DAS attribute
2160  uint8_t category_flag = 0;
2161 
2162  unsigned int num_attr_elems = dtp->get_attr_num(attr_name);
2163  vector<string> attr_values;
2164  size_t total_attr_values_size = 0;
2165  for (unsigned int i = 0; i <num_attr_elems;i++){
2166  attr_values.push_back((*(dtp->get_attr_vector(attr_name)))[i]);
2167  total_attr_values_size += attr_values[i].size();
2168  }
2169  // Need to add a flag, value as 0 to indicate the attribute.
2170  // DAS: category flag, sizeof attirubte name, attribute name, size of attribute type, attribute type
2171  size_t bytes_to_write_attr = 1 + attr_name.size() + attr_type.size() + 2* sizeof(size_t);
2172 
2173  // One unsigned int to store the number of element elements i
2174  // + sizeof(size_t) * number of elements to store the number of characters for each attribute
2175  // (in DAP, every attribute is in string format)
2176  // +total size of all attribute values
2177  bytes_to_write_attr += sizeof(unsigned int) + num_attr_elems*sizeof(size_t)+total_attr_values_size;
2178 
2179  vector<char>attr_buf;
2180  attr_buf.resize(bytes_to_write_attr);
2181  char* temp_attrp =attr_buf.data();
2182 
2183  // The attribute flag
2184  memcpy((void*)temp_attrp,(void*)&category_flag,1);
2185  temp_attrp++;
2186 
2187  // The attribute name and type
2188  temp_attrp=copy_str(temp_attrp,attr_name);
2189  temp_attrp=copy_str(temp_attrp,attr_type);
2190 
2191  // Number of elements
2192  memcpy((void*)temp_attrp,(void*)&num_attr_elems,sizeof(unsigned int));
2193  temp_attrp+=sizeof(unsigned int);
2194 
2195  // All attributes
2196  for (unsigned int i = 0; i <num_attr_elems;i++)
2197  temp_attrp=copy_str(temp_attrp,(*(dtp->get_attr_vector(attr_name)))[i]);
2198 
2199  size_t bytes_to_be_written = fwrite((const void*)attr_buf.data(),1,bytes_to_write_attr,das_file);
2200  if(bytes_to_be_written != bytes_to_write_attr)
2201  throw InternalErr(__FILE__, __LINE__,"Failed to write a DAS attribute to a cache");
2202 
2203  return;
2204 
2205 }
2206 
2207 // Read DDS from a disk cache, this function is not used by default.
2208 void HDF5RequestHandler::read_dds_from_disk_cache(BESDDSResponse* bdds, BESDataDDSResponse* data_bdds,
2209  bool build_data,const string & container_name,const string & h5_fname,
2210  const string & dds_cache_fname,const string &das_cache_fname, hid_t h5_fd,
2211  bool das_from_dc) {
2212 
2213 
2214  BESDEBUG(HDF5_NAME, prolog << "BEGIN dds_cache_fname: " << dds_cache_fname << endl);
2215 
2216  DDS *dds;
2217  if(true == build_data)
2218  dds = data_bdds->get_dds();
2219  else
2220  dds = bdds->get_dds();
2221 
2222  // write a function to pass the following with the lock.
2223  BaseTypeFactory tf;
2224  DDS tdds(&tf,name_path(h5_fname),"3.2");
2225  tdds.filename(h5_fname);
2226 
2227  FILE *dds_file = fopen(dds_cache_fname.c_str(),"r");
2228  tdds.parse(dds_file);
2229  auto cache_dds = new DDS(tdds);
2230 #if 0
2231 cerr<<"before dds "<<endl;
2232 dds->dump(cerr);
2233 cerr<<"after dds "<<endl;
2234 cerr<<"before tdds "<<endl;
2235 cache_dds->dump(cerr);
2236 cerr<<"after tdds "<<endl;
2237 #endif
2238  if(dds != nullptr)
2239  delete dds;
2240 
2241  Ancillary::read_ancillary_dds( *cache_dds, h5_fname ) ;
2242 
2243  add_das_to_dds(cache_dds,container_name,h5_fname,das_cache_fname,h5_fd,das_from_dc);
2244  if(true == build_data)
2245  data_bdds->set_dds(cache_dds);
2246  else
2247  bdds->set_dds(cache_dds);
2248  if(dds_file !=nullptr)
2249  fclose(dds_file);
2250 
2251  if (dds_cache) {
2252  // add a copy
2253  BESDEBUG(HDF5_NAME, prolog << "For memory cache, DDS added to the cache for : " << h5_fname << endl);
2254  dds_cache->add(new DDS(*cache_dds), h5_fname);
2255  }
2256 
2257 }
2258 
2259 // Add DAS to DDS.
2260 void HDF5RequestHandler::add_das_to_dds(DDS *dds, const string &/*container_name*/, const string &filename,
2261  const string &das_cache_fname, hid_t h5_fd, bool das_from_dc) {
2262 
2263  BESDEBUG(HDF5_NAME, prolog << "BEGIN" << endl);
2264 
2265  // Check DAS memory cache
2266  DAS *das = nullptr ;
2267  bool use_das_cache = false;
2268  if (das_cache)
2269  das = static_cast<DAS*>(das_cache->get(filename));
2270  if (das)
2271  use_das_cache = true;
2272 
2273  if (true == use_das_cache) {
2274  BESDEBUG(HDF5_NAME, prolog << "DAS Cached hit for : " << filename << endl);
2275  dds->transfer_attributes(das); // no need to copy the cached DAS
2276  }
2277 
2278  else {
2279 
2280  das = new DAS ;
2281 
2282  // The following block is commented out because the attribute containers in DDX disappear
2283  // when the container_name of DAS is added. Without adding the container_name of DAS,
2284  // the attribute containers show up in DDX. This information is re-discovered while working on
2285  // https://bugs.earthdata.nasa.gov/browse/HYRAX-714 although the following code was commented
2286  // out long time ago. KY 2022-05-27
2287 #if 0
2288  if (!container_name.empty())
2289  das->container_name(container_name);
2290 #endif
2291  if(das_from_dc == true)
2292  read_das_from_disk_cache(das_cache_fname,das);
2293  else {
2294  // This bool is for the case, when DDS is read from a cache then we need to open the HDF5 file.
2295  bool h5_file_open = true;
2296  if(h5_fd == -1)
2297  h5_file_open = false;
2298  if (true == _usecf) {
2299  // go to the CF option
2300  if(h5_file_open == false)
2301  h5_fd = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
2302 
2303  read_cfdas( *das,filename,h5_fd);
2304  if(h5_file_open == false)
2305  H5Fclose(h5_fd);
2306  }
2307  else {
2308  if(h5_file_open == false)
2309  h5_fd = get_fileid(filename.c_str());
2310  find_gloattr(h5_fd, *das);
2311  depth_first(h5_fd, "/", *das);
2312  if(h5_file_open == false)
2313  close_fileid(h5_fd);
2314  }
2315 
2316  Ancillary::read_ancillary_das( *das, filename ) ;
2317 
2318  if(das_cache_fname!="" && das_from_dc == false)
2319  write_das_to_disk_cache(das_cache_fname,das);
2320  }
2321 
2322  dds->transfer_attributes(das);
2323 
2324  if (das_cache) {
2325  // add a copy
2326  BESDEBUG(HDF5_NAME, prolog << "For memory cache, DAS added to the cache for : " << filename << endl);
2327  das_cache->add(new DAS(*das), filename);
2328  }
2329  delete das;
2330 
2331  }
2332 
2333 }
2334 
2335 bool obtain_beskeys_info(const string& key, bool & has_key) {
2336 
2337  bool ret_value = false;
2338  string doset ="";
2339  TheBESKeys::TheKeys()->get_value( key, doset, has_key ) ;
2340  if(has_key) {
2341  const string dosettrue ="true";
2342  const string dosetyes = "yes";
2343  doset = BESUtil::lowercase(doset) ;
2344  ret_value = (dosettrue == doset || dosetyes == doset);
2345  }
2346  return ret_value;
2347 }
2348 
2349 #if 0
2350 bool is_beskeys_set_true(const string& bes_value) {
2351 
2352  const string dosettrue ="true";
2353  const string dosetyes = "yes";
2354  string doset = BESUtil::lowercase(bes_value) ;
2355  return (dosettrue == doset || dosetyes == doset);
2356 
2357 }
2358 bool check_and_set_beskeys(const string key) {
2359 
2360  bool found = false;
2361  string doset ="";
2362  const string dosettrue ="true";
2363  const string dosetyes = "yes";
2364 
2365  TheBESKeys::TheKeys()->get_value( key, doset, found ) ;
2366  if( true == found ) {
2367  doset = BESUtil::lowercase( doset ) ;
2368  }
2369  BESDEBUG(HDF5_NAME, prolog << "Key: " << key << (found?(" was found. value: "+doset):" was not found.") << endl);
2370  return found && (dosettrue == doset || dosetyes == doset);
2371 
2372 }
2373 #endif
2374 
2375 // get_uint_key and get_float_key are copied from the netCDF handler.
2376 
2377 static unsigned int get_uint_key(const string &key, unsigned int def_val)
2378 {
2379  bool found = false;
2380  string doset = "";
2381 
2382  TheBESKeys::TheKeys()->get_value(key, doset, found);
2383  if (true == found) {
2384  // In C++11, stoi is better.
2385  return atoi(doset.c_str()); // use better code TODO
2386  }
2387  else {
2388  return def_val;
2389  }
2390 }
2391 
2392 static unsigned long get_ulong_key(const string &key, unsigned long def_val)
2393 {
2394  bool found = false;
2395  string doset = "";
2396 
2397  TheBESKeys::TheKeys()->get_value(key, doset, found);
2398  if (true == found) {
2399  // In C++11, stoull is better.
2400  return atol(doset.c_str()); // use better code TODO
2401  }
2402  else {
2403  return def_val;
2404  }
2405 }
2406 static float get_float_key(const string &key, float def_val)
2407 {
2408  bool found = false;
2409  string doset = "";
2410 
2411  TheBESKeys::TheKeys()->get_value(key, doset, found);
2412  if (true == found) {
2413  return atof(doset.c_str()); // use better code TODO
2414  }
2415  else {
2416  return def_val;
2417  }
2418 }
2419 
2420 static string get_beskeys(const string &key) {
2421 
2422  bool found = false;
2423  string ret_value ="";
2424 
2425  TheBESKeys::TheKeys()->get_value( key, ret_value, found ) ;
2426  return ret_value;
2427 
2428 }
2429 
2430 // The function to copy a string to a memory buffer.
2431 char* copy_str(char*temp_ptr,const string & str) {
2432 
2433  size_t str_size=str.size();
2434  memcpy((void*)temp_ptr,(void*)&str_size,sizeof(size_t));
2435  temp_ptr+=sizeof(size_t);
2436  vector<char>temp_vc2(str.begin(),str.end());
2437  memcpy((void*)temp_ptr,(void*)temp_vc2.data(),str.size());
2438  temp_ptr+=str.size();
2439  return temp_ptr;
2440 
2441 }
2442 
2443 
2444 // Obtain the string from a memory buffer.
2445 // Note: both char* and string(as a reference) will be returned
2446 // The attribute binary first stores the size of the string, then the string itself
2447 char* obtain_str(char*temp_ptr,string & str) {
2448 
2449  size_t oname_size = *((size_t *)temp_ptr);
2450  temp_ptr = temp_ptr + sizeof(size_t);
2451  string oname;
2452  for(unsigned int i =0; i<oname_size; i++){
2453  oname.push_back(*temp_ptr);
2454  ++temp_ptr;
2455  }
2456  str = oname;
2457  return temp_ptr;
2458 
2459 }
2460 
2461 // For our case, there are no global attributes for DAS.
2462 // The global attribures are always under HDF_GLOBAL.
2463 // The main function to obtain the DAS info. from the cache.
2464 char* get_attr_info_from_dc(char*temp_pointer,DAS *das,AttrTable *at_par) {
2465 
2466  // 3 is only for the code to come into the loop.
2467  uint8_t flag =3;
2468  while(flag !=2) {
2469  flag = *((uint8_t*)(temp_pointer));
2470  BESDEBUG(HDF5_NAME, prolog << "Build DAS from the disk cache file flag: "
2471  <<" flag = 0, attribute; flag = 1, container; flag =2; end of container;"
2472  <<" flag = 3; the initial value to get the attribute retrieval process started."
2473  <<" The flag value is "
2474  << (int)flag <<endl);
2475  temp_pointer++;
2476 
2477  if(flag ==1) {
2478  string container_name;
2479  temp_pointer = obtain_str(temp_pointer,container_name);
2480  BESDEBUG(HDF5_NAME, prolog << "DAS from the disk cache, container name is " << container_name << endl);
2481 
2482  // Remember the current Attribute table state
2483  AttrTable*temp_at_par = at_par;
2484  if(at_par == nullptr)
2485  at_par = das->add_table(container_name, new AttrTable);
2486  else
2487  at_par = at_par->append_container(container_name);
2488 
2489  temp_pointer = get_attr_info_from_dc(temp_pointer,das,at_par);
2490  // MUST resume the original state
2491  at_par = temp_at_par;
2492 
2493  }
2494  else if(flag == 0) {
2495  // The attribute must have a table.
2496  if(at_par ==nullptr)
2497  throw BESInternalError( "The AttrTable must exist for DAS attributes", __FILE__, __LINE__ ) ;
2498 
2499  // Attribute name
2500  string attr_name;
2501  temp_pointer = obtain_str(temp_pointer,attr_name);
2502  BESDEBUG(HDF5_NAME, prolog << "DAS from the disk cache, attr name is: " << attr_name << endl);
2503 
2504  // Attribute type
2505  string attr_type;
2506  temp_pointer = obtain_str(temp_pointer,attr_type);
2507  BESDEBUG(HDF5_NAME, prolog << "DAS from the disk cache, attr type is: " << attr_type << endl);
2508 
2509  // Attribute values
2510  unsigned int num_values = *((unsigned int*)(temp_pointer));
2511  BESDEBUG(HDF5_NAME, prolog << "DAS from the disk cache, number of attribute values is: " << num_values << endl);
2512  temp_pointer+=sizeof(unsigned int);
2513 
2514  vector <string> attr_values;
2515 
2516  for(unsigned int i = 0; i<num_values; i++) {
2517  string attr_value;
2518  temp_pointer = obtain_str(temp_pointer,attr_value);
2519  attr_values.push_back(attr_value);
2520  BESDEBUG(HDF5_NAME, prolog << "DAS from the disk cache, attribute value is: " << attr_value << endl);
2521  }
2522 
2523  at_par->append_attr(attr_name,attr_type,&attr_values);
2524  }
2525 
2526  }
2527  return temp_pointer;
2528 
2529 }
2530 
2531 // The debugging function to get attribute info.
2532 void get_attr_contents(AttrTable*temp_table) {
2533  if(temp_table !=nullptr) {
2534  AttrTable::Attr_iter top_startit = temp_table->attr_begin();
2535  AttrTable::Attr_iter top_endit = temp_table->attr_end();
2536  AttrTable::Attr_iter top_it = top_startit;
2537  while(top_it !=top_endit) {
2538  AttrType atype = temp_table->get_attr_type(top_it);
2539  if(atype == Attr_unknown)
2540  cerr<<"unsupported DAS attributes" <<endl;
2541  else if(atype!=Attr_container) {
2542 
2543  cerr<<"Attribute name is "<<temp_table->get_name(top_it)<<endl;
2544  cerr<<"Attribute type is "<<temp_table->get_type(top_it)<<endl;
2545  unsigned int num_attrs = temp_table->get_attr_num(temp_table->get_name(top_it));
2546  cerr<<"Attribute values are "<<endl;
2547  for (unsigned int i = 0; i <num_attrs;i++)
2548  cerr<<(*(temp_table->get_attr_vector(temp_table->get_name(top_it))))[i]<<" ";
2549  cerr<<endl;
2550  }
2551  else {
2552  cerr<<"Coming to the attribute container. "<<endl;
2553  cerr<<"container name is "<<(*top_it)->name <<endl;
2554  AttrTable* sub_table = temp_table->get_attr_table(top_it);
2555  cerr<<"container table name is "<<sub_table->get_name() <<endl;
2556  get_attr_contents(sub_table);
2557  }
2558  ++top_it;
2559  }
2560 
2561  }
2562 }
2563 
2564 
2565 void HDF5RequestHandler::add_attributes(BESDataHandlerInterface &dhi) {
2566 
2567 
2568  BESResponseObject *response = dhi.response_handler->get_response_object();
2569  BESDataDDSResponse *bdds = dynamic_cast<BESDataDDSResponse *>(response);
2570  if (!bdds)
2571  throw BESInternalError("cast error", __FILE__, __LINE__);
2572  DDS *dds = bdds->get_dds();
2573  string container_name = bdds->get_explicit_containers() ? dhi.container->get_symbolic_name(): "";
2574  string filename = dhi.container->access();
2575  DAS* das = nullptr;
2576  bool das_from_mcache = false;
2577  if(das_cache) {
2578  das = static_cast<DAS*>(das_cache->get(filename));
2579  if(das) {
2580  BESDEBUG(HDF5_NAME, prolog << "DAS Cached hit for : " << filename << endl);
2581  dds->transfer_attributes(das); // no need to copy the cached DAS
2582  das_from_mcache = true;
2583  }
2584  }
2585 
2586  if(false == das_from_mcache) {
2587  das = new DAS;
2588  // This looks at the 'use explicit containers' prop, and if true
2589  // sets the current container for the DAS.
2590  if (!container_name.empty()) das->container_name(container_name);
2591 
2592  hid_t h5_fd =-1;
2593  if (true == _usecf) {
2594  // go to the CF option
2595  h5_fd = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
2596 
2597  read_cfdas( *das,filename,h5_fd);
2598 
2599  H5Fclose(h5_fd);
2600  }
2601  else {
2602  h5_fd = get_fileid(filename.c_str());
2603  find_gloattr(h5_fd, *das);
2604  depth_first(h5_fd, "/", *das);
2605  close_fileid(h5_fd);
2606  }
2607 
2608 
2609  Ancillary::read_ancillary_das(*das, filename);
2610 
2611  dds->transfer_attributes(das);
2612 
2613  // Only free the DAS if it's not added to the cache
2614  if (das_cache) {
2615  // add a copy
2616  BESDEBUG(HDF5_NAME, prolog << "DAS added to the cache for : " << filename << endl);
2617  das_cache->add(das, filename);
2618  }
2619  else {
2620  delete das;
2621  }
2622  }
2623  BESDEBUG(HDF5_NAME, prolog << "Data ACCESS in add_attributes(): set the including attribute flag to true: "<<filename << endl);
2624  bdds->set_ia_flag(true);
2625  return;
2626 
2627 }
2628 
2629 
HDF5RequestHandler.h
include the entry functions to execute the handlers
BESContainer::get_symbolic_name
std::string get_symbolic_name() const
retrieve the symbolic name for this container
Definition: BESContainer.h:221
BESContainer::access
virtual std::string access()=0
returns the true name of this container
BESDASResponse
Represents an OPeNDAP DAS DAP2 data object within the BES.
Definition: BESDASResponse.h:44
BESDASResponse::clear_container
virtual void clear_container()
clear the container in the DAP response object
Definition: BESDASResponse.cc:58
BESDASResponse::set_container
virtual void set_container(const std::string &cn)
set the container in the DAP response object
Definition: BESDASResponse.cc:49
BESDDSResponse
Holds a DDS object within the BES.
Definition: BESDDSResponse.h:50
BESDDSResponse::set_container
virtual void set_container(const std::string &cn)
set the container in the DAP response object
Definition: BESDDSResponse.cc:64
BESDDSResponse::set_dds
void set_dds(libdap::DDS *ddsIn)
Definition: BESDDSResponse.h:71
BESDDSResponse::clear_container
virtual void clear_container()
clear the container in the DAP response object
Definition: BESDDSResponse.cc:73
BESDDSResponse::get_dds
libdap::DDS * get_dds()
Definition: BESDDSResponse.h:80
BESDMRResponse
Represents an OPeNDAP DMR DAP4 data object within the BES.
Definition: BESDMRResponse.h:49
BESDapError
error object created from libdap error objects and can handle those errors
Definition: BESDapError.h:59
BESDapResponse::set_dap4_function
virtual void set_dap4_function(BESDataHandlerInterface &dhi)
set the constraint depending on the context
Definition: BESDapResponse.cc:173
BESDapResponse::set_dap4_constraint
virtual void set_dap4_constraint(BESDataHandlerInterface &dhi)
set the constraint depending on the context
Definition: BESDapResponse.cc:156
BESDapResponse::set_constraint
virtual void set_constraint(BESDataHandlerInterface &dhi)
set the constraint depending on the context
Definition: BESDapResponse.cc:134
BESDapResponse::get_explicit_containers
bool get_explicit_containers() const
Should containers be explicitly represented in the DD* responses?
Definition: BESDapResponse.h:70
BESDapResponse::get_request_xml_base
std::string get_request_xml_base() const
Return the xml:base URL for this request.
Definition: BESDapResponse.h:76
BESDataDDSResponse
Represents an OPeNDAP DataDDS DAP2 data object within the BES.
Definition: BESDataDDSResponse.h:46
BESDataDDSResponse::set_dds
void set_dds(libdap::DDS *ddsIn)
Definition: BESDataDDSResponse.h:73
BESDataDDSResponse::set_container
virtual void set_container(const std::string &cn)
set the container in the DAP response object
Definition: BESDataDDSResponse.cc:50
BESDataDDSResponse::clear_container
virtual void clear_container()
clear the container in the DAP response object
Definition: BESDataDDSResponse.cc:59
BESDataHandlerInterface
Structure storing information used by the BES to handle the request.
Definition: BESDataHandlerInterface.h:56
BESDataHandlerInterface::container
BESContainer * container
pointer to current container in this interface
Definition: BESDataHandlerInterface.h:75
BESError
Base exception class for the BES with basic string message.
Definition: BESError.h:59
BESError::get_message
std::string get_message() const
get the error message for this exception
Definition: BESError.h:111
BESInfo
informational response object
Definition: BESInfo.h:63
BESInfo::add_data
virtual void add_data(const std::string &s)
add data to this informational object. If buffering is not set then the information is output directl...
Definition: BESInfo.cc:160
BESInternalError
exception thrown if internal error encountered
Definition: BESInternalError.h:43
BESInternalFatalError
exception thrown if an internal error is found and is fatal to the BES
Definition: BESInternalFatalError.h:43
BESRequestHandler
Represents a specific data type request handler.
Definition: BESRequestHandler.h:74
BESResponseHandler::get_response_object
virtual BESResponseObject * get_response_object()
return the current response object
Definition: BESResponseHandler.cc:82
BESResponseObject
Abstract base class representing a specific set of information in response to a request to the BES.
Definition: BESResponseObject.h:46
BESServiceRegistry::services_handled
virtual void services_handled(const std::string &handler, std::list< std::string > &services)
returns the list of servies provided by the handler in question
Definition: BESServiceRegistry.cc:347
BESStopWatch::start
virtual bool start(std::string name)
Definition: BESStopWatch.cc:67
BESUtil::lowercase
static std::string lowercase(const std::string &s)
Definition: BESUtil.cc:254
BESUtil::implode
static std::string implode(const std::list< std::string > &values, char delim)
Definition: BESUtil.cc:617
ObjMemCache
An in-memory cache for DapObj (DAS, DDS, ...) objects.
Definition: ObjMemCache.h:84
ObjMemCache::add
virtual void add(libdap::DapObj *obj, const std::string &key)
Add an object to the cache and associate it with a key.
Definition: ObjMemCache.cc:63
ObjMemCache::get
virtual libdap::DapObj * get(const std::string &key)
Get the cached pointer.
Definition: ObjMemCache.cc:105
TheBESKeys::get_value
void get_value(const std::string &s, std::string &val, bool &found)
Retrieve the value of a given key, if set.
Definition: TheBESKeys.cc:340
TheBESKeys::TheKeys
static TheBESKeys * TheKeys()
Definition: TheBESKeys.cc:71
h5cfdaputil.h
Helper functions for generating DAS attributes and a function to check BES Key.
depth_first
void depth_first(hid_t pid, const char *gname, DAS &das)
Definition: h5das.cc:62
find_gloattr
void find_gloattr(hid_t file, DAS &das)
Definition: h5das.cc:482
breadth_first
bool breadth_first(const hid_t file_id, hid_t pid, const char *gname, D4Group *par_grp, const char *fname, bool use_dimscale, vector< link_info_t > &hdf5_hls)
Definition: h5dmr.cc:321
get_fileid
hid_t get_fileid(const char *filename)
Definition: h5get.cc:412
close_fileid
void close_fileid(hid_t fid)
Definition: h5get.cc:434
hdf5_handler.h
The main header of the HDF5 OPeNDAP handler.