bes  Updated for version 3.20.13
HDF4RequestHandler.cc
1 
2 // -*- mode: c++; c-basic-offset:4 -*-
3 
4 // This file is part of hdf4_handler, a data handler for the OPeNDAP data
5 // server.
6 
7 // Copyright (c) 2002,2003 OPeNDAP, Inc.
8 // Author: James Gallagher <jgallagher@opendap.org>
9 // Author: Muqun Yang <myang6@opendap.org>
10 // This is free software; you can redistribute it and/or modify it under the
11 // terms of the GNU Lesser General Public License as published by the Free
12 // Software Foundation; either version 2.1 of the License, or (at your
13 // option) any later version.
14 //
15 // This software is distributed in the hope that it will be useful, but
16 // WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
17 // or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
18 // License for more details.
19 //
20 // You should have received a copy of the GNU Lesser General Public
21 // License along with this library; if not, write to the Free Software
22 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
23 //
24 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
25 
26 // HDF4RequestHandler.cc
27 
28 #include <cstdlib>
29 #include <sys/types.h>
30 #include <sys/stat.h>
31 #include <fcntl.h>
32 #include <cerrno>
33 #include <unistd.h>
34 #include <iostream>
35 #include <sstream>
36 
37 #include <libdap/DMR.h>
38 #include <libdap/D4BaseTypeFactory.h>
39 #include <BESDMRResponse.h>
40 #include <libdap/mime_util.h>
41 #include <libdap/InternalErr.h>
42 #include <libdap/Ancillary.h>
43 #include <libdap/debug.h>
44 
45 #include "HDF4RequestHandler.h"
46 #include <BESResponseNames.h>
47 #include <BESDapNames.h>
48 #include <BESDASResponse.h>
49 #include <BESDDSResponse.h>
50 #include <BESDataDDSResponse.h>
51 #include <BESInfo.h>
52 #include <BESResponseHandler.h>
53 #include <BESVersionInfo.h>
54 #include <BESServiceRegistry.h>
55 #include <BESUtil.h>
56 #include <TheBESKeys.h>
57 #include <libdap/InternalErr.h>
58 #include <BESInternalError.h>
59 #include <BESDapError.h>
60 #include <BESStopWatch.h>
61 #include <BESDebug.h>
62 #include "BESDataNames.h"
63 //#include <libdap/ConstraintEvaluator.h>
64 #include <libdap/Ancillary.h>
65 #include "config_hdf.h"
66 
67 #define HDF4_NAME "h4"
68 #include "HE2CF.h"
69 #include "HDF4_DDS.h"
70 
71 #include "HDF4_DMR.h"
72 
73 //#include "HDFCFUtil.h"
74 #include "HDFFloat32.h"
75 #include "HDFSPArray_RealField.h"
76 
77 #include "dodsutil.h"
78 //#if 0
79 #include <sys/time.h>
80 //#endif
81 
82 using namespace std;
83 using namespace libdap;
84 
85 bool check_beskeys(const string &);
86 bool get_beskeys(const string &,string &);
87 
88 extern void read_das(DAS & das, const string & filename);
89 extern void read_dds(DDS & dds, const string & filename);
90 
91 extern bool read_dds_hdfsp(DDS & dds, const string & filename,int32 sdfd, int32 fileid,HDFSP::File*h4file);
92 
93 extern bool read_das_hdfsp(DAS & das, const string & filename,int32 sdfd, int32 fileid,HDFSP::File**h4fileptr);
94 
95 extern void read_das_sds(DAS & das, const string & filename,int32 sdfd, bool ecs_metadata,HDFSP::File**h4fileptr);
96 extern void read_dds_sds(DDS &dds, const string & filename,int32 sdfd, HDFSP::File*h4file,bool dds_set_cache);
97 
98 #ifdef USE_HDFEOS2_LIB
99 
100 void read_das_use_eos2lib(DAS & das, const string & filename,int32 sdfd,int32 fileid, int32 gridfd, int32 swathfd,bool ecs_metadata,HDFSP::File**h4file,HDFEOS2::File**eosfile);
101 void read_dds_use_eos2lib(DDS & dds, const string & filename,int32 sdfd,int32 fileid, int32 gridfd, int32 swathfd,HDFSP::File*h4file,HDFEOS2::File*eosfile);
102 void close_fileid(const int sdfd, const int fileid,const int gridfd, const int swathfd,HDFSP::File*h4file,HDFEOS2::File*eosfile);
103 
104 #endif
105 
106 void close_hdf4_fileid(const int sdfd,const int fileid,HDFSP::File*h4file);
107 bool rw_das_cache_file(const string & filename, DAS *das_ptr,bool rw_flag);
108 bool r_dds_cache_file(const string & cache_filename, DDS *dds_ptr,const string & hdf4_filename);
109 
110 // CF key
111 bool HDF4RequestHandler::_usecf = false;
112 
113 // Keys to tune the performance -general
114 bool HDF4RequestHandler::_pass_fileid = false;
115 bool HDF4RequestHandler::_disable_structmeta = false;
116 bool HDF4RequestHandler::_enable_special_eos = false;
117 bool HDF4RequestHandler::_disable_scaleoffset_comp = false;
118 bool HDF4RequestHandler::_disable_ecsmetadata_min = false;
119 bool HDF4RequestHandler::_disable_ecsmetadata_all = false;
120 
121 
122 // Keys to tune the performance - cache
123 bool HDF4RequestHandler::_enable_eosgeo_cachefile = false;
124 bool HDF4RequestHandler::_enable_data_cachefile = false;
125 bool HDF4RequestHandler::_enable_metadata_cachefile= false;
126 
127 // Keys to handle vdata and vgroups
128 bool HDF4RequestHandler::_enable_hybrid_vdata = false;
129 bool HDF4RequestHandler::_enable_ceres_vdata = false;
130 bool HDF4RequestHandler::_enable_vdata_attr = false;
131 bool HDF4RequestHandler::_enable_vdata_desc_attr = false;
132 bool HDF4RequestHandler::_disable_vdata_nameclashing_check = false;
133 bool HDF4RequestHandler::_enable_vgroup_attr = false;
134 
135 // Misc. keys
136 bool HDF4RequestHandler::_enable_check_modis_geo_file = false;
137 bool HDF4RequestHandler::_enable_swath_grid_attr = false;
138 bool HDF4RequestHandler::_enable_ceres_merra_short_name = false;
139 bool HDF4RequestHandler::_enable_check_scale_offset_type = false;
140 bool HDF4RequestHandler::_disable_swath_dim_map = false;
141 
142 // Cache path,prefix and size
143 bool HDF4RequestHandler::_cache_latlon_path_exist =false;
144 string HDF4RequestHandler::_cache_latlon_path ="";
145 bool HDF4RequestHandler::_cache_latlon_prefix_exist =false;
146 string HDF4RequestHandler::_cache_latlon_prefix ="";
147 bool HDF4RequestHandler::_cache_latlon_size_exist =false;
148 long HDF4RequestHandler::_cache_latlon_size =0;
149 bool HDF4RequestHandler::_cache_metadata_path_exist =false;
150 string HDF4RequestHandler::_cache_metadata_path ="";
151 
152 HDF4RequestHandler::HDF4RequestHandler(const string & name) :
153  BESRequestHandler(name) {
154  add_handler(DAS_RESPONSE, HDF4RequestHandler::hdf4_build_das);
155  add_handler(DDS_RESPONSE, HDF4RequestHandler::hdf4_build_dds);
156  add_handler(DATA_RESPONSE, HDF4RequestHandler::hdf4_build_data);
157  add_handler(DMR_RESPONSE, HDF4RequestHandler::hdf4_build_dmr);
158  add_handler(DAP4DATA_RESPONSE, HDF4RequestHandler::hdf4_build_dmr);
159  add_handler(HELP_RESPONSE, HDF4RequestHandler::hdf4_build_help);
160  add_handler(VERS_RESPONSE, HDF4RequestHandler::hdf4_build_version);
161 
162  _usecf = check_beskeys("H4.EnableCF");
163 
164  // The following keys are only effective when usecf is true.
165  // Keys to tune the performance -general
166  _pass_fileid = check_beskeys("H4.EnablePassFileID");
167  _disable_structmeta = check_beskeys("H4.DisableStructMetaAttr");
168  _enable_special_eos = check_beskeys("H4.EnableSpecialEOS");
169  _disable_scaleoffset_comp = check_beskeys("H4.DisableScaleOffsetComp");
170  _disable_ecsmetadata_min = check_beskeys("H4.DisableECSMetaDataMin");
171  _disable_ecsmetadata_all = check_beskeys("H4.DisableECSMetaDataAll");
172 
173 
174  // Keys to tune the performance - cache
175  _enable_eosgeo_cachefile = check_beskeys("H4.EnableEOSGeoCacheFile");
176  _enable_data_cachefile = check_beskeys("H4.EnableDataCacheFile");
177  _enable_metadata_cachefile = check_beskeys("H4.EnableMetaDataCacheFile");
178 
179  // Keys to handle vdata and vgroups
180  _enable_hybrid_vdata = check_beskeys("H4.EnableHybridVdata");
181  _enable_ceres_vdata = check_beskeys("H4.EnableCERESVdata");
182  _enable_vdata_attr = check_beskeys("H4.EnableVdata_to_Attr");
183  _enable_vdata_desc_attr = check_beskeys("H4.EnableVdataDescAttr");
184  _disable_vdata_nameclashing_check = check_beskeys("H4.DisableVdataNameclashingCheck");
185  _enable_vgroup_attr = check_beskeys("H4.EnableVgroupAttr");
186 
187  // Misc. keys
188  _enable_check_modis_geo_file = check_beskeys("H4.EnableCheckMODISGeoFile");
189  _enable_swath_grid_attr = check_beskeys("H4.EnableSwathGridAttr");
190  _enable_ceres_merra_short_name = check_beskeys("H4.EnableCERESMERRAShortName");
191  _enable_check_scale_offset_type = check_beskeys("H4.EnableCheckScaleOffsetType");
192 
193  _disable_swath_dim_map = check_beskeys("H4.DisableSwathDimMap");
194 
195  // Cache path etc.
196  _cache_latlon_path_exist =get_beskeys("HDF4.Cache.latlon.path",_cache_latlon_path);
197  _cache_latlon_prefix_exist =get_beskeys("HDF4.Cache.latlon.prefix",_cache_latlon_prefix);
198  string temp_cache_latlon_size;
199  _cache_latlon_size_exist =get_beskeys("HDF4.Cache.latlon.size",temp_cache_latlon_size);
200  if(_cache_latlon_size_exist == true) {
201  istringstream iss(temp_cache_latlon_size);
202  iss >> _cache_latlon_size;
203  }
204 
205  _cache_metadata_path_exist =get_beskeys("H4.Cache.metadata.path",_cache_metadata_path);
206 
207 }
208 
209 #if 0
210 HDF4RequestHandler::~HDF4RequestHandler() {
211 }
212 #endif
213 
214 bool HDF4RequestHandler::hdf4_build_das(BESDataHandlerInterface & dhi) {
215 
216 
217  BESStopWatch sw;
218  if (BESDebug::IsSet(TIMING_LOG_KEY))
219  sw.start("HDF4RequestHandler::hdf4_build_das", dhi.data[REQUEST_ID]);
220 
221  if(true == _usecf) {
222 
223  // Build DAP response only based on the HDF4 SD interfaces. Doing this
224  // way will save the use of other library open calls. Other library open
225  // calls may be expensive
226  // for an HDF4 file that only has variables created by SD interfaces.
227  // This optimization may be very useful for the aggreagation case that
228  // has many variables.
229  // Currently we only handle AIRS version 6 products. AIRS products
230  // are identified by their file names.
231  // We only obtain the filename. The path is stripped off.
232 
233  string base_file_name = basename(dhi.container->access());
234 
235  //AIRS.2015.01.24.L3.RetStd_IR008.v6.0.11.0.G15041143400.hdf
236  // Identify this file from product name: AIRS, product level: .L3. or .L2. and version .v6.
237  if((base_file_name.size() >12) && (base_file_name.compare(0,4,"AIRS") == 0)
238  && ((base_file_name.find(".L3.")!=string::npos) || (base_file_name.find(".L2.")!=string::npos))
239  && (base_file_name.find(".v6.")!=string::npos)) {
240  return hdf4_build_das_cf_sds(dhi);
241 
242  }
243  }
244 
245  BESResponseObject *response = dhi.response_handler->get_response_object();
246  auto bdas = dynamic_cast<BESDASResponse *> (response);
247  if (!bdas)
248  throw BESInternalError("cast error", __FILE__, __LINE__);
249 
250  try {
251  bdas->set_container(dhi.container->get_symbolic_name());
252  DAS *das = bdas->get_das();
253 
254  string base_file_name = basename(dhi.container->access());
255 
256  string accessed = dhi.container->access();
257 
258  if (true == _usecf) {
259 
260  int32 sdfd = -1;
261  int32 fileid = -1;
262 
263  HDFSP::File *h4file = nullptr;
264 
265  // Obtain HDF4 file IDs
266  //SDstart
267  sdfd = SDstart (accessed.c_str(), DFACC_READ);
268  if( -1 == sdfd){
269  string invalid_file_msg="HDF4 SDstart error for the file ";
270  invalid_file_msg +=accessed;
271  invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
272  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
273  }
274 
275  // H open
276  fileid = Hopen(accessed.c_str(), DFACC_READ,0);
277  if (-1 == fileid) {
278  SDend(sdfd);
279  string invalid_file_msg="HDF4 Hopen error for the file ";
280  invalid_file_msg +=accessed;
281  invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
282  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
283  }
284 
285 #ifdef USE_HDFEOS2_LIB
286 
287  int32 gridfd = -1;
288  int32 swathfd = -1;
289  HDFEOS2::File *eosfile = nullptr;
290  // Obtain HDF-EOS2 file IDs with the file open APIs.
291 
292  // Grid open
293  gridfd = GDopen(const_cast < char *>(accessed.c_str()), DFACC_READ);
294  if (-1 == gridfd) {
295  SDend(sdfd);
296  Hclose(fileid);
297  string invalid_file_msg="HDF-EOS GDopen error for the file ";
298  invalid_file_msg +=accessed;
299  invalid_file_msg +=". It is very possible that this file is not an HDF-EOS2 file. ";
300  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
301  }
302 
303  // Swath open
304  swathfd = SWopen(const_cast < char *>(accessed.c_str()), DFACC_READ);
305  if (-1 == swathfd) {
306  SDend(sdfd);
307  Hclose(fileid);
308  GDclose(gridfd);
309  string invalid_file_msg="HDF-EOS SWopen error for the file ";
310  invalid_file_msg +=accessed;
311  invalid_file_msg +=". It is very possible that this file is not an HDF-EOS2 file. ";
312  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
313  }
314 
315  try {
316  bool ecs_metadata = !_disable_ecsmetadata_all;
317 #if 0
318 if(ecs_metadata == true)
319 cerr<<"output ecs metadata "<<endl;
320 else
321 cerr<<"Don't output ecs metadata "<<endl;
322 #endif
323  read_das_use_eos2lib(*das, accessed,sdfd,fileid,gridfd,swathfd,ecs_metadata,&h4file,&eosfile);
324  }
325  catch(...) {
326  close_fileid(sdfd,fileid,gridfd,swathfd,h4file,eosfile);
327  throw;
328 #if 0
329  //throw InternalErr(__FILE__,__LINE__,"read_das_use_eos2lib error");
330 #endif
331  }
332  if(eosfile != nullptr)
333  delete eosfile;
334  GDclose(gridfd);
335  SWclose(swathfd);
336 
337 #else
338  try {
339  read_das_hdfsp(*das,accessed,sdfd,fileid,&h4file);
340  }
341  catch(...) {
342  close_hdf4_fileid(sdfd,fileid,h4file);
343  throw;
344  //throw InternalErr(__FILE__,__LINE__,"read_das_hdfsp error");
345  }
346 #endif
347  close_hdf4_fileid(sdfd,fileid,h4file);
348  }
349  else
350  read_das(*das,accessed);
351 
352  Ancillary::read_ancillary_das(*das, accessed);
353  bdas->clear_container();
354  }
355 
356  catch (BESError & e) {
357  throw;
358  }
359  catch (InternalErr & e) {
360  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
361  __FILE__, __LINE__);
362  }
363  catch (Error & e) {
364  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
365  __FILE__, __LINE__);
366  }
367  catch (...) {
368  string s = "unknown exception caught building HDF4 DAS";
369  throw BESDapError(s, true, unknown_error, __FILE__, __LINE__);
370  }
371 
372  return true;
373 }
374 
375 bool HDF4RequestHandler::hdf4_build_dds(BESDataHandlerInterface & dhi) {
376 
377 
378 
379  BESStopWatch sw;
380  if (BESDebug::IsSet(TIMING_LOG_KEY))
381  sw.start("HDF4RequestHandler::hdf4_build_das", dhi.data[REQUEST_ID]);
382 
383 
384  if(true == _usecf) {
385  // Build DAP response only based on the HDF4 SD interfaces. Doing this
386  // way will save the use of other library open calls. Other library open
387  // calls may be expensive
388  // for an HDF4 file that only has variables created by SD interfaces.
389  // This optimization may be very useful for the aggreagation case that
390  // has many variables.
391  // Currently we only handle AIRS version 6 products. AIRS products
392  // are identified by their file names.
393  // We only obtain the filename. The path is stripped off.
394 
395  string base_file_name = basename(dhi.container->access());
396 
397  //AIRS.2015.01.24.L3.RetStd_IR008.v6.0.11.0.G15041143400.hdf
398  // Identify this file from product name: AIRS, product level: .L3. or .L2. and version .v6.
399  if((base_file_name.size() >12) && (base_file_name.compare(0,4,"AIRS") == 0)
400  && ((base_file_name.find(".L3.")!=string::npos) || (base_file_name.find(".L2.")!=string::npos))
401  && (base_file_name.find(".v6.")!=string::npos)) {
402  return hdf4_build_dds_cf_sds(dhi);
403 
404  }
405  }
406 
407 // This is for the performance check. Leave it now for future use. KY 2014-10-23
408 #ifdef KENT2
409 struct timeval start_time,end_time;
410 gettimeofday(&start_time,nullptr);
411 #endif
412 
413  BESResponseObject *response = dhi.response_handler->get_response_object();
414  auto bdds = dynamic_cast<BESDDSResponse *> (response);
415  if (!bdds)
416  throw BESInternalError("cast error", __FILE__, __LINE__);
417 
418  try {
419  bdds->set_container(dhi.container->get_symbolic_name());
420  DDS *dds = bdds->get_dds();
421 #if 0
422  //ConstraintEvaluator & ce = bdds->get_ce();
423 #endif
424 
425  string accessed = dhi.container->access();
426  dds->filename(accessed);
427 
428  auto das = new DAS;
429  BESDASResponse bdas(das);
430  bdas.set_container(dhi.container->get_symbolic_name());
431 
432  if (true == _usecf) {
433 
434  int32 sdfd = -1;
435  int32 fileid = -1;
436  HDFSP::File *h4file = nullptr;
437 
438  // Obtain HDF4 file IDs
439  //SDstart
440  sdfd = SDstart (accessed.c_str(), DFACC_READ);
441  if( -1 == sdfd){
442  string invalid_file_msg="HDF4 SDstart error for the file ";
443  invalid_file_msg +=accessed;
444  invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
445  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
446  }
447 
448  // H open
449  fileid = Hopen(accessed.c_str(), DFACC_READ,0);
450  if (-1 == fileid) {
451  SDend(sdfd);
452  string invalid_file_msg="HDF4 Hopen error for the file ";
453  invalid_file_msg +=accessed;
454  invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
455  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
456  }
457 
458 #ifdef USE_HDFEOS2_LIB
459 
460  int32 gridfd = -1;
461  int32 swathfd = -1;
462 
463  HDFEOS2::File *eosfile = nullptr;
464 
465  // Obtain HDF-EOS2 file IDs with the file open APIs.
466  // Grid open
467  gridfd = GDopen(const_cast < char *>(accessed.c_str()), DFACC_READ);
468  if (-1 == gridfd) {
469  SDend(sdfd);
470  Hclose(fileid);
471  string invalid_file_msg="HDF-EOS GDopen error for the file ";
472  invalid_file_msg +=accessed;
473  invalid_file_msg +=". It is very possible that this file is not an HDF-EOS2 file. ";
474  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
475  }
476 
477  // Swath open
478  swathfd = SWopen(const_cast < char *>(accessed.c_str()), DFACC_READ);
479  if (-1 == swathfd) {
480  SDend(sdfd);
481  Hclose(fileid);
482  GDclose(gridfd);
483  string invalid_file_msg="HDF-EOS SWopen error for the file ";
484  invalid_file_msg +=accessed;
485  invalid_file_msg +=". It is very possible that this file is not an HDF-EOS2 file. ";
486  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
487  }
488 
489  try {
490  bool ecs_metadata = !_disable_ecsmetadata_all;
491  read_das_use_eos2lib(*das, accessed,sdfd,fileid,gridfd,swathfd,ecs_metadata,&h4file,&eosfile);
492  Ancillary::read_ancillary_das(*das, accessed);
493 
494  // Pass file pointer(h4file, eosfile) from DAS to DDS.
495  read_dds_use_eos2lib(*dds, accessed,sdfd,fileid,gridfd,swathfd,h4file,eosfile);
496  }
497  catch(...) {
498  close_fileid(sdfd,fileid,gridfd,swathfd,h4file,eosfile);
499  throw;
500  }
501 
502  if(eosfile != nullptr)
503  delete eosfile;
504 
505  GDclose(gridfd);
506  SWclose(swathfd);
507 
508 #else
509  try {
510  read_das_hdfsp(*das, accessed,sdfd,fileid,&h4file);
511  Ancillary::read_ancillary_das(*das, accessed);
512 
513  // Pass file pointer(h4file) from DAS to DDS.
514  read_dds_hdfsp(*dds, accessed,sdfd,fileid,h4file);
515  }
516  catch(...) {
517  close_hdf4_fileid(sdfd,fileid,h4file);
518  throw;
519  }
520 
521 #endif
522  close_hdf4_fileid(sdfd,fileid,h4file);
523  }
524  else {
525  read_das(*das, accessed);
526  Ancillary::read_ancillary_das(*das, accessed);
527  read_dds(*dds, accessed);
528  }
529 
530 // Leave it for future performance tests. KY 2014-10-23
531 #ifdef KENT2
532 gettimeofday(&end_time,nullptr);
533 int total_time_spent = (end_time.tv_sec - start_time.tv_sec)*1000000 +end_time.tv_usec-start_time.tv_usec;
534 cerr<<"total time spent for DDS buld is "<<total_time_spent<< "micro seconds "<<endl;
535 #endif
536 
537  Ancillary::read_ancillary_dds(*dds, accessed);
538 
539  dds->transfer_attributes(das);
540 
541  bdds->set_constraint(dhi);
542 
543  bdds->clear_container();
544  }
545  catch (BESError & e) {
546  throw;
547  }
548  catch (InternalErr & e) {
549  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
550  __FILE__, __LINE__);
551  }
552  catch (Error & e) {
553  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
554  __FILE__, __LINE__);
555  }
556  catch (...) {
557  string s = "unknown exception caught building HDF4 DDS";
558  throw BESDapError(s, true, unknown_error, __FILE__, __LINE__);
559  }
560 
561  return true;
562 }
563 
564 bool HDF4RequestHandler::hdf4_build_data(BESDataHandlerInterface & dhi) {
565 
566 
567  BESStopWatch sw;
568  if (BESDebug::IsSet(TIMING_LOG_KEY))
569  sw.start("HDF4RequestHandler::hdf4_build_data", dhi.data[REQUEST_ID]);
570 
571 
572  int32 sdfd = -1;
573  int32 fileid = -1;
574 
575  // Since passing file IDs requires to use the derived class and it
576  // causes the management of code structure messy, we first handle this with
577  // another method.
578  if(true == _usecf) {
579  // Build DAP response only based on the HDF4 SD interfaces. Doing this
580  // way will save the use of other library open calls. Other library open
581  // calls may be expensive
582  // for an HDF4 file that only has variables created by SD interfaces.
583  // This optimization may be very useful for the aggreagation case that
584  // has many variables.
585  // Currently we only handle AIRS version 6 products. AIRS products
586  // are identified by their file names.
587  // We only obtain the filename. The path is stripped off.
588 
589  string base_file_name = basename(dhi.container->access());
590  //AIRS.2015.01.24.L3.RetStd_IR008.v6.0.11.0.G15041143400.hdf
591  // Identify this file from product name: AIRS, product level: .L3. or .L2. and version .v6.
592  if((base_file_name.size() >12) && (base_file_name.compare(0,4,"AIRS") == 0)
593  && ((base_file_name.find(".L3.")!=string::npos) || (base_file_name.find(".L2.")!=string::npos))
594  && (base_file_name.find(".v6.")!=string::npos)) {
595 
596  BESDEBUG("h4", "Coming to read the data of AIRS level 3 or level 2 products." << endl);
597 
598  if(true == _pass_fileid)
599  return hdf4_build_data_cf_sds_with_IDs(dhi);
600  else
601  return hdf4_build_data_cf_sds(dhi);
602 
603  }
604 
605  if(true == _pass_fileid)
606  return hdf4_build_data_with_IDs(dhi);
607 
608  }
609 
610  BESResponseObject *response = dhi.response_handler->get_response_object();
611  auto bdds = dynamic_cast<BESDataDDSResponse *> (response);
612 
613  if (!bdds)
614  throw BESInternalError("cast error", __FILE__, __LINE__);
615 
616  try {
617  bdds->set_container(dhi.container->get_symbolic_name());
618 
619  DDS *dds = bdds->get_dds();
620 
621  // Not sure why keep the following line, it is not used.
622  //ConstraintEvaluator & ce = bdds->get_ce();
623 
624  string accessed = dhi.container->access();
625  dds->filename(accessed);
626 
627  auto das = new DAS;
628  BESDASResponse bdas(das);
629  bdas.set_container(dhi.container->get_symbolic_name());
630 
631  if (true == _usecf) {
632 
633  HDFSP::File *h4file = nullptr;
634 
635  // Obtain HDF4 file IDs
636  //SDstart
637  sdfd = SDstart (accessed.c_str(), DFACC_READ);
638  if( -1 == sdfd) {
639  string invalid_file_msg="HDF4 SDstart error for the file ";
640  invalid_file_msg +=accessed;
641  invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
642  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
643  }
644 
645  // H open
646  fileid = Hopen(accessed.c_str(), DFACC_READ,0);
647  if (-1 == fileid) {
648  SDend(sdfd);
649  string invalid_file_msg="HDF4 Hopen error for the file ";
650  invalid_file_msg +=accessed;
651  invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
652  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
653  }
654 
655 
656 #ifdef USE_HDFEOS2_LIB
657 
658  int32 gridfd = -1;
659  int32 swathfd = -1;
660  HDFEOS2::File *eosfile = nullptr;
661  // Obtain HDF-EOS2 file IDs with the file open APIs.
662 
663  // Grid open
664  gridfd = GDopen(const_cast < char *>(accessed.c_str()), DFACC_READ);
665  if (-1 == gridfd) {
666  SDend(sdfd);
667  Hclose(fileid);
668  string invalid_file_msg="HDF-EOS GDopen error for the file ";
669  invalid_file_msg +=accessed;
670  invalid_file_msg +=". It is very possible that this file is not an HDF-EOS2 file. ";
671  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
672  }
673 
674  // Swath open
675  swathfd = SWopen(const_cast < char *>(accessed.c_str()), DFACC_READ);
676  if (-1 == swathfd) {
677  SDend(sdfd);
678  Hclose(fileid);
679  GDclose(gridfd);
680  string invalid_file_msg="HDF-EOS SWopen error for the file ";
681  invalid_file_msg +=accessed;
682  invalid_file_msg +=". It is very possible that this file is not an HDF-EOS2 file. ";
683  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
684 
685  }
686 
687  try {
688 
689  // Here we will check if ECS_Metadata key if set. For DataDDS,
690  // if either H4.DisableECSMetaDataMin or H4.DisableECSMetaDataAll is set,
691  // the HDF-EOS2 coremetadata or archivemetadata will not be passed to DAP.
692  bool ecs_metadata = true;
693 #if 0
694  if((true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataMin"))
695  || (true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataAll")))
696  ecs_metadata = false;
697 #endif
698  if((true == _disable_ecsmetadata_min)
699  || (true == _disable_ecsmetadata_all))
700  ecs_metadata = false;
701 
702  read_das_use_eos2lib(*das, accessed,sdfd,fileid,gridfd,swathfd,ecs_metadata,&h4file,&eosfile);
703  Ancillary::read_ancillary_das(*das, accessed);
704 
705  // Pass file pointer(h4file, eosfile) from DAS to DDS.
706  read_dds_use_eos2lib(*dds, accessed,sdfd,fileid,gridfd,swathfd,h4file,eosfile);
707  }
708  catch(...) {
709  close_fileid(sdfd,fileid,gridfd,swathfd,h4file,eosfile);
710  throw;
711  }
712 
713  if(eosfile != nullptr)
714  delete eosfile;
715  GDclose(gridfd);
716  SWclose(swathfd);
717 
718 #else
719  try {
720  read_das_hdfsp(*das, accessed,sdfd,fileid,&h4file);
721  Ancillary::read_ancillary_das(*das, accessed);
722 
723  // Pass file pointer(h4file) from DAS to DDS.
724  read_dds_hdfsp(*dds, accessed,sdfd,fileid,h4file);
725  }
726  catch(...) {
727  close_hdf4_fileid(sdfd,fileid,h4file);
728  throw;
729  }
730 #endif
731  close_hdf4_fileid(sdfd,fileid,h4file);
732  }
733  else {
734  read_das(*das, accessed);
735  Ancillary::read_ancillary_das(*das, accessed);
736  read_dds(*dds, accessed);
737  }
738 
739  Ancillary::read_ancillary_dds(*dds, accessed);
740  dds->transfer_attributes(das);
741  bdds->set_constraint(dhi);
742  bdds->clear_container();
743 
744  }
745 
746  catch (BESError & e) {
747  throw;
748  }
749  catch (InternalErr & e) {
750  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
751  __FILE__, __LINE__);
752  }
753  catch (Error & e) {
754  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
755  __FILE__, __LINE__);
756  }
757  catch (...) {
758  string s = "unknown exception caught building DAP2 Data Response from an HDF4 data resource";
759  throw BESDapError(s, true, unknown_error, __FILE__, __LINE__);
760  }
761 
762  return true;
763 }
764 
765 bool HDF4RequestHandler::hdf4_build_data_with_IDs(BESDataHandlerInterface & dhi) {
766 
767  BESStopWatch sw;
768  if (BESDebug::IsSet(TIMING_LOG_KEY))
769  sw.start("HDF4RequestHandler::hdf4_build_data_with_IDs", dhi.data[REQUEST_ID]);
770 
771  int32 sdfd = -1;
772  int32 fileid = -1;
773  HDFSP::File *h4file = nullptr;
774 #ifdef USE_HDFEOS2_LIB
775  int32 gridfd = -1;
776  int32 swathfd = -1;
777  HDFEOS2::File *eosfile = nullptr;
778 #endif
779 
780  BESResponseObject *response = dhi.response_handler->get_response_object();
781  auto bdds = dynamic_cast<BESDataDDSResponse *> (response);
782 
783  if (!bdds)
784  throw BESInternalError("cast error", __FILE__, __LINE__);
785 
786  try {
787  bdds->set_container(dhi.container->get_symbolic_name());
788 
789  // Create a new HDF4DDS object.
790  HDF4DDS *hdds = new HDF4DDS(bdds->get_dds());
791 
792  // delete the old object.
793  delete bdds->get_dds();
794 
795  bdds->set_dds(hdds);
796 
797 #if 0
798  //ConstraintEvaluator & ce = bdds->get_ce();
799 #endif
800 
801  string accessed = dhi.container->access();
802  hdds->filename(accessed);
803 
804  auto das = new DAS;
805  BESDASResponse bdas(das);
806  bdas.set_container(dhi.container->get_symbolic_name());
807 
808 
809  // Obtain HDF4 file IDs
810  //SDstart
811  sdfd = SDstart (accessed.c_str(), DFACC_READ);
812  if( -1 == sdfd) {
813  string invalid_file_msg="HDF4 SDstart error for the file ";
814  invalid_file_msg +=accessed;
815  invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
816  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
817  }
818  // H open
819  fileid = Hopen(accessed.c_str(), DFACC_READ,0);
820  if (-1 == fileid) {
821  SDend(sdfd);
822  string invalid_file_msg="HDF4 Hopen error for the file ";
823  invalid_file_msg +=accessed;
824  invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
825  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
826  }
827 
828 #ifdef USE_HDFEOS2_LIB
829 
830 
831  // Obtain HDF-EOS2 file IDs with the file open APIs.
832 
833  // Grid open
834  gridfd = GDopen(const_cast < char *>(accessed.c_str()), DFACC_READ);
835  if (-1 == gridfd) {
836  SDend(sdfd);
837  Hclose(fileid);
838  string invalid_file_msg="HDF-EOS GDopen error for the file ";
839  invalid_file_msg +=accessed;
840  invalid_file_msg +=". It is very possible that this file is not an HDF-EOS2 file. ";
841  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
842  }
843 
844  // Swath open
845  swathfd = SWopen(const_cast < char *>(accessed.c_str()), DFACC_READ);
846  if (-1 == swathfd) {
847  SDend(sdfd);
848  Hclose(fileid);
849  GDclose(gridfd);
850  string invalid_file_msg="HDF-EOS SWopen error for the file ";
851  invalid_file_msg +=accessed;
852  invalid_file_msg +=". It is very possible that this file is not an HDF-EOS2 file. ";
853  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
854  }
855 
856  hdds->setHDF4Dataset(sdfd,fileid,gridfd,swathfd);
857 
858  // Here we will check if ECS_Metadata key if set. For DataDDS,
859  // if either H4.DisableECSMetaDataMin or H4.DisableECSMetaDataAll is set,
860  // the HDF-EOS2 coremetadata or archivemetadata will not be passed to DAP.
861  bool ecs_metadata = true;
862 #if 0
863  if((true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataMin"))
864  || (true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataAll")))
865 #endif
866  if((true == _disable_ecsmetadata_min)
867  || (true == _disable_ecsmetadata_all))
868  ecs_metadata = false;
869 
870  read_das_use_eos2lib(*das, accessed,sdfd,fileid,gridfd,swathfd,ecs_metadata,&h4file,&eosfile);
871 
872 #if 0
873  //read_das_use_eos2lib(*das, accessed,sdfd,fileid,gridfd,swathfd,true);
874 #endif
875 
876  Ancillary::read_ancillary_das(*das, accessed);
877 
878  // Pass file pointer(h4file, eosfile) from DAS to DDS.
879  read_dds_use_eos2lib(*hdds, accessed,sdfd,fileid,gridfd,swathfd,h4file,eosfile);
880 
881  if(eosfile != nullptr)
882  delete eosfile;
883 
884 #else
885  hdds->setHDF4Dataset(sdfd,fileid);
886  read_das_hdfsp(*das, accessed,sdfd,fileid,&h4file);
887  Ancillary::read_ancillary_das(*das, accessed);
888 
889  // Pass file pointer(h4file) from DAS to DDS.
890  read_dds_hdfsp(*hdds, accessed,sdfd,fileid,h4file);
891 #endif
892  if(h4file != nullptr)
893  delete h4file;
894 
895  Ancillary::read_ancillary_dds(*hdds, accessed);
896 
897  hdds->transfer_attributes(das);
898 
899  bdds->set_constraint(dhi);
900 
901  bdds->clear_container();
902 
903 
904 // File IDs are closed by the derived class.
905 #if 0
906  if(true == usecf) {
907 #ifdef USE_HDFEOS2_LIB
908  GDclose(gridfd);
909  SWclose(swathfd);
910 
911 #endif
912  SDend(sdfd);
913  Hclose(fileid);
914  }
915 #endif
916  }
917 
918  catch (InternalErr & e) {
919  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
920  __FILE__, __LINE__);
921  }
922  catch (Error & e) {
923  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
924  __FILE__, __LINE__);
925  }
926  catch (...) {
927 #ifdef USE_HDFEOS2_LIB
928  close_fileid(sdfd,fileid,gridfd,swathfd,h4file,eosfile);
929 #else
930  close_hdf4_fileid(sdfd,fileid,h4file);
931 #endif
932  string s = "unknown exception caught building DAP2 data response from an HDF4 data resource";
933  throw BESDapError(s, true, unknown_error, __FILE__, __LINE__);
934  }
935 
936  return true;
937 }
938 
939 // Special function to build DDS for HDF4 SDS-only DDS. One can turn
940 // on cache for this function. Currently only AIRS version 6 is supported.
941 bool HDF4RequestHandler::hdf4_build_dds_cf_sds(BESDataHandlerInterface &dhi){
942 
943  int32 sdfd = -1;
944  HDFSP::File *h4file = nullptr;
945 
946  BESResponseObject *response = dhi.response_handler->get_response_object();
947  auto bdds = dynamic_cast<BESDDSResponse *> (response);
948 
949  if (!bdds)
950  throw BESInternalError("cast error", __FILE__, __LINE__);
951 
952  try {
953  bdds->set_container(dhi.container->get_symbolic_name());
954 
955  DDS *dds = bdds->get_dds();
956 
957 #if 0
958  //BaseTypeFactory* factory = new BaseTypeFactory ;
959  //dds->set_factory( factory ) ;
960 #endif
961 
962  string accessed = dhi.container->access();
963  dds->filename(accessed);
964 
965  auto das = new DAS;
966  BESDASResponse bdas(das);
967 
968  // Check and set up dds and das cache files.
969  string base_file_name = basename(dhi.container->access());
970  bool das_set_cache = false;
971  bool dds_set_cache = false;
972  bool dds_das_get_cache = false;
973  string das_filename;
974  string dds_filename;
975 
976  // Check if the EnableMetaData key is set.
977 #if 0
978  string check_enable_mcache_key="H4.EnableMetaDataCacheFile";
979  bool turn_on_enable_mcache_key= false;
980  turn_on_enable_mcache_key = HDFCFUtil::check_beskeys(check_enable_mcache_key);
981  if(true == turn_on_enable_mcache_key) {// the EnableMetaData key is set
982 #endif
983  if(true == _enable_metadata_cachefile) {// the EnableMetaData key is set
984 
985 
986 #if 0
987  string md_cache_dir;
988  bool found = false;
989  string key = "H4.Cache.metadata.path";
990  TheBESKeys::TheKeys()->get_value(key,md_cache_dir,found);
991 #endif
992  if(true == _cache_metadata_path_exist) {// the metadata.path key is set
993 
994  // Create DAS and DDS cache file names
995  das_filename = _cache_metadata_path + "/" + base_file_name +"_das";
996  dds_filename = _cache_metadata_path + "/" + base_file_name +"_dds";
997 
998  // When the returned value das_set_cache is false, read DAS from the cache file.
999  // Otherwise, do nothing.
1000  das_set_cache = rw_das_cache_file(das_filename,das,false);
1001 
1002  // When the returned value dds_set_cache is false, read DDS from the cache file.
1003  // Otherwise, do nothing.
1004  dds_set_cache = r_dds_cache_file(dds_filename,dds,accessed);
1005 
1006  // Set the flag to obtain DDS and DAS from the cache files.
1007  // Here, we require that both DDS and DAS should be cached.
1008  // We don't support caching one and not caching another.
1009  if((false == das_set_cache)&&(false == dds_set_cache))
1010  dds_das_get_cache = true;
1011  }
1012  }
1013 
1014 
1015  // We need to go back to retrieve DDS, DAS from the HDF file.
1016  if(false == dds_das_get_cache) {
1017 
1018  // Obtain SD ID, this is the only ID we need to use.
1019  sdfd = SDstart (accessed.c_str(), DFACC_READ);
1020  if( -1 == sdfd) {
1021 
1022  string invalid_file_msg="HDF4 SDstart error for the file ";
1023  invalid_file_msg +=accessed;
1024  invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
1025  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1026  }
1027 
1028 
1029  // Here we will check if ECS_Metadata key if set. For DDS and DAS,
1030  // only when the DisableECSMetaDataAll key is set, ecs_metadata is off.
1031  bool ecs_metadata = !_disable_ecsmetadata_all;
1032 #if 0
1033  bool ecs_metadata = true;
1034  if((true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataMin"))
1035  || (true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataAll")))
1036  ecs_metadata = false;
1037 #endif
1038 
1039  read_das_sds(*das, accessed,sdfd,ecs_metadata,&h4file);
1040 
1041  Ancillary::read_ancillary_das(*das, accessed);
1042 
1043  // Pass file pointer(h4file) from DAS to DDS, also pass the flag
1044  // dds_set_cache to indicate if the handler needs to cache DDS.
1045  read_dds_sds(*dds, accessed,sdfd,h4file,dds_set_cache);
1046 
1047  // We also need to cache DAS if das_set_cache is set.
1048  if(true == das_set_cache) {
1049  if(das_filename =="")
1050  throw InternalErr(__FILE__,__LINE__,"DAS cache file name should be set ");
1051  rw_das_cache_file(das_filename,das,true);
1052  }
1053 
1054  }
1055 
1056  Ancillary::read_ancillary_dds(*dds, accessed);
1057  dds->transfer_attributes(das);
1058  bdds->set_constraint(dhi);
1059 
1060  bdds->clear_container();
1061 
1062  if(h4file != nullptr)
1063  delete h4file;
1064 
1065  if(sdfd != -1)
1066  SDend(sdfd);
1067 #if 0
1068  // Cannot(should not) delete factory here. factory will be released by DAP? or errors occur.
1069  //if(factory != nullptr)
1070  // delete factory;
1071 #endif
1072 
1073  }
1074 
1075  catch (BESError & e) {
1076  throw;
1077  }
1078  catch (InternalErr & e) {
1079  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
1080  __FILE__, __LINE__);
1081  }
1082  catch (Error & e) {
1083  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
1084  __FILE__, __LINE__);
1085  }
1086  catch (...) {
1087  if(sdfd != -1)
1088  SDend(sdfd);
1089  if(h4file != nullptr)
1090  delete h4file;
1091  string s = "unknown exception caught building HDF4 DataDDS";
1092  throw BESDapError(s, true, unknown_error, __FILE__, __LINE__);
1093  }
1094 
1095  return true;
1096 }
1097 
1098 // Special function to build DAS for HDF4 SDS-only DDS. One can turn
1099 // on cache for this function. Currently only AIRS version 6 is supported.
1100 bool HDF4RequestHandler::hdf4_build_das_cf_sds(BESDataHandlerInterface &dhi){
1101 
1102  int32 sdfd = -1;
1103  HDFSP::File *h4file = nullptr;
1104 
1105  BESResponseObject *response = dhi.response_handler->get_response_object();
1106  auto bdas = dynamic_cast<BESDASResponse *> (response);
1107 
1108  if (!bdas)
1109  throw BESInternalError("cast error", __FILE__, __LINE__);
1110 
1111  try {
1112  bdas->set_container(dhi.container->get_symbolic_name());
1113 
1114  auto das = bdas->get_das();
1115  string base_file_name = basename(dhi.container->access());
1116 
1117  string accessed = dhi.container->access();
1118 
1119  // Check if the enable metadata cache key is set and set the key appropriate.
1120  bool das_set_cache = false;
1121  bool das_get_cache = false;
1122  string das_filename;
1123 #if 0
1124  string check_enable_mcache_key="H4.EnableMetaDataCacheFile";
1125  bool turn_on_enable_mcache_key= false;
1126  turn_on_enable_mcache_key = HDFCFUtil::check_beskeys(check_enable_mcache_key);
1127  if(true == turn_on_enable_mcache_key) { // find the metadata cache key.
1128 #endif
1129  if(true == _enable_metadata_cachefile) { // find the metadata cache key.
1130 #if 0
1131  string md_cache_dir;
1132  string key = "H4.Cache.metadata.path";
1133  bool found = false;
1134  TheBESKeys::TheKeys()->get_value(key,md_cache_dir,found);
1135  if(true == found) { // Also find the metadata cache.
1136 #endif
1137  if(true == _cache_metadata_path_exist) {
1138 
1139  // Create the DAS cache file name.
1140  das_filename = _cache_metadata_path + "/" + base_file_name +"_das";
1141 
1142  // Read the DAS from the cached file, if das_set_cache is false.
1143  // When the das_set_cache is true, need to create a das cache file.
1144  das_set_cache = rw_das_cache_file(das_filename,das,false);
1145 
1146  // Not set cache, must get the das from cache, so das_get_cache should be true.
1147  if(false == das_set_cache)
1148  das_get_cache = true;
1149  }
1150  }
1151 
1152  // Need to retrieve DAS from the HDF4 file.
1153  if(false == das_get_cache) {
1154  // Obtain SD ID.
1155  sdfd = SDstart (accessed.c_str(), DFACC_READ);
1156  if( -1 == sdfd){
1157  string invalid_file_msg="HDF4 SDstart error for the file ";
1158  invalid_file_msg +=accessed;
1159  invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
1160  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1161  }
1162 
1163 
1164 #if 0
1165  bool ecs_metadata = true;
1166  if((true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataMin"))
1167  || (true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataAll")))
1168  ecs_metadata = false;
1169 #endif
1170  // Here we will check if ECS_Metadata key if set. For DDS and DAS,
1171  // only when the DisableECSMetaDataAll key is set, ecs_metadata is off.
1172  bool ecs_metadata = !_disable_ecsmetadata_all;
1173 
1174  read_das_sds(*das, accessed,sdfd,ecs_metadata,&h4file);
1175  Ancillary::read_ancillary_das(*das, accessed);
1176 
1177  // Generate DAS cache file if the cache flag is set.
1178  if(true == das_set_cache)
1179  rw_das_cache_file(das_filename,das,true);
1180 
1181  }
1182 
1183  bdas->clear_container();
1184 
1185  if(h4file != nullptr)
1186  delete h4file;
1187 
1188  if(sdfd != -1)
1189  SDend(sdfd);
1190 
1191  }
1192 
1193  catch (BESError & e) {
1194  throw;
1195  }
1196  catch (InternalErr & e) {
1197  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
1198  __FILE__, __LINE__);
1199  }
1200  catch (Error & e) {
1201  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
1202  __FILE__, __LINE__);
1203  }
1204  catch (...) {
1205  if(sdfd != -1)
1206  SDend(sdfd);
1207  if(h4file != nullptr)
1208  delete h4file;
1209  string s = "unknown exception caught building HDF4 DataDDS";
1210  throw BESDapError(s, true, unknown_error, __FILE__, __LINE__);
1211  }
1212  return true;
1213 }
1214 // Special function to build data for HDF4 SDS-only DDS. One can turn
1215 // on cache for this function. Currently only AIRS version 6 is supported.
1216 bool HDF4RequestHandler::hdf4_build_data_cf_sds(BESDataHandlerInterface &dhi){
1217 
1218  int32 sdfd = -1;
1219  HDFSP::File *h4file = nullptr;
1220 
1221  BESResponseObject *response = dhi.response_handler->get_response_object();
1222  auto bdds = dynamic_cast<BESDataDDSResponse *> (response);
1223 
1224  if (!bdds)
1225  throw BESInternalError("cast error", __FILE__, __LINE__);
1226 
1227  try {
1228  bdds->set_container(dhi.container->get_symbolic_name());
1229 
1230  DDS *dds = bdds->get_dds();
1231 
1232  string accessed = dhi.container->access();
1233  dds->filename(accessed);
1234 
1235  auto das = new DAS;
1236  BESDASResponse bdas(das);
1237  bdas.set_container(dhi.container->get_symbolic_name());
1238 
1239 #if 0
1240  //BaseTypeFactory* factory = new BaseTypeFactory ;
1241  //dds->set_factory( factory ) ;
1242 #endif
1243 
1244  string base_file_name = basename(dhi.container->access());
1245 
1246  bool das_set_cache = false;
1247  bool dds_set_cache = false;
1248  bool dds_das_get_cache = false;
1249  string das_filename;
1250  string dds_filename;
1251 
1252 #if 0
1253  string check_enable_mcache_key="H4.EnableMetaDataCacheFile";
1254  bool _disable_ecsmetadata_allturn_on_enable_mcache_key= false;
1255  turn_on_enable_mcache_key = HDFCFUtil::check_beskeys(check_enable_mcache_key);
1256  if(true == turn_on_enable_mcache_key) {
1257 #endif
1258  if(true == _enable_metadata_cachefile) {
1259 
1260  if(true == _cache_metadata_path_exist) {
1261  BESDEBUG("h4", "H4.Cache.metadata.path key is set and metadata cache key is set." << endl);
1262 
1263  // Notice, since the DAS output may be different between DAS/DDS service and DataDDS service.
1264  // See comments about ecs_metadata below.
1265  // So we create a different DAS file. This can be optimized in the future if necessary.
1266  das_filename = _cache_metadata_path + "/" + base_file_name +"_das_dd";
1267  dds_filename = _cache_metadata_path + "/" + base_file_name +"_dds";
1268 
1269  // If das_set_cache is true, data is read from the DAS cache.
1270  das_set_cache = rw_das_cache_file(das_filename,das,false);
1271 
1272  // If dds_set_cache is true, data is read from the DDS cache.
1273  dds_set_cache = r_dds_cache_file(dds_filename,dds,accessed);
1274  // Need to set a flag to generate DAS and DDS cache files
1275  if((false == das_set_cache)&&(false == dds_set_cache))
1276  dds_das_get_cache = true;
1277  }
1278  }
1279  if(false == dds_das_get_cache) {
1280 
1281  // Obtain HDF4 SD ID
1282  sdfd = SDstart (accessed.c_str(), DFACC_READ);
1283  if( -1 == sdfd){
1284  string invalid_file_msg="HDF4 SDstart error for the file ";
1285  invalid_file_msg +=accessed;
1286  invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
1287  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1288  }
1289 
1290 
1291  // Here we will check if ECS_Metadata key if set. For DataDDS,
1292  // if either H4.DisableECSMetaDataMin or H4.DisableECSMetaDataAll is set,
1293  // the HDF-EOS2 coremetadata or archivemetadata will not be passed to DAP.
1294  bool ecs_metadata = true;
1295 #if 0
1296  if((true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataMin"))
1297  || (true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataAll")))
1298 #endif
1299  if((true == _disable_ecsmetadata_min)
1300  || (true == _disable_ecsmetadata_all))
1301  ecs_metadata = false;
1302 
1303  read_das_sds(*das, accessed,sdfd,ecs_metadata,&h4file);
1304 
1305  Ancillary::read_ancillary_das(*das, accessed);
1306 
1307  // Need to write DAS to a cache file.
1308  if(true == das_set_cache) {
1309  rw_das_cache_file(das_filename,das,true);
1310  }
1311 
1312  // Pass file pointer h4file from DAS to DDS. Also need to pass the
1313  // flag that indicates if a DDS file needs to be created.
1314  read_dds_sds(*dds, accessed,sdfd,h4file,dds_set_cache);
1315 
1316  }
1317 
1318  Ancillary::read_ancillary_dds(*dds, accessed);
1319 
1320  dds->transfer_attributes(das);
1321 
1322  bdds->set_constraint(dhi);
1323 
1324  bdds->clear_container();
1325 
1326  if(h4file != nullptr)
1327  delete h4file;
1328 
1329  if(sdfd != -1)
1330  SDend(sdfd);
1331  }
1332 
1333  catch (BESError & e) {
1334  throw;
1335  }
1336  catch (InternalErr & e) {
1337  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
1338  __FILE__, __LINE__);
1339  }
1340  catch (Error & e) {
1341  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
1342  __FILE__, __LINE__);
1343  }
1344  catch (...) {
1345  if(sdfd != -1)
1346  SDend(sdfd);
1347  if(h4file != nullptr)
1348  delete h4file;
1349  string s = "unknown exception caught building HDF4 DataDDS";
1350  throw BESDapError(s, true, unknown_error, __FILE__, __LINE__);
1351  }
1352 
1353 
1354  return true;
1355 }
1356 
1357 // Notice the metadata cache doesn't apply when the EnableFileID key is set.
1358 // This is because the file ID generated by creating DDS/DAS WO cache is needed to access the data.
1359 // So to make this work at any time, we have to create SDS ID even when the EnableFileID key is set.
1360 // This is against the purpose of EnableFileID key. To acheieve the same purpose for AIRS,
1361 // one can set the DataCache key, the performance is similar or even better than just using the EnableFileID key.
1362 bool HDF4RequestHandler::hdf4_build_data_cf_sds_with_IDs(BESDataHandlerInterface &dhi){
1363 
1364  int32 sdfd = -1;
1365  HDFSP::File *h4file = nullptr;
1366 
1367  BESResponseObject *response = dhi.response_handler->get_response_object();
1368  auto bdds = dynamic_cast<BESDataDDSResponse *> (response);
1369 
1370  if (!bdds)
1371  throw BESInternalError("cast error", __FILE__, __LINE__);
1372 
1373  try {
1374  bdds->set_container(dhi.container->get_symbolic_name());
1375 
1376  auto hdds = new HDF4DDS(bdds->get_dds());
1377 
1378 
1379  delete bdds->get_dds();
1380 
1381  bdds->set_dds(hdds);
1382 
1383 
1384  string accessed = dhi.container->access();
1385  hdds->filename(accessed);
1386 
1387  auto das = new DAS;
1388  BESDASResponse bdas(das);
1389  bdas.set_container(dhi.container->get_symbolic_name());
1390 
1391  //Obtain SD ID.
1392  sdfd = SDstart (accessed.c_str(), DFACC_READ);
1393  if( -1 == sdfd) {
1394  string invalid_file_msg="HDF4 SDstart error for the file ";
1395  invalid_file_msg +=accessed;
1396  invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
1397  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1398  }
1399 
1400  hdds->setHDF4Dataset(sdfd,-1);
1401 
1402  // Here we will check if ECS_Metadata key if set. For DataDDS,
1403  // if either H4.DisableECSMetaDataMin or H4.DisableECSMetaDataAll is set,
1404  // the HDF-EOS2 coremetadata or archivemetadata will not be passed to DAP.
1405  bool ecs_metadata = true;
1406 #if 0
1407  if((true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataMin"))
1408  || (true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataAll")))
1409 #endif
1410  if((true == _disable_ecsmetadata_min)
1411  || (true == _disable_ecsmetadata_all))
1412  ecs_metadata = false;
1413 
1414  read_das_sds(*das, accessed,sdfd,ecs_metadata,&h4file);
1415 #if 0
1416  //read_das_use_eos2lib(*das, accessed,sdfd,fileid,gridfd,swathfd,true);
1417 #endif
1418 
1419  Ancillary::read_ancillary_das(*das, accessed);
1420 
1421  // Pass file pointer(h4file, eosfile) from DAS to DDS.
1422  read_dds_sds(*hdds, accessed,sdfd,h4file,false);
1423 
1424  if(h4file != nullptr)
1425  delete h4file;
1426 
1427  Ancillary::read_ancillary_dds(*hdds, accessed);
1428 
1429  hdds->transfer_attributes(das);
1430 
1431  bdds->set_constraint(dhi);
1432 
1433  bdds->clear_container();
1434 
1435  }
1436 
1437  catch (BESError & e) {
1438  throw;
1439  }
1440  catch (InternalErr & e) {
1441  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
1442  __FILE__, __LINE__);
1443  }
1444  catch (Error & e) {
1445  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
1446  __FILE__, __LINE__);
1447  }
1448  catch (...) {
1449  if(sdfd != -1)
1450  SDend(sdfd);
1451  if(h4file != nullptr)
1452  delete h4file;
1453  string s = "unknown exception caught building HDF4 DataDDS";
1454  throw BESDapError(s, true, unknown_error, __FILE__, __LINE__);
1455  }
1456 
1457 
1458  return true;
1459 }
1460 bool HDF4RequestHandler::hdf4_build_dmr(BESDataHandlerInterface &dhi)
1461 {
1462 
1463  BESStopWatch sw;
1464  if (BESDebug::IsSet(TIMING_LOG_KEY))
1465  sw.start("HDF4RequestHandler::hdf4_build_dmr", dhi.data[REQUEST_ID]);
1466 
1467  // Because this code does not yet know how to build a DMR directly, use
1468  // the DMR ctor that builds a DMR using a 'full DDS' (a DDS with attributes).
1469  // First step, build the 'full DDS'
1470  string data_path = dhi.container->access();
1471 
1472  BaseTypeFactory factory;
1473  DDS dds(&factory, name_path(data_path), "3.2");
1474  dds.filename(data_path);
1475 
1476  DAS das;
1477 
1478  int32 sdfd = -1;
1479  int32 fileid = -1;
1480 
1481  // Since passing file IDs requires to use the derived class and it
1482  // causes the management of code structure messy, we first handle this with
1483  // another method.
1484  if(true == _usecf) {
1485 
1486  if(true == _pass_fileid)
1487  return hdf4_build_dmr_with_IDs(dhi);
1488 
1489  }
1490 
1491 
1492  try {
1493 
1494  if (true == _usecf) {
1495 
1496  HDFSP::File *h4file = nullptr;
1497 
1498  // Obtain HDF4 file IDs
1499  //SDstart
1500  sdfd = SDstart (data_path.c_str(), DFACC_READ);
1501  if( -1 == sdfd){
1502  string invalid_file_msg="HDF4 SDstart error for the file ";
1503  invalid_file_msg +=data_path;
1504  invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
1505  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1506  }
1507 
1508  // H open
1509  fileid = Hopen(data_path.c_str(), DFACC_READ,0);
1510  if (-1 == fileid) {
1511  SDend(sdfd);
1512  string invalid_file_msg="HDF4 Hopen error for the file ";
1513  invalid_file_msg +=data_path;
1514  invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
1515  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1516  }
1517 
1518 #ifdef USE_HDFEOS2_LIB
1519 
1520  int32 gridfd = -1;
1521  int32 swathfd = -1;
1522 
1523  HDFEOS2::File *eosfile = nullptr;
1524 
1525  // Obtain HDF-EOS2 file IDs with the file open APIs.
1526  // Grid open
1527  gridfd = GDopen(const_cast < char *>(data_path.c_str()), DFACC_READ);
1528  if (-1 == gridfd) {
1529  SDend(sdfd);
1530  Hclose(fileid);
1531  string invalid_file_msg="HDF-EOS GDopen error for the file ";
1532  invalid_file_msg +=data_path;
1533  invalid_file_msg +=". It is very possible that this file is not an HDF-EOS2 file. ";
1534  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1535  }
1536 
1537  // Swath open
1538  swathfd = SWopen(const_cast < char *>(data_path.c_str()), DFACC_READ);
1539  if (-1 == swathfd) {
1540  SDend(sdfd);
1541  Hclose(fileid);
1542  GDclose(gridfd);
1543  string invalid_file_msg="HDF-EOS SWopen error for the file ";
1544  invalid_file_msg +=data_path;
1545  invalid_file_msg +=". It is very possible that this file is not an HDF-EOS2 file. ";
1546  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1547  }
1548 
1549 
1550  // Here we will check if ECS_Metadata key if set. For DAP4's DMR,
1551  // if either H4.DisableECSMetaDataMin or H4.DisableECSMetaDataAll is set,
1552  // the HDF-EOS2 coremetadata or archivemetadata will not be passed to DAP.
1553  // This is one difference between DAP2 and DAP4 mapping. Since
1554  // people can use BES key to turn on the ECS metadata, so this is okay.
1555  // KY 2014-10-23
1556  bool ecs_metadata = true;
1557 #if 0
1558  if((true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataMin"))
1559  || (true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataAll")))
1560 #endif
1561  if((true == _disable_ecsmetadata_min)
1562  || (true == _disable_ecsmetadata_all))
1563  ecs_metadata = false;
1564 
1565  try {
1566 
1567  read_das_use_eos2lib(das, data_path,sdfd,fileid,gridfd,swathfd,ecs_metadata,&h4file,&eosfile);
1568 #if 0
1569  //read_das_use_eos2lib(*das, data_path,sdfd,fileid,gridfd,swathfd,true);
1570 #endif
1571  Ancillary::read_ancillary_das(das, data_path);
1572 
1573  // Pass file pointer(h4file, eosfile) from DAS to DDS
1574  read_dds_use_eos2lib(dds, data_path,sdfd,fileid,gridfd,swathfd,h4file,eosfile);
1575  }
1576 
1577  catch(...) {
1578  close_fileid(sdfd,fileid,gridfd,swathfd,h4file,eosfile);
1579  throw;
1580  }
1581  if(eosfile != nullptr)
1582  delete eosfile;
1583  GDclose(gridfd);
1584  SWclose(swathfd);
1585 
1586 #else
1587  try {
1588  read_das_hdfsp(das, data_path,sdfd,fileid,&h4file);
1589  Ancillary::read_ancillary_das(das, data_path);
1590 
1591  // Pass file pointer(h4file) from DAS to DDS.
1592  read_dds_hdfsp(dds, data_path,sdfd,fileid,h4file);
1593  }
1594  catch(...) {
1595  close_hdf4_fileid(sdfd,fileid,h4file);
1596  throw;
1597  //throw InternalErr(__FILE__,__LINE__,"read_dds_hdfsp error");
1598  }
1599 #endif
1600  close_hdf4_fileid(sdfd,fileid,h4file);
1601 
1602  }
1603  else {
1604  read_das(das, data_path);
1605  Ancillary::read_ancillary_das(das, data_path);
1606  read_dds(dds, data_path);
1607  }
1608 
1609 
1610  Ancillary::read_ancillary_dds(dds, data_path);
1611 
1612  dds.transfer_attributes(&das);
1613 
1614  }
1615 
1616  catch (InternalErr & e) {
1617  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
1618  __FILE__, __LINE__);
1619  }
1620  catch (Error & e) {
1621  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
1622  __FILE__, __LINE__);
1623  }
1624  catch (...) {
1625  string s = "unknown exception caught building HDF4 DataDDS";
1626  throw BESDapError(s, true, unknown_error, __FILE__, __LINE__);
1627  }
1628 
1629 #if 0
1630  //dds.print(cout);
1631  //dds.print_das(cout);
1632 #endif
1633  //
1634  // Extract the DMR Response object - this holds the DMR used by the
1635  // other parts of the framework.
1636  BESResponseObject *response = dhi.response_handler->get_response_object();
1637  BESDMRResponse &bes_dmr = dynamic_cast<BESDMRResponse &>(*response);
1638 
1639  DMR *dmr = bes_dmr.get_dmr();
1640 
1641  D4BaseTypeFactory MyD4TypeFactory;
1642  dmr->set_factory(&MyD4TypeFactory);
1643 #if 0
1644  //dmr->set_factory(new D4BaseTypeFactory);
1645 #endif
1646  dmr->build_using_dds(dds);
1647 
1648 #if 0
1649  //dmr->print(cout);
1650 #endif
1651 
1652  // Instead of fiddling with the internal storage of the DHI object,
1653  // (by setting dhi.data[DAP4_CONSTRAINT], etc., directly) use these
1654  // methods to set the constraints. But, why? Ans: from Patrick is that
1655  // in the 'container' mode of BES each container can have a different
1656  // CE.
1657  bes_dmr.set_dap4_constraint(dhi);
1658  bes_dmr.set_dap4_function(dhi);
1659  dmr->set_factory(nullptr);
1660 
1661  return true;
1662 }
1663 
1664 bool HDF4RequestHandler::hdf4_build_dmr_with_IDs(BESDataHandlerInterface & dhi) {
1665 
1666  BESStopWatch sw;
1667  if (BESDebug::IsSet(TIMING_LOG_KEY))
1668  sw.start("HDF4RequestHandler::hdf4_build_dmr_with_IDs", dhi.data[REQUEST_ID]);
1669 
1670  // Because this code does not yet know how to build a DMR directly, use
1671  // the DMR ctor that builds a DMR using a 'full DDS' (a DDS with attributes).
1672  // First step, build the 'full DDS'
1673  string data_path = dhi.container->access();
1674 
1675  BaseTypeFactory factory;
1676  DDS dds(&factory, name_path(data_path), "3.2");
1677  dds.filename(data_path);
1678 
1679  DAS das;
1680 
1681  int32 sdfd = -1;
1682  int32 fileid = -1;
1683 
1684  HDFSP::File *h4file = nullptr;
1685 
1686  // Obtain HDF4 file IDs
1687  //SDstart
1688  sdfd = SDstart (data_path.c_str(), DFACC_READ);
1689  if( -1 == sdfd){
1690  string invalid_file_msg="HDF4 SDstart error for the file ";
1691  invalid_file_msg +=data_path;
1692  invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
1693  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1694  }
1695 
1696  // H open
1697  fileid = Hopen(data_path.c_str(), DFACC_READ,0);
1698  if (-1 == fileid) {
1699  SDend(sdfd);
1700  string invalid_file_msg="HDF4 SDstart error for the file ";
1701  invalid_file_msg +=data_path;
1702  invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
1703  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1704  }
1705 
1706 
1707 #ifdef USE_HDFEOS2_LIB
1708 
1709  int32 gridfd = -1;
1710  int32 swathfd = -1;
1711 
1712  HDFEOS2::File *eosfile = nullptr;
1713  // Obtain HDF-EOS2 file IDs with the file open APIs.
1714  // Grid open
1715  gridfd = GDopen(const_cast < char *>(data_path.c_str()), DFACC_READ);
1716  if (-1 == gridfd) {
1717  SDend(sdfd);
1718  Hclose(fileid);
1719  string invalid_file_msg="HDF4 SDstart error for the file ";
1720  invalid_file_msg +=data_path;
1721  invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
1722  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1723 
1724  }
1725 
1726  // Swath open
1727  swathfd = SWopen(const_cast < char *>(data_path.c_str()), DFACC_READ);
1728  if (-1 == swathfd) {
1729  SDend(sdfd);
1730  Hclose(fileid);
1731  GDclose(gridfd);
1732  string invalid_file_msg="HDF-EOS SWopen error for the file ";
1733  invalid_file_msg +=data_path;
1734  invalid_file_msg +=". It is very possible that this file is not an HDF-EOS2 file. ";
1735  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1736 
1737  }
1738 
1739  // Here we will check if ECS_Metadata key if set. For DAP4's DMR,
1740  // if either H4.DisableECSMetaDataMin or H4.DisableECSMetaDataAll is set,
1741  // the HDF-EOS2 coremetadata or archivemetadata will not be passed to DAP.
1742  // This is one difference between DAP2 and DAP4 mapping. Since
1743  // people can use BES key to turn on the ECS metadata, so this is okay.
1744  // KY 2014-10-23
1745  bool ecs_metadata = true;
1746 #if 0
1747  if((true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataMin"))
1748  || (true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataAll")))
1749 #endif
1750  if((true == _disable_ecsmetadata_min)
1751  || (true == _disable_ecsmetadata_all))
1752  ecs_metadata = false;
1753 
1754  try {
1755 
1756  read_das_use_eos2lib(das, data_path,sdfd,fileid,gridfd,swathfd,ecs_metadata,&h4file,&eosfile);
1757 #if 0
1758  //read_das_use_eos2lib(*das, data_path,sdfd,fileid,gridfd,swathfd,true);
1759 #endif
1760  Ancillary::read_ancillary_das(das, data_path);
1761 
1762  // Pass file pointer(h4file, eosfile) from DAS to DDS.
1763  read_dds_use_eos2lib(dds, data_path,sdfd,fileid,gridfd,swathfd,h4file,eosfile);
1764  }
1765 
1766  catch(...) {
1767  close_fileid(sdfd,fileid,gridfd,swathfd,h4file,eosfile);
1768  throw;
1769  }
1770  if(eosfile != nullptr)
1771  delete eosfile;
1772 
1773 #else
1774  try {
1775  read_das_hdfsp(das, data_path,sdfd,fileid,&h4file);
1776  Ancillary::read_ancillary_das(das, data_path);
1777 
1778  // Pass file pointer(h4file) from DAS to DDS.
1779  read_dds_hdfsp(dds, data_path,sdfd,fileid,h4file);
1780  }
1781  catch(...) {
1782  close_hdf4_fileid(sdfd,fileid,h4file);
1783  throw;
1784  }
1785 #endif
1786  if(h4file != nullptr)
1787  delete h4file;
1788 
1789  Ancillary::read_ancillary_dds(dds, data_path);
1790 
1791  dds.transfer_attributes(&das);
1792 
1793 
1794 // File IDs are closed by the derived class.
1795 #if 0
1796  if(true == usecf) {
1797 #ifdef USE_HDFEOS2_LIB
1798  GDclose(gridfd);
1799  SWclose(swathfd);
1800 
1801 #endif
1802  SDend(sdfd);
1803  Hclose(fileid);
1804  }
1805 
1806  //dds.print(cout);
1807  //dds.print_das(cout);
1808 #endif
1809  // Extract the DMR Response object - this holds the DMR used by the
1810  // other parts of the framework.
1811  BESResponseObject *response = dhi.response_handler->get_response_object();
1812  BESDMRResponse &bes_dmr = dynamic_cast<BESDMRResponse &>(*response);
1813 
1814  // In this handler we use a different pattern since the handler specializes the DDS/DMR.
1815  // First, build the DMR adding the open handle to the HDF4 dataset, then free the DMR
1816  // the BES built and add this one. The HDF4DMR object will close the open dataset when
1817  // the BES runs the DMR's destructor.
1818  DMR *dmr = bes_dmr.get_dmr();
1819  D4BaseTypeFactory MyD4TypeFactory;
1820  dmr->set_factory(&MyD4TypeFactory);
1821  dmr->build_using_dds(dds);
1822  auto hdf4_dmr = new HDF4DMR(dmr);
1823 #ifdef USE_HDFEOS2_LIB
1824  hdf4_dmr->setHDF4Dataset(sdfd,fileid,gridfd,swathfd);
1825 #else
1826  hdf4_dmr->setHDF4Dataset(sdfd,fileid);
1827 #endif
1828  delete dmr; // The call below will make 'dmr' unreachable; delete it now to avoid a leak.
1829  bes_dmr.set_dmr(hdf4_dmr); // BESDMRResponse will delete hdf4_dmr
1830 
1831  // Instead of fiddling with the internal storage of the DHI object,
1832  // (by setting dhi.data[DAP4_CONSTRAINT], etc., directly) use these
1833  // methods to set the constraints. But, why? Ans: from Patrick is that
1834  // in the 'container' mode of BES each container can have a different
1835  // CE.
1836  bes_dmr.set_dap4_constraint(dhi);
1837  bes_dmr.set_dap4_function(dhi);
1838  hdf4_dmr->set_factory(nullptr);
1839 
1840  return true;
1841 
1842 }
1843 
1844 bool HDF4RequestHandler::hdf4_build_help(BESDataHandlerInterface & dhi) {
1845  BESResponseObject *response = dhi.response_handler->get_response_object();
1846  auto info = dynamic_cast<BESInfo *> (response);
1847  if (!info)
1848  throw BESInternalError("cast error", __FILE__, __LINE__);
1849 
1850  map < string, string > attrs;
1851  attrs["name"] = MODULE_NAME ;
1852  attrs["version"] = MODULE_VERSION ;
1853  list < string > services;
1854  BESServiceRegistry::TheRegistry()->services_handled(HDF4_NAME, services);
1855  if (services.empty() == false) {
1856  string handles = BESUtil::implode(services, ',');
1857  attrs["handles"] = handles;
1858  }
1859  info->begin_tag("module", &attrs);
1860  info->end_tag("module");
1861 
1862  return true;
1863 }
1864 
1865 bool HDF4RequestHandler::hdf4_build_version(BESDataHandlerInterface & dhi) {
1866  BESResponseObject *response = dhi.response_handler->get_response_object();
1867  auto info = dynamic_cast<BESVersionInfo *> (response);
1868  if (!info)
1869  throw BESInternalError("cast error", __FILE__, __LINE__);
1870 
1871  info->add_module(MODULE_NAME, MODULE_VERSION);
1872 
1873  return true;
1874 }
1875 
1876 #ifdef USE_HDFEOS2_LIB
1877 void close_fileid(int sdfd, int fileid,int gridfd, int swathfd,HDFSP:: File* h4file, HDFEOS2::File*eosfile) {
1878  if(h4file !=nullptr)
1879  delete h4file;
1880  if(sdfd != -1)
1881  SDend(sdfd);
1882  if(fileid != -1)
1883  Hclose(fileid);
1884 
1885 
1886  if(eosfile !=nullptr)
1887  delete eosfile;
1888  if(gridfd != -1)
1889  GDclose(gridfd);
1890  if(swathfd != -1)
1891  SWclose(swathfd);
1892 
1893 }
1894 #endif
1895 void close_hdf4_fileid(int sdfd, int fileid,HDFSP::File*h4file) {
1896 
1897  if(h4file !=nullptr)
1898  delete h4file;
1899 
1900  if(sdfd != -1)
1901  SDend(sdfd);
1902  if(fileid != -1)
1903  Hclose(fileid);
1904 
1905 }
1906 
1907 // Handling DAS and DDS cache. Currently we only apply the cache to special products(AIRS version 6 level 3 or 2).
1908 // read or write DAS from/to a cache file.
1909 bool rw_das_cache_file(const string & filename, DAS *das_ptr,bool w_flag) {
1910 
1911  bool das_set_cache = false;
1912  FILE *das_file = nullptr;
1913 
1914  if(false == w_flag) // open a cache file for reading.
1915  das_file = fopen(filename.c_str(),"r");
1916  else
1917  das_file = fopen(filename.c_str(),"w");
1918 
1919  if(nullptr == das_file) {
1920  if(ENOENT == errno) {
1921  // Since the das service always tries to read the data from a cache and if the cache file doesn't exist,
1922  // it will generates a cache file, so here we set a flag to indicate if a cache file needs to be generated.
1923  if(false == w_flag) {
1924  BESDEBUG("h4", "DAS set cache key is true." << endl);
1925  das_set_cache = true;
1926  }
1927  }
1928  else
1929  throw BESInternalError( "An error occurred trying to open a das cache file " + get_errno(), __FILE__, __LINE__);
1930  }
1931  else {
1932 
1933  int fd_das = fileno(das_file);
1934 
1935  // Set a corresponding read(shared) or write(exclusive) lock.
1936  struct flock *l_das;
1937  if(false == w_flag)
1938  l_das = lock(F_RDLCK);
1939  else
1940  l_das = lock(F_WRLCK);
1941 
1942  // Hold a lock.
1943  if(fcntl(fd_das,F_SETLKW,l_das) == -1) {
1944  fclose(das_file);
1945  ostringstream oss;
1946  oss << "cache process: " << l_das->l_pid << " triggered a locking error: " << get_errno();
1947  throw BESInternalError( oss.str(), __FILE__, __LINE__);
1948  }
1949 
1950  if(false == w_flag){
1951  // Read DAS from a cache file
1952  BESDEBUG("h4", "Obtaining DAS from the cache file" << endl);
1953  try {
1954  das_ptr->parse(das_file);
1955  }
1956  catch(...) {
1957  if(fcntl(fd_das,F_SETLK,lock(F_UNLCK)) == -1) {
1958  fclose(das_file);
1959  throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1960  }
1961  fclose(das_file);
1962  throw InternalErr(__FILE__,__LINE__,"Fail to parse the das from a das file.");
1963  }
1964  }
1965  else {
1966  // Write DAS to a cache file
1967  BESDEBUG("h4", "write DAS to a cache file" << endl);
1968  try {
1969  das_ptr->print(das_file);
1970  }
1971  catch(...) {
1972  if(fcntl(fd_das,F_SETLK,lock(F_UNLCK)) == -1) {
1973  fclose(das_file);
1974  throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1975  }
1976  fclose(das_file);
1977  throw InternalErr(__FILE__,__LINE__,"Fail to generate a das cache file.");
1978  }
1979 
1980  }
1981 
1982  // Unlock the cache file
1983  if(fcntl(fd_das,F_SETLK,lock(F_UNLCK)) == -1) {
1984  fclose(das_file);
1985  throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1986  }
1987  fclose(das_file);
1988 
1989  }
1990 
1991  return das_set_cache;
1992 
1993 }
1994 
1995 // Read dds from a cache file.
1996 bool r_dds_cache_file(const string & cache_filename, DDS *dds_ptr,const string & hdf4_filename) {
1997 
1998  bool dds_set_cache = false;
1999  FILE *dds_file = nullptr;
2000  dds_file = fopen(cache_filename.c_str(),"rb");
2001 
2002  if(nullptr == dds_file) {
2003  if(ENOENT == errno) {
2004  // Since the das service always tries to read the data from a cache and if the cache file doesn't exist,
2005  // it is supposed that the handler should generate a cache file,
2006  // so set a flag to indicate if a cache file needs to be generated.
2007  dds_set_cache = true;
2008  }
2009  else
2010  throw BESInternalError( "An error occurred trying to open a dds cache file " + get_errno(), __FILE__, __LINE__);
2011  }
2012  else {
2013 
2014  int fd_dds = fileno(dds_file);
2015  struct flock *l_dds;
2016  l_dds = lock(F_RDLCK);
2017 
2018  // hold a read(shared) lock to read dds from a file.
2019  if(fcntl(fd_dds,F_SETLKW,l_dds) == -1) {
2020  fclose(dds_file);
2021  ostringstream oss;
2022  oss << "cache process: " << l_dds->l_pid << " triggered a locking error: " << get_errno();
2023  throw BESInternalError( oss.str(), __FILE__, __LINE__);
2024  }
2025 
2026  try {
2027  HDFCFUtil::read_sp_sds_dds_cache(dds_file,dds_ptr,cache_filename,hdf4_filename);
2028  }
2029  catch(...) {
2030  if(fcntl(fd_dds,F_SETLK,lock(F_UNLCK)) == -1) {
2031  fclose(dds_file);
2032  throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
2033  }
2034 
2035  fclose(dds_file);
2036  throw InternalErr(__FILE__,__LINE__,"Fail to generate a dds cache file.");
2037  }
2038 
2039  if(fcntl(fd_dds,F_SETLK,lock(F_UNLCK)) == -1) {
2040  fclose(dds_file);
2041  throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
2042  }
2043 
2044  fclose(dds_file);
2045 
2046  }
2047 
2048  return dds_set_cache;
2049 
2050 }
2051 
2052 bool check_beskeys(const string & key) {
2053 
2054  bool found = false;
2055  string doset ="";
2056  const string dosettrue ="true";
2057  const string dosetyes = "yes";
2058 
2059  TheBESKeys::TheKeys()->get_value( key, doset, found ) ;
2060  if( true == found ) {
2061  doset = BESUtil::lowercase( doset ) ;
2062  if( dosettrue == doset || dosetyes == doset )
2063  return true;
2064  }
2065  return false;
2066 
2067 }
2068 
2069 bool get_beskeys(const string & key,string &key_value) {
2070 
2071  bool found = false;
2072 
2073  TheBESKeys::TheKeys()->get_value( key, key_value, found ) ;
2074  return found;
2075 
2076 }
2077 
2078 #if 0
2079 int get_cachekey_int(const string key) {
2080 
2081  bool found = false;
2082  int ret_value = 0;
2083  string doset ="";
2084 
2085  TheBESKeys::TheKeys()->get_value( key, doset, found ) ;
2086  if( true == found ) {
2087  istringstream iss(doset);
2088  iss >>ret_value;
2089  }
2090  return ret_value;
2091 
2092 }
2093 #endif
2094 #if 0
2095 void test_func(HDFSP::File**h4file) {
2096 cerr<<"OK to pass pointer of a nullptr pointer "<<endl;
2097 
2098 }
2099 #endif
BESContainer::get_symbolic_name
std::string get_symbolic_name() const
retrieve the symbolic name for this container
Definition: BESContainer.h:221
BESContainer::access
virtual std::string access()=0
returns the true name of this container
BESDASResponse
Represents an OPeNDAP DAS DAP2 data object within the BES.
Definition: BESDASResponse.h:44
BESDDSResponse
Holds a DDS object within the BES.
Definition: BESDDSResponse.h:50
BESDMRResponse
Represents an OPeNDAP DMR DAP4 data object within the BES.
Definition: BESDMRResponse.h:49
BESDapError
error object created from libdap error objects and can handle those errors
Definition: BESDapError.h:59
BESDapResponse::set_dap4_function
virtual void set_dap4_function(BESDataHandlerInterface &dhi)
set the constraint depending on the context
Definition: BESDapResponse.cc:173
BESDapResponse::set_dap4_constraint
virtual void set_dap4_constraint(BESDataHandlerInterface &dhi)
set the constraint depending on the context
Definition: BESDapResponse.cc:156
BESDataDDSResponse
Represents an OPeNDAP DataDDS DAP2 data object within the BES.
Definition: BESDataDDSResponse.h:46
BESDataHandlerInterface
Structure storing information used by the BES to handle the request.
Definition: BESDataHandlerInterface.h:56
BESDataHandlerInterface::data
std::map< std::string, std::string > data
the map of string data that will be required for the current request.
Definition: BESDataHandlerInterface.h:90
BESDataHandlerInterface::container
BESContainer * container
pointer to current container in this interface
Definition: BESDataHandlerInterface.h:75
BESDebug::IsSet
static bool IsSet(const std::string &flagName)
see if the debug context flagName is set to true
Definition: BESDebug.h:168
BESError
Base exception class for the BES with basic string message.
Definition: BESError.h:59
BESInfo
informational response object
Definition: BESInfo.h:63
BESInternalError
exception thrown if internal error encountered
Definition: BESInternalError.h:43
BESRequestHandler
Represents a specific data type request handler.
Definition: BESRequestHandler.h:74
BESResponseHandler::get_response_object
virtual BESResponseObject * get_response_object()
return the current response object
Definition: BESResponseHandler.cc:82
BESResponseObject
Abstract base class representing a specific set of information in response to a request to the BES.
Definition: BESResponseObject.h:46
BESServiceRegistry::services_handled
virtual void services_handled(const std::string &handler, std::list< std::string > &services)
returns the list of servies provided by the handler in question
Definition: BESServiceRegistry.cc:347
BESStopWatch::start
virtual bool start(std::string name)
Definition: BESStopWatch.cc:67
BESUtil::lowercase
static std::string lowercase(const std::string &s)
Definition: BESUtil.cc:254
BESUtil::implode
static std::string implode(const std::list< std::string > &values, char delim)
Definition: BESUtil.cc:617
TheBESKeys::get_value
void get_value(const std::string &s, std::string &val, bool &found)
Retrieve the value of a given key, if set.
Definition: TheBESKeys.cc:340
TheBESKeys::TheKeys
static TheBESKeys * TheKeys()
Definition: TheBESKeys.cc:71
close_fileid
void close_fileid(hid_t fid)
Definition: h5get.cc:434