bes  Updated for version 3.20.13
HDF5GMCF.cc
Go to the documentation of this file.
1 // This file is part of the hdf5_handler implementing for the CF-compliant
2 // Copyright (c) 2011-2016 The HDF Group, Inc. and OPeNDAP, Inc.
3 //
4 // This is free software; you can redistribute it and/or modify it under the
5 // terms of the GNU Lesser General Public License as published by the Free
6 // Software Foundation; either version 2.1 of the License, or (at your
7 // option) any later version.
8 //
9 // This software is distributed in the hope that it will be useful, but
10 // WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
11 // or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
12 // License for more details.
13 //
14 // You should have received a copy of the GNU Lesser General Public
15 // License along with this library; if not, write to the Free Software
16 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 //
18 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
19 // You can contact The HDF Group, Inc. at 1800 South Oak Street,
20 // Suite 203, Champaign, IL 61820
21 
36 
37 #include "HDF5CF.h"
38 #include "HDF5RequestHandler.h"
39 #include "h5apicompatible.h"
40 #include <BESDebug.h>
41 #include <algorithm>
42 
43 using namespace std;
44 using namespace libdap;
45 using namespace HDF5CF;
46 
47 // Copier function.
48 GMCVar::GMCVar(Var*var) {
49 
50  BESDEBUG("h5", "Coming to GMCVar()"<<endl);
51  newname = var->newname;
52  name = var->name;
53  fullpath = var->fullpath;
54  rank = var->rank;
55  total_elems = var->total_elems;
56  zero_storage_size = var->zero_storage_size;
57  dtype = var->dtype;
58  unsupported_attr_dtype = var->unsupported_attr_dtype;
59  unsupported_dspace = var->unsupported_dspace;
60  coord_attr_add_path = false;
61 
62  for (auto ira = var->attrs.begin();
63  ira!=var->attrs.end(); ++ira) {
64  auto attr= new Attribute();
65  attr->name = (*ira)->name;
66  attr->newname = (*ira)->newname;
67  attr->dtype =(*ira)->dtype;
68  attr->count =(*ira)->count;
69  attr->strsize = (*ira)->strsize;
70  attr->fstrsize = (*ira)->fstrsize;
71  attr->value =(*ira)->value;
72  attrs.push_back(attr);
73  }
74 
75  for (auto ird = var->dims.begin();
76  ird!=var->dims.end(); ++ird) {
77  auto dim = new Dimension((*ird)->size);
78  dim->name = (*ird)->name;
79  dim->newname = (*ird)->newname;
80  dim->unlimited_dim = (*ird)->unlimited_dim;
81  dims.push_back(dim);
82  }
83 
84  product_type = General_Product;
85 
86 }
87 #if 0
88 GMCVar::GMCVar(GMCVar*cvar) {
89 
90  newname = cvar->newname;
91  name = cvar->name;
92  fullpath = cvar->fullpath;
93  rank = cvar->rank;
94  dtype = cvar->dtype;
95  unsupported_attr_dtype = cvar->unsupported_attr_dtype;
96  unsupported_dspace = cvar->unsupported_dspace;
97 
98  for (vector<Attribute*>::iterator ira = cvar->attrs.begin();
99  ira!=cvar->attrs.end(); ++ira) {
100  Attribute* attr= new Attribute();
101  attr->name = (*ira)->name;
102  attr->newname = (*ira)->newname;
103  attr->dtype =(*ira)->dtype;
104  attr->count =(*ira)->count;
105  attr->strsize = (*ira)->strsize;
106  attr->fstrsize = (*ira)->fstrsize;
107  attr->value =(*ira)->value;
108  attrs.push_back(attr);
109  }
110 
111  for (vector<Dimension*>::iterator ird = cvar->dims.begin();
112  ird!=cvar->dims.end(); ++ird) {
113  Dimension *dim = new Dimension((*ird)->size);
114 //"h5","dim->name "<< (*ird)->name <<endl;
115 //"h5","dim->newname "<< (*ird)->newname <<endl;
116  dim->name = (*ird)->name;
117  dim->newname = (*ird)->newname;
118  dims.push_back(dim);
119  }
120 
121  GMcvar->cfdimname = latdim0;
122  GMcvar->cvartype = CV_EXIST;
123  GMcvar->product_type = product_type;
124 
125 
126 }
127 #endif
128 
129 //Copier function of a special variable.
130 GMSPVar::GMSPVar(Var*var) {
131 
132  BESDEBUG("h5", "Coming to GMSPVar()"<<endl);
133  fullpath = var->fullpath;
134  rank = var->rank;
135  total_elems = var->total_elems;
136  zero_storage_size = var->zero_storage_size;
137  unsupported_attr_dtype = var->unsupported_attr_dtype;
138  unsupported_dspace = var->unsupported_dspace;
139  coord_attr_add_path = var->coord_attr_add_path;
140  // The caller of this function should change the following fields.
141  // This is just to make data coverity happy.
142  otype = H5UNSUPTYPE;
143  sdbit = -1;
144  numofdbits = -1;
145 
146  for (auto ira = var->attrs.begin();
147  ira!=var->attrs.end(); ++ira) {
148  auto attr= new Attribute();
149  attr->name = (*ira)->name;
150  attr->newname = (*ira)->newname;
151  attr->dtype =(*ira)->dtype;
152  attr->count =(*ira)->count;
153  attr->strsize = (*ira)->strsize;
154  attr->fstrsize = (*ira)->fstrsize;
155  attr->value =(*ira)->value;
156  attrs.push_back(attr);
157  }
158 
159  for (auto ird = var->dims.begin();
160  ird!=var->dims.end(); ++ird) {
161  auto dim = new Dimension((*ird)->size);
162  dim->name = (*ird)->name;
163  dim->newname = (*ird)->newname;
164  dim->unlimited_dim = (*ird)->unlimited_dim;
165  dims.push_back(dim);
166  }
167 }
168 
169 
170 GMFile::GMFile(const char*file_fullpath, hid_t file_id, H5GCFProduct product_type, GMPattern gproduct_pattern):
171 File(file_fullpath,file_id), product_type(product_type),gproduct_pattern(gproduct_pattern)
172 {
173 
174 }
175 
176 // destructor
177 GMFile::~GMFile()
178 {
179 
180  if (!this->cvars.empty()){
181  for (auto i= this->cvars.begin(); i!=this->cvars.end(); ++i) {
182  delete *i;
183  }
184  }
185 
186  if (!this->spvars.empty()){
187  for (auto i= this->spvars.begin(); i!=this->spvars.end(); ++i) {
188  delete *i;
189  }
190  }
191 
192 }
193 
194 // Get CF string
195 string GMFile::get_CF_string(string s) {
196 
197  // HDF5 group or variable path always starts with '/'. When CF naming rule is applied,
198  // the first '/' is always changes to "_", this is not necessary. However,
199  // to keep the backward compatiablity, I use a BES key for people to go back with the original name.
200 
201  if(s[0] !='/')
202  return File::get_CF_string(s);
203  else if (General_Product == product_type && OTHERGMS == gproduct_pattern) {
204 
205  if(true == HDF5RequestHandler::get_keep_var_leading_underscore())
206  return File::get_CF_string(s);
207  else {
208  s.erase(0,1);
209  return File::get_CF_string(s);
210  }
211  }
212  else {
213  // The leading underscore should be removed from all supported products
214  s.erase(0,1);
215  return File::get_CF_string(s);
216  }
217 }
218 
219 // Retrieve all the HDF5 information.
220 void GMFile::Retrieve_H5_Info(const char *file_fullpath,
221  hid_t file_id, bool include_attr) {
222 
223  BESDEBUG("h5", "Coming to Retrieve_H5_Info()"<<endl);
224  // GPM needs the attribute info. to obtain the lat/lon.
225  // So set the include_attr to be true for these products.
226  if (product_type == Mea_SeaWiFS_L2 || product_type == Mea_SeaWiFS_L3
227  || GPMS_L3 == product_type || GPMM_L3 == product_type || GPM_L1 == product_type || OBPG_L3 == product_type
228  || Mea_Ozone == product_type || General_Product == product_type)
229  File::Retrieve_H5_Info(file_fullpath,file_id,true);
230  else
231  File::Retrieve_H5_Info(file_fullpath,file_id,include_attr);
232 
233 }
234 
235 // Update the product type. This is because the file structure may change across different versions of products
236 // I need to handle them differently and still support different versions. The goal is to support two versions in a row.
237 // Currently GPM level 3 is changed.
238 // This routine should be called right after Retrieve_H5_Info.
240 
241  BESDEBUG("h5", "Coming to Update_Product_Type()"<<endl);
242  if(GPMS_L3 == this->product_type || GPMM_L3 == this->product_type) {
243 
244  // Check Dimscale attributes
245 #if 0
246  //Check_General_Product_Pattern();
247 #endif
248  Check_Dimscale_General_Product_Pattern();
249  if(GENERAL_DIMSCALE == this->gproduct_pattern){
250  if(GPMS_L3 == this->product_type) {
251  for (auto irv = this->vars.begin();
252  irv != this->vars.end(); ++irv)
253  (*irv)->newname = (*irv)->name;
254  }
255  this->product_type = General_Product;
256  }
257  }
258  else if(General_Product == this->product_type)
259  Check_General_Product_Pattern();
260 }
261 
263 
264  BESDEBUG("h5", "Coming to Remove_Unneeded_Objects()"<<endl);
265  if(General_Product == this->product_type) {
266  string file_path = this->path;
267  if(HDF5CFUtil::obtain_string_after_lastslash(file_path).find("OMPS-NPP")==0)
268  Remove_OMPSNPP_InputPointers();
269  }
270  if((General_Product == this->product_type) && (GENERAL_DIMSCALE == this->gproduct_pattern)) {
271  set<string> nc4_non_coord_set;
272  string nc4_non_coord="_nc4_non_coord_";
273  size_t nc4_non_coord_size= nc4_non_coord.size();
274  for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
275  if((*irv)->name.find(nc4_non_coord)==0)
276  nc4_non_coord_set.insert((*irv)->name.substr(nc4_non_coord_size,(*irv)->name.size()-nc4_non_coord_size));
277 
278  }
279 
280  for (auto irv = this->vars.begin(); irv != this->vars.end();) {
281  if(nc4_non_coord_set.find((*irv)->name)!=nc4_non_coord_set.end()){
282  delete(*irv);
283  irv=this->vars.erase(irv);
284  }
285  else
286  ++irv;
287  }
288 
289  if(nc4_non_coord_set.empty()==false)
290  this->have_nc4_non_coord = true;
291  }
292  else if(GPM_L3_New == this->product_type) {
293  for (auto irg = this->groups.begin(); irg != this->groups.end(); ) {
294  if((*irg)->attrs.empty()) {
295  delete(*irg);
296  irg = this->groups.erase(irg);
297 
298  }
299  else
300  ++irg;
301  }
302  }
303 }
304 
305 void GMFile::Remove_OMPSNPP_InputPointers() {
306  // Here I don't check whether this is a netCDF file by
307  // using Check_Dimscale_General_Product_Pattern() to see if it returns true.
308  // We will see if we need this.
309  for (auto irg = this->groups.begin(); irg != this->groups.end(); ) {
310  if((*irg)->path.find("/InputPointers")==0) {
311  delete(*irg);
312  irg = this->groups.erase(irg);
313 
314  }
315  else
316  ++irg;
317  }
318 
319  for (auto irv = this->vars.begin(); irv != this->vars.end(); ) {
320  if((*irv)->fullpath.find("/InputPointers")==0) {
321  delete(*irv);
322  irv = this->vars.erase(irv);
323 
324  }
325  else
326  ++irv;
327  }
328 }
330 
331  for (auto ircv = this->cvars.begin(); ircv != this->cvars.end(); ++ircv) {
332 
333  if ((*ircv)->cvartype != CV_NONLATLON_MISS){
334  for (auto ira = (*ircv)->attrs.begin();
335  ira != (*ircv)->attrs.end(); ++ira) {
336  Retrieve_H5_Attr_Value(*ira,(*ircv)->fullpath);
337  }
338  }
339  }
340 }
341 
342 // Retrieve HDF5 supported attribute values.
344 
345  BESDEBUG("h5", "Coming to Retrieve_H5_Supported_Attr_Values()"<<endl);
346 
347  // General attributes
349 
350  //Coordinate variable attributes
351  for (auto ircv = this->cvars.begin(); ircv != this->cvars.end(); ++ircv) {
352  if ((*ircv)->cvartype != CV_NONLATLON_MISS){
353  for (auto ira = (*ircv)->attrs.begin();
354  ira != (*ircv)->attrs.end(); ++ira) {
355  Retrieve_H5_Attr_Value(*ira,(*ircv)->fullpath);
356  }
357  }
358  }
359 
360  // Special variable attributes
361  for (auto irspv = this->spvars.begin(); irspv != this->spvars.end(); ++irspv) {
362  for (auto ira = (*irspv)->attrs.begin();
363  ira != (*irspv)->attrs.end(); ++ira) {
364  Retrieve_H5_Attr_Value(*ira,(*irspv)->fullpath);
365  Adjust_H5_Attr_Value(*ira);
366  }
367  }
368 }
369 
370 // Adjust attribute values. Currently this is only for ACOS and OCO2.
371 // Reason: DAP2 doesn't support 64-bit integer and they have 64-bit integer data
372 // in these files. Chop them to two 32-bit integers following the data producer's information.
374 
375  BESDEBUG("h5", "Coming to Adjust_H5_Attr_Value()"<<endl);
376  if (product_type == ACOS_L2S_OR_OCO2_L1B) {
377  if (("Type" == attr->name) && (H5VSTRING == attr->dtype)) {
378  string orig_attrvalues(attr->value.begin(),attr->value.end());
379  if (orig_attrvalues != "Signed64") return;
380  string new_attrvalues = "Signed32";
381  // Since the new_attrvalues size is the same as the orig_attrvalues size
382  // No need to adjust the strsize and fstrsize. KY 2011-2-1
383  attr->value.clear();
384  attr->value.resize(new_attrvalues.size());
385  copy(new_attrvalues.begin(),new_attrvalues.end(),attr->value.begin());
386  }
387  }
388 }
389 
390 // Unsupported datatype
391 void GMFile:: Handle_Unsupported_Dtype(bool include_attr) {
392 
393  BESDEBUG("h5", "Coming to Handle_Unsupported_Dtype()"<<endl);
394  if(true == check_ignored) {
395  Gen_Unsupported_Dtype_Info(include_attr);
396  }
397  File::Handle_Unsupported_Dtype(include_attr);
398  Handle_GM_Unsupported_Dtype(include_attr);
399 }
400 
401 // Unsupported datatype for general data products.
402 void GMFile:: Handle_GM_Unsupported_Dtype(bool include_attr) {
403 
404  BESDEBUG("h5", "Coming to Handle_GM_Unsupported_Dtype()"<<endl);
405  for (auto ircv = this->cvars.begin(); ircv != this->cvars.end(); ) {
406  if (true == include_attr) {
407  for (auto ira = (*ircv)->attrs.begin(); ira != (*ircv)->attrs.end(); ) {
408  H5DataType temp_dtype = (*ira)->getType();
409  if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4)) {
410  delete (*ira);
411  ira = (*ircv)->attrs.erase(ira);
412  }
413  else {
414  ++ira;
415  }
416  }
417  }
418  H5DataType temp_dtype = (*ircv)->getType();
419  if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4)) {
420 
421  // This may need to be checked carefully in the future,
422  // My current understanding is that the coordinate variable can
423  // be ignored if the corresponding variable is ignored.
424  // Currently we don't find any NASA files in this category.
425  // KY 2012-5-21
426  delete (*ircv);
427  ircv = this->cvars.erase(ircv);
428  }
429  else {
430  ++ircv;
431  }
432 
433  }
434 
435  for (auto ircv = this->spvars.begin(); ircv != this->spvars.end(); ) {
436  if (true == include_attr) {
437  for (auto ira = (*ircv)->attrs.begin(); ira != (*ircv)->attrs.end(); ) {
438  H5DataType temp_dtype = (*ira)->getType();
439  if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4)) {
440  delete (*ira);
441  ira = (*ircv)->attrs.erase(ira);
442  }
443  else {
444  ++ira;
445  }
446  }
447  }
448  H5DataType temp_dtype = (*ircv)->getType();
449  if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4)) {
450  delete (*ircv);
451  ircv = this->spvars.erase(ircv);
452  }
453  else {
454  ++ircv;
455  }
456 
457  }
458 }
459 
460 // Datatype ignore information.
461 void GMFile:: Gen_Unsupported_Dtype_Info(bool include_attr) {
462 
463  BESDEBUG("h5", "GMFile::Coming to Gen_Unsupported_Dtype_Info()"<<endl);
464  if(true == include_attr) {
465 
466  File::Gen_Group_Unsupported_Dtype_Info();
467  File::Gen_Var_Unsupported_Dtype_Info();
468  Gen_VarAttr_Unsupported_Dtype_Info();
469  }
470 
471 }
472 
473 // Datatype ignored information for variable ttributes
474 void GMFile:: Gen_VarAttr_Unsupported_Dtype_Info() {
475 
476  BESDEBUG("h5", "GMFile::Coming to Gen_Unsupported_Dtype_Info()"<<endl);
477  // First general variables(non-CV and non-special variable) that use dimension scales.
478  if((General_Product == this->product_type && GENERAL_DIMSCALE== this->gproduct_pattern)
479  || (Mea_Ozone == this->product_type) || (Mea_SeaWiFS_L2 == this->product_type) || (Mea_SeaWiFS_L3 == this->product_type)
480  || (OBPG_L3 == this->product_type)) {
481  Gen_DimScale_VarAttr_Unsupported_Dtype_Info();
482  }
483 
484  else
485  File::Gen_VarAttr_Unsupported_Dtype_Info();
486 
487  // CV and special variables
488  Gen_GM_VarAttr_Unsupported_Dtype_Info();
489 
490 }
491 
492 // Generate ignored object,attribute information for the CVs and special variables of general supported products.
493 void GMFile:: Gen_GM_VarAttr_Unsupported_Dtype_Info(){
494 
495  BESDEBUG("h5", "GMFile::Coming to Gen_GM_VarAttr_Unsupported_Dtype_Info()"<<endl);
496  if((General_Product == this->product_type && GENERAL_DIMSCALE== this->gproduct_pattern)
497  || (Mea_Ozone == this->product_type) || (Mea_SeaWiFS_L2 == this->product_type) || (Mea_SeaWiFS_L3 == this->product_type)
498  || (OBPG_L3 == this->product_type)) {
499 
500  for (auto irv = this->cvars.begin(); irv != this->cvars.end(); ++irv) {
501  // If the attribute REFERENCE_LIST comes with the attribut CLASS, the
502  // attribute REFERENCE_LIST is okay to ignore. No need to report.
503  bool is_ignored = ignored_dimscale_ref_list((*irv));
504  if (false == (*irv)->attrs.empty()) {
505  for (auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira) {
506  H5DataType temp_dtype = (*ira)->getType();
507  // TODO: check why 64-bit integer is included here.
508  if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4) || temp_dtype == H5INT64 || temp_dtype == H5UINT64) {
509  // "DIMENSION_LIST" is okay to ignore and "REFERENCE_LIST"
510  // is okay to ignore if the variable has another attribute
511  // CLASS="DIMENSION_SCALE"
512  if (("DIMENSION_LIST" !=(*ira)->name) &&
513  ("REFERENCE_LIST" != (*ira)->name || true == is_ignored))
514  this->add_ignored_info_attrs(false,(*irv)->fullpath,(*ira)->name);
515  }
516  }
517  }
518  }
519 
520  for (auto irv = this->spvars.begin(); irv != this->spvars.end(); ++irv) {
521  // If the attribute REFERENCE_LIST comes with the attribut CLASS, the
522  // attribute REFERENCE_LIST is okay to ignore. No need to report.
523  bool is_ignored = ignored_dimscale_ref_list((*irv));
524  if (false == (*irv)->attrs.empty()) {
525 #if 0
526  //if (true == (*irv)->unsupported_attr_dtype)
527 #endif
528  for (auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira) {
529  H5DataType temp_dtype = (*ira)->getType();
530  //TODO; check why 64-bit integer is included here.
531  if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4) || temp_dtype == H5INT64 || temp_dtype == H5UINT64) {
532  // "DIMENSION_LIST" is okay to ignore and "REFERENCE_LIST"
533  // is okay to ignore if the variable has another attribute
534  // CLASS="DIMENSION_SCALE"
535  if (("DIMENSION_LIST" !=(*ira)->name) &&
536  ("REFERENCE_LIST" != (*ira)->name || true == is_ignored))
537  this->add_ignored_info_attrs(false,(*irv)->fullpath,(*ira)->name);
538  }
539  }
540  }
541  }
542  }// "if((General_Product == ......)"
543  else {
544  for (auto irv = this->cvars.begin(); irv != this->cvars.end(); ++irv) {
545  if (false == (*irv)->attrs.empty()) {
546 #if 0
547  //if (true == (*irv)->unsupported_attr_dtype)
548 #endif
549  for (auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira) {
550  H5DataType temp_dtype = (*ira)->getType();
551  // TODO: check why 64-bit integer is included here.
552  if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4) || temp_dtype == H5INT64 || temp_dtype == H5UINT64)
553  this->add_ignored_info_attrs(false,(*irv)->fullpath,(*ira)->name);
554  }
555  }
556  }
557 
558  for (auto irv = this->spvars.begin(); irv != this->spvars.end(); ++irv) {
559  if (false == (*irv)->attrs.empty()) {
560 #if 0
561  //if (true == (*irv)->unsupported_attr_dtype)
562 #endif
563  for (auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira) {
564  H5DataType temp_dtype = (*ira)->getType();
565  //TODO: check why 64-bit integer is included here.
566  if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4) || temp_dtype == H5INT64 || temp_dtype == H5UINT64) {
567  this->add_ignored_info_attrs(false,(*irv)->fullpath,(*ira)->name);
568  }
569  }
570  }
571  }
572 
573  }// else
574 
575 }
576 
577 // Unsupported data space
578 void GMFile:: Handle_Unsupported_Dspace(bool include_attr) {
579 
580  BESDEBUG("h5", "Coming to GMFile:Handle_Unsupported_Dspace()"<<endl);
581  if(true == check_ignored)
582  Gen_Unsupported_Dspace_Info();
583 
584  File::Handle_Unsupported_Dspace(include_attr);
585  Handle_GM_Unsupported_Dspace(include_attr);
586 
587 }
588 
589 // Unsupported data space for coordinate variables and special variables of general products
590 void GMFile:: Handle_GM_Unsupported_Dspace(bool include_attr) {
591 
592  BESDEBUG("h5", "Coming to GMFile:Handle_GM_Unsupported_Dspace()"<<endl);
593  if(true == this->unsupported_var_dspace) {
594  for (auto ircv = this->cvars.begin(); ircv != this->cvars.end(); ) {
595  if (true == (*ircv)->unsupported_dspace ) {
596 
597  // This may need to be checked carefully in the future,
598  // My current understanding is that the coordinate variable can
599  // be ignored if the corresponding variable is ignored.
600  // Currently we don't find any NASA files in this category.
601  // KY 2012-5-21
602  delete (*ircv);
603  ircv = this->cvars.erase(ircv);
604  }
605  else {
606  ++ircv;
607  }
608  }
609 
610  for (auto ircv = this->spvars.begin(); ircv != this->spvars.end(); ) {
611  if (true == (*ircv)->unsupported_dspace) {
612  delete (*ircv);
613  ircv = this->spvars.erase(ircv);
614  }
615  else {
616  ++ircv;
617  }
618 
619  }
620  }// if
621 
622  if(true == include_attr) {
623  if(true == this->unsupported_var_attr_dspace) {
624  for (auto ircv = this->cvars.begin(); ircv != this->cvars.end(); ++ircv) {
625  if (false == (*ircv)->attrs.empty()) {
626  if (true == (*ircv)->unsupported_attr_dspace) {
627  for (auto ira = (*ircv)->attrs.begin(); ira != (*ircv)->attrs.end(); ) {
628  if (0 == (*ira)->count) {
629  delete (*ira);
630  ira = (*ircv)->attrs.erase(ira);
631  }
632  else {
633  ++ira;
634  }
635  }
636  }
637  }
638  }
639 
640  for (auto ircv = this->spvars.begin(); ircv != this->spvars.end(); ++ircv) {
641  if (false == (*ircv)->attrs.empty()) {
642  if (true == (*ircv)->unsupported_attr_dspace) {
643  for (auto ira = (*ircv)->attrs.begin(); ira != (*ircv)->attrs.end(); ) {
644  if (0 == (*ira)->count) {
645  delete (*ira);
646  ira = (*ircv)->attrs.erase(ira);
647  }
648  else {
649  ++ira;
650  }
651  }
652  }
653  }
654  } // for
655  }// if
656  }// if
657 
658 }
659 
660 // Generate unsupported data space information
661 void GMFile:: Gen_Unsupported_Dspace_Info() {
662 
663  File::Gen_Unsupported_Dspace_Info();
664 
665 }
666 
667 // Handle other unsupported objects
668 void GMFile:: Handle_Unsupported_Others(bool include_attr) {
669 
670  BESDEBUG("h5", "Coming to GMFile:Handle_Unsupported_Others()"<<endl);
671  File::Handle_Unsupported_Others(include_attr);
672 
673  // Add the removal of CLASS=DIM_SCALE attribute if this is a netCDF-4-like attribute.
674  //
675  if(General_Product != this->product_type
676  || (General_Product == this->product_type && OTHERGMS != this->gproduct_pattern)){
677  //
678 #if 0
679  if((General_Product == this->product_type && GENERAL_DIMSCALE== this->gproduct_pattern)
680  || (Mea_Ozone == this->product_type) || (Mea_SeaWiFS_L2 == this->product_type)
681  || (Mea_SeaWiFS_L3 == this->product_type)
682  || (OBPG_L3 == this->product_type))
683 #endif
684  remove_netCDF_internal_attributes(include_attr);
685  if(include_attr == true) {
686  // We also need to remove the _nc3_strict from the root attributes
687  for (auto ira = this->root_attrs.begin(); ira != this->root_attrs.end();) {
688 
689  if((*ira)->name == "_nc3_strict") {
690  delete(*ira);
691  ira =this->root_attrs.erase(ira);
692  //If we have other root attributes to remove, remove the break statement.
693  }
694  else if((*ira)->name == "_NCProperties") {
695  delete(*ira);
696  ira =this->root_attrs.erase(ira);
697  }
698  else if((*ira)->name == "_Netcdf4Coordinates") {
699  delete(*ira);
700  ira =this->root_attrs.erase(ira);
701  }
702 
703  else {
704  ++ira;
705  }
706  }
707  for (auto irv = this->cvars.begin(); irv != this->cvars.end(); ++irv) {
708  for(auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end();) {
709  if((*ira)->name == "CLASS") {
710  string class_value = Retrieve_Str_Attr_Value(*ira,(*irv)->fullpath);
711 
712  // Compare the attribute "CLASS" value with "DIMENSION_SCALE". We only compare the string with the size of
713  // "DIMENSION_SCALE", which is 15.
714  if (0 == class_value.compare(0,15,"DIMENSION_SCALE")) {
715  delete(*ira);
716  ira = (*irv)->attrs.erase(ira);
717  // Add another block to set a key
718  }
719  else {
720  ++ira;
721  }
722  }
723  else if((*ira)->name == "NAME") {// Add a BES Key later
724  delete(*ira);
725  ira =(*irv)->attrs.erase(ira);
726  //"NAME" attribute causes the file netCDF-4 failed.
727 #if 0
728 
729  string name_value = Retrieve_Str_Attr_Value(*ira,(*irv)->fullpath);
730  if( 0 == name_value.compare(0,(*irv)->name.size(),(*irv)->name)) {
731  delete(*ira);
732  ira =(*irv)->attrs.erase(ira);
733  }
734  else {
735  string netcdf_dim_mark= "This is a netCDF dimension but not a netCDF variable";
736  if( 0 == name_value.compare(0,netcdf_dim_mark.size(),netcdf_dim_mark)) {
737  delete((*ira));
738  ira =(*irv)->attrs.erase(ira);
739  }
740  else {
741  ++ira;
742  }
743  }
744 #endif
745  }
746  else if((*ira)->name == "_Netcdf4Dimid") {
747  delete(*ira);
748  ira =(*irv)->attrs.erase(ira);
749  }
750  else if((*ira)->name == "_Netcdf4Coordinates") {
751  delete(*ira);
752  ira =(*irv)->attrs.erase(ira);
753  }
754 
755 #if 0
756  else if((*ira)->name == "_nc3_strict") {
757  delete((*ira));
758  ira =(*irv)->attrs.erase(ira);
759  }
760 #endif
761  else {
762  ++ira;
763  }
764  }
765  }
766  }
767  }
768  // netCDF Java lifts the string size limitation. All the string attributes can be
769  // represented by netCDF Java. So comment out the code. KY 2018/08/10
770 #if 0
771  if(true == this->check_ignored && true == include_attr) {
772  if(true == HDF5RequestHandler::get_drop_long_string()){
773  for (vector<GMCVar *>::iterator irv = this->cvars.begin();
774  irv != this->cvars.end(); ++irv) {
775  for(vector<Attribute *>::iterator ira = (*irv)->attrs.begin();
776  ira != (*irv)->attrs.end();++ira) {
777  if(true == Check_DropLongStr((*irv),(*ira))) {
778  this->add_ignored_droplongstr_hdr();
779  this->add_ignored_var_longstr_info((*irv),(*ira));
780  }
781  }
782  }
783 
784  for (vector<GMSPVar *>::iterator irv = this->spvars.begin();
785  irv != this->spvars.end(); ++irv) {
786  for(vector<Attribute *>::iterator ira = (*irv)->attrs.begin();
787  ira != (*irv)->attrs.end();++ira) {
788  if(true == Check_DropLongStr((*irv),(*ira))) {
789  this->add_ignored_droplongstr_hdr();
790  this->add_ignored_var_longstr_info((*irv),(*ira));
791  }
792  }
793 
794  }
795  }
796  }
797 #endif
798 
799  if(false == this->have_ignored)
800  this->add_no_ignored_info();
801 
802 }
803 
804 // Add dimension names
806 
807  BESDEBUG("h5", "Coming to GMFile:Add_Dim_Name()"<<endl);
808  switch(product_type) {
809  case Mea_SeaWiFS_L2:
810  case Mea_SeaWiFS_L3:
811  Add_Dim_Name_Mea_SeaWiFS();
812  break;
813  case Aqu_L3:
814  Add_Dim_Name_Aqu_L3();
815  break;
816  case OSMAPL2S:
817  Add_Dim_Name_OSMAPL2S();
818  break;
819  case ACOS_L2S_OR_OCO2_L1B:
820  Add_Dim_Name_ACOS_L2S_OCO2_L1B();
821  break;
822  case Mea_Ozone:
823  Add_Dim_Name_Mea_Ozonel3z();
824  break;
825  case GPMS_L3:
826  case GPMM_L3:
827  case GPM_L1:
828  case GPM_L3_New:
829  Add_Dim_Name_GPM();
830  break;
831  case OBPG_L3:
832  Add_Dim_Name_OBPG_L3();
833  break;
834  case General_Product:
835  Add_Dim_Name_General_Product();
836  break;
837  default:
838  throw1("Cannot generate dim. names for unsupported datatype");
839  }
840 
841 // Just for debugging
842 #if 0
843 for (vector<Var*>::iterator irv2 = this->vars.begin();
844  irv2 != this->vars.end(); irv2++) {
845  for (vector<Dimension *>::iterator ird = (*irv2)->dims.begin();
846  ird !=(*irv2)->dims.end(); ird++) {
847  cerr<<"Dimension name afet Add_Dim_Name "<<(*ird)->newname <<endl;
848  }
849 }
850 #endif
851 
852 }
853 
854 //Add Dim. Names for OBPG level 3 product
855 void GMFile::Add_Dim_Name_OBPG_L3() {
856 
857  BESDEBUG("h5", "Coming to Add_Dim_Name_OBPG_L3()"<<endl);
858  // netCDF-4 like structure
859  // Note: We need to change the product type to netCDF-4 like product type and pattern.
860  Check_General_Product_Pattern();
861  Add_Dim_Name_General_Product();
862 }
863 
864 //Add Dim. Names for MeaSures SeaWiFS. Future: May combine with the handling of netCDF-4 products
865 void GMFile::Add_Dim_Name_Mea_SeaWiFS() {
866 
867  BESDEBUG("h5", "Coming to Add_Dim_Name_Mea_SeaWiFS()"<<endl);
868  pair<set<string>::iterator,bool> setret;
869  if (Mea_SeaWiFS_L3 == product_type)
870  iscoard = true;
871  for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
872  Handle_UseDimscale_Var_Dim_Names_Mea_SeaWiFS_Ozone((*irv));
873  for (auto ird = (*irv)->dims.begin(); ird !=(*irv)->dims.end();++ird) {
874  setret = dimnamelist.insert((*ird)->name);
875  if (true == setret.second)
876  Insert_One_NameSizeMap_Element((*ird)->name,(*ird)->size,(*ird)->unlimited_dim);
877  }
878  }
879 
880  if (true == dimnamelist.empty())
881  throw1("This product should have the dimension names, but no dimension names are found");
882 }
883 
884 // Handle Dimension scales for MEasUREs SeaWiFS and OZone.
885 void GMFile::Handle_UseDimscale_Var_Dim_Names_Mea_SeaWiFS_Ozone(Var* var)
886 {
887 
888  BESDEBUG("h5", "Coming to Handle_UseDimscale_Var_Dim_Names_Mea_SeaWiFS_Ozone()"<<endl);
889  Attribute* dimlistattr = nullptr;
890  bool has_dimlist = false;
891  bool has_class = false;
892  bool has_reflist = false;
893 
894  for(auto ira = var->attrs.begin(); ira != var->attrs.end();ira++) {
895  if ("DIMENSION_LIST" == (*ira)->name) {
896  dimlistattr = *ira;
897  has_dimlist = true;
898  }
899  if ("CLASS" == (*ira)->name)
900  has_class = true;
901  if ("REFERENCE_LIST" == (*ira)->name)
902  has_reflist = true;
903 
904  if (true == has_dimlist)
905  break;
906  if (true == has_class && true == has_reflist)
907  break;
908  }
909 
910  if (true == has_dimlist)
911  Add_UseDimscale_Var_Dim_Names_Mea_SeaWiFS_Ozone(var,dimlistattr);
912 
913  // Dim name is the same as the variable name for dimscale variable
914  else if(true == has_class && true == has_reflist) {
915  if (var->dims.size() !=1)
916  throw2("dimension scale dataset must be 1 dimension, this is not true for variable ",
917  var->name);
918 
919  // The var name is the object name, however, we would like the dimension name to be full path.
920  // so that the dim name can be served as the key for future handling.
921  (var->dims)[0]->name = var->fullpath;
922  (var->dims)[0]->newname = var->fullpath;
923  pair<set<string>::iterator,bool> setret;
924  setret = dimnamelist.insert((var->dims)[0]->name);
925  if (true == setret.second)
926  Insert_One_NameSizeMap_Element((var->dims)[0]->name,(var->dims)[0]->size,(var->dims)[0]->unlimited_dim);
927  }
928 
929  // No dimension, add fake dim names, this may never happen for MeaSure
930  // but just for coherence and completeness.
931  // For Fake dimesnion
932  else {
933 
934  set<hsize_t> fakedimsize;
935  pair<set<hsize_t>::iterator,bool> setsizeret;
936  for (auto ird= var->dims.begin(); ird != var->dims.end(); ++ird) {
937  Add_One_FakeDim_Name(*ird);
938  setsizeret = fakedimsize.insert((*ird)->size);
939  if (false == setsizeret.second)
940  Adjust_Duplicate_FakeDim_Name(*ird);
941  }
942 // Just for debugging
943 #if 0
944  for (int i = 0; i < var->dims.size(); ++i) {
945  Add_One_FakeDim_Name((var->dims)[i]);
946  bool gotoMainLoop = false;
947  for (int j =i-1; j>=0 && !gotoMainLoop; --j) {
948  if (((var->dims)[i])->size == ((var->dims)[j])->size){
949  Adjust_Duplicate_FakeDim_Name((var->dims)[i]);
950  gotoMainLoop = true;
951  }
952  }
953  }
954 #endif
955 
956  }//end of else
957 }
958 
959 // Helper function to support dimensions of MeaSUrES SeaWiFS and OZone products
960 void GMFile::Add_UseDimscale_Var_Dim_Names_Mea_SeaWiFS_Ozone(Var *var,const Attribute*dimlistattr)
961 {
962 
963  BESDEBUG("h5", "Coming to Add_UseDimscale_Var_Dim_Names_Mea_SeaWiFS_Ozone()"<<endl);
964  ssize_t objnamelen = -1;
965  hobj_ref_t rbuf;
966 
967  vector<hvl_t> vlbuf;
968 
969  hid_t dset_id = -1;
970  hid_t attr_id = -1;
971  hid_t atype_id = -1;
972  hid_t amemtype_id = -1;
973  hid_t aspace_id = -1;
974  hid_t ref_dset = -1;
975 
976 
977  if(nullptr == dimlistattr)
978  throw2("Cannot obtain the dimension list attribute for variable ",var->name);
979 
980  if (0==var->rank)
981  throw2("The number of dimension should NOT be 0 for the variable ",var->name);
982 
983  try {
984 
985  vlbuf.resize(var->rank);
986 
987  dset_id = H5Dopen(this->fileid,(var->fullpath).c_str(),H5P_DEFAULT);
988  if (dset_id < 0)
989  throw2("Cannot open the dataset ",var->fullpath);
990 
991  attr_id = H5Aopen(dset_id,(dimlistattr->name).c_str(),H5P_DEFAULT);
992  if (attr_id <0 )
993  throw4("Cannot open the attribute ",dimlistattr->name," of HDF5 dataset ",var->fullpath);
994 
995  atype_id = H5Aget_type(attr_id);
996  if (atype_id <0)
997  throw4("Cannot obtain the datatype of the attribute ",dimlistattr->name," of HDF5 dataset ",var->fullpath);
998 
999  amemtype_id = H5Tget_native_type(atype_id, H5T_DIR_ASCEND);
1000 
1001  if (amemtype_id < 0)
1002  throw2("Cannot obtain the memory datatype for the attribute ",dimlistattr->name);
1003 
1004 
1005  if (H5Aread(attr_id,amemtype_id,vlbuf.data()) <0)
1006  throw2("Cannot obtain the referenced object for the variable ",var->name);
1007 
1008 
1009  vector<char> objname;
1010  int vlbuf_index = 0;
1011 
1012  // The dimension names of variables will be the HDF5 dataset names dereferenced from the DIMENSION_LIST attribute.
1013  for (auto ird = var->dims.begin(); ird != var->dims.end(); ++ird) {
1014 
1015  if(vlbuf[vlbuf_index].p== nullptr)
1016  throw4("The dimension doesn't exist. Var name is ",var->name,"; the dimension index is ",vlbuf_index);
1017 
1018  rbuf =((hobj_ref_t*)vlbuf[vlbuf_index].p)[0];
1019  if ((ref_dset = H5RDEREFERENCE(attr_id, H5R_OBJECT, &rbuf)) < 0)
1020  throw2("Cannot dereference from the DIMENSION_LIST attribute for the variable ",var->name);
1021  // The above code works with dereferencing references generated with new H5R(H5R_ref_t) APIs with 1.12 and 1.13.
1022  // However, in case this h5rdeference2 API stops working with new APIs, the following #if 0 #endif block is a
1023  // way to handle this issue.
1024 #if 0
1025 
1026  rbuf =((hobj_ref_t*)vl_ref[0].p)[0];
1027  H5E_BEGIN_TRY {
1028  dset1 = H5Rdereference2(attr_id,H5P_DEFAULT,H5R_OBJECT,&ds_ref_buf);
1029  } H5E_END_TRY;
1030 
1031  H5R_ref_t new_rbuf =((H5R_ref_t*)vlbuf[vlbuf_index].p)[0];
1032  if ((ref_dset = H5Ropen_object((H5R_ref_t *)&new_rbuf, H5P_DEFAULT, H5P_DEFAULT))<0)
1033  throw2("Cannot dereference from the DIMENSION_LIST attribute for the variable ",var->name);
1034  H5Rdestroy(&new_rbuf);
1035 
1036 #endif
1037  if ((objnamelen= H5Iget_name(ref_dset,nullptr,0))<=0)
1038  throw2("Cannot obtain the dataset name dereferenced from the DIMENSION_LIST attribute for the variable ",var->name);
1039  objname.resize(objnamelen+1);
1040  if ((objnamelen= H5Iget_name(ref_dset,objname.data(),objnamelen+1))<=0)
1041  throw2("Cannot obtain the dataset name dereferenced from the DIMENSION_LIST attribute for the variable ",var->name);
1042 
1043  auto objname_str = string(objname.begin(),objname.end());
1044  string trim_objname = objname_str.substr(0,objnamelen);
1045  (*ird)->name = string(trim_objname.begin(),trim_objname.end());
1046 
1047  pair<set<string>::iterator,bool> setret;
1048  setret = dimnamelist.insert((*ird)->name);
1049  if (true == setret.second)
1050  Insert_One_NameSizeMap_Element((*ird)->name,(*ird)->size,(*ird)->unlimited_dim);
1051  (*ird)->newname = (*ird)->name;
1052  H5Dclose(ref_dset);
1053  objname.clear();
1054  vlbuf_index++;
1055  }// end of for
1056 
1057  if(vlbuf.size()!= 0) {
1058 
1059  if ((aspace_id = H5Aget_space(attr_id)) < 0)
1060  throw2("Cannot get hdf5 dataspace id for the attribute ",dimlistattr->name);
1061 
1062  if (H5Dvlen_reclaim(amemtype_id,aspace_id,H5P_DEFAULT,(void*)vlbuf.data())<0)
1063  throw2("Cannot successfully clean up the variable length memory for the variable ",var->name);
1064 
1065  H5Sclose(aspace_id);
1066 
1067  }
1068 
1069  H5Tclose(atype_id);
1070  H5Tclose(amemtype_id);
1071  H5Aclose(attr_id);
1072  H5Dclose(dset_id);
1073 
1074  }
1075 
1076  catch(...) {
1077 
1078  if(atype_id != -1)
1079  H5Tclose(atype_id);
1080 
1081  if(amemtype_id != -1)
1082  H5Tclose(amemtype_id);
1083 
1084  if(aspace_id != -1)
1085  H5Sclose(aspace_id);
1086 
1087  if(attr_id != -1)
1088  H5Aclose(attr_id);
1089 
1090  if(dset_id != -1)
1091  H5Dclose(dset_id);
1092 
1093  throw;
1094  }
1095 
1096 }
1097 
1098 // Add MeaSURES OZone level 3Z dimension names
1099 void GMFile::Add_Dim_Name_Mea_Ozonel3z() {
1100 
1101  BESDEBUG("h5", "Coming to Add_Dim_Name_Mea_Ozonel3z()"<<endl);
1102  iscoard = true;
1103  bool use_dimscale = false;
1104 
1105  for (auto irg = this->groups.begin(); irg != this->groups.end(); ++ irg) {
1106  if ("/Dimensions" == (*irg)->path) {
1107  use_dimscale = true;
1108  break;
1109  }
1110  }
1111 
1112  if (false == use_dimscale) {
1113 
1114  bool has_dimlist = false;
1115  bool has_class = false;
1116  bool has_reflist = false;
1117 
1118  for (auto irv = this->vars.begin(); irv != this->vars.end(); irv++) {
1119  for(auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end();ira++) {
1120  if ("DIMENSION_LIST" == (*ira)->name)
1121  has_dimlist = true;
1122  }
1123  if (true == has_dimlist)
1124  break;
1125  }
1126 
1127  if (true == has_dimlist) {
1128  for (auto irv = this->vars.begin(); irv != this->vars.end(); irv++) {
1129 
1130  for(auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end();ira++) {
1131  if ("CLASS" == (*ira)->name)
1132  has_class = true;
1133  if ("REFERENCE_LIST" == (*ira)->name)
1134  has_reflist = true;
1135  if (true == has_class && true == has_reflist)
1136  break;
1137  }
1138 
1139  if (true == has_class &&
1140  true == has_reflist)
1141  break;
1142 
1143  }
1144  if (true == has_class && true == has_reflist)
1145  use_dimscale = true;
1146  } // end of if (true == has_dimlist)
1147  } // end of if (false == use_dimscale)
1148 
1149  if (true == use_dimscale) {
1150 
1151  pair<set<string>::iterator,bool> setret;
1152  for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1153  Handle_UseDimscale_Var_Dim_Names_Mea_SeaWiFS_Ozone((*irv));
1154  for (auto ird = (*irv)->dims.begin(); ird !=(*irv)->dims.end();++ird) {
1155  setret = dimnamelist.insert((*ird)->name);
1156  if(true == setret.second)
1157  Insert_One_NameSizeMap_Element((*ird)->name,(*ird)->size,(*ird)->unlimited_dim);
1158  }
1159  }
1160 
1161  if (true == dimnamelist.empty())
1162  throw1("This product should have the dimension names, but no dimension names are found");
1163  } // end of if (true == use_dimscale)
1164 
1165  else {
1166 
1167  // Since the dim. size of each dimension of 2D lat/lon may be the same, so use multimap.
1168  multimap<hsize_t,string> ozonedimsize_to_dimname;
1169  pair<multimap<hsize_t,string>::iterator,multimap<hsize_t,string>::iterator> mm_er_ret;
1170  multimap<hsize_t,string>::iterator irmm;
1171 
1172  for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1173  bool is_cv = check_cv((*irv)->name);
1174  if (true == is_cv) {
1175  if ((*irv)->dims.size() != 1)
1176  throw3("The coordinate variable", (*irv)->name," must be one dimension for the zonal average product");
1177  ozonedimsize_to_dimname.insert(pair<hsize_t,string>(((*irv)->dims)[0]->size,(*irv)->fullpath));
1178  }
1179  }// end of for
1180 
1181  set<hsize_t> fakedimsize;
1182  pair<set<hsize_t>::iterator,bool> setsizeret;
1183  pair<set<string>::iterator,bool> setret;
1184  pair<set<string>::iterator,bool> tempsetret;
1185  set<string> tempdimnamelist;
1186  bool fakedimflag = false;
1187 
1188  for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1189 
1190  for (auto ird = (*irv)->dims.begin(); ird != (*irv)->dims.end(); ++ird) {
1191 
1192  fakedimflag = true;
1193  mm_er_ret = ozonedimsize_to_dimname.equal_range((*ird)->size);
1194  for (irmm = mm_er_ret.first; irmm!=mm_er_ret.second;irmm++) {
1195  setret = tempdimnamelist.insert(irmm->second);
1196  if (true == setret.second) {
1197  (*ird)->name = irmm->second;
1198  (*ird)->newname = (*ird)->name;
1199  setret = dimnamelist.insert((*ird)->name);
1200  if(setret.second) Insert_One_NameSizeMap_Element((*ird)->name,(*ird)->size,(*ird)->unlimited_dim);
1201  fakedimflag = false;
1202  break;
1203  }
1204  }
1205 
1206  if (true == fakedimflag) {
1207  Add_One_FakeDim_Name(*ird);
1208  setsizeret = fakedimsize.insert((*ird)->size);
1209  if (false == setsizeret.second)
1210  Adjust_Duplicate_FakeDim_Name(*ird);
1211  }
1212 
1213  } // end of for
1214  tempdimnamelist.clear();
1215  fakedimsize.clear();
1216  } // end of for
1217  } // end of else
1218 }
1219 
1220 // This is a special helper function for MeaSURES ozone products
1221 bool GMFile::check_cv(const string & varname) const {
1222 
1223  BESDEBUG("h5", "Coming to check_cv()"<<endl);
1224  const string lat_name ="Latitude";
1225  const string time_name ="Time";
1226  const string ratio_pressure_name ="MixingRatioPressureLevels";
1227  const string profile_pressure_name ="ProfilePressureLevels";
1228  const string wave_length_name ="Wavelength";
1229 
1230  if (lat_name == varname)
1231  return true;
1232  else if (time_name == varname)
1233  return true;
1234  else if (ratio_pressure_name == varname)
1235  return true;
1236  else if (profile_pressure_name == varname)
1237  return true;
1238  else if (wave_length_name == varname)
1239  return true;
1240  else
1241  return false;
1242 }
1243 
1244 // Add Dimension names for GPM products
1245 void GMFile::Add_Dim_Name_GPM()
1246 {
1247 
1248  BESDEBUG("h5", "Coming to Add_Dim_Name_GPM()"<<endl);
1249  // This is used to create a dimension name set.
1250  pair<set<string>::iterator,bool> setret;
1251 
1252  // The commented code is for an old version of GPM products. May remove them later. KY 2015-06-16
1253  // We need to create a fakedim name to fill in. To make the dimension name unique, we use a counter.
1254 #if 0
1255  // int dim_count = 0;
1256  // map<string,string> varname_to_fakedim;
1257  // map<int,string> gpm_dimsize_to_fakedimname;
1258 #endif
1259 
1260  // We find that GPM has an attribute DimensionNames(nlon,nlat) in this case.
1261  // We will use this attribute to specify the dimension names.
1262  for (auto irv = this->vars.begin(); irv != this->vars.end(); irv++) {
1263 
1264  for (auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira) {
1265 
1266  if("DimensionNames" == (*ira)->name) {
1267 
1268  Retrieve_H5_Attr_Value(*ira,(*irv)->fullpath);
1269  string dimname_value((*ira)->value.begin(),(*ira)->value.end());
1270 
1271  vector<string> ind_elems;
1272  char sep=',';
1273  HDF5CFUtil::Split(dimname_value.data(),sep,ind_elems);
1274 
1275  if(ind_elems.size() != (size_t)((*irv)->getRank())) {
1276  throw2("The number of dims obtained from the <DimensionNames> attribute is not equal to the rank ",
1277  (*irv)->name);
1278  }
1279 
1280  for(unsigned int i = 0; i<ind_elems.size(); ++i) {
1281 
1282  ((*irv)->dims)[i]->name = ind_elems[i];
1283 
1284  // Generate a dimension name if the dimension name is missing.
1285  // The routine will ensure that the fakeDim name is unique.
1286  if(((*irv)->dims)[i]->name==""){
1287  Add_One_FakeDim_Name(((*irv)->dims)[i]);
1288 // For debugging
1289 #if 0
1290  string fakedim = "FakeDim";
1291  stringstream sdim_count;
1292  sdim_count << dim_count;
1293  fakedim = fakedim + sdim_count.str();
1294  dim_count++;
1295  ((*irv)->dims)[i]->name = fakedim;
1296  ((*irv)->dims)[i]->newname = fakedim;
1297  ind_elems[i] = fakedim;
1298 #endif
1299  }
1300 
1301  else {
1302  ((*irv)->dims)[i]->newname = ind_elems[i];
1303  setret = dimnamelist.insert(((*irv)->dims)[i]->name);
1304 
1305  if (true == setret.second) {
1306  Insert_One_NameSizeMap_Element(((*irv)->dims)[i]->name,
1307  ((*irv)->dims)[i]->size,
1308  ((*irv)->dims)[i]->unlimited_dim);
1309  }
1310  else {
1311  if(dimname_to_dimsize[((*irv)->dims)[i]->name] !=((*irv)->dims)[i]->size)
1312  throw5("Dimension ",((*irv)->dims)[i]->name, "has two sizes",
1313  ((*irv)->dims)[i]->size,dimname_to_dimsize[((*irv)->dims)[i]->name]);
1314 
1315  }
1316  }
1317 
1318  }// end of for
1319  break;
1320 
1321  } //end of if
1322  } //end of for
1323 
1324 #if 0
1325  if(false == has_dim_name_attr) {
1326 
1327  throw4( "The variable ", (*irv)->name, " doesn't have the DimensionNames attribute.",
1328  "We currently don't support this case. Please report to the NASA data center.");
1329  }
1330 
1331 #endif
1332  } //end of for
1333 
1334 }
1335 
1336 // Add Dimension names for Aquarius level 3 products
1337 void GMFile::Add_Dim_Name_Aqu_L3()
1338 {
1339  BESDEBUG("h5", "Coming to Add_Dim_Name_Aqu_L3()"<<endl);
1340  for (auto irv = this->vars.begin(); irv != this->vars.end(); irv++) {
1341  if ("l3m_data" == (*irv)->name) {
1342  ((*irv)->dims)[0]->name = "lat";
1343  ((*irv)->dims)[0]->newname = "lat";
1344  ((*irv)->dims)[1]->name = "lon";
1345  ((*irv)->dims)[1]->newname = "lon";
1346  break;
1347  }
1348 
1349 // For the time being, don't assign dimension names to palette,
1350 // we will see if tools can pick up l3m and then make decisions.
1351 #if 0
1352  if ("palette" == (*irv)->name) {
1353 //"h5","coming to palette" <<endl;
1354  ((*irv)->dims)[0]->name = "paldim0";
1355  ((*irv)->dims)[0]->newname = "paldim0";
1356  ((*irv)->dims)[1]->name = "paldim1";
1357  ((*irv)->dims)[1]->newname = "paldim1";
1358  }
1359 #endif
1360 
1361  }// end of for
1362 }
1363 
1364 // Add dimension names for OSMAPL2S(note: the SMAP change their structures. The code doesn't not apply to them.)
1365 void GMFile::Add_Dim_Name_OSMAPL2S(){
1366 
1367  BESDEBUG("h5", "Coming to Add_Dim_Name_OSMAPL2S()"<<endl);
1368  string tempvarname ="";
1369  string key = "_lat";
1370  string osmapl2sdim0 ="YDim";
1371  string osmapl2sdim1 ="XDim";
1372 
1373  // Since the dim. size of each dimension of 2D lat/lon may be the same, so use multimap.
1374  multimap<hsize_t,string> osmapl2sdimsize_to_dimname;
1375  pair<multimap<hsize_t,string>::iterator,multimap<hsize_t,string>::iterator> mm_er_ret;
1376  multimap<hsize_t,string>::iterator irmm;
1377 
1378  // Generate dimension names based on the size of "???_lat"(one coordinate variable)
1379  for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1380  tempvarname = (*irv)->name;
1381  if ((tempvarname.size() > key.size())&&
1382  (key == tempvarname.substr(tempvarname.size()-key.size(),key.size()))){
1383  if ((*irv)->dims.size() !=2)
1384  throw1("Currently only 2D lat/lon is supported for OSMAPL2S");
1385  osmapl2sdimsize_to_dimname.insert(pair<hsize_t,string>(((*irv)->dims)[0]->size,osmapl2sdim0));
1386  osmapl2sdimsize_to_dimname.insert(pair<hsize_t,string>(((*irv)->dims)[1]->size,osmapl2sdim1));
1387  break;
1388  }
1389  }
1390 
1391  set<hsize_t> fakedimsize;
1392  pair<set<hsize_t>::iterator,bool> setsizeret;
1393  pair<set<string>::iterator,bool> setret;
1394  pair<set<string>::iterator,bool> tempsetret;
1395  set<string> tempdimnamelist;
1396  bool fakedimflag = false;
1397 
1398 
1399  for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1400  for (auto ird = (*irv)->dims.begin(); ird != (*irv)->dims.end(); ++ird) {
1401 
1402  fakedimflag = true;
1403  mm_er_ret = osmapl2sdimsize_to_dimname.equal_range((*ird)->size);
1404  for (irmm = mm_er_ret.first; irmm!=mm_er_ret.second;irmm++) {
1405  setret = tempdimnamelist.insert(irmm->second);
1406  if (setret.second) {
1407  (*ird)->name = irmm->second;
1408  (*ird)->newname = (*ird)->name;
1409  setret = dimnamelist.insert((*ird)->name);
1410  if(setret.second) Insert_One_NameSizeMap_Element((*ird)->name,(*ird)->size,(*ird)->unlimited_dim);
1411  fakedimflag = false;
1412  break;
1413  }
1414  }
1415 
1416  if (true == fakedimflag) {
1417  Add_One_FakeDim_Name(*ird);
1418  setsizeret = fakedimsize.insert((*ird)->size);
1419  if (!setsizeret.second)
1420  Adjust_Duplicate_FakeDim_Name(*ird);
1421  }
1422  } // end of for ird
1423  tempdimnamelist.clear();
1424  fakedimsize.clear();
1425  } // end of for irv
1426 }
1427 
1428 //Add dimension names for ACOS level2S or OCO2 level1B products
1429 void GMFile::Add_Dim_Name_ACOS_L2S_OCO2_L1B(){
1430 
1431  BESDEBUG("h5", "Coming to Add_Dim_Name_ACOS_L2S_OCO2_L1B()"<<endl);
1432  for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1433  set<hsize_t> fakedimsize;
1434  pair<set<hsize_t>::iterator,bool> setsizeret;
1435  for (auto ird= (*irv)->dims.begin(); ird != (*irv)->dims.end(); ++ird) {
1436  Add_One_FakeDim_Name(*ird);
1437  setsizeret = fakedimsize.insert((*ird)->size);
1438  if (false == setsizeret.second)
1439  Adjust_Duplicate_FakeDim_Name(*ird);
1440  }
1441  } // end of for irv
1442 }
1443 
1444 // Add dimension names for general products. Read the descrption of Check_General_Product_Pattern() for different patterns we support.
1445 void GMFile::Add_Dim_Name_General_Product(){
1446 
1447  BESDEBUG("h5", "Coming to Add_Dim_Name_General_Product()"<<endl);
1448 
1449  // This general product should follow the HDF5 dimension scale model.
1450  if (GENERAL_DIMSCALE == this->gproduct_pattern){
1451  Add_Dim_Name_Dimscale_General_Product();
1452 }
1453  // This general product has 2-D latitude,longitude
1454  else if (GENERAL_LATLON2D == this->gproduct_pattern)
1455  Add_Dim_Name_LatLon2D_General_Product();
1456  // This general product has 1-D latitude,longitude
1457  else if (GENERAL_LATLON1D == this->gproduct_pattern || GENERAL_LATLON_COOR_ATTR == this->gproduct_pattern)
1458  Add_Dim_Name_LatLon1D_Or_CoordAttr_General_Product();
1459 
1460 
1461 }
1462 
1463 // We check four patterns under the General_Product category
1464 // 1. General products that uses HDF5 dimension scales following netCDF-4 data model
1465 // 2. General products that have 2-D lat/lon variables(lat/lon variable names are used to identify the case) under the root group or
1466 // a special geolocation group
1467 // 3. General products that have 1-D lat/lon variables(lat/lon variable names are used to identify the case) under the root group or
1468 // a special geolocation group
1469 // 4. General products that have some variables containing CF "coordinates" attributes. We can support some products if the "coordinates"
1470 // attribute contains CF lat/lon units and the variable ranks are 2 or 1.
1471 void GMFile::Check_General_Product_Pattern() {
1472 
1473  BESDEBUG("h5", "Coming to Check_General_Product_Pattern()"<<endl);
1474  if (false == Check_Dimscale_General_Product_Pattern()) {
1475  //HERE add a check for the GPM. (choose 5 variables equally distance for the attribute)
1476  if (false == Check_And_Update_New_GPM_L3())
1477  if (false == Check_LatLon2D_General_Product_Pattern())
1478  if (false == Check_LatLon1D_General_Product_Pattern())
1479  Check_LatLon_With_Coordinate_Attr_General_Product_Pattern();
1480  }
1481 
1482 }
1483 
1484 // Check if this general product is netCDF4-like HDF5 file.
1485 // We only need to check "DIMENSION_LIST","CLASS" and CLASS values.
1486 bool GMFile::Check_Dimscale_General_Product_Pattern() {
1487 
1488  BESDEBUG("h5", "Coming to Check_Dimscale_General_Product_Pattern()"<<endl);
1489  bool ret_value = false;
1490  bool has_dimlist = false;
1491  bool has_dimscalelist = false;
1492 
1493  // Check if containing the "DIMENSION_LIST" attribute;
1494  for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1495  for(auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end();ira++) {
1496  if ("DIMENSION_LIST" == (*ira)->name) {
1497  has_dimlist = true;
1498  break;
1499  }
1500  }
1501  if (true == has_dimlist)
1502  break;
1503  }
1504 
1505  // Check if containing both the attribute "CLASS" and the attribute "REFERENCE_LIST" for the same variable.
1506  // This is the dimension scale.
1507  // Actually "REFERENCE_LIST" is not necessary for a dimension scale dataset. If a dimension scale doesn't
1508  // have a "REFERENCE_LIST", it is still valid. But no other variables use this dimension scale. We found
1509  // such a case in a matched_airs_aqua product. KY 2012-12-03
1510  for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1511  for(auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end();ira++) {
1512  if ("CLASS" == (*ira)->name) {
1513 
1514  Retrieve_H5_Attr_Value(*ira,(*irv)->fullpath);
1515  string class_value;
1516  class_value.resize((*ira)->value.size());
1517  copy((*ira)->value.begin(),(*ira)->value.end(),class_value.begin());
1518 
1519  // Compare the attribute "CLASS" value with "DIMENSION_SCALE". We only compare the string with the size of
1520  // "DIMENSION_SCALE", which is 15.
1521  if (0 == class_value.compare(0,15,"DIMENSION_SCALE")) {
1522  has_dimscalelist = true;
1523  break;
1524  }
1525  }
1526  }
1527  if (true == has_dimscalelist)
1528  break;
1529  }
1530 
1531  if (true == has_dimscalelist) {
1532  if (true == has_dimlist ) {
1533  this->gproduct_pattern = GENERAL_DIMSCALE;
1534  ret_value = true;
1535  }
1536  else {
1537  //May fall into the single dimension scale case.
1538  //This is really, really rare,but we do have to check.
1539  // Check if NAME and _Netcdf4Dimid exists for this variable.
1540 
1541  bool is_general_dimscale = false;
1542 
1543  for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1544 
1545  bool has_class_dscale = false;
1546  bool has_name = false;
1547  bool has_netcdf4_id = false;
1548 
1549  for (auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end();ira++) {
1550 
1551  if ("CLASS" == (*ira)->name) {
1552  Retrieve_H5_Attr_Value(*ira,(*irv)->fullpath);
1553  string class_value;
1554  class_value.resize((*ira)->value.size());
1555  copy((*ira)->value.begin(),(*ira)->value.end(),class_value.begin());
1556 
1557  // Compare the attribute "CLASS" value with "DIMENSION_SCALE". We only compare the string with the size of
1558  // "DIMENSION_SCALE", which is 15.
1559  if (0 == class_value.compare(0,15,"DIMENSION_SCALE"))
1560  has_class_dscale= true;
1561  }
1562  else if ("NAME" == (*ira)->name)
1563  has_name = true;
1564  else if ("_Netcdf4Dimid" == (*ira)->name)
1565  has_netcdf4_id = true;
1566  if(true == has_class_dscale && true == has_name && true == has_netcdf4_id)
1567  is_general_dimscale = true;
1568  }
1569 
1570  if(true == is_general_dimscale)
1571  break;
1572  }
1573 
1574  if (true == is_general_dimscale) {
1575  this->gproduct_pattern = GENERAL_DIMSCALE;
1576  ret_value = true;
1577  }
1578  }
1579  }
1580 
1581  return ret_value;
1582 }
1583 
1584 bool GMFile::Check_And_Update_New_GPM_L3() {
1585 
1586  bool is_new_gpm_l3 = false;
1587  unsigned num_vars = this->vars.size();
1588  unsigned sel_steps = num_vars/5;
1589  string dim_name="DimensionNames";
1590  bool has_dim_name = false;
1591  if(sel_steps == 0)
1592  sel_steps = 1;
1593 
1594  // Given DimensionNames exists in almost every variable in the new GPM product,
1595  // We will check the existence of this attribute for at most 5 variables.
1596 
1597  vector<Var *>::iterator it_var_end;
1598 
1599  if(sel_steps ==1)
1600  it_var_end = this->vars.end();
1601  else
1602  it_var_end = this->vars.begin()+5*sel_steps;
1603 
1604  for (auto irv = this->vars.begin(); irv != it_var_end; irv+=sel_steps) {
1605  for(auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end();ira++) {
1606 
1607  if(H5FSTRING == (*ira)->getType()) {
1608  if((*ira)->name == dim_name){
1609  has_dim_name = true;
1610  break;
1611  }
1612  }
1613  }
1614  if(true == has_dim_name)
1615  break;
1616  }
1617 
1618 
1619 
1620  // Files that can go to this step should be a small subset, now
1621  // we will check the "??GridHeader" for all the groups.
1622  if(true == has_dim_name) {
1623  string attr_name_subset = "GridHeader";
1624  BESDEBUG("h5", "GMFile::Check_And_Update_New_GPM_L3() has attribute <DimensionNames>. "<<endl);
1625  for (auto irg = this->groups.begin(); irg != this->groups.end(); ++ irg) {
1626  for (auto ira = (*irg)->attrs.begin(); ira != (*irg)->attrs.end();ira++) {
1627 
1628  string attr_name = (*ira)->name;
1629 
1630  // We identify this as a new GPM level 3 product.
1631  if (attr_name.find(attr_name_subset)!=string::npos) {
1632  this->product_type = GPM_L3_New;
1633  is_new_gpm_l3 = true;
1634  break;
1635  }
1636  }
1637  if (true == is_new_gpm_l3)
1638  break;
1639  }
1640  }
1641  return is_new_gpm_l3;
1642 }
1643 
1644 // If having 2-D latitude/longitude,set the general product pattern.
1645 // In this version, we only check if we have "latitude,longitude","Latitude,Longitude","lat,lon" and "cell_lat,cell_lon"names.
1646 // The "cell_lat" and "cell_lon" come from SMAP. KY 2015-12-2
1647 bool GMFile::Check_LatLon2D_General_Product_Pattern() {
1648 
1649  BESDEBUG("h5", "Coming to Check_LatLon2D_General_Product_Pattern()"<<endl);
1650  bool ret_value = false;
1651 
1652  ret_value = Check_LatLon2D_General_Product_Pattern_Name_Size("latitude","longitude");
1653  if(false == ret_value) {
1654  ret_value = Check_LatLon2D_General_Product_Pattern_Name_Size("Latitude","Longitude");
1655  if(false == ret_value) {
1656  ret_value = Check_LatLon2D_General_Product_Pattern_Name_Size("lat","lon");
1657  if(false == ret_value)
1658  ret_value = Check_LatLon2D_General_Product_Pattern_Name_Size("cell_lat","cell_lon");
1659  }
1660  }
1661 
1662  // Make sure set the general product pattern flag for this case.
1663  if(true == ret_value)
1664  this->gproduct_pattern = GENERAL_LATLON2D;
1665  return ret_value;
1666 
1667 }
1668 
1669 // Helper function for Check_LatLon2D_General_Product_Pattern,we assume the lat and lon only present either under the root or
1670 // a specific group Geolocation.
1671 bool GMFile::Check_LatLon2D_General_Product_Pattern_Name_Size(const string & latname,const string & lonname) {
1672 
1673  BESDEBUG("h5", "Coming to Check_LatLon2D_General_Product_Pattern_Name_Size()"<<endl);
1674  bool ret_value = false;
1675  bool ll_flag = false;
1676 
1677  vector<size_t>lat_size(2,0);
1678  vector<size_t>lon_size(2,0);
1679 
1680  const string designed_group1 = "/";
1681  const string designed_group2 = "/Geolocation/";
1682 
1683  bool lat_flag_g1 = false;
1684  bool lon_flag_g1 = false;
1685  bool lat_flag_g2 = false;
1686  bool lon_flag_g2 = false;
1687 
1688 
1689  // This case allows to have both "lat and lon" under either group 1 or group 2 but on not both group 1 and 2.
1690  // This case doesn't allow "lat" and "lon" under separate groups.
1691  // Check if we have lat and lon at the only designated group,group 1 "/"
1692  lat_flag_g1 = is_var_under_group(latname,designed_group1,2,lat_size);
1693  lon_flag_g1 = is_var_under_group(lonname,designed_group1,2,lon_size);
1694  if(lat_flag_g1 == true && lon_flag_g1 == true) {
1695 
1696  // Make sure the group 2 "/Geolocation" doesn't have the lat/lon
1697  lat_flag_g2 = is_var_under_group(latname,designed_group2,2,lat_size);
1698  if(lat_flag_g2 == false) {
1699  lon_flag_g2 = is_var_under_group(lonname,designed_group2,2,lon_size);
1700  if(lon_flag_g2 == false)
1701  ll_flag = true;
1702  }
1703  }// If the root doesn't have lat/lon, check the group 2 "/Geolocation".
1704  else if(lat_flag_g1 == false && lon_flag_g1 == false) {
1705  lat_flag_g2 = is_var_under_group(latname,designed_group2,2,lat_size);
1706  if(lat_flag_g2 == true) {
1707  lon_flag_g2 = is_var_under_group(lonname,designed_group2,2,lon_size);
1708  if(lon_flag_g2 == true)
1709  ll_flag = true;
1710  }
1711  }
1712 
1713  // We are loose here since this is just to support some NASA products in a customized way.
1714  // If the first two cases don't exist, we allow to check another group"GeolocationData" and
1715  // see if Latitude and Longitude are present. (4 years ? from the first implementation, we got this case.)
1716  // KY 2020-02-27
1717  if(false == ll_flag) {
1718 
1719  const string designed_group3 = "/GeolocationData/";
1720  if(is_var_under_group(latname,designed_group3,2,lat_size) &&
1721  is_var_under_group(lonname,designed_group3,2,lon_size))
1722  ll_flag = true;
1723  }
1724 
1725 #if 0
1726 
1727  for (vector<Var *>::iterator irv = this->vars.begin();
1728  irv != this->vars.end(); ++irv) {
1729 
1730  if((*irv)->rank == 2) {
1731  if((*irv)->name == latname) {
1732 
1733  // Obtain the variable path
1734  string lat_path =HDF5CFUtil::obtain_string_before_lastslash((*irv)->fullpath);
1735 
1736  // Tackle only the root group or the name of the group as "/Geolocation"
1737  // By doing this, we assume that the file has lat/lon either under the root or under the "Geolocation
1738  // but not BOTH. The following code may generate wrong results if the file contains lat/lon under
1739  // both the root and /Geolocation. This is documented in https://jira.hdfgroup.org/browse/HFVHANDLER-175
1740  bool has_right_lat = false;
1741  if("/" == lat_path || "/Geolocation/" == lat_path)
1742  if("/" == lat_path || "/Geolocation/" == lat_path) {
1743  ll_flag++;
1744  lat_size[0] = (*irv)->getDimensions()[0]->size;
1745  lat_size[1] = (*irv)->getDimensions()[1]->size;
1746  }
1747 
1748  }
1749  else if((*irv)->name == lonname) {
1750  string lon_path = HDF5CFUtil::obtain_string_before_lastslash((*irv)->fullpath);
1751  if("/" == lon_path || "/Geolocation/" == lon_path) {
1752  ll_flag++;
1753  lon_size[0] = (*irv)->getDimensions()[0]->size;
1754  lon_size[1] = (*irv)->getDimensions()[1]->size;
1755  }
1756  }
1757  if(2 == ll_flag)
1758  break;
1759  } // if((*irv)->rank == 2)
1760  } // for (vector<Var *>::iterator irv = this->vars.begin();
1761 
1762 #endif
1763 
1764  // Only when both lat/lon are found can we support this case.
1765  // Before that, we also need to check if the lat/lon shares the same dimensions.
1766  if(true == ll_flag) {
1767 
1768  bool latlon_size_match = true;
1769  for (unsigned int size_index = 0; size_index <lat_size.size();size_index++) {
1770  if(lat_size[size_index] != lon_size[size_index]){
1771  latlon_size_match = false;
1772  break;
1773  }
1774  }
1775  if (true == latlon_size_match) {
1776  // If we do find the lat/lon pair, save them for later use.
1777  gp_latname = latname;
1778  gp_lonname = lonname;
1779  ret_value = true;
1780  }
1781 
1782  }
1783 
1784  return ret_value;
1785 
1786 }
1787 
1788 // If having 1-D latitude/longitude,set the general product pattern.
1789 // In this version, we only check if we have "latitude,longitude","Latitude,Longitude","lat,lon" and "cell_lat,cell_lon"names.
1790 // The "cell_lat" and "cell_lon" come from SMAP. KY 2015-12-2
1791 bool GMFile::Check_LatLon1D_General_Product_Pattern() {
1792 
1793  BESDEBUG("h5", "Coming to Check_LatLon1D_General_Product_Pattern()"<<endl);
1794  bool ret_value = false;
1795 
1796  ret_value = Check_LatLon1D_General_Product_Pattern_Name_Size("latitude","longitude");
1797  if(false == ret_value) {
1798  ret_value = Check_LatLon1D_General_Product_Pattern_Name_Size("Latitude","Longitude");
1799  if(false == ret_value) {
1800  ret_value = Check_LatLon1D_General_Product_Pattern_Name_Size("lat","lon");
1801  if(false == ret_value)
1802  ret_value = Check_LatLon1D_General_Product_Pattern_Name_Size("cell_lat","cell_lon");
1803  }
1804  }
1805 
1806  if(true == ret_value)
1807  this->gproduct_pattern = GENERAL_LATLON1D;
1808  return ret_value;
1809 
1810 }
1811 
1812 // Helper function for Check_LatLon1D_General_Product_Pattern.
1813 // We only check if the lat/lon etc. pairs are under "/" or "/Geolocation". Other cases can be easily added.
1814 bool GMFile::Check_LatLon1D_General_Product_Pattern_Name_Size(const string & latname,const string & lonname) {
1815 
1816  BESDEBUG("h5", "Coming to Check_LatLon1D_General_Product_Pattern_Name_Size()"<<endl);
1817  bool ret_value = false;
1818  short ll_flag = 0;
1819  size_t lat_size = 0;
1820  size_t lon_size = 0;
1821 
1822  for (vector<Var *>::iterator irv = this->vars.begin();
1823  irv != this->vars.end(); ++irv) {
1824 
1825  if((*irv)->rank == 1) {
1826  if((*irv)->name == latname) {
1827 
1828  string lat_path =HDF5CFUtil::obtain_string_before_lastslash((*irv)->fullpath);
1829 
1830  // Tackle only the root group or the name of the group as "/Geolocation"
1831  // May not generate the correct output. See https://jira.hdfgroup.org/browse/HFVHANDLER-175
1832  if("/" == lat_path || "/Geolocation/" == lat_path) {
1833  ll_flag++;
1834  lat_size = (*irv)->getDimensions()[0]->size;
1835  }
1836  }
1837  else if((*irv)->name == lonname) {
1838  string lon_path = HDF5CFUtil::obtain_string_before_lastslash((*irv)->fullpath);
1839  if("/" == lon_path || "/Geolocation/" == lon_path) {
1840  ll_flag++;
1841  lon_size = (*irv)->getDimensions()[0]->size;
1842  }
1843  }
1844  if(2 == ll_flag)
1845  break;
1846  }
1847  }
1848 
1849  if(2 == ll_flag) {
1850 
1851  bool latlon_size_match_grid = true;
1852 
1853  // When the size of latitude is equal to the size of longitude for a 1-D lat/lon, it is very possible
1854  // that this is not a regular grid but rather a profile with the lat,lon recorded as the function of time.
1855  // Adding the coordinate/dimension as the normal grid is wrong, so check out this case.
1856  // KY 2015-12-2
1857  if(lat_size == lon_size) {
1858 
1859  // It is very unusual that lat_size = lon_size for a grid.
1860  latlon_size_match_grid = false;
1861 
1862  // For a normal grid, a >2D variable should exist to have both lat and lon size,
1863  // if such a variable that has the same size exists, we will treat it as a normal grid.
1864  for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1865  if((*irv)->rank >=2) {
1866  short ll_size_flag = 0;
1867  for (auto ird= (*irv)->dims.begin(); ird != (*irv)->dims.end(); ++ird) {
1868  if(lat_size == (*ird)->size) {
1869  ll_size_flag++;
1870  if(2 == ll_size_flag){
1871  break;
1872  }
1873  }
1874  }
1875  if(2 == ll_size_flag) {
1876  latlon_size_match_grid = true;
1877  break;
1878  }
1879  }
1880  }
1881  }
1882 
1883  // If the sizes of lat and lon match the grid, this is the lat/lon candidate.
1884  // Save the latitude and longitude names for later use.
1885  if (true == latlon_size_match_grid) {
1886  gp_latname = latname;
1887  gp_lonname = lonname;
1888  ret_value = true;
1889  }
1890  }
1891 
1892  return ret_value;
1893 }
1894 
1895 // This function checks if this general product contains "coordinates" attributes in some variables
1896 // that can be used to handle CF friendly.
1897 bool GMFile::Check_LatLon_With_Coordinate_Attr_General_Product_Pattern() {
1898 
1899  BESDEBUG("h5", "Coming to Check_LatLon_With_Coordinate_Attr_General_Product_Pattern()"<<endl);
1900  bool ret_value = false;
1901  string co_attrname = "coordinates";
1902  string co_attrvalue="";
1903  string unit_attrname = "units";
1904  string lat_unit_attrvalue ="degrees_north";
1905  string lon_unit_attrvalue ="degrees_east";
1906 
1907  bool coor_has_lat_flag = false;
1908  bool coor_has_lon_flag = false;
1909 
1910  vector<Var*> tempvar_lat;
1911  vector<Var*> tempvar_lon;
1912 
1913  // Check if having both lat, lon names stored in the coordinate attribute value by looping through rank >1 variables.
1914  for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1915 
1916  if((*irv)->rank >=2) {
1917  for (auto ira =(*irv)->attrs.begin(); ira !=(*irv)->attrs.end();++ira) {
1918 
1919  // If having attribute "coordinates" for this variable, checking the values and
1920  // see if having lat/lon,latitude/longitude, Latitude/Longitude pairs.
1921  if((*ira)->name == co_attrname) {
1922  Retrieve_H5_Attr_Value((*ira),(*irv)->fullpath);
1923  string orig_attr_value((*ira)->value.begin(),(*ira)->value.end());
1924  vector<string> coord_values;
1925  char sep=' ';
1926  HDF5CFUtil::Split_helper(coord_values,orig_attr_value,sep);
1927 
1928  for(auto irs=coord_values.begin();irs!=coord_values.end();++irs) {
1929  string coord_value_suffix1;
1930  string coord_value_suffix2;
1931  string coord_value_suffix3;
1932 
1933  if((*irs).size() >=3) {
1934 
1935  // both "lat" and "lon" have 3 characters.
1936  coord_value_suffix1 = (*irs).substr((*irs).size()-3,3);
1937 
1938  // The word "latitude" has 8 characters and the word "longitude" has 9 characters.
1939  if((*irs).size() >=8){
1940  coord_value_suffix2 = (*irs).substr((*irs).size()-8,8);
1941  if((*irs).size() >=9)
1942  coord_value_suffix3 = (*irs).substr((*irs).size()-9,9);
1943  }
1944  }
1945 
1946  // lat/longitude or latitude/lon pairs in theory are fine.
1947  if(coord_value_suffix1=="lat" || coord_value_suffix2 =="latitude" || coord_value_suffix2 == "Latitude")
1948  coor_has_lat_flag = true;
1949  else if(coord_value_suffix1=="lon" || coord_value_suffix3 =="longitude" || coord_value_suffix3 == "Longitude")
1950  coor_has_lon_flag = true;
1951  }
1952 
1953  if(true == coor_has_lat_flag && true == coor_has_lon_flag)
1954  break;
1955  }// end of if((*ira)->name
1956  }// end of for (ira)
1957 
1958  if(true == coor_has_lat_flag && true == coor_has_lon_flag)
1959  break;
1960  else {
1961  coor_has_lat_flag = false;
1962  coor_has_lon_flag = false;
1963  }
1964  } // end of if
1965  }// end of for
1966 
1967  // Check the variable names that include latitude and longitude suffixes such as lat,latitude and Latitude.
1968  if(true == coor_has_lat_flag && true == coor_has_lon_flag) {
1969 
1970  for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1971 
1972  bool var_is_lat = false;
1973  bool var_is_lon = false;
1974 
1975  string varname = (*irv)->name;
1976  string ll_ssuffix;
1977  string ll_lsuffix1;
1978  string ll_lsuffix2;
1979  if(varname.size() >=3) {//lat/lon
1980  ll_ssuffix = varname.substr(varname.size()-3,3);
1981  if(varname.size() >=8) {//latitude/Latitude
1982  ll_lsuffix1 = varname.substr(varname.size()-8,8);
1983  if(varname.size() >=9)//Longitude/longitude
1984  ll_lsuffix2 = varname.substr(varname.size()-9,9);
1985  }
1986  }
1987  if(ll_ssuffix=="lat" || ll_lsuffix1 =="latitude" || ll_lsuffix1 == "Latitude")
1988  var_is_lat = true;
1989  else if(ll_ssuffix=="lon" || ll_lsuffix2 =="longitude" || ll_lsuffix2 == "Longitude")
1990  var_is_lon = true;
1991 
1992  // Find the lat/lon candidate, save them to temporary vectors
1993  if(true == var_is_lat) {
1994  if((*irv)->rank > 0) {
1995  auto lat = new Var(*irv);
1996  tempvar_lat.push_back(lat);
1997  }
1998  }
1999  else if(true == var_is_lon) {
2000  if((*irv)->rank >0) {
2001  auto lon = new Var(*irv);
2002  tempvar_lon.push_back(lon);
2003  }
2004  }
2005  }// for of irv
2006 
2007  // Build up latloncv_candidate_pairs, Name_Size_2Pairs struct,
2008  // 1) Compare the rank, dimension sizes and the dimension orders of tempvar_lon against tempvar_lat
2009  // rank >=2 the sizes,orders, should be consistent
2010  // rank =1, no check issue.
2011  // 2) If the conditions are fulfilled, save them to the Name_Size struct
2012  for (auto irlat = tempvar_lat.begin(); irlat!=tempvar_lat.end();++irlat) {
2013 
2014  // Check the rank =1 case
2015  if((*irlat)->rank == 1)
2016  Build_lat1D_latlon_candidate(*irlat,tempvar_lon);
2017 
2018  // Check the reank>=2 case
2019  else if((*irlat)->rank >1)
2020  Build_latg1D_latlon_candidate(*irlat,tempvar_lon);
2021  }
2022 
2023 #if 0
2024 for(vector<struct Name_Size_2Pairs>::iterator ivs=latloncv_candidate_pairs.begin(); ivs!=latloncv_candidate_pairs.end();++ivs) {
2025 cerr<<"struct lat lon names are " <<(*ivs).name1 <<" and " << (*ivs).name2 <<endl;
2026 }
2027 #endif
2028 
2029  // Check if there is duplicate latitude variables for one longitude variable in the latloncv_candidate_pairs.
2030  // if yes, remove the ones that have duplicate latitude variables.
2031  // This will assure that the latloncv_candidate_pairs is one-to-one mapping between latitude and longitude.
2032  Build_unique_latlon_candidate();
2033 
2034 
2035  // Even if we find that there are qualified geo-location coordinate pairs, we still need to check
2036  // the geo-location variable rank.
2037  // If the rank of any one-pair is 2, this case is qualified for the category GENERAL_LATLON_COOR_ATTR.
2038  // If the rank of any one-pair is 1,
2039  // we will check if the sizes of the lat and the lon in a pair are the same.
2040  // If they are not the same, this case is qualified for the category GENERAL_LATLON_COOR_ATTR
2041  // else check if there is any variable that has the "coordinates" attribute and the "coordinates" attribute includes
2042  // the paths of this lat/lon pair. If the dimensions of such a variable have two sizes that are equal to the size of the lat,
2043  // this case is still qualfied for the category GENERAL_LATLON_COOR_ATTR.
2044  // NOTE: here we deliberately ignore the case when the rank of lat/lon is >2. In some recent developments, we find that
2045  // there are 3D lat/lon and some tools like Panoply can visualize those data. So maybe we need to accept some 3D lat/lon in the futurei(KY 2016-07-07).
2046  if(latloncv_candidate_pairs.size() >0) {
2047  int num_1d_rank = 0;
2048  int num_2d_rank = 0;
2049  int num_g2d_rank = 0;
2050  vector<struct Name_Size_2Pairs> temp_1d_latlon_pairs;
2051  for(auto ivs=latloncv_candidate_pairs.begin(); ivs!=latloncv_candidate_pairs.end();++ivs) {
2052  if(1 == (*ivs).rank) {
2053  num_1d_rank++;
2054  temp_1d_latlon_pairs.push_back(*ivs);
2055  }
2056  else if(2 == (*ivs).rank)
2057  num_2d_rank++;
2058  else if((*ivs).rank >2)
2059  num_g2d_rank++;
2060  }
2061 
2062  // This is the GENERAL_LATLON_COOR_ATTR case.
2063  if (num_2d_rank !=0)
2064  ret_value = true;
2065  else if(num_1d_rank!=0) {
2066 
2067  // Check if lat and lon share the same size and the dimension of a variable
2068  // that has the "coordinates" only holds one size.
2069  for(auto ivs=temp_1d_latlon_pairs.begin(); ivs!=temp_1d_latlon_pairs.end();++ivs) {
2070  if((*ivs).size1 != (*ivs).size2) {
2071  ret_value = true;
2072  break;
2073  }
2074  else {
2075 
2076  // If 1-D lat and lon share the same size,we need to check if there is a variable
2077  // that has both lat and lon as the coordinates but only has one dimension that holds the size.
2078  // If this is true, this is not the GENERAL_LATLON_COOR_ATTR case(SMAP level 2 follows into the category).
2079 
2080  ret_value = true;
2081  for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
2082 
2083  if((*irv)->rank >=2) {
2084  for (auto ira =(*irv)->attrs.begin(); ira !=(*irv)->attrs.end();++ira) {
2085 
2086  // Check if this variable has the "coordinates" attribute
2087  if((*ira)->name == co_attrname) {
2088  Retrieve_H5_Attr_Value((*ira),(*irv)->fullpath);
2089  string orig_attr_value((*ira)->value.begin(),(*ira)->value.end());
2090  vector<string> coord_values;
2091  char sep=' ';
2092  HDF5CFUtil::Split_helper(coord_values,orig_attr_value,sep);
2093  bool has_lat_flag = false;
2094  bool has_lon_flag = false;
2095  for (auto itcv=coord_values.begin();itcv!=coord_values.end();++itcv) {
2096  if((*ivs).name1 == (*itcv))
2097  has_lat_flag = true;
2098  else if((*ivs).name2 == (*itcv))
2099  has_lon_flag = true;
2100  }
2101  // Find both lat and lon, now check the dim. size
2102  if(true == has_lat_flag && true == has_lon_flag) {
2103  short has_same_ll_size = 0;
2104  for(auto ird = (*irv)->dims.begin();ird!=(*irv)->dims.end();++ird){
2105  if((*ird)->size == (*ivs).size1)
2106  has_same_ll_size++;
2107  }
2108  if(has_same_ll_size!=2){
2109  ret_value = false;
2110  break;
2111  }
2112  }
2113  }
2114  }// end of for ira
2115 
2116  if(false == ret_value)
2117  break;
2118  }// end of if irv
2119  }// end of for irv
2120 
2121  if(true == ret_value)
2122  break;
2123  }// else
2124  }// end of for ivs
2125  } // else of if num_1d_rank
2126  }// end of if of latloncv_candidate_pairs
2127 
2128  release_standalone_var_vector(tempvar_lat);
2129  release_standalone_var_vector(tempvar_lon);
2130 
2131  }
2132 #if 0
2133 if(true == ret_value)
2134 cerr<<"This product is the coordinate type "<<endl;
2135 #endif
2136  // Don't forget to set the flag for this general product pattern.
2137  if(true == ret_value)
2138  this->gproduct_pattern = GENERAL_LATLON_COOR_ATTR;
2139 
2140  return ret_value;
2141 }
2142 
2143 // Build 1-D latlon coordinate variables candidate for GENERAL_LATLON_COOR_ATTR.
2144 void GMFile::Build_lat1D_latlon_candidate(const Var *lat,const vector<Var*> &lon_vec) {
2145 
2146  BESDEBUG("h5", "Coming to Build_lat1D_latlon_candidate()"<<endl);
2147  set<string> lon_candidate_path;
2148  vector< pair<string,hsize_t> > lon_path_size_vec;
2149 
2150  // Obtain the path and the size info. from all the potential qualified longitude candidate.
2151  for(auto irlon = lon_vec.begin(); irlon!=lon_vec.end();++irlon) {
2152 
2153  if (lat->rank == (*irlon)->rank) {
2154  pair<string,hsize_t>lon_path_size;
2155  lon_path_size.first = (*irlon)->fullpath;
2156  lon_path_size.second = (*irlon)->getDimensions()[0]->size;
2157  lon_path_size_vec.push_back(lon_path_size);
2158  }
2159  }
2160 
2161  // If there is only one potential qualified longitude for this latitude, just save this pair.
2162  if(lon_path_size_vec.size() == 1) {
2163 
2164  Name_Size_2Pairs latlon_pair;
2165  latlon_pair.name1 = lat->fullpath;
2166  latlon_pair.name2 = lon_path_size_vec[0].first;
2167  latlon_pair.size1 = lat->getDimensions()[0]->size;
2168  latlon_pair.size2 = lon_path_size_vec[0].second;
2169  latlon_pair.rank = lat->rank;
2170  latloncv_candidate_pairs.push_back(latlon_pair);
2171 
2172  }
2173  else if(lon_path_size_vec.size() >1) {
2174 
2175  // For more than one potential qualified longitude, we can still find a qualified one
2176  // if we find there is only one longitude under the same group of this latitude.
2177  string lat_path = HDF5CFUtil::obtain_string_before_lastslash(lat->fullpath);
2178  pair<string,hsize_t> lon_final_path_size;
2179  short num_lon_match = 0;
2180  for (auto islon =lon_path_size_vec.begin();islon!=lon_path_size_vec.end();++islon) {
2181  // Search the longitude path and see if it matches with the latitude.
2182  if(HDF5CFUtil::obtain_string_before_lastslash((*islon).first)==lat_path) {
2183  num_lon_match++;
2184  if(1 == num_lon_match)
2185  lon_final_path_size = *islon;
2186  else if(num_lon_match > 1)
2187  break;
2188  }
2189  }
2190  if(num_lon_match ==1) {// insert this lat/lon pair to the struct
2191  Name_Size_2Pairs latlon_pair;
2192  latlon_pair.name1 = lat->fullpath;
2193  latlon_pair.name2 = lon_final_path_size.first;
2194  latlon_pair.size1 = lat->getDimensions()[0]->size;
2195  latlon_pair.size2 = lon_final_path_size.second;
2196  latlon_pair.rank = lat->rank;
2197  latloncv_candidate_pairs.push_back(latlon_pair);
2198  }
2199  }
2200 
2201 }
2202 
2203 // Build >1D latlon coordinate variables candidate for GENERAL_LATLON_COOR_ATTR.
2204 void GMFile::Build_latg1D_latlon_candidate(Var *lat,const vector<Var*> & lon_vec) {
2205 
2206  BESDEBUG("h5", "Coming to Build_latg1D_latlon_candidate()"<<endl);
2207  set<string> lon_candidate_path;
2208 
2209  // We will check if the longitude shares the same dimensions of the latitude
2210  for(auto irlon = lon_vec.begin(); irlon!=lon_vec.end();++irlon) {
2211 
2212  if (lat->rank == (*irlon)->rank) {
2213 
2214  // Check the dim order and size.
2215  bool same_dim = true;
2216  for(int dim_index = 0; dim_index <lat->rank; dim_index++) {
2217  if(lat->getDimensions()[dim_index]->size !=
2218  (*irlon)->getDimensions()[dim_index]->size){
2219  same_dim = false;
2220  break;
2221  }
2222  }
2223  if(true == same_dim)
2224  lon_candidate_path.insert((*irlon)->fullpath);
2225  }
2226  }
2227 
2228  // Check the size of the lon., if the size is not 1, see if we can find the pair under the same group.
2229  if(lon_candidate_path.size() > 1) {
2230 
2231  string lat_path = HDF5CFUtil::obtain_string_before_lastslash(lat->fullpath);
2232  vector <string> lon_final_candidate_path_vec;
2233  for(auto islon_path =lon_candidate_path.begin();islon_path!=lon_candidate_path.end();++islon_path) {
2234 
2235  // Search the path.
2236  if(HDF5CFUtil::obtain_string_before_lastslash(*islon_path)==lat_path)
2237  lon_final_candidate_path_vec.push_back(*islon_path);
2238  }
2239 
2240  if(lon_final_candidate_path_vec.size() == 1) {// insert this lat/lon pair to the struct
2241 
2242  Name_Size_2Pairs latlon_pair;
2243 
2244  latlon_pair.name1 = lat->fullpath;
2245  latlon_pair.name2 = lon_final_candidate_path_vec[0];
2246  latlon_pair.size1 = lat->getDimensions()[0]->size;
2247  latlon_pair.size2 = lat->getDimensions()[1]->size;
2248  latlon_pair.rank = lat->rank;
2249  latloncv_candidate_pairs.push_back(latlon_pair);
2250  }
2251  else if(lon_final_candidate_path_vec.size() >1) {
2252 
2253  // Under the same group, if we have two pairs lat/lon such as foo1_lat,foo1_lon, foo2_lat,foo2_lon, we will
2254  // treat {foo1_lat,foo1_lon} and {foo2_lat,foo2_lon} as two lat,lon coordinate candidates. This is essentially the SMAP L1B case.
2255  // We only compare three potential suffixes, lat/lon, latitude/longitude,Latitude/Longitude. We will treat the pair
2256  // latitude/Longitude and Latitude/longitude as a valid one.
2257 
2258  string lat_name = HDF5CFUtil::obtain_string_after_lastslash(lat->fullpath);
2259  string lat_name_prefix1;
2260  string lat_name_prefix2;
2261 
2262  // name prefix before the pair lat,note: no need to check if the last 3 characters are lat or lon. We've checked already.
2263  if(lat_name.size() >3) {
2264  lat_name_prefix1 = lat_name.substr(0,lat_name.size()-3);
2265  if(lat_name.size() >8)
2266  lat_name_prefix2 = lat_name.substr(0,lat_name.size()-8);
2267  }
2268  string lon_name_prefix1;
2269  string lon_name_prefix2;
2270 
2271  for(auto ilon = lon_final_candidate_path_vec.begin(); ilon!=lon_final_candidate_path_vec.end();++ilon) {
2272  string lon_name = HDF5CFUtil::obtain_string_after_lastslash(*ilon);
2273  if(lon_name.size() >3) {
2274  lon_name_prefix1 = lon_name.substr(0,lon_name.size()-3);
2275  if(lon_name.size() >9)
2276  lon_name_prefix2 = lon_name.substr(0,lon_name.size()-9);
2277  }
2278  if((lat_name_prefix1 !="" && lat_name_prefix1 == lon_name_prefix1) ||
2279  (lat_name_prefix2 !="" && lat_name_prefix2 == lon_name_prefix2)) {// match lat,lon this one is the candidate
2280 
2281  Name_Size_2Pairs latlon_pair;
2282  latlon_pair.name1 = lat->fullpath;
2283  latlon_pair.name2 = *ilon;
2284  latlon_pair.size1 = lat->getDimensions()[0]->size;
2285  latlon_pair.size2 = lat->getDimensions()[1]->size;
2286  latlon_pair.rank = lat->rank;
2287  latloncv_candidate_pairs.push_back(latlon_pair);
2288 
2289  }
2290  }
2291  }// end else of if lon_final_candidate_path_vec
2292  }// end of if(lon_candidate_path.size() > 1)
2293 
2294  else if(lon_candidate_path.size() == 1) {//insert this lat/lon pair to the struct
2295 
2296  Name_Size_2Pairs latlon_pair;
2297 
2298  latlon_pair.name1 = lat->fullpath;
2299  latlon_pair.name2 = *(lon_candidate_path.begin());
2300  latlon_pair.size1 = lat->getDimensions()[0]->size;
2301  latlon_pair.size2 = lat->getDimensions()[1]->size;
2302  latlon_pair.rank = lat->rank;
2303  latloncv_candidate_pairs.push_back(latlon_pair);
2304 
2305  }
2306 
2307 }
2308 
2309 // We need to make sure that one lat maps to one lon in the lat/lon pairs.
2310 // This routine removes the duplicate ones like (lat1,lon1) and (lat2,lon1).
2311 void GMFile::Build_unique_latlon_candidate() {
2312 
2313  BESDEBUG("h5", "Coming to Build_unique_latlon_candidate()"<<endl);
2314  set<int> duplicate_index;
2315  for(unsigned int i= 0; i<latloncv_candidate_pairs.size();i++) {
2316  for(unsigned int j=i+1;j<latloncv_candidate_pairs.size();j++) {
2317  if(latloncv_candidate_pairs[i].name2 == latloncv_candidate_pairs[j].name2) {
2318  duplicate_index.insert(i);
2319  duplicate_index.insert(j);
2320  }
2321  }
2322  }
2323 
2324  // set is pre-sorted. we used a quick way to remove multiple elements.
2325  for(auto its= duplicate_index.rbegin();its!=duplicate_index.rend();++its) {
2326  latloncv_candidate_pairs[*its] = latloncv_candidate_pairs.back();
2327  latloncv_candidate_pairs.pop_back();
2328  }
2329 }
2330 // Leave the following code for the time being.
2331 #if 0
2332 // In this version, we only check if we have "latitude,longitude","Latitude,Longitude","lat,lon" names.
2333 // This routine will check this case.
2334 bool GMFile::Check_LatLonName_General_Product(int ll_rank) {
2335 
2336  if(ll_rank <1 || ll_rank >2)
2337  throw2("Only support rank = 1 or 2 lat/lon case for the general product. The current rank is ",ll_rank);
2338  bool ret_value = false;
2339  size_t lat2D_dimsize0 = 0;
2340  size_t lat2D_dimsize1 = 0;
2341  size_t lon2D_dimsize0 = 0;
2342  size_t lon2D_dimsize1 = 0;
2343 
2344  // The element order is latlon_flag,latilong_flag and LatLon_flag.
2345  vector<short>ll_flag(3,0);
2346 
2347  vector<size_t>lat_size;
2348  vector<size_t>lon_size;
2349 
2350  // We only need to check 2-D latlon
2351  if(2 == ll_rank) {
2352  //lat/lon is 2-D array, so the size is doubled.
2353  lat_size.assign(6,0);
2354  lon_size.assign(6,0);
2355  }
2356 
2357  for (vector<Var *>::iterator irv = this->vars.begin();
2358  irv != this->vars.end(); ++irv) {
2359 
2360  if((*irv)->rank == ll_rank) {
2361  if((*irv)->name == "lat") {
2362  ll_flag[0]++;
2363  if(ll_rank == 2) {
2364  lat_size[0] = (*irv)->getDimensions()[0]->size;
2365  lat_size[1] = (*irv)->getDimensions()[1]->size;
2366 
2367  }
2368 
2369  }
2370  else if((*irv)->name == "lon") {
2371  ll_flag[0]++;
2372  if(ll_rank == 2) {
2373  lon_size[0] = (*irv)->getDimensions()[0]->size;
2374  lon_size[1] = (*irv)->getDimensions()[1]->size;
2375 
2376  }
2377 
2378  }
2379  else if((*irv)->name == "latitude"){
2380  ll_flag[1]++;
2381  if(ll_rank == 2) {
2382  lat_size[2] = (*irv)->getDimensions()[0]->size;
2383  lat_size[3] = (*irv)->getDimensions()[1]->size;
2384 
2385  }
2386  }
2387  else if((*irv)->name == "longitude"){
2388  ll_flag[1]++;
2389  if(ll_rank == 2) {
2390  lon_size[2] = (*irv)->getDimensions()[0]->size;
2391  lon_size[3] = (*irv)->getDimensions()[1]->size;
2392 
2393  }
2394 
2395  }
2396  else if((*irv)->name == "Latitude"){
2397  ll_flag[2]++;
2398  if(ll_rank == 2) {
2399  lat_size[4] = (*irv)->getDimensions()[0]->size;
2400  lat_size[5] = (*irv)->getDimensions()[1]->size;
2401 
2402  }
2403 
2404  }
2405  else if((*irv)->name == "Longitude"){
2406  ll_flag[2]++;
2407  if(ll_rank == 2) {
2408  lon_size[4] = (*irv)->getDimensions()[0]->size;
2409  lon_size[5] = (*irv)->getDimensions()[1]->size;
2410  }
2411  }
2412  }
2413  }
2414 
2415  int total_llflag = 0;
2416  for (int i = 0; i < ll_flag.size();i++)
2417  if(2 == ll_flag[i])
2418  total_llflag ++;
2419 
2420  // We only support 1 (L)lat(i)/(L)lon(g) pair.
2421  if(1 == total_llflag) {
2422  bool latlon_size_match = true;
2423  if(2 == ll_rank) {
2424  for (int size_index = 0; size_index <lat_size.size();size_index++) {
2425  if(lat_size[size_index] != lon_size[size_index]){
2426  latlon_size_match = false;
2427  break;
2428  }
2429  }
2430  }
2431 
2432  if(true == latlon_size_match) {
2433  ret_value = true;
2434  if(2 == ll_flag[0]) {
2435  gp_latname = "lat";
2436  gp_lonname = "lon";
2437  }
2438  else if ( 2 == ll_flag[1]) {
2439  gp_latname = "latitude";
2440  gp_lonname = "longitude";
2441  }
2442 
2443  else if (2 == ll_flag[2]){
2444  gp_latname = "Latitude";
2445  gp_lonname = "Longitude";
2446  }
2447  }
2448  }
2449 
2450  return ret_value;
2451 }
2452 #endif
2453 
2454 // Add dimension names for the case that has 2-D lat/lon.
2455 void GMFile::Add_Dim_Name_LatLon2D_General_Product() {
2456 
2457  BESDEBUG("h5", "Coming to Add_Dim_Name_LatLon2D_General_Product()"<<endl);
2458  string latdimname0;
2459  string latdimname1;
2460  size_t latdimsize0 = 0;
2461  size_t latdimsize1 = 0;
2462 
2463  // Need to generate fake dimensions.
2464  for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
2465 
2466  set<hsize_t> fakedimsize;
2467  pair<set<hsize_t>::iterator,bool> setsizeret;
2468  int num_dup_dim_size = 0;
2469  for (auto ird= (*irv)->dims.begin(); ird != (*irv)->dims.end(); ++ird) {
2470  Add_One_FakeDim_Name(*ird);
2471  setsizeret = fakedimsize.insert((*ird)->size);
2472 
2473  // Avoid the same size dimension sharing the same dimension name.
2474  if (false == setsizeret.second){
2475  num_dup_dim_size++;
2476  Adjust_Duplicate_FakeDim_Name2(*ird,num_dup_dim_size);
2477  }
2478 
2479 #if 0
2480  if (false == setsizeret.second)
2481  Adjust_Duplicate_FakeDim_Name(*ird);
2482 #endif
2483  }
2484 
2485  // Find variable name that is latitude or lat or Latitude
2486  // Note that we don't need to check longitude since longitude dim. sizes should be the same as the latitude for this case.
2487  if((*irv)->name == gp_latname) {
2488  if((*irv)->rank != 2) {
2489  throw4("coordinate variables ",gp_latname,
2490  " must have rank 2 for the 2-D latlon case , the current rank is ",
2491  (*irv)->rank);
2492  }
2493  latdimname0 = (*irv)->getDimensions()[0]->name;
2494  latdimsize0 = (*irv)->getDimensions()[0]->size;
2495 
2496  latdimname1 = (*irv)->getDimensions()[1]->name;
2497  latdimsize1 = (*irv)->getDimensions()[1]->size;
2498  }
2499  }
2500 
2501 
2502  // Now we need to change a dimension of a general variable that shares the same size of lat
2503  // to the dimension name of the lat.
2504  for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
2505  int lat_dim0_index = 0;
2506  int lat_dim1_index = 0;
2507  bool has_lat_dims_size = false;
2508 
2509  for (unsigned int dim_index = 0; dim_index <(*irv)->dims.size(); dim_index++) {
2510 
2511  // Find if having the first dimension size of lat
2512  if(((*irv)->dims[dim_index])->size == latdimsize0) {
2513 
2514  // Find if having the second dimension size of lat
2515  lat_dim0_index = dim_index;
2516  for(unsigned int dim_index2 = dim_index+1;dim_index2 < (*irv)->dims.size();dim_index2++) {
2517  if(((*irv)->dims[dim_index2])->size == latdimsize1) {
2518  lat_dim1_index = dim_index2;
2519  has_lat_dims_size = true;
2520  break;
2521  }
2522  }
2523  }
2524  if(true == has_lat_dims_size)
2525  break;
2526  }
2527  // Find the lat's dimension sizes, change the (fake) dimension names.
2528  if(true == has_lat_dims_size) {
2529 
2530  ((*irv)->dims[lat_dim0_index])->name = latdimname0;
2531 #if 0
2532  //((*irv)->dims[lat_dim0_index])->newname = latdimname0;
2533 #endif
2534  ((*irv)->dims[lat_dim1_index])->name = latdimname1;
2535 #if 0
2536  //((*irv)->dims[lat_dim1_index])->newname = latdimname1;
2537 #endif
2538 
2539  }
2540  }
2541 
2542  //When we generate Fake dimensions, we may encounter discontiguous Fake dimension names such
2543  // as FakeDim0, FakeDim9 etc. We would like to make Fake dimension names in contiguous order
2544  // FakeDim0,FakeDim1,etc.
2545 
2546  // Obtain the tempdimnamelist set.
2547  set<string>tempdimnamelist;
2548 
2549  for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
2550  for (auto ird= (*irv)->dims.begin(); ird != (*irv)->dims.end(); ++ird)
2551  tempdimnamelist.insert((*ird)->name);
2552  }
2553 
2554  // Generate the final dimnamelist,it is a contiguous order: FakeDim0,FakeDim1 etc.
2555  set<string>finaldimnamelist;
2556  string finaldimname_base = "FakeDim";
2557 
2558  for(unsigned int i = 0; i<tempdimnamelist.size();i++) {
2559  stringstream sfakedimindex;
2560  sfakedimindex << i;
2561  string finaldimname = finaldimname_base + sfakedimindex.str();
2562  finaldimnamelist.insert(finaldimname);
2563  }
2564 
2565  // If the original tempdimnamelist is not the same as the finaldimnamelist,
2566  // we need to generate a map from original name to the final name.
2567  if(finaldimnamelist != tempdimnamelist) {
2568  map<string,string> tempdimname_to_finaldimname;
2569  auto tempit = tempdimnamelist.begin();
2570  auto finalit = finaldimnamelist.begin();
2571  while(tempit != tempdimnamelist.end()) {
2572  tempdimname_to_finaldimname[*tempit] = *finalit;
2573  tempit++;
2574  finalit++;
2575  }
2576 
2577  // Change the dimension names of every variable to the final dimension name list.
2578  for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
2579  for (auto ird= (*irv)->dims.begin(); ird != (*irv)->dims.end(); ++ird) {
2580  if(tempdimname_to_finaldimname.find((*ird)->name) !=tempdimname_to_finaldimname.end())
2581  (*ird)->name = tempdimname_to_finaldimname[(*ird)->name];
2582  else
2583  throw3("The dimension names ",(*ird)->name, "cannot be found in the dim. name list.");
2584  }
2585  }
2586  }
2587 
2588 
2589  dimnamelist.clear();
2590  dimnamelist = finaldimnamelist;
2591 
2592  // We need to update dimname_to_dimsize map. This may be used in the future.
2593  dimname_to_dimsize.clear();
2594  for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
2595  for (auto ird= (*irv)->dims.begin(); ird != (*irv)->dims.end(); ++ird) {
2596  if(finaldimnamelist.find((*ird)->name)!=finaldimnamelist.end()) {
2597  dimname_to_dimsize[(*ird)->name] = (*ird)->size;
2598  dimname_to_unlimited[(*ird)->name] = (*ird)->unlimited_dim;
2599  finaldimnamelist.erase((*ird)->name);
2600  }
2601 
2602  }
2603  if(true == finaldimnamelist.empty())
2604  break;
2605  }
2606 
2607  // Finally set dimension newname
2608  for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
2609  for (auto ird= (*irv)->dims.begin(); ird != (*irv)->dims.end(); ++ird)
2610  (*ird)->newname = (*ird)->name;
2611  }
2612 
2613 }
2614 
2615 // Add dimension names for the case that has 1-D lat/lon or CoordAttr..
2616 //
2617 void GMFile::Add_Dim_Name_LatLon1D_Or_CoordAttr_General_Product() {
2618 
2619  BESDEBUG("h5", "Coming to Add_Dim_Name_LatLon1D_Or_CoordAttr_General_Product()"<<endl);
2620  // Only need to add the fake dimension names
2621  for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
2622  set<hsize_t> fakedimsize;
2623  pair<set<hsize_t>::iterator,bool> setsizeret;
2624  int num_dup_dim_size = 0;
2625  for (auto ird= (*irv)->dims.begin(); ird != (*irv)->dims.end(); ++ird) {
2626  Add_One_FakeDim_Name(*ird);
2627  setsizeret = fakedimsize.insert((*ird)->size);
2628  // Avoid the same size dimension sharing the same dimension name.
2629  // HYRAX-629, we not only want to have the unique dimension name for
2630  // the same dimension size of different dimensions in a var, we also
2631  // want to reduce the number of dimension names.
2632  // So foo[100][100], foo2[100][100][100] should be something like
2633  // foo[FakeDim0][FakeDim1] foo2[FakeDim0][FakeDim1][FakeDim2]
2634  if (false == setsizeret.second){
2635  num_dup_dim_size++;
2636  Adjust_Duplicate_FakeDim_Name2(*ird,num_dup_dim_size);
2637  }
2638  // Comment out the original code for the time being.
2639 #if 0
2640  if (false == setsizeret.second){
2641  num_dup_dim_size++;
2642  Adjust_Duplicate_FakeDim_Name(*ird,num_dup_dim_size);
2643  }
2644 #endif
2645  }
2646  }
2647 
2648 // leave it here for debugging purpose
2649 #if 0
2650 for (vector<Var *>::iterator irv = this->vars.begin();
2651  irv != this->vars.end(); ++irv) {
2652 cerr<<"Var name is "<<(*irv)->newname<<endl;
2653  for (vector<Dimension *>::iterator ird= (*irv)->dims.begin();
2654  ird != (*irv)->dims.end(); ++ird)
2655 cerr<<"Dimension name is "<<(*ird)->newname <<endl;
2656 }
2657 #endif
2658 
2659 }
2660 
2661 // For netCDF-4-like HDF5 products, we need to add the dimension scales.
2662 void GMFile::Add_Dim_Name_Dimscale_General_Product() {
2663 
2664  BESDEBUG("h5", "Coming to Add_Dim_Name_Dimscale_General_Product()"<<endl);
2665  //cerr<<"coming to Add_Dim_Name_Dimscale_General_Product"<<endl;
2666  pair<set<string>::iterator,bool> setret;
2667  this->iscoard = true;
2668 
2669  for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
2670 
2671  // Obtain all the dimension names for this variable
2672  Handle_UseDimscale_Var_Dim_Names_General_Product((*irv));
2673 
2674  // Need to update dimenamelist and dimname_to_dimsize and dimname_to_unlimited maps for future use.
2675  for (auto ird = (*irv)->dims.begin(); ird !=(*irv)->dims.end();++ird) {
2676  setret = dimnamelist.insert((*ird)->name);
2677  if (true == setret.second)
2678  Insert_One_NameSizeMap_Element((*ird)->name,(*ird)->size,(*ird)->unlimited_dim);
2679  }
2680  } // end of for
2681 
2682  if (true == dimnamelist.empty())
2683  throw1("This product should have the dimension names, but no dimension names are found");
2684 
2685 }
2686 
2687 // Obtain dimension names for this variable when netCDF-4 model(using dimension scales) is followed.
2688 void GMFile::Handle_UseDimscale_Var_Dim_Names_General_Product(Var *var) {
2689 
2690  BESDEBUG("h5", "Coming to Handle_UseDimscale_Var_Dim_Names_General_Product()"<<endl);
2691  Attribute* dimlistattr = nullptr;
2692  bool has_dimlist = false;
2693  bool has_dimclass = false;
2694 
2695  for(auto ira = var->attrs.begin(); ira != var->attrs.end();ira++) {
2696  if ("DIMENSION_LIST" == (*ira)->name) {
2697  dimlistattr = *ira;
2698  has_dimlist = true;
2699  }
2700  if ("CLASS" == (*ira)->name) {
2701 
2702  Retrieve_H5_Attr_Value(*ira,var->fullpath);
2703  string class_value;
2704  class_value.resize((*ira)->value.size());
2705  copy((*ira)->value.begin(),(*ira)->value.end(),class_value.begin());
2706 
2707  // Compare the attribute "CLASS" value with "DIMENSION_SCALE". We only compare the string with the size of
2708  // "DIMENSION_SCALE", which is 15.
2709  if (0 == class_value.compare(0,15,"DIMENSION_SCALE")) {
2710  has_dimclass = true;
2711  break;
2712  }
2713  }
2714 
2715  } // end of for
2716 
2717  // This is a general variable, we need to find the corresponding coordinate variables.
2718  if (true == has_dimlist)
2719  Add_UseDimscale_Var_Dim_Names_General_Product(var,dimlistattr);
2720 
2721  // Dim name is the same as the variable name for dimscale variable
2722  else if(true == has_dimclass) {
2723  if (var->dims.size() !=1)
2724  throw2("Currently dimension scale dataset must be 1 dimension, this is not true for the dataset ",
2725  var->name);
2726 
2727  // The var name is the object name, however, we would like the dimension name to be the full path.
2728  // so that the dim name can be served as the key for future handling.
2729  (var->dims)[0]->name = var->fullpath;
2730  (var->dims)[0]->newname = var->fullpath;
2731  pair<set<string>::iterator,bool> setret;
2732  setret = dimnamelist.insert((var->dims)[0]->name);
2733  if (true == setret.second)
2734  Insert_One_NameSizeMap_Element((var->dims)[0]->name,(var->dims)[0]->size,(var->dims)[0]->unlimited_dim);
2735  }
2736 
2737  // No dimension, add fake dim names, this will rarely happen.
2738  else {
2739 
2740  set<hsize_t> fakedimsize;
2741  pair<set<hsize_t>::iterator,bool> setsizeret;
2742  for (vector<Dimension *>::iterator ird= var->dims.begin();
2743  ird != var->dims.end(); ++ird) {
2744  Add_One_FakeDim_Name(*ird);
2745  setsizeret = fakedimsize.insert((*ird)->size);
2746  // Avoid the same size dimension sharing the same dimension name.
2747  if (false == setsizeret.second)
2748  Adjust_Duplicate_FakeDim_Name(*ird);
2749  }
2750  }
2751 
2752 }
2753 
2754 // Add dimension names for the case when HDF5 dimension scale is followed(netCDF4-like)
2755 void GMFile::Add_UseDimscale_Var_Dim_Names_General_Product(Var *var,Attribute*dimlistattr)
2756 {
2757 
2758  BESDEBUG("h5", "Coming to Add_UseDimscale_Var_Dim_Names_General_Product()"<<endl);
2759  ssize_t objnamelen = -1;
2760  hobj_ref_t rbuf;
2761  //hvl_t *vlbuf = nullptr;
2762  vector<hvl_t> vlbuf;
2763 
2764  hid_t dset_id = -1;
2765  hid_t attr_id = -1;
2766  hid_t atype_id = -1;
2767  hid_t amemtype_id = -1;
2768  hid_t aspace_id = -1;
2769  hid_t ref_dset = -1;
2770 
2771  if(nullptr == dimlistattr)
2772  throw2("Cannot obtain the dimension list attribute for variable ",var->name);
2773 
2774  else if (0==var->rank)
2775  throw2("The number of dimension should NOT be 0 for the variable ",var->name);
2776 
2777  else {
2778  try {
2779 
2780  vlbuf.resize(var->rank);
2781 
2782  dset_id = H5Dopen(this->fileid,(var->fullpath).c_str(),H5P_DEFAULT);
2783  if (dset_id < 0)
2784  throw2("Cannot open the dataset ",var->fullpath);
2785 
2786  attr_id = H5Aopen(dset_id,(dimlistattr->name).c_str(),H5P_DEFAULT);
2787  if (attr_id <0 )
2788  throw4("Cannot open the attribute ",dimlistattr->name," of HDF5 dataset ",var->fullpath);
2789 
2790  atype_id = H5Aget_type(attr_id);
2791  if (atype_id <0)
2792  throw4("Cannot obtain the datatype of the attribute ",dimlistattr->name," of HDF5 dataset ",var->fullpath);
2793 
2794  amemtype_id = H5Tget_native_type(atype_id, H5T_DIR_ASCEND);
2795 
2796  if (amemtype_id < 0)
2797  throw2("Cannot obtain the memory datatype for the attribute ",dimlistattr->name);
2798 
2799 
2800  if (H5Aread(attr_id,amemtype_id,vlbuf.data()) <0)
2801  throw2("Cannot obtain the referenced object for the variable ",var->name);
2802 
2803 
2804  vector<char> objname;
2805  int vlbuf_index = 0;
2806 
2807  // The dimension names of variables will be the HDF5 dataset names dereferenced from the DIMENSION_LIST attribute.
2808  for (auto ird = var->dims.begin(); ird != var->dims.end(); ++ird) {
2809 
2810  if(vlbuf[vlbuf_index].p== nullptr)
2811  throw4("The dimension doesn't exist. Var name is ",var->name,"; the dimension index is ",vlbuf_index);
2812  rbuf =((hobj_ref_t*)vlbuf[vlbuf_index].p)[0];
2813  if ((ref_dset = H5RDEREFERENCE(attr_id, H5R_OBJECT, &rbuf)) < 0)
2814  throw2("Cannot dereference from the DIMENSION_LIST attribute for the variable ",var->name);
2815 
2816  if ((objnamelen= H5Iget_name(ref_dset,nullptr,0))<=0)
2817  throw2("Cannot obtain the dataset name dereferenced from the DIMENSION_LIST attribute for the variable ",var->name);
2818  objname.resize(objnamelen+1);
2819  if ((objnamelen= H5Iget_name(ref_dset,objname.data(),objnamelen+1))<=0)
2820  throw2("Cannot obtain the dataset name dereferenced from the DIMENSION_LIST attribute for the variable ",var->name);
2821 
2822  auto objname_str = string(objname.begin(),objname.end());
2823 
2824  // We need to remove the first character of the object name since the first character
2825  // of the object full path is always "/" and this will be changed to "_".
2826  // The convention of handling the dimension-scale general product is to remove the first "_".
2827  // Check the get_CF_string function of HDF5GMCF.cc.
2828  string trim_objname = objname_str.substr(0,objnamelen);
2829  (*ird)->name = string(trim_objname.begin(),trim_objname.end());
2830 
2831  pair<set<string>::iterator,bool> setret;
2832  setret = dimnamelist.insert((*ird)->name);
2833  if (true == setret.second)
2834  Insert_One_NameSizeMap_Element((*ird)->name,(*ird)->size,(*ird)->unlimited_dim);
2835  (*ird)->newname = (*ird)->name;
2836  H5Dclose(ref_dset);
2837 #if 0
2838  ref_dset = -1;
2839 #endif
2840  objname.clear();
2841  vlbuf_index++;
2842  }// end of for ird
2843 
2844  if(vlbuf.empty() == false) {
2845 
2846  if ((aspace_id = H5Aget_space(attr_id)) < 0)
2847  throw2("Cannot get hdf5 dataspace id for the attribute ",dimlistattr->name);
2848 
2849  if (H5Dvlen_reclaim(amemtype_id,aspace_id,H5P_DEFAULT,(void*)vlbuf.data())<0)
2850  throw2("Cannot successfully clean up the variable length memory for the variable ",var->name);
2851 
2852  H5Sclose(aspace_id);
2853 
2854  }
2855 
2856  H5Tclose(atype_id);
2857  H5Tclose(amemtype_id);
2858  H5Aclose(attr_id);
2859  H5Dclose(dset_id);
2860  }
2861 
2862  catch(...) {
2863 
2864  if(atype_id != -1)
2865  H5Tclose(atype_id);
2866 
2867  if(amemtype_id != -1)
2868  H5Tclose(amemtype_id);
2869 
2870  if(aspace_id != -1)
2871  H5Sclose(aspace_id);
2872 
2873  if(attr_id != -1)
2874  H5Aclose(attr_id);
2875 
2876  if(dset_id != -1)
2877  H5Dclose(dset_id);
2878 
2879  throw;
2880  }
2881  }
2882 
2883 }
2884 
2885 // Handle coordinate variables
2887 
2888  BESDEBUG("h5", "GMFile:: Coming to Handle_CVar()"<<endl);
2889  // No coordinate variables are generated for ACOS_L2S or OCO2_L1B
2890  // Currently we support the three patterns for the general products:
2891  // 1) Dimensions follow HDF5 dimension scale specification
2892  // 2) Dimensions don't follow HDF5 dimension scale specification but have 1D lat/lon
2893  // 3) Dimensions don't follow HDF5 dimension scale specification bu have 2D lat/lon
2894  if (General_Product == this->product_type ||
2895  ACOS_L2S_OR_OCO2_L1B == this->product_type) {
2896  if (GENERAL_DIMSCALE == this->gproduct_pattern)
2897  Handle_CVar_Dimscale_General_Product();
2898  else if (GENERAL_LATLON1D == this->gproduct_pattern)
2899  Handle_CVar_LatLon1D_General_Product();
2900  else if (GENERAL_LATLON2D == this->gproduct_pattern)
2901  Handle_CVar_LatLon2D_General_Product();
2902  return;
2903  }
2904 
2905  else if (Mea_SeaWiFS_L2 == this->product_type ||
2906  Mea_SeaWiFS_L3 == this->product_type)
2907  Handle_CVar_Mea_SeaWiFS();
2908  else if (Aqu_L3 == this->product_type)
2909  Handle_CVar_Aqu_L3();
2910  else if (OBPG_L3 == this->product_type)
2911  Handle_CVar_OBPG_L3();
2912  else if (OSMAPL2S == this->product_type)
2913  Handle_CVar_OSMAPL2S();
2914  else if (Mea_Ozone == this->product_type)
2915  Handle_CVar_Mea_Ozone();
2916  else if (GPMS_L3 == this->product_type || GPMM_L3 == this->product_type
2917  || GPM_L3_New == this->product_type )
2918  Handle_CVar_GPM_L3();
2919  else if (GPM_L1 == this->product_type)
2920  Handle_CVar_GPM_L1();
2921 }
2922 
2923 // Handle GPM level 1 coordinate variables
2924 void GMFile::Handle_CVar_GPM_L1() {
2925 
2926  BESDEBUG("h5", "Coming to Handle_CVar_GPM_L1()"<<endl);
2927 #if 0
2928  // Loop through the variable list to build the coordinates.
2929  for (vector<Var *>::iterator irv = this->vars.begin();
2930  irv != this->vars.end(); ++irv) {
2931  if((*irv)->name=="AlgorithmRuntimeInfo") {
2932  delete(*irv);
2933  this->vars.erase(irv);
2934  break;
2935  }
2936  }
2937 #endif
2938 
2939  // Loop through all variables to check 2-D "Latitude" and "Longitude".
2940  // Create coordinate variables based on 2-D "Latitude" and "Longitude".
2941  // Latitude[Xdim][YDim] Longitude[Xdim][YDim], Latitude <->Xdim, Longitude <->YDim.
2942  // Make sure to build cf dimension names cfdimname = latpath+ the lat dimension name.
2943  // We want to save dimension names of Latitude and Longitude since
2944  // the fake coordinate variables of these two dimensions should not be generated.
2945  // So we need to remember these dimension names.
2946  set<string> ll_dim_set;
2947  for (auto irv = this->vars.begin(); irv != this->vars.end(); ) {
2948  if((*irv)->rank == 2 && (*irv)->name == "Latitude") {
2949  auto GMcvar = new GMCVar(*irv);
2950  size_t lat_pos = (*irv)->fullpath.rfind("Latitude");
2951  string lat_path = (*irv)->fullpath.substr(0,lat_pos);
2952  GMcvar->cfdimname = lat_path + ((*irv)->dims)[0]->name;
2953  ll_dim_set.insert(((*irv)->dims)[0]->name);
2954  GMcvar->cvartype = CV_EXIST;
2955  GMcvar->product_type = product_type;
2956  this->cvars.push_back(GMcvar);
2957  delete(*irv);
2958  irv = this->vars.erase(irv);
2959  }
2960 
2961  if((*irv)->rank == 2 && (*irv)->name == "Longitude") {
2962  auto GMcvar = new GMCVar(*irv);
2963  size_t lon_pos = (*irv)->fullpath.rfind("Longitude");
2964  string lon_path = (*irv)->fullpath.substr(0,lon_pos);
2965  GMcvar->cfdimname = lon_path + ((*irv)->dims)[1]->name;
2966  ll_dim_set.insert(((*irv)->dims)[1]->name);
2967  GMcvar->cvartype = CV_EXIST;
2968  GMcvar->product_type = product_type;
2969  this->cvars.push_back(GMcvar);
2970  delete(*irv);
2971  irv = this->vars.erase(irv);
2972  }
2973  else {
2974  ++irv;
2975  }
2976  }// end of for irv
2977 
2978 #if 0
2979  // Loop through all variables and create a dim set.
2980  set<string> cvdimset;
2981  pair<set<string>::iterator,bool> setret;
2982  for (vector<Var *>::iterator irv = this->vars.begin();
2983  irv != this->vars.end(); ++irv) {
2984  for(vector<Dimension *>::iterator ird = (*irv)->dims.begin();
2985  ird != (*irv)->dims.end(); ++ird) {
2986  setret = cvdimset.insert((*ird)->name);
2987 cerr<<"var name is "<<(*irv)->fullpath <<endl;
2988  if (true == setret.second) {
2989 cerr<<"dim name is "<<(*ird)->name <<endl;
2990  Insert_One_NameSizeMap_Element((*ird)->name,(*ird)->size);
2991  }
2992  }
2993  }// for (vector<Var *>::iterator irv = this->vars.begin();...
2994 #endif
2995 
2996  // For each dimension, create a coordinate variable.
2997  // Here we just need to loop through the map dimname_to_dimsize,
2998  // use the name and the size to create coordinate variables.
2999  for (map<string,hsize_t>::const_iterator itd = dimname_to_dimsize.begin();
3000  itd!=dimname_to_dimsize.end();++itd) {
3001  // We will not create fake coordinate variables for the
3002  // dimensions of latitude and longitude.
3003  if((ll_dim_set.find(itd->first)) == ll_dim_set.end()) {
3004  auto GMcvar = new GMCVar();
3005  Create_Missing_CV(GMcvar,itd->first);
3006  this->cvars.push_back(GMcvar);
3007  }
3008  }//end of for (map<string,hsize_t>.
3009 
3010 }
3011 
3012 // Handle coordinate variables for GPM level 3
3013 void GMFile::Handle_CVar_GPM_L3() {
3014 
3015  BESDEBUG("h5", "Coming to Handle_CVar_GPM_L3()"<<endl);
3016  iscoard = true;
3017 
3018  // Here we just need to loop through the map dimname_to_dimsize,
3019  // use the name and the size to create coordinate variables.
3020  for (map<string,hsize_t>::const_iterator itd = dimname_to_dimsize.begin();
3021  itd!=dimname_to_dimsize.end();++itd) {
3022 
3023  auto GMcvar = new GMCVar();
3024  if("nlon" == itd->first || "nlat" == itd->first
3025  || "lnH" == itd->first || "ltH" == itd->first
3026  || "lnL" == itd->first || "ltL" == itd->first) {
3027  GMcvar->name = itd->first;
3028  GMcvar->newname = GMcvar->name;
3029  GMcvar->fullpath = GMcvar->name;
3030  GMcvar->rank = 1;
3031  GMcvar->dtype = H5FLOAT32;
3032  auto gmcvar_dim = new Dimension(itd->second);
3033  gmcvar_dim->name = GMcvar->name;
3034  gmcvar_dim->newname = gmcvar_dim->name;
3035  GMcvar->dims.push_back(gmcvar_dim);
3036  GMcvar->cfdimname = gmcvar_dim->name;
3037  if ("nlat" ==GMcvar->name || "ltH" == GMcvar->name
3038  || "ltL" == GMcvar->name)
3039  GMcvar->cvartype = CV_LAT_MISS;
3040  else if ("nlon" == GMcvar->name || "lnH" == GMcvar->name
3041  || "lnL" == GMcvar->name)
3042  GMcvar->cvartype = CV_LON_MISS;
3043  GMcvar->product_type = product_type;
3044  }
3045  else if (("nlayer" == itd->first && (28 == itd->second || 19 == itd->second)) ||
3046  ("hgt" == itd->first && 5 == itd->second) ||
3047  ("nalt" == itd->first && 5 == itd->second)){
3048  GMcvar->name = itd->first;
3049  GMcvar->newname = GMcvar->name;
3050  GMcvar->fullpath = GMcvar->name;
3051  GMcvar->rank = 1;
3052  GMcvar->dtype = H5FLOAT32;
3053  auto gmcvar_dim = new Dimension(itd->second);
3054  gmcvar_dim->name = GMcvar->name;
3055  gmcvar_dim->newname = gmcvar_dim->name;
3056  GMcvar->dims.push_back(gmcvar_dim);
3057  GMcvar->cfdimname = gmcvar_dim->name;
3058  GMcvar->cvartype = CV_SPECIAL;
3059  GMcvar->product_type = product_type;
3060  }
3061  else
3062  Create_Missing_CV(GMcvar,itd->first);
3063  this->cvars.push_back(GMcvar);
3064  }//end of for
3065 }
3066 
3067 // Handle Coordinate variables for MeaSuRES SeaWiFS
3068 void GMFile::Handle_CVar_Mea_SeaWiFS() {
3069 
3070  BESDEBUG("h5", "Coming to Handle_CVar_Mea_SeaWiFS()"<<endl);
3071  pair<set<string>::iterator,bool> setret;
3072  set<string>tempdimnamelist = dimnamelist;
3073 
3074  for (auto irs = dimnamelist.begin(); irs != dimnamelist.end();++irs) {
3075  for (auto irv = this->vars.begin(); irv != this->vars.end(); ) {
3076  if ((*irs)== (*irv)->fullpath) {
3077 
3078  if (!iscoard && (("/natrack" == (*irs))
3079  || "/nxtrack" == (*irs))) {
3080  ++irv;
3081  continue;
3082  }
3083 
3084  if((*irv)->dims.size()!=1)
3085  throw3("Coard coordinate variable ",(*irv)->name, "is not 1D");
3086 
3087  // Create Coordinate variables.
3088  tempdimnamelist.erase(*irs);
3089  auto GMcvar = new GMCVar(*irv);
3090  GMcvar->cfdimname = *irs;
3091  GMcvar->cvartype = CV_EXIST;
3092  GMcvar->product_type = product_type;
3093  this->cvars.push_back(GMcvar);
3094  delete(*irv);
3095  irv = this->vars.erase(irv);
3096  } // end of if irs
3097  else if(false == iscoard) {
3098  // 2-D lat/lon, natrack maps to lat, nxtrack maps to lon.
3099  if ((((*irs) =="/natrack") && ((*irv)->fullpath == "/latitude"))
3100  ||(((*irs) =="/nxtrack") && ((*irv)->fullpath == "/longitude"))) {
3101  tempdimnamelist.erase(*irs);
3102  auto GMcvar = new GMCVar(*irv);
3103  GMcvar->cfdimname = *irs;
3104  GMcvar->cvartype = CV_EXIST;
3105  GMcvar->product_type = product_type;
3106  this->cvars.push_back(GMcvar);
3107  delete(*irv);
3108  irv = this->vars.erase(irv);
3109  }
3110  else {
3111  ++irv;
3112  }
3113 
3114  }// end of else if(false == iscoard)
3115  else {
3116  ++irv;
3117  }
3118  } // end of for irv ...
3119  } // end of for irs ...
3120 
3121  // Creating the missing "third-dimension" according to the dimension names.
3122  // This may never happen for the current MeaSure SeaWiFS, but put it here for code coherence and completeness.
3123  // KY 12-30-2011
3124  for (set<string>::iterator irs = tempdimnamelist.begin();
3125  irs != tempdimnamelist.end();++irs) {
3126  auto GMcvar = new GMCVar();
3127  Create_Missing_CV(GMcvar,*irs);
3128  this->cvars.push_back(GMcvar);
3129  }
3130 }
3131 
3132 // Handle Coordinate varibles for OSMAPL2S(Note: this function doesn't apply to SMAP)
3133 void GMFile::Handle_CVar_OSMAPL2S() {
3134 
3135  BESDEBUG("h5", "Coming to Handle_CVar_OSMAPL2S()"<<endl);
3136  pair<set<string>::iterator,bool> setret;
3137  set<string>tempdimnamelist = dimnamelist;
3138  string tempvarname;
3139  string key0 = "_lat";
3140  string key1 = "_lon";
3141  string osmapl2sdim0 ="YDim";
3142  string osmapl2sdim1 ="XDim";
3143 
3144  bool foundkey0 = false;
3145  bool foundkey1 = false;
3146 
3147  set<string> itset;
3148 
3149  for (auto irv = this->vars.begin(); irv != this->vars.end(); ) {
3150 
3151  tempvarname = (*irv)->name;
3152 
3153  if ((tempvarname.size() > key0.size())&&
3154  (key0 == tempvarname.substr(tempvarname.size()-key0.size(),key0.size()))){
3155 
3156  foundkey0 = true;
3157 
3158  if (dimnamelist.find(osmapl2sdim0)== dimnamelist.end())
3159  throw5("variable ",tempvarname," must have dimension ",osmapl2sdim0," , but not found ");
3160 
3161  tempdimnamelist.erase(osmapl2sdim0);
3162  auto GMcvar = new GMCVar(*irv);
3163  GMcvar->newname = GMcvar->name; // Remove the path, just use the variable name
3164  GMcvar->cfdimname = osmapl2sdim0;
3165  GMcvar->cvartype = CV_EXIST;
3166  GMcvar->product_type = product_type;
3167  this->cvars.push_back(GMcvar);
3168  delete(*irv);
3169  irv = this->vars.erase(irv);
3170  }// end of if
3171 
3172  else if ((tempvarname.size() > key1.size())&&
3173  (key1 == tempvarname.substr(tempvarname.size()-key1.size(),key1.size()))){
3174 
3175  foundkey1 = true;
3176 
3177  if (dimnamelist.find(osmapl2sdim1)== dimnamelist.end())
3178  throw5("variable ",tempvarname," must have dimension ",osmapl2sdim1," , but not found ");
3179 
3180  tempdimnamelist.erase(osmapl2sdim1);
3181 
3182  auto GMcvar = new GMCVar(*irv);
3183  GMcvar->newname = GMcvar->name;
3184  GMcvar->cfdimname = osmapl2sdim1;
3185  GMcvar->cvartype = CV_EXIST;
3186  GMcvar->product_type = product_type;
3187  this->cvars.push_back(GMcvar);
3188  delete(*irv);
3189  irv = this->vars.erase(irv);
3190  }// end of else if ((tempvarname.size() > key1.size())&& ...
3191  else {
3192  ++irv;
3193  }
3194  if (true == foundkey0 && true == foundkey1)
3195  break;
3196  } // end of for
3197 
3198  for (auto irs = tempdimnamelist.begin(); irs != tempdimnamelist.end();++irs) {
3199 
3200  auto GMcvar = new GMCVar();
3201  Create_Missing_CV(GMcvar,*irs);
3202  this->cvars.push_back(GMcvar);
3203  }
3204 
3205 }
3206 
3207 // Handle coordinate variables for Aquarius level 3 products
3208 void GMFile::Handle_CVar_Aqu_L3() {
3209 
3210  BESDEBUG("h5", "Coming to Handle_CVar_Aqu_L3()"<<endl);
3211  iscoard = true;
3212  for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
3213 
3214  if ( "l3m_data" == (*irv)->name) {
3215  for (auto ird = (*irv)->dims.begin(); ird != (*irv)->dims.end(); ++ird) {
3216  auto GMcvar = new GMCVar();
3217  GMcvar->name = (*ird)->name;
3218  GMcvar->newname = GMcvar->name;
3219  GMcvar->fullpath = GMcvar->name;
3220  GMcvar->rank = 1;
3221  GMcvar->dtype = H5FLOAT32;
3222  auto gmcvar_dim = new Dimension((*ird)->size);
3223  gmcvar_dim->name = GMcvar->name;
3224  gmcvar_dim->newname = gmcvar_dim->name;
3225  GMcvar->dims.push_back(gmcvar_dim);
3226  GMcvar->cfdimname = gmcvar_dim->name;
3227  if ("lat" ==GMcvar->name ) GMcvar->cvartype = CV_LAT_MISS;
3228  if ("lon" == GMcvar->name ) GMcvar->cvartype = CV_LON_MISS;
3229  GMcvar->product_type = product_type;
3230  this->cvars.push_back(GMcvar);
3231  } // end of for
3232  } // end of if
3233  }//end of for
3234 
3235 }
3236 
3237 //Handle coordinate variables for MeaSuRES Ozone products
3238 void GMFile::Handle_CVar_Mea_Ozone() {
3239 
3240  BESDEBUG("h5", "Coming to Handle_CVar_Mea_Ozone()"<<endl);
3241  pair<set<string>::iterator,bool> setret;
3242  set<string>tempdimnamelist = dimnamelist;
3243 
3244  if(false == iscoard)
3245  throw1("Measure Ozone level 3 zonal average product must follow COARDS conventions");
3246 
3247  for (auto irs = dimnamelist.begin(); irs != dimnamelist.end();++irs) {
3248  for (auto irv = this->vars.begin(); irv != this->vars.end(); ) {
3249  if ((*irs)== (*irv)->fullpath) {
3250 
3251  if((*irv)->dims.size()!=1)
3252  throw3("Coard coordinate variable",(*irv)->name, "is not 1D");
3253 
3254  // Create Coordinate variables.
3255  tempdimnamelist.erase(*irs);
3256  auto GMcvar = new GMCVar(*irv);
3257  GMcvar->cfdimname = *irs;
3258  GMcvar->cvartype = CV_EXIST;
3259  GMcvar->product_type = product_type;
3260  this->cvars.push_back(GMcvar);
3261  delete(*irv);
3262  irv = this->vars.erase(irv);
3263  } // end of if
3264  else {
3265  ++irv;
3266  }
3267  } // end of for irv
3268  } // end of for irs
3269 
3270  for (auto irs = tempdimnamelist.begin(); irs != tempdimnamelist.end();irs++) {
3271 
3272  auto GMcvar = new GMCVar();
3273  Create_Missing_CV(GMcvar,*irs);
3274  this->cvars.push_back(GMcvar);
3275  }
3276 }
3277 
3278 // Handle coordinate variables for general products that use HDF5 dimension scales.
3279 void GMFile::Handle_CVar_Dimscale_General_Product() {
3280 
3281  BESDEBUG("h5", "Coming to Handle_CVar_Dimscale_General_Product"<<endl);
3282  pair<set<string>::iterator,bool> setret;
3283  set<string>tempdimnamelist = dimnamelist;
3284 
3285  for (auto irs = dimnamelist.begin(); irs != dimnamelist.end();++irs) {
3286  for (auto irv = this->vars.begin(); irv != this->vars.end(); ) {
3287 
3288  // This is the dimension scale dataset; it should be changed to a coordinate variable.
3289  if ((*irs)== (*irv)->fullpath) {
3290  if((*irv)->dims.size()!=1)
3291  throw3("COARDS coordinate variable",(*irv)->name, "is not 1D");
3292 
3293  // Create Coordinate variables.
3294  tempdimnamelist.erase(*irs);
3295  auto GMcvar = new GMCVar(*irv);
3296  GMcvar->cfdimname = *irs;
3297 
3298  // Check if this is just a netCDF-4 dimension.
3299  bool is_netcdf_dimension = Is_netCDF_Dimension(*irv);
3300 
3301  // If this is just the netcdf dimension, we
3302  // will fill in the index numbers.
3303  if (true == is_netcdf_dimension)
3304  GMcvar->cvartype = CV_FILLINDEX;
3305  else
3306  GMcvar->cvartype = CV_EXIST;
3307  GMcvar->product_type = product_type;
3308  this->cvars.push_back(GMcvar);
3309  delete(*irv);
3310  irv = this->vars.erase(irv);
3311  } // end of if irs
3312  else {
3313  ++irv;
3314  }
3315  } // end of for irv
3316  } // end of for irs
3317 
3318  // Check if we have 2-D lat/lon CVs, and if yes, add those to the CV list.
3319  Update_M2DLatLon_Dimscale_CVs();
3320 
3321  // Add other missing coordinate variables.
3322  for (auto irs = tempdimnamelist.begin(); irs != tempdimnamelist.end();irs++) {
3323  auto GMcvar = new GMCVar();
3324  Create_Missing_CV(GMcvar,*irs);
3325  this->cvars.push_back(GMcvar);
3326  }
3327 
3328 
3329 //Debugging
3330 #if 0
3331 for (set<string>::iterator irs = dimnamelist.begin();
3332  irs != dimnamelist.end();irs++) {
3333 cerr<<"dimension name is "<<(*irs)<<endl;
3334 }
3335 #endif
3336 
3337 }
3338 
3339 
3340 // Check if we have 2-D lat/lon CVs in a netCDF-4-like file, and if yes, add those to the CV list.
3341 // This routine is a really complicate one. There are 9 steps to generate right 2-D lat/lon CVs.
3342 void GMFile::Update_M2DLatLon_Dimscale_CVs() {
3343 
3344  BESDEBUG("h5", "Coming to Update_M2DLatLon_Dimscale_CVs()"<<endl);
3345  // If this is not a file that only includes 1-D lat/lon CVs
3346  if(false == Check_1DGeolocation_Dimscale()) {
3347 
3348  // Define temporary vectors to store 1-D lat/lon CVs
3349  vector<GMCVar*> tempcvar_1dlat;
3350  vector<GMCVar*> tempcvar_1dlon;
3351 
3352  // 1. Obtain 1-D lat/lon CVs(only search the CF units and the reserved lat/lon names)
3353  Obtain_1DLatLon_CVs(tempcvar_1dlat,tempcvar_1dlon);
3354 
3355  // Define temporary vectors to store 2-D lat/lon Vars
3356  vector<Var*> tempcvar_2dlat;
3357  vector<Var*> tempcvar_2dlon;
3358 
3359  // This map remembers the positions of the latlon vars in the vector var.
3360  // Remembering the positions avoids the searching of these lat and lon again when
3361  // deleting them for the var vector and adding them(only the CVs) to the CV vector.
3362  // KY 2015-12-23
3363  map<string,int> latlon2d_path_to_index;
3364 
3365  // 2. Obtain 2-D lat/lon variables(only search the CF units and the reserved names)
3366  Obtain_2DLatLon_Vars(tempcvar_2dlat,tempcvar_2dlon,latlon2d_path_to_index);
3367 
3368 #if 0
3369 for(vector<GMCVar *>::iterator irv = tempcvar_1dlat.begin();irv != tempcvar_1dlat.end();++irv)
3370 cerr<<"1-D lat variable full path is "<<(*irv)->fullpath <<endl;
3371 for(vector<GMCVar *>::iterator irv = tempcvar_1dlon.begin();irv != tempcvar_1dlon.end();++irv)
3372 cerr<<"1-D lon variable full path is "<<(*irv)->fullpath <<endl;
3373 
3374 for(vector<Var *>::iterator irv = tempcvar_2dlat.begin();irv != tempcvar_2dlat.end();++irv)
3375 cerr<<"2-D lat variable full path is "<<(*irv)->fullpath <<endl;
3376 for(vector<Var *>::iterator irv = tempcvar_2dlon.begin();irv != tempcvar_2dlon.end();++irv)
3377 cerr<<"2-D lon variable full path is "<<(*irv)->fullpath <<endl;
3378 #endif
3379 
3380  // 3. Sequeeze the 2-D lat/lon vectors by removing the ones that share the same dims with 1-D lat/lon CVs.
3381  Obtain_2DLLVars_With_Dims_not_1DLLCVars(tempcvar_2dlat,tempcvar_2dlon,tempcvar_1dlat,tempcvar_1dlon,latlon2d_path_to_index);
3382 
3383 #if 0
3384 for(vector<Var *>::iterator irv = tempcvar_2dlat.begin();irv != tempcvar_2dlat.end();++irv)
3385 cerr<<"2-D Left lat variable full path is "<<(*irv)->fullpath <<endl;
3386 for(vector<Var *>::iterator irv = tempcvar_2dlon.begin();irv != tempcvar_2dlon.end();++irv)
3387 cerr<<"2-D Left lon variable full path is "<<(*irv)->fullpath <<endl;
3388 #endif
3389 
3390  // 4. Assemble the final 2-D lat/lon CV candidate vectors by checking if the corresponding 2-D lon of a 2-D lat shares
3391  // the same dimension and under the same group and if there is another pair of 2-D lat/lon under the same group.
3392  Obtain_2DLLCVar_Candidate(tempcvar_2dlat,tempcvar_2dlon,latlon2d_path_to_index);
3393 
3394 #if 0
3395 for(vector<Var *>::iterator irv = tempcvar_2dlat.begin();irv != tempcvar_2dlat.end();++irv)
3396 cerr<<"Final candidate 2-D Left lat variable full path is "<<(*irv)->fullpath <<endl;
3397 for(vector<Var *>::iterator irv = tempcvar_2dlon.begin();irv != tempcvar_2dlon.end();++irv)
3398 cerr<<"Final candidate 2-D Left lon variable full path is "<<(*irv)->fullpath <<endl;
3399 #endif
3400 
3401  // 5. Remove the 2-D lat/lon variables that are to be used as CVs from the vector that stores general variables
3402  // var2d_index, remembers the index of the 2-D lat/lon CVs in the original vector of vars.
3403  vector<int> var2d_index;
3404  for (map<string,int>::const_iterator it= latlon2d_path_to_index.begin();it!=latlon2d_path_to_index.end();++it)
3405  var2d_index.push_back(it->second);
3406 
3407  Remove_2DLLCVar_Final_Candidate_from_Vars(var2d_index);
3408 
3409  // 6. If we have 2-D CVs, COARDS should be turned off.
3410  if(tempcvar_2dlat.size()>0)
3411  iscoard = false;
3412 
3413  // 7. Add the CVs based on the final 2-D lat/lon CV candidates.
3414  // We need to remember the dim names that the 2-D lat/lon CVs are associated with.
3415  set<string>dim_names_2d_cvs;
3416 
3417  for(auto irv = tempcvar_2dlat.begin();irv != tempcvar_2dlat.end();++irv){
3418  auto lat = new GMCVar(*irv);
3419  // Latitude is always corresponding to the first dimension.
3420  lat->cfdimname = (*irv)->getDimensions()[0]->name;
3421  dim_names_2d_cvs.insert(lat->cfdimname);
3422  lat->cvartype = CV_EXIST;
3423  lat->product_type = product_type;
3424  this->cvars.push_back(lat);
3425  }
3426  for(auto irv = tempcvar_2dlon.begin();irv != tempcvar_2dlon.end();++irv){
3427  auto lon = new GMCVar(*irv);
3428  // Longitude is always corresponding to the second dimension.
3429  lon->cfdimname = (*irv)->getDimensions()[1]->name;
3430  dim_names_2d_cvs.insert(lon->cfdimname);
3431  lon->cvartype = CV_EXIST;
3432  lon->product_type = product_type;
3433  this->cvars.push_back(lon);
3434  }
3435 
3436  // 8. Move the originally assigned 1-D CVs that are replaced by 2-D CVs back to the general variable list.
3437  // Also remove the CV created by the pure dimensions.
3438  // Dimension names are used to identify those 1-D CVs.
3439  for(auto ircv= this->cvars.begin();ircv !=this->cvars.end();) {
3440  if(1 == (*ircv)->rank) {
3441  if(dim_names_2d_cvs.find((*ircv)->cfdimname)!=dim_names_2d_cvs.end()) {
3442  if(CV_FILLINDEX == (*ircv)->cvartype) {// This is pure dimension
3443  delete(*ircv);
3444  ircv = this->cvars.erase(ircv);
3445  }
3446  else if(CV_EXIST == (*ircv)->cvartype) {// This var exists already
3447 
3448  // Add this var. to the var list.
3449  Var *var = new Var(*ircv);
3450  this->vars.push_back(var);
3451 
3452  // Remove this var. from the GMCVar list.
3453  delete(*ircv);
3454  ircv = this->cvars.erase(ircv);
3455 
3456  }
3457  else {// the removed 1-D coordinate variable should be either the CV_FILLINDEX or CV_EXIST.
3458  if(CV_LAT_MISS == (*ircv)->cvartype)
3459  throw3("For the 2-D lat/lon case, the latitude dimension name ",(*ircv)->cfdimname, "is a coordinate variable of type CV_LAT_MISS");
3460  else if(CV_LON_MISS == (*ircv)->cvartype)
3461  throw3("For the 2-D lat/lon case, the latitude dimension name ",(*ircv)->cfdimname, "is a coordinate variable of type CV_LON_MISS");
3462  else if(CV_NONLATLON_MISS == (*ircv)->cvartype)
3463  throw3("For the 2-D lat/lon case, the latitude dimension name ",(*ircv)->cfdimname, "is a coordinate variable of type CV_NONLATLON_MISS");
3464  else if(CV_MODIFY == (*ircv)->cvartype)
3465  throw3("For the 2-D lat/lon case, the latitude dimension name ",(*ircv)->cfdimname, "is a coordinate variable of type CV_MODIFY");
3466  else if(CV_SPECIAL == (*ircv)->cvartype)
3467  throw3("For the 2-D lat/lon case, the latitude dimension name ",(*ircv)->cfdimname, "is a coordinate variable of type CV_SPECIAL");
3468  else
3469  throw3("For the 2-D lat/lon case, the latitude dimension name ",(*ircv)->cfdimname, "is a coordinate variable of type CV_UNSUPPORTED");
3470  }
3471 
3472  }
3473  else
3474  ++ircv;
3475 
3476  }
3477  else
3478  ++ircv;
3479 
3480  }
3481 
3482 
3483 #if 0
3484 //if(iscoard == true)
3485 //cerr<<"COARD is true"<<endl;
3486 for(set<string>::iterator irs = grp_cv_paths.begin();irs != grp_cv_paths.end();++irs) {
3487 cerr<<"group path is "<< (*irs)<<endl;
3488 
3489 }
3490 #endif
3491 
3492 #if 0
3493 //Print CVs
3494 cerr<<"File name is "<< this->path <<endl;
3495 cerr<<"CV names are the following "<<endl;
3496 for (vector<GMCVar *>:: iterator i= this->cvars.begin(); i!=this->cvars.end(); ++i)
3497 cerr<<(*i)->fullpath <<endl;
3498 #endif
3499 
3500 
3501  // 9. release the resources allocated by the temporary vectors.
3502  release_standalone_GMCVar_vector(tempcvar_1dlat);
3503  release_standalone_GMCVar_vector(tempcvar_1dlon);
3504  release_standalone_var_vector(tempcvar_2dlat);
3505  release_standalone_var_vector(tempcvar_2dlon);
3506  }// if(false == Check_1DGeolocation_Dimscale())
3507 #if 0
3508 for (vector<GMCVar *>:: iterator i= this->cvars.begin(); i!=this->cvars.end(); ++i)
3509 cerr<<(*i)->fullpath <<endl;
3510 #endif
3511 
3512 }
3513 
3514 // If Check_1DGeolocation_Dimscale() is true, no need to build 2-D lat/lon coordinate variables.
3515 // This function is introduced to avoid the performance penalty caused by handling the general 2-D lat/lon case.
3516 bool GMFile::Check_1DGeolocation_Dimscale() {
3517 
3518  BESDEBUG("h5", "Coming to Check_1DGeolocation_Dimscale()"<<endl);
3519  bool has_only_1d_geolocation_cv = false;
3520  bool has_1d_lat_cv_flag = false;
3521  bool has_1d_lon_cv_flag = false;
3522 
3523  string lat_dimname;
3524  hsize_t lat_size = 0;
3525 
3526  string lon_dimname;
3527  hsize_t lon_size = 0;
3528 
3529  // We need to consider both 1-D lat/lon and the 1-D zonal average case(1-D lat only).
3530  for (auto ircv = this->cvars.begin(); ircv != this->cvars.end(); ++ircv) {
3531  if((*ircv)->cvartype == CV_EXIST) {
3532  string attr_name ="units";
3533  string lat_unit_value = "degrees_north";
3534  string lon_unit_value = "degrees_east";
3535 
3536  for(auto ira = (*ircv)->attrs.begin(); ira != (*ircv)->attrs.end();ira++) {
3537 
3538  if(true == Is_Str_Attr(*ira,(*ircv)->fullpath,attr_name,lat_unit_value)) {
3539  lat_size = (*ircv)->getDimensions()[0]->size;
3540  lat_dimname = (*ircv)->getDimensions()[0]->name;
3541  has_1d_lat_cv_flag = true;
3542  break;
3543  }
3544  else if(true == Is_Str_Attr(*ira,(*ircv)->fullpath,attr_name,lon_unit_value)){
3545  lon_size = (*ircv)->getDimensions()[0]->size;
3546  lon_dimname = (*ircv)->getDimensions()[0]->name;
3547  has_1d_lon_cv_flag = true;
3548  break;
3549  }
3550  }
3551  }
3552  }
3553 
3554  // If having 1-D lat/lon CVs, this is a good sign for only 1-D lat/lon CVs ,
3555  // just need to have a couple of checks.
3556  if(true == has_1d_lat_cv_flag ) {
3557 
3558  if(true == has_1d_lon_cv_flag) {
3559 
3560  // Come to the possible classic netCDF-4 case,
3561  if(0 == this->groups.size()) {
3562 
3563  // Rarely happens when lat_size is the same as the lon_size.
3564  // However, still want to make sure there is a 2-D variable that uses both lat and lon dims.
3565  if(lat_size == lon_size) {
3566  bool var_has_latdim = false;
3567  bool var_has_londim = false;
3568  for (auto irv = this->vars.begin();
3569  irv != this->vars.end(); ++irv) {
3570  if((*irv)->rank >= 2) {
3571  for (auto ird = (*irv)->dims.begin(); ird !=(*irv)->dims.end();++ird) {
3572  if((*ird)->name == lat_dimname)
3573  var_has_latdim = true;
3574  else if((*ird)->name == lon_dimname)
3575  var_has_londim = true;
3576  }
3577  if(true == var_has_latdim && true == var_has_londim) {
3578  has_only_1d_geolocation_cv = true;
3579  break;
3580  }
3581  else {
3582  var_has_latdim = false;
3583  var_has_londim = false;
3584  }
3585  }
3586  }
3587  }
3588  else
3589  has_only_1d_geolocation_cv = true;
3590  }// if(0 == this->groups.size())
3591  else {
3592  // Multiple groups, need to check if having 2-D lat/lon pairs
3593  bool has_2d_latname_flag = false;
3594  bool has_2d_lonname_flag = false;
3595  for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
3596  if((*irv)->rank == 2) {
3597 
3598  //Note: When the 2nd parameter is true in the function Is_geolatlon, it checks the lat/latitude/Latitude
3599  if(true == Is_geolatlon((*irv)->name,true))
3600  has_2d_latname_flag = true;
3601 
3602  //Note: When the 2nd parameter is false in the function Is_geolatlon, it checks the lon/longitude/Longitude
3603  else if(true == Is_geolatlon((*irv)->name,false))
3604  has_2d_lonname_flag = true;
3605 
3606  if((true == has_2d_latname_flag) && (true == has_2d_lonname_flag))
3607  break;
3608  }
3609  }
3610 
3611  if(has_2d_latname_flag != true || has_2d_lonname_flag != true) {
3612 
3613  //Check if having the 2-D lat/lon by checking if having lat/lon CF units(lon's units: degrees_east lat's units: degrees_north)
3614  has_2d_latname_flag = false;
3615  has_2d_lonname_flag = false;
3616 
3617  for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
3618  if((*irv)->rank == 2) {
3619  for (auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira) {
3620 
3621  if (false == has_2d_latname_flag) {
3622 
3623  // When the third parameter of the function has_latlon_cf_units is set to true, it checks latitude
3624  has_2d_latname_flag = has_latlon_cf_units((*ira),(*irv)->fullpath,true);
3625  if(true == has_2d_latname_flag)
3626  break;
3627  else if(false == has_2d_lonname_flag) {
3628 
3629  // When the third parameter of the function has_latlon_cf_units is set to false, it checks longitude
3630  has_2d_lonname_flag = has_latlon_cf_units((*ira),(*irv)->fullpath,false);
3631  if(true == has_2d_lonname_flag)
3632  break;
3633  }
3634  }
3635  else if(false == has_2d_lonname_flag) {
3636 
3637  // Now has_2d_latname_flag is true, just need to check the has_2d_lonname_flag
3638  // When the third parameter of has_latlon_cf_units is set to false, it checks longitude
3639  has_2d_lonname_flag = has_latlon_cf_units((*ira),(*irv)->fullpath,false);
3640  if(true == has_2d_lonname_flag)
3641  break;
3642  }
3643  }
3644  if(true == has_2d_latname_flag && true == has_2d_lonname_flag)
3645  break;
3646  }
3647  }
3648  }// if(has_2d_latname_flag != true || has_2d_lonname_flag != true)
3649 
3650  // If we cannot find either of 2-D any lat/lon variables, this file is treated as having only 1-D lat/lon.
3651  if(has_2d_latname_flag != true || has_2d_lonname_flag != true)
3652  has_only_1d_geolocation_cv = true;
3653  }
3654 
3655  }//
3656  else {//Zonal average case, we do not need to find 2-D lat/lon CVs.
3657  has_only_1d_geolocation_cv = true;
3658  }
3659 
3660  }
3661 
3662 #if 0
3663 if(has_only_1d_geolocation_cv == true)
3664 cerr <<"has only 1D lat/lon CVs. "<<endl;
3665 else
3666 cerr<<"Possibly has 2D lat/lon CVs. "<<endl;
3667 #endif
3668 
3669  return has_only_1d_geolocation_cv;
3670 
3671 }
3672 
3673 // Obtain the originally assigned 1-D lat/lon coordinate variables.
3674 // This function should be used before generating any 2-D lat/lon CVs.
3675 void GMFile::Obtain_1DLatLon_CVs(vector<GMCVar*> &cvar_1dlat,vector<GMCVar*> &cvar_1dlon) {
3676 
3677  BESDEBUG("h5", "Coming to Obtain_1DLatLon_CVs()"<<endl);
3678  for (auto ircv = this->cvars.begin(); ircv != this->cvars.end(); ++ircv) {
3679 
3680  if((*ircv)->cvartype == CV_EXIST) {
3681 
3682  string attr_name ="units";
3683  string lat_unit_value = "degrees_north";
3684  string lon_unit_value = "degrees_east";
3685 
3686  for(auto ira = (*ircv)->attrs.begin(); ira != (*ircv)->attrs.end();ira++) {
3687 
3688  // 1-D latitude
3689  if(true == Is_Str_Attr(*ira,(*ircv)->fullpath,attr_name,lat_unit_value)) {
3690  auto lat = new GMCVar(*ircv);
3691  lat->cfdimname = (*ircv)->getDimensions()[0]->name;
3692  lat->cvartype = (*ircv)->cvartype;
3693  lat->product_type = (*ircv)->product_type;
3694  cvar_1dlat.push_back(lat);
3695  }
3696  // 1-D longitude
3697  else if(true == Is_Str_Attr(*ira,(*ircv)->fullpath,attr_name,lon_unit_value)){
3698  auto lon = new GMCVar(*ircv);
3699  lon->cfdimname = (*ircv)->getDimensions()[0]->name;
3700  lon->cvartype = (*ircv)->cvartype;
3701  lon->product_type = (*ircv)->product_type;
3702  cvar_1dlon.push_back(lon);
3703  }
3704  }
3705  }// end of if((*ircv)->cvartype == CV_EXIST)
3706  }// end of for (ircv = this->cvars.begin();
3707 
3708 }
3709 
3710 // Obtain all 2-D lat/lon variables. We first check the lat/latitude/Latitude names, if not found, we check if CF lat/lon units are present.
3711 // Latitude variables are saved in the vector var_2dlat. Longitude variables are saved in the vector var_2dlon.
3712 // We also remember the index of these lat/lon in the original var vector.
3713 void GMFile::Obtain_2DLatLon_Vars(vector<Var*> &var_2dlat,vector<Var*> &var_2dlon,map<string,int> & latlon2d_path_to_index) {
3714 
3715  BESDEBUG("h5", "Coming to Obtain_2DLatLon_Vars()"<<endl);
3716  for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
3717  if((*irv)->rank == 2) {
3718 
3719  //Note: When the 2nd parameter is true in the function Is_geolatlon, it checks the lat/latitude/Latitude
3720  if(true == Is_geolatlon((*irv)->name,true)) {
3721  Var *lat = new Var(*irv);
3722  var_2dlat.push_back(lat);
3723  latlon2d_path_to_index[(*irv)->fullpath]= distance(this->vars.begin(),irv);
3724  continue;
3725  }
3726  else {
3727 
3728  bool has_2dlat = false;
3729  for (auto ira = (*irv)->attrs.begin();
3730  ira != (*irv)->attrs.end(); ++ira) {
3731 
3732  // When the third parameter of has_latlon_cf_units is set to true, it checks latitude
3733  if(true == has_latlon_cf_units((*ira),(*irv)->fullpath,true)) {
3734  Var *lat = new Var(*irv);
3735  var_2dlat.push_back(lat);
3736  latlon2d_path_to_index[(*irv)->fullpath] = distance(this->vars.begin(),irv);
3737  has_2dlat = true;
3738  break;
3739  }
3740  }
3741 
3742  if(true == has_2dlat)
3743  continue;
3744  }
3745 
3746  //Note: When the 2nd parameter is false in the function Is_geolatlon, it checks the lon/longitude/Longitude
3747  if(true == Is_geolatlon((*irv)->name,false)) {
3748  Var *lon = new Var(*irv);
3749  latlon2d_path_to_index[(*irv)->fullpath] = distance(this->vars.begin(),irv);
3750  var_2dlon.push_back(lon);
3751  }
3752  else {
3753  for (auto ira = (*irv)->attrs.begin();
3754  ira != (*irv)->attrs.end(); ++ira) {
3755 
3756  // When the third parameter of has_latlon_cf_units is set to false, it checks longitude
3757  if(true == has_latlon_cf_units((*ira),(*irv)->fullpath,false)) {
3758  Var *lon = new Var(*irv);
3759  latlon2d_path_to_index[(*irv)->fullpath] = distance(this->vars.begin(),irv);
3760  var_2dlon.push_back(lon);
3761  break;
3762  }
3763  }
3764  }
3765  } // if((*irv)->rank == 2)
3766  } // for (auto irv
3767 }
3768 
3769 // Sequeeze the 2-D lat/lon vectors by removing the ones that share the same dims with 1-D lat/lon CVs.
3770 // The latlon2d_path_to_index map also needs to be updated.
3771 void GMFile::Obtain_2DLLVars_With_Dims_not_1DLLCVars(vector<Var*> &var_2dlat,
3772  vector<Var*> &var_2dlon,
3773  vector<GMCVar*> &cvar_1dlat,
3774  vector<GMCVar*> &cvar_1dlon,
3775  map<string,int> &latlon2d_path_to_index) {
3776 
3777  BESDEBUG("h5", "Coming to Obtain_2DLLVars_With_Dims_not_1DLLCVars()"<<endl);
3778  // First latitude at var_2dlat
3779  for(auto irv = var_2dlat.begin();irv != var_2dlat.end();) {
3780  bool remove_2dlat = false;
3781  for(auto ircv = cvar_1dlat.begin();ircv != cvar_1dlat.end();++ircv) {
3782  for (auto ird = (*irv)->dims.begin(); ird!=(*irv)->dims.end(); ++ird) {
3783  if((*ird)->name == (*ircv)->getDimensions()[0]->name &&
3784  (*ird)->size == (*ircv)->getDimensions()[0]->size) {
3785  latlon2d_path_to_index.erase((*irv)->fullpath);
3786  delete(*irv);
3787  irv = var_2dlat.erase(irv);
3788  remove_2dlat = true;
3789  break;
3790  }
3791  }
3792  if(true == remove_2dlat)
3793  break;
3794  }
3795 
3796  if(false == remove_2dlat)
3797  ++irv;
3798  }// end of for(auto irv = var_2dlat.begin()
3799 
3800  // Second longitude
3801  for(auto irv = var_2dlon.begin();irv != var_2dlon.end();) {
3802  bool remove_2dlon = false;
3803  for(auto ircv = cvar_1dlon.begin();ircv != cvar_1dlon.end();++ircv) {
3804  for (auto ird = (*irv)->dims.begin();
3805  ird!=(*irv)->dims.end(); ++ird) {
3806  if((*ird)->name == (*ircv)->getDimensions()[0]->name &&
3807  (*ird)->size == (*ircv)->getDimensions()[0]->size) {
3808  latlon2d_path_to_index.erase((*irv)->fullpath);
3809  delete(*irv);
3810  irv = var_2dlon.erase(irv);
3811  remove_2dlon = true;
3812  break;
3813  }
3814  }
3815  if(true == remove_2dlon)
3816  break;
3817  }
3818 
3819  if(false == remove_2dlon)
3820  ++irv;
3821  } // end of for(auto irv = var_2dlon.begin()
3822 
3823 }
3824 
3825 //Out of the collected 2-D lat/lon variables, we will select the final qualified 2-D lat/lon as CVs.
3826 void GMFile::Obtain_2DLLCVar_Candidate(vector<Var*> &var_2dlat,
3827  vector<Var*> &var_2dlon,
3828  map<string,int>& latlon2d_path_to_index) {
3829  BESDEBUG("h5", "Coming to Obtain_2DLLCVar_Candidate()"<<endl);
3830  // First check 2-D lat, see if we have the corresponding 2-D lon(same dims, under the same group).
3831  // If no, remove that lat from the vector.
3832  vector<string> lon2d_group_paths;
3833 
3834  for(auto irv_2dlat = var_2dlat.begin();irv_2dlat !=var_2dlat.end();) {
3835  for(auto irv_2dlon = var_2dlon.begin();irv_2dlon != var_2dlon.end();++irv_2dlon) {
3836  if(((*irv_2dlat)->getDimensions()[0]->name == (*irv_2dlon)->getDimensions()[0]->name) &&
3837  ((*irv_2dlat)->getDimensions()[0]->size == (*irv_2dlon)->getDimensions()[0]->size) &&
3838  ((*irv_2dlat)->getDimensions()[1]->name == (*irv_2dlon)->getDimensions()[1]->name) &&
3839  ((*irv_2dlat)->getDimensions()[1]->size == (*irv_2dlon)->getDimensions()[1]->size))
3840  lon2d_group_paths.push_back(HDF5CFUtil::obtain_string_before_lastslash((*irv_2dlon)->fullpath));
3841  }
3842  // Doesn't find any lons that shares the same dims,remove this lat from the 2dlat vector,
3843  // also update the latlon2d_path_to_index map
3844  if(0 == lon2d_group_paths.size()) {
3845  latlon2d_path_to_index.erase((*irv_2dlat)->fullpath);
3846  delete(*irv_2dlat);
3847  irv_2dlat = var_2dlat.erase(irv_2dlat);
3848  }
3849  else {// Find lons,check if they are under the same group
3850 #if 0
3851  //string lat2d_group_path = (*irv_2dlat)->fullpath.substr(0,(*irv_2dlat)->fullpath.find_last_of("/"));
3852 #endif
3853  string lat2d_group_path = HDF5CFUtil::obtain_string_before_lastslash((*irv_2dlat)->fullpath);
3854 
3855  // Check how many lon2d shares the same group with the lat2d
3856  short lon2d_has_lat2d_group_path_flag = 0;
3857  for(auto ivs = lon2d_group_paths.begin();ivs!=lon2d_group_paths.end();++ivs) {
3858  if((*ivs)==lat2d_group_path)
3859  lon2d_has_lat2d_group_path_flag++;
3860  }
3861 
3862  // No lon2d shares the same group with the lat2d, remove this lat2d
3863  if(0 == lon2d_has_lat2d_group_path_flag) {
3864  latlon2d_path_to_index.erase((*irv_2dlat)->fullpath);
3865  delete(*irv_2dlat);
3866  irv_2dlat = var_2dlat.erase(irv_2dlat);
3867  }
3868  // Only one lon2d, yes, keep it.
3869  else if (1== lon2d_has_lat2d_group_path_flag) {
3870  ++irv_2dlat;
3871  }
3872  // More than 1 lon2d, we will remove the lat2d, but save the group path so that we may
3873  // flatten the variable path stored in the coordinates attribute under this group.
3874  else {
3875  // Save the group path for the future use.
3876  grp_cv_paths.insert(lat2d_group_path);
3877  latlon2d_path_to_index.erase((*irv_2dlat)->fullpath);
3878  delete(*irv_2dlat);
3879  irv_2dlat = var_2dlat.erase(irv_2dlat);
3880  }
3881  }
3882 
3883  //Clear the vector that stores the same dim. since it is only applied to this lat,
3884  lon2d_group_paths.clear();
3885  }
3886 
3887 #if 0
3888 for(auto irv_2dlat = var_2dlat.begin();irv_2dlat !=var_2dlat.end();++irv_2dlat)
3889 cerr<<"2 left 2-D lat variable full path is: "<<(*irv_2dlat)->fullpath <<endl;
3890 #endif
3891 
3892 
3893  // Second check 2-D lon, see if we have the corresponding 2-D lat(same dims, under the same group).
3894  // If no, remove that lon from the vector.
3895  vector<string> lat2d_group_paths;
3896 
3897  // Check the longitude
3898  for(auto irv_2dlon = var_2dlon.begin();irv_2dlon !=var_2dlon.end();) {
3899  for(auto irv_2dlat = var_2dlat.begin();irv_2dlat != var_2dlat.end();++irv_2dlat) {
3900  if(((*irv_2dlat)->getDimensions()[0]->name == (*irv_2dlon)->getDimensions()[0]->name) &&
3901  ((*irv_2dlat)->getDimensions()[0]->size == (*irv_2dlon)->getDimensions()[0]->size) &&
3902  ((*irv_2dlat)->getDimensions()[1]->name == (*irv_2dlon)->getDimensions()[1]->name) &&
3903  ((*irv_2dlat)->getDimensions()[1]->size == (*irv_2dlon)->getDimensions()[1]->size))
3904  lat2d_group_paths.push_back(HDF5CFUtil::obtain_string_before_lastslash((*irv_2dlat)->fullpath));
3905 #if 0
3906  //lat2d_group_paths.push_back((*irv_2dlat)->fullpath.substr(0,(*irv_2dlat)->fullpath.find_last_of("/")));
3907 #endif
3908  }
3909  // Doesn't find any lats that shares the same dims,remove this lon from this vector
3910  if(lat2d_group_paths.empty()) {
3911  latlon2d_path_to_index.erase((*irv_2dlon)->fullpath);
3912  delete(*irv_2dlon);
3913  irv_2dlon = var_2dlon.erase(irv_2dlon);
3914  }
3915  else {
3916  string lon2d_group_path = HDF5CFUtil::obtain_string_before_lastslash((*irv_2dlon)->fullpath);
3917 
3918  // Check how many lat2d shares the same group with the lon2d
3919  short lat2d_has_lon2d_group_path_flag = 0;
3920  for(auto ivs = lat2d_group_paths.begin();ivs!=lat2d_group_paths.end();++ivs) {
3921  if((*ivs)==lon2d_group_path)
3922  lat2d_has_lon2d_group_path_flag++;
3923  }
3924 
3925  // No lat2d shares the same group with the lon2d, remove this lon2d
3926  if(0 == lat2d_has_lon2d_group_path_flag) {
3927  latlon2d_path_to_index.erase((*irv_2dlon)->fullpath);
3928  delete(*irv_2dlon);
3929  irv_2dlon = var_2dlon.erase(irv_2dlon);
3930  }
3931  // Only one lat2d shares the same group with the lon2d, yes, keep it.
3932  else if (1== lat2d_has_lon2d_group_path_flag) {
3933  ++irv_2dlon;
3934  }
3935  // more than 1 lat2d, we will remove the lon2d, but save the group path so that we can
3936  // change the coordinates attribute for variables under this group later.
3937  else {
3938  // Save the group path for future "coordinates" modification.
3939  grp_cv_paths.insert(lon2d_group_path);
3940  latlon2d_path_to_index.erase((*irv_2dlon)->fullpath);
3941  delete(*irv_2dlon);
3942  irv_2dlon = var_2dlon.erase(irv_2dlon);
3943  }
3944  }
3945  //Clear the vector that stores the same dim. since it is only applied to this lon,
3946  lat2d_group_paths.clear();
3947  }
3948 #if 0
3949 for(vector<Var*>::iterator itv = var_2dlat.begin(); itv!= var_2dlat.end();++itv) {
3950 cerr<<"Before unique, 2-D CV latitude name is "<<(*itv)->fullpath <<endl;
3951 }
3952 for(vector<Var*>::iterator itv = var_2dlon.begin(); itv!= var_2dlon.end();++itv) {
3953 cerr<<"Before unique, 2-D CV longitude name is "<<(*itv)->fullpath <<endl;
3954 }
3955 #endif
3956 
3957  // Final check var_2dlat and var_2dlon to remove non-qualified CVs.
3958  Obtain_unique_2dCV(var_2dlat,latlon2d_path_to_index);
3959  Obtain_unique_2dCV(var_2dlon,latlon2d_path_to_index);
3960 #if 0
3961 for(vector<Var*>::iterator itv = var_2dlat.begin(); itv!= var_2dlat.end();++itv) {
3962 cerr<<"2-D CV latitude name is "<<(*itv)->fullpath <<endl;
3963 }
3964 for(vector<Var*>::iterator itv = var_2dlon.begin(); itv!= var_2dlon.end();++itv) {
3965 cerr<<"2-D CV longitude name is "<<(*itv)->fullpath <<endl;
3966 }
3967 #endif
3968 
3969  // This is to serve as a sanity check. This can help us find bugs in the first place.
3970  if(var_2dlat.size() != var_2dlon.size()) {
3971  throw1("Error in generating 2-D lat/lon CVs. The size of 2d-lat should be the same as that of 2d-lon.");
3972  }
3973 }
3974 
3975 // If two vars in the 2-D lat or 2-D lon CV candidate vector share the same dim. , these two vars cannot be CVs.
3976 // The group they belong to is the group candidate that the coordinates attribute of the variable under that group may be modified..
3977 void GMFile::Obtain_unique_2dCV(vector<Var*> &var_ll,map<string,int>&latlon2d_path_to_index){
3978 
3979  BESDEBUG("h5", "Coming to Obtain_unique_2dCV()"<<endl);
3980  vector<bool> var_share_dims(var_ll.size(),false);
3981 
3982  for(unsigned int i = 0; i <var_ll.size();i++) {
3983 
3984  // obtain the path of var_ll
3985  string var_ll_i_path = HDF5CFUtil::obtain_string_before_lastslash(var_ll[i]->fullpath);
3986 
3987  // Check if two vars share the same dims.
3988  for(unsigned int j = i+1; j<var_ll.size();j++) {
3989  if((var_ll[i]->getDimensions()[0]->name == var_ll[j]->getDimensions()[0]->name)
3990  ||(var_ll[i]->getDimensions()[0]->name == var_ll[j]->getDimensions()[1]->name)
3991  ||(var_ll[i]->getDimensions()[1]->name == var_ll[j]->getDimensions()[0]->name)
3992  ||(var_ll[i]->getDimensions()[1]->name == var_ll[j]->getDimensions()[1]->name)){
3993  string var_ll_j_path = HDF5CFUtil::obtain_string_before_lastslash(var_ll[j]->fullpath);
3994 
3995  // Compare var_ll_i_path and var_ll_j_path,only set the child group path be true and remember the path.
3996  // The variable at the parent group can be the coordinate variable.
3997  // Obtain the string size,
3998  // compare the string size, long.compare(0,shortlength,short)==0,
3999  // yes, save the long path(child group path), set the long path one true. Else save two paths, set both true
4000  if(var_ll_i_path.size() > var_ll_j_path.size()) {
4001 
4002  // If var_ll_j_path is the parent group of var_ll_i_path,
4003  // set the shared dim. be true for the child group only,remember the path.
4004  if(var_ll_i_path.compare(0,var_ll_j_path.size(),var_ll_j_path)==0) {
4005  var_share_dims[i] = true;
4006  grp_cv_paths.insert(var_ll_i_path);
4007  }
4008  else {// Save both as shared, they cannot be CVs.
4009  var_share_dims[i] = true;
4010  var_share_dims[j] = true;
4011 
4012  grp_cv_paths.insert(var_ll_i_path);
4013  grp_cv_paths.insert(var_ll_j_path);
4014  }
4015  }
4016  else if (var_ll_i_path.size() == var_ll_j_path.size()) {// Share the same group, remember both group paths.
4017  var_share_dims[i] = true;
4018  var_share_dims[j] = true;
4019  if(var_ll_i_path == var_ll_j_path)
4020  grp_cv_paths.insert(var_ll_i_path);
4021  else {
4022  grp_cv_paths.insert(var_ll_i_path);
4023  grp_cv_paths.insert(var_ll_j_path);
4024  }
4025  }
4026  else {
4027  // var_ll_i_path is the parent group of var_ll_j_path,
4028  // set the shared dim. be true for the child group,remember the path.
4029  if(var_ll_j_path.compare(0,var_ll_i_path.size(),var_ll_i_path)==0) {
4030  var_share_dims[j] = true;
4031  grp_cv_paths.insert(var_ll_j_path);
4032  }
4033  else {// Save both as shared, they cannot be CVs.
4034  var_share_dims[i] = true;
4035  var_share_dims[j] = true;
4036 
4037  grp_cv_paths.insert(var_ll_i_path);
4038  grp_cv_paths.insert(var_ll_j_path);
4039 
4040  }
4041  }
4042  }// end of if((var_ll[i]->getDimensions()[0]->name == var_ll[j]->getDimensions()[0]->name)
4043  }// end of for(int j = i+1; j<var_ll.size();j++)
4044  }// end of for( int i = 0; i <var_ll.size();i++)
4045 
4046  // Remove the shared 2-D lat/lon CVs from the 2-D lat/lon CV candidates.
4047  int var_index = 0;
4048  for(auto itv = var_ll.begin(); itv!= var_ll.end();) {
4049  if(true == var_share_dims[var_index]) {
4050  latlon2d_path_to_index.erase((*itv)->fullpath);
4051  delete(*itv);
4052  itv = var_ll.erase(itv);
4053  }
4054  else {
4055  ++itv;
4056  }
4057  ++var_index;
4058  }
4059 
4060 }
4061 
4062 // When promoting a 2-D lat or lon to a coordinate variable, we need to remove them from the general variable vector.
4063 void GMFile::Remove_2DLLCVar_Final_Candidate_from_Vars(vector<int> &var2d_index) {
4064 
4065  BESDEBUG("h5", "Coming to Remove_2DLLCVar_Final_Candidate_from_Vars()"<<endl);
4066  //Sort the 2-D lat/lon var index according to the ascending order before removing the 2-D lat/lon vars
4067  sort(var2d_index.begin(),var2d_index.end());
4068  auto it = this->vars.begin();
4069 
4070  // This is a performance optimiziation operation.
4071  // We find it is typical for swath files that have many many general variables but only have very few lat/lon CVs.
4072  // To reduce the looping through all variables and comparing the fullpath(string), we use index and remember
4073  // the position of 2-D CVs in the iterator. In this way, only a few operations are needed.
4074  for (unsigned int i = 0; i <var2d_index.size();i++) {
4075  if ( i == 0)
4076  advance(it,var2d_index[i]);
4077  else
4078  advance(it,var2d_index[i]-var2d_index[i-1]-1);
4079 
4080  if(it == this->vars.end())
4081  throw1("Out of range to obtain 2D lat/lon variables");
4082  else {
4083  delete(*it);
4084  it = this->vars.erase(it);
4085  }
4086  }
4087 }
4088 
4089 //This function is for generating the coordinates attribute for the 2-D lat/lon.
4090 //It will check if this var can keep its "coordinates" attribute rather than rebuilding it.
4091 //This function is used by Handle_Coor_Attr().
4092 bool GMFile::Check_Var_2D_CVars(Var *var) {
4093 
4094  BESDEBUG("h5", "Coming to Check_Var_2D_CVars()"<<endl);
4095  bool ret_value = true;
4096  for (auto ircv = this->cvars.begin(); ircv != this->cvars.end(); ++ircv) {
4097  if((*ircv)->rank==2) {
4098  short first_dim_index = 0;
4099  short first_dim_times = 0;
4100  short second_dim_index = 0;
4101  short second_dim_times = 0;
4102  for (auto ird = var->dims.begin(); ird != var->dims.end(); ++ird) {
4103  if((*ird)->name == ((*ircv)->getDimensions()[0])->name) {
4104  first_dim_index = distance(var->dims.begin(),ird);
4105  first_dim_times++;
4106  }
4107  else if((*ird)->name == ((*ircv)->getDimensions()[1])->name) {
4108  second_dim_index = distance(var->dims.begin(),ird);
4109  second_dim_times++;
4110  }
4111  }
4112  // The 2-D CV dimensions must only appear once as the dimension of the variable
4113  // It also must follow the dimension order of the 2-D lat/lon dimensions.
4114  if(first_dim_times == 1 && second_dim_times == 1) {
4115  if(first_dim_index < second_dim_index) {
4116  ret_value = false;
4117  break;
4118  }
4119  }
4120  }
4121  }
4122  return ret_value;
4123 
4124 }
4125 
4126 // This function flattens the variable path in the "coordinates" attribute.
4127 // It is also used by Handle_Coor_Attr().
4128 bool GMFile::Flatten_VarPath_In_Coordinates_Attr(Var *var) {
4129 
4130  BESDEBUG("h5", "Coming to Flatten_VarPath_In_Coordinates_Attr()"<<endl);
4131  string co_attrname = "coordinates";
4132  bool need_flatten_coor_attr = false;
4133  string orig_coor_value;
4134  string flatten_coor_value;
4135  // Assume the separator is always a space.
4136  char sc = ' ';
4137  char backslash = '/';
4138 
4139  for (auto ira =var->attrs.begin(); ira !=var->attrs.end();) {
4140 
4141  // We only check the original attribute name
4142  // Remove the original "coordinates" attribute.
4143  // There is a case that the "coordinates" attribute doesn't need to be flattened.
4144  // We will skip this case. This is necessary since the "coordinates" may be revised
4145  // to add a path to follow the CF. Without skipping the case, the "coordinates" is
4146  // actually not handled but "makes the impression" it is handled. That makes the
4147  // this case ignored when the EnableCoorattrAddPath BES key is supposed to work on.
4148  // https://bugs.earthdata.nasa.gov/browse/HDFFEATURE-45
4149  if((*ira)->name == co_attrname) {
4150  Retrieve_H5_Attr_Value((*ira),var->fullpath);
4151  string orig_attr_value((*ira)->value.begin(),(*ira)->value.end());
4152  if(orig_attr_value.find_first_of(backslash)!=string::npos){
4153  orig_coor_value = orig_attr_value;
4154  need_flatten_coor_attr = true;
4155  delete(*ira);
4156  ira = var->attrs.erase(ira);
4157  }
4158  break;
4159  }
4160  else
4161  ++ira;
4162  }
4163 
4164  if(true == need_flatten_coor_attr) {
4165 
4166  // We need to loop through each element in the "coordinates".
4167  size_t ele_start_pos = 0;
4168  size_t cur_pos = orig_coor_value.find_first_of(sc);
4169  while(cur_pos !=string::npos) {
4170  string tempstr = orig_coor_value.substr(ele_start_pos,cur_pos-ele_start_pos);
4171  tempstr = get_CF_string(tempstr);
4172  flatten_coor_value += tempstr + sc;
4173  ele_start_pos = cur_pos+1;
4174  cur_pos = orig_coor_value.find_first_of(sc,cur_pos+1);
4175  }
4176  // Only one element
4177  if(ele_start_pos == 0)
4178  flatten_coor_value = get_CF_string(orig_coor_value);
4179  else // Add the last element
4180  flatten_coor_value += get_CF_string(orig_coor_value.substr(ele_start_pos));
4181 
4182  // Generate the new "coordinates" attribute.
4183  auto attr = new Attribute();
4184  Add_Str_Attr(attr,co_attrname,flatten_coor_value);
4185  var->attrs.push_back(attr);
4186  var->coord_attr_add_path = false;
4187 
4188  }
4189 
4190  return true;
4191 
4192 }
4193 // This function flattens the variable path in the "coordinates" attribute for hybrid EOS5.
4194 // Will implement it later.
4195 // It is also used by Handle_Coor_Attr().
4196 #if 0
4197 bool GMFile::Flatten_VarPath_In_Coordinates_Attr_EOS5(Var *var) {
4198 
4199  BESDEBUG("h5", "Coming to Flatten_VarPath_In_Coordinates_Attr_EOS5()"<<endl);
4200  string co_attrname = "coordinates";
4201  bool has_coor_attr = false;
4202  string orig_coor_value;
4203  string flatten_coor_value;
4204  // Assume the separator is always a space.
4205  char sc = ' ';
4206 
4207  for (auto ira =var->attrs.begin(); ira !=var->attrs.end();) {
4208 
4209  // We only check the original attribute name
4210  // Remove the original "coordinates" attribute.
4211  if((*ira)->name == co_attrname) {
4212  Retrieve_H5_Attr_Value((*ira),var->fullpath);
4213 
4214  string orig_attr_value((*ira)->value.begin(),(*ira)->value.end());
4215  orig_coor_value = orig_attr_value;
4216  has_coor_attr = true;
4217  delete(*ira);
4218  ira = var->attrs.erase(ira);
4219  break;
4220  }
4221  else
4222  ++ira;
4223  }
4224 
4225  if(true == has_coor_attr) {
4226 
4227  // We need to loop through each element in the "coordinates".
4228  // For EOS5: we need to find the swath name.
4229  size_t ele_start_pos = 0;
4230 // cerr<<"orig_coor_value is "<<orig_coor_value <<endl;
4231  size_t cur_pos = orig_coor_value.find_first_of(sc);
4232  while(cur_pos !=string::npos) {
4233  string tempstr = orig_coor_value.substr(ele_start_pos,cur_pos-ele_start_pos);
4234  // Find the swath name
4235  // tempstr = "swath name" +"_"+tempstr;
4236  tempstr = get_CF_string(tempstr);
4237  flatten_coor_value += tempstr + sc;
4238  ele_start_pos = cur_pos+1;
4239  cur_pos = orig_coor_value.find_first_of(sc,cur_pos+1);
4240  }
4241  // Only one element
4242  if(ele_start_pos == 0) {
4243  // Find the swath name
4244  // tempstr = "swath name" +"_"+tempstr;
4245  flatten_coor_value = get_CF_string(tempstr);
4246  }
4247  else // Add the last element
4248  flatten_coor_value += get_CF_string(orig_coor_value.substr(ele_start_pos));
4249 
4250  // Generate the new "coordinates" attribute.
4251  Attribute *attr = new Attribute();
4252  Add_Str_Attr(attr,co_attrname,flatten_coor_value);
4253  var->attrs.push_back(attr);
4254  }
4255 
4256  return true;
4257 
4258 }
4259 #endif
4260 
4261 
4262 // The following two routines only handle one 2-D lat/lon CVs. It is replaced by the more general
4263 // multi 2-D lat/lon CV routines. Leave it here just for references.
4264 #if 0
4265 bool GMFile::Check_2DLatLon_Dimscale(string & latname, string &lonname) {
4266 
4267  // New code to support 2-D lat/lon, still in development.
4268  // Need to handle 2-D latitude and longitude cases.
4269  // 1. Searching only the coordinate variables and if getting either of the following, keep the current way,
4270  // (A) GMcvar no CV_FILLINDEX:(The 2-D latlon case should have fake CVs)
4271  // (B) CV_EXIST: Attributes contain units and units value is degrees_east or degrees_north(have lat/lon)
4272  // (B) CV_EXIST: variables have name pair{lat/latitude/Latitude,lon/longitude/Longitude}(have lat/lon)
4273  //
4274  // 2. if not 1), searching all the variables and see if finding variables {lat/latitude/Latitude,lon/longitude/Longitude};
4275  // If finding {lat/lon},{latitude,longitude},{latitude,Longitude} pair,
4276  // if the number of dimension of either variable is not 2, keep the current way.
4277  // else check if the dimension name of latitude and longitude are the same, not, keep the current way
4278  // check the units of this CV pair, if units of the latitude is not degrees_north,
4279  // change it to degrees_north.
4280  // if units of the longitude is not degrees_east, change it to degrees_east.
4281  // make iscoard false.
4282 
4283  bool latlon_2d_cv_check1 = false;
4284 
4285  // Some products(TOM MEaSURE) provide the true dimension scales for 2-D lat,lon. So relax this check.
4286  latlon_2d_cv_check1 = true;
4287 #if 0
4288  // If having 2-D lat/lon, the corresponding dimension must be pure and the CV type must be FILLINDEX.
4289  for (auto ircv = this->cvars.begin();
4290  ircv != this->cvars.end(); ++ircv) {
4291  if((*ircv)->cvartype == CV_FILLINDEX){
4292  latlon_2d_cv_check1 = true;
4293  break;
4294  }
4295  }
4296 #endif
4297 
4298  bool latlon_2d_cv_check2 = true;
4299 
4300  // There may still not be 2-D lat/lon. Check the units attributes and lat/lon pairs.
4301  if(true == latlon_2d_cv_check1) {
4302  BESDEBUG("h5","Coming to check if having 2d latlon coordinates for a netCDF-4 like product. "<<endl);
4303 
4304  // check if units attribute values have CF lat/lon units "degrees_north" or "degrees_east".
4305  for (auto ircv = this->cvars.begin();
4306  ircv != this->cvars.end(); ++ircv) {
4307  if((*ircv)->cvartype == CV_EXIST) {
4308  for(auto ira = (*ircv)->attrs.begin();
4309  ira != (*ircv)->attrs.end();ira++) {
4310  string attr_name ="units";
4311  string lat_unit_value = "degrees_north";
4312  string lon_unit_value = "degrees_east";
4313 
4314  // Considering the cross-section case, either is fine.
4315  if((true == Is_Str_Attr(*ira,(*ircv)->fullpath,attr_name,lat_unit_value)) ||
4316  (true == Is_Str_Attr(*ira,(*ircv)->fullpath,attr_name,lon_unit_value))) {
4317  latlon_2d_cv_check2= false;
4318  break;
4319  }
4320  }
4321  }
4322 
4323  if(false == latlon_2d_cv_check2)
4324  break;
4325  }
4326  }
4327 
4328  bool latlon_2d_cv_check3 = true;
4329 
4330  // Even we cannot find the CF lat/lon attributes, we may still find lat/lon etc pairs.
4331  if(true == latlon_2d_cv_check1 && true == latlon_2d_cv_check2) {
4332 
4333  short latlon_flag = 0;
4334  short LatLon_flag = 0;
4335  short latilong_flag = 0;
4336 
4337  for (auto ircv = this->cvars.begin();
4338  ircv != this->cvars.end(); ++ircv) {
4339  if((*ircv)->cvartype == CV_EXIST) {
4340  if((*ircv)->name == "lat")
4341  latlon_flag++;
4342  else if((*ircv)->name == "lon")
4343  latlon_flag++;
4344  else if((*ircv)->name == "latitude")
4345  latilong_flag++;
4346  else if((*ircv)->name == "longitude")
4347  latilong_flag++;
4348  else if((*ircv)->name == "Latitude")
4349  LatLon_flag++;
4350  else if((*ircv)->name == "Longitude")
4351  LatLon_flag++;
4352  }
4353 
4354  }
4355  if((2== latlon_flag) || (2 == latilong_flag) || (2 == LatLon_flag ))
4356  latlon_2d_cv_check3 = false;
4357  }
4358 
4359  bool latlon_2d = false;
4360  short latlon_flag = 0;
4361  string latdim1,latdim2,londim1,londim2;
4362 
4363  short LatLon_flag = 0;
4364  string Latdim1,Latdim2,Londim1,Londim2;
4365 
4366  short latilong_flag = 0;
4367  string latidim1,latidim2,longdim1,longdim2;
4368 
4369 
4370  // Final check, we need to check if we have 2-D {lat/latitude/Latitude, lon/longitude/Longitude}
4371  // in the general variable list.
4372  // Here, depending on the future support, lat/lon pairs with other names(cell_lat,cell_lon etc) may be supported.
4373  // KY 2015-12-03
4374  if(true == latlon_2d_cv_check1 && true == latlon_2d_cv_check2 && true == latlon_2d_cv_check3) {
4375 
4376  for (auto irv = this->vars.begin();
4377  irv != this->vars.end(); ++irv) {
4378 
4379  //
4380  if((*irv)->rank == 2) {
4381  if((*irv)->name == "lat") {
4382  latlon_flag++;
4383  latdim1 = (*irv)->getDimensions()[0]->name;
4384  latdim2 = (*irv)->getDimensions()[1]->name;
4385 
4386  }
4387  else if((*irv)->name == "lon") {
4388  latlon_flag++;
4389  londim1 = (*irv)->getDimensions()[0]->name;
4390  londim2 = (*irv)->getDimensions()[1]->name;
4391  }
4392  else if((*irv)->name == "latitude"){
4393  latilong_flag++;
4394  latidim1 = (*irv)->getDimensions()[0]->name;
4395  latidim2 = (*irv)->getDimensions()[1]->name;
4396  }
4397  else if((*irv)->name == "longitude"){
4398  latilong_flag++;
4399  longdim1 = (*irv)->getDimensions()[0]->name;
4400  longdim2 = (*irv)->getDimensions()[1]->name;
4401 
4402  }
4403  else if((*irv)->name == "Latitude"){
4404  LatLon_flag++;
4405  Latdim1 = (*irv)->getDimensions()[0]->name;
4406  Latdim2 = (*irv)->getDimensions()[1]->name;
4407 
4408  }
4409  else if((*irv)->name == "Longitude"){
4410  LatLon_flag++;
4411  Londim1 = (*irv)->getDimensions()[0]->name;
4412  Londim2 = (*irv)->getDimensions()[1]->name;
4413  }
4414 
4415  }
4416  }
4417 
4418  // Here we ensure that only one lat/lon(lati/long,Lati/Long) is in the file.
4419  // If we find >=2 pairs lat/lon and latitude/longitude, or Latitude/Longitude,
4420  // we will not treat this as a 2-D latlon Dimscale case. The data producer
4421  // should correct their mistakes.
4422  if(2 == latlon_flag) {
4423  if((2 == latilong_flag) || ( 2 == LatLon_flag))
4424  latlon_2d = false;
4425  else if((latdim1 == londim1) && (latdim2 == londim2)) {
4426  latname = "lat";
4427  lonname = "lon";
4428  latlon_2d = true;
4429  }
4430  }
4431  else if ( 2 == latilong_flag) {
4432  if( 2 == LatLon_flag)
4433  latlon_2d = false;
4434  else if ((latidim1 == longdim1) ||(latidim2 == longdim2)) {
4435  latname = "latitude";
4436  lonname = "longitude";
4437  latlon_2d = true;
4438  }
4439  }
4440  else if (2 == LatLon_flag){
4441  if ((Latdim1 == Londim1) ||(Latdim2 == Londim2)) {
4442  latname = "Latitude";
4443  lonname = "Longitude";
4444  latlon_2d = true;
4445  }
4446  }
4447  }
4448 
4449  return latlon_2d;
4450 }
4451 
4452 
4453 // Update the coordinate variables for files that use HDF5 dimension scales and have 2-D lat/lon.
4454 void GMFile::Update_2DLatLon_Dimscale_CV(const string &latname,const string &lonname) {
4455 
4456  iscoard = false;
4457 
4458  // Update latitude.
4459  for (auto irv = this->vars.begin();
4460  irv != this->vars.end(); ++irv) {
4461 
4462  if((*irv)->rank == 2) {
4463 
4464  // Find 2-D latitude
4465  if((*irv)->name == latname) {
4466 
4467  // Obtain the first dimension of this variable
4468  string latdim0 = (*irv)->getDimensions()[0]->name;
4469 //cerr<<"latdim0 is "<<latdim0 <<endl;
4470 
4471  // Remove the CV corresponding to latdim0
4472  for (vector<GMCVar *>:: iterator i= this->cvars.begin(); i!=this->cvars.end(); ) {
4473  if((*i)->cfdimname == latdim0) {
4474  if(CV_FILLINDEX == (*i)->cvartype) {
4475  delete(*i);
4476  i = this->cvars.erase(i);
4477  }
4478  else if(CV_EXIST == (*i)->cvartype) {
4479  // Add this var. to the var list.
4480  Var *var = new Var(*i);
4481  this->vars.push_back(var);
4482  // Remove this var. from the GMCVar list.
4483  delete(*i);
4484  i = this->cvars.erase(i);
4485 
4486  }
4487  else {// the latdimname should be either the CV_FILLINDEX or CV_EXIST.
4488  if(CV_LAT_MISS == (*i)->cvartype)
4489  throw3("For the 2-D lat/lon case, the latitude dimension name ",latdim0, "is a coordinate variable of type CV_LAT_MISS");
4490  else if(CV_LON_MISS == (*i)->cvartype)
4491  throw3("For the 2-D lat/lon case, the latitude dimension name ",latdim0, "is a coordinate variable of type CV_LON_MISS");
4492  else if(CV_NONLATLON_MISS == (*i)->cvartype)
4493  throw3("For the 2-D lat/lon case, the latitude dimension name ",latdim0, "is a coordinate variable of type CV_NONLATLON_MISS");
4494  else if(CV_MODIFY == (*i)->cvartype)
4495  throw3("For the 2-D lat/lon case, the latitude dimension name ",latdim0, "is a coordinate variable of type CV_MODIFY");
4496  else if(CV_SPECIAL == (*i)->cvartype)
4497  throw3("For the 2-D lat/lon case, the latitude dimension name ",latdim0, "is a coordinate variable of type CV_SPECIAL");
4498  else
4499  throw3("For the 2-D lat/lon case, the latitude dimension name ",latdim0, "is a coordinate variable of type CV_UNSUPPORTED");
4500 
4501  }
4502  }
4503  else
4504  ++i;
4505  }
4506  // Add the 2-D latitude(latname) to the CV list.
4507  GMCVar* GMcvar = new GMCVar(*irv);
4508  GMcvar->cfdimname = latdim0;
4509  GMcvar->cvartype = CV_EXIST;
4510  GMcvar->product_type = product_type;
4511  this->cvars.push_back(GMcvar);
4512  delete(*irv);
4513  this->vars.erase(irv);
4514  break;
4515  }
4516  }
4517  }
4518 
4519  // Update longitude.
4520  for (auto irv = this->vars.begin();
4521  irv != this->vars.end(); ++irv) {
4522 
4523  if((*irv)->rank == 2) {
4524 
4525  // Find 2-D longitude
4526  if((*irv)->name == lonname) {
4527 
4528  // Obtain the second dimension of this variable
4529  string londim0 = (*irv)->getDimensions()[1]->name;
4530 
4531  // Remove the CV corresponding to londim0
4532  for (vector<GMCVar *>:: iterator i= this->cvars.begin(); i!=this->cvars.end(); ) {
4533  // NEED more work!!! should also remove ntime from the GMCVar list but add it to the cvar list.Same for Lon.
4534  if((*i)->cfdimname == londim0) {
4535  if(CV_FILLINDEX == (*i)->cvartype) {
4536  delete(*i);
4537  i= this->cvars.erase(i);
4538  }
4539  else if(CV_EXIST == (*i)->cvartype) {
4540  // Add this var. to the var list.
4541  Var *var = new Var(*i);
4542  this->vars.push_back(var);
4543  // Remove this var. from the GMCVar list.
4544  delete(*i);
4545  i = this->cvars.erase(i);
4546  }
4547  else {// the latdimname should be either the CV_FILLINDEX or CV_EXIST.
4548  if(CV_LAT_MISS == (*i)->cvartype)
4549  throw3("For the 2-D lat/lon case, the longitude dimension name ",londim0, "is a coordinate variable of type CV_LAT_MISS");
4550  else if(CV_LON_MISS == (*i)->cvartype)
4551  throw3("For the 2-D lat/lon case, the longitude dimension name ",londim0, "is a coordinate variable of type CV_LON_MISS");
4552  else if(CV_NONLATLON_MISS == (*i)->cvartype)
4553  throw3("For the 2-D lat/lon case, the longitude dimension name ",londim0, "is a coordinate variable of type CV_NONLATLON_MISS");
4554  else if(CV_MODIFY == (*i)->cvartype)
4555  throw3("For the 2-D lat/lon case, the longitude dimension name ",londim0, "is a coordinate variable of type CV_MODIFY");
4556  else if(CV_SPECIAL == (*i)->cvartype)
4557  throw3("For the 2-D lat/lon case, the longitude dimension name ",londim0, "is a coordinate variable of type CV_SPECIAL");
4558  else
4559  throw3("For the 2-D lat/lon case, the longitude dimension name ",londim0, "is a coordinate variable of type CV_UNSUPPORTED");
4560  }
4561  }
4562  else
4563  ++i;
4564  }
4565 
4566  // Add the 2-D longitude(lonname) to the CV list.
4567  GMCVar* GMcvar = new GMCVar(*irv);
4568  GMcvar->cfdimname = londim0;
4569  GMcvar->cvartype = CV_EXIST;
4570  GMcvar->product_type = product_type;
4571  this->cvars.push_back(GMcvar);
4572  delete(*irv);
4573  this->vars.erase(irv);
4574  break;
4575  }
4576  }
4577  }
4578 }
4579 #endif
4580 
4581 // Handle coordinate variables for general HDF5 products that have 1-D lat/lon
4582 void GMFile::Handle_CVar_LatLon1D_General_Product() {
4583 
4584  BESDEBUG("h5", "Coming to Handle_CVar_LatLon1D_General_Product()"<<endl);
4585  this->iscoard = true;
4586  Handle_CVar_LatLon_General_Product();
4587 
4588 }
4589 
4590 // Handle coordinate variables for general HDF5 products that have 2-D lat/lon
4591 void GMFile::Handle_CVar_LatLon2D_General_Product() {
4592 
4593  BESDEBUG("h5", "Coming to Handle_CVar_LatLon2D_General_Product()"<<endl);
4594  Handle_CVar_LatLon_General_Product();
4595 
4596 }
4597 
4598 // Routine to handle coordinate variables for general HDF5 product
4599 // that have either 1-D or 2-D lat/lon
4600 void GMFile::Handle_CVar_LatLon_General_Product() {
4601 
4602  BESDEBUG("h5", "Coming to Handle_CVar_LatLon_General_Product()"<<endl);
4603  if((GENERAL_LATLON2D != this->gproduct_pattern)
4604  && GENERAL_LATLON1D != this->gproduct_pattern)
4605  throw1("This function only supports latlon 1D or latlon 2D general products");
4606 
4607  pair<set<string>::iterator,bool> setret;
4608  set<string>tempdimnamelist = dimnamelist;
4609 
4610  for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
4611 
4612  // This is the dimension scale dataset; it should be changed to a coordinate variable.
4613  if (gp_latname== (*irv)->name) {
4614 
4615  // For latitude, regardless 1D or 2D, the first dimension needs to be updated.
4616  // Create Coordinate variables.
4617  tempdimnamelist.erase(((*irv)->dims[0])->name);
4618  auto GMcvar = new GMCVar(*irv);
4619  GMcvar->cfdimname = ((*irv)->dims[0])->name;
4620  GMcvar->cvartype = CV_EXIST;
4621  GMcvar->product_type = product_type;
4622  this->cvars.push_back(GMcvar);
4623  delete(*irv);
4624  this->vars.erase(irv);
4625  break;
4626  } // end of if
4627  } // end of for
4628 
4629  for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
4630 
4631  // This is the dimension scale dataset; it should be changed to a coordinate variable.
4632  if (gp_lonname== (*irv)->name) {
4633 
4634  // For 2-D lat/lon, the londimname should be the second dimension of the longitude
4635  // For 1-D lat/lon, the londimname should be the first dimension of the longitude
4636  // Create Coordinate variables.
4637  string londimname;
4638  if(GENERAL_LATLON2D == this->gproduct_pattern)
4639  londimname = ((*irv)->dims[1])->name;
4640  else
4641  londimname = ((*irv)->dims[0])->name;
4642 
4643  tempdimnamelist.erase(londimname);
4644  auto GMcvar = new GMCVar(*irv);
4645  GMcvar->cfdimname = londimname;
4646  GMcvar->cvartype = CV_EXIST;
4647  GMcvar->product_type = product_type;
4648  this->cvars.push_back(GMcvar);
4649  delete(*irv);
4650  this->vars.erase(irv);
4651  break;
4652  } // end of if
4653  } // end of for
4654 
4655  //
4656  // Add other missing coordinate variables.
4657  for (auto irs = tempdimnamelist.begin(); irs != tempdimnamelist.end();irs++) {
4658  auto GMcvar = new GMCVar();
4659  Create_Missing_CV(GMcvar,*irs);
4660  this->cvars.push_back(GMcvar);
4661  }
4662 
4663 }
4664 
4665 // Handle coordinate variables for OBPG level 3
4666 void GMFile::Handle_CVar_OBPG_L3() {
4667 
4668  BESDEBUG("h5", "Coming to Handle_CVar_OBPG_L3()"<<endl);
4669  if (GENERAL_DIMSCALE == this->gproduct_pattern)
4670  Handle_CVar_Dimscale_General_Product();
4671 
4672  // Change the CV Type of the corresponding CVs of lat and lon from CV_FILLINDEX to CV_LATMISS or CV_LONMISS
4673  for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
4674 
4675  // Here I try to avoid using the dimension name row and column to find the lat/lon dimension size.
4676  // So I am looking for a 2-D floating-point array or a 2-D array under the group geophsical_data.
4677  // This may be subject to change if OBPG level 3 change its arrangement of variables.
4678  // KY 2014-09-29
4679 
4680  if((*irv)->rank == 2) {
4681 
4682  if(((*irv)->fullpath.find("/geophsical_data") == 0) || ((*irv)->dtype == H5FLOAT32)) {
4683 
4684  size_t lat_size = (*irv)->getDimensions()[0]->size;
4685  string lat_name = (*irv)->getDimensions()[0]->name;
4686  size_t lon_size = (*irv)->getDimensions()[1]->size;
4687  string lon_name = (*irv)->getDimensions()[1]->name;
4688  size_t temp_size = 0;
4689  string temp_name;
4690  H5DataType ll_dtype = (*irv)->dtype;
4691 
4692  // We always assume that longitude size is greater than latitude size.
4693  if(lat_size >lon_size) {
4694  temp_size = lon_size;
4695  temp_name = lon_name;
4696  lon_size = lat_size;
4697  lon_name = lat_name;
4698  lat_size = temp_size;
4699  lat_name = temp_name;
4700  }
4701  for (auto ircv = this->cvars.begin(); ircv != this->cvars.end(); ++ircv) {
4702  if((*ircv)->cvartype == CV_FILLINDEX) {
4703  if((*ircv)->getDimensions()[0]->size == lat_size &&
4704  (*ircv)->getDimensions()[0]->name == lat_name) {
4705  (*ircv)->cvartype = CV_LAT_MISS;
4706  (*ircv)->dtype = ll_dtype;
4707  for (auto ira = (*ircv)->attrs.begin(); ira != (*ircv)->attrs.end(); ++ira) {
4708  if ((*ira)->name == "NAME") {
4709  delete (*ira);
4710  (*ircv)->attrs.erase(ira);
4711  break;
4712  }
4713  }
4714  }
4715  else if((*ircv)->getDimensions()[0]->size == lon_size &&
4716  (*ircv)->getDimensions()[0]->name == lon_name) {
4717  (*ircv)->cvartype = CV_LON_MISS;
4718  (*ircv)->dtype = ll_dtype;
4719  for (auto ira = (*ircv)->attrs.begin(); ira != (*ircv)->attrs.end(); ++ira) {
4720  if ((*ira)->name == "NAME") {
4721  delete (*ira);
4722  (*ircv)->attrs.erase(ira);
4723  break;
4724  }
4725  }
4726  }
4727  }
4728  }
4729  break;
4730 
4731  } // end of if(((*irv)->fullpath.find("/geophsical_data") == 0) || ((*irv)->dtype == H5FLOAT32))
4732  } // end of if((*irv)->rank == 2)
4733  } // end of for (auto irv = this->vars.begin();
4734 
4735 }
4736 
4737 // Handle some special variables. Currently only GPM and ACOS have these variables.
4739 
4740  BESDEBUG("h5", "Coming to Handle_SpVar()"<<endl);
4741  if (ACOS_L2S_OR_OCO2_L1B == product_type)
4742  Handle_SpVar_ACOS_OCO2();
4743  else if(GPM_L1 == product_type) {
4744  // Loop through the variable list to build the coordinates.
4745  // These variables need to be removed.
4746  for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
4747  if((*irv)->name=="AlgorithmRuntimeInfo") {
4748  delete(*irv);
4749  this->vars.erase(irv);
4750  break;
4751  }
4752  }
4753  }
4754 
4755  // GPM level-3 These variables need to be removed.
4756  else if(GPMM_L3 == product_type || GPMS_L3 == product_type || GPM_L3_New==product_type) {
4757 
4758  for (auto irv = this->vars.begin(); irv != this->vars.end(); ) {
4759  if((*irv)->name=="InputFileNames") {
4760  delete(*irv);
4761  irv = this->vars.erase(irv);
4762  }
4763  else if((*irv)->name=="InputAlgorithmVersions") {
4764  delete(*irv);
4765  irv = this->vars.erase(irv);
4766  }
4767  else if((*irv)->name=="InputGenerationDateTimes") {
4768  delete(*irv);
4769  irv = this->vars.erase(irv);
4770  }
4771  else {
4772  ++irv;
4773  }
4774  }
4775  }
4776 
4777 }
4778 
4779 // Handle special variables for ACOS.
4780 void GMFile::Handle_SpVar_ACOS_OCO2() {
4781 
4782  BESDEBUG("h5", "Coming to Handle_SpVar_ACOS_OCO2()"<<endl);
4783  //The ACOS or OCO2 have 64-bit variables. DAP2 doesn't support 64-bit variables.
4784  // So we will not handle attributes yet.
4785  for (auto irv = this->vars.begin(); irv != this->vars.end(); ) {
4786 
4787  if (H5INT64 == (*irv)->getType()) {
4788 
4789  // First: Time Part of soundingid
4790  auto spvar = new GMSPVar(*irv);
4791  spvar->name = (*irv)->name +"_Time";
4792  spvar->newname = (*irv)->newname+"_Time";
4793  spvar->dtype = H5INT32;
4794  spvar->otype = (*irv)->getType();
4795  spvar->sdbit = 1;
4796 
4797  // 2 digit hour, 2 digit min, 2 digit seconds
4798  spvar->numofdbits = 6;
4799  this->spvars.push_back(spvar);
4800 
4801  // Second: Date Part of soundingid
4802  auto spvar2 = new GMSPVar(*irv);
4803  spvar2->name = (*irv)->name +"_Date";
4804  spvar2->newname = (*irv)->newname+"_Date";
4805  spvar2->dtype = H5INT32;
4806  spvar2->otype = (*irv)->getType();
4807  spvar2->sdbit = 7;
4808 
4809  // 4 digit year, 2 digit month, 2 digit day
4810  spvar2->numofdbits = 8;
4811  this->spvars.push_back(spvar2);
4812 
4813  delete(*irv);
4814  irv = this->vars.erase(irv);
4815  } // end of if (H5INT64 == (*irv)->getType())
4816  else {
4817  ++irv;
4818  }
4819  } // end of for (auto irv = this->vars.begin(); ...
4820 }
4821 
4822 // Adjust Object names, For some products, NASA data centers don't need
4823 // the fullpath of objects.
4825 
4826  BESDEBUG("h5", "Coming to Adjust_Obj_Name()"<<endl);
4827  if(Mea_Ozone == product_type)
4828  Adjust_Mea_Ozone_Obj_Name();
4829 
4830  if(GPMS_L3 == product_type || GPMM_L3 == product_type)
4831  Adjust_GPM_L3_Obj_Name();
4832 
4833 // Just for debugging
4834 #if 0
4835 for (auto irv2 = this->vars.begin();
4836  irv2 != this->vars.end(); irv2++) {
4837  for (auto ird = (*irv2)->dims.begin();
4838  ird !=(*irv2)->dims.end(); ird++) {
4839  cerr<<"Dimension name afet Adjust_Obj_Name "<<(*ird)->newname <<endl;
4840  }
4841 }
4842 #endif
4843 
4844 }
4845 
4846 // Adjust object names for GPM level 3 products
4847 void GMFile:: Adjust_GPM_L3_Obj_Name() {
4848 
4849  BESDEBUG("h5", "Coming to Adjust_GPM_L3_Obj_Name()"<<endl);
4850  string objnewname;
4851  // In this definition, root group is not considered as a group.
4852  if(this->groups.size() <= 1) {
4853  for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
4854  objnewname = HDF5CFUtil::obtain_string_after_lastslash((*irv)->newname);
4855  if (objnewname !="")
4856  (*irv)->newname = objnewname;
4857  }
4858  }
4859  else {
4860  for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
4861  size_t grid_group_path_pos = ((*irv)->newname.substr(1)).find_first_of("/");
4862  objnewname = ((*irv)->newname).substr(grid_group_path_pos+2);
4863  (*irv)->newname = objnewname;
4864  }
4865  }
4866 }
4867 
4868 // Adjust object names for MeaSUREs OZone
4869 void GMFile:: Adjust_Mea_Ozone_Obj_Name() {
4870 
4871  BESDEBUG("h5", "Coming to Adjust_Mea_Ozone_Obj_Name()"<<endl);
4872  string objnewname;
4873  for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
4874  objnewname = HDF5CFUtil::obtain_string_after_lastslash((*irv)->newname);
4875  if (objnewname !="")
4876  (*irv)->newname = objnewname;
4877 
4878 #if 0
4879 //Just for debugging
4880 for (auto ird = (*irv)->dims.begin();
4881  ird !=(*irv)->dims.end();++ird) {
4882  cerr<<"Ozone dim. name "<<(*ird)->name <<endl;
4883  cerr<<"Ozone dim. new name "<<(*ird)->newname <<endl;
4884 }
4885 #endif
4886 
4887  }
4888 
4889  for (auto irv = this->cvars.begin(); irv != this->cvars.end(); ++irv) {
4890  objnewname = HDF5CFUtil::obtain_string_after_lastslash((*irv)->newname);
4891  if (objnewname !="")
4892  (*irv)->newname = objnewname;
4893 #if 0
4894  //Just for debugging
4895 for (auto ird = (*irv)->dims.begin();
4896  ird !=(*irv)->dims.end();++ird) {
4897  cerr<<"Ozone CV dim. name "<<(*ird)->name <<endl;
4898  cerr<<"Ozone CV dim. new name "<<(*ird)->newname <<endl;
4899 }
4900 #endif
4901  }
4902 }
4903 
4904 // Flatten object names.
4905 void GMFile::Flatten_Obj_Name(bool include_attr) {
4906 
4907  BESDEBUG("h5", "GMFile::Coming to Flatten_Obj_Name()"<<endl);
4908  // General variables
4909  File::Flatten_Obj_Name(include_attr);
4910 
4911  // Coordinate variables
4912  for (auto irv = this->cvars.begin(); irv != this->cvars.end(); ++irv) {
4913  (*irv)->newname = get_CF_string((*irv)->newname);
4914 
4915  for (auto ird = (*irv)->dims.begin(); ird != (*irv)->dims.end(); ++ird)
4916  (*ird)->newname = get_CF_string((*ird)->newname);
4917 
4918  if (true == include_attr) {
4919  for (auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira)
4920  (*ira)->newname = File::get_CF_string((*ira)->newname);
4921  }
4922 
4923  }
4924 
4925  // Special variables
4926  for (auto irv = this->spvars.begin(); irv != this->spvars.end(); ++irv) {
4927  (*irv)->newname = get_CF_string((*irv)->newname);
4928 
4929  for (auto ird = (*irv)->dims.begin(); ird != (*irv)->dims.end(); ++ird)
4930  (*ird)->newname = get_CF_string((*ird)->newname);
4931 
4932  if (true == include_attr) {
4933  for (auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira)
4934  (*ira)->newname = File::get_CF_string((*ira)->newname);
4935  }
4936  }
4937 
4938 // Just for debugging
4939 #if 0
4940 for (auto irv2 = this->vars.begin();
4941  irv2 != this->vars.end(); irv2++) {
4942  for (auto ird = (*irv2)->dims.begin();
4943  ird !=(*irv2)->dims.end(); ird++) {
4944  cerr<<"Dimension name afet Flatten_Obj_Name "<<(*ird)->newname <<endl;
4945  }
4946 }
4947 #endif
4948 
4949 
4950 }
4951 
4952 // Rarely object name clashings may occur. This routine makes sure
4953 // all object names are unique.
4954 void GMFile::Handle_Obj_NameClashing(bool include_attr) {
4955 
4956  BESDEBUG("h5", "GMFile::Coming to Handle_Obj_NameClashing()"<<endl);
4957  // objnameset will be filled with all object names that we are going to check the name clashing.
4958  // For example, we want to see if there are any name clashings for all variable names in this file.
4959  // objnameset will include all variable names. If a name clashing occurs, we can figure out from the set operation immediately.
4960  set<string>objnameset;
4961  Handle_GMCVar_NameClashing(objnameset);
4962  Handle_GMSPVar_NameClashing(objnameset);
4963  File::Handle_GeneralObj_NameClashing(include_attr,objnameset);
4964  if (true == include_attr) {
4965  Handle_GMCVar_AttrNameClashing();
4966  Handle_GMSPVar_AttrNameClashing();
4967  }
4968  // Moving to h5gmcfdap.cc, right after Adjust_Dim_Name
4969  //Handle_DimNameClashing();
4970 }
4971 
4972 // Name clashings for coordinate variables
4973 void GMFile::Handle_GMCVar_NameClashing(set<string> &objnameset ) {
4974 
4975  GMHandle_General_NameClashing(objnameset,this->cvars);
4976 }
4977 
4978 // Name clashings for special variables(like 64-bit integer variables)
4979 void GMFile::Handle_GMSPVar_NameClashing(set<string> &objnameset ) {
4980 
4981  GMHandle_General_NameClashing(objnameset,this->spvars);
4982 }
4983 
4984 // This routine handles attribute name clashings for coordinate variables.
4985 void GMFile::Handle_GMCVar_AttrNameClashing() {
4986 
4987  BESDEBUG("h5", "Coming to Handle_GMCVar_AttrNameClashing()"<<endl);
4988  set<string> objnameset;
4989 
4990  for (auto irv = this->cvars.begin(); irv != this->cvars.end(); ++irv) {
4991  Handle_General_NameClashing(objnameset,(*irv)->attrs);
4992  objnameset.clear();
4993  }
4994 }
4995 
4996 // Attribute name clashings for special variables
4997 void GMFile::Handle_GMSPVar_AttrNameClashing() {
4998 
4999  BESDEBUG("h5", "Coming to Handle_GMSPVar_AttrNameClashing()"<<endl);
5000  set<string> objnameset;
5001 
5002  for (auto irv = this->spvars.begin(); irv != this->spvars.end(); ++irv) {
5003  Handle_General_NameClashing(objnameset,(*irv)->attrs);
5004  objnameset.clear();
5005  }
5006 }
5007 
5008 //class T must have member string newname
5009 // The subroutine to handle name clashings,
5010 // it builds up a map from original object names to clashing-free object names.
5011 template<class T> void
5012 GMFile::GMHandle_General_NameClashing(set <string>&objnameset, vector<T*>& objvec) {
5013 
5014  BESDEBUG("h5", "Coming to GMHandle_General_NameClashing()"<<endl);
5015  pair<set<string>::iterator,bool> setret;
5016  set<string>::iterator iss;
5017 
5018  vector<string> clashnamelist;
5019 
5020  map<int,int> cl_to_ol;
5021  int ol_index = 0;
5022  int cl_index = 0;
5023 
5024  typename vector<T*>::iterator irv;
5025 
5026  for (irv = objvec.begin(); irv != objvec.end(); ++irv) {
5027 
5028  setret = objnameset.insert((*irv)->newname);
5029  if (false == setret.second ) {
5030  clashnamelist.insert(clashnamelist.end(),(*irv)->newname);
5031  cl_to_ol[cl_index] = ol_index;
5032  cl_index++;
5033  }
5034  ol_index++;
5035  }
5036 
5037 
5038  // Now change the clashed elements to unique elements;
5039  // Generate the set which has the same size as the original vector.
5040  for (auto ivs=clashnamelist.begin(); ivs!=clashnamelist.end(); ++ivs) {
5041  int clash_index = 1;
5042  string temp_clashname = *ivs +'_';
5043  HDF5CFUtil::gen_unique_name(temp_clashname,objnameset,clash_index);
5044  *ivs = temp_clashname;
5045  }
5046 
5047 
5048  // Now go back to the original vector, make it unique.
5049  for (unsigned int i =0; i <clashnamelist.size(); i++)
5050  objvec[cl_to_ol[i]]->newname = clashnamelist[i];
5051 
5052 }
5053 
5054 // Handle dimension name clashings
5056 
5057 
5058  BESDEBUG("h5", "GMFile: Coming to Handle_DimNameClashing()"<<endl);
5059  // ACOS L2S or OCO2 L1B products doesn't need the dimension name clashing check based on our current understanding. KY 2012-5-16
5060  if (ACOS_L2S_OR_OCO2_L1B == product_type)
5061  return;
5062 
5063  map<string,string>dimname_to_dimnewname;
5064  pair<map<string,string>::iterator,bool>mapret;
5065  set<string> dimnameset;
5066  vector<Dimension*>vdims;
5067  set<string> dimnewnameset;
5068  pair<set<string>::iterator,bool> setret;
5069 
5070  // First: Generate the dimset/dimvar based on coordinate variables.
5071  for (auto irv = this->cvars.begin(); irv !=this->cvars.end(); ++irv) {
5072  for (auto ird = (*irv)->dims.begin(); ird !=(*irv)->dims.end();++ird) {
5073  setret = dimnameset.insert((*ird)->name);
5074  if (true == setret.second)
5075  vdims.push_back(*ird);
5076  }
5077  }
5078 
5079  // For some cases, dimension names are provided but there are no corresponding coordinate
5080  // variables. For now, we will assume no such cases.
5081  // Actually, we find such a case in our fake testsuite. So we need to fix it.
5082  for(auto irv= this->vars.begin(); irv != this->vars.end();++irv) {
5083  for (auto ird = (*irv)->dims.begin(); ird !=(*irv)->dims.end();++ird) {
5084 #if 0
5085  //setret = dimnameset.insert((*ird)->newname);
5086 #endif
5087  setret = dimnameset.insert((*ird)->name);
5088  if (setret.second) vdims.push_back(*ird);
5089  }
5090  }
5091 
5092  GMHandle_General_NameClashing(dimnewnameset,vdims);
5093 
5094  // Third: Make dimname_to_dimnewname map
5095  for (auto ird = vdims.begin();ird!=vdims.end();++ird) {
5096  mapret = dimname_to_dimnewname.insert(pair<string,string>((*ird)->name,(*ird)->newname));
5097  if (false == mapret.second)
5098  throw4("The dimension name ",(*ird)->name," should map to ",
5099  (*ird)->newname);
5100  }
5101 
5102  // Fourth: Change the original dimension new names to the unique dimension new names
5103  for (auto irv = this->cvars.begin(); irv !=this->cvars.end(); ++irv)
5104  for (auto ird = (*irv)->dims.begin(); ird!=(*irv)->dims.end();++ird)
5105  (*ird)->newname = dimname_to_dimnewname[(*ird)->name];
5106 
5107  for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv)
5108  for (auto ird = (*irv)->dims.begin(); ird !=(*irv)->dims.end();++ird)
5109  (*ird)->newname = dimname_to_dimnewname[(*ird)->name];
5110 
5111 }
5112 
5113 // For COARDS, dim. names need to be the same as obj. names.
5115 
5116  BESDEBUG("h5", "GMFile:Coming to Adjust_Dim_Name()"<<endl);
5117 #if 0
5118  // Just for debugging
5119 for (auto irv2 = this->vars.begin();
5120  irv2 != this->vars.end(); irv2++) {
5121  for (auto ird = (*irv2)->dims.begin();
5122  ird !=(*irv2)->dims.end(); ird++) {
5123  cerr<<"Dimension new name "<<(*ird)->newname <<endl;
5124  }
5125 }
5126 #endif
5127 
5128  // Only need for COARD conventions.
5129  if( true == iscoard) {
5130  for (auto irv = this->cvars.begin(); irv !=this->cvars.end(); ++irv) {
5131 #if 0
5132 cerr<<"1D Cvariable name is "<<(*irv)->name <<endl;
5133 cerr<<"1D Cvariable new name is "<<(*irv)->newname <<endl;
5134 cerr<<"1D Cvariable dim name is "<<((*irv)->dims)[0]->name <<endl;
5135 cerr<<"1D Cvariable dim new name is "<<((*irv)->dims)[0]->newname <<endl;
5136 #endif
5137  if ((*irv)->dims.size()!=1)
5138  throw3("Coard coordinate variable ",(*irv)->name, "is not 1D");
5139  if ((*irv)->newname != (((*irv)->dims)[0]->newname)) {
5140  ((*irv)->dims)[0]->newname = (*irv)->newname;
5141 
5142  // For all variables that have this dimension,the dimension newname should also change.
5143  for (auto irv2 = this->vars.begin(); irv2 != this->vars.end(); ++irv2) {
5144  for (auto ird = (*irv2)->dims.begin(); ird !=(*irv2)->dims.end(); ++ird) {
5145  // This is the key, the dimension name of this dimension
5146  // should be equal to the dimension name of the coordinate variable.
5147  // Then the dimension name matches and the dimension name should be changed to
5148  // the new dimension name.
5149  if ((*ird)->name == ((*irv)->dims)[0]->name)
5150  (*ird)->newname = ((*irv)->dims)[0]->newname;
5151  }
5152  }
5153  } // end of if ((*irv)->newname != (((*irv)->dims)[0]->newname))
5154  }// end of for (auto irv = this->cvars.begin(); ...
5155  } // end of if( true == iscoard)
5156 
5157 // Just for debugging
5158 #if 0
5159 for (auto irv2 = this->vars.begin();
5160  irv2 != this->vars.end(); irv2++) {
5161  for (auto ird = (*irv2)->dims.begin();
5162  ird !=(*irv2)->dims.end(); ird++) {
5163  cerr<<"Dimension name afet Adjust_Dim_Name "<<(*ird)->newname <<endl;
5164  }
5165 }
5166 #endif
5167 
5168 
5169 }
5170 
5171 // Add supplemental CF attributes for some products.
5172 void
5174 
5175  BESDEBUG("h5", "GMFile::Coming to Add_Supplement_Attrs()"<<endl);
5176  if (General_Product == product_type || true == add_path) {
5177  File::Add_Supplement_Attrs(add_path);
5178 
5179  if(true == add_path) {
5180  // Adding variable original name(origname) and full path(fullpath)
5181  for (auto irv = this->cvars.begin(); irv != this->cvars.end(); ++irv) {
5182  if (((*irv)->cvartype == CV_EXIST) || ((*irv)->cvartype == CV_MODIFY)) {
5183  auto attr = new Attribute();
5184  const string varname = (*irv)->name;
5185  const string attrname = "origname";
5186  Add_Str_Attr(attr,attrname,varname);
5187  (*irv)->attrs.push_back(attr);
5188  }
5189  }
5190 
5191  for (auto irv = this->cvars.begin();
5192  irv != this->cvars.end(); ++irv) {
5193  // Turn off the fullnamepath attribute when zero_storage_size is 0.
5194  // Use the BES key since quite a few testing cases will be affected.
5195  // KY 2020-03-23
5196  if((*irv)->zero_storage_size == false
5197  || HDF5RequestHandler::get_no_zero_size_fullnameattr() == false) {
5198  if (((*irv)->cvartype == CV_EXIST) || ((*irv)->cvartype == CV_MODIFY)) {
5199  auto attr = new Attribute();
5200  const string varname = (*irv)->fullpath;
5201  const string attrname = "fullnamepath";
5202  Add_Str_Attr(attr,attrname,varname);
5203  (*irv)->attrs.push_back(attr);
5204  }
5205  }
5206  }
5207 
5208  for (auto irv = this->spvars.begin(); irv != this->spvars.end(); ++irv) {
5209  auto attr = new Attribute();
5210  const string varname = (*irv)->name;
5211  const string attrname = "origname";
5212  Add_Str_Attr(attr,attrname,varname);
5213  (*irv)->attrs.push_back(attr);
5214  }
5215 
5216  for (auto irv = this->spvars.begin(); irv != this->spvars.end(); ++irv) {
5217  // Turn off the fullnamepath attribute when zero_storage_size is 0.
5218  // Use the BES key since quite a few testing cases will be affected.
5219  // KY 2020-03-23
5220  if((*irv)->zero_storage_size == false
5221  || HDF5RequestHandler::get_no_zero_size_fullnameattr() == false) {
5222  auto attr = new Attribute();
5223  const string varname = (*irv)->fullpath;
5224  const string attrname = "fullnamepath";
5225  Add_Str_Attr(attr,attrname,varname);
5226  (*irv)->attrs.push_back(attr);
5227  }
5228  }
5229  }
5230  } // if (General_Product == product_type || true == add_path)
5231 
5232  if(GPM_L1 == product_type || GPMS_L3 == product_type || GPMM_L3 == product_type)
5233  Add_GPM_Attrs();
5234  else if (Aqu_L3 == product_type)
5235  Add_Aqu_Attrs();
5236  else if (Mea_SeaWiFS_L2 == product_type || Mea_SeaWiFS_L3 == product_type)
5237  Add_SeaWiFS_Attrs();
5238 
5239 }
5240 
5241 // Add CF attributes for GPM products
5242 void
5243 GMFile:: Add_GPM_Attrs() {
5244 
5245  BESDEBUG("h5", "Coming to Add_GPM_Attrs()"<<endl);
5246  vector<HDF5CF::Var *>::const_iterator it_v;
5247  vector<HDF5CF::Attribute *>::const_iterator ira;
5248  const string attr_name_be_replaced = "CodeMissingValue";
5249  const string attr_new_name = "_FillValue";
5250  const string attr2_name_be_replaced = "Units";
5251  const string attr2_new_name ="units";
5252 
5253  // Need to convert String type CodeMissingValue to the corresponding _FilLValue
5254  // Create a function at HDF5CF.cc. use strtod,strtof,strtol etc. function to convert
5255  // string to the corresponding type.
5256  for (it_v = vars.begin(); it_v != vars.end(); ++it_v) {
5257  bool has_fvalue_attr = false;
5258  for(ira = (*it_v)->attrs.begin(); ira!= (*it_v)->attrs.end();ira++) {
5259  if(attr_new_name == (*ira)->name) {
5260  has_fvalue_attr = true;
5261  break;
5262  }
5263  }
5264 
5265  if(false == has_fvalue_attr) {
5266  for(ira = (*it_v)->attrs.begin(); ira!= (*it_v)->attrs.end();ira++) {
5267  if(attr_name_be_replaced == (*ira)->name) {
5268  if((*ira)->dtype == H5FSTRING)
5269  Change_Attr_One_Str_to_Others((*ira),(*it_v));
5270  (*ira)->name = attr_new_name;
5271  (*ira)->newname = attr_new_name;
5272  }
5273  }
5274  }
5275 
5276  }
5277 
5278 
5279  for (auto irv = this->cvars.begin();
5280  irv != this->cvars.end(); ++irv) {
5281  bool has_fvalue_attr = false;
5282 
5283  for(ira = (*irv)->attrs.begin(); ira!= (*irv)->attrs.end();ira++) {
5284 
5285  if(attr_new_name == (*ira)->name) {
5286  has_fvalue_attr = true;
5287  break;
5288  }
5289  }
5290  if(false == has_fvalue_attr) {
5291  for(ira = (*irv)->attrs.begin(); ira!= (*irv)->attrs.end();ira++) {
5292 
5293  if(attr_name_be_replaced == (*ira)->name) {
5294  if((*ira)->dtype == H5FSTRING)
5295  Change_Attr_One_Str_to_Others((*ira),(*irv));
5296  (*ira)->name = attr_new_name;
5297  (*ira)->newname = attr_new_name;
5298  break;
5299  }
5300  }
5301  }
5302 
5303 
5304  if(product_type == GPM_L1) {
5305 
5306  if ((*irv)->cvartype == CV_EXIST) {
5307  if((*irv)->name.find("Latitude") !=string::npos) {
5308  string unit_value = "degrees_north";
5309  Correct_GPM_L1_LatLon_units(*irv,unit_value);
5310 
5311  }
5312  else if((*irv)->name.find("Longitude") !=string::npos) {
5313  string unit_value = "degrees_east";
5314  Correct_GPM_L1_LatLon_units(*irv,unit_value);
5315  }
5316  }
5317 
5318 
5319  else if ((*irv)->cvartype == CV_NONLATLON_MISS) {
5320 
5321  string comment;
5322  const string attrname = "comment";
5323  auto attr = new Attribute();
5324 
5325  {
5326  if((*irv)->name == "nchannel1")
5327  comment = "Number of Swath S1 channels (10V 10H 19V 19H 23V 37V 37H 89V 89H).";
5328  else if((*irv)->name == "nchannel2")
5329  comment = "Number of Swath S2 channels (166V 166H 183+/-3V 183+/-8V).";
5330  else if((*irv)->name == "nchan1")
5331  comment = "Number of channels in Swath 1.";
5332  else if((*irv)->name == "nchan2")
5333  comment = "Number of channels in Swath 2.";
5334  else if((*irv)->name == "VH")
5335  comment = "Number of polarizations.";
5336  else if((*irv)->name == "GMIxyz")
5337  comment = "x, y, z components in GMI instrument coordinate system.";
5338  else if((*irv)->name == "LNL")
5339  comment = "Linear and non-linear.";
5340  else if((*irv)->name == "nscan")
5341  comment = "Number of scans in the granule.";
5342  else if((*irv)->name == "nscan1")
5343  comment = "Typical number of Swath S1 scans in the granule.";
5344  else if((*irv)->name == "nscan2")
5345  comment = "Typical number of Swath S2 scans in the granule.";
5346  else if((*irv)->name == "npixelev")
5347  comment = "Number of earth view pixels in one scan.";
5348  else if((*irv)->name == "npixelht")
5349  comment = "Number of hot load pixels in one scan.";
5350  else if((*irv)->name == "npixelcs")
5351  comment = "Number of cold sky pixels in one scan.";
5352  else if((*irv)->name == "npixelfr")
5353  comment = "Number of full rotation earth view pixels in one scan.";
5354  else if((*irv)->name == "nfreq1")
5355  comment = "Number of frequencies in Swath 1.";
5356  else if((*irv)->name == "nfreq2")
5357  comment = "Number of frequencies in Swath 2.";
5358  else if((*irv)->name == "npix1")
5359  comment = "Number of pixels in Swath 1.";
5360  else if((*irv)->name == "npix2")
5361  comment = "Number of pixels in Swath 2.";
5362  else if((*irv)->name == "npix3")
5363  comment = "Number of pixels in Swath 3.";
5364  else if((*irv)->name == "npix4")
5365  comment = "Number of pixels in Swath 4.";
5366  else if((*irv)->name == "ncolds1")
5367  comment = "Maximum number of cold samples in Swath 1.";
5368  else if((*irv)->name == "ncolds2")
5369  comment = "Maximum number of cold samples in Swath 2.";
5370  else if((*irv)->name == "nhots1")
5371  comment = "Maximum number of hot samples in Swath 1.";
5372  else if((*irv)->name == "nhots2")
5373  comment = "Maximum number of hot samples in Swath 2.";
5374  else if((*irv)->name == "ntherm")
5375  comment = "Number of hot load thermisters.";
5376  else if((*irv)->name == "ntach")
5377  comment = "Number of tachometer readings.";
5378  else if((*irv)->name == "nsamt"){
5379  comment = "Number of sample types. ";
5380  comment = +"The types are: total science GSDR, earthview,hot load, cold sky.";
5381  }
5382  else if((*irv)->name == "nndiode")
5383  comment = "Number of noise diodes.";
5384  else if((*irv)->name == "n7")
5385  comment = "Number seven.";
5386  else if((*irv)->name == "nray")
5387  comment = "Number of angle bins in each NS scan.";
5388  else if((*irv)->name == "nrayMS")
5389  comment = "Number of angle bins in each MS scan.";
5390  else if((*irv)->name == "nrayHS")
5391  comment = "Number of angle bins in each HS scan.";
5392  else if((*irv)->name == "nbin")
5393  comment = "Number of range bins in each NS and MS ray. Bin interval is 125m.";
5394  else if((*irv)->name == "nbinHS")
5395  comment = "Number of range bins in each HS ray. Bin interval is 250m.";
5396  else if((*irv)->name == "nbinSZP")
5397  comment = "Number of range bins for sigmaZeroProfile.";
5398  else if((*irv)->name == "nbinSZPHS")
5399  comment = "Number of range bins for sigmaZeroProfile in each HS scan.";
5400  else if((*irv)->name == "nNP")
5401  comment = "Number of NP kinds.";
5402  else if((*irv)->name == "nearFar")
5403  comment = "Near reference, Far reference.";
5404  else if((*irv)->name == "foreBack")
5405  comment = "Forward, Backward.";
5406  else if((*irv)->name == "method")
5407  comment = "Number of SRT methods.";
5408  else if((*irv)->name == "nNode")
5409  comment = "Number of binNode.";
5410  else if((*irv)->name == "nDSD")
5411  comment = "Number of DSD parameters. Parameters are N0 and D0";
5412  else if((*irv)->name == "LS")
5413  comment = "Liquid, solid.";
5414  }
5415 
5416  if(""==comment)
5417  delete attr;
5418  else {
5419  Add_Str_Attr(attr,attrname,comment);
5420  (*irv)->attrs.push_back(attr);
5421  }
5422 
5423  }
5424  }
5425 
5426  if(product_type == GPMS_L3 || product_type == GPMM_L3) {
5427  if ((*irv)->cvartype == CV_NONLATLON_MISS) {
5428 
5429  string comment;
5430  const string attrname = "comment";
5431  auto attr = new Attribute();
5432 
5433  {
5434  if((*irv)->name == "chn")
5435  comment = "Number of channels:Ku,Ka,KaHS,DPR.";
5436  else if((*irv)->name == "inst")
5437  comment = "Number of instruments:Ku,Ka,KaHS.";
5438  else if((*irv)->name == "tim")
5439  comment = "Number of hours(local time).";
5440  else if((*irv)->name == "ang"){
5441  comment = "Number of angles.The meaning of ang is different for each channel.";
5442  comment +=
5443  "For Ku channel all indices are used with the meaning 0,1,2,..6 =angle bins 24,";
5444  comment +=
5445  "(20,28),(16,32),(12,36),(8,40),(3,44),and (0,48).";
5446  comment +=
5447  "For Ka channel 4 indices are used with the meaning 0,1,2,3 = angle bins 12,(8,16),";
5448  comment +=
5449  "(4,20),and (0,24). For KaHS channel 4 indices are used with the meaning 0,1,2,3 =";
5450  comment += "angle bins(11,2),(7,16),(3,20),and (0.23).";
5451 
5452  }
5453  else if((*irv)->name == "rt")
5454  comment = "Number of rain types: stratiform, convective,all.";
5455  else if((*irv)->name == "st")
5456  comment = "Number of surface types:ocean,land,all.";
5457  else if((*irv)->name == "bin"){
5458  comment = "Number of bins in histogram. The thresholds are different for different";
5459  comment +=" variables. see the file specification for this algorithm.";
5460  }
5461  else if((*irv)->name == "nvar") {
5462  comment = "Number of phase bins. Bins are counts of phase less than 100, ";
5463  comment +="counts of phase greater than or equal to 100 and less than 200, ";
5464  comment +="counts of phase greater than or equal to 200.";
5465  }
5466  else if((*irv)->name == "AD")
5467  comment = "Ascending or descending half of the orbit.";
5468  }
5469 
5470  if(""==comment)
5471  delete attr;
5472  else {
5473  Add_Str_Attr(attr,attrname,comment);
5474  (*irv)->attrs.push_back(attr);
5475  }
5476 
5477  }
5478  }
5479 
5480 
5481  if ((*irv)->cvartype == CV_SPECIAL) {
5482  if((*irv)->name == "nlayer" || (*irv)->name == "hgt"
5483  || (*irv)->name == "nalt") {
5484  auto attr = new Attribute();
5485  string unit_value = "km";
5486  Add_Str_Attr(attr,attr2_new_name,unit_value);
5487  (*irv)->attrs.push_back(attr);
5488 
5489  auto attr1 = new Attribute();
5490  string attr1_axis="axis";
5491  string attr1_value = "Z";
5492  Add_Str_Attr(attr1,attr1_axis,attr1_value);
5493  (*irv)->attrs.push_back(attr1);
5494 
5495  auto attr2 = new Attribute();
5496  string attr2_positive="positive";
5497  string attr2_value = "up";
5498  Add_Str_Attr(attr2,attr2_positive,attr2_value);
5499  (*irv)->attrs.push_back(attr2);
5500 
5501  }
5502  if((*irv)->name == "hgt" || (*irv)->name == "nalt"){
5503  auto attr1 = new Attribute();
5504  string comment ="Number of heights above the earth ellipsoid";
5505  Add_Str_Attr(attr1,"comment",comment);
5506  (*irv)->attrs.push_back(attr1);
5507  }
5508 
5509  }
5510 
5511  }
5512 
5513 
5514 // Old code, leave it for the time being
5515 #if 0
5516  const string fill_value_attr_name = "_FillValue";
5517  vector<HDF5CF::Var *>::const_iterator it_v;
5518  vector<HDF5CF::Attribute *>::const_iterator ira;
5519 
5520  for (it_v = vars.begin();
5521  it_v != vars.end(); ++it_v) {
5522 
5523  bool has_fillvalue = false;
5524  for(ira = (*it_v)->attrs.begin(); ira!= (*it_v)->attrs.end();ira++) {
5525  if (fill_value_attr_name == (*ira)->name){
5526  has_fillvalue = true;
5527  break;
5528  }
5529 
5530  }
5531 
5532  // Add the fill value
5533  if (has_fillvalue != true ) {
5534 
5535  if(H5FLOAT32 == (*it_v)->dtype) {
5536  Attribute* attr = new Attribute();
5537  float _FillValue = -9999.9;
5538  Add_One_Float_Attr(attr,fill_value_attr_name,_FillValue);
5539  (*it_v)->attrs.push_back(attr);
5540  }
5541  }
5542  }// for (it_v = vars.begin(); ...
5543 #endif
5544 
5545 }
5546 
5547 // For GPM level 1 data, var must have names that contains either "Latitude" nor "Longitude".
5548 void
5549 GMFile:: Correct_GPM_L1_LatLon_units(Var *var, const string unit_value) {
5550 
5551  BESDEBUG("h5", "Coming to Correct_GPM_L1_LatLon_units()"<<endl);
5552  const string Unit_name = "Units";
5553  const string unit_name = "units";
5554 
5555  // Delete "units" and "Units"
5556  for(auto ira = var->attrs.begin(); ira!= var->attrs.end();) {
5557  if(unit_name == (*ira)->name) {
5558  delete(*ira);
5559  ira = var->attrs.erase(ira);
5560  }
5561  else if(Unit_name == (*ira)->name) {
5562  delete(*ira);
5563  ira = var->attrs.erase(ira);
5564  }
5565  else
5566  ++ira;
5567  }
5568  // Add the correct units for Latitude and Longitude
5569  // Note: the reason we do this way, for some versions of GPM, units is degrees,
5570  // rather than degrees_north.. So units also needs to be corrected to follow CF.
5571  auto attr = new Attribute();
5572  Add_Str_Attr(attr,unit_name,unit_value);
5573  var->attrs.push_back(attr);
5574 }
5575 
5576 
5577 
5578 // Add attributes for Aquarius products
5579 void
5580 GMFile:: Add_Aqu_Attrs() {
5581 
5582  BESDEBUG("h5", "Coming to Add_Aqu_Attrs()"<<endl);
5583  vector<HDF5CF::Var *>::const_iterator it_v;
5584  vector<HDF5CF::Attribute *>::const_iterator ira;
5585 
5586  const string orig_longname_attr_name = "Parameter";
5587  const string longname_attr_name ="long_name";
5588  string longname_value;
5589 
5590 
5591  const string orig_units_attr_name = "Units";
5592  const string units_attr_name = "units";
5593  string units_value;
5594 
5595  const string orig_valid_min_attr_name = "Data Minimum";
5596  const string valid_min_attr_name = "valid_min";
5597  float valid_min_value = 0;
5598 
5599  const string orig_valid_max_attr_name = "Data Maximum";
5600  const string valid_max_attr_name = "valid_max";
5601  float valid_max_value = 0;
5602 
5603  // The fill value is -32767.0. However, No _FillValue attribute is added.
5604  // So add it here. KY 2012-2-16
5605 
5606  const string fill_value_attr_name = "_FillValue";
5607  float _FillValue = -32767.0;
5608 
5609  for (ira = this->root_attrs.begin(); ira != this->root_attrs.end(); ++ira) {
5610  if (orig_longname_attr_name == (*ira)->name) {
5611  Retrieve_H5_Attr_Value(*ira,"/");
5612  longname_value.resize((*ira)->value.size());
5613  copy((*ira)->value.begin(),(*ira)->value.end(),longname_value.begin());
5614 
5615  }
5616  else if (orig_units_attr_name == (*ira)->name) {
5617  Retrieve_H5_Attr_Value(*ira,"/");
5618  units_value.resize((*ira)->value.size());
5619  copy((*ira)->value.begin(),(*ira)->value.end(),units_value.begin());
5620 
5621  }
5622  else if (orig_valid_min_attr_name == (*ira)->name) {
5623  Retrieve_H5_Attr_Value(*ira,"/");
5624  memcpy(&valid_min_value,(void*)(&((*ira)->value[0])),(*ira)->value.size());
5625  }
5626 
5627  else if (orig_valid_max_attr_name == (*ira)->name) {
5628  Retrieve_H5_Attr_Value(*ira,"/");
5629  memcpy(&valid_max_value,(void*)(&((*ira)->value[0])),(*ira)->value.size());
5630  }
5631 
5632  }// end of for (ira = this->root_attrs.begin(); ira != this->root_attrs.end(); ++ira)
5633 
5634  // New version Aqu(Q20112132011243.L3m_MO_SCI_V3.0_SSS_1deg.bz2) files seem to have CF attributes added.
5635  // In this case, we should not add extra CF attributes, or duplicate values may appear. KY 2015-06-20
5636  bool has_long_name = false;
5637  bool has_units = false;
5638  bool has_valid_min = false;
5639  bool has_valid_max = false;
5640  bool has_fillvalue = false;
5641 
5642  for (it_v = vars.begin(); it_v != vars.end(); ++it_v) {
5643  if ("l3m_data" == (*it_v)->name) {
5644  for (ira = (*it_v)->attrs.begin(); ira != (*it_v)->attrs.end(); ++ira) {
5645  if (longname_attr_name == (*ira)->name)
5646  has_long_name = true;
5647  else if(units_attr_name == (*ira)->name)
5648  has_units = true;
5649  else if(valid_min_attr_name == (*ira)->name)
5650  has_valid_min = true;
5651  else if(valid_max_attr_name == (*ira)->name)
5652  has_valid_max = true;
5653  else if(fill_value_attr_name == (*ira)->name)
5654  has_fillvalue = true;
5655  }
5656  break;
5657  }
5658  } // end of for (it_v = vars.begin(); ...
5659 
5660 
5661  // Level 3 variable name is l3m_data
5662  for (it_v = vars.begin(); it_v != vars.end(); ++it_v) {
5663  if ("l3m_data" == (*it_v)->name) {
5664 
5665  Attribute *attr = nullptr;
5666  // 1. Add the long_name attribute if no
5667  if(false == has_long_name) {
5668  attr = new Attribute();
5669  Add_Str_Attr(attr,longname_attr_name,longname_value);
5670  (*it_v)->attrs.push_back(attr);
5671  }
5672 
5673  // 2. Add the units attribute
5674  if(false == has_units) {
5675  attr = new Attribute();
5676  Add_Str_Attr(attr,units_attr_name,units_value);
5677  (*it_v)->attrs.push_back(attr);
5678  }
5679 
5680  // 3. Add the valid_min attribute
5681  if(false == has_valid_min) {
5682  attr = new Attribute();
5683  Add_One_Float_Attr(attr,valid_min_attr_name,valid_min_value);
5684  (*it_v)->attrs.push_back(attr);
5685  }
5686 
5687  // 4. Add the valid_max attribute
5688  if(false == has_valid_max) {
5689  attr = new Attribute();
5690  Add_One_Float_Attr(attr,valid_max_attr_name,valid_max_value);
5691  (*it_v)->attrs.push_back(attr);
5692  }
5693 
5694  // 5. Add the _FillValue attribute
5695  if(false == has_fillvalue) {
5696  attr = new Attribute();
5697  Add_One_Float_Attr(attr,fill_value_attr_name,_FillValue);
5698  (*it_v)->attrs.push_back(attr);
5699  }
5700 
5701  break;
5702  }
5703  } // for (it_v = vars.begin(); ...
5704 }
5705 
5706 // Add SeaWiFS attributes
5707 void
5708 GMFile:: Add_SeaWiFS_Attrs() {
5709 
5710  BESDEBUG("h5", "Coming to Add_SeaWiFS_Attrs()"<<endl);
5711  // The fill value is -999.0. However, No _FillValue attribute is added.
5712  // So add it here. KY 2012-2-16
5713  const string fill_value_attr_name = "_FillValue";
5714  float _FillValue = -999.0;
5715  const string valid_range_attr_name = "valid_range";
5716  vector<HDF5CF::Var *>::const_iterator it_v;
5717  vector<HDF5CF::Attribute *>::const_iterator ira;
5718 
5719 
5720  for (it_v = vars.begin(); it_v != vars.end(); ++it_v) {
5721  if (H5FLOAT32 == (*it_v)->dtype) {
5722  bool has_fillvalue = false;
5723  bool has_validrange = false;
5724  for(ira = (*it_v)->attrs.begin(); ira!= (*it_v)->attrs.end();ira++) {
5725  if (fill_value_attr_name == (*ira)->name){
5726  has_fillvalue = true;
5727  break;
5728  }
5729 
5730  else if(valid_range_attr_name == (*ira)->name) {
5731  has_validrange = true;
5732  break;
5733  }
5734 
5735  }
5736  // Add the fill value
5737  if (has_fillvalue != true && has_validrange != true ) {
5738  auto attr = new Attribute();
5739  Add_One_Float_Attr(attr,fill_value_attr_name,_FillValue);
5740  (*it_v)->attrs.push_back(attr);
5741  }
5742  }
5743  }
5744 }
5745 
5746 // Leave the following code for the time being
5747 #if 0
5748 // Handle the "coordinates" and "units" attributes of coordinate variables.
5750 
5751  string co_attrname = "coordinates";
5752  string co_attrvalue="";
5753  string unit_attrname = "units";
5754  string nonll_unit_attrvalue ="level";
5755  string lat_unit_attrvalue ="degrees_north";
5756  string lon_unit_attrvalue ="degrees_east";
5757 
5758  for (auto ircv = this->cvars.begin();
5759  ircv != this->cvars.end(); ++ircv) {
5760 //cerr<<"CV name is "<<(*ircv)->name << " cv type is "<<(*ircv)->cvartype <<endl;
5761 
5762  if ((*ircv)->cvartype == CV_NONLATLON_MISS) {
5763  Attribute * attr = new Attribute();
5764  Add_Str_Attr(attr,unit_attrname,nonll_unit_attrvalue);
5765  (*ircv)->attrs.push_back(attr);
5766  }
5767 
5768  else if ((*ircv)->cvartype == CV_LAT_MISS) {
5769 //cerr<<"Should add new attribute "<<endl;
5770  Attribute * attr = new Attribute();
5771 // float temp = -999.9;
5772 // Add_One_Float_Attr(attr,unit_attrname,temp);
5773  Add_Str_Attr(attr,unit_attrname,lat_unit_attrvalue);
5774  (*ircv)->attrs.push_back(attr);
5775 //cerr<<"After adding new attribute "<<endl;
5776  }
5777 
5778  else if ((*ircv)->cvartype == CV_LON_MISS) {
5779  Attribute * attr = new Attribute();
5780  Add_Str_Attr(attr,unit_attrname,lon_unit_attrvalue);
5781  (*ircv)->attrs.push_back(attr);
5782  }
5783  } // for (auto ircv = this->cvars.begin(); ...
5784 
5785  // No need to handle MeaSUREs SeaWiFS level 2 products
5786  if(product_type == Mea_SeaWiFS_L2)
5787  return;
5788 
5789  // GPM level 1 needs to be handled separately
5790  else if(product_type == GPM_L1) {
5791  Handle_GPM_l1_Coor_Attr();
5792  return;
5793  }
5794  // No need to handle products that follow COARDS.
5795  else if (true == iscoard) {
5796  // May need to check coordinates for 2-D lat/lon but cannot treat those lat/lon as CV case. KY 2015-12-10-TEMPPP
5797  return;
5798  }
5799 
5800 
5801  // Now handle the 2-D lat/lon case(note: this only applies to the one that dim. scale doesn't apply)
5802  for (auto ircv = this->cvars.begin();
5803  ircv != this->cvars.end(); ++ircv) {
5804  if((*ircv)->rank == 2) {
5805 
5806  // The following code makes sure that the replacement only happens with the general 2-D lat/lon case.
5807  if(gp_latname == (*ircv)->name)
5808  Replace_Var_Str_Attr((*ircv),unit_attrname,lat_unit_attrvalue);
5809  else if(gp_lonname ==(*ircv)->name)
5810  Replace_Var_Str_Attr((*ircv),unit_attrname,lon_unit_attrvalue);
5811  }
5812  }
5813 
5814  // Check the dimension names of 2-D lat/lon CVs
5815  string ll2d_dimname0,ll2d_dimname1;
5816  bool has_ll2d_coords = false;
5817  for (auto ircv = this->cvars.begin();
5818  ircv != this->cvars.end(); ++ircv) {
5819  if((*ircv)->rank == 2) {
5820  // Note: we should still use the original dim. name to match the general variables.
5821  ll2d_dimname0 = (*ircv)->getDimensions()[0]->name;
5822  ll2d_dimname1 = (*ircv)->getDimensions()[1]->name;
5823  if(ll2d_dimname0 !="" && ll2d_dimname1 !="")
5824  has_ll2d_coords = true;
5825  break;
5826  }
5827  }
5828 
5829  if(true == has_ll2d_coords) {
5830 
5831  for (auto irv = this->vars.begin();
5832  irv != this->vars.end(); ++irv) {
5833 
5834  bool coor_attr_keep_exist = false;
5835 
5836  // May need to delete only the "coordinates" with both 2-D lat/lon dim. KY 2015-12-07
5837  if(((*irv)->rank >=2)) {
5838 
5839  short ll2dim_flag = 0;
5840  for (auto ird = (*irv)->dims.begin();
5841  ird != (*irv)->dims.end(); ++ ird) {
5842  if((*ird)->name == ll2d_dimname0)
5843  ll2dim_flag++;
5844  else if((*ird)->name == ll2d_dimname1)
5845  ll2dim_flag++;
5846  }
5847 
5848  if(ll2dim_flag != 2)
5849  coor_attr_keep_exist = true;
5850 
5851  // The following line doesn't apply to SMAP,it applies to Old SMAP Level 2 Simulation files.
5852  if(product_type == OSMAPL2S)
5853  coor_attr_keep_exist = true;
5854 
5855  if (false == coor_attr_keep_exist) {
5856  for (auto ira =(*irv)->attrs.begin();
5857  ira !=(*irv)->attrs.end();) {
5858  if ((*ira)->newname == co_attrname) {
5859  delete (*ira);
5860  ira = (*irv)->attrs.erase(ira);
5861  }
5862  else {
5863  ++ira;
5864  }
5865  }// for (auto ira =(*irv)->attrs.begin(); ...
5866 
5867  // Generate the "coordinates" attribute only for variables that have both 2-D lat/lon dim. names.
5868  for (auto ird = (*irv)->dims.begin();
5869  ird != (*irv)->dims.end(); ++ ird) {
5870  for (auto ircv = this->cvars.begin();
5871  ircv != this->cvars.end(); ++ircv) {
5872  if ((*ird)->name == (*ircv)->cfdimname)
5873  co_attrvalue = (co_attrvalue.empty())
5874  ?(*ircv)->newname:co_attrvalue + " "+(*ircv)->newname;
5875  }
5876  }
5877 
5878  if (false == co_attrvalue.empty()) {
5879  Attribute * attr = new Attribute();
5880  Add_Str_Attr(attr,co_attrname,co_attrvalue);
5881  (*irv)->attrs.push_back(attr);
5882  }
5883 
5884  co_attrvalue.clear();
5885  } // for (auto irv = this->vars.begin(); ...
5886  }
5887  }
5888  }
5889 }
5890 #endif
5891 
5892 
5893 // Handle the "coordinates" and "units" attributes of coordinate variables.
5895 
5896  BESDEBUG("h5", "GMFile::Coming to Handle_Coor_Attr()"<<endl);
5897  string co_attrname = "coordinates";
5898  string co_attrvalue="";
5899  string unit_attrname = "units";
5900  string nonll_unit_attrvalue ="level";
5901  string lat_unit_attrvalue ="degrees_north";
5902  string lon_unit_attrvalue ="degrees_east";
5903 
5904  // Attribute units should be added for coordinate variables that
5905  // have the type CV_NONLATLON_MISS,CV_LAT_MISS and CV_LON_MISS.
5906  for (auto ircv = this->cvars.begin(); ircv != this->cvars.end(); ++ircv) {
5907 
5908  if ((*ircv)->cvartype == CV_NONLATLON_MISS) {
5909  auto attr = new Attribute();
5910  Add_Str_Attr(attr,unit_attrname,nonll_unit_attrvalue);
5911  (*ircv)->attrs.push_back(attr);
5912  }
5913  else if ((*ircv)->cvartype == CV_LAT_MISS) {
5914  auto attr = new Attribute();
5915  Add_Str_Attr(attr,unit_attrname,lat_unit_attrvalue);
5916  (*ircv)->attrs.push_back(attr);
5917  }
5918  else if ((*ircv)->cvartype == CV_LON_MISS) {
5919  auto attr = new Attribute();
5920  Add_Str_Attr(attr,unit_attrname,lon_unit_attrvalue);
5921  (*ircv)->attrs.push_back(attr);
5922  }
5923  } // end of for
5924 
5925  // No need to handle MeaSUREs SeaWiFS level 2 products
5926  if(product_type == Mea_SeaWiFS_L2)
5927  return;
5928 
5929  // GPM level 1 needs to be handled separately
5930  else if(product_type == GPM_L1) {
5931  Handle_GPM_l1_Coor_Attr();
5932  return;
5933  }
5934 
5935  // Handle Lat/Lon with "coordinates" attribute.
5936  else if(product_type == General_Product && gproduct_pattern == GENERAL_LATLON_COOR_ATTR){
5937  Handle_LatLon_With_CoordinateAttr_Coor_Attr();
5938  return;
5939  }
5940  // No need to handle products that follow COARDS.
5941  else if (true == iscoard) {
5942 
5943  // If we find that there are groups that should check the coordinates attribute of the variable.
5944  // We should flatten the path inside the coordinates.(this is the case mainly for netcdf-4 2D lat/lon case)
5945  if(grp_cv_paths.size() >0) {
5946  for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
5947  if(grp_cv_paths.find(HDF5CFUtil::obtain_string_before_lastslash((*irv)->fullpath)) != grp_cv_paths.end()){
5948 
5949  // Check the "coordinates" attribute and flatten the values.
5950  Flatten_VarPath_In_Coordinates_Attr(*irv);
5951  }
5952  }
5953  }
5954  return;
5955  }
5956 
5957  // Now handle the 2-D lat/lon case
5958  for (auto ircv = this->cvars.begin(); ircv != this->cvars.end(); ++ircv) {
5959 
5960  if((*ircv)->rank == 2 && (*ircv)->cvartype == CV_EXIST) {
5961 
5962  //Note: When the 2nd parameter is true in the function Is_geolatlon, it checks the lat/latitude/Latitude
5963  // When the 2nd parameter is false in the function Is_geolatlon, it checks the lon/longitude/Longitude
5964  // The following code makes sure that the replacement only happens with the general 2-D lat/lon case.
5965  // The following code is commented out since we find an OMPS-NPP case that has the no-CF unit for
5966  // "Latitude". So just to check the latitude and longitude and if the units are not CF-compliant,
5967  // change them. KY 2020-02-27
5968 #if 0
5969  if(gp_latname == (*ircv)->name) {
5970  // Only if gp_latname is not lat/latitude/Latitude, change the units
5971  if(false == Is_geolatlon(gp_latname,true))
5972  Replace_Var_Str_Attr((*ircv),unit_attrname,lat_unit_attrvalue);
5973  }
5974  else if(gp_lonname ==(*ircv)->name) {
5975  // Only if gp_lonname is not lon/longitude/Longitude, change the units
5976  if(false == Is_geolatlon(gp_lonname,false))
5977  Replace_Var_Str_Attr((*ircv),unit_attrname,lon_unit_attrvalue);
5978  }
5979 #endif
5980 
5981  // We meet several products that miss the 2-D latitude and longitude CF units although they
5982  // have the CV names like latitude/longitude, we should double check this case,
5983  // and add the correct CF units if possible. We will watch if this is the right way.
5984  //else if(true == Is_geolatlon((*ircv)->name,true))
5985  if(true == Is_geolatlon((*ircv)->name,true))
5986  Replace_Var_Str_Attr((*ircv),unit_attrname,lat_unit_attrvalue);
5987  else if(true == Is_geolatlon((*ircv)->name,false))
5988  Replace_Var_Str_Attr((*ircv),unit_attrname,lon_unit_attrvalue);
5989  }
5990  } // for (auto ircv = this->cvars.begin()
5991 
5992  // If we find that there are groups that we should check the coordinates attribute of the variable under,
5993  // we should flatten the path inside the coordinates. Note this is for 2D-latlon CV netCDF-4-like case.
5994  if(grp_cv_paths.size() >0) {
5995  for (auto irv = this->vars.begin();
5996  irv != this->vars.end(); ++irv) {
5997  if(grp_cv_paths.find(HDF5CFUtil::obtain_string_before_lastslash((*irv)->fullpath)) != grp_cv_paths.end()){
5998 
5999  // Check the "coordinates" attribute and flatten the values.
6000  Flatten_VarPath_In_Coordinates_Attr(*irv);
6001  }
6002  }
6003  }
6004 
6005  // Check if having 2-D lat/lon CVs
6006  bool has_ll2d_coords = false;
6007 
6008  // Since iscoard is false up to this point, So the netCDF-4 like 2-D lat/lon case must fulfill if the program comes here.
6009  if(General_Product == this->product_type && GENERAL_DIMSCALE == this->gproduct_pattern)
6010  has_ll2d_coords = true;
6011  else {// For other cases. Need to see if there is a case. KY 2016-07-07
6012  string ll2d_dimname0;
6013  string ll2d_dimname1;
6014  for (auto ircv = this->cvars.begin();
6015  ircv != this->cvars.end(); ++ircv) {
6016  if((*ircv)->rank == 2) {
6017  // Note: we should still use the original dim. name to match the general variables.
6018  ll2d_dimname0 = (*ircv)->getDimensions()[0]->name;
6019  ll2d_dimname1 = (*ircv)->getDimensions()[1]->name;
6020  if(ll2d_dimname0 !="" && ll2d_dimname1 !="")
6021  has_ll2d_coords = true;
6022  break;
6023  }
6024  }
6025  }
6026 
6027  // We now walk through all the >=2 vars and flatten the "coordinates"
6028  if(true == has_ll2d_coords) {
6029 
6030  // For some netCDF-4-like 2-D lat/lon cases, we may need to forcely flatten the coordinates.
6031  // This case usually happens when the data producers follow the CF and the NASA DIWG guideline to
6032  // provide the absolute path of the coordinates as the value of the "coordinates" attribute.
6033  // The handler doesn't need to figure out the contents of the coordinates attribute but to
6034  // flatten the path inside.
6035  // However, the BES Key FORCENDCoorAttr must be set.
6036  bool force_flatten_coor_attr = HDF5RequestHandler::get_force_flatten_coor_attr();
6037 
6038  // We also need to find if we have coordinates attribute for >=2D variables.
6039  // If not, the handler has to figure out the coordinates.
6040  bool has_coor_attr_ge_2d_vars = false;
6041  for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
6042  if((*irv)->rank >=2){
6043  for (auto ira =(*irv)->attrs.begin(); ira !=(*irv)->attrs.end();++ira) {
6044  // We will check if we have the coordinate attribute
6045  if((*ira)->name == co_attrname) {
6046  has_coor_attr_ge_2d_vars = true;
6047  break;
6048  }
6049  }
6050  if(has_coor_attr_ge_2d_vars == true)
6051  break;
6052  }
6053  }
6054 #if 0
6055  // Here we may need to consider the special case for HDF-EOS5. The "Data Fields" etc should not be
6056  // in the group path. May need to let DIWG provide a guideline for this issue.
6057  bool is_hybrid_eos5= false;
6058  if(force_flatten_coor_attr == true && has_coor_attr_ge_2d_vars == true)
6059  is_hybrid_eos5 = Is_Hybrid_EOS5();
6060 #endif
6061  for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
6062 
6063  bool has_coor = false;
6064  for (auto ira =(*irv)->attrs.begin(); ira !=(*irv)->attrs.end();++ira) {
6065  // We will check if we have the coordinate attribute
6066  if((*ira)->name == co_attrname) {
6067  has_coor = true;
6068  break;
6069  }
6070  }
6071 
6072  // The coordinates attribute is flattened by force.
6073  if(true == force_flatten_coor_attr && true == has_coor) {
6074 #if 0
6075  if(is_hybrid_eos5 == true) {
6076  Flatten_VarPath_In_Coordinates_Attr_EOS5((*irv));
6077  }
6078  else
6079 #endif
6080  Flatten_VarPath_In_Coordinates_Attr((*irv));
6081  }
6082 
6083  else if(((*irv)->rank >=2) && (has_coor_attr_ge_2d_vars == false || false == force_flatten_coor_attr)) {
6084 
6085  bool coor_attr_keep_exist = false;
6086 
6087  // Check if this var is under group_cv_paths, no, then check if this var's dims are the same as the dims of 2-D CVars
6088  if(grp_cv_paths.find(HDF5CFUtil::obtain_string_before_lastslash((*irv)->fullpath)) == grp_cv_paths.end())
6089 
6090  // If finding this var is associated with 2-D lat/lon CVs, not keep the original "coordinates" attribute.
6091  coor_attr_keep_exist = Check_Var_2D_CVars(*irv);
6092  else {
6093  coor_attr_keep_exist = true;
6094  }
6095 
6096  // The following two lines are just for old smap level 2 case.
6097  if(product_type == OSMAPL2S)
6098  coor_attr_keep_exist = true;
6099 
6100  // Need to delete the original "coordinates" and rebuild the "coordinates" if this var is associated with the 2-D lat/lon CVs.
6101  if (false == coor_attr_keep_exist) {
6102  for (auto ira =(*irv)->attrs.begin(); ira !=(*irv)->attrs.end();) {
6103  if ((*ira)->newname == co_attrname) {
6104  delete (*ira);
6105  ira = (*irv)->attrs.erase(ira);
6106  }
6107  else {
6108  ++ira;
6109  }
6110  }// for (auto ira =(*irv)->attrs.begin(); ...
6111 
6112  // Generate the new "coordinates" attribute.
6113  for (auto ird = (*irv)->dims.begin(); ird != (*irv)->dims.end(); ++ ird) {
6114  for (auto ircv = this->cvars.begin(); ircv != this->cvars.end(); ++ircv) {
6115  if ((*ird)->name == (*ircv)->cfdimname)
6116  co_attrvalue = (co_attrvalue.empty())
6117  ?(*ircv)->newname:co_attrvalue + " "+(*ircv)->newname;
6118  }
6119  }
6120 
6121  if (false == co_attrvalue.empty()) {
6122  auto attr = new Attribute();
6123  Add_Str_Attr(attr,co_attrname,co_attrvalue);
6124  (*irv)->attrs.push_back(attr);
6125  }
6126 
6127  co_attrvalue.clear();
6128  (*irv)->coord_attr_add_path = false;
6129  } // for (auto irv = this->vars.begin(); ...
6130  }
6131  }
6132  }
6133 }
6134 
6135 // Handle GPM level 1 coordiantes attributes.
6136 void GMFile:: Handle_GPM_l1_Coor_Attr() {
6137 
6138  BESDEBUG("h5", "Coming to Handle_GPM_l1_Coor_Attr()"<<endl);
6139  // Build a map from CFdimname to 2-D lat/lon variable name, should be something like: aa_list[cfdimname]=s1_latitude .
6140  // Loop all variables
6141  // Inner loop: for all dims of a var
6142  // if(dimname matches the dim(not cfdim) name of one of 2-D lat/lon,
6143  // check if the variable's full path contains the path of one of 2-D lat/lon,
6144  // yes, build its cfdimname = path+ dimname, check this cfdimname with the cfdimname of the corresponding 2-D lat/lon
6145  // If matched, save this latitude variable name as one of the coordinate variable.
6146  // else this is a 3rd-dimension cv, just use the dimension name(or the corresponding cv name maybe through a map).
6147 
6148  // Prepare 1) 2-D CVar(lat,lon) corresponding dimension name set.
6149  // 2) cfdim name to cvar name map(don't need to use a map, just a holder. It should be fine.
6150 
6151  // "coordinates" attribute name and value. We only need to provide this atttribute for variables that have 2-D lat/lon
6152  string co_attrname = "coordinates";
6153  string co_attrvalue="";
6154 
6155  // 2-D cv dimname set.
6156  set<string> cvar_2d_dimset;
6157 
6158  pair<map<string,string>::iterator,bool>mapret;
6159 
6160  // Hold the mapping from cfdimname to 2-D cvar name. Something like nscan->lat, npixel->lon
6161  map<string,string>cfdimname_to_cvar2dname;
6162 
6163  // Loop through cv variables to build 2-D cv dimname set and the mapping from cfdimname to 2-D cvar name.
6164  for (auto irv = this->cvars.begin(); irv != this->cvars.end(); ++irv) {
6165 
6166  //This CVar must be 2-D array.
6167  if((*irv)->rank == 2) {
6168 
6169 #if 0
6170 //cerr<<"2-D cv name is "<<(*irv)->name <<endl;
6171 //cerr<<"2-D cv new name is "<<(*irv)->newname <<endl;
6172 //cerr<<"(*irv)->cfdimname is "<<(*irv)->cfdimname <<endl;
6173 #endif
6174 
6175  for(auto ird = (*irv)->dims.begin(); ird != (*irv)->dims.end(); ++ird)
6176  cvar_2d_dimset.insert((*ird)->name);
6177 
6178  // Generate cfdimname to cvar2d map
6179  mapret = cfdimname_to_cvar2dname.insert(pair<string,string>((*irv)->cfdimname,(*irv)->newname));
6180  if (false == mapret.second)
6181  throw4("The cf dimension name ",(*irv)->cfdimname," should map to 2-D coordinate variable",
6182  (*irv)->newname);
6183  }
6184  }
6185 
6186  // Loop through the variable list to build the coordinates.
6187  for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
6188 
6189  // Only apply to >2D variables.
6190  if((*irv)->rank >=2) {
6191 
6192  // The variable dimension names must be found in the 2D cvar dim. nameset.
6193  // The flag must be at least 2.
6194  short have_2d_dimnames_flag = 0;
6195  for (auto ird = (*irv)->dims.begin(); ird !=(*irv)->dims.end();++ird) {
6196  if (cvar_2d_dimset.find((*ird)->name)!=cvar_2d_dimset.end())
6197  have_2d_dimnames_flag++;
6198  }
6199 
6200  // Final candidates to have 2-D CVar coordinates.
6201  if(have_2d_dimnames_flag >=2) {
6202 
6203  // Obtain the variable path
6204  string var_path;
6205  if((*irv)->fullpath.size() > (*irv)->name.size())
6206  var_path=(*irv)->fullpath.substr(0,(*irv)->fullpath.size()-(*irv)->name.size());
6207  else
6208  throw4("The variable full path ",(*irv)->fullpath," doesn't contain the variable name ",
6209  (*irv)->name);
6210 
6211  // A flag to identify if this variable really needs the 2-D coordinate variables.
6212  short cv_2d_flag = 0;
6213 
6214  // 2-D coordinate variable names for the potential variable candidate
6215  vector<string> cv_2d_names;
6216 
6217  // Dimension names of the 2-D coordinate variables.
6218  set<string> cv_2d_dimnames;
6219 
6220  // Loop through the map from dim. name to coordinate name.
6221  for(map<string,string>::const_iterator itm = cfdimname_to_cvar2dname.begin();
6222  itm != cfdimname_to_cvar2dname.end();++itm) {
6223 
6224  // Obtain the dimension name from the cfdimname.
6225  string reduced_dimname = HDF5CFUtil::obtain_string_after_lastslash(itm->first);
6226  string cfdim_path;
6227  if(itm->first.size() <= reduced_dimname.size())
6228  throw2("The cf dim. name of this dimension is not right.",itm->first);
6229  else
6230  cfdim_path= itm->first.substr(0,itm->first.size() - reduced_dimname.size());
6231  // cfdim_path will not be nullptr only when the cfdim name is for the 2-D cv var.
6232 
6233  // Find the correct path,
6234  // Note:
6235  // var_path doesn't have to be the same as cfdim_path
6236  // consider the variable /a1/a2/foo and the latitude /a1/latitude(cfdimpath is /a1)
6237  // If there is no /a1/a2/latitude, the /a1/latitude can be used as the coordinate of /a1/a2/foo.
6238  // But we want to check if var_path is the same as cfdim_path first. So we check cfdimname_to_cvarname again.
6239  if(var_path == cfdim_path) {
6240  for (auto ird = (*irv)->dims.begin(); ird!=(*irv)->dims.end();++ird) {
6241  if(reduced_dimname == (*ird)->name) {
6242  cv_2d_flag++;
6243  cv_2d_names.push_back(itm->second);
6244  cv_2d_dimnames.insert((*ird)->name);
6245  }
6246  }
6247  }
6248  }
6249 
6250  // Note:
6251  // var_path doesn't have to be the same as cfdim_path
6252  // consider the variable /a1/a2/foo and the latitude /a1/latitude(cfdimpath is /a1)
6253  // If there is no /a1/a2/latitude, the /a1/latitude can be used as the coordinate of /a1/a2/foo.
6254  // The variable has 2 coordinates(dimensions) if the flag is 2
6255  // If the flag is not 2, we will see if the above case stands.
6256  if(cv_2d_flag != 2) {
6257  cv_2d_flag = 0;
6258  // Loop through the map from dim. name to coordinate name.
6259  for(map<string,string>::const_iterator itm = cfdimname_to_cvar2dname.begin();
6260  itm != cfdimname_to_cvar2dname.end();++itm) {
6261  // Obtain the dimension name from the cfdimname.
6262  string reduced_dimname = HDF5CFUtil::obtain_string_after_lastslash(itm->first);
6263  string cfdim_path;
6264  if(itm->first.size() <= reduced_dimname.size())
6265  throw2("The cf dim. name of this dimension is not right.",itm->first);
6266  else
6267  cfdim_path= itm->first.substr(0,itm->first.size() - reduced_dimname.size());
6268  // cfdim_path will not be nullptr only when the cfdim name is for the 2-D cv var.
6269 
6270  // Find the correct path,
6271  // Note:
6272  // var_path doesn't have to be the same as cfdim_path
6273  // consider the variable /a1/a2/foo and the latitude /a1/latitude(cfdimpath is /a1)
6274  // If there is no /a1/a2/latitude, the /a1/latitude can be used as the coordinate of /a1/a2/foo.
6275  //
6276  if(var_path.find(cfdim_path)!=string::npos) {
6277  for (auto ird = (*irv)->dims.begin(); ird!=(*irv)->dims.end();++ird) {
6278  if(reduced_dimname == (*ird)->name) {
6279  cv_2d_flag++;
6280  cv_2d_names.push_back(itm->second);
6281  cv_2d_dimnames.insert((*ird)->name);
6282  }
6283  }
6284  }
6285 
6286  }
6287  }
6288 
6289  // Now we got all cases.
6290  if(2 == cv_2d_flag) {
6291 
6292  // Add latitude and longitude to the 'coordinates' attribute.
6293  co_attrvalue = cv_2d_names[0] + " " + cv_2d_names[1];
6294  if((*irv)->rank >2) {
6295  for (auto ird = (*irv)->dims.begin(); ird !=(*irv)->dims.end();++ird) {
6296 
6297  // Add 3rd-dimension to the 'coordinates' attribute.
6298  if(cv_2d_dimnames.find((*ird)->name) == cv_2d_dimnames.end())
6299  co_attrvalue = co_attrvalue + " " +(*ird)->newname;
6300  }
6301  }
6302  auto attr = new Attribute();
6303  Add_Str_Attr(attr,co_attrname,co_attrvalue);
6304  (*irv)->attrs.push_back(attr);
6305  (*irv)->coord_attr_add_path = false;
6306  }
6307  }
6308  }
6309  }
6310 }
6311 
6312 // This routine is for handling "coordinates" for the GENERAL_LATLON_COOR_ATTR pattern of General_Product.
6313 void GMFile::Handle_LatLon_With_CoordinateAttr_Coor_Attr() {
6314 
6315  BESDEBUG("h5", "Coming to Handle_LatLon_With_CoordinateAttr_Coor_Attr()"<<endl);
6316  string co_attrname = "coordinates";
6317 
6318  // Loop through all rank >1 variables
6319  for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
6320  if((*irv)->rank >= 2) {
6321  for (auto ira =(*irv)->attrs.begin(); ira !=(*irv)->attrs.end(); ++ira) {
6322  if((*ira)->name == co_attrname) {
6323  // If having the coordinates attribute, check if the "coordinates" variables match 2-D lat/lon CV condition,
6324  // if yes, flatten the coordinates attribute.
6325  string coor_value = Retrieve_Str_Attr_Value(*ira,(*irv)->fullpath);
6326  if(Coord_Match_LatLon_NameSize(coor_value) == true)
6327  Flatten_VarPath_In_Coordinates_Attr(*irv);
6328  // If the "coordinates" variables don't match the first condition, we can still check
6329  // if we can find the corresponding "coordinates" variables that match the names under the same group,
6330  // if yes, we add the path to the attribute "coordinates".
6331  else if(Coord_Match_LatLon_NameSize_Same_Group(coor_value,HDF5CFUtil::obtain_string_before_lastslash((*irv)->fullpath)) == true)
6332  Add_VarPath_In_Coordinates_Attr(*irv,coor_value);
6333  // For other cases, we don't do anything with the "coordinates".
6334  break;
6335  }
6336  }
6337  }
6338  }
6339 
6340 }
6341 
6342 // We will check the "coordinates variables" stored in the coordinate attribute match the
6343 // checked latlon_name_pairs for the GENERAL_LATLON_COOR_ATTR case.
6344 bool GMFile::Coord_Match_LatLon_NameSize(const string & coord_values) {
6345 
6346  BESDEBUG("h5", "Coming to Coord_Match_LatLon_NameSize()"<<endl);
6347  bool ret_value =false;
6348  vector<string> coord_values_vec;
6349  char sep=' ';
6350  int match_lat_name_pair_index = -1;
6351  int match_lon_name_pair_index = -2;
6352  int num_match_lat = 0;
6353  int num_match_lon = 0;
6354 
6355 
6356  // Decompose the coordinates attribute into a string vector.
6357  HDF5CFUtil::Split_helper(coord_values_vec,coord_values,sep);
6358 
6359  // Some products ignore the first "/" of the coordinate path in the coordinate attribute, we will correct this
6360  if((coord_values_vec[0])[0] !='/') {
6361  for(auto irs=coord_values_vec.begin();irs!=coord_values_vec.end();++irs){
6362  if(((*irs).find_first_of('/'))!=string::npos) {
6363  *irs = '/' + (*irs);
6364  }
6365  }
6366  }
6367 
6368  //Loop through all coordinate path stored in the coordinate patch vector,
6369  for(auto irs=coord_values_vec.begin();irs!=coord_values_vec.end();++irs){
6370 
6371  // Loop through all the lat/lon pairs generated in the Check_LatLon_With_Coordinate_Attr routine
6372  // Remember the index and number appeared for both lat and lon.
6373  for(auto ivs=latloncv_candidate_pairs.begin(); ivs!=latloncv_candidate_pairs.end();++ivs) {
6374  if((*irs) == (*ivs).name1){
6375  match_lat_name_pair_index = distance(latloncv_candidate_pairs.begin(),ivs);
6376  num_match_lat++;
6377  }
6378  else if ((*irs) == (*ivs).name2) {
6379  match_lon_name_pair_index = distance(latloncv_candidate_pairs.begin(),ivs);
6380  num_match_lon++;
6381  }
6382  }
6383  }
6384  //Only when both index and the number of appearence match, we can set this be true.
6385  if((match_lat_name_pair_index == match_lon_name_pair_index) && (num_match_lat ==1) && (num_match_lon ==1))
6386  ret_value = true;
6387 
6388  return ret_value;
6389 
6390 }
6391 
6392 //Some products only store the coordinate name(not full path) in the attribute coordinates, as
6393 //long as it is valid, we should add the path to this coordinates.
6394 bool GMFile::Coord_Match_LatLon_NameSize_Same_Group(const string &coord_values,const string &var_path) {
6395 
6396  BESDEBUG("h5", "Coming to Coord_Match_LatLon_NameSize_Same_Group()"<<endl);
6397  bool ret_value =false;
6398  vector<string> coord_values_vec;
6399  char sep=' ';
6400  int match_lat_name_pair_index = -1;
6401  int match_lon_name_pair_index = -2;
6402  int num_match_lat = 0;
6403  int num_match_lon = 0;
6404 
6405  HDF5CFUtil::Split_helper(coord_values_vec,coord_values,sep);
6406 
6407  // Assume the 3rd-variable is also located under the same group if rank >=2
6408  for(auto irs=coord_values_vec.begin();irs!=coord_values_vec.end();++irs){
6409 //cerr<<"coordinate values are "<<*irs <<endl;
6410  for(auto ivs=latloncv_candidate_pairs.begin(); ivs!=latloncv_candidate_pairs.end();++ivs) {
6411  string lat_name = HDF5CFUtil::obtain_string_after_lastslash((*ivs).name1);
6412  string lat_path = HDF5CFUtil::obtain_string_before_lastslash((*ivs).name1);
6413  string lon_name = HDF5CFUtil::obtain_string_after_lastslash((*ivs).name2);
6414  string lon_path = HDF5CFUtil::obtain_string_before_lastslash((*ivs).name2);
6415  if((*irs) == lat_name && lat_path == var_path){
6416  match_lat_name_pair_index = distance(latloncv_candidate_pairs.begin(),ivs);
6417  num_match_lat++;
6418  }
6419  else if ((*irs) == lon_name && lon_path == var_path) {
6420  match_lon_name_pair_index = distance(latloncv_candidate_pairs.begin(),ivs);
6421  num_match_lon++;
6422  }
6423  }
6424  }
6425 
6426  if((match_lat_name_pair_index == match_lon_name_pair_index) && (num_match_lat ==1) && (num_match_lon ==1))
6427  ret_value = true;
6428 
6429  return ret_value;
6430 }
6431 
6432 // This is for the GENERAL_LATLON_COOR_ATTR pattern of General_Product.
6433 void GMFile::Add_VarPath_In_Coordinates_Attr(Var *var, const string &coor_value) {
6434 
6435  BESDEBUG("h5", "Coming to Add_VarPath_In_Coordinates_Attr()"<<endl);
6436  string new_coor_value;
6437  char sep =' ';
6438  string var_path = HDF5CFUtil::obtain_string_before_lastslash(var->fullpath) ;
6439  string var_flatten_path = get_CF_string(var_path);
6440 
6441  // We need to loop through each element in the "coor_value".
6442  size_t ele_start_pos = 0;
6443  size_t cur_pos = coor_value.find_first_of(sep);
6444  while(cur_pos !=string::npos) {
6445  string tempstr = coor_value.substr(ele_start_pos,cur_pos-ele_start_pos);
6446  tempstr = var_flatten_path + tempstr;
6447  new_coor_value += tempstr + sep;
6448  ele_start_pos = cur_pos+1;
6449  cur_pos = coor_value.find_first_of(sep,cur_pos+1);
6450  }
6451 
6452  if(ele_start_pos == 0)
6453  new_coor_value = var_flatten_path + coor_value;
6454  else
6455  new_coor_value += var_flatten_path + coor_value.substr(ele_start_pos);
6456 
6457  string coor_attr_name = "coordinates";
6458  Replace_Var_Str_Attr(var,coor_attr_name,new_coor_value);
6459  var->coord_attr_add_path = false;
6460 
6461 }
6462 
6463 // Create Missing coordinate variables. Index numbers are used for these variables.
6464 void GMFile:: Create_Missing_CV(GMCVar *GMcvar, const string& dimname) {
6465 
6466  BESDEBUG("h5", "GMFile::Coming to Create_Missing_CV()"<<endl);
6467 
6468  GMcvar->name = dimname;
6469  GMcvar->newname = GMcvar->name;
6470  GMcvar->fullpath = GMcvar->name;
6471  GMcvar->rank = 1;
6472  GMcvar->dtype = H5INT32;
6473  hsize_t gmcvar_dimsize = dimname_to_dimsize[dimname];
6474  bool unlimited_flag = dimname_to_unlimited[dimname];
6475  Dimension* gmcvar_dim = new Dimension(gmcvar_dimsize);
6476  gmcvar_dim->unlimited_dim = unlimited_flag;
6477  gmcvar_dim->name = dimname;
6478  gmcvar_dim->newname = dimname;
6479  GMcvar->dims.push_back(gmcvar_dim);
6480  GMcvar->cfdimname = dimname;
6481  GMcvar->cvartype = CV_NONLATLON_MISS;
6482  GMcvar->product_type = product_type;
6483 }
6484 
6485  // Check if this is just a netCDF-4 dimension. We need to check the dimension scale dataset attribute "NAME",
6486  // the value should start with "This is a netCDF dimension but not a netCDF variable".
6487 bool GMFile::Is_netCDF_Dimension(Var *var) {
6488 
6489  string netcdf_dim_mark = "This is a netCDF dimension but not a netCDF variable";
6490 
6491  bool is_only_dimension = false;
6492 
6493  for(auto ira = var->attrs.begin(); ira != var->attrs.end();ira++) {
6494 
6495  if ("NAME" == (*ira)->name) {
6496 
6497  Retrieve_H5_Attr_Value(*ira,var->fullpath);
6498  string name_value;
6499  name_value.resize((*ira)->value.size());
6500  copy((*ira)->value.begin(),(*ira)->value.end(),name_value.begin());
6501 
6502  // Compare the attribute "NAME" value with the string netcdf_dim_mark. We only compare the string with the size of netcdf_dim_mark
6503  if (0 == name_value.compare(0,netcdf_dim_mark.size(),netcdf_dim_mark))
6504  is_only_dimension = true;
6505 
6506  break;
6507  }
6508  } // for(auto ira = var->attrs.begin(); ...
6509 
6510  return is_only_dimension;
6511 }
6512 
6513 // Handle attributes for special variables.
6514 void
6516 
6517 }
6518 
6519 bool
6520 GMFile::Is_Hybrid_EOS5() {
6521 
6522  bool has_group_hdfeos = false;
6523  bool has_group_hdfeos_info = false;
6524 
6525  // Too costly to check the dataset.
6526  // We will just check the attribute under /HDFEOS INFORMATION.
6527 
6528  // First check if the HDFEOS groups are included
6529  for (auto irg = this->groups.begin(); irg != this->groups.end(); ++ irg) {
6530  if ("/HDFEOS" == (*irg)->path)
6531  has_group_hdfeos = true;
6532  else if("/HDFEOS INFORMATION" == (*irg)->path) {
6533  for(auto ira = (*irg)->attrs.begin();
6534  ira != (*irg)->attrs.end();ira++) {
6535  if("HDFEOSVersion" == (*ira)->name)
6536  has_group_hdfeos_info = true;
6537  }
6538  }
6539  if(true == has_group_hdfeos && true == has_group_hdfeos_info)
6540  break;
6541  }
6542 
6543 
6544  if(true == has_group_hdfeos && true == has_group_hdfeos_info)
6545  return true;
6546  else
6547  return false;
6548 }
6549 
6550 void GMFile::Handle_Hybrid_EOS5() {
6551 
6552  string eos_str="HDFEOS_";
6553  string eos_info_str="HDFEOS_INFORMATION_";
6554  string grid_str="GRIDS_";
6555  string swath_str="SWATHS_";
6556  string zas_str="ZAS_";
6557  string df_str="Data_Fields_";
6558  string gf_str="Geolocation_Fields_";
6559  for (auto irv = this->vars.begin(); irv != this->vars.end(); irv++) {
6560  string temp_var_name = (*irv)->newname;
6561 
6562  bool remove_eos = Remove_EOS5_Strings(temp_var_name);
6563 
6564  if(true == remove_eos)
6565  (*irv)->newname = get_CF_string(temp_var_name);
6566  else {//HDFEOS info and extra fields
6567  string::size_type eos_info_pos = temp_var_name.find(eos_info_str);
6568  if(eos_info_pos !=string::npos)
6569  (*irv)->newname = temp_var_name.erase(eos_info_pos,eos_info_str.size());
6570  else {// Check the extra fields
6571  if(Remove_EOS5_Strings_NonEOS_Fields(temp_var_name)==true)
6572  (*irv)->newname = get_CF_string(temp_var_name);
6573  }
6574  }
6575  }
6576 
6577  // Now we need to handle the dimension names.
6578  for (auto irv = this->vars.begin(); irv != this->vars.end(); irv++) {
6579  for (auto ird = (*irv)->dims.begin(); ird!=(*irv)->dims.end(); ++ird) {
6580  string temp_dim_name = (*ird)->newname;
6581  bool remove_eos = Remove_EOS5_Strings(temp_dim_name);
6582 
6583  if(true == remove_eos)
6584  (*ird)->newname = get_CF_string(temp_dim_name);
6585  else {//HDFEOS info and extra fields
6586  string::size_type eos_info_pos = temp_dim_name.find(eos_info_str);
6587  if(eos_info_pos !=string::npos)
6588  (*ird)->newname = temp_dim_name.erase(eos_info_pos,eos_info_str.size());
6589  else {// Check the extra fields
6590  if(Remove_EOS5_Strings_NonEOS_Fields(temp_dim_name)==true)
6591  (*ird)->newname = get_CF_string(temp_dim_name);
6592  }
6593  }
6594  }
6595  }
6596 
6597  // We have to loop through all CVs.
6598  for (auto irv = this->cvars.begin(); irv != this->cvars.end(); ++irv) {
6599  string temp_var_name = (*irv)->newname;
6600 
6601  bool remove_eos = Remove_EOS5_Strings(temp_var_name);
6602 
6603  if(true == remove_eos)
6604  (*irv)->newname = get_CF_string(temp_var_name);
6605  else {//HDFEOS info and extra "fields"
6606  string::size_type eos_info_pos = temp_var_name.find(eos_info_str);
6607  if(eos_info_pos !=string::npos)
6608  (*irv)->newname = temp_var_name.erase(eos_info_pos,eos_info_str.size());
6609  else {// Check the extra fields
6610  if(Remove_EOS5_Strings_NonEOS_Fields(temp_var_name)==true)
6611  (*irv)->newname = get_CF_string(temp_var_name);
6612  }
6613  }
6614  }
6615  // Now we need to handle the dimension names.
6616  for (auto irv = this->cvars.begin(); irv != this->cvars.end(); irv++) {
6617  for (auto ird = (*irv)->dims.begin(); ird!=(*irv)->dims.end(); ++ird) {
6618  string temp_dim_name = (*ird)->newname;
6619  bool remove_eos = Remove_EOS5_Strings(temp_dim_name);
6620 
6621  if(true == remove_eos)
6622  (*ird)->newname = get_CF_string(temp_dim_name);
6623  else {// HDFEOS info and extra "fields"
6624  string::size_type eos_info_pos = temp_dim_name.find(eos_info_str);
6625  if(eos_info_pos !=string::npos)
6626  (*ird)->newname = temp_dim_name.erase(eos_info_pos,eos_info_str.size());
6627  else {// Check the extra "fields"
6628  if(Remove_EOS5_Strings_NonEOS_Fields(temp_dim_name)==true)
6629  (*ird)->newname = get_CF_string(temp_dim_name);
6630  }
6631  }
6632  }
6633  }
6634 
6635  // Update the coordinate attribute values
6636  // We need to remove the HDFEOS special information from the coordinates attributes
6637  // since the variable names in the DAP output are already updated.
6638  for (auto irv = this->vars.begin(); irv != this->vars.end(); irv++) {
6639  for(auto ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end();ira++) {
6640  // We cannot use Retrieve_Str_Attr_value for "coordinates" since "coordinates" may be added by the handler.
6641  // KY 2017-11-3
6642  if((*ira)->name == "coordinates") {
6643  string cor_values((*ira)->value.begin(),(*ira)->value.end()) ;
6644  bool change_cor_values = false;
6645  // Find the HDFEOS string
6646  if(cor_values.find(eos_str)==0) {
6647  if(cor_values.find(grid_str)!=string::npos) {// Grid
6648  cor_values = HDF5CFUtil::remove_substrings(cor_values,eos_str);
6649  cor_values = HDF5CFUtil::remove_substrings(cor_values,grid_str);
6650  string new_cor_values = HDF5CFUtil::remove_substrings(cor_values,df_str);
6651  if(new_cor_values.size() < cor_values.size()){//df_str is also removed.
6652  change_cor_values = true;
6653  cor_values = new_cor_values;
6654  }
6655  }
6656  else if(cor_values.find(zas_str)!=string::npos) {//ZA
6657  cor_values = HDF5CFUtil::remove_substrings(cor_values,eos_str);
6658  cor_values = HDF5CFUtil::remove_substrings(cor_values,zas_str);
6659  string new_cor_values = HDF5CFUtil::remove_substrings(cor_values,df_str);
6660  if(new_cor_values.size() < cor_values.size()){//df_str is also removed.
6661  change_cor_values = true;
6662  cor_values = new_cor_values;
6663  }
6664  }
6665  else if(cor_values.find(swath_str)!=string::npos) {//Swath
6666  cor_values = HDF5CFUtil::remove_substrings(cor_values,eos_str);
6667  cor_values = HDF5CFUtil::remove_substrings(cor_values,swath_str);
6668  string new_cor_values = HDF5CFUtil::remove_substrings(cor_values,df_str);
6669  if(new_cor_values.size() < cor_values.size()){//df_str is also removed.
6670  change_cor_values = true;
6671  cor_values = new_cor_values;
6672  }
6673  else {
6674  new_cor_values = HDF5CFUtil::remove_substrings(cor_values,gf_str);
6675  if(new_cor_values.size() < cor_values.size()){//gf_str is also removed.
6676  change_cor_values = true;
6677  cor_values = new_cor_values;
6678  }
6679  }
6680  }
6681  }
6682  if(true == change_cor_values) {//Update the coordinate values
6683  (*ira)->value.resize(cor_values.size());
6684  (*ira)->fstrsize=cor_values.size();
6685  (*ira)->strsize[0] = cor_values.size();
6686  copy(cor_values.begin(), cor_values.end(), (*ira)->value.begin());
6687  (*irv)->coord_attr_add_path = false;
6688  }
6689 
6690  break;
6691  }
6692  }
6693 
6694  }
6695 }
6696 
6697 // This routine is for handling the hybrid-HDFEOS5 products that have to be treated as "general products"
6698 bool GMFile:: Remove_EOS5_Strings(string &var_name) {
6699 
6700  string eos_str="HDFEOS_";
6701  string grid_str="GRIDS_";
6702  string swath_str="SWATHS_";
6703  string zas_str="ZAS_";
6704  string df_str="Data_Fields_";
6705  string gf_str="Geolocation_Fields_";
6706  string temp_var_name = var_name;
6707 
6708  bool remove_eos = false;
6709 
6710  string::size_type eos_pos = temp_var_name.find(eos_str);
6711  if(eos_pos!=string::npos) {
6712  temp_var_name.erase(eos_pos,eos_str.size());
6713  // Check grid,swath and zonal
6714  string::size_type grid_pos=temp_var_name.find(grid_str);
6715  string::size_type grid_df_pos=string::npos;
6716  if(grid_pos!=string::npos)
6717  grid_df_pos = temp_var_name.find(df_str,grid_pos);
6718  string::size_type zas_pos = string::npos;
6719  string::size_type zas_df_pos=string::npos;
6720  if(grid_pos==string::npos || grid_df_pos ==string::npos)
6721  zas_pos=temp_var_name.find(zas_str);
6722  if(zas_pos!=string::npos)
6723  zas_df_pos=temp_var_name.find(df_str,zas_pos);
6724 
6725  if(grid_pos !=string::npos && grid_df_pos!=string::npos) {
6726  temp_var_name.erase(grid_pos,grid_str.size());
6727  grid_df_pos = temp_var_name.find(df_str);
6728  temp_var_name.erase(grid_df_pos,df_str.size());
6729  remove_eos = true;
6730  }
6731  else if(zas_pos!=string::npos && zas_df_pos!=string::npos){
6732  temp_var_name.erase(zas_pos,zas_str.size());
6733  zas_df_pos = temp_var_name.find(df_str);
6734  temp_var_name.erase(zas_df_pos,df_str.size());
6735  remove_eos = true;
6736  }
6737  else {//Check both Geolocation and Data for Swath
6738 
6739  string::size_type swath_pos=temp_var_name.find(swath_str);
6740  string::size_type swath_df_pos=string::npos;
6741  if(swath_pos!=string::npos)
6742  swath_df_pos=temp_var_name.find(df_str,swath_pos);
6743 
6744  string::size_type swath_gf_pos=string::npos;
6745  if(swath_pos!=string::npos && swath_df_pos == string::npos)
6746  swath_gf_pos=temp_var_name.find(gf_str,swath_pos);
6747 
6748  if(swath_pos !=string::npos) {
6749 
6750  if(swath_df_pos!=string::npos) {
6751  temp_var_name.erase(swath_pos,swath_str.size());
6752  swath_df_pos = temp_var_name.find(df_str);
6753  temp_var_name.erase(swath_df_pos,df_str.size());
6754  remove_eos = true;
6755  }
6756  else if(swath_gf_pos!=string::npos) {
6757  temp_var_name.erase(swath_pos,swath_str.size());
6758  swath_gf_pos = temp_var_name.find(gf_str);
6759  temp_var_name.erase(swath_gf_pos,gf_str.size());
6760  remove_eos = true;
6761  }
6762  }
6763  }
6764  }
6765  if(true == remove_eos)
6766  var_name = temp_var_name;
6767 
6768  return remove_eos;
6769 }
6770 
6771 bool GMFile:: Remove_EOS5_Strings_NonEOS_Fields(string &var_name) {
6772 
6773  string eos_str="HDFEOS_";
6774  string grid_str="GRIDS_";
6775  string swath_str="SWATHS_";
6776  string zas_str="ZAS_";
6777  string temp_var_name = var_name;
6778 
6779  bool remove_eos = false;
6780 
6781  string::size_type eos_pos = temp_var_name.find(eos_str);
6782  if(eos_pos!=string::npos) {
6783  temp_var_name.erase(eos_pos,eos_str.size());
6784  remove_eos = true;
6785 
6786  // See if we need to further remove some fields
6787  if(temp_var_name.find(grid_str)==0)
6788  temp_var_name.erase(0,grid_str.size());
6789  else if(temp_var_name.find(swath_str)==0)
6790  temp_var_name.erase(0,swath_str.size());
6791  else if(temp_var_name.find(zas_str)==0)
6792  temp_var_name.erase(0,zas_str.size());
6793  }
6794  if(true == remove_eos)
6795  var_name = temp_var_name;
6796 
6797 
6798  return remove_eos;
6799 }
6800 
6801 // We do have an AirMSPI HDF-EOS5 hybrid UTM product that has grid_mapping attribute.
6804 }
6805 
6808 }
6809 
6811 
6812  // We need to remove the FakeDim added for the unsupported variables.
6813  // We found such a case in the AirMSPR product. A compound dataype array
6814  // is assigned to a FakeDim. We need to remove them.
6815  // KY 2017-11-2: no need to even check the unsupported_var_dspace now.
6816  if(this->unsupported_var_dtype == true) {
6817 
6818  // Need to check if we have coordinate variables such as FakeDim?
6819  for (auto ircv = this->cvars.begin(); ircv != this->cvars.end();) {
6820  if((*ircv)->newname.find("FakeDim")==0) {
6821  bool var_has_fakedim = false;
6822  for (auto irv2 = this->vars.begin(); irv2 != this->vars.end(); irv2++) {
6823  for (auto ird = (*irv2)->dims.begin(); ird !=(*irv2)->dims.end(); ird++) {
6824  if((*ird)->newname == (*ircv)->newname){
6825  var_has_fakedim = true;
6826  break;
6827  }
6828  }
6829  if(var_has_fakedim == true)
6830  break;
6831  }
6832  if(var_has_fakedim == false) {
6833  // Remove this cv, the variable is unsupported.
6834  delete(*ircv);
6835  ircv = this->cvars.erase(ircv);
6836  }
6837  else
6838  ++ircv;
6839  }
6840  else
6841  ++ircv;
6842  }
6843 #if 0
6844  // We need to handle unlimited dimensions
6845  //if(removed_fakedim_vars.size()!=0) {
6846  //}
6847 #endif
6848  }
6849 
6850 }
6851 
6852 //Rename NC4 NonCoordVars back to the original name. This is detected by CAR_ARCTAS files.
6853 //By handling this way, the output will be the same as the netCDF handler output.
6854 //Check HFVHANDLER-254 for more information.
6856 
6857  if(true == this->have_nc4_non_coord) {
6858  string nc4_non_coord="_nc4_non_coord_";
6859  size_t nc4_non_coord_size= nc4_non_coord.size();
6860  for (auto irv = this->vars.begin(); irv != this->vars.end(); irv++) {
6861  if((*irv)->name.find(nc4_non_coord)==0)
6862  (*irv)->newname = (*irv)->newname.substr(nc4_non_coord_size,(*irv)->newname.size()-nc4_non_coord_size);
6863  }
6864 
6865  for (auto ircv = this->cvars.begin(); ircv != this->cvars.end();++ircv) {
6866  if((*ircv)->name.find(nc4_non_coord)==0)
6867  (*ircv)->newname = (*ircv)->newname.substr(nc4_non_coord_size,(*ircv)->newname.size()-nc4_non_coord_size);
6868  }
6869  }
6870 
6871 }
6872 
6874 
6875  BESDEBUG("h5", "GMFile::Coming to Add_Path_Coor_Attr()"<<endl);
6876  string co_attrname = "coordinates";
6877  for (auto irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
6878  if((*irv)->coord_attr_add_path == true) {
6879  for (auto ira =(*irv)->attrs.begin(); ira !=(*irv)->attrs.end();++ira) {
6880  // We will check if we have the coordinate attribute
6881  if((*ira)->name == co_attrname) {
6882  string coor_value = Retrieve_Str_Attr_Value(*ira,(*irv)->fullpath);
6883  char sep=' ';
6884  vector<string>cvalue_vec;
6885  HDF5CFUtil::Split_helper(cvalue_vec,coor_value,sep);
6886  string new_coor_value;
6887  for (int i = 0; i<cvalue_vec.size();i++) {
6888  HDF5CFUtil::cha_co(cvalue_vec[i],(*irv)->fullpath);
6889  cvalue_vec[i] = get_CF_string(cvalue_vec[i]);
6890  if(i == 0)
6891  new_coor_value = cvalue_vec[i];
6892  else
6893  new_coor_value += sep+cvalue_vec[i];
6894 
6895 #if 0
6896 //cout<<"co1["<<i<<"]= "<<cvalue_vec[i]<<endl;
6897 #endif
6898  }
6899 #if 0
6900 //cout<<"new_coor_value is "<<new_coor_value<<endl;
6901 #endif
6902  Replace_Var_Str_Attr((*irv),co_attrname,new_coor_value);
6903  break;
6904  }
6905  }
6906 
6907  }
6908  }
6909 }
6910 
6911 
6912 
6913 
6914 // We will create some temporary coordinate variables. The resource allocoated
6915 // for these variables need to be released.
6916 void
6917 GMFile::release_standalone_GMCVar_vector(vector<GMCVar*>&tempgc_vars){
6918 
6919  for (auto i = tempgc_vars.begin(); i != tempgc_vars.end(); ) {
6920  delete(*i);
6921  i = tempgc_vars.erase(i);
6922  }
6923 
6924 }
6925 
6926 #if 0
6927 void
6928 GMFile::add_ignored_info_attrs(bool is_grp,bool is_first){
6929 
6930 }
6931 void
6932 GMFile::add_ignored_info_objs(bool is_dim_related, bool is_first) {
6933 
6934 }
6935 #endif
6936 
6937 #if 0
6938 bool
6939 GMFile::ignored_dimscale_ref_list(Var *var) {
6940 
6941  bool ignored_dimscale = true;
6942  if(General_Product == this->product_type && GENERAL_DIMSCALE== this->gproduct_pattern) {
6943 
6944  bool has_dimscale = false;
6945  bool has_reference_list = false;
6946 
6947  for(auto ira = var->attrs.begin(); ira != var->attrs.end();ira++) {
6948 
6949  if((*ira)->name == "REFERENCE_LIST" &&
6950  false == HDF5CFUtil::cf_strict_support_type((*ira)->getType()))
6951  has_reference_list = true;
6952  if((*ira)->name == "CLASS") {
6953  Retrieve_H5_Attr_Value(*ira,var->fullpath);
6954  string class_value;
6955  class_value.resize((*ira)->value.size());
6956  copy((*ira)->value.begin(),(*ira)->value.end(),class_value.begin());
6957 
6958  // Compare the attribute "CLASS" value with "DIMENSION_SCALE". We only compare the string with the size of
6959  // "DIMENSION_SCALE", which is 15.
6960  if (0 == class_value.compare(0,15,"DIMENSION_SCALE")) {
6961  has_dimscale = true;
6962  }
6963  }
6964 
6965  if(true == has_dimscale && true == has_reference_list) {
6966  ignored_dimscale= false;
6967  break;
6968  }
6969 
6970  }
6971  }
6972  return ignored_dimscale;
6973 }
6974 
6975 #endif
This class specifies the core engineering of mapping HDF5 to DAP by following CF.
#define throw1(a1)
The followings are convenient functions to throw exceptions with different.
Definition: HDF5CF.h:128
include the entry functions to execute the handlers
This class represents one attribute.
Definition: HDF5CF.h:189
This class repersents one dimension of an HDF5 dataset(variable).
Definition: HDF5CF.h:145
This class retrieves all information from an HDF5 file.
Definition: HDF5CF.h:563
std::vector< Group * > groups
Non-root group vectors.
Definition: HDF5CF.h:795
virtual void Handle_Unsupported_Dspace(bool)
Handle unsupported HDF5 dataspaces for datasets.
Definition: HDF5CF.cc:1273
virtual void Handle_Grid_Mapping_Vars()
Handle Grid Mapping Vars.
Definition: HDF5CF.cc:2208
virtual void Handle_Unsupported_Others(bool)
Handle other unmapped objects/attributes.
Definition: HDF5CF.cc:1320
virtual void Retrieve_H5_Supported_Attr_Values()
Retrieve attribute values for the supported HDF5 datatypes.
Definition: HDF5CF.cc:727
std::vector< Var * > vars
Var vectors.
Definition: HDF5CF.h:789
virtual void Add_Supplement_Attrs(bool)
Add supplemental attributes such as fullpath and original name.
Definition: HDF5CF.cc:2027
virtual void Retrieve_H5_Info(const char *path, hid_t file_id, bool)
Definition: HDF5CF.cc:168
std::vector< Attribute * > root_attrs
Root attribute vectors.
Definition: HDF5CF.h:792
virtual void Handle_Unsupported_Dtype(bool)
Handle unsupported HDF5 datatypes.
Definition: HDF5CF.cc:916
virtual void Flatten_Obj_Name(bool)
Flatten the object name.
Definition: HDF5CF.cc:1370
virtual bool Have_Grid_Mapping_Attrs()
Check if having Grid Mapping Attrs.
Definition: HDF5CF.cc:2188
This class is a derived class of CVar. It represents a coordinate variable for general HDF5 files.
Definition: HDF5CF.h:413
void Add_Supplement_Attrs(bool) override
Add supplemental attributes such as fullpath and original name for general NASA HDF5 products.
Definition: HDF5GMCF.cc:5173
void Add_Path_Coord_Attr()
Update the coordinate attribute to include path and also flatten.
Definition: HDF5GMCF.cc:6873
void Handle_Obj_NameClashing(bool)
Handle object name clashing for general NASA HDF5 products.
Definition: HDF5GMCF.cc:4954
void Remove_Unused_FakeDimVars()
Unsupported datatype array may generate FakeDim. Remove them.
Definition: HDF5GMCF.cc:6810
void Update_Product_Type()
Update "product type" attributes for general HDF5 products.
Definition: HDF5GMCF.cc:239
bool Have_Grid_Mapping_Attrs() override
Check if having Grid Mapping Attrs.
Definition: HDF5GMCF.cc:6802
void Handle_SpVar_Attr() override
Handle special variable attributes for general NASA HDF5 products.
Definition: HDF5GMCF.cc:6515
void Handle_DimNameClashing() override
Definition: HDF5GMCF.cc:5055
void Handle_Grid_Mapping_Vars() override
Handle Grid Mapping Vars.
Definition: HDF5GMCF.cc:6806
void Adjust_H5_Attr_Value(Attribute *attr)
Adjust attribute values for general HDF5 products.
Definition: HDF5GMCF.cc:373
void Retrieve_H5_CVar_Supported_Attr_Values() override
Retrieve coordinate variable attributes.
Definition: HDF5GMCF.cc:329
void Retrieve_H5_Info(const char *path, hid_t file_id, bool include_attr) override
Retrieve DDS information from the HDF5 file; real implementation for general HDF5 products.
Definition: HDF5GMCF.cc:220
void Handle_SpVar() override
Handle special variables for general NASA HDF5 products.
Definition: HDF5GMCF.cc:4738
void Remove_Unneeded_Objects()
Remove unneeded objects.
Definition: HDF5GMCF.cc:262
void Handle_CVar() override
Handle coordinate variables for general NASA HDF5 products.
Definition: HDF5GMCF.cc:2886
void Add_Dim_Name()
Add dimension name.
Definition: HDF5GMCF.cc:805
void Retrieve_H5_Supported_Attr_Values() override
Retrieve attribute values for the supported HDF5 datatypes for general HDF5 products.
Definition: HDF5GMCF.cc:343
void Handle_Coor_Attr() override
Handle "coordinates" attributes for general HDF5 products.
Definition: HDF5GMCF.cc:5894
void Rename_NC4_NonCoordVars()
Remove the _nc4_non_coord from the variable new names.
Definition: HDF5GMCF.cc:6855
void Handle_Unsupported_Dspace(bool) override
Handle unsupported HDF5 dataspaces for general HDF5 products.
Definition: HDF5GMCF.cc:578
void Handle_Unsupported_Others(bool) override
Handle other unmapped objects/attributes for general HDF5 products.
Definition: HDF5GMCF.cc:668
void Handle_Unsupported_Dtype(bool) override
Handle unsupported HDF5 datatypes for general HDF5 products.
Definition: HDF5GMCF.cc:391
void Flatten_Obj_Name(bool include_attr) override
Flatten the object name for general NASA HDF5 products.
Definition: HDF5GMCF.cc:4905
void Adjust_Dim_Name() override
Adjust dimension name for general NASA HDF5 products.
Definition: HDF5GMCF.cc:5114
void Adjust_Obj_Name() override
Adjust object names based on different general NASA HDF5 products.
Definition: HDF5GMCF.cc:4824
This class is a derived class of Var. It represents a special general HDF5 product(currently ACOS and...
Definition: HDF5CF.h:382
This class represents one HDF5 dataset(CF variable)
Definition: HDF5CF.h:256
const std::vector< Dimension * > & getDimensions() const
Get the list of the dimensions.
Definition: HDF5CF.h:315
static void Split(const char *s, int len, char sep, std::vector< std::string > &names)
Definition: HDF5CFUtil.cc:343