BESUtil.cc

// BESUtil.cc
 
// This file is part of bes, A C++ back-end server implementation framework
// for the OPeNDAP Data Access Protocol.
 
// Copyright (c) 2004-2009 University Corporation for Atmospheric Research
// Author: Patrick West <pwest@ucar.edu> and Jose Garcia <jgarcia@ucar.edu>
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
//
// You can contact University Corporation for Atmospheric Research at
// 3080 Center Green Drive, Boulder, CO 80301
 
// (c) COPYRIGHT University Corporation for Atmospheric Research 2004-2005
// Please read the full copyright statement in the file COPYRIGHT_UCAR.
//
// Authors:
//      pwest       Patrick West <pwest@ucar.edu>
//      jgarcia     Jose Garcia <jgarcia@ucar.edu>
 
#include "config.h"
 
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/resource.h>
 
#include <fcntl.h>
#if HAVE_UNISTD_H
#include <unistd.h>
#endif
 
 
#include <thread>         // std::this_thread::sleep_for
#include <chrono>         // std::chrono::seconds
#include <string>     // std::string, std::stol
#include <cstdio>
#include <cerrno>
#include <cstring>
#include <cstdlib>
#include <ctime>
#include <cassert>
#include <vector>
#include <list>
#include <sstream>
#include <iostream>
#include <algorithm>
 
#include "TheBESKeys.h"
#include "BESUtil.h"
#include "BESDebug.h"
#include "BESForbiddenError.h"
#include "BESNotFoundError.h"
#include "BESInternalError.h"
#include "BESLog.h"
#include "BESCatalogList.h"
 
#include "BESInternalFatalError.h"
#include "RequestServiceTimer.h"
 
using namespace std;
 
#define CRLF "\r\n"
 
#define MODULE "util"
#define prolog string("BESUtil::").append(__func__).append("() - ")
 
const string BES_KEY_TIMEOUT_CANCEL = "BES.CancelTimeoutOnSend";
 
long
BESUtil::get_current_memory_usage() noexcept
{
    struct rusage usage;
    if (getrusage(RUSAGE_SELF, &usage) == 0) { // getrusage()  successful?
#ifdef __APPLE__
        // get the max size (man page says it is in bytes). This function returns the
        // size in KB like Linux. jhrg 3/29/22
        return usage.ru_maxrss / 1024;
#else
        return usage.ru_maxrss; // get the max size (man page says it is in kilobytes)
#endif
    }
    else {
        return 0;
    }
}
 
void BESUtil::trim_if_trailing_slash(string &value)
{
    if (!value.empty() && value.back() == '/')
        value.pop_back();   // requires C++-11
    // value.erase(value.end () -1);
}
 
void BESUtil::trim_if_surrounding_quotes(std::string &value)
{
    if (!value.empty() && value[0] == '"')
        value.erase(0, 1);
    if (!value.empty() && value.back() == '"')
        value.pop_back();   // requires C++-11
    // value.erase(value.end () -1);
}
 
void BESUtil::set_mime_text(ostream &strm)
{
    strm << "HTTP/1.0 200 OK" << CRLF;
    strm << "XBES-Server: " << PACKAGE_STRING << CRLF;
 
    const time_t t = time(0);
    strm << "Date: " << rfc822_date(t).c_str() << CRLF;
    strm << "Last-Modified: " << rfc822_date(t).c_str() << CRLF;
 
    strm << "Content-Type: text/plain" << CRLF;
    // Note that Content-Description is from RFC 2045 (MIME, pt 1), not 2616.
    strm << "Content-Description: unknown" << CRLF;
    strm << CRLF;
}
 
void BESUtil::set_mime_html(ostream &strm)
{
    strm << "HTTP/1.0 200 OK" << CRLF;
    strm << "XBES-Server: " << PACKAGE_STRING << CRLF;
 
    const time_t t = time(0);
    strm << "Date: " << rfc822_date(t).c_str() << CRLF;
    strm << "Last-Modified: " << rfc822_date(t).c_str() << CRLF;
 
    strm << "Content-type: text/html" << CRLF;
    // Note that Content-Description is from RFC 2045 (MIME, pt 1), not 2616.
    strm << "Content-Description: unknown" << CRLF;
    strm << CRLF;
}
 
// Return a MIME rfc-822 date. The grammar for this is:
//       date-time   =  [ day "," ] date time        ; dd mm yy
//                                                   ;  hh:mm:ss zzz
//
//       day         =  "Mon"  / "Tue" /  "Wed"  / "Thu"
//                   /  "Fri"  / "Sat" /  "Sun"
//
//       date        =  1*2DIGIT month 2DIGIT        ; day month year
//                                                   ;  e.g. 20 Jun 82
//                   NB: year is 4 digit; see RFC 1123. 11/30/99 jhrg
//
//       month       =  "Jan"  /  "Feb" /  "Mar"  /  "Apr"
//                   /  "May"  /  "Jun" /  "Jul"  /  "Aug"
//                   /  "Sep"  /  "Oct" /  "Nov"  /  "Dec"
//
//       time        =  hour zone                    ; ANSI and Military
//
//       hour        =  2DIGIT ":" 2DIGIT [":" 2DIGIT]
//                                                   ; 00:00:00 - 23:59:59
//
//       zone        =  "UT"  / "GMT"                ; Universal Time
//                                                   ; North American : UT
//                   /  "EST" / "EDT"                ;  Eastern:  - 5/ - 4
//                   /  "CST" / "CDT"                ;  Central:  - 6/ - 5
//                   /  "MST" / "MDT"                ;  Mountain: - 7/ - 6
//                   /  "PST" / "PDT"                ;  Pacific:  - 8/ - 7
//                   /  1ALPHA                       ; Military: Z = UT;
//                                                   ;  A:-1; (J not used)
//                                                   ;  M:-12; N:+1; Y:+12
//                   / ( ("+" / "-") 4DIGIT )        ; Local differential
//                                                   ;  hours+min. (HHMM)
 
static const char *days[] = { "Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat" };
static const char *months[] = { "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" };
 
string BESUtil::rfc822_date(const time_t t)
{
    struct tm stm{};
    gmtime_r(&t, &stm);
    char d[256];
 
    snprintf(d, 255, "%s, %02d %s %4d %02d:%02d:%02d GMT", days[stm.tm_wday], stm.tm_mday,
             months[stm.tm_mon], 1900 + stm.tm_year, stm.tm_hour, stm.tm_min, stm.tm_sec);
    d[255] = '\0';
    return {d};
}
 
string BESUtil::unhexstring(const string& s)
{
    int val;
    istringstream ss(s);
    ss >> std::hex >> val;
    char tmp_str[2];
    tmp_str[0] = static_cast<char>(val);
    tmp_str[1] = '\0';
    return {tmp_str};
}
 
// I modified this to mirror the version in libdap. The change allows several
// escape sequences to by listed in 'except'. jhrg 2/18/09
string BESUtil::www2id(const string &in, const string &escape, const string &except)
{
    string::size_type i = 0;
    string res = in;
    while ((i = res.find_first_of(escape, i)) != string::npos) {
        if (except.find(res.substr(i, 3)) != string::npos) {
            i += 3;
            continue;
        }
        res.replace(i, 3, unhexstring(res.substr(i + 1, 2)));
    }
 
    return res;
}
 
string BESUtil::lowercase(const string &s)
{
    string return_string = s;
    for (int j = 0; j < static_cast<int>(return_string.length()); j++) {
        return_string[j] = (char) tolower(return_string[j]);
    }
 
    return return_string;
}
 
string BESUtil::unescape(const string &s)
{
    bool done = false;
    string::size_type index = 0;
    /* string::size_type new_index = 0 ; */
    string new_str;
    while (!done) {
        string::size_type bs = s.find('\\', index);
        if (bs == string::npos) {
            new_str += s.substr(index, s.length() - index);
            done = true;
        }
        else {
            new_str += s.substr(index, bs - index);
            new_str += s[bs + 1];
            index = bs + 2;
        }
    }
 
    return new_str;
}
 
static void throw_access_error(const string &pathname, long error_number)
{
    switch(error_number) {
        case ENOENT:
        case ENOTDIR: {
            string message = string("Failed to locate '").append(pathname).append("'");
            INFO_LOG(message);
            throw BESNotFoundError(message, __FILE__, __LINE__);
        }
 
        default: {
            string message = string("Not allowed to access '").append(pathname).append("'");
            INFO_LOG(message);
            throw BESForbiddenError(message, __FILE__, __LINE__);
        }
    }
}
 
bool pathname_contains_symlink(const string &path, int search_limit)
{
    // This kludge to remove a trailing '/' is needed because lstat and readlinkat fail
    // to detect a dir symlink when the dir name ends in '/'. On OSX readlinkat (and readlink)
    // does detect embedded links, but not on Linux. The lstat() service doesn't detect
    // embedded links anywhere. jhrg 1/3/22
    string pathname = path;
    if (!pathname.empty() && pathname.back() == '/') {
        pathname.pop_back();
    }
 
    bool is_link = false;
    size_t pos;
    int i = 0; // used with search_limit
    do {
        // test pathname
        struct stat buf;
        int status = lstat(pathname.c_str(), &buf);
        if (status == 0) {
            is_link = S_ISLNK(buf.st_mode);
        }
        else {
            string msg = "Could not resolve path when testing for symbolic links: ";
            msg.append(strerror(errno));
            BESDEBUG(MODULE, prolog << msg << endl);
            throw BESInternalError(msg, __FILE__, __LINE__);
        }
 
        // remove the last part of pathname, including the trailing '/'
        pos = pathname.find_last_of('/');
        if (pos != string::npos)    // find_last_of returns npos if the char is not found
            pathname.erase(pos);
    } while (++i < search_limit && !is_link && pos != string::npos && !pathname.empty());
 
    return is_link;
 
#if 0
    // ssize_t readlink(const char *restrict pathname, char *restrict buf, size_t bufsiz);
    // readlinkat (or readlink) can be used to detect sym links in a path or to get the path
    // to the linked file. Here we used it to test for sym links. 1/3/22 jhrg
    ssize_t len = readlinkat(AT_FDCWD, pathname.c_str(), nullptr, 0);
    if (len == -1) {
        // either errno is EINVAL meaning this is not a link or there's really an error
        switch (errno) {
            case EINVAL:
                return false;
            default:
                string msg = "Could not resolve path when testing for symbolic links: ";
                msg.append(strerror(errno));
                throw BESInternalError(msg, __FILE__, __LINE__);
        }
    }
 
    return true;    // If readlinkat() does not return -1, it's a symlink
#endif
}
 
void BESUtil::check_path(const string &path, const string &root, bool follow_sym_links) {
    // if nothing is passed in path, then the path checks out since root is assumed to be valid.
    if (path == "") return;
 
    if (path.find("..") != string::npos) {
        throw_access_error(path, EACCES);   // use the code for 'access would be denied'
    }
 
    // Check if the combination of root + path exists on this machine. If so, check if it
    // has symbolic links. Return BESNotFoundError if it does not exist and BESForbiddenError
    // if it does exist but contains symbolic links and follow_sym_links is false. jhrg 12/30/21
 
    string pathname = root;
 
    if (pathname.back() != '/' && path.front() != '/')
        pathname.append("/");
 
    pathname.append(path);
    if (access(pathname.c_str(), R_OK) != 0) {
        throw_access_error(pathname, errno);
    }
 
    if (follow_sym_links == false) {
        auto n = count(path.begin(), path.end(), '/');
        // using 'n' for the search_limit may not be optimal (when path ends in '/', an extra
        // component may be searched) but it's better than testing for a trailing '/' on every call.
        if (pathname_contains_symlink(pathname, n)) {
            throw_access_error(pathname, EACCES);   // use the code for 'access would be denied'
        }
    }
}
 
char *
BESUtil::fastpidconverter(char *buf, int base)
{
    return fastpidconverter(getpid(), buf, base);
}
 
char *
BESUtil::fastpidconverter(long val, /* value to be converted */
char *buf, /* output string         */
int base) /* conversion base       */
{
    ldiv_t r; /* result of val / base  */
 
    if (base > 36 || base < 2) /* no conversion if wrong base */
    {
        *buf = '\0';
        return buf;
    }
    if (val < 0) *buf++ = '-';
    r = ldiv(labs(val), base);
 
    /* output digits of val/base first */
 
    if (r.quot > 0) buf = fastpidconverter(r.quot, buf, base);
    /* output last digit */
 
    *buf++ = "0123456789abcdefghijklmnopqrstuvwxyz"[(int) r.rem];
    *buf = '\0';
    return buf;
}
 
void BESUtil::removeLeadingAndTrailingBlanks(string &key)
{
    if (!key.empty()) {
        string::size_type first = key.find_first_not_of(" \t\n\r");
        string::size_type last = key.find_last_not_of(" \t\n\r");
        if (first == string::npos)
            key = "";
        else {
            string::size_type num = last - first + 1;
            string new_key = key.substr(first, num);
            key = new_key;
        }
    }
}
 
string BESUtil::entity(char c)
{
    switch (c) {
    case '>':
        return "&gt;";
    case '<':
        return "&lt;";
    case '&':
        return "&amp;";
    case '\'':
        return "&apos;";
    case '\"':
        return "&quot;";
    default:
        return string(1, c); // is this proper default, just the char?
    }
}
 
string BESUtil::id2xml(string in, const string &not_allowed)
{
    string::size_type i = 0;
 
    while ((i = in.find_first_of(not_allowed, i)) != string::npos) {
        in.replace(i, 1, entity(in[i]));
        i++;
    }
 
    return in;
}
 
string BESUtil::xml2id(string in)
{
    string::size_type i = 0;
 
    while ((i = in.find("&gt;", i)) != string::npos)
        in.replace(i, 4, ">");
 
    i = 0;
    while ((i = in.find("&lt;", i)) != string::npos)
        in.replace(i, 4, "<");
 
    i = 0;
    while ((i = in.find("&amp;", i)) != string::npos)
        in.replace(i, 5, "&");
 
    i = 0;
    while ((i = in.find("&apos;", i)) != string::npos)
        in.replace(i, 6, "'");
 
    i = 0;
    while ((i = in.find("&quot;", i)) != string::npos)
        in.replace(i, 6, "\"");
 
    return in;
}
 
void BESUtil::explode(char delim, const string &str, list<string> &values)
{
    std::string::size_type start = 0;
    std::string::size_type qstart = 0;
    std::string::size_type adelim = 0;
    std::string::size_type aquote = 0;
    bool done = false;
    while (!done) {
        string aval;
        if (str[start] == '"') {
            bool endquote = false;
            qstart = start + 1;
            while (!endquote) {
                aquote = str.find('"', qstart);
                if (aquote == string::npos) {
                    string currval = str.substr(start, str.length() - start);
                    string err = "BESUtil::explode - No end quote after value " + currval;
                    throw BESInternalError(err, __FILE__, __LINE__);
                }
                // could be an escaped escape character and an escaped
                // quote, or an escaped escape character and a quote
                if (str[aquote - 1] == '\\') {
                    if (str[aquote - 2] == '\\') {
                        endquote = true;
                        qstart = aquote + 1;
                    }
                    else {
                        qstart = aquote + 1;
                    }
                }
                else {
                    endquote = true;
                    qstart = aquote + 1;
                }
            }
            if (str[qstart] != delim && qstart != str.length()) {
                string currval = str.substr(start, qstart - start);
                string err = "BESUtil::explode - No delim after end quote " + currval;
                throw BESInternalError(err, __FILE__, __LINE__);
            }
            if (qstart == str.length()) {
                adelim = string::npos;
            }
            else {
                adelim = qstart;
            }
        }
        else {
            adelim = str.find(delim, start);
        }
        if (adelim == string::npos) {
            aval = str.substr(start, str.length() - start);
            done = true;
        }
        else {
            aval = str.substr(start, adelim - start);
        }
 
        values.push_back(aval);
        start = adelim + 1;
        if (start == str.length()) {
            values.push_back("");
            done = true;
        }
    }
}
 
string BESUtil::implode(const list<string> &values, char delim)
{
    string result;
    list<string>::const_iterator i = values.begin();
    list<string>::const_iterator e = values.end();
    bool first = true;
    string::size_type d; // = string::npos ;
    for (; i != e; i++) {
        if (!first) result += delim;
        d = (*i).find(delim);
        if (d != string::npos && (*i)[0] != '"') {
            string err = (string) "BESUtil::implode - delimiter exists in value " + (*i);
            throw BESInternalError(err, __FILE__, __LINE__);
        }
        //d = string::npos ;
        result += (*i);
        first = false;
    }
    return result;
}
 
void BESUtil::url_explode(const string &url_str, BESUtil::url &url_parts)
{
    string rest;
 
    string::size_type colon = url_str.find(":");
    if (colon == string::npos) {
        string err = "BESUtil::url_explode: missing colon for protocol";
        throw BESInternalError(err, __FILE__, __LINE__);
    }
 
    url_parts.protocol = url_str.substr(0, colon);
 
    if (url_str.substr(colon, 3) != "://") {
        string err = "BESUtil::url_explode: no :// in the URL";
        throw BESInternalError(err, __FILE__, __LINE__);
    }
 
    colon += 3;
    rest = url_str.substr(colon);
 
    string::size_type slash = rest.find("/");
    if (slash == string::npos) slash = rest.length();
 
    string::size_type at = rest.find("@");
    if ((at != string::npos) && (at < slash)) {
        // everything before the @ is username:password
        string up = rest.substr(0, at);
        colon = up.find(":");
        if (colon != string::npos) {
            url_parts.uname = up.substr(0, colon);
            url_parts.psswd = up.substr(colon + 1);
        }
        else {
            url_parts.uname = up;
        }
        // everything after the @ is domain/path
        rest = rest.substr(at + 1);
    }
    slash = rest.find("/");
    if (slash == string::npos) slash = rest.length();
    colon = rest.find(":");
    if ((colon != string::npos) && (colon < slash)) {
        // everything before the colon is the domain
        url_parts.domain = rest.substr(0, colon);
        // everything after the folon is port/path
        rest = rest.substr(colon + 1);
        slash = rest.find("/");
        if (slash != string::npos) {
            url_parts.port = rest.substr(0, slash);
            url_parts.path = rest.substr(slash + 1);
        }
        else {
            url_parts.port = rest;
            url_parts.path = "";
        }
    }
    else {
        slash = rest.find("/");
        if (slash != string::npos) {
            url_parts.domain = rest.substr(0, slash);
            url_parts.path = rest.substr(slash + 1);
        }
        else {
            url_parts.domain = rest;
        }
    }
}
 
string BESUtil::url_create(BESUtil::url &url_parts)
{
    string url = url_parts.protocol + "://";
    if (!url_parts.uname.empty()) {
        url += url_parts.uname;
        if (!url_parts.psswd.empty()) url += ":" + url_parts.psswd;
        url += "@";
    }
    url += url_parts.domain;
    if (!url_parts.port.empty()) url += ":" + url_parts.port;
    if (!url_parts.path.empty()) url += "/" + url_parts.path;
 
    return url;
}
 
 
string BESUtil::pathConcat(const string &firstPart, const string &secondPart, char separator)
{
    string first = firstPart;
    string second = secondPart;
    string sep(1,separator);
 
    // make sure there are not multiple slashes at the end of the first part...
    // Note that this removes all the slashes. jhrg 9/27/16
    while (!first.empty() && *first.rbegin() == separator) {
        // C++-11 first.pop_back();
        first = first.substr(0, first.length() - 1);
    }
    // make sure second part does not BEGIN with a slash
    while (!second.empty() && second[0] == separator) {
        // erase is faster? second = second.substr(1);
        second.erase(0, 1);
    }
    string newPath;
    if (first.empty()) {
        newPath = second;
    }
    else if (second.empty()) {
        newPath = first;
    }
    else {
        newPath = first.append(sep).append(second);
    }
    return newPath;
}
string BESUtil::assemblePath(const string &firstPart, const string &secondPart, bool leadingSlash, bool trailingSlash)
{
    BESDEBUG(MODULE, prolog << "firstPart:  '" << firstPart << "'" << endl);
    BESDEBUG(MODULE, prolog << "secondPart: '" << secondPart << "'" << endl);
 
    string newPath = BESUtil::pathConcat(firstPart, secondPart);
    if (leadingSlash) {
        if (newPath.empty()) {
            newPath = "/";
        }
        else if (newPath.front() != '/') {
            newPath = "/" + newPath;
        }
    }
 
    if (trailingSlash) {
        if (newPath.empty() || newPath.back() != '/') {
            newPath.append("/");
        }
    }
    else {
        while (!newPath.empty() && newPath.back() == '/')
            newPath.erase(newPath.length()-1);
    }
 
    BESDEBUG(MODULE, prolog << "newPath: " << newPath << endl);
    return newPath;
}
 
bool BESUtil::endsWith(string const &fullString, string const &ending)
{
    if (fullString.length() >= ending.length()) {
        return (0 == fullString.compare(fullString.length() - ending.length(), ending.length(), ending));
    }
    else {
        return false;
    }
}
 
void BESUtil::exit_on_request_timeout()
{
    if (RequestServiceTimer::TheTimer()->is_expired()) {
        stringstream msg;
        msg << "The submitted request took too long to service.";
        throw BESInternalFatalError(msg.str(), __FILE__, __LINE__);
    }
}
 
void BESUtil::conditional_timeout_cancel()
{
    const string false_str = "false";
    const string no_str = "no";
 
    bool cancel_timeout_on_send = true;
    bool found = false;
    string value;
 
    TheBESKeys::TheKeys()->get_value(BES_KEY_TIMEOUT_CANCEL, value, found);
    if (found) {
        value = BESUtil::lowercase(value);
        if ( value == false_str || value == no_str) cancel_timeout_on_send = false;
    }
    BESDEBUG(MODULE, __func__ << "() - cancel_timeout_on_send: " << (cancel_timeout_on_send ? "true" : "false") << endl);
    if (cancel_timeout_on_send) {
        RequestServiceTimer::TheTimer()->disable_timeout();
        alarm(0);
    }
}
 
unsigned int BESUtil::replace_all(string &s, string find_this, string replace_with_this)
{
    unsigned int replace_count = 0;
    size_t pos = s.find(find_this);
    while (pos != string::npos) {
        // Replace current matching substring
        s.replace(pos, find_this.size(), replace_with_this);
        // Get the next occurrence from current position
        pos = s.find(find_this, pos + replace_with_this.size());
        replace_count++;
    }
    return replace_count;
}
 
string BESUtil::normalize_path(const string &raw_path, bool leading_separator, bool trailing_separator, const string separator /* = "/" */)
{
    if (separator.length() != 1)
        throw BESInternalError("Path separators must be a single character. The string '" + separator + "' does not qualify.", __FILE__, __LINE__);
    char separator_char = separator[0];
    string double_separator;
    double_separator = double_separator.append(separator).append(separator);
 
    string path(raw_path);
 
    replace_all(path, double_separator, separator);
 
    if (path.empty()) {
        path = separator;
    }
    if (path == separator) {
        return path;
    }
    if (leading_separator) {
        if (path[0] != separator_char) {
            path = string(separator).append(path);
        }
    }
    else {
        if (path[0] == separator_char) {
            path = path.substr(1);
        }
    }
    if (trailing_separator) {
        if (*path.rbegin() != separator_char) {
            path = path.append(separator);
        }
    }
    else {
        if (*path.rbegin() == separator_char) {
            path = path.substr(0, path.length() - 1);
        }
    }
    return path;
}
 
void BESUtil::tokenize(const string& str, vector<string>& tokens, const string& delimiters /* = "/" */)
{
    // Skip delimiters at beginning.
    string::size_type lastPos = str.find_first_not_of(delimiters, 0);
    // Find first "non-delimiter".
    string::size_type pos = str.find_first_of(delimiters, lastPos);
    while (string::npos != pos || string::npos != lastPos) {
        // Found a token, add it to the vector.
        tokens.push_back(str.substr(lastPos, pos - lastPos));
        // Skip delimiters.  Note the "not_of"
        lastPos = str.find_first_not_of(delimiters, pos);
        // Find next "non-delimiter"
        pos = str.find_first_of(delimiters, lastPos);
    }
}
 
string BESUtil::get_time(bool use_local_time)
{
    return get_time(time(0), use_local_time);
}
 
string BESUtil::get_time(time_t the_time, bool use_local_time)
{
    char buf[sizeof "YYYY-MM-DDTHH:MM:SS zones"];
    int status = 0;
 
    // From StackOverflow:
    // This will work too, if your compiler doesn't support %F or %T:
    // strftime(buf, sizeof buf, "%Y-%m-%dT%H:%M:%S%Z", gmtime(&now));
    //
    // UTC is the default. Override to local time based on the
    // passed parameter 'use_local_time'
    struct tm result{};
    if (!use_local_time) {
        gmtime_r(&the_time, &result);
        status = strftime(buf, sizeof buf, "%FT%T%Z", &result);
    }
    else {
        localtime_r(&the_time, &result);
        status = strftime(buf, sizeof buf, "%FT%T%Z", &result);
    }
 
    if (!status) {
        ERROR_LOG(prolog + "Error formatting time value!");
        return "date-format-error";
    }
 
    return buf;
}
 
vector<string> BESUtil::split(const string &s, char delim /* '/' */, bool skip_empty /* true */)
{
    stringstream ss(s);
    string item;
    vector<string> tokens;
 
    while (getline(ss, item, delim)) {
 
        if (item.empty() && skip_empty)
            continue;
 
        tokens.push_back(item);
    }
 
    return tokens;
}
 
BESCatalog *BESUtil::separateCatalogFromPath(std::string &ppath)
{
    BESCatalog *catalog = 0;    // pointer to a singleton; do not delete
    vector<string> path_tokens;
 
    // BESUtil::normalize_path() removes duplicate separators and adds leading and trailing separators as directed.
    string path = BESUtil::normalize_path(ppath, false, false);
    BESDEBUG(MODULE, prolog << "Normalized path: " << path << endl);
 
    // Because we may need to alter the container/file/resource name by removing
    // a catalog name from the first node in the path we use "use_container" to store
    // the altered container path.
    string use_container = ppath;
 
    // Breaks path into tokens
    BESUtil::tokenize(path, path_tokens);
    if (!path_tokens.empty()) {
        BESDEBUG(MODULE, "First path token: " << path_tokens[0] << endl);
        catalog = BESCatalogList::TheCatalogList()->find_catalog(path_tokens[0]);
        if (catalog) {
            BESDEBUG(MODULE, prolog << "Located catalog " << catalog->get_catalog_name() << " from path component" << endl);
            // Since the catalog name is in the path we
            // need to drop it this should leave container
            // with a leading
            ppath = BESUtil::normalize_path(path.substr(path_tokens[0].length()), true, false);
            BESDEBUG(MODULE, prolog << "Modified container/path value to:  " << use_container << endl);
        }
    }
 
    return catalog;
}
 
void ios_state_msg(std::ios &ios_ref, std::stringstream &msg) {
    msg << " {ios.good()=" << (ios_ref.good() ? "true" : "false") << "}";
    msg << " {ios.eof()="  <<  (ios_ref.eof()?"true":"false") << "}";
    msg << " {ios.fail()=" << (ios_ref.fail()?"true":"false") << "}";
    msg << " {ios.bad()="  <<  (ios_ref.bad()?"true":"false") << "}";
}
 
// size of the buffer used to read from the temporary file built on disk and
// send data to the client over the network connection (socket/stream)
#define OUTPUT_FILE_BLOCK_SIZE 4096
 
void BESUtil::file_to_stream(const std::string &file_name, std::ostream &o_strm)
{
    stringstream msg;
    msg << prolog << "Using ostream: " << (void *) &o_strm << " cout: " << (void *) &cout << endl;
    BESDEBUG(MODULE,  msg.str());
    INFO_LOG( msg.str());
 
    vector<char> rbuffer(OUTPUT_FILE_BLOCK_SIZE);
    std::ifstream i_stream(file_name, std::ios_base::in | std::ios_base::binary);  // Use binary mode so we can
 
    // good() returns true if !(eofbit || badbit || failbit)
    if(!i_stream.good()){
        stringstream msg;
        msg << prolog << "Failed to open file " << file_name;
        ios_state_msg(i_stream, msg);
        BESDEBUG(MODULE, msg.str() << endl);
        throw BESInternalError(msg.str(),__FILE__,__LINE__);
    }
 
    // good() returns true if !(eofbit || badbit || failbit)
    if(!o_strm.good()){
        stringstream msg;
        msg << prolog << "Problem with ostream. " << file_name;
        ios_state_msg(i_stream, msg);
        BESDEBUG(MODULE, msg.str() << endl);
        throw BESInternalError(msg.str(),__FILE__,__LINE__);
    }
 
    //vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv
    // This is where the file is copied.
    uint64_t tcount = 0;
    while (i_stream.good() && o_strm.good()){
        i_stream.read(rbuffer.data(), OUTPUT_FILE_BLOCK_SIZE);      // Read at most n bytes into
        o_strm.write(rbuffer.data(), i_stream.gcount()); // buf, then write the buf to
        tcount += i_stream.gcount();
    }
    o_strm.flush();
 
    // fail() is true if failbit || badbit got set, but does not consider eofbit
    if(i_stream.fail() && !i_stream.eof()){
        stringstream msg;
        msg << prolog << "There was an ifstream error when reading from: " << file_name;
        ios_state_msg(i_stream, msg);
        msg << " last_lap: " << i_stream.gcount() << " bytes";
        msg << " total_read: " << tcount << " bytes";
        BESDEBUG(MODULE, msg.str() << endl);
        throw BESInternalError(msg.str(),__FILE__,__LINE__);
    }
 
    // If we're not at the eof of the input stream then we have failed.
    if (!i_stream.eof()){
        stringstream msg;
        msg << prolog << "Failed to reach EOF on source file: " << file_name;
        ios_state_msg(i_stream, msg);
        msg << " last_lap: " << i_stream.gcount() << " bytes";
        msg << " total_read: " << tcount << " bytes";
        BESDEBUG(MODULE, msg.str() << endl);
        throw BESInternalError(msg.str(),__FILE__,__LINE__);
    }
 
    // And if something went wrong on the output stream we have failed.
    if(!o_strm.good()){
        stringstream msg;
        msg << prolog << "There was an ostream error during transmit. Transmitted " << tcount  << " bytes.";
        ios_state_msg(o_strm, msg);
        auto crntpos = o_strm.tellp();
        msg << " current_position: " << crntpos << endl;
        BESDEBUG(MODULE, msg.str());
        ERROR_LOG(msg.str());
    }
 
    msg.str("");
    msg << prolog << "Sent "<< tcount << " bytes from file '" << file_name<< "'. " << endl;
    BESDEBUG(MODULE,msg.str());
    INFO_LOG(msg.str());
}
 
uint64_t BESUtil::file_to_stream_helper(const std::string &file_name, std::ostream &o_strm, uint64_t byteCount){
 
    stringstream msg;
    msg << prolog << "Using ostream: " << (void *) &o_strm << " cout: " << (void *) &cout << endl;
    BESDEBUG(MODULE,  msg.str());
    INFO_LOG( msg.str());
 
    vector<char> rbuffer(OUTPUT_FILE_BLOCK_SIZE);
    std::ifstream i_stream(file_name, std::ios_base::in | std::ios_base::binary);  // Use binary mode so we can
 
    // good() returns true if !(eofbit || badbit || failbit)
    if(!i_stream.good()){
        stringstream msg;
        msg << prolog << "Failed to open file " << file_name;
        ios_state_msg(i_stream, msg);
        BESDEBUG(MODULE, msg.str() << endl);
        throw BESInternalError(msg.str(),__FILE__,__LINE__);
    }
 
    // good() returns true if !(eofbit || badbit || failbit)
    if(!o_strm.good()){
        stringstream msg;
        msg << prolog << "Problem with ostream. " << file_name;
        ios_state_msg(i_stream, msg);
        BESDEBUG(MODULE, msg.str() << endl);
        throw BESInternalError(msg.str(),__FILE__,__LINE__);
    }
 
    // this is where we advance to the last byte that was read
    i_stream.seekg(byteCount);
 
    //vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv
    // This is where the file is copied.
    while (i_stream.good() && o_strm.good()){
        i_stream.read(rbuffer.data(), OUTPUT_FILE_BLOCK_SIZE);      // Read at most n bytes into
        o_strm.write(rbuffer.data(), i_stream.gcount()); // buf, then write the buf to
        BESDEBUG(MODULE, "i_stream: " << i_stream.gcount() << endl);
        byteCount += i_stream.gcount();
    }
    o_strm.flush();
 
    // fail() is true if failbit || badbit got set, but does not consider eofbit
    if(i_stream.fail() && !i_stream.eof()){
        stringstream msg;
        msg << prolog << "There was an ifstream error when reading from: " << file_name;
        ios_state_msg(i_stream, msg);
        msg << " last_lap: " << i_stream.gcount() << " bytes";
        msg << " total_read: " << byteCount << " bytes";
        BESDEBUG(MODULE, msg.str() << endl);
        throw BESInternalError(msg.str(),__FILE__,__LINE__);
    }
 
    // If we're not at the eof of the input stream then we have failed.
    if (!i_stream.eof()){
        stringstream msg;
        msg << prolog << "Failed to reach EOF on source file: " << file_name;
        ios_state_msg(i_stream, msg);
        msg << " last_lap: " << i_stream.gcount() << " bytes";
        msg << " total_read: " << byteCount << " bytes";
        BESDEBUG(MODULE, msg.str() << endl);
        throw BESInternalError(msg.str(),__FILE__,__LINE__);
    }
 
    // And if something went wrong on the output stream we have failed.
    if(!o_strm.good()){
        stringstream msg;
        msg << prolog << "There was an ostream error during transmit. Transmitted " << byteCount  << " bytes.";
        ios_state_msg(o_strm, msg);
        auto crntpos = o_strm.tellp();
        msg << " current_position: " << crntpos << endl;
        BESDEBUG(MODULE, msg.str());
        ERROR_LOG(msg.str());
    }
 
    msg.str(prolog);
    msg << "Sent "<< byteCount << " bytes from file '" << file_name<< "'. " << endl;
    BESDEBUG(MODULE,msg.str());
    INFO_LOG(msg.str());
 
    i_stream.close();
 
    return byteCount;
}
 
 
// I added this because maybe using the low-level file calls was important. I'm not
// sure and the iostreams in C++ are safer. jhrg 6/4/21
#define FILE_CALLS 0
 
uint64_t BESUtil::file_to_stream_task(const std::string &file_name, std::atomic<bool> &file_write_done, std::ostream &o_strm) {
    stringstream msg;
    msg << prolog << "Using ostream: " << (void *) &o_strm << " cout: " << (void *) &cout << endl;
    BESDEBUG(MODULE, msg.str());
    INFO_LOG(msg.str());
 
    vector<char> rbuffer(OUTPUT_FILE_BLOCK_SIZE);
 
    std::ifstream i_stream(file_name, std::ios_base::in | std::ios_base::binary);
#if FILE_CALLS
    int fd = open(file_name.c_str(), O_RDONLY | O_NONBLOCK);
    int eof = false;
#endif
 
    //vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv
    // This is where the file is copied.
    BESDEBUG(MODULE, "Starting transfer" << endl);
    uint64_t tcount = 0;
    while (!i_stream.bad() && !i_stream.fail() && o_strm.good()) {
        if (file_write_done && i_stream.eof()) {
            BESDEBUG(MODULE, "breaking out of loop" << endl);
            break;
        }
        else {
            i_stream.read(rbuffer.data(), OUTPUT_FILE_BLOCK_SIZE);      // Read at most n bytes into
 
#if FILE_CALLS
            int status = read(fd, rbuffer.data(), OUTPUT_FILE_BLOCK_SIZE);
            if (status == 0) {
                eof = true;
            }
            else if (status == -1) {
                BESDEBUG(MODULE, "read() call error: " << errno << endl);
            }
 
            o_strm.write(rbuffer.data(), status); // buf, then write the buf to
            tcount += status;
#endif
 
            o_strm.write(rbuffer.data(), i_stream.gcount()); // buf, then write the buf to
            tcount += i_stream.gcount();
            BESDEBUG(MODULE, "transfer bytes " << tcount << endl);
        }
    }
 
#if FILE_CALLS
    close(fd);
#endif
    o_strm.flush();
 
    // And if something went wrong on the output stream we have failed.
    if(!o_strm.good()){
        stringstream msg;
        msg << prolog << "There was an ostream error during transmit. Transmitted " << tcount  << " bytes.";
        ios_state_msg(o_strm, msg);
        auto crntpos = o_strm.tellp();
        msg << " current_position: " << crntpos << endl;
        BESDEBUG(MODULE, msg.str());
        INFO_LOG(msg.str());
    }
 
    msg.str(prolog);
    msg << "Sent "<< tcount << " bytes from file '" << file_name<< "'. " << endl;
    BESDEBUG(MODULE,msg.str());
    INFO_LOG(msg.str());
 
    return tcount;
}
 
#if 0
 
void BESUtil::split(const string &s, const string &delimiter, vector<uint64_t> &res)
{
    const size_t delim_len = delimiter.length();
 
    size_t pos_start = 0, pos_end;
 
    while ((pos_end = s.find (delimiter, pos_start)) != string::npos) {
        res.push_back (stoull(s.substr(pos_start, pos_end - pos_start)));
        pos_start = pos_end + delim_len;
    }
 
    res.push_back (stoull(s.substr (pos_start)));
}
 
void BESUtil::split(const string &s, const string &delimiter, vector<string> &res)
{
    const size_t delim_len = delimiter.length();
 
    size_t pos_start = 0, pos_end;
 
    while ((pos_end = s.find (delimiter, pos_start)) != string::npos) {
        res.push_back(s.substr(pos_start, pos_end - pos_start));
        pos_start = pos_end + delim_len;
    }
 
    res.push_back(s.substr (pos_start));
}
#endif