From 458bbd41c8f075bf7fa21e3bdc0325a849941de1 Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Tue, 28 Oct 2025 15:45:31 -0400 Subject: [PATCH 01/77] wip get urls from dmrpp --- http/EffectiveUrlCache.cc | 28 +++++++++++++++++++ http/EffectiveUrlCache.h | 13 +++++++++ modules/dmrpp_module/Chunk.cc | 11 ++++++-- .../ngap_container/NgapOwnedContainer.cc | 18 ++++++++++++ .../ngap_container/NgapOwnedContainer.h | 1 + 5 files changed, 69 insertions(+), 2 deletions(-) diff --git a/http/EffectiveUrlCache.cc b/http/EffectiveUrlCache.cc index ef318a442d..2a00c40dd0 100644 --- a/http/EffectiveUrlCache.cc +++ b/http/EffectiveUrlCache.cc @@ -160,6 +160,34 @@ shared_ptr EffectiveUrlCache::get_effective_url(shared_ptr return effective_url; } +// TODO: docstring +void EffectiveUrlCache::cache_signed_url_components(const std::string &key, const std::string &s3_url, const std::string &s3credentials_url) { + // TODO: if urls are empty, do....something?? skip it or something + d_href_to_s3credentials_cache[key] = s3_url; + d_href_to_s3_cache[key] = s3credentials_url; +} + +/** + * TODO + * + * @param source_url + * @returns The effective (signed) URL +*/ +shared_ptr EffectiveUrlCache::get_signed_url(shared_ptr source_url) { + BESDEBUG(MODULE, prolog << "GET SIGNED URL NOT YET IMPLEMENTED, falling back to get_effective_url." << endl); + + auto s3_url = d_href_to_s3credentials_cache[source_url->str()]; + auto s3credentials_url = d_href_to_s3_cache[source_url->str()]; + //TODO-implement! Copy get_effective_url boilerplate, then... + /* + 1. Get s3credentials endpoint for source_url + 2. Get values from s3credentials enpoint (pre-cached, but for now can get direct) + 3. Get s3 url for source_url + 4. use aws sdk to sign the url + */ + return get_effective_url(source_url); +} + /** * @return Is the cache enabled (set in the bes.conf file)? */ diff --git a/http/EffectiveUrlCache.h b/http/EffectiveUrlCache.h index 27f85d4d36..8689833cb6 100644 --- a/http/EffectiveUrlCache.h +++ b/http/EffectiveUrlCache.h @@ -58,6 +58,10 @@ class EffectiveUrlCache : public BESObj { std::map> d_effective_urls; + // TODO- Will be moved to new class before merge + std::map d_href_to_s3credentials_cache; + std::map d_href_to_s3_cache; + // URLs that match are not cached. std::unique_ptr d_skip_regex = nullptr; @@ -94,6 +98,15 @@ class EffectiveUrlCache : public BESObj { ~EffectiveUrlCache() override = default; std::shared_ptr get_effective_url(std::shared_ptr source_url); + + /* + * TODO: temporarily lives here; move into own class before merge! + */ + std::shared_ptr get_signed_url(std::shared_ptr source_url); + void cache_signed_url_components(const std::string &key, const std::string &s3_url, const std::string &s3credentials_url); + /* + * End temp contents! + */ void dump(std::ostream &strm) const override; diff --git a/modules/dmrpp_module/Chunk.cc b/modules/dmrpp_module/Chunk.cc index e384298f7a..e09c819fee 100644 --- a/modules/dmrpp_module/Chunk.cc +++ b/modules/dmrpp_module/Chunk.cc @@ -1348,7 +1348,8 @@ string Chunk::to_string() const { * This method returns the data URL for this chunk. If the data URL is not * set, it returns nullptr. * - * @note The call to get_effective_url() will call EffectiveUrlCache::get_effective_url() + * @note The call to get_signed_url() will first attempt to create a locally-signed url; if + * that fails, it will fall through to calling EffectiveUrlCache::get_effective_url() * which will call CurlUtils.cc get_redirect_url() which will call gru_mk_attempt() and * will look for an HTTP 302 response and return the redirect URL in that response. * @@ -1359,7 +1360,13 @@ std::shared_ptr Chunk::get_data_url() const { // The d_data_url may be nullptr(fillvalue case). if (d_data_url == nullptr) return d_data_url; - std::shared_ptr effective_url = EffectiveUrlCache::TheCache()->get_effective_url(d_data_url); + + std::shared_ptr effective_url = EffectiveUrlCache::TheCache()->get_signed_url(d_data_url); + + if (effective_url == nullptr) { + BESDEBUG(MODULE, prolog << "No signed url generated; constructing effective_url via redirects." << endl); + effective_url = EffectiveUrlCache::TheCache()->get_signed_url(d_data_url); + } BESDEBUG(MODULE, prolog << "Using data_url: " << effective_url->str() << endl); #if ENABLE_TRACKING_QUERY_PARAMETER diff --git a/modules/dmrpp_module/ngap_container/NgapOwnedContainer.cc b/modules/dmrpp_module/ngap_container/NgapOwnedContainer.cc index 5b90362860..feebcaf00a 100644 --- a/modules/dmrpp_module/ngap_container/NgapOwnedContainer.cc +++ b/modules/dmrpp_module/ngap_container/NgapOwnedContainer.cc @@ -35,6 +35,7 @@ #include #include +#include #include "BESStopWatch.h" #include "BESUtil.h" @@ -45,6 +46,7 @@ #include "TheBESKeys.h" #include "BESSyntaxUserError.h" #include "BESDebug.h" +#include "EffectiveUrlCache.h" #include "NgapOwnedContainer.h" #include "NgapNames.h" @@ -61,6 +63,7 @@ using namespace std; using namespace bes; +using http::EffectiveUrlCache; namespace ngap { @@ -559,6 +562,15 @@ bool NgapOwnedContainer::get_dmrpp_from_cache_or_remote_source(string &dmrpp_str return true; } +// TODO-docstring +NgapApi::DataAccessUrls NgapOwnedContainer::extract_s3_data_urls_from_dmrpp(const string &dmrpp_string) { + + //TODO: pull from DMZ::build_thin_dmr code + // TODO: handle the case where nothing is found in dmrpp + + return tie("foo_href", "foo_s3", "foo_s3credentials"); +} + /** * @brief Get the DMR++ from a remote source or a local cache * @@ -578,6 +590,12 @@ string NgapOwnedContainer::access() { // get the remote DMR++. jhrg 4/29/24 get_dmrpp_from_cache_or_remote_source(dmrpp_string); + // To sign urls locally, we need access to the credential info that has been previously + // injected into the dmrpp. Extract that now, in preparation for upcoming url signing. + auto urls = extract_s3_data_urls_from_dmrpp(dmrpp_string); + assert(get<1>(urls) == get_real_name()); // TODO: better way to do this?? + EffectiveUrlCache::TheCache()->cache_signed_url_components(get_real_name(), get<1>(urls), get<2>(urls)); + set_attributes("as-string"); // This means access() returns a string. jhrg 10/19/23 // Originally, this was either hard-coded (as it is now) or was set using the 'extension' // on the URL. But it's always a DMR++. jhrg 11/16/23 diff --git a/modules/dmrpp_module/ngap_container/NgapOwnedContainer.h b/modules/dmrpp_module/ngap_container/NgapOwnedContainer.h index 3be72acd60..115c18f1e6 100644 --- a/modules/dmrpp_module/ngap_container/NgapOwnedContainer.h +++ b/modules/dmrpp_module/ngap_container/NgapOwnedContainer.h @@ -87,6 +87,7 @@ class NgapOwnedContainer: public BESContainer { static FileCache d_dmrpp_file_cache; bool get_dmrpp_from_cache_or_remote_source(std::string &dmrpp_string) const; + static NgapApi::DataAccessUrls extract_s3_data_urls_from_dmrpp(const std::string &dmrpp_string); // I made these statics so that they will be in the class' namespace but still // easy to test in the unit tests. jhrg 4/29/24 From be8ecf48c61c90cd4672bdc84cf7910ad6a489c8 Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Tue, 28 Oct 2025 17:26:31 -0400 Subject: [PATCH 02/77] urls available from EffectiveUrlCache --- http/EffectiveUrlCache.cc | 23 +++++---- modules/dmrpp_module/Chunk.cc | 2 +- .../dmrpp_module/ngap_container/Makefile.am | 2 +- .../ngap_container/NgapOwnedContainer.cc | 51 ++++++++++++++++--- 4 files changed, 59 insertions(+), 19 deletions(-) diff --git a/http/EffectiveUrlCache.cc b/http/EffectiveUrlCache.cc index 2a00c40dd0..cd925ddf01 100644 --- a/http/EffectiveUrlCache.cc +++ b/http/EffectiveUrlCache.cc @@ -163,8 +163,8 @@ shared_ptr EffectiveUrlCache::get_effective_url(shared_ptr // TODO: docstring void EffectiveUrlCache::cache_signed_url_components(const std::string &key, const std::string &s3_url, const std::string &s3credentials_url) { // TODO: if urls are empty, do....something?? skip it or something - d_href_to_s3credentials_cache[key] = s3_url; - d_href_to_s3_cache[key] = s3credentials_url; + d_href_to_s3_cache[key] = s3_url; + d_href_to_s3credentials_cache[key] = s3credentials_url; } /** @@ -176,15 +176,18 @@ void EffectiveUrlCache::cache_signed_url_components(const std::string &key, cons shared_ptr EffectiveUrlCache::get_signed_url(shared_ptr source_url) { BESDEBUG(MODULE, prolog << "GET SIGNED URL NOT YET IMPLEMENTED, falling back to get_effective_url." << endl); - auto s3_url = d_href_to_s3credentials_cache[source_url->str()]; - auto s3credentials_url = d_href_to_s3_cache[source_url->str()]; //TODO-implement! Copy get_effective_url boilerplate, then... - /* - 1. Get s3credentials endpoint for source_url - 2. Get values from s3credentials enpoint (pre-cached, but for now can get direct) - 3. Get s3 url for source_url - 4. use aws sdk to sign the url - */ + + auto s3_url = d_href_to_s3_cache[source_url->str()]; + auto s3credentials_url = d_href_to_s3credentials_cache[source_url->str()]; + if (s3_url.empty() || s3credentials_url.empty()) { + // TODO: warn here? + return get_effective_url(source_url); + } + + // TODO-1. Get values from s3credentials endpoint (and cache them eventually, but for now can get direct) + // TODO-2: use aws sdk to sign the url + return get_effective_url(source_url); } diff --git a/modules/dmrpp_module/Chunk.cc b/modules/dmrpp_module/Chunk.cc index e09c819fee..c1cd68cb2f 100644 --- a/modules/dmrpp_module/Chunk.cc +++ b/modules/dmrpp_module/Chunk.cc @@ -1365,7 +1365,7 @@ std::shared_ptr Chunk::get_data_url() const { if (effective_url == nullptr) { BESDEBUG(MODULE, prolog << "No signed url generated; constructing effective_url via redirects." << endl); - effective_url = EffectiveUrlCache::TheCache()->get_signed_url(d_data_url); + effective_url = EffectiveUrlCache::TheCache()->get_effective_url(d_data_url); } BESDEBUG(MODULE, prolog << "Using data_url: " << effective_url->str() << endl); diff --git a/modules/dmrpp_module/ngap_container/Makefile.am b/modules/dmrpp_module/ngap_container/Makefile.am index ecd6d3c059..20cb182370 100644 --- a/modules/dmrpp_module/ngap_container/Makefile.am +++ b/modules/dmrpp_module/ngap_container/Makefile.am @@ -7,7 +7,7 @@ AUTOMAKE_OPTIONS = foreign ACLOCAL_AMFLAGS = -I conf AM_CPPFLAGS = -I$(top_srcdir) -I$(top_srcdir)/dispatch -I$(top_srcdir)/dap -I$(top_srcdir)/xmlcommand \ --I$(top_srcdir)/http $(DAP_CFLAGS) +-I$(top_srcdir)/http -I$(top_srcdir)/pugixml/src $(DAP_CFLAGS) LIBADD = diff --git a/modules/dmrpp_module/ngap_container/NgapOwnedContainer.cc b/modules/dmrpp_module/ngap_container/NgapOwnedContainer.cc index feebcaf00a..e0381a91ba 100644 --- a/modules/dmrpp_module/ngap_container/NgapOwnedContainer.cc +++ b/modules/dmrpp_module/ngap_container/NgapOwnedContainer.cc @@ -35,7 +35,6 @@ #include #include -#include #include "BESStopWatch.h" #include "BESUtil.h" @@ -51,6 +50,10 @@ #include "NgapOwnedContainer.h" #include "NgapNames.h" +#define PUGIXML_NO_XPATH +#define PUGIXML_HEADER_ONLY +#include + #define prolog std::string("NgapOwnedContainer::").append(__func__).append("() - ") // CACHE_LOG is defined separately from INFO_LOG so that we can turn it off easily. jhrg 11/19/23 @@ -64,6 +67,7 @@ using namespace std; using namespace bes; using http::EffectiveUrlCache; +using namespace pugi; namespace ngap { @@ -562,13 +566,47 @@ bool NgapOwnedContainer::get_dmrpp_from_cache_or_remote_source(string &dmrpp_str return true; } +static inline bool is_eq(const char *value, const char *key) { + return strcmp(value, key) == 0; +} + // TODO-docstring +// Implementation borrowed from DMZ::process_dataset NgapApi::DataAccessUrls NgapOwnedContainer::extract_s3_data_urls_from_dmrpp(const string &dmrpp_string) { + // If the dmrpp is invalid, we will have hit a failure before now---so we can assume + // it's generally safe---so do basically no additional safety checking + + // Load the xml document + pugi::xml_document result_xml_doc; + pugi::xml_parse_result result = result_xml_doc.load_string(dmrpp_string.c_str(), pugi::parse_default | pugi::parse_ws_pcdata_single); + auto xml_root_node = result_xml_doc.first_child(); + + // Pull the expected data values from the xml + string href_attr; + string s3_attr; + string s3credentials_attr; + for (xml_attribute attr = xml_root_node.first_attribute(); attr; attr = attr.next_attribute()) { + if (is_eq(attr.name(), "dmrpp:href")) { + href_attr = attr.value(); + } + else if (is_eq(attr.name(), "dmrpp:s3")) { + s3_attr = attr.value(); + } + else if (is_eq(attr.name(), "dmrpp:s3credentials")) { + s3credentials_attr = attr.value(); + } + if (!href_attr.empty() && !s3_attr.empty() && !s3credentials_attr.empty()) { + break; + } + } - //TODO: pull from DMZ::build_thin_dmr code - // TODO: handle the case where nothing is found in dmrpp - - return tie("foo_href", "foo_s3", "foo_s3credentials"); + if (s3_attr.empty()) { + BESDEBUG(MODULE, prolog << "DMR++ XML dataset element dmrpp:s3 is missing" << endl); + } + if (s3_attr.empty()) { + BESDEBUG(MODULE, prolog << "DMR++ XML dataset element dmrpp:s3credentials is missing" << endl); + } + return tie(href_attr, s3_attr, s3credentials_attr); } /** @@ -593,8 +631,7 @@ string NgapOwnedContainer::access() { // To sign urls locally, we need access to the credential info that has been previously // injected into the dmrpp. Extract that now, in preparation for upcoming url signing. auto urls = extract_s3_data_urls_from_dmrpp(dmrpp_string); - assert(get<1>(urls) == get_real_name()); // TODO: better way to do this?? - EffectiveUrlCache::TheCache()->cache_signed_url_components(get_real_name(), get<1>(urls), get<2>(urls)); + EffectiveUrlCache::TheCache()->cache_signed_url_components(get<0>(urls), get<1>(urls), get<2>(urls)); set_attributes("as-string"); // This means access() returns a string. jhrg 10/19/23 // Originally, this was either hard-coded (as it is now) or was set using the 'extension' From 61ffd336c5b3e33030c16086b9ca8a1e559d3476 Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Tue, 28 Oct 2025 22:01:06 -0400 Subject: [PATCH 03/77] parse s3 credentials from tea endpoint --- http/EffectiveUrlCache.cc | 92 +++++++++++++++++++++++++++++++++-- http/EffectiveUrlCache.h | 9 +++- modules/dmrpp_module/Chunk.cc | 2 +- 3 files changed, 95 insertions(+), 8 deletions(-) diff --git a/http/EffectiveUrlCache.cc b/http/EffectiveUrlCache.cc index cd925ddf01..e32e8466f3 100644 --- a/http/EffectiveUrlCache.cc +++ b/http/EffectiveUrlCache.cc @@ -41,6 +41,8 @@ #include "EffectiveUrl.h" #include "EffectiveUrlCache.h" +#include "rapidjson/document.h" + using namespace std; constexpr auto MODULE = "euc"; @@ -163,32 +165,112 @@ shared_ptr EffectiveUrlCache::get_effective_url(shared_ptr // TODO: docstring void EffectiveUrlCache::cache_signed_url_components(const std::string &key, const std::string &s3_url, const std::string &s3credentials_url) { // TODO: if urls are empty, do....something?? skip it or something + INFO_LOG(prolog + "Caching signed url components for key: " + key); d_href_to_s3_cache[key] = s3_url; d_href_to_s3credentials_cache[key] = s3credentials_url; } +shared_ptr EffectiveUrlCache::get_cache_s3credentials(std::string const &s3credentials_url) { + //TODO-future: make cache for these credentials, check if they've already been retrieved + + // 1. Get the credentials from TEA + std::string s3credentials_json_string; + try { + BES_PROFILE_TIMING(string("Request s3 credentials from TEA - ") + s3credentials_url); + curl::http_get(s3credentials_url, s3credentials_json_string); + } + catch (http::HttpError &http_error) { + string err_msg = prolog + "Encountered an error while " + "attempting to retrieve s3 credentials from TEA. " + http_error.get_message(); + INFO_LOG(err_msg); + return nullptr; + } + if (s3credentials_json_string.empty()) { + string err_msg = prolog + "Unable to retrieve s3 credentials from TEA endpoint " + s3credentials_url; + INFO_LOG(err_msg); + return nullptr; + } + + // 2. Parse the response to pull out the credentials + auto credentials = get_s3_credentials_from_tea_endpoint_json(s3credentials_json_string); + + //TODO-future: add these credentials to the cache + + return credentials; +} + + +// Lightly adapted from get_urls_from_granules_umm_json_v1_4 +std::shared_ptr EffectiveUrlCache::get_s3_credentials_from_tea_endpoint_json(std::string const &s3credentials_json_string) { + rapidjson::Document s3credentials_response; + s3credentials_response.Parse(s3credentials_json_string.c_str()); + + string access_key_id; + string secret_access_key; + string session_token; + string expiration; + + auto itr = s3credentials_response.FindMember("accessKeyId"); + if (itr != s3credentials_response.MemberEnd()) { + access_key_id = itr->value.GetString(); + } + + itr = s3credentials_response.FindMember("secretAccessKey"); + if (itr != s3credentials_response.MemberEnd()) { + secret_access_key = itr->value.GetString(); + } + + itr = s3credentials_response.FindMember("sessionToken"); + if (itr != s3credentials_response.MemberEnd()) { + session_token = itr->value.GetString(); + } + + itr = s3credentials_response.FindMember("expiration"); + if (itr != s3credentials_response.MemberEnd()) { + expiration = itr->value.GetString(); + } + + if (access_key_id.empty() || secret_access_key.empty() || session_token.empty() || expiration.empty()) { + return nullptr; + } + + return make_shared(access_key_id, + secret_access_key, + session_token, + expiration); +} + + + /** * TODO * * @param source_url * @returns The effective (signed) URL */ -shared_ptr EffectiveUrlCache::get_signed_url(shared_ptr source_url) { - BESDEBUG(MODULE, prolog << "GET SIGNED URL NOT YET IMPLEMENTED, falling back to get_effective_url." << endl); +std::shared_ptr EffectiveUrlCache::get_signed_url(std::shared_ptr source_url) { + INFO_LOG(prolog + "GET SIGNED URL NOT YET IMPLEMENTED, falling back to get_effective_url; returns nullptr"); //TODO-implement! Copy get_effective_url boilerplate, then... + // 1. Get relevant urls auto s3_url = d_href_to_s3_cache[source_url->str()]; auto s3credentials_url = d_href_to_s3credentials_cache[source_url->str()]; if (s3_url.empty() || s3credentials_url.empty()) { - // TODO: warn here? + INFO_LOG(prolog + "No url available for TEA s3credentials endpoint."); + return get_effective_url(source_url); + } + + // 2. Get values from s3credentials endpoint + auto s3_access_key_tuple = get_cache_s3credentials(s3credentials_url); + if (s3_access_key_tuple == nullptr) { return get_effective_url(source_url); } + - // TODO-1. Get values from s3credentials endpoint (and cache them eventually, but for now can get direct) // TODO-2: use aws sdk to sign the url - return get_effective_url(source_url); + return nullptr; } /** diff --git a/http/EffectiveUrlCache.h b/http/EffectiveUrlCache.h index 8689833cb6..06fcd8cd58 100644 --- a/http/EffectiveUrlCache.h +++ b/http/EffectiveUrlCache.h @@ -58,9 +58,14 @@ class EffectiveUrlCache : public BESObj { std::map> d_effective_urls; - // TODO- Will be moved to new class before merge + // TODO: The following will be moved to new class before merge +public: + typedef std::tuple S3AccessKeyTuple; +private: std::map d_href_to_s3credentials_cache; std::map d_href_to_s3_cache; + std::shared_ptr get_cache_s3credentials(std::string const &s3credentials_url); + static std::shared_ptr get_s3_credentials_from_tea_endpoint_json(std::string const &s3credentials_json_string); // URLs that match are not cached. std::unique_ptr d_skip_regex = nullptr; @@ -101,7 +106,7 @@ class EffectiveUrlCache : public BESObj { /* * TODO: temporarily lives here; move into own class before merge! - */ + */ std::shared_ptr get_signed_url(std::shared_ptr source_url); void cache_signed_url_components(const std::string &key, const std::string &s3_url, const std::string &s3credentials_url); /* diff --git a/modules/dmrpp_module/Chunk.cc b/modules/dmrpp_module/Chunk.cc index c1cd68cb2f..88ab79b9f7 100644 --- a/modules/dmrpp_module/Chunk.cc +++ b/modules/dmrpp_module/Chunk.cc @@ -1364,7 +1364,7 @@ std::shared_ptr Chunk::get_data_url() const { std::shared_ptr effective_url = EffectiveUrlCache::TheCache()->get_signed_url(d_data_url); if (effective_url == nullptr) { - BESDEBUG(MODULE, prolog << "No signed url generated; constructing effective_url via redirects." << endl); + INFO_LOG(prolog + "Failed to locally sign url; constructing effective_url via redirects."); effective_url = EffectiveUrlCache::TheCache()->get_effective_url(d_data_url); } BESDEBUG(MODULE, prolog << "Using data_url: " << effective_url->str() << endl); From 60271a24b92cf03ef66bb4e987d87c15b0669a42 Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Tue, 28 Oct 2025 23:01:01 -0400 Subject: [PATCH 04/77] Split SignedUrlCache out of EffectiveUrlCache --- CMakeLists.txt | 3 + http/EffectiveUrlCache.cc | 113 ----- http/EffectiveUrlCache.h | 18 - http/Makefile.am | 2 + http/SignedUrlCache.cc | 327 +++++++++++++ http/SignedUrlCache.h | 121 +++++ http/unit-tests/.gitignore | 1 + http/unit-tests/Makefile.am | 5 +- http/unit-tests/SignedUrlCacheTest.cc | 443 ++++++++++++++++++ modules/dmrpp_module/Chunk.cc | 18 +- .../ngap_container/NgapOwnedContainer.cc | 4 +- 11 files changed, 914 insertions(+), 141 deletions(-) create mode 100644 http/SignedUrlCache.cc create mode 100644 http/SignedUrlCache.h create mode 100644 http/unit-tests/SignedUrlCacheTest.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index 9dae673f29..e6ee70d6d8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -528,7 +528,10 @@ add_executable( http/EffectiveUrl.h http/EffectiveUrlCache.cc http/EffectiveUrlCache.h + http/SignedUrlCache.cc + http/SignedUrlCache.h http/unit-tests/EffectiveUrlCacheTest.cc + http/unit-tests/SignedUrlCacheTest.cc http/unit-tests/HttpUrlTest.cc http/unit-tests/RemoteResourceTest.cc http/unit-tests/AllowedHostsTest.cc diff --git a/http/EffectiveUrlCache.cc b/http/EffectiveUrlCache.cc index e32e8466f3..ef318a442d 100644 --- a/http/EffectiveUrlCache.cc +++ b/http/EffectiveUrlCache.cc @@ -41,8 +41,6 @@ #include "EffectiveUrl.h" #include "EffectiveUrlCache.h" -#include "rapidjson/document.h" - using namespace std; constexpr auto MODULE = "euc"; @@ -162,117 +160,6 @@ shared_ptr EffectiveUrlCache::get_effective_url(shared_ptr return effective_url; } -// TODO: docstring -void EffectiveUrlCache::cache_signed_url_components(const std::string &key, const std::string &s3_url, const std::string &s3credentials_url) { - // TODO: if urls are empty, do....something?? skip it or something - INFO_LOG(prolog + "Caching signed url components for key: " + key); - d_href_to_s3_cache[key] = s3_url; - d_href_to_s3credentials_cache[key] = s3credentials_url; -} - -shared_ptr EffectiveUrlCache::get_cache_s3credentials(std::string const &s3credentials_url) { - //TODO-future: make cache for these credentials, check if they've already been retrieved - - // 1. Get the credentials from TEA - std::string s3credentials_json_string; - try { - BES_PROFILE_TIMING(string("Request s3 credentials from TEA - ") + s3credentials_url); - curl::http_get(s3credentials_url, s3credentials_json_string); - } - catch (http::HttpError &http_error) { - string err_msg = prolog + "Encountered an error while " - "attempting to retrieve s3 credentials from TEA. " + http_error.get_message(); - INFO_LOG(err_msg); - return nullptr; - } - if (s3credentials_json_string.empty()) { - string err_msg = prolog + "Unable to retrieve s3 credentials from TEA endpoint " + s3credentials_url; - INFO_LOG(err_msg); - return nullptr; - } - - // 2. Parse the response to pull out the credentials - auto credentials = get_s3_credentials_from_tea_endpoint_json(s3credentials_json_string); - - //TODO-future: add these credentials to the cache - - return credentials; -} - - -// Lightly adapted from get_urls_from_granules_umm_json_v1_4 -std::shared_ptr EffectiveUrlCache::get_s3_credentials_from_tea_endpoint_json(std::string const &s3credentials_json_string) { - rapidjson::Document s3credentials_response; - s3credentials_response.Parse(s3credentials_json_string.c_str()); - - string access_key_id; - string secret_access_key; - string session_token; - string expiration; - - auto itr = s3credentials_response.FindMember("accessKeyId"); - if (itr != s3credentials_response.MemberEnd()) { - access_key_id = itr->value.GetString(); - } - - itr = s3credentials_response.FindMember("secretAccessKey"); - if (itr != s3credentials_response.MemberEnd()) { - secret_access_key = itr->value.GetString(); - } - - itr = s3credentials_response.FindMember("sessionToken"); - if (itr != s3credentials_response.MemberEnd()) { - session_token = itr->value.GetString(); - } - - itr = s3credentials_response.FindMember("expiration"); - if (itr != s3credentials_response.MemberEnd()) { - expiration = itr->value.GetString(); - } - - if (access_key_id.empty() || secret_access_key.empty() || session_token.empty() || expiration.empty()) { - return nullptr; - } - - return make_shared(access_key_id, - secret_access_key, - session_token, - expiration); -} - - - -/** - * TODO - * - * @param source_url - * @returns The effective (signed) URL -*/ -std::shared_ptr EffectiveUrlCache::get_signed_url(std::shared_ptr source_url) { - INFO_LOG(prolog + "GET SIGNED URL NOT YET IMPLEMENTED, falling back to get_effective_url; returns nullptr"); - - //TODO-implement! Copy get_effective_url boilerplate, then... - - // 1. Get relevant urls - auto s3_url = d_href_to_s3_cache[source_url->str()]; - auto s3credentials_url = d_href_to_s3credentials_cache[source_url->str()]; - if (s3_url.empty() || s3credentials_url.empty()) { - INFO_LOG(prolog + "No url available for TEA s3credentials endpoint."); - return get_effective_url(source_url); - } - - // 2. Get values from s3credentials endpoint - auto s3_access_key_tuple = get_cache_s3credentials(s3credentials_url); - if (s3_access_key_tuple == nullptr) { - return get_effective_url(source_url); - } - - - // TODO-2: use aws sdk to sign the url - - return nullptr; -} - /** * @return Is the cache enabled (set in the bes.conf file)? */ diff --git a/http/EffectiveUrlCache.h b/http/EffectiveUrlCache.h index 06fcd8cd58..27f85d4d36 100644 --- a/http/EffectiveUrlCache.h +++ b/http/EffectiveUrlCache.h @@ -58,15 +58,6 @@ class EffectiveUrlCache : public BESObj { std::map> d_effective_urls; - // TODO: The following will be moved to new class before merge -public: - typedef std::tuple S3AccessKeyTuple; -private: - std::map d_href_to_s3credentials_cache; - std::map d_href_to_s3_cache; - std::shared_ptr get_cache_s3credentials(std::string const &s3credentials_url); - static std::shared_ptr get_s3_credentials_from_tea_endpoint_json(std::string const &s3credentials_json_string); - // URLs that match are not cached. std::unique_ptr d_skip_regex = nullptr; @@ -103,15 +94,6 @@ class EffectiveUrlCache : public BESObj { ~EffectiveUrlCache() override = default; std::shared_ptr get_effective_url(std::shared_ptr source_url); - - /* - * TODO: temporarily lives here; move into own class before merge! - */ - std::shared_ptr get_signed_url(std::shared_ptr source_url); - void cache_signed_url_components(const std::string &key, const std::string &s3_url, const std::string &s3credentials_url); - /* - * End temp contents! - */ void dump(std::ostream &strm) const override; diff --git a/http/Makefile.am b/http/Makefile.am index 631c4c6d3a..94789462d7 100644 --- a/http/Makefile.am +++ b/http/Makefile.am @@ -41,6 +41,7 @@ SRCS = CurlUtils.cc \ HttpUtils.cc \ ProxyConfig.cc \ EffectiveUrlCache.cc \ + SignedUrlCache.cc \ url_impl.cc \ EffectiveUrl.cc \ AllowedHosts.cc \ @@ -55,6 +56,7 @@ HDRS = CurlUtils.h \ ProxyConfig.h \ HttpNames.h \ EffectiveUrlCache.h \ + SignedUrlCache.h \ url_impl.h \ EffectiveUrl.h \ AllowedHosts.h \ diff --git a/http/SignedUrlCache.cc b/http/SignedUrlCache.cc new file mode 100644 index 0000000000..1f297e50f5 --- /dev/null +++ b/http/SignedUrlCache.cc @@ -0,0 +1,327 @@ +// -*- mode: c++; c-basic-offset:4 -*- + +// This file is part of the BES http package, part of the Hyrax data server. + +// Copyright (c) 2020 OPeNDAP, Inc. +// Author: Nathan Potter +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +// +// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112. + +// Authors: +// ndp Nathan Potter + +#include "config.h" + +#include + +#include +#include + +#include "TheBESKeys.h" +#include "BESDebug.h" +#include "BESStopWatch.h" +#include "BESUtil.h" +#include "CurlUtils.h" +#include "HttpError.h" +#include "HttpNames.h" +#include "EffectiveUrl.h" +#include "SignedUrlCache.h" + +#include "rapidjson/document.h" + +using namespace std; + +constexpr auto MODULE = "euc"; +constexpr auto MODULE_TIMER = "euc:timer"; +constexpr auto MODULE_DUMPER = "euc:dump"; + +#define prolog std::string("SignedUrlCache::").append(__func__).append("() - ") + +namespace http { + +/** + * @brief Get the cached signed URL. + * @param url_key Key to a cached signed URL. + * @note This method is not, itself, thread safe. + */ +shared_ptr SignedUrlCache::get_cached_signed_url(string const &url_key) { + shared_ptr signed_url(nullptr); + auto it = d_signed_urls.find(url_key); + if (it != d_signed_urls.end()) { + signed_url = (*it).second; + } + return signed_url; +} + +/** + * Find the terminal (effective) url for the source_url. If the source_url matches the + * skip_regex then it will not be cached. + * + * @param source_url + * @returns The effective URL +*/ +shared_ptr SignedUrlCache::get_signed_url(shared_ptr source_url) { + + BESDEBUG(MODULE, prolog << "BEGIN url: " << source_url->str() << endl); + BESDEBUG(MODULE_DUMPER, prolog << "dump: " << endl << dump() << endl); + + // Lock access to the cache, the d_signed_urls map. Released when the lock goes out of scope. + std::lock_guard lock_me(d_cache_lock_mutex); + + if (!is_enabled()) { + BESDEBUG(MODULE, prolog << "CACHE IS DISABLED." << endl); + return shared_ptr(new EffectiveUrl(source_url)); + } + + // if it's not an HTTP url there is nothing to cache. + if (source_url->str().find(HTTP_PROTOCOL) != 0 && source_url->str().find(HTTPS_PROTOCOL) != 0) { + BESDEBUG(MODULE, prolog << "END Not an HTTP request, SKIPPING." << endl); + return shared_ptr(new EffectiveUrl(source_url)); + } + + if (!d_skip_regex) { + set_skip_regex(); + } + + if (d_skip_regex) { + size_t match_length = 0; + match_length = d_skip_regex->match(source_url->str().c_str(), (int) source_url->str().size()); + if (match_length == source_url->str().size()) { + BESDEBUG(MODULE, prolog << "END Candidate url matches the " + "no_redirects_regex_pattern [" << d_skip_regex->pattern() << + "][match_length=" << match_length << "] SKIPPING." << endl); + return shared_ptr(new EffectiveUrl(source_url)); + } + BESDEBUG(MODULE, prolog << "Candidate url: '" << source_url->str() + << "' does NOT match the skip_regex pattern [" << d_skip_regex->pattern() << "]" + << endl); + } else { + BESDEBUG(MODULE, prolog << "The cache_effective_urls_skip_regex() was NOT SET " << endl); + } + + shared_ptr signed_url = get_cached_signed_url(source_url->str()); + bool retrieve_and_cache = !signed_url || signed_url->is_expired(); + // TODO-H: make sure that `signed_url->is_expired();` handles correctly for the signed urls + + // It not found or expired, (re)load. + if (retrieve_and_cache) { + BESDEBUG(MODULE, prolog << "Acquiring signed URL for " << source_url->str() << endl); + { + BES_STOPWATCH_START(MODULE_TIMER, prolog + "Retrieve and cache signed url for source url: " + source_url->str()); + signed_url = sign_url_with_fetched_credentials(source_url); + + if (signed_url == nullptr) { + return nullptr; + } + } + BESDEBUG(MODULE, prolog << " source_url: " << source_url->str() << " (" + << (source_url->is_trusted() ? "" : "NOT ") << "trusted)" << endl); + BESDEBUG(MODULE, prolog << "signed_url: " << signed_url->dump() << " (" + << (source_url->is_trusted() ? "" : "NOT ") << "trusted)" << endl); + + d_signed_urls[source_url->str()] = signed_url; + + BESDEBUG(MODULE, prolog << "Updated record for " << source_url->str() << " cache size: " + << d_signed_urls.size() << endl); + + // Since we don't want there to be a concurrency issue when we release the lock, we don't + // return the instance of shared_ptr that we placed in the cache. Rather + // we make a clone and return that. It will have its own lifecycle independent of + // the instance we placed in the cache - it can be modified and the one in the cache + // is unchanged. Trusted state was established from source_url when signed_url was + // created in sign_url() + signed_url = make_shared(signed_url); + } else { + // Here we have a !expired instance of a shared_ptr retrieved from the cache. + // Now we need to make a copy to return, inheriting trust from the requesting URL. + signed_url = make_shared(signed_url, source_url->is_trusted()); + } + + BESDEBUG(MODULE_DUMPER, prolog << "dump: " << endl << dump() << endl); + BESDEBUG(MODULE, prolog << "END" << endl); + + return signed_url; +} + +// TODO: docstring +void SignedUrlCache::cache_signed_url_components(const std::string &key, const std::string &s3_url, const std::string &s3credentials_url) { + // TODO: if urls are empty, do....something?? skip it or something + INFO_LOG(prolog + "Caching signed url components for key: " + key); + d_href_to_s3_cache[key] = s3_url; + d_href_to_s3credentials_cache[key] = s3credentials_url; +} + +shared_ptr SignedUrlCache::get_cache_s3credentials(std::string const &s3credentials_url) { + //TODO-future: make cache for these credentials, check if they've already been retrieved + + // 1. Get the credentials from TEA + std::string s3credentials_json_string; + try { + BES_PROFILE_TIMING(string("Request s3 credentials from TEA - ") + s3credentials_url); + curl::http_get(s3credentials_url, s3credentials_json_string); + } + catch (http::HttpError &http_error) { + string err_msg = prolog + "Encountered an error while " + "attempting to retrieve s3 credentials from TEA. " + http_error.get_message(); + INFO_LOG(err_msg); + return nullptr; + } + if (s3credentials_json_string.empty()) { + string err_msg = prolog + "Unable to retrieve s3 credentials from TEA endpoint " + s3credentials_url; + INFO_LOG(err_msg); + return nullptr; + } + + // 2. Parse the response to pull out the credentials + auto credentials = get_s3_credentials_from_tea_endpoint_json(s3credentials_json_string); + + //TODO-future: add these credentials to the cache + + return credentials; +} + + +// Lightly adapted from get_urls_from_granules_umm_json_v1_4 +std::shared_ptr SignedUrlCache::get_s3_credentials_from_tea_endpoint_json(std::string const &s3credentials_json_string) { + rapidjson::Document s3credentials_response; + s3credentials_response.Parse(s3credentials_json_string.c_str()); + + string access_key_id; + string secret_access_key; + string session_token; + string expiration; + + auto itr = s3credentials_response.FindMember("accessKeyId"); + if (itr != s3credentials_response.MemberEnd()) { + access_key_id = itr->value.GetString(); + } + + itr = s3credentials_response.FindMember("secretAccessKey"); + if (itr != s3credentials_response.MemberEnd()) { + secret_access_key = itr->value.GetString(); + } + + itr = s3credentials_response.FindMember("sessionToken"); + if (itr != s3credentials_response.MemberEnd()) { + session_token = itr->value.GetString(); + } + + itr = s3credentials_response.FindMember("expiration"); + if (itr != s3credentials_response.MemberEnd()) { + expiration = itr->value.GetString(); + } + + if (access_key_id.empty() || secret_access_key.empty() || session_token.empty() || expiration.empty()) { + return nullptr; + } + return make_shared(access_key_id, + secret_access_key, + session_token, + expiration); +} + + + +/** + * TODO + * + * @param source_url + * @returns The effective (signed) URL, which may be a nullptr if signing credentials are not found +*/ +std::shared_ptr SignedUrlCache::sign_url_with_fetched_credentials(std::shared_ptr source_url) { + INFO_LOG(prolog + "SIGN URL ONLY PARTIALLY IMPLEMENTED; for now, always returns nullptr"); + + // 1. Get relevant urls + // TODO: safety first! check that keys exist in maps THEN check for empty + auto s3_url = d_href_to_s3_cache[source_url->str()]; + auto s3credentials_url = d_href_to_s3credentials_cache[source_url->str()]; + if (s3_url.empty() || s3credentials_url.empty()) { + INFO_LOG(prolog + "No url available for TEA s3credentials endpoint."); + return nullptr; + } + + // 2. Get access credentials from s3credentials endpoint + auto s3_access_key_tuple = get_cache_s3credentials(s3credentials_url); + if (s3_access_key_tuple == nullptr) { + return nullptr; + } + + // 3: Use aws sdk to sign the url + return sign_url(s3_url, s3_access_key_tuple); +} + +/** + * TODO + * + * @param source_url + * @returns The effective (signed) URL, which may be a nullptr if signing credentials are not found +*/ +std::shared_ptr SignedUrlCache::sign_url(const std::string &s3_url, const std::shared_ptr credentials) { + // TODO: actually do this! :) + + return nullptr; +} + +/** + * @return Is the cache enabled (set in the bes.conf file)? + * Follows the same settings as the EffectiveUrlsCache + */ +bool SignedUrlCache::is_enabled() { + // The first time here, the value of d_enabled is -1. Once we check for it in TheBESKeys + // The value will be 0 (false) or 1 (true) and TheBESKeys will not be checked again. + if (d_enabled < 0) { + string value = TheBESKeys::TheKeys()->read_string_key(HTTP_CACHE_EFFECTIVE_URLS_KEY, "false"); + d_enabled = BESUtil::lowercase(value) == "true"; + } + BESDEBUG(MODULE, prolog << "d_enabled: " << (d_enabled ? "true" : "false") << endl); + return d_enabled; +} + +// * Follows the same settings as the EffectiveUrlsCache +void SignedUrlCache::set_skip_regex() { + if (!d_skip_regex) { + string pattern = TheBESKeys::TheKeys()->read_string_key(HTTP_CACHE_EFFECTIVE_URLS_SKIP_REGEX_KEY, ""); + if (!pattern.empty()) { + d_skip_regex.reset(new BESRegex(pattern.c_str())); + } + BESDEBUG(MODULE, prolog << "d_skip_regex: " + << (d_skip_regex ? d_skip_regex->pattern() : "Value has not been set.") << endl); + } +} + +/** + * @brief dumps information about this object + * @param strm C++ i/o stream to dump the information to + */ +void SignedUrlCache::dump(ostream &strm) const { + strm << BESIndent::LMarg << prolog << "(this: " << (void *) this << ")" << endl; + BESIndent::Indent(); + strm << BESIndent::LMarg << "d_skip_regex: " << (d_skip_regex ? d_skip_regex->pattern() : "WAS NOT SET") << endl; + if (!d_signed_urls.empty()) { + strm << BESIndent::LMarg << "effective url list:" << endl; + BESIndent::Indent(); + for (auto const &i: d_signed_urls) { + strm << BESIndent::LMarg << i.first << " --> " << i.second->str(); + } + BESIndent::UnIndent(); + } else { + strm << BESIndent::LMarg << "effective url list: EMPTY" << endl; + } + BESIndent::UnIndent(); +} + +} // namespace http diff --git a/http/SignedUrlCache.h b/http/SignedUrlCache.h new file mode 100644 index 0000000000..f8487e4d2a --- /dev/null +++ b/http/SignedUrlCache.h @@ -0,0 +1,121 @@ +// -*- mode: c++; c-basic-offset:4 -*- + +// This file is part of the BES http package, part of the Hyrax data server. + +// Copyright (c) 2020 OPeNDAP, Inc. +// Author: Nathan Potter +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +// +// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112. + +// Authors: +// ndp Nathan Potter + +#ifndef _bes_http_SignedUrlCache_h_ +#define _bes_http_SignedUrlCache_h_ 1 + +#include +#include +#include +#include +#include + +#include "BESObj.h" +#include "BESRegex.h" // for std::unique_ptr + +namespace http { + +class EffectiveUrl; +class url; + +/** + * This is a singleton class. It is used to associate a URL with its "pre-signed" AWS s3 URL. This means that + * a URL is signed locally rather than sent through a potentially large number of external redirect actions, as + * in EffectiveUrlCache.h. This url location plus the requisite AWS signature headers, from which the requested bytes + * are transmitted, is termed the "effective url" and is stored in an in memory cache (std::map) so that later + * requests may skip the external signing service and just get required bytes from the actual source. + */ +class SignedUrlCache : public BESObj { +public: + typedef std::tuple S3AccessKeyTuple; + +private: + SignedUrlCache() = default; + + std::mutex d_cache_lock_mutex; + + std::map> d_signed_urls; + std::map d_href_to_s3credentials_cache; + std::map d_href_to_s3_cache; + + std::shared_ptr get_cache_s3credentials(std::string const &s3credentials_url); + static std::shared_ptr get_s3_credentials_from_tea_endpoint_json(std::string const &s3credentials_json_string); + + // URLs that match are not cached. + std::unique_ptr d_skip_regex = nullptr; + + int d_enabled = -1; + + // TODO: for all helper functions here, think carefully about static/const/etc + // TODO: think a little more about splitting functionality to revert control to the caller + std::shared_ptr sign_url_with_fetched_credentials(std::shared_ptr source_url); + static std::shared_ptr sign_url(const std::string &s3_url, const std::shared_ptr credentials); + std::shared_ptr get_cached_signed_url(std::string const &url_key); + + void set_skip_regex(); + + bool is_enabled(); + + friend class SignedUrlCacheTest; + +public: + /** @brief Get the singleton SignedUrlCache instance. + * + * This static method returns the instance of this singleton class. + * The implementation will only build one instance of SignedUrlCache and + * thereafter return a pointer to that instance. + * + * Thread safe with C++-11 and greater. + * + * @return A pointer to the SignedUrlCache singleton + */ + static SignedUrlCache *TheCache() { + // Create a local static object the first time the function is called + static SignedUrlCache instance; + return &instance; + } + + SignedUrlCache(const SignedUrlCache &src) = delete; + SignedUrlCache &operator=(const SignedUrlCache &rhs) = delete; + + ~SignedUrlCache() override = default; + + void cache_signed_url_components(const std::string &key, const std::string &s3_url, const std::string &s3credentials_url); + std::shared_ptr get_signed_url(std::shared_ptr source_url); + + void dump(std::ostream &strm) const override; + + std::string dump() const { + std::stringstream sstrm; + dump(sstrm); + return sstrm.str(); + } +}; + +} // namespace http + +#endif // _bes_http_SignedUrlCache_h_ + diff --git a/http/unit-tests/.gitignore b/http/unit-tests/.gitignore index b623b0b70c..6e7bb2da4d 100644 --- a/http/unit-tests/.gitignore +++ b/http/unit-tests/.gitignore @@ -3,6 +3,7 @@ /CurlUtilsTest /CurlSListTest /EffectiveUrlCacheTest +/SignedUrlCacheTest /HttpUrlTest /HttpErrorTest /AllowedHostsTest diff --git a/http/unit-tests/Makefile.am b/http/unit-tests/Makefile.am index a63447c538..eca33225a2 100644 --- a/http/unit-tests/Makefile.am +++ b/http/unit-tests/Makefile.am @@ -74,7 +74,7 @@ bes_ngap_s3_creds.conf: bes_ngap_s3_creds.conf.in $(top_srcdir)/configure.ac if CPPUNIT -UNIT_TESTS = HttpUtilsTest HttpErrorTest RemoteResourceTest EffectiveUrlCacheTest HttpUrlTest \ +UNIT_TESTS = HttpUtilsTest HttpErrorTest RemoteResourceTest EffectiveUrlCacheTest HttpUrlTest SignedUrlCacheTest \ AllowedHostsTest awsv4_test CurlUtilsTest CurlSListTest # CredentialsManagerTest was removed because it was testing our S3 signing code and @@ -122,6 +122,9 @@ CurlUtilsTest_LDADD = $(LIBADD) EffectiveUrlCacheTest_SOURCES = EffectiveUrlCacheTest.cc EffectiveUrlCacheTest_LDADD = $(LIBADD) +SignedUrlCacheTest_SOURCES = SignedUrlCacheTest.cc +SignedUrlCacheTest_LDADD = $(LIBADD) + HttpUrlTest_SOURCES = HttpUrlTest.cc HttpUrlTest_LDADD = $(LIBADD) diff --git a/http/unit-tests/SignedUrlCacheTest.cc b/http/unit-tests/SignedUrlCacheTest.cc new file mode 100644 index 0000000000..ed0d5d694e --- /dev/null +++ b/http/unit-tests/SignedUrlCacheTest.cc @@ -0,0 +1,443 @@ +// -*- mode: c++; c-basic-offset:4 -*- + +// This file is part of the BES component of the Hyrax Data Server. + +// Copyright (c) 2018 OPeNDAP, Inc. +// Author: Nathan Potter +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +// +// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112. + +#include "config.h" + +#include +#include + +#include +#include +#include + +#include + +#include "BESError.h" +#include "BESDebug.h" +#include "BESUtil.h" +#include "BESCatalogList.h" +#include "TheBESKeys.h" +#include "BESContextManager.h" + +#include "HttpNames.h" +#include "url_impl.h" +#include "EffectiveUrl.h" +#include "SignedUrlCache.h" + +#include "test_config.h" + +using namespace std; + +static bool debug = false; +static bool Debug = false; +static bool bes_debug = false; +static bool ngap_tests = false; +static std::string token; + +#undef DBG +#define DBG(x) do { if (debug) x; } while(false) +#define prolog std::string("SignedUrlCacheTest::").append(__func__).append("() - ") + +namespace http { + +class SignedUrlCacheTest : public CppUnit::TestFixture { +private: + string d_data_dir = TEST_DATA_DIR; + + void show_file(string filename) { + ifstream t(filename.c_str()); + + if (t.is_open()) { + string file_content((istreambuf_iterator(t)), istreambuf_iterator()); + t.close(); + cerr << endl << "#############################################################################" << endl; + cerr << "file: " << filename << endl; + cerr << ". . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . " << endl; + cerr << file_content << endl; + cerr << "#############################################################################" << endl; + } else { + cerr << "FAILED TO OPEN FILE: " << filename << endl; + } + } + +public: + // Called once before everything gets tested + SignedUrlCacheTest() = default; + + // Called at the end of the tests + ~SignedUrlCacheTest() override = default; + + // Called before each test + void setUp() override { + DBG(cerr << endl); + DBG(cerr << prolog << "BEGIN" << endl); + DBG(cerr << prolog << "data_dir: " << d_data_dir << endl); + string bes_conf = BESUtil::assemblePath(TEST_BUILD_DIR, "bes.conf"); + DBG(cerr << prolog << "Using BES configuration: " << bes_conf << endl); + if (Debug) show_file(bes_conf); + TheBESKeys::ConfigFile = bes_conf; + + if (bes_debug) BESDebug::SetUp("cerr,bes,euc,http,curl"); + + // Clear the cache for the next test. + SignedUrlCache *theCache = SignedUrlCache::TheCache(); + theCache->d_signed_urls.clear(); + + if (!token.empty()) { + DBG(cerr << "Setting BESContext " << EDL_AUTH_TOKEN_KEY << " to: '" << token << "'" << endl); + BESContextManager::TheManager()->set_context(EDL_AUTH_TOKEN_KEY, token); + } + DBG(cerr << prolog << "END" << endl); + } + +/*##################################################################################################*/ +/* TESTS BEGIN */ + + void is_cache_disabled_test() { + DBG(cerr << prolog << "SignedUrlCache::TheCache()->is_enabled(): " + << (SignedUrlCache::TheCache()->is_enabled() ? "true" : "false") << endl); + CPPUNIT_ASSERT(!SignedUrlCache::TheCache()->is_enabled()); + + shared_ptr src_url_00(new http::url("http://started_here.com")); + auto effective_url_00 = shared_ptr(new http::EffectiveUrl("https://ended_here.com")); + + SignedUrlCache::TheCache()->d_signed_urls.insert( + pair>(src_url_00->str(), effective_url_00)); + CPPUNIT_ASSERT(SignedUrlCache::TheCache()->d_signed_urls.size() == 1); + + // This one does not add the URL or even check it because it _should_ be matching the skip regex. + auto result_url = SignedUrlCache::TheCache()->get_signed_url(src_url_00); + CPPUNIT_ASSERT(result_url->str() == src_url_00->str()); + } + + void skip_regex_test_01() { + DBG(cerr << prolog << "BEGIN" << endl); + try { + // The cache is disabled in bes.conf, so we need to turn it on. + SignedUrlCache::TheCache()->d_enabled = true; + + // This one does not add the URL or even check it because it _should_ be matching the skip regex + // in the bes.conf + shared_ptr src_url(new http::url("https://foobar.com/opendap/data/nc/fnoc1.nc?dap4.ce=u;v")); + auto result_url = SignedUrlCache::TheCache()->get_signed_url(src_url); + CPPUNIT_ASSERT(SignedUrlCache::TheCache()->d_signed_urls.empty()); + CPPUNIT_ASSERT(result_url->str() == src_url->str()); + } + catch (const BESError &be) { + stringstream msg; + msg << prolog << "ERROR! Caught BESError. Message: " << be.get_message() << endl; + CPPUNIT_FAIL(msg.str()); + } + } + + void cache_test_00() { + DBG(cerr << prolog << "BEGIN" << endl); + //string source_url; + //string value; + try { + // The cache is disabled in bes.conf, so we need to turn it on. + SignedUrlCache::TheCache()->d_enabled = true; + +#if 0 + string src_url_00 = "https://d1jecqxxv88lkr.cloudfront.net/ghrcwuat-protected/rss_demo/rssmif16d__7/f1" + "6_ssmis_20040107v7.nc"; + string eurl_str = "https://ghrcwuat-protected.s3.us-west-2.amazonaws.com/rss_demo/rssmif16d__7/f16_ssm" + "is_20031229v7.nc?A-userid=hyrax&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=A" + "SIASF4N-AWS-Creds-00808%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Date=20200808T032623Z" + "&X-Amz-Expires=86400&X-Amz-Security-Token=FwoGZXIvYXdzE-AWS-Sec-Token-MWRLIZGYvDx1O" + "Nzd0ffK8VtxO8JP7thrGIQ%3D%3D&X-Amz-SignedHeaders=host&X-Amz-Signature=260a7c4dd4-AW" + "S-SIGGY-0c7a39ee899"; + auto effective_url_00 = shared_ptr(new http::EffectiveUrl(eurl_str)); + SignedUrlCache::TheCache()->d_signed_urls.insert(pair>(src_url_00, effective_url_00)); + CPPUNIT_ASSERT(SignedUrlCache::TheCache()->d_signed_urls.size() == 1); +#endif + + string src_url_01 = "http://test.opendap.org/data/httpd_catalog/READTHIS"; + string eurl_str = "https://test.opendap.org/data/httpd_catalog/READTHIS"; + auto effective_url_01 = shared_ptr(new http::EffectiveUrl(eurl_str)); + SignedUrlCache::TheCache()->d_signed_urls.insert( + pair>(src_url_01, effective_url_01)); + CPPUNIT_ASSERT(SignedUrlCache::TheCache()->d_signed_urls.size() == 1); + + // This one actually does the thing + eurl_str = "http://test.opendap.org/opendap/"; + auto expected_url_02 = std::unique_ptr(new http::EffectiveUrl(eurl_str)); + + shared_ptr src_url_02(new http::url("http://test.opendap.org/opendap")); + DBG(cerr << prolog << "Retrieving effective URL for: " << src_url_02->str() << endl); + auto result_url = SignedUrlCache::TheCache()->get_signed_url(src_url_02); + CPPUNIT_ASSERT(SignedUrlCache::TheCache()->d_signed_urls.size() == 2); + + DBG(cerr << prolog << "SignedUrlCache::TheCache()->get_signed_url() returned: " << result_url + << endl); + CPPUNIT_ASSERT(result_url->str() == expected_url_02->str()); + } + catch (const BESError &be) { + stringstream msg; + msg << prolog << "ERROR! Caught BESError. Message: " << be.get_message() << endl; + CPPUNIT_FAIL(msg.str()); + + } + DBG(cerr << prolog << "END" << endl); + } + + + void cache_test_01() { + DBG(cerr << prolog << "BEGIN" << endl); + string source_url; + string value; + string result_url; + try { + // The cache is disabled in bes.conf so we need to turn it on. + SignedUrlCache::TheCache()->d_enabled = true; + + std::map d_signed_urls; + source_url = "http://someURL"; + + http::EffectiveUrl first_eu("http://someOtherUrl"); + d_signed_urls[source_url] = &first_eu; + DBG(cerr << prolog << "source_url: " << source_url << endl); + DBG(cerr << prolog << "first_eu: " << first_eu.str() << endl); + + CPPUNIT_ASSERT(d_signed_urls[source_url] == &first_eu); + + http::EffectiveUrl second_eu("http://someMoreUrlLovin"); + d_signed_urls[source_url] = &second_eu; + DBG(cerr << prolog << "source_url: " << source_url << endl); + DBG(cerr << prolog << "second_eu: " << second_eu.str() << endl); + + CPPUNIT_ASSERT(d_signed_urls[source_url] == &second_eu); + } + catch (const BESError &be) { + stringstream msg; + msg << prolog << "ERROR! Caught BESError. Message: " << be.get_message() << endl; + CPPUNIT_FAIL(msg.str()); + + } + DBG(cerr << prolog << "END" << endl); + } + + void euc_ghrc_tea_url_test() { + if (!ngap_tests) { + DBG(cerr << prolog << "SKIPPING." << endl); + return; + } + DBG(cerr << prolog << "BEGIN" << endl); + string source_url; + string value; + try { + // The cache is disabled in bes.conf so we need to turn it on. + SignedUrlCache::TheCache()->d_enabled = true; + + shared_ptr thing1(new http::url( + "https://d1jecqxxv88lkr.cloudfront.net/ghrcwuat-protected/rss_demo/rssmif16d__7/f16_ssmis_20031026v7.nc")); + string thing1_out_of_region_effective_url_prefix = "https://d1jecqxxv88lkr.cloudfront.net/s3"; + string thing1_in_region_effective_url_prefix = "https://ghrcwuat-protected.s3.us-west-2.amazonaws.com/"; + + DBG(cerr << prolog << "Retrieving signed URL for: " << thing1->str() << endl); + auto result_url = SignedUrlCache::TheCache()->get_signed_url(thing1); + CPPUNIT_ASSERT(SignedUrlCache::TheCache()->d_signed_urls.size() == 1); + + DBG(cerr << prolog << "SignedUrlCache::TheCache()->get_signed_url() returned: " << result_url + << endl); + CPPUNIT_ASSERT(result_url->str().rfind(thing1_in_region_effective_url_prefix, 0) == 0 + || result_url->str().rfind(thing1_out_of_region_effective_url_prefix, 0) == 0); + + // result_url = SignedUrlCache::TheCache()->get_signed_url(thing1); + } + catch (const BESError &be) { + stringstream msg; + msg << prolog << "ERROR! Caught BESError. Message: " << be.get_message() << endl; + CPPUNIT_FAIL(msg.str()); + + } + DBG(cerr << prolog << "END" << endl); + } + + void euc_harmony_url_test() { + if (!ngap_tests) { + DBG(cerr << prolog << "SKIPPING." << endl); + return; + } + DBG(cerr << prolog << "BEGIN" << endl); + string source_url; + string value; + try { + // The cache is disabled in bes.conf, so we need to turn it on. + SignedUrlCache::TheCache()->d_enabled = true; + + shared_ptr thing1( + new http::url("https://harmony.uat.earthdata.nasa.gov/service-results/harmony-uat-staging/public/" + "sds/staged/ATL03_20200714235814_03000802_003_01.h5")); + string thing1_out_of_region_effective_url_prefix = "https://djpip0737hawz.cloudfront.net/s3"; + string thing1_in_region_effective_url_prefix = "https://harmony-uat-staging.s3.us-west-2.amazonaws.com/public/"; + + DBG(cerr << prolog << "Retrieving signed URL for: " << thing1->str() << endl); + auto result_url = SignedUrlCache::TheCache()->get_signed_url(thing1); + CPPUNIT_ASSERT(SignedUrlCache::TheCache()->d_signed_urls.size() == 1); + + DBG(cerr << prolog << "SignedUrlCache::TheCache()->get_signed_url() returned: " << result_url + << endl); + + CPPUNIT_ASSERT(result_url->str().rfind(thing1_in_region_effective_url_prefix, 0) == 0 + || result_url->str().rfind(thing1_out_of_region_effective_url_prefix, 0) == 0); + + // TODO ??? result_url = SignedUrlCache::TheCache()->get_signed_url(thing1); + } + catch (const BESError &be) { + stringstream msg; + msg << prolog << "ERROR! Caught BESError. Message: " << be.get_message() << endl; + CPPUNIT_FAIL(msg.str()); + + } + DBG(cerr << prolog << "END" << endl); + } + + void trusted_url_test_01() { + DBG(cerr << prolog << "BEGIN" << endl); + string url_str = "http://test.opendap.org/data/nothing_is_here.html"; + string result_url_str = "http://test.opendap.org/data/httpd_catalog/READTHIS"; + shared_ptr trusted_src_url(new http::url(url_str, true)); + shared_ptr untrusted_src_url(new http::url(url_str, false)); + + DBG(cerr << prolog << "Retrieving signed URL for: " << trusted_src_url->str() << endl); + try { + // The cache is disabled in bes.conf so we need to turn it on. + SignedUrlCache::TheCache()->d_enabled = true; + + shared_ptr result_url; + + result_url = SignedUrlCache::TheCache()->get_signed_url(untrusted_src_url); + DBG(cerr << prolog << "source_url: " << untrusted_src_url->str() << " is " + << (untrusted_src_url->is_trusted() ? "" : "NOT ") << "trusted." << endl); + DBG(cerr << prolog << "result_url: " << result_url->str() << " is " + << (result_url->is_trusted() ? "" : "NOT ") << "trusted." << endl); + DBG(cerr << prolog << "SignedUrlCache::TheCache()->d_signed_urls.size(): " + << SignedUrlCache::TheCache()->d_signed_urls.size() << endl); + CPPUNIT_ASSERT(SignedUrlCache::TheCache()->d_signed_urls.size() == 1); + CPPUNIT_ASSERT(result_url->str() == result_url_str); + CPPUNIT_ASSERT(!result_url->is_trusted()); + + result_url = SignedUrlCache::TheCache()->get_signed_url(trusted_src_url); + DBG(cerr << prolog << "source_url: " << trusted_src_url->str() << " is " + << (trusted_src_url->is_trusted() ? "" : "NOT ") << "trusted." << endl); + DBG(cerr << prolog << "result_url: " << result_url->str() << " is " + << (result_url->is_trusted() ? "" : "NOT ") << "trusted." << endl); + DBG(cerr << prolog << "SignedUrlCache::TheCache()->d_signed_urls.size(): " + << SignedUrlCache::TheCache()->d_signed_urls.size() << endl); + CPPUNIT_ASSERT(SignedUrlCache::TheCache()->d_signed_urls.size() == 1); + CPPUNIT_ASSERT(result_url->str() == result_url_str); + CPPUNIT_ASSERT(result_url->is_trusted()); + + result_url = SignedUrlCache::TheCache()->get_signed_url(untrusted_src_url); + DBG(cerr << prolog << "source_url: " << untrusted_src_url->str() << " is " + << (untrusted_src_url->is_trusted() ? "" : "NOT ") << "trusted." << endl); + DBG(cerr << prolog << "result_url: " << result_url->str() << " is " + << (result_url->is_trusted() ? "" : "NOT ") << "trusted." << endl); + DBG(cerr << prolog << "SignedUrlCache::TheCache()->d_signed_urls.size(): " + << SignedUrlCache::TheCache()->d_signed_urls.size() << endl); + CPPUNIT_ASSERT(SignedUrlCache::TheCache()->d_signed_urls.size() == 1); + CPPUNIT_ASSERT(result_url->str() == result_url_str); + CPPUNIT_ASSERT(!result_url->is_trusted()); + + } + catch (const BESError &be) { + stringstream msg; + msg << prolog << "ERROR! Caught BESError. Message: " << be.get_message() << endl; + CPPUNIT_FAIL(msg.str()); + } + DBG(cerr << prolog << "END" << endl); + } + +/* TESTS END */ +/*##################################################################################################*/ + +CPPUNIT_TEST_SUITE(SignedUrlCacheTest); + + // TODO: fix up tests, enable/replace!! add more!! + // CPPUNIT_TEST(is_cache_disabled_test); + // CPPUNIT_TEST(cache_test_00); + // CPPUNIT_TEST(cache_test_01); + // CPPUNIT_TEST(skip_regex_test_01); + // CPPUNIT_TEST(euc_ghrc_tea_url_test); + // CPPUNIT_TEST(euc_harmony_url_test); + // CPPUNIT_TEST(trusted_url_test_01); + + CPPUNIT_TEST_SUITE_END(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(SignedUrlCacheTest); + +} // namespace httpd_catalog + +int main(int argc, char *argv[]) { + CppUnit::TextTestRunner runner; + runner.addTest(CppUnit::TestFactoryRegistry::getRegistry().makeTest()); + + int option_char; + while ((option_char = getopt(argc, argv, "dbDPt:N")) != -1) + switch (option_char) { + case 'd': + debug = true; // debug is a static global + cerr << "debug enabled" << endl; + break; + case 'D': + Debug = true; // Debug is a static global + cerr << "Debug enabled" << endl; + break; + case 'b': + bes_debug = true; // debug is a static global + cerr << "bes_debug enabled" << endl; + break; + case 'N': + ngap_tests = true; // ngap_tests is a static global + cerr << "NGAP Tests Enabled." << token << endl; + break; + case 't': + token = optarg; // token is a static global + cerr << "Authorization header value: " << token << endl; + break; + default: + break; + } + + argc -= optind; + argv += optind; + + bool wasSuccessful = true; + string test; + if (0 == argc) { + // run them all + wasSuccessful = runner.run(""); + } else { + int i = 0; + while (i < argc) { + if (debug) cerr << "Running " << argv[i] << endl; + test = http::SignedUrlCacheTest::suite()->getName().append("::").append(argv[i]); + wasSuccessful = wasSuccessful && runner.run(test); + ++i; + } + } + + return wasSuccessful ? 0 : 1; +} diff --git a/modules/dmrpp_module/Chunk.cc b/modules/dmrpp_module/Chunk.cc index 88ab79b9f7..1fcc1407bf 100644 --- a/modules/dmrpp_module/Chunk.cc +++ b/modules/dmrpp_module/Chunk.cc @@ -44,6 +44,7 @@ #include "CurlUtils.h" #include "CurlHandlePool.h" #include "EffectiveUrlCache.h" +#include "SignedUrlCache.h" #include "DmrppRequestHandler.h" #include "DmrppNames.h" #include "byteswap_compat.h" @@ -51,6 +52,7 @@ using namespace std; using http::EffectiveUrlCache; +using http::SignedUrlCache; #define prolog std::string("Chunk::").append(__func__).append("() - ") @@ -1348,8 +1350,8 @@ string Chunk::to_string() const { * This method returns the data URL for this chunk. If the data URL is not * set, it returns nullptr. * - * @note The call to get_signed_url() will first attempt to create a locally-signed url; if - * that fails, it will fall through to calling EffectiveUrlCache::get_effective_url() + * @note The call to get_signed_url() will first attempt to create a locally-signed url via SignedUrlCache::; if + * that fails, it will fall through to calling EffectiveUrlCache::get_signed_url() * which will call CurlUtils.cc get_redirect_url() which will call gru_mk_attempt() and * will look for an HTTP 302 response and return the redirect URL in that response. * @@ -1361,19 +1363,19 @@ std::shared_ptr Chunk::get_data_url() const { if (d_data_url == nullptr) return d_data_url; - std::shared_ptr effective_url = EffectiveUrlCache::TheCache()->get_signed_url(d_data_url); + std::shared_ptr url = SignedUrlCache::TheCache()->get_signed_url(d_data_url); - if (effective_url == nullptr) { + if (url == nullptr) { INFO_LOG(prolog + "Failed to locally sign url; constructing effective_url via redirects."); - effective_url = EffectiveUrlCache::TheCache()->get_effective_url(d_data_url); + url = EffectiveUrlCache::TheCache()->get_effective_url(d_data_url); } - BESDEBUG(MODULE, prolog << "Using data_url: " << effective_url->str() << endl); + BESDEBUG(MODULE, prolog << "Using data_url: " << url->str() << endl); #if ENABLE_TRACKING_QUERY_PARAMETER //A conditional call to void Chunk::add_tracking_query_param() // here for the NASA cost model work THG's doing. jhrg 8/7/18 if (!d_query_marker.empty()) { - string url_str = effective_url->str(); + string url_str = url->str(); if(url_str.find('?') != string::npos){ url_str.append("&"); } @@ -1386,7 +1388,7 @@ std::shared_ptr Chunk::get_data_url() const { } #endif - return effective_url; + return url; } } // namespace dmrpp diff --git a/modules/dmrpp_module/ngap_container/NgapOwnedContainer.cc b/modules/dmrpp_module/ngap_container/NgapOwnedContainer.cc index e0381a91ba..4953fbe74f 100644 --- a/modules/dmrpp_module/ngap_container/NgapOwnedContainer.cc +++ b/modules/dmrpp_module/ngap_container/NgapOwnedContainer.cc @@ -46,6 +46,7 @@ #include "BESSyntaxUserError.h" #include "BESDebug.h" #include "EffectiveUrlCache.h" +#include "SignedUrlCache.h" #include "NgapOwnedContainer.h" #include "NgapNames.h" @@ -67,6 +68,7 @@ using namespace std; using namespace bes; using http::EffectiveUrlCache; +using http::SignedUrlCache; using namespace pugi; namespace ngap { @@ -631,7 +633,7 @@ string NgapOwnedContainer::access() { // To sign urls locally, we need access to the credential info that has been previously // injected into the dmrpp. Extract that now, in preparation for upcoming url signing. auto urls = extract_s3_data_urls_from_dmrpp(dmrpp_string); - EffectiveUrlCache::TheCache()->cache_signed_url_components(get<0>(urls), get<1>(urls), get<2>(urls)); + SignedUrlCache::TheCache()->cache_signed_url_components(get<0>(urls), get<1>(urls), get<2>(urls)); set_attributes("as-string"); // This means access() returns a string. jhrg 10/19/23 // Originally, this was either hard-coded (as it is now) or was set using the 'extension' From a19ee90022b942c2b78cefdcfc8d0d027d8a6f50 Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Tue, 28 Oct 2025 23:01:01 -0400 Subject: [PATCH 05/77] Split SignedUrlCache out of EffectiveUrlCache --- CMakeLists.txt | 3 + http/EffectiveUrlCache.cc | 113 ----- http/EffectiveUrlCache.h | 18 - http/Makefile.am | 2 + http/SignedUrlCache.cc | 360 ++++++++++++++ http/SignedUrlCache.h | 126 +++++ http/unit-tests/.gitignore | 1 + http/unit-tests/Makefile.am | 5 +- http/unit-tests/SignedUrlCacheTest.cc | 443 ++++++++++++++++++ modules/dmrpp_module/Chunk.cc | 18 +- .../ngap_container/NgapOwnedContainer.cc | 4 +- 11 files changed, 952 insertions(+), 141 deletions(-) create mode 100644 http/SignedUrlCache.cc create mode 100644 http/SignedUrlCache.h create mode 100644 http/unit-tests/SignedUrlCacheTest.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index 9dae673f29..e6ee70d6d8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -528,7 +528,10 @@ add_executable( http/EffectiveUrl.h http/EffectiveUrlCache.cc http/EffectiveUrlCache.h + http/SignedUrlCache.cc + http/SignedUrlCache.h http/unit-tests/EffectiveUrlCacheTest.cc + http/unit-tests/SignedUrlCacheTest.cc http/unit-tests/HttpUrlTest.cc http/unit-tests/RemoteResourceTest.cc http/unit-tests/AllowedHostsTest.cc diff --git a/http/EffectiveUrlCache.cc b/http/EffectiveUrlCache.cc index e32e8466f3..ef318a442d 100644 --- a/http/EffectiveUrlCache.cc +++ b/http/EffectiveUrlCache.cc @@ -41,8 +41,6 @@ #include "EffectiveUrl.h" #include "EffectiveUrlCache.h" -#include "rapidjson/document.h" - using namespace std; constexpr auto MODULE = "euc"; @@ -162,117 +160,6 @@ shared_ptr EffectiveUrlCache::get_effective_url(shared_ptr return effective_url; } -// TODO: docstring -void EffectiveUrlCache::cache_signed_url_components(const std::string &key, const std::string &s3_url, const std::string &s3credentials_url) { - // TODO: if urls are empty, do....something?? skip it or something - INFO_LOG(prolog + "Caching signed url components for key: " + key); - d_href_to_s3_cache[key] = s3_url; - d_href_to_s3credentials_cache[key] = s3credentials_url; -} - -shared_ptr EffectiveUrlCache::get_cache_s3credentials(std::string const &s3credentials_url) { - //TODO-future: make cache for these credentials, check if they've already been retrieved - - // 1. Get the credentials from TEA - std::string s3credentials_json_string; - try { - BES_PROFILE_TIMING(string("Request s3 credentials from TEA - ") + s3credentials_url); - curl::http_get(s3credentials_url, s3credentials_json_string); - } - catch (http::HttpError &http_error) { - string err_msg = prolog + "Encountered an error while " - "attempting to retrieve s3 credentials from TEA. " + http_error.get_message(); - INFO_LOG(err_msg); - return nullptr; - } - if (s3credentials_json_string.empty()) { - string err_msg = prolog + "Unable to retrieve s3 credentials from TEA endpoint " + s3credentials_url; - INFO_LOG(err_msg); - return nullptr; - } - - // 2. Parse the response to pull out the credentials - auto credentials = get_s3_credentials_from_tea_endpoint_json(s3credentials_json_string); - - //TODO-future: add these credentials to the cache - - return credentials; -} - - -// Lightly adapted from get_urls_from_granules_umm_json_v1_4 -std::shared_ptr EffectiveUrlCache::get_s3_credentials_from_tea_endpoint_json(std::string const &s3credentials_json_string) { - rapidjson::Document s3credentials_response; - s3credentials_response.Parse(s3credentials_json_string.c_str()); - - string access_key_id; - string secret_access_key; - string session_token; - string expiration; - - auto itr = s3credentials_response.FindMember("accessKeyId"); - if (itr != s3credentials_response.MemberEnd()) { - access_key_id = itr->value.GetString(); - } - - itr = s3credentials_response.FindMember("secretAccessKey"); - if (itr != s3credentials_response.MemberEnd()) { - secret_access_key = itr->value.GetString(); - } - - itr = s3credentials_response.FindMember("sessionToken"); - if (itr != s3credentials_response.MemberEnd()) { - session_token = itr->value.GetString(); - } - - itr = s3credentials_response.FindMember("expiration"); - if (itr != s3credentials_response.MemberEnd()) { - expiration = itr->value.GetString(); - } - - if (access_key_id.empty() || secret_access_key.empty() || session_token.empty() || expiration.empty()) { - return nullptr; - } - - return make_shared(access_key_id, - secret_access_key, - session_token, - expiration); -} - - - -/** - * TODO - * - * @param source_url - * @returns The effective (signed) URL -*/ -std::shared_ptr EffectiveUrlCache::get_signed_url(std::shared_ptr source_url) { - INFO_LOG(prolog + "GET SIGNED URL NOT YET IMPLEMENTED, falling back to get_effective_url; returns nullptr"); - - //TODO-implement! Copy get_effective_url boilerplate, then... - - // 1. Get relevant urls - auto s3_url = d_href_to_s3_cache[source_url->str()]; - auto s3credentials_url = d_href_to_s3credentials_cache[source_url->str()]; - if (s3_url.empty() || s3credentials_url.empty()) { - INFO_LOG(prolog + "No url available for TEA s3credentials endpoint."); - return get_effective_url(source_url); - } - - // 2. Get values from s3credentials endpoint - auto s3_access_key_tuple = get_cache_s3credentials(s3credentials_url); - if (s3_access_key_tuple == nullptr) { - return get_effective_url(source_url); - } - - - // TODO-2: use aws sdk to sign the url - - return nullptr; -} - /** * @return Is the cache enabled (set in the bes.conf file)? */ diff --git a/http/EffectiveUrlCache.h b/http/EffectiveUrlCache.h index 06fcd8cd58..27f85d4d36 100644 --- a/http/EffectiveUrlCache.h +++ b/http/EffectiveUrlCache.h @@ -58,15 +58,6 @@ class EffectiveUrlCache : public BESObj { std::map> d_effective_urls; - // TODO: The following will be moved to new class before merge -public: - typedef std::tuple S3AccessKeyTuple; -private: - std::map d_href_to_s3credentials_cache; - std::map d_href_to_s3_cache; - std::shared_ptr get_cache_s3credentials(std::string const &s3credentials_url); - static std::shared_ptr get_s3_credentials_from_tea_endpoint_json(std::string const &s3credentials_json_string); - // URLs that match are not cached. std::unique_ptr d_skip_regex = nullptr; @@ -103,15 +94,6 @@ class EffectiveUrlCache : public BESObj { ~EffectiveUrlCache() override = default; std::shared_ptr get_effective_url(std::shared_ptr source_url); - - /* - * TODO: temporarily lives here; move into own class before merge! - */ - std::shared_ptr get_signed_url(std::shared_ptr source_url); - void cache_signed_url_components(const std::string &key, const std::string &s3_url, const std::string &s3credentials_url); - /* - * End temp contents! - */ void dump(std::ostream &strm) const override; diff --git a/http/Makefile.am b/http/Makefile.am index 631c4c6d3a..94789462d7 100644 --- a/http/Makefile.am +++ b/http/Makefile.am @@ -41,6 +41,7 @@ SRCS = CurlUtils.cc \ HttpUtils.cc \ ProxyConfig.cc \ EffectiveUrlCache.cc \ + SignedUrlCache.cc \ url_impl.cc \ EffectiveUrl.cc \ AllowedHosts.cc \ @@ -55,6 +56,7 @@ HDRS = CurlUtils.h \ ProxyConfig.h \ HttpNames.h \ EffectiveUrlCache.h \ + SignedUrlCache.h \ url_impl.h \ EffectiveUrl.h \ AllowedHosts.h \ diff --git a/http/SignedUrlCache.cc b/http/SignedUrlCache.cc new file mode 100644 index 0000000000..99bdd8d36d --- /dev/null +++ b/http/SignedUrlCache.cc @@ -0,0 +1,360 @@ +// -*- mode: c++; c-basic-offset:4 -*- + +// This file is part of the BES http package, part of the Hyrax data server. + +// Copyright (c) 2020 OPeNDAP, Inc. +// Author: Nathan Potter +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +// +// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112. + +// Authors: +// ndp Nathan Potter + +#include "config.h" + +#include + +#include +#include + +#include "TheBESKeys.h" +#include "BESDebug.h" +#include "BESStopWatch.h" +#include "BESUtil.h" +#include "CurlUtils.h" +#include "HttpError.h" +#include "HttpNames.h" +#include "EffectiveUrl.h" +#include "SignedUrlCache.h" + +#include "rapidjson/document.h" + +using namespace std; + +constexpr auto MODULE = "euc"; +constexpr auto MODULE_TIMER = "euc:timer"; +constexpr auto MODULE_DUMPER = "euc:dump"; + +#define prolog std::string("SignedUrlCache::").append(__func__).append("() - ") + +namespace http { + +/** + * @brief Get the cached signed URL. + * @param url_key Key to a cached signed URL. + * @note This method is not, itself, thread safe. + */ +shared_ptr SignedUrlCache::get_cached_signed_url(string const &url_key) { + shared_ptr signed_url(nullptr); + auto it = d_signed_urls.find(url_key); + if (it != d_signed_urls.end()) { + signed_url = (*it).second; + } + return signed_url; +} + +// TODO-docstring +bool SignedUrlCache::are_s3credentials_expired(std::shared_ptr const credentials) { + // TODO: implement! + return false; +} + +/** + * @brief Get the cached signed URL. + * @param url_key Key to a cached signed URL. + * @note This method is not, itself, thread safe. + */ +shared_ptr SignedUrlCache::retrieve_cached_s3credentials(string const &url_key) { + shared_ptr s3_access_key_tuple(nullptr); + auto it = d_s3credentials_cache.find(url_key); + if (it != d_s3credentials_cache.end()) { + // Is it expired? If so, erase it + if (are_s3credentials_expired(it->second)) { + d_s3credentials_cache.erase(it); + } else { + s3_access_key_tuple = it->second; + } + } + return s3_access_key_tuple; +} + +/** + * Find the terminal (effective) url for the source_url. If the source_url matches the + * skip_regex then it will not be cached. + * + * Unlike EffectiveUrlCache, return nullptr instead of making a new EffectiveUrl(source_url); + * + * @param source_url + * @returns The signed effective URL, nullptr if none able to be created +*/ +shared_ptr SignedUrlCache::get_signed_url(shared_ptr source_url) { + + BESDEBUG(MODULE, prolog << "BEGIN url: " << source_url->str() << endl); + BESDEBUG(MODULE_DUMPER, prolog << "dump: " << endl << dump() << endl); + + // Lock access to the cache, the d_signed_urls map. Released when the lock goes out of scope. + std::lock_guard lock_me(d_cache_lock_mutex); + + if (!is_enabled()) { + BESDEBUG(MODULE, prolog << "CACHE IS DISABLED." << endl); + return nullptr; + } + + // if it's not an HTTP url there is nothing to cache. + if (source_url->str().find(HTTP_PROTOCOL) != 0 && source_url->str().find(HTTPS_PROTOCOL) != 0) { + BESDEBUG(MODULE, prolog << "END Not an HTTP request, SKIPPING." << endl); + return nullptr; + } + + if (!d_skip_regex) { + set_skip_regex(); + } + + if (d_skip_regex) { + size_t match_length = 0; + match_length = d_skip_regex->match(source_url->str().c_str(), (int) source_url->str().size()); + if (match_length == source_url->str().size()) { + BESDEBUG(MODULE, prolog << "END Candidate url matches the " + "no_redirects_regex_pattern [" << d_skip_regex->pattern() << + "][match_length=" << match_length << "] SKIPPING." << endl); + return nullptr; + } + BESDEBUG(MODULE, prolog << "Candidate url: '" << source_url->str() + << "' does NOT match the skip_regex pattern [" << d_skip_regex->pattern() << "]" + << endl); + } else { + BESDEBUG(MODULE, prolog << "The cache_effective_urls_skip_regex() was NOT SET " << endl); + } + + shared_ptr signed_url = get_cached_signed_url(source_url->str()); + bool retrieve_and_cache = !signed_url || signed_url->is_expired(); + // TODO-H: make sure that `signed_url->is_expired();` handles correctly for the signed urls + + // It not found or expired, (re)load. + if (retrieve_and_cache) { + BESDEBUG(MODULE, prolog << "Acquiring signed URL for " << source_url->str() << endl); + { + BES_STOPWATCH_START(MODULE_TIMER, prolog + "Retrieve and cache signed url for source url: " + source_url->str()); + + // 1. Get requisite s3 data urls... + string s3_url; + string s3credentials_url; + tie(s3_url, s3credentials_url) = retrieve_cached_signed_url_components(source_url->str()); + if (s3_url.empty() || s3credentials_url.empty()) { + return nullptr; + } + + // 2. Get unexpired access credentials from cache or s3credentials endpoint... + auto s3_access_key_tuple = retrieve_cached_s3credentials(s3credentials_url); + if (!s3_access_key_tuple) { + s3_access_key_tuple = get_s3credentials_from_endpoint(s3credentials_url); + } + if (!s3_access_key_tuple) { + return nullptr; + } + + // 3: ...and use them to create a signed url + signed_url = sign_url(s3_url, s3_access_key_tuple); + if (!signed_url) { + return nullptr; + } + d_signed_urls[source_url->str()] = signed_url; + } + BESDEBUG(MODULE, prolog << " source_url: " << source_url->str() << " (" + << (source_url->is_trusted() ? "" : "NOT ") << "trusted)" << endl); + BESDEBUG(MODULE, prolog << "signed_url: " << signed_url->dump() << " (" + << (source_url->is_trusted() ? "" : "NOT ") << "trusted)" << endl); + + + BESDEBUG(MODULE, prolog << "Updated record for " << source_url->str() << " cache size: " + << d_signed_urls.size() << endl); + + // Since we don't want there to be a concurrency issue when we release the lock, we don't + // return the instance of shared_ptr that we placed in the cache. Rather + // we make a clone and return that. It will have its own lifecycle independent of + // the instance we placed in the cache - it can be modified and the one in the cache + // is unchanged. Trusted state was established from source_url when signed_url was + // created in sign_url() + signed_url = make_shared(signed_url); + } else { + // Here we have a !expired instance of a shared_ptr retrieved from the cache. + // Now we need to make a copy to return, inheriting trust from the requesting URL. + signed_url = make_shared(signed_url, source_url->is_trusted()); + } + + BESDEBUG(MODULE_DUMPER, prolog << "dump: " << endl << dump() << endl); + BESDEBUG(MODULE, prolog << "END" << endl); + + return signed_url; +} + +// TODO: docstring +void SignedUrlCache::cache_signed_url_components(const std::string &key_href_url, const std::string &s3_url, const std::string &s3credentials_url) { + if (s3_url.empty() || s3credentials_url.empty() ) { + // Don't cache either if one is empty. + return; + } + d_href_to_s3_cache[key_href_url] = s3_url; + d_href_to_s3credentials_cache[key_href_url] = s3credentials_url; +} + +// TODO: docstring +std::pair SignedUrlCache::retrieve_cached_signed_url_components(const std::string &key_href_url) const { + auto it_s3_url = d_href_to_s3_cache.find(key_href_url); + auto it_s3credentials_url = d_href_to_s3credentials_cache.find(key_href_url); + if (it_s3_url != d_href_to_s3_cache.end() || it_s3credentials_url != d_href_to_s3credentials_cache.end() ) { + INFO_LOG(prolog + "No url available for TEA s3credentials endpoint."); + return std::pair("", ""); + } + return std::pair(it_s3_url->second, it_s3credentials_url->second); +} + +// TODO: docstring +shared_ptr SignedUrlCache::get_s3credentials_from_endpoint(std::string const &s3credentials_url) { + // 1. Get the credentials from TEA + std::string s3credentials_json_string; + try { + BES_PROFILE_TIMING(string("Request s3 credentials from TEA - ") + s3credentials_url); + curl::http_get(s3credentials_url, s3credentials_json_string); + } + catch (http::HttpError &http_error) { + string err_msg = prolog + "Encountered an error while " + "attempting to retrieve s3 credentials from TEA. " + http_error.get_message(); + INFO_LOG(err_msg); + return nullptr; + } + if (s3credentials_json_string.empty()) { + string err_msg = prolog + "Unable to retrieve s3 credentials from TEA endpoint " + s3credentials_url; + INFO_LOG(err_msg); + return nullptr; + } + + // 2. Parse the response to pull out the credentials + auto credentials = extract_s3_credentials_from_response_json(s3credentials_json_string); + if (credentials) { + // Store credentials if any were retrieved + d_s3credentials_cache[s3credentials_url] = credentials; + } + return credentials; +} + + +// Lightly adapted from get_urls_from_granules_umm_json_v1_4 +std::shared_ptr SignedUrlCache::extract_s3_credentials_from_response_json(std::string const &s3credentials_json_string) { + rapidjson::Document s3credentials_response; + s3credentials_response.Parse(s3credentials_json_string.c_str()); + + string access_key_id; + string secret_access_key; + string session_token; + string expiration; + + auto itr = s3credentials_response.FindMember("accessKeyId"); + if (itr != s3credentials_response.MemberEnd()) { + access_key_id = itr->value.GetString(); + } + + itr = s3credentials_response.FindMember("secretAccessKey"); + if (itr != s3credentials_response.MemberEnd()) { + secret_access_key = itr->value.GetString(); + } + + itr = s3credentials_response.FindMember("sessionToken"); + if (itr != s3credentials_response.MemberEnd()) { + session_token = itr->value.GetString(); + } + + itr = s3credentials_response.FindMember("expiration"); + if (itr != s3credentials_response.MemberEnd()) { + expiration = itr->value.GetString(); + } + + if (access_key_id.empty() || secret_access_key.empty() || session_token.empty() || expiration.empty()) { + return nullptr; + } + return make_shared(access_key_id, + secret_access_key, + session_token, + expiration); +} + + +/** + * TODO + * + * @param source_url + * @returns The effective (signed) URL, which may be a nullptr if signing credentials are not found +*/ +std::shared_ptr SignedUrlCache::sign_url(std::string const &s3_url, std::shared_ptr const s3_access_key_tuple) { + + // TODO-HERE: actually sign this!! + + return nullptr; +} + +/** + * @return Is the cache enabled (set in the bes.conf file)? + * Follows the same settings as the EffectiveUrlsCache + */ +bool SignedUrlCache::is_enabled() { + // The first time here, the value of d_enabled is -1. Once we check for it in TheBESKeys + // The value will be 0 (false) or 1 (true) and TheBESKeys will not be checked again. + if (d_enabled < 0) { + string value = TheBESKeys::TheKeys()->read_string_key(HTTP_CACHE_EFFECTIVE_URLS_KEY, "false"); + d_enabled = BESUtil::lowercase(value) == "true"; + } + BESDEBUG(MODULE, prolog << "d_enabled: " << (d_enabled ? "true" : "false") << endl); + return d_enabled; +} + +// * Follows the same settings as the EffectiveUrlsCache +void SignedUrlCache::set_skip_regex() { + if (!d_skip_regex) { + string pattern = TheBESKeys::TheKeys()->read_string_key(HTTP_CACHE_EFFECTIVE_URLS_SKIP_REGEX_KEY, ""); + if (!pattern.empty()) { + d_skip_regex.reset(new BESRegex(pattern.c_str())); + } + BESDEBUG(MODULE, prolog << "d_skip_regex: " + << (d_skip_regex ? d_skip_regex->pattern() : "Value has not been set.") << endl); + } +} + +/** + * @brief dumps information about this object + * @param strm C++ i/o stream to dump the information to + */ +void SignedUrlCache::dump(ostream &strm) const { + strm << BESIndent::LMarg << prolog << "(this: " << (void *) this << ")" << endl; + BESIndent::Indent(); + strm << BESIndent::LMarg << "d_skip_regex: " << (d_skip_regex ? d_skip_regex->pattern() : "WAS NOT SET") << endl; + if (!d_signed_urls.empty()) { + strm << BESIndent::LMarg << "signed url list:" << endl; + BESIndent::Indent(); + for (auto const &i: d_signed_urls) { + strm << BESIndent::LMarg << i.first << " --> " << i.second->str(); + } + BESIndent::UnIndent(); + } else { + strm << BESIndent::LMarg << "signed url list: EMPTY" << endl; + } + + //TODO: add other three caches here: + + BESIndent::UnIndent(); +} + +} // namespace http diff --git a/http/SignedUrlCache.h b/http/SignedUrlCache.h new file mode 100644 index 0000000000..dc76feed79 --- /dev/null +++ b/http/SignedUrlCache.h @@ -0,0 +1,126 @@ +// -*- mode: c++; c-basic-offset:4 -*- + +// This file is part of the BES http package, part of the Hyrax data server. + +// Copyright (c) 2020 OPeNDAP, Inc. +// Author: Nathan Potter +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +// +// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112. + +// Authors: +// ndp Nathan Potter + +#ifndef _bes_http_SignedUrlCache_h_ +#define _bes_http_SignedUrlCache_h_ 1 + +#include +#include +#include +#include +#include + +#include "BESObj.h" +#include "BESRegex.h" // for std::unique_ptr + +namespace http { + +class EffectiveUrl; +class url; + +/** + * This is a singleton class. It is used to associate a URL with its "pre-signed" AWS s3 URL. This means that + * a URL is signed locally rather than sent through a potentially large number of external redirect actions, as + * in EffectiveUrlCache.h. This url location plus the requisite AWS signature headers, from which the requested bytes + * are transmitted, is termed the "effective url" and is stored in an in memory cache (std::map) so that later + * requests may skip the external signing service and just get required bytes from the actual source. + */ +class SignedUrlCache : public BESObj { +public: + typedef std::tuple S3AccessKeyTuple; + +private: + SignedUrlCache() = default; + + std::mutex d_cache_lock_mutex; + + std::map> d_signed_urls; + + std::map d_href_to_s3credentials_cache; + std::map d_href_to_s3_cache; + + std::shared_ptr get_s3credentials_from_endpoint(std::string const &s3credentials_url); + static std::shared_ptr extract_s3_credentials_from_response_json(std::string const &s3credentials_json_string); + + std::map> d_s3credentials_cache; + std::shared_ptr retrieve_cached_s3credentials(std::string const &url_key); + static bool are_s3credentials_expired(std::shared_ptr const credentials); + + // URLs that match are not cached. + std::unique_ptr d_skip_regex = nullptr; + + int d_enabled = -1; + + // TODO: for all helper functions here, think carefully about static/const/etc + // TODO: think a little more about splitting functionality to invert control to the caller + std::shared_ptr sign_url(std::string const &s3_url, std::shared_ptr const s3_access_key_tuple); + std::shared_ptr get_cached_signed_url(std::string const &url_key); + + void set_skip_regex(); + + bool is_enabled(); + + friend class SignedUrlCacheTest; + +public: + /** @brief Get the singleton SignedUrlCache instance. + * + * This static method returns the instance of this singleton class. + * The implementation will only build one instance of SignedUrlCache and + * thereafter return a pointer to that instance. + * + * Thread safe with C++-11 and greater. + * + * @return A pointer to the SignedUrlCache singleton + */ + static SignedUrlCache *TheCache() { + // Create a local static object the first time the function is called + static SignedUrlCache instance; + return &instance; + } + + SignedUrlCache(const SignedUrlCache &src) = delete; + SignedUrlCache &operator=(const SignedUrlCache &rhs) = delete; + + ~SignedUrlCache() override = default; + + void cache_signed_url_components(const std::string &key_href_url, const std::string &s3_url, const std::string &s3credentials_url); + std::pair retrieve_cached_signed_url_components(const std::string &key_href_url) const; + std::shared_ptr get_signed_url(std::shared_ptr source_url); + + void dump(std::ostream &strm) const override; + + std::string dump() const { + std::stringstream sstrm; + dump(sstrm); + return sstrm.str(); + } +}; + +} // namespace http + +#endif // _bes_http_SignedUrlCache_h_ + diff --git a/http/unit-tests/.gitignore b/http/unit-tests/.gitignore index b623b0b70c..6e7bb2da4d 100644 --- a/http/unit-tests/.gitignore +++ b/http/unit-tests/.gitignore @@ -3,6 +3,7 @@ /CurlUtilsTest /CurlSListTest /EffectiveUrlCacheTest +/SignedUrlCacheTest /HttpUrlTest /HttpErrorTest /AllowedHostsTest diff --git a/http/unit-tests/Makefile.am b/http/unit-tests/Makefile.am index a63447c538..eca33225a2 100644 --- a/http/unit-tests/Makefile.am +++ b/http/unit-tests/Makefile.am @@ -74,7 +74,7 @@ bes_ngap_s3_creds.conf: bes_ngap_s3_creds.conf.in $(top_srcdir)/configure.ac if CPPUNIT -UNIT_TESTS = HttpUtilsTest HttpErrorTest RemoteResourceTest EffectiveUrlCacheTest HttpUrlTest \ +UNIT_TESTS = HttpUtilsTest HttpErrorTest RemoteResourceTest EffectiveUrlCacheTest HttpUrlTest SignedUrlCacheTest \ AllowedHostsTest awsv4_test CurlUtilsTest CurlSListTest # CredentialsManagerTest was removed because it was testing our S3 signing code and @@ -122,6 +122,9 @@ CurlUtilsTest_LDADD = $(LIBADD) EffectiveUrlCacheTest_SOURCES = EffectiveUrlCacheTest.cc EffectiveUrlCacheTest_LDADD = $(LIBADD) +SignedUrlCacheTest_SOURCES = SignedUrlCacheTest.cc +SignedUrlCacheTest_LDADD = $(LIBADD) + HttpUrlTest_SOURCES = HttpUrlTest.cc HttpUrlTest_LDADD = $(LIBADD) diff --git a/http/unit-tests/SignedUrlCacheTest.cc b/http/unit-tests/SignedUrlCacheTest.cc new file mode 100644 index 0000000000..ed0d5d694e --- /dev/null +++ b/http/unit-tests/SignedUrlCacheTest.cc @@ -0,0 +1,443 @@ +// -*- mode: c++; c-basic-offset:4 -*- + +// This file is part of the BES component of the Hyrax Data Server. + +// Copyright (c) 2018 OPeNDAP, Inc. +// Author: Nathan Potter +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +// +// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112. + +#include "config.h" + +#include +#include + +#include +#include +#include + +#include + +#include "BESError.h" +#include "BESDebug.h" +#include "BESUtil.h" +#include "BESCatalogList.h" +#include "TheBESKeys.h" +#include "BESContextManager.h" + +#include "HttpNames.h" +#include "url_impl.h" +#include "EffectiveUrl.h" +#include "SignedUrlCache.h" + +#include "test_config.h" + +using namespace std; + +static bool debug = false; +static bool Debug = false; +static bool bes_debug = false; +static bool ngap_tests = false; +static std::string token; + +#undef DBG +#define DBG(x) do { if (debug) x; } while(false) +#define prolog std::string("SignedUrlCacheTest::").append(__func__).append("() - ") + +namespace http { + +class SignedUrlCacheTest : public CppUnit::TestFixture { +private: + string d_data_dir = TEST_DATA_DIR; + + void show_file(string filename) { + ifstream t(filename.c_str()); + + if (t.is_open()) { + string file_content((istreambuf_iterator(t)), istreambuf_iterator()); + t.close(); + cerr << endl << "#############################################################################" << endl; + cerr << "file: " << filename << endl; + cerr << ". . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . " << endl; + cerr << file_content << endl; + cerr << "#############################################################################" << endl; + } else { + cerr << "FAILED TO OPEN FILE: " << filename << endl; + } + } + +public: + // Called once before everything gets tested + SignedUrlCacheTest() = default; + + // Called at the end of the tests + ~SignedUrlCacheTest() override = default; + + // Called before each test + void setUp() override { + DBG(cerr << endl); + DBG(cerr << prolog << "BEGIN" << endl); + DBG(cerr << prolog << "data_dir: " << d_data_dir << endl); + string bes_conf = BESUtil::assemblePath(TEST_BUILD_DIR, "bes.conf"); + DBG(cerr << prolog << "Using BES configuration: " << bes_conf << endl); + if (Debug) show_file(bes_conf); + TheBESKeys::ConfigFile = bes_conf; + + if (bes_debug) BESDebug::SetUp("cerr,bes,euc,http,curl"); + + // Clear the cache for the next test. + SignedUrlCache *theCache = SignedUrlCache::TheCache(); + theCache->d_signed_urls.clear(); + + if (!token.empty()) { + DBG(cerr << "Setting BESContext " << EDL_AUTH_TOKEN_KEY << " to: '" << token << "'" << endl); + BESContextManager::TheManager()->set_context(EDL_AUTH_TOKEN_KEY, token); + } + DBG(cerr << prolog << "END" << endl); + } + +/*##################################################################################################*/ +/* TESTS BEGIN */ + + void is_cache_disabled_test() { + DBG(cerr << prolog << "SignedUrlCache::TheCache()->is_enabled(): " + << (SignedUrlCache::TheCache()->is_enabled() ? "true" : "false") << endl); + CPPUNIT_ASSERT(!SignedUrlCache::TheCache()->is_enabled()); + + shared_ptr src_url_00(new http::url("http://started_here.com")); + auto effective_url_00 = shared_ptr(new http::EffectiveUrl("https://ended_here.com")); + + SignedUrlCache::TheCache()->d_signed_urls.insert( + pair>(src_url_00->str(), effective_url_00)); + CPPUNIT_ASSERT(SignedUrlCache::TheCache()->d_signed_urls.size() == 1); + + // This one does not add the URL or even check it because it _should_ be matching the skip regex. + auto result_url = SignedUrlCache::TheCache()->get_signed_url(src_url_00); + CPPUNIT_ASSERT(result_url->str() == src_url_00->str()); + } + + void skip_regex_test_01() { + DBG(cerr << prolog << "BEGIN" << endl); + try { + // The cache is disabled in bes.conf, so we need to turn it on. + SignedUrlCache::TheCache()->d_enabled = true; + + // This one does not add the URL or even check it because it _should_ be matching the skip regex + // in the bes.conf + shared_ptr src_url(new http::url("https://foobar.com/opendap/data/nc/fnoc1.nc?dap4.ce=u;v")); + auto result_url = SignedUrlCache::TheCache()->get_signed_url(src_url); + CPPUNIT_ASSERT(SignedUrlCache::TheCache()->d_signed_urls.empty()); + CPPUNIT_ASSERT(result_url->str() == src_url->str()); + } + catch (const BESError &be) { + stringstream msg; + msg << prolog << "ERROR! Caught BESError. Message: " << be.get_message() << endl; + CPPUNIT_FAIL(msg.str()); + } + } + + void cache_test_00() { + DBG(cerr << prolog << "BEGIN" << endl); + //string source_url; + //string value; + try { + // The cache is disabled in bes.conf, so we need to turn it on. + SignedUrlCache::TheCache()->d_enabled = true; + +#if 0 + string src_url_00 = "https://d1jecqxxv88lkr.cloudfront.net/ghrcwuat-protected/rss_demo/rssmif16d__7/f1" + "6_ssmis_20040107v7.nc"; + string eurl_str = "https://ghrcwuat-protected.s3.us-west-2.amazonaws.com/rss_demo/rssmif16d__7/f16_ssm" + "is_20031229v7.nc?A-userid=hyrax&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=A" + "SIASF4N-AWS-Creds-00808%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Date=20200808T032623Z" + "&X-Amz-Expires=86400&X-Amz-Security-Token=FwoGZXIvYXdzE-AWS-Sec-Token-MWRLIZGYvDx1O" + "Nzd0ffK8VtxO8JP7thrGIQ%3D%3D&X-Amz-SignedHeaders=host&X-Amz-Signature=260a7c4dd4-AW" + "S-SIGGY-0c7a39ee899"; + auto effective_url_00 = shared_ptr(new http::EffectiveUrl(eurl_str)); + SignedUrlCache::TheCache()->d_signed_urls.insert(pair>(src_url_00, effective_url_00)); + CPPUNIT_ASSERT(SignedUrlCache::TheCache()->d_signed_urls.size() == 1); +#endif + + string src_url_01 = "http://test.opendap.org/data/httpd_catalog/READTHIS"; + string eurl_str = "https://test.opendap.org/data/httpd_catalog/READTHIS"; + auto effective_url_01 = shared_ptr(new http::EffectiveUrl(eurl_str)); + SignedUrlCache::TheCache()->d_signed_urls.insert( + pair>(src_url_01, effective_url_01)); + CPPUNIT_ASSERT(SignedUrlCache::TheCache()->d_signed_urls.size() == 1); + + // This one actually does the thing + eurl_str = "http://test.opendap.org/opendap/"; + auto expected_url_02 = std::unique_ptr(new http::EffectiveUrl(eurl_str)); + + shared_ptr src_url_02(new http::url("http://test.opendap.org/opendap")); + DBG(cerr << prolog << "Retrieving effective URL for: " << src_url_02->str() << endl); + auto result_url = SignedUrlCache::TheCache()->get_signed_url(src_url_02); + CPPUNIT_ASSERT(SignedUrlCache::TheCache()->d_signed_urls.size() == 2); + + DBG(cerr << prolog << "SignedUrlCache::TheCache()->get_signed_url() returned: " << result_url + << endl); + CPPUNIT_ASSERT(result_url->str() == expected_url_02->str()); + } + catch (const BESError &be) { + stringstream msg; + msg << prolog << "ERROR! Caught BESError. Message: " << be.get_message() << endl; + CPPUNIT_FAIL(msg.str()); + + } + DBG(cerr << prolog << "END" << endl); + } + + + void cache_test_01() { + DBG(cerr << prolog << "BEGIN" << endl); + string source_url; + string value; + string result_url; + try { + // The cache is disabled in bes.conf so we need to turn it on. + SignedUrlCache::TheCache()->d_enabled = true; + + std::map d_signed_urls; + source_url = "http://someURL"; + + http::EffectiveUrl first_eu("http://someOtherUrl"); + d_signed_urls[source_url] = &first_eu; + DBG(cerr << prolog << "source_url: " << source_url << endl); + DBG(cerr << prolog << "first_eu: " << first_eu.str() << endl); + + CPPUNIT_ASSERT(d_signed_urls[source_url] == &first_eu); + + http::EffectiveUrl second_eu("http://someMoreUrlLovin"); + d_signed_urls[source_url] = &second_eu; + DBG(cerr << prolog << "source_url: " << source_url << endl); + DBG(cerr << prolog << "second_eu: " << second_eu.str() << endl); + + CPPUNIT_ASSERT(d_signed_urls[source_url] == &second_eu); + } + catch (const BESError &be) { + stringstream msg; + msg << prolog << "ERROR! Caught BESError. Message: " << be.get_message() << endl; + CPPUNIT_FAIL(msg.str()); + + } + DBG(cerr << prolog << "END" << endl); + } + + void euc_ghrc_tea_url_test() { + if (!ngap_tests) { + DBG(cerr << prolog << "SKIPPING." << endl); + return; + } + DBG(cerr << prolog << "BEGIN" << endl); + string source_url; + string value; + try { + // The cache is disabled in bes.conf so we need to turn it on. + SignedUrlCache::TheCache()->d_enabled = true; + + shared_ptr thing1(new http::url( + "https://d1jecqxxv88lkr.cloudfront.net/ghrcwuat-protected/rss_demo/rssmif16d__7/f16_ssmis_20031026v7.nc")); + string thing1_out_of_region_effective_url_prefix = "https://d1jecqxxv88lkr.cloudfront.net/s3"; + string thing1_in_region_effective_url_prefix = "https://ghrcwuat-protected.s3.us-west-2.amazonaws.com/"; + + DBG(cerr << prolog << "Retrieving signed URL for: " << thing1->str() << endl); + auto result_url = SignedUrlCache::TheCache()->get_signed_url(thing1); + CPPUNIT_ASSERT(SignedUrlCache::TheCache()->d_signed_urls.size() == 1); + + DBG(cerr << prolog << "SignedUrlCache::TheCache()->get_signed_url() returned: " << result_url + << endl); + CPPUNIT_ASSERT(result_url->str().rfind(thing1_in_region_effective_url_prefix, 0) == 0 + || result_url->str().rfind(thing1_out_of_region_effective_url_prefix, 0) == 0); + + // result_url = SignedUrlCache::TheCache()->get_signed_url(thing1); + } + catch (const BESError &be) { + stringstream msg; + msg << prolog << "ERROR! Caught BESError. Message: " << be.get_message() << endl; + CPPUNIT_FAIL(msg.str()); + + } + DBG(cerr << prolog << "END" << endl); + } + + void euc_harmony_url_test() { + if (!ngap_tests) { + DBG(cerr << prolog << "SKIPPING." << endl); + return; + } + DBG(cerr << prolog << "BEGIN" << endl); + string source_url; + string value; + try { + // The cache is disabled in bes.conf, so we need to turn it on. + SignedUrlCache::TheCache()->d_enabled = true; + + shared_ptr thing1( + new http::url("https://harmony.uat.earthdata.nasa.gov/service-results/harmony-uat-staging/public/" + "sds/staged/ATL03_20200714235814_03000802_003_01.h5")); + string thing1_out_of_region_effective_url_prefix = "https://djpip0737hawz.cloudfront.net/s3"; + string thing1_in_region_effective_url_prefix = "https://harmony-uat-staging.s3.us-west-2.amazonaws.com/public/"; + + DBG(cerr << prolog << "Retrieving signed URL for: " << thing1->str() << endl); + auto result_url = SignedUrlCache::TheCache()->get_signed_url(thing1); + CPPUNIT_ASSERT(SignedUrlCache::TheCache()->d_signed_urls.size() == 1); + + DBG(cerr << prolog << "SignedUrlCache::TheCache()->get_signed_url() returned: " << result_url + << endl); + + CPPUNIT_ASSERT(result_url->str().rfind(thing1_in_region_effective_url_prefix, 0) == 0 + || result_url->str().rfind(thing1_out_of_region_effective_url_prefix, 0) == 0); + + // TODO ??? result_url = SignedUrlCache::TheCache()->get_signed_url(thing1); + } + catch (const BESError &be) { + stringstream msg; + msg << prolog << "ERROR! Caught BESError. Message: " << be.get_message() << endl; + CPPUNIT_FAIL(msg.str()); + + } + DBG(cerr << prolog << "END" << endl); + } + + void trusted_url_test_01() { + DBG(cerr << prolog << "BEGIN" << endl); + string url_str = "http://test.opendap.org/data/nothing_is_here.html"; + string result_url_str = "http://test.opendap.org/data/httpd_catalog/READTHIS"; + shared_ptr trusted_src_url(new http::url(url_str, true)); + shared_ptr untrusted_src_url(new http::url(url_str, false)); + + DBG(cerr << prolog << "Retrieving signed URL for: " << trusted_src_url->str() << endl); + try { + // The cache is disabled in bes.conf so we need to turn it on. + SignedUrlCache::TheCache()->d_enabled = true; + + shared_ptr result_url; + + result_url = SignedUrlCache::TheCache()->get_signed_url(untrusted_src_url); + DBG(cerr << prolog << "source_url: " << untrusted_src_url->str() << " is " + << (untrusted_src_url->is_trusted() ? "" : "NOT ") << "trusted." << endl); + DBG(cerr << prolog << "result_url: " << result_url->str() << " is " + << (result_url->is_trusted() ? "" : "NOT ") << "trusted." << endl); + DBG(cerr << prolog << "SignedUrlCache::TheCache()->d_signed_urls.size(): " + << SignedUrlCache::TheCache()->d_signed_urls.size() << endl); + CPPUNIT_ASSERT(SignedUrlCache::TheCache()->d_signed_urls.size() == 1); + CPPUNIT_ASSERT(result_url->str() == result_url_str); + CPPUNIT_ASSERT(!result_url->is_trusted()); + + result_url = SignedUrlCache::TheCache()->get_signed_url(trusted_src_url); + DBG(cerr << prolog << "source_url: " << trusted_src_url->str() << " is " + << (trusted_src_url->is_trusted() ? "" : "NOT ") << "trusted." << endl); + DBG(cerr << prolog << "result_url: " << result_url->str() << " is " + << (result_url->is_trusted() ? "" : "NOT ") << "trusted." << endl); + DBG(cerr << prolog << "SignedUrlCache::TheCache()->d_signed_urls.size(): " + << SignedUrlCache::TheCache()->d_signed_urls.size() << endl); + CPPUNIT_ASSERT(SignedUrlCache::TheCache()->d_signed_urls.size() == 1); + CPPUNIT_ASSERT(result_url->str() == result_url_str); + CPPUNIT_ASSERT(result_url->is_trusted()); + + result_url = SignedUrlCache::TheCache()->get_signed_url(untrusted_src_url); + DBG(cerr << prolog << "source_url: " << untrusted_src_url->str() << " is " + << (untrusted_src_url->is_trusted() ? "" : "NOT ") << "trusted." << endl); + DBG(cerr << prolog << "result_url: " << result_url->str() << " is " + << (result_url->is_trusted() ? "" : "NOT ") << "trusted." << endl); + DBG(cerr << prolog << "SignedUrlCache::TheCache()->d_signed_urls.size(): " + << SignedUrlCache::TheCache()->d_signed_urls.size() << endl); + CPPUNIT_ASSERT(SignedUrlCache::TheCache()->d_signed_urls.size() == 1); + CPPUNIT_ASSERT(result_url->str() == result_url_str); + CPPUNIT_ASSERT(!result_url->is_trusted()); + + } + catch (const BESError &be) { + stringstream msg; + msg << prolog << "ERROR! Caught BESError. Message: " << be.get_message() << endl; + CPPUNIT_FAIL(msg.str()); + } + DBG(cerr << prolog << "END" << endl); + } + +/* TESTS END */ +/*##################################################################################################*/ + +CPPUNIT_TEST_SUITE(SignedUrlCacheTest); + + // TODO: fix up tests, enable/replace!! add more!! + // CPPUNIT_TEST(is_cache_disabled_test); + // CPPUNIT_TEST(cache_test_00); + // CPPUNIT_TEST(cache_test_01); + // CPPUNIT_TEST(skip_regex_test_01); + // CPPUNIT_TEST(euc_ghrc_tea_url_test); + // CPPUNIT_TEST(euc_harmony_url_test); + // CPPUNIT_TEST(trusted_url_test_01); + + CPPUNIT_TEST_SUITE_END(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION(SignedUrlCacheTest); + +} // namespace httpd_catalog + +int main(int argc, char *argv[]) { + CppUnit::TextTestRunner runner; + runner.addTest(CppUnit::TestFactoryRegistry::getRegistry().makeTest()); + + int option_char; + while ((option_char = getopt(argc, argv, "dbDPt:N")) != -1) + switch (option_char) { + case 'd': + debug = true; // debug is a static global + cerr << "debug enabled" << endl; + break; + case 'D': + Debug = true; // Debug is a static global + cerr << "Debug enabled" << endl; + break; + case 'b': + bes_debug = true; // debug is a static global + cerr << "bes_debug enabled" << endl; + break; + case 'N': + ngap_tests = true; // ngap_tests is a static global + cerr << "NGAP Tests Enabled." << token << endl; + break; + case 't': + token = optarg; // token is a static global + cerr << "Authorization header value: " << token << endl; + break; + default: + break; + } + + argc -= optind; + argv += optind; + + bool wasSuccessful = true; + string test; + if (0 == argc) { + // run them all + wasSuccessful = runner.run(""); + } else { + int i = 0; + while (i < argc) { + if (debug) cerr << "Running " << argv[i] << endl; + test = http::SignedUrlCacheTest::suite()->getName().append("::").append(argv[i]); + wasSuccessful = wasSuccessful && runner.run(test); + ++i; + } + } + + return wasSuccessful ? 0 : 1; +} diff --git a/modules/dmrpp_module/Chunk.cc b/modules/dmrpp_module/Chunk.cc index 88ab79b9f7..1fcc1407bf 100644 --- a/modules/dmrpp_module/Chunk.cc +++ b/modules/dmrpp_module/Chunk.cc @@ -44,6 +44,7 @@ #include "CurlUtils.h" #include "CurlHandlePool.h" #include "EffectiveUrlCache.h" +#include "SignedUrlCache.h" #include "DmrppRequestHandler.h" #include "DmrppNames.h" #include "byteswap_compat.h" @@ -51,6 +52,7 @@ using namespace std; using http::EffectiveUrlCache; +using http::SignedUrlCache; #define prolog std::string("Chunk::").append(__func__).append("() - ") @@ -1348,8 +1350,8 @@ string Chunk::to_string() const { * This method returns the data URL for this chunk. If the data URL is not * set, it returns nullptr. * - * @note The call to get_signed_url() will first attempt to create a locally-signed url; if - * that fails, it will fall through to calling EffectiveUrlCache::get_effective_url() + * @note The call to get_signed_url() will first attempt to create a locally-signed url via SignedUrlCache::; if + * that fails, it will fall through to calling EffectiveUrlCache::get_signed_url() * which will call CurlUtils.cc get_redirect_url() which will call gru_mk_attempt() and * will look for an HTTP 302 response and return the redirect URL in that response. * @@ -1361,19 +1363,19 @@ std::shared_ptr Chunk::get_data_url() const { if (d_data_url == nullptr) return d_data_url; - std::shared_ptr effective_url = EffectiveUrlCache::TheCache()->get_signed_url(d_data_url); + std::shared_ptr url = SignedUrlCache::TheCache()->get_signed_url(d_data_url); - if (effective_url == nullptr) { + if (url == nullptr) { INFO_LOG(prolog + "Failed to locally sign url; constructing effective_url via redirects."); - effective_url = EffectiveUrlCache::TheCache()->get_effective_url(d_data_url); + url = EffectiveUrlCache::TheCache()->get_effective_url(d_data_url); } - BESDEBUG(MODULE, prolog << "Using data_url: " << effective_url->str() << endl); + BESDEBUG(MODULE, prolog << "Using data_url: " << url->str() << endl); #if ENABLE_TRACKING_QUERY_PARAMETER //A conditional call to void Chunk::add_tracking_query_param() // here for the NASA cost model work THG's doing. jhrg 8/7/18 if (!d_query_marker.empty()) { - string url_str = effective_url->str(); + string url_str = url->str(); if(url_str.find('?') != string::npos){ url_str.append("&"); } @@ -1386,7 +1388,7 @@ std::shared_ptr Chunk::get_data_url() const { } #endif - return effective_url; + return url; } } // namespace dmrpp diff --git a/modules/dmrpp_module/ngap_container/NgapOwnedContainer.cc b/modules/dmrpp_module/ngap_container/NgapOwnedContainer.cc index e0381a91ba..4953fbe74f 100644 --- a/modules/dmrpp_module/ngap_container/NgapOwnedContainer.cc +++ b/modules/dmrpp_module/ngap_container/NgapOwnedContainer.cc @@ -46,6 +46,7 @@ #include "BESSyntaxUserError.h" #include "BESDebug.h" #include "EffectiveUrlCache.h" +#include "SignedUrlCache.h" #include "NgapOwnedContainer.h" #include "NgapNames.h" @@ -67,6 +68,7 @@ using namespace std; using namespace bes; using http::EffectiveUrlCache; +using http::SignedUrlCache; using namespace pugi; namespace ngap { @@ -631,7 +633,7 @@ string NgapOwnedContainer::access() { // To sign urls locally, we need access to the credential info that has been previously // injected into the dmrpp. Extract that now, in preparation for upcoming url signing. auto urls = extract_s3_data_urls_from_dmrpp(dmrpp_string); - EffectiveUrlCache::TheCache()->cache_signed_url_components(get<0>(urls), get<1>(urls), get<2>(urls)); + SignedUrlCache::TheCache()->cache_signed_url_components(get<0>(urls), get<1>(urls), get<2>(urls)); set_attributes("as-string"); // This means access() returns a string. jhrg 10/19/23 // Originally, this was either hard-coded (as it is now) or was set using the 'extension' From a926716e6fa48a600ef36ff8b94dcb8d52cd042b Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Wed, 29 Oct 2025 16:49:10 -0400 Subject: [PATCH 06/77] wip start adding aws sdk dependence --- http/SignedUrlCache.cc | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/http/SignedUrlCache.cc b/http/SignedUrlCache.cc index 99bdd8d36d..5deb6a5e96 100644 --- a/http/SignedUrlCache.cc +++ b/http/SignedUrlCache.cc @@ -31,6 +31,7 @@ #include #include +#include "AWS_SDK.h" // TODO-include aws appropriately #include "TheBESKeys.h" #include "BESDebug.h" #include "BESStopWatch.h" @@ -301,9 +302,20 @@ std::shared_ptr SignedUrlCache::extract_s3_cre */ std::shared_ptr SignedUrlCache::sign_url(std::string const &s3_url, std::shared_ptr const s3_access_key_tuple) { - // TODO-HERE: actually sign this!! + AWS_SDK aws_sdk; + string id = get<0>(*s3_access_key_tuple); + string secret = get<1>(*s3_access_key_tuple); + aws_sdk.initialize_s3_client("us-east-1", id, secret); // TODO: uhhhhh where was the region from?????? can we get it from s3_url, I hope???? - return nullptr; + std::string bucket = s3_url.substr(5, s3_url.find("/")); // Split off "s3://" which we know is here or the path wouldn't have been cached in the first place + std::string object = s3_url.substr(s3_url.find(bucket) + 1); // Add 1 to cover the "/" after the bucket + const uint64_t expiration_seconds = 60; // TODO: get from doing math on s3_access_key_tuple ! + + const Aws::String url = aws_sdk.s3_generate_presigned_object_url(bucket, object, expiration_seconds); + + // TODO: how might that fail? is it ever null or bad or...does it throw? if so, return nullptr first.... + + return make_shared(url->str()); // TODO: get string out of AWS url??? or decide to return it..... } /** From db5fb39b53eae9ab0fdc6e8e426cd7d0d7f3d23d Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Thu, 30 Oct 2025 13:40:57 -0400 Subject: [PATCH 07/77] Add unit test: cache_disabled --- http/unit-tests/SignedUrlCacheTest.cc | 55 ++++++++++++++++++++------- 1 file changed, 41 insertions(+), 14 deletions(-) diff --git a/http/unit-tests/SignedUrlCacheTest.cc b/http/unit-tests/SignedUrlCacheTest.cc index ed0d5d694e..312adfbe32 100644 --- a/http/unit-tests/SignedUrlCacheTest.cc +++ b/http/unit-tests/SignedUrlCacheTest.cc @@ -118,16 +118,25 @@ class SignedUrlCacheTest : public CppUnit::TestFixture { << (SignedUrlCache::TheCache()->is_enabled() ? "true" : "false") << endl); CPPUNIT_ASSERT(!SignedUrlCache::TheCache()->is_enabled()); - shared_ptr src_url_00(new http::url("http://started_here.com")); - auto effective_url_00 = shared_ptr(new http::EffectiveUrl("https://ended_here.com")); + auto input_url = make_shared("http://started_here.com"); + auto output_url = make_shared("http://started_here.com?signed-now"); SignedUrlCache::TheCache()->d_signed_urls.insert( - pair>(src_url_00->str(), effective_url_00)); + pair>(input_url->str(), output_url)); CPPUNIT_ASSERT(SignedUrlCache::TheCache()->d_signed_urls.size() == 1); - // This one does not add the URL or even check it because it _should_ be matching the skip regex. - auto result_url = SignedUrlCache::TheCache()->get_signed_url(src_url_00); - CPPUNIT_ASSERT(result_url->str() == src_url_00->str()); + // When the cache is disabled, we return a nullptr---always. + // (In comparison, the EffectiveUrlCache creates an EffectiveUrl around the raw input url) + auto result_when_disabled = SignedUrlCache::TheCache()->get_signed_url(input_url); + CPPUNIT_ASSERT(result_when_disabled == nullptr); + + // When the cache is enabled, we return the cached value + // The cache is disabled in bes.conf, so we need to turn it on. + SignedUrlCache::TheCache()->d_enabled = true; + CPPUNIT_ASSERT(SignedUrlCache::TheCache()->is_enabled()); + + auto result_when_enabled = SignedUrlCache::TheCache()->get_signed_url(input_url); + CPPUNIT_ASSERT(result_when_enabled->str() == output_url->str()); } void skip_regex_test_01() { @@ -374,14 +383,32 @@ class SignedUrlCacheTest : public CppUnit::TestFixture { CPPUNIT_TEST_SUITE(SignedUrlCacheTest); - // TODO: fix up tests, enable/replace!! add more!! - // CPPUNIT_TEST(is_cache_disabled_test); - // CPPUNIT_TEST(cache_test_00); - // CPPUNIT_TEST(cache_test_01); - // CPPUNIT_TEST(skip_regex_test_01); - // CPPUNIT_TEST(euc_ghrc_tea_url_test); - // CPPUNIT_TEST(euc_harmony_url_test); - // CPPUNIT_TEST(trusted_url_test_01); + // Test behavior analogous to that of the EffectiveUrlCache: + CPPUNIT_TEST(is_cache_disabled_test); + // CPPUNIT_TEST(cache_test_00); + // CPPUNIT_TEST(cache_test_01); + // CPPUNIT_TEST(skip_regex_test_01); + // CPPUNIT_TEST(euc_ghrc_tea_url_test); + // CPPUNIT_TEST(euc_harmony_url_test); + // CPPUNIT_TEST(trusted_url_test_01); + // - set_skip_regex + // - dump + + // Test behavior specific novel to url signing: + + // - get_s3credentials_from_endpoint + // - extract_s3_credentials_from_response_json + // - retrieve_cached_s3credentials + // - are_s3credentials_expired + + // - sign_url + // - get_cached_signed_url + + // - cache_signed_url_components + // - retrieve_cached_signed_url_components + // - get_signed_url + + CPPUNIT_TEST_SUITE_END(); }; From 2e0c4b7e1b1c65b6a05e4182d12714ef7da0e44e Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Thu, 30 Oct 2025 13:49:34 -0400 Subject: [PATCH 08/77] Update test: set_skip_regex --- http/unit-tests/SignedUrlCacheTest.cc | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/http/unit-tests/SignedUrlCacheTest.cc b/http/unit-tests/SignedUrlCacheTest.cc index 312adfbe32..52e77ece6e 100644 --- a/http/unit-tests/SignedUrlCacheTest.cc +++ b/http/unit-tests/SignedUrlCacheTest.cc @@ -139,7 +139,7 @@ class SignedUrlCacheTest : public CppUnit::TestFixture { CPPUNIT_ASSERT(result_when_enabled->str() == output_url->str()); } - void skip_regex_test_01() { + void set_skip_regex_test() { DBG(cerr << prolog << "BEGIN" << endl); try { // The cache is disabled in bes.conf, so we need to turn it on. @@ -147,10 +147,19 @@ class SignedUrlCacheTest : public CppUnit::TestFixture { // This one does not add the URL or even check it because it _should_ be matching the skip regex // in the bes.conf - shared_ptr src_url(new http::url("https://foobar.com/opendap/data/nc/fnoc1.nc?dap4.ce=u;v")); + auto src_url = make_shared("https://foobar.com/opendap/data/nc/fnoc1.nc?dap4.ce=u;v"); auto result_url = SignedUrlCache::TheCache()->get_signed_url(src_url); CPPUNIT_ASSERT(SignedUrlCache::TheCache()->d_signed_urls.empty()); - CPPUNIT_ASSERT(result_url->str() == src_url->str()); + CPPUNIT_ASSERT(result_url == nullptr); + + // Similarly, skipped even when that url has been previously + // added to the cache somehow + auto output_url = make_shared("http://started_here.com?signed-now"); + SignedUrlCache::TheCache()->d_signed_urls.insert( + pair>(src_url->str(), output_url)); + CPPUNIT_ASSERT(SignedUrlCache::TheCache()->d_signed_urls.size() == 1); + auto result_url2 = SignedUrlCache::TheCache()->get_signed_url(src_url); + CPPUNIT_ASSERT(result_url2 == nullptr); } catch (const BESError &be) { stringstream msg; @@ -385,26 +394,26 @@ CPPUNIT_TEST_SUITE(SignedUrlCacheTest); // Test behavior analogous to that of the EffectiveUrlCache: CPPUNIT_TEST(is_cache_disabled_test); + CPPUNIT_TEST(set_skip_regex_test); // CPPUNIT_TEST(cache_test_00); // CPPUNIT_TEST(cache_test_01); - // CPPUNIT_TEST(skip_regex_test_01); + // CPPUNIT_TEST(euc_ghrc_tea_url_test); // CPPUNIT_TEST(euc_harmony_url_test); // CPPUNIT_TEST(trusted_url_test_01); - // - set_skip_regex // - dump // Test behavior specific novel to url signing: // - get_s3credentials_from_endpoint // - extract_s3_credentials_from_response_json - // - retrieve_cached_s3credentials + // - retrieve_cached_s3credentials // - are_s3credentials_expired // - sign_url // - get_cached_signed_url - - // - cache_signed_url_components + + // - cache_signed_url_components // - retrieve_cached_signed_url_components // - get_signed_url From 1bce3926c142cc314855259ca6198b701175ad59 Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Thu, 30 Oct 2025 14:34:31 -0400 Subject: [PATCH 09/77] Add test: dump --- http/unit-tests/SignedUrlCacheTest.cc | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/http/unit-tests/SignedUrlCacheTest.cc b/http/unit-tests/SignedUrlCacheTest.cc index 52e77ece6e..c9d039a790 100644 --- a/http/unit-tests/SignedUrlCacheTest.cc +++ b/http/unit-tests/SignedUrlCacheTest.cc @@ -168,6 +168,22 @@ class SignedUrlCacheTest : public CppUnit::TestFixture { } } + void dump_test() { + // Add values to each type of subcache + SignedUrlCache::TheCache()->d_signed_urls.insert( + pair>("www.foo.com", make_shared("http://www.bar.com"))); + + // Check to make sure dump includes them + auto strm = std::ostringstream(); + SignedUrlCache::TheCache()->dump(strm); + // Remove start of string to skip address that varies + auto result = strm.str().substr(49); + std::string expected_str = string("d_skip_regex: ") + + "\n signed url list:" + + "\n www.foo.com --> http://www.bar.com"; + CPPUNIT_ASSERT_MESSAGE("The dump should be `" + expected_str + "`; was `" + result + "`", expected_str == result); + } + void cache_test_00() { DBG(cerr << prolog << "BEGIN" << endl); //string source_url; @@ -395,13 +411,12 @@ CPPUNIT_TEST_SUITE(SignedUrlCacheTest); // Test behavior analogous to that of the EffectiveUrlCache: CPPUNIT_TEST(is_cache_disabled_test); CPPUNIT_TEST(set_skip_regex_test); + CPPUNIT_TEST(dump_test); // CPPUNIT_TEST(cache_test_00); // CPPUNIT_TEST(cache_test_01); - // CPPUNIT_TEST(euc_ghrc_tea_url_test); // CPPUNIT_TEST(euc_harmony_url_test); // CPPUNIT_TEST(trusted_url_test_01); - // - dump // Test behavior specific novel to url signing: From 06520662b3719e4585a1ea3b27e771e5d837abb0 Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Thu, 30 Oct 2025 17:48:55 -0400 Subject: [PATCH 10/77] Fill in credential expiration wip --- http/SignedUrlCache.cc | 36 +++++++++++++++++---- http/SignedUrlCache.h | 2 +- http/unit-tests/SignedUrlCacheTest.cc | 45 +++++++++++++++++++++++---- 3 files changed, 70 insertions(+), 13 deletions(-) diff --git a/http/SignedUrlCache.cc b/http/SignedUrlCache.cc index 4e2d18d56d..f92775e06a 100644 --- a/http/SignedUrlCache.cc +++ b/http/SignedUrlCache.cc @@ -67,10 +67,32 @@ shared_ptr SignedUrlCache::get_cached_signed_url(string const &ur return signed_url; } -// TODO-docstring -bool SignedUrlCache::are_s3credentials_expired(std::shared_ptr const credentials) { - // TODO: implement! - return false; +// TODO-docstring; note if timestamp isn't valid, we return that we are expired +// We intentionally copy here because we need to delete a character +// Could alternatively convert to datetime when we originally cache it.... +bool SignedUrlCache::is_timestamp_after_now(std::string ×tamp_str) { + + auto now = std::chrono::system_clock::now(); + auto now_secs = std::chrono::time_point_cast(now); + + if (timestamp_str.size() == 25) { + // Hack to handle fact that s3credentials from aws include an + // extra colon in their timezone field + // This changes "1980-07-16 18:40:58+00:00" to "1980-07-16 18:40:58+0000" + timestamp_str.erase(22, 1); + } + std::tm timestamp_time = {}; + auto time_parse_result = strptime(timestamp_str.c_str(), "%F %T%z", ×tamp_time); + if (time_parse_result == nullptr) { + // TODO: do we want to warn or anything here?? We wouldn't expect to be + // here unless we truly thought we had a valid timestamp str... + return false; + } + auto timestamp_time_point = std::chrono::system_clock::from_time_t(std::mktime(×tamp_time)); + auto timestamp_secs = std::chrono::time_point_cast(timestamp_time_point); + + // TODO: do we want to add a buffer of a second or two here? + return timestamp_secs > now_secs; } /** @@ -82,8 +104,10 @@ shared_ptr SignedUrlCache::retrieve_cached_s3c shared_ptr s3_access_key_tuple(nullptr); auto it = d_s3credentials_cache.find(url_key); if (it != d_s3credentials_cache.end()) { - // Is it expired? If so, erase it - if (are_s3credentials_expired(it->second)) { + // Is it expired? If so, erase it! + auto timestamp_str = get<3>(*(it->second)); + if (!is_timestamp_after_now(timestamp_str)) { + // Expired! d_s3credentials_cache.erase(it); } else { s3_access_key_tuple = it->second; diff --git a/http/SignedUrlCache.h b/http/SignedUrlCache.h index dc76feed79..3e3bb516f5 100644 --- a/http/SignedUrlCache.h +++ b/http/SignedUrlCache.h @@ -67,7 +67,7 @@ class SignedUrlCache : public BESObj { std::map> d_s3credentials_cache; std::shared_ptr retrieve_cached_s3credentials(std::string const &url_key); - static bool are_s3credentials_expired(std::shared_ptr const credentials); + static bool is_timestamp_after_now(std::string ×tamp); // URLs that match are not cached. std::unique_ptr d_skip_regex = nullptr; diff --git a/http/unit-tests/SignedUrlCacheTest.cc b/http/unit-tests/SignedUrlCacheTest.cc index c9d039a790..bab242f83c 100644 --- a/http/unit-tests/SignedUrlCacheTest.cc +++ b/http/unit-tests/SignedUrlCacheTest.cc @@ -184,6 +184,40 @@ class SignedUrlCacheTest : public CppUnit::TestFixture { CPPUNIT_ASSERT_MESSAGE("The dump should be `" + expected_str + "`; was `" + result + "`", expected_str == result); } + void is_timestamp_after_now_test() { + std::string str_old("1980-07-16 18:40:58+00:00"); + CPPUNIT_ASSERT_MESSAGE("Ancient timestamp is before now", !SignedUrlCache::is_timestamp_after_now(str_old)); + + std::string str_future("3035-07-16 02:20:33+00:00"); + CPPUNIT_ASSERT_MESSAGE("Future timestamp is after now", SignedUrlCache::is_timestamp_after_now(str_future)); + + std::string str_invalid("invalid timestamp woo hooray huzzah"); + CPPUNIT_ASSERT_MESSAGE("Invalid timestamp is not after now", !SignedUrlCache::is_timestamp_after_now(str_invalid)); + } + + void retrieve_cached_s3credentials_test() { + std::string key("i_am_a_key"); + auto result_not_in_cache = SignedUrlCache::TheCache()->retrieve_cached_s3credentials(key); + CPPUNIT_ASSERT_MESSAGE("Cache miss should return null", result_not_in_cache == nullptr); + + auto value = make_shared("a man", "a plan", "a canal", "3035-07-16 02:20:33+00:00"); + SignedUrlCache::TheCache()->d_s3credentials_cache.insert(pair>(key, value)); + auto result_in_cache = SignedUrlCache::TheCache()->retrieve_cached_s3credentials(key); + CPPUNIT_ASSERT_MESSAGE("Cache hit should successfully retrieve result", result_in_cache == value); + } + + void retrieve_cached_s3credentials_test_expired_credentials() { + std::string key("i_am_a_key"); + std::string expired_time("1980-07-16 18:40:58+00:00"); + auto value = make_shared("foo", "bar", "bat", expired_time); + SignedUrlCache::TheCache()->d_s3credentials_cache.insert(pair>(key, value)); + + auto result = SignedUrlCache::TheCache()->retrieve_cached_s3credentials(key); + CPPUNIT_ASSERT_MESSAGE("Cached expired result should not be retrieved", result == nullptr); + CPPUNIT_ASSERT_MESSAGE("Expired result should have been removed from cache", SignedUrlCache::TheCache()->d_s3credentials_cache.empty()); + } + +/* void cache_test_00() { DBG(cerr << prolog << "BEGIN" << endl); //string source_url; @@ -402,6 +436,7 @@ class SignedUrlCacheTest : public CppUnit::TestFixture { } DBG(cerr << prolog << "END" << endl); } +*/ /* TESTS END */ /*##################################################################################################*/ @@ -418,12 +453,12 @@ CPPUNIT_TEST_SUITE(SignedUrlCacheTest); // CPPUNIT_TEST(euc_harmony_url_test); // CPPUNIT_TEST(trusted_url_test_01); - // Test behavior specific novel to url signing: - + // Test behavior specific to url signing: // - get_s3credentials_from_endpoint // - extract_s3_credentials_from_response_json - // - retrieve_cached_s3credentials - // - are_s3credentials_expired + CPPUNIT_TEST(is_timestamp_after_now_test); + CPPUNIT_TEST(retrieve_cached_s3credentials_test); + CPPUNIT_TEST(retrieve_cached_s3credentials_test_expired_credentials); // - sign_url // - get_cached_signed_url @@ -432,8 +467,6 @@ CPPUNIT_TEST_SUITE(SignedUrlCacheTest); // - retrieve_cached_signed_url_components // - get_signed_url - - CPPUNIT_TEST_SUITE_END(); }; From 19a92400905c9e809f548d95e003c9de65377cd6 Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Fri, 31 Oct 2025 11:18:27 -0400 Subject: [PATCH 11/77] Clear all caches for unit test setup --- http/unit-tests/SignedUrlCacheTest.cc | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/http/unit-tests/SignedUrlCacheTest.cc b/http/unit-tests/SignedUrlCacheTest.cc index bab242f83c..0cfc8030f5 100644 --- a/http/unit-tests/SignedUrlCacheTest.cc +++ b/http/unit-tests/SignedUrlCacheTest.cc @@ -102,6 +102,9 @@ class SignedUrlCacheTest : public CppUnit::TestFixture { // Clear the cache for the next test. SignedUrlCache *theCache = SignedUrlCache::TheCache(); theCache->d_signed_urls.clear(); + theCache->d_href_to_s3credentials_cache.clear(); + theCache->d_href_to_s3_cache.clear(); + theCache->d_s3credentials_cache.clear(); if (!token.empty()) { DBG(cerr << "Setting BESContext " << EDL_AUTH_TOKEN_KEY << " to: '" << token << "'" << endl); @@ -206,10 +209,10 @@ class SignedUrlCacheTest : public CppUnit::TestFixture { CPPUNIT_ASSERT_MESSAGE("Cache hit should successfully retrieve result", result_in_cache == value); } - void retrieve_cached_s3credentials_test_expired_credentials() { + void retrieve_cached_s3credentials_expired_credentials_test() { std::string key("i_am_a_key"); - std::string expired_time("1980-07-16 18:40:58+00:00"); - auto value = make_shared("foo", "bar", "bat", expired_time); + std::string expiration_time("1980-07-16 18:40:58+00:00"); + auto value = make_shared("https://www.foo", "https://www.bar", "https://www.bat", expiration_time); SignedUrlCache::TheCache()->d_s3credentials_cache.insert(pair>(key, value)); auto result = SignedUrlCache::TheCache()->retrieve_cached_s3credentials(key); @@ -458,7 +461,7 @@ CPPUNIT_TEST_SUITE(SignedUrlCacheTest); // - extract_s3_credentials_from_response_json CPPUNIT_TEST(is_timestamp_after_now_test); CPPUNIT_TEST(retrieve_cached_s3credentials_test); - CPPUNIT_TEST(retrieve_cached_s3credentials_test_expired_credentials); + CPPUNIT_TEST(retrieve_cached_s3credentials_expired_credentials_test); // - sign_url // - get_cached_signed_url From ff57cd35890687a63bb57ce4c33cae6a35146de3 Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Fri, 31 Oct 2025 12:21:08 -0400 Subject: [PATCH 12/77] Add test and fix behavior: invalid response json --- http/SignedUrlCache.cc | 20 +++++++++------ http/unit-tests/SignedUrlCacheTest.cc | 36 +++++++++++++++++++++++++-- 2 files changed, 46 insertions(+), 10 deletions(-) diff --git a/http/SignedUrlCache.cc b/http/SignedUrlCache.cc index f92775e06a..50f7f65a6e 100644 --- a/http/SignedUrlCache.cc +++ b/http/SignedUrlCache.cc @@ -282,38 +282,42 @@ std::shared_ptr SignedUrlCache::extract_s3_cre rapidjson::Document s3credentials_response; s3credentials_response.Parse(s3credentials_json_string.c_str()); + if (s3credentials_response.HasParseError()) { + return nullptr; + } + string access_key_id; string secret_access_key; string session_token; string expiration; auto itr = s3credentials_response.FindMember("accessKeyId"); - if (itr != s3credentials_response.MemberEnd()) { + if (itr != s3credentials_response.MemberEnd() && itr->value.IsString()) { access_key_id = itr->value.GetString(); } itr = s3credentials_response.FindMember("secretAccessKey"); - if (itr != s3credentials_response.MemberEnd()) { + if (itr != s3credentials_response.MemberEnd() && itr->value.IsString()) { secret_access_key = itr->value.GetString(); } itr = s3credentials_response.FindMember("sessionToken"); - if (itr != s3credentials_response.MemberEnd()) { + if (itr != s3credentials_response.MemberEnd() && itr->value.IsString()) { session_token = itr->value.GetString(); } itr = s3credentials_response.FindMember("expiration"); - if (itr != s3credentials_response.MemberEnd()) { + if (itr != s3credentials_response.MemberEnd() && itr->value.IsString()) { expiration = itr->value.GetString(); } if (access_key_id.empty() || secret_access_key.empty() || session_token.empty() || expiration.empty()) { return nullptr; } - return make_shared(access_key_id, - secret_access_key, - session_token, - expiration); + return make_shared(access_key_id, + secret_access_key, + session_token, + expiration); } diff --git a/http/unit-tests/SignedUrlCacheTest.cc b/http/unit-tests/SignedUrlCacheTest.cc index 0cfc8030f5..71f6b2b890 100644 --- a/http/unit-tests/SignedUrlCacheTest.cc +++ b/http/unit-tests/SignedUrlCacheTest.cc @@ -220,6 +220,38 @@ class SignedUrlCacheTest : public CppUnit::TestFixture { CPPUNIT_ASSERT_MESSAGE("Expired result should have been removed from cache", SignedUrlCache::TheCache()->d_s3credentials_cache.empty()); } + void extract_s3_credentials_from_response_json_test() { + + std::string access_key("i_am_an_access_key_id"); + std::string valid_response( + string("{\n\"accessKeyId\": \"") + access_key + "\",\n" + + "\"secretAccessKey\": \"i_am_a_secret_access_key\",\n" + + "\"sessionToken\": \"i_am_a_fake_token\",\n" + + "\"expiration\": \"3025-09-30 18:40:58+00:00\"\n" + + "} "); + auto result = SignedUrlCache::extract_s3_credentials_from_response_json(valid_response); + CPPUNIT_ASSERT_MESSAGE("Valid json should not return nullptr", result != nullptr); + CPPUNIT_ASSERT_MESSAGE("Access key should be returned as first value", access_key == get<0>(*result)); + + CPPUNIT_ASSERT_MESSAGE("Empty string should return nullptr", SignedUrlCache::extract_s3_credentials_from_response_json("") == nullptr); + CPPUNIT_ASSERT_MESSAGE("Invalid json should return nullptr", SignedUrlCache::extract_s3_credentials_from_response_json("{foo}") == nullptr); + + std::string invalid_response( + string("{\n\"accessKeyId\": \"") + access_key + "\",\n" + + "\"secretAccessKey\": \"i_am_a_secret_access_key\",\n" + + "\"sessionToken\": \"i_am_a_fake_token\",\n" + + "} "); + CPPUNIT_ASSERT_MESSAGE("Response missing field should return nullptr", SignedUrlCache::extract_s3_credentials_from_response_json(invalid_response) == nullptr); + + std::string invalid_response_contents( + string("{\n\"accessKeyId\": \"") + access_key + "\",\n" + + "\"secretAccessKey\": [3, 4, 5],\n" + // Oh no! An array instead of a string!! Horrors! + "\"sessionToken\": \"i_am_a_fake_token\",\n" + + "\"expiration\": \"3025-09-30 18:40:58+00:00\"\n" + + "} "); + CPPUNIT_ASSERT_MESSAGE("Field with non-string response should return nullptr", SignedUrlCache::extract_s3_credentials_from_response_json(invalid_response_contents) == nullptr); + } + /* void cache_test_00() { DBG(cerr << prolog << "BEGIN" << endl); @@ -457,11 +489,11 @@ CPPUNIT_TEST_SUITE(SignedUrlCacheTest); // CPPUNIT_TEST(trusted_url_test_01); // Test behavior specific to url signing: - // - get_s3credentials_from_endpoint - // - extract_s3_credentials_from_response_json CPPUNIT_TEST(is_timestamp_after_now_test); CPPUNIT_TEST(retrieve_cached_s3credentials_test); CPPUNIT_TEST(retrieve_cached_s3credentials_expired_credentials_test); + CPPUNIT_TEST(extract_s3_credentials_from_response_json_test); + // - get_s3credentials_from_endpoint // - sign_url // - get_cached_signed_url From 2466331484c8db472ce03bbb4263c55fc3ca8761 Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Fri, 31 Oct 2025 12:58:02 -0400 Subject: [PATCH 13/77] Test non-http behavior --- http/unit-tests/SignedUrlCacheTest.cc | 51 +++++++++++++++++++-------- 1 file changed, 37 insertions(+), 14 deletions(-) diff --git a/http/unit-tests/SignedUrlCacheTest.cc b/http/unit-tests/SignedUrlCacheTest.cc index 71f6b2b890..718952726e 100644 --- a/http/unit-tests/SignedUrlCacheTest.cc +++ b/http/unit-tests/SignedUrlCacheTest.cc @@ -99,8 +99,12 @@ class SignedUrlCacheTest : public CppUnit::TestFixture { if (bes_debug) BESDebug::SetUp("cerr,bes,euc,http,curl"); - // Clear the cache for the next test. + // Reset to same starting point every time + // (It's a singleton so resetting it is important for test determinism) SignedUrlCache *theCache = SignedUrlCache::TheCache(); + theCache->d_enabled = -1; + + // ...and clear the caches theCache->d_signed_urls.clear(); theCache->d_href_to_s3credentials_cache.clear(); theCache->d_href_to_s3_cache.clear(); @@ -116,6 +120,25 @@ class SignedUrlCacheTest : public CppUnit::TestFixture { /*##################################################################################################*/ /* TESTS BEGIN */ + void get_cached_signed_url_test() { + // The cache is disabled in bes.conf, so we need to turn it on. + SignedUrlCache::TheCache()->d_enabled = true; + + auto input_url = make_shared("http://started_here.com"); + auto output_url = make_shared("http://started_here.com?signed-now"); + + SignedUrlCache::TheCache()->d_signed_urls.insert( + pair>(input_url->str(), output_url)); + auto result = SignedUrlCache::TheCache()->get_signed_url(input_url); + CPPUNIT_ASSERT(result->str() == output_url->str()); + + std::string non_http_key("foo"); + SignedUrlCache::TheCache()->d_signed_urls.insert( + pair>(non_http_key, output_url)); + auto result2 = SignedUrlCache::TheCache()->get_signed_url(make_shared(non_http_key)); + CPPUNIT_ASSERT_MESSAGE("Non-url key returns nullptr", result2 == nullptr); + } + void is_cache_disabled_test() { DBG(cerr << prolog << "SignedUrlCache::TheCache()->is_enabled(): " << (SignedUrlCache::TheCache()->is_enabled() ? "true" : "false") << endl); @@ -125,7 +148,7 @@ class SignedUrlCacheTest : public CppUnit::TestFixture { auto output_url = make_shared("http://started_here.com?signed-now"); SignedUrlCache::TheCache()->d_signed_urls.insert( - pair>(input_url->str(), output_url)); + pair>(input_url->str(), output_url)); CPPUNIT_ASSERT(SignedUrlCache::TheCache()->d_signed_urls.size() == 1); // When the cache is disabled, we return a nullptr---always. @@ -133,8 +156,7 @@ class SignedUrlCacheTest : public CppUnit::TestFixture { auto result_when_disabled = SignedUrlCache::TheCache()->get_signed_url(input_url); CPPUNIT_ASSERT(result_when_disabled == nullptr); - // When the cache is enabled, we return the cached value - // The cache is disabled in bes.conf, so we need to turn it on. + // ...if we now enable the cache is enabled, we return the previously cached value SignedUrlCache::TheCache()->d_enabled = true; CPPUNIT_ASSERT(SignedUrlCache::TheCache()->is_enabled()); @@ -479,28 +501,29 @@ class SignedUrlCacheTest : public CppUnit::TestFixture { CPPUNIT_TEST_SUITE(SignedUrlCacheTest); // Test behavior analogous to that of the EffectiveUrlCache: + CPPUNIT_TEST(get_cached_signed_url_test); CPPUNIT_TEST(is_cache_disabled_test); CPPUNIT_TEST(set_skip_regex_test); CPPUNIT_TEST(dump_test); - // CPPUNIT_TEST(cache_test_00); - // CPPUNIT_TEST(cache_test_01); - // CPPUNIT_TEST(euc_ghrc_tea_url_test); - // CPPUNIT_TEST(euc_harmony_url_test); - // CPPUNIT_TEST(trusted_url_test_01); - // Test behavior specific to url signing: + // Test behavior specific to SignedUrlCache: CPPUNIT_TEST(is_timestamp_after_now_test); CPPUNIT_TEST(retrieve_cached_s3credentials_test); CPPUNIT_TEST(retrieve_cached_s3credentials_expired_credentials_test); CPPUNIT_TEST(extract_s3_credentials_from_response_json_test); // - get_s3credentials_from_endpoint - - // - sign_url - // - get_cached_signed_url - // - cache_signed_url_components // - retrieve_cached_signed_url_components + + // ...and, specifically, the signing itself: + // TODO-future: will add/update these tests once signing behavior is implemented! + // - sign_url // - get_signed_url + // CPPUNIT_TEST(cache_test_00); + // CPPUNIT_TEST(cache_test_01); + // CPPUNIT_TEST(euc_ghrc_tea_url_test); + // CPPUNIT_TEST(euc_harmony_url_test); + // CPPUNIT_TEST(trusted_url_test_01); CPPUNIT_TEST_SUITE_END(); }; From b384fd9415d321b4251cba8be581cab64a58ce0b Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Fri, 31 Oct 2025 13:08:56 -0400 Subject: [PATCH 14/77] Add test: cache signed url components --- http/SignedUrlCache.cc | 4 ++-- http/unit-tests/SignedUrlCacheTest.cc | 20 +++++++++++++++++++- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/http/SignedUrlCache.cc b/http/SignedUrlCache.cc index 50f7f65a6e..cc6ed81ae8 100644 --- a/http/SignedUrlCache.cc +++ b/http/SignedUrlCache.cc @@ -228,8 +228,8 @@ shared_ptr SignedUrlCache::get_signed_url(shared_ptr source // TODO: docstring void SignedUrlCache::cache_signed_url_components(const std::string &key_href_url, const std::string &s3_url, const std::string &s3credentials_url) { - if (s3_url.empty() || s3credentials_url.empty() ) { - // Don't cache either if one is empty. + if (key_href_url.empty() || s3_url.empty() || s3credentials_url.empty() ) { + // Don't cache either if any is empty. return; } d_href_to_s3_cache[key_href_url] = s3_url; diff --git a/http/unit-tests/SignedUrlCacheTest.cc b/http/unit-tests/SignedUrlCacheTest.cc index 718952726e..0c6c499fbd 100644 --- a/http/unit-tests/SignedUrlCacheTest.cc +++ b/http/unit-tests/SignedUrlCacheTest.cc @@ -274,6 +274,24 @@ class SignedUrlCacheTest : public CppUnit::TestFixture { CPPUNIT_ASSERT_MESSAGE("Field with non-string response should return nullptr", SignedUrlCache::extract_s3_credentials_from_response_json(invalid_response_contents) == nullptr); } + void cache_signed_url_components_test() { + + SignedUrlCache *theCache = SignedUrlCache::TheCache(); + + theCache->cache_signed_url_components("", "foo", "bar"); + CPPUNIT_ASSERT_MESSAGE("Empty key_href_url results in no caching", theCache->d_href_to_s3_cache.empty() && theCache->d_href_to_s3credentials_cache.empty()); + + theCache->cache_signed_url_components("foo", "", "bar"); + CPPUNIT_ASSERT_MESSAGE("Empty s3_url results in no caching", theCache->d_href_to_s3_cache.empty() && theCache->d_href_to_s3credentials_cache.empty()); + + theCache->cache_signed_url_components("foo", "bar", ""); + CPPUNIT_ASSERT_MESSAGE("Empty s3credentials_url results in no caching", theCache->d_href_to_s3_cache.empty() && theCache->d_href_to_s3credentials_cache.empty()); + + theCache->cache_signed_url_components("foo", "bar", "bat"); + CPPUNIT_ASSERT_MESSAGE("s3_url should be cached", theCache->d_href_to_s3_cache["foo"] == "bar"); + CPPUNIT_ASSERT_MESSAGE("s3credentials_url should be cached", theCache->d_href_to_s3credentials_cache["foo"] == "bat"); + } + /* void cache_test_00() { DBG(cerr << prolog << "BEGIN" << endl); @@ -511,8 +529,8 @@ CPPUNIT_TEST_SUITE(SignedUrlCacheTest); CPPUNIT_TEST(retrieve_cached_s3credentials_test); CPPUNIT_TEST(retrieve_cached_s3credentials_expired_credentials_test); CPPUNIT_TEST(extract_s3_credentials_from_response_json_test); + CPPUNIT_TEST(cache_signed_url_components_test); // - get_s3credentials_from_endpoint - // - cache_signed_url_components // - retrieve_cached_signed_url_components // ...and, specifically, the signing itself: From 7a3e8a30a0bcf0d6e81dad303b27e9c9fcd59ac2 Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Fri, 31 Oct 2025 13:28:36 -0400 Subject: [PATCH 15/77] Add test: retrieved signed url components --- http/SignedUrlCache.cc | 2 +- http/unit-tests/SignedUrlCacheTest.cc | 18 ++++++++++++++++-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/http/SignedUrlCache.cc b/http/SignedUrlCache.cc index cc6ed81ae8..b08770bfb1 100644 --- a/http/SignedUrlCache.cc +++ b/http/SignedUrlCache.cc @@ -240,7 +240,7 @@ void SignedUrlCache::cache_signed_url_components(const std::string &key_href_url std::pair SignedUrlCache::retrieve_cached_signed_url_components(const std::string &key_href_url) const { auto it_s3_url = d_href_to_s3_cache.find(key_href_url); auto it_s3credentials_url = d_href_to_s3credentials_cache.find(key_href_url); - if (it_s3_url != d_href_to_s3_cache.end() || it_s3credentials_url != d_href_to_s3credentials_cache.end() ) { + if (it_s3_url == d_href_to_s3_cache.end() || it_s3credentials_url == d_href_to_s3credentials_cache.end() ) { INFO_LOG(prolog + "No url available for TEA s3credentials endpoint."); return std::pair("", ""); } diff --git a/http/unit-tests/SignedUrlCacheTest.cc b/http/unit-tests/SignedUrlCacheTest.cc index 0c6c499fbd..fbb51fd840 100644 --- a/http/unit-tests/SignedUrlCacheTest.cc +++ b/http/unit-tests/SignedUrlCacheTest.cc @@ -275,7 +275,6 @@ class SignedUrlCacheTest : public CppUnit::TestFixture { } void cache_signed_url_components_test() { - SignedUrlCache *theCache = SignedUrlCache::TheCache(); theCache->cache_signed_url_components("", "foo", "bar"); @@ -292,6 +291,21 @@ class SignedUrlCacheTest : public CppUnit::TestFixture { CPPUNIT_ASSERT_MESSAGE("s3credentials_url should be cached", theCache->d_href_to_s3credentials_cache["foo"] == "bat"); } + void retrieve_cached_signed_url_components_test() { + SignedUrlCache *theCache = SignedUrlCache::TheCache(); + + theCache->cache_signed_url_components("two fish", "red fish", "blue fish"); + auto retrieved = theCache->retrieve_cached_signed_url_components("two fish"); + auto expected = std::pair("red fish", "blue fish"); + CPPUNIT_ASSERT_MESSAGE("Cached components should be retrieved", expected == retrieved); + + std::string key("little_bo_peep"); + theCache->d_href_to_s3_cache.insert(pair(key, "goat")); + auto retrieved2 = theCache->retrieve_cached_signed_url_components(key); + auto empty_pair = std::pair("", ""); + CPPUNIT_ASSERT_MESSAGE("If both urls were not cached, no response is returned", retrieved2 == empty_pair); + } + /* void cache_test_00() { DBG(cerr << prolog << "BEGIN" << endl); @@ -530,8 +544,8 @@ CPPUNIT_TEST_SUITE(SignedUrlCacheTest); CPPUNIT_TEST(retrieve_cached_s3credentials_expired_credentials_test); CPPUNIT_TEST(extract_s3_credentials_from_response_json_test); CPPUNIT_TEST(cache_signed_url_components_test); + CPPUNIT_TEST(retrieve_cached_signed_url_components_test); // - get_s3credentials_from_endpoint - // - retrieve_cached_signed_url_components // ...and, specifically, the signing itself: // TODO-future: will add/update these tests once signing behavior is implemented! From 0142bededd386cab8d3acdaffe73028cdd80d816 Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Fri, 31 Oct 2025 13:33:10 -0400 Subject: [PATCH 16/77] Add messages to unit test asserts --- http/unit-tests/SignedUrlCacheTest.cc | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/http/unit-tests/SignedUrlCacheTest.cc b/http/unit-tests/SignedUrlCacheTest.cc index fbb51fd840..3387fa81bf 100644 --- a/http/unit-tests/SignedUrlCacheTest.cc +++ b/http/unit-tests/SignedUrlCacheTest.cc @@ -130,7 +130,7 @@ class SignedUrlCacheTest : public CppUnit::TestFixture { SignedUrlCache::TheCache()->d_signed_urls.insert( pair>(input_url->str(), output_url)); auto result = SignedUrlCache::TheCache()->get_signed_url(input_url); - CPPUNIT_ASSERT(result->str() == output_url->str()); + CPPUNIT_ASSERT_MESSAGE("Cached url should be retrievable", result->str() == output_url->str()); std::string non_http_key("foo"); SignedUrlCache::TheCache()->d_signed_urls.insert( @@ -142,26 +142,26 @@ class SignedUrlCacheTest : public CppUnit::TestFixture { void is_cache_disabled_test() { DBG(cerr << prolog << "SignedUrlCache::TheCache()->is_enabled(): " << (SignedUrlCache::TheCache()->is_enabled() ? "true" : "false") << endl); - CPPUNIT_ASSERT(!SignedUrlCache::TheCache()->is_enabled()); + CPPUNIT_ASSERT_MESSAGE("Cache is disabled", !SignedUrlCache::TheCache()->is_enabled()); auto input_url = make_shared("http://started_here.com"); auto output_url = make_shared("http://started_here.com?signed-now"); SignedUrlCache::TheCache()->d_signed_urls.insert( pair>(input_url->str(), output_url)); - CPPUNIT_ASSERT(SignedUrlCache::TheCache()->d_signed_urls.size() == 1); + CPPUNIT_ASSERT_MESSAGE("Cache contains single item", SignedUrlCache::TheCache()->d_signed_urls.size() == 1); // When the cache is disabled, we return a nullptr---always. // (In comparison, the EffectiveUrlCache creates an EffectiveUrl around the raw input url) auto result_when_disabled = SignedUrlCache::TheCache()->get_signed_url(input_url); - CPPUNIT_ASSERT(result_when_disabled == nullptr); + CPPUNIT_ASSERT_MESSAGE("When cache is disabled, nullptr is returned", result_when_disabled == nullptr); // ...if we now enable the cache is enabled, we return the previously cached value SignedUrlCache::TheCache()->d_enabled = true; - CPPUNIT_ASSERT(SignedUrlCache::TheCache()->is_enabled()); + CPPUNIT_ASSERT_MESSAGE("Cache is enabled", SignedUrlCache::TheCache()->is_enabled()); auto result_when_enabled = SignedUrlCache::TheCache()->get_signed_url(input_url); - CPPUNIT_ASSERT(result_when_enabled->str() == output_url->str()); + CPPUNIT_ASSERT_MESSAGE("When cache is re-enabled, value is returned", result_when_enabled->str() == output_url->str()); } void set_skip_regex_test() { @@ -174,17 +174,16 @@ class SignedUrlCacheTest : public CppUnit::TestFixture { // in the bes.conf auto src_url = make_shared("https://foobar.com/opendap/data/nc/fnoc1.nc?dap4.ce=u;v"); auto result_url = SignedUrlCache::TheCache()->get_signed_url(src_url); - CPPUNIT_ASSERT(SignedUrlCache::TheCache()->d_signed_urls.empty()); - CPPUNIT_ASSERT(result_url == nullptr); + CPPUNIT_ASSERT_MESSAGE("When key matches skip regex, value is not cached", SignedUrlCache::TheCache()->d_signed_urls.empty()); + CPPUNIT_ASSERT_MESSAGE("When key matches skip regex, nullptr is returned", result_url == nullptr); // Similarly, skipped even when that url has been previously // added to the cache somehow auto output_url = make_shared("http://started_here.com?signed-now"); SignedUrlCache::TheCache()->d_signed_urls.insert( pair>(src_url->str(), output_url)); - CPPUNIT_ASSERT(SignedUrlCache::TheCache()->d_signed_urls.size() == 1); auto result_url2 = SignedUrlCache::TheCache()->get_signed_url(src_url); - CPPUNIT_ASSERT(result_url2 == nullptr); + CPPUNIT_ASSERT_MESSAGE("When key matches skip regex, even if it exists in the cache, the value is not returned", result_url2 == nullptr); } catch (const BESError &be) { stringstream msg; From 4792441a3d534948173d5a3161db96d5e0cfccaf Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Fri, 31 Oct 2025 14:15:36 -0400 Subject: [PATCH 17/77] add test for gets3credentials from endpoing --- http/SignedUrlCache.h | 5 ++--- http/unit-tests/SignedUrlCacheTest.cc | 19 ++++++++++++++++++- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/http/SignedUrlCache.h b/http/SignedUrlCache.h index 3e3bb516f5..746ebc097a 100644 --- a/http/SignedUrlCache.h +++ b/http/SignedUrlCache.h @@ -74,9 +74,8 @@ class SignedUrlCache : public BESObj { int d_enabled = -1; - // TODO: for all helper functions here, think carefully about static/const/etc - // TODO: think a little more about splitting functionality to invert control to the caller - std::shared_ptr sign_url(std::string const &s3_url, std::shared_ptr const s3_access_key_tuple); + std::shared_ptr sign_url(std::string const &s3_url, + std::shared_ptr const s3_access_key_tuple); std::shared_ptr get_cached_signed_url(std::string const &url_key); void set_skip_regex(); diff --git a/http/unit-tests/SignedUrlCacheTest.cc b/http/unit-tests/SignedUrlCacheTest.cc index 3387fa81bf..b5f6f526ea 100644 --- a/http/unit-tests/SignedUrlCacheTest.cc +++ b/http/unit-tests/SignedUrlCacheTest.cc @@ -305,6 +305,23 @@ class SignedUrlCacheTest : public CppUnit::TestFixture { CPPUNIT_ASSERT_MESSAGE("If both urls were not cached, no response is returned", retrieved2 == empty_pair); } + void get_s3credentials_from_endpoint_test() { + SignedUrlCache *theCache = SignedUrlCache::TheCache(); + + CPPUNIT_ASSERT_MESSAGE("Credentials cache is empty", theCache->d_s3credentials_cache.empty()); + + auto result_bad = theCache->get_s3credentials_from_endpoint("http://badurl"); + CPPUNIT_ASSERT_MESSAGE("After attempting fetch from invalid url, credentials cache is still empty", theCache->d_s3credentials_cache.empty()); + CPPUNIT_ASSERT_MESSAGE("Fetch from invalid url returns nullptr", result_bad == nullptr); + + // TODO: think about whether it's worth mocking out a test url with fake credentials... + // auto result_good = theCache->get_s3credentials_from_endpoint("http://badurl"); + // CPPUNIT_ASSERT_MESSAGE("Fetch from valid url returns credentials", result_good != nullptr); + + // auto result_good_from_cache = theCache->retrieve_cached_signed_url_components(key); + // CPPUNIT_ASSERT_MESSAGE("After attempting fetch from valid url, credentials cache contains the result", result_good == result_good_from_cache); + } + /* void cache_test_00() { DBG(cerr << prolog << "BEGIN" << endl); @@ -544,7 +561,7 @@ CPPUNIT_TEST_SUITE(SignedUrlCacheTest); CPPUNIT_TEST(extract_s3_credentials_from_response_json_test); CPPUNIT_TEST(cache_signed_url_components_test); CPPUNIT_TEST(retrieve_cached_signed_url_components_test); - // - get_s3credentials_from_endpoint + CPPUNIT_TEST(get_s3credentials_from_endpoint_test); // ...and, specifically, the signing itself: // TODO-future: will add/update these tests once signing behavior is implemented! From 53e288cf422884937877bd9cc8036b2b7996ff42 Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Fri, 31 Oct 2025 14:17:53 -0400 Subject: [PATCH 18/77] remove stale tests --- http/unit-tests/SignedUrlCacheTest.cc | 226 -------------------------- 1 file changed, 226 deletions(-) diff --git a/http/unit-tests/SignedUrlCacheTest.cc b/http/unit-tests/SignedUrlCacheTest.cc index b5f6f526ea..10be69a3a3 100644 --- a/http/unit-tests/SignedUrlCacheTest.cc +++ b/http/unit-tests/SignedUrlCacheTest.cc @@ -322,227 +322,6 @@ class SignedUrlCacheTest : public CppUnit::TestFixture { // CPPUNIT_ASSERT_MESSAGE("After attempting fetch from valid url, credentials cache contains the result", result_good == result_good_from_cache); } -/* - void cache_test_00() { - DBG(cerr << prolog << "BEGIN" << endl); - //string source_url; - //string value; - try { - // The cache is disabled in bes.conf, so we need to turn it on. - SignedUrlCache::TheCache()->d_enabled = true; - -#if 0 - string src_url_00 = "https://d1jecqxxv88lkr.cloudfront.net/ghrcwuat-protected/rss_demo/rssmif16d__7/f1" - "6_ssmis_20040107v7.nc"; - string eurl_str = "https://ghrcwuat-protected.s3.us-west-2.amazonaws.com/rss_demo/rssmif16d__7/f16_ssm" - "is_20031229v7.nc?A-userid=hyrax&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=A" - "SIASF4N-AWS-Creds-00808%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Date=20200808T032623Z" - "&X-Amz-Expires=86400&X-Amz-Security-Token=FwoGZXIvYXdzE-AWS-Sec-Token-MWRLIZGYvDx1O" - "Nzd0ffK8VtxO8JP7thrGIQ%3D%3D&X-Amz-SignedHeaders=host&X-Amz-Signature=260a7c4dd4-AW" - "S-SIGGY-0c7a39ee899"; - auto effective_url_00 = shared_ptr(new http::EffectiveUrl(eurl_str)); - SignedUrlCache::TheCache()->d_signed_urls.insert(pair>(src_url_00, effective_url_00)); - CPPUNIT_ASSERT(SignedUrlCache::TheCache()->d_signed_urls.size() == 1); -#endif - - string src_url_01 = "http://test.opendap.org/data/httpd_catalog/READTHIS"; - string eurl_str = "https://test.opendap.org/data/httpd_catalog/READTHIS"; - auto effective_url_01 = shared_ptr(new http::EffectiveUrl(eurl_str)); - SignedUrlCache::TheCache()->d_signed_urls.insert( - pair>(src_url_01, effective_url_01)); - CPPUNIT_ASSERT(SignedUrlCache::TheCache()->d_signed_urls.size() == 1); - - // This one actually does the thing - eurl_str = "http://test.opendap.org/opendap/"; - auto expected_url_02 = std::unique_ptr(new http::EffectiveUrl(eurl_str)); - - shared_ptr src_url_02(new http::url("http://test.opendap.org/opendap")); - DBG(cerr << prolog << "Retrieving effective URL for: " << src_url_02->str() << endl); - auto result_url = SignedUrlCache::TheCache()->get_signed_url(src_url_02); - CPPUNIT_ASSERT(SignedUrlCache::TheCache()->d_signed_urls.size() == 2); - - DBG(cerr << prolog << "SignedUrlCache::TheCache()->get_signed_url() returned: " << result_url - << endl); - CPPUNIT_ASSERT(result_url->str() == expected_url_02->str()); - } - catch (const BESError &be) { - stringstream msg; - msg << prolog << "ERROR! Caught BESError. Message: " << be.get_message() << endl; - CPPUNIT_FAIL(msg.str()); - - } - DBG(cerr << prolog << "END" << endl); - } - - - void cache_test_01() { - DBG(cerr << prolog << "BEGIN" << endl); - string source_url; - string value; - string result_url; - try { - // The cache is disabled in bes.conf so we need to turn it on. - SignedUrlCache::TheCache()->d_enabled = true; - - std::map d_signed_urls; - source_url = "http://someURL"; - - http::EffectiveUrl first_eu("http://someOtherUrl"); - d_signed_urls[source_url] = &first_eu; - DBG(cerr << prolog << "source_url: " << source_url << endl); - DBG(cerr << prolog << "first_eu: " << first_eu.str() << endl); - - CPPUNIT_ASSERT(d_signed_urls[source_url] == &first_eu); - - http::EffectiveUrl second_eu("http://someMoreUrlLovin"); - d_signed_urls[source_url] = &second_eu; - DBG(cerr << prolog << "source_url: " << source_url << endl); - DBG(cerr << prolog << "second_eu: " << second_eu.str() << endl); - - CPPUNIT_ASSERT(d_signed_urls[source_url] == &second_eu); - } - catch (const BESError &be) { - stringstream msg; - msg << prolog << "ERROR! Caught BESError. Message: " << be.get_message() << endl; - CPPUNIT_FAIL(msg.str()); - - } - DBG(cerr << prolog << "END" << endl); - } - - void euc_ghrc_tea_url_test() { - if (!ngap_tests) { - DBG(cerr << prolog << "SKIPPING." << endl); - return; - } - DBG(cerr << prolog << "BEGIN" << endl); - string source_url; - string value; - try { - // The cache is disabled in bes.conf so we need to turn it on. - SignedUrlCache::TheCache()->d_enabled = true; - - shared_ptr thing1(new http::url( - "https://d1jecqxxv88lkr.cloudfront.net/ghrcwuat-protected/rss_demo/rssmif16d__7/f16_ssmis_20031026v7.nc")); - string thing1_out_of_region_effective_url_prefix = "https://d1jecqxxv88lkr.cloudfront.net/s3"; - string thing1_in_region_effective_url_prefix = "https://ghrcwuat-protected.s3.us-west-2.amazonaws.com/"; - - DBG(cerr << prolog << "Retrieving signed URL for: " << thing1->str() << endl); - auto result_url = SignedUrlCache::TheCache()->get_signed_url(thing1); - CPPUNIT_ASSERT(SignedUrlCache::TheCache()->d_signed_urls.size() == 1); - - DBG(cerr << prolog << "SignedUrlCache::TheCache()->get_signed_url() returned: " << result_url - << endl); - CPPUNIT_ASSERT(result_url->str().rfind(thing1_in_region_effective_url_prefix, 0) == 0 - || result_url->str().rfind(thing1_out_of_region_effective_url_prefix, 0) == 0); - - // result_url = SignedUrlCache::TheCache()->get_signed_url(thing1); - } - catch (const BESError &be) { - stringstream msg; - msg << prolog << "ERROR! Caught BESError. Message: " << be.get_message() << endl; - CPPUNIT_FAIL(msg.str()); - - } - DBG(cerr << prolog << "END" << endl); - } - - void euc_harmony_url_test() { - if (!ngap_tests) { - DBG(cerr << prolog << "SKIPPING." << endl); - return; - } - DBG(cerr << prolog << "BEGIN" << endl); - string source_url; - string value; - try { - // The cache is disabled in bes.conf, so we need to turn it on. - SignedUrlCache::TheCache()->d_enabled = true; - - shared_ptr thing1( - new http::url("https://harmony.uat.earthdata.nasa.gov/service-results/harmony-uat-staging/public/" - "sds/staged/ATL03_20200714235814_03000802_003_01.h5")); - string thing1_out_of_region_effective_url_prefix = "https://djpip0737hawz.cloudfront.net/s3"; - string thing1_in_region_effective_url_prefix = "https://harmony-uat-staging.s3.us-west-2.amazonaws.com/public/"; - - DBG(cerr << prolog << "Retrieving signed URL for: " << thing1->str() << endl); - auto result_url = SignedUrlCache::TheCache()->get_signed_url(thing1); - CPPUNIT_ASSERT(SignedUrlCache::TheCache()->d_signed_urls.size() == 1); - - DBG(cerr << prolog << "SignedUrlCache::TheCache()->get_signed_url() returned: " << result_url - << endl); - - CPPUNIT_ASSERT(result_url->str().rfind(thing1_in_region_effective_url_prefix, 0) == 0 - || result_url->str().rfind(thing1_out_of_region_effective_url_prefix, 0) == 0); - - // TODO ??? result_url = SignedUrlCache::TheCache()->get_signed_url(thing1); - } - catch (const BESError &be) { - stringstream msg; - msg << prolog << "ERROR! Caught BESError. Message: " << be.get_message() << endl; - CPPUNIT_FAIL(msg.str()); - - } - DBG(cerr << prolog << "END" << endl); - } - - void trusted_url_test_01() { - DBG(cerr << prolog << "BEGIN" << endl); - string url_str = "http://test.opendap.org/data/nothing_is_here.html"; - string result_url_str = "http://test.opendap.org/data/httpd_catalog/READTHIS"; - shared_ptr trusted_src_url(new http::url(url_str, true)); - shared_ptr untrusted_src_url(new http::url(url_str, false)); - - DBG(cerr << prolog << "Retrieving signed URL for: " << trusted_src_url->str() << endl); - try { - // The cache is disabled in bes.conf so we need to turn it on. - SignedUrlCache::TheCache()->d_enabled = true; - - shared_ptr result_url; - - result_url = SignedUrlCache::TheCache()->get_signed_url(untrusted_src_url); - DBG(cerr << prolog << "source_url: " << untrusted_src_url->str() << " is " - << (untrusted_src_url->is_trusted() ? "" : "NOT ") << "trusted." << endl); - DBG(cerr << prolog << "result_url: " << result_url->str() << " is " - << (result_url->is_trusted() ? "" : "NOT ") << "trusted." << endl); - DBG(cerr << prolog << "SignedUrlCache::TheCache()->d_signed_urls.size(): " - << SignedUrlCache::TheCache()->d_signed_urls.size() << endl); - CPPUNIT_ASSERT(SignedUrlCache::TheCache()->d_signed_urls.size() == 1); - CPPUNIT_ASSERT(result_url->str() == result_url_str); - CPPUNIT_ASSERT(!result_url->is_trusted()); - - result_url = SignedUrlCache::TheCache()->get_signed_url(trusted_src_url); - DBG(cerr << prolog << "source_url: " << trusted_src_url->str() << " is " - << (trusted_src_url->is_trusted() ? "" : "NOT ") << "trusted." << endl); - DBG(cerr << prolog << "result_url: " << result_url->str() << " is " - << (result_url->is_trusted() ? "" : "NOT ") << "trusted." << endl); - DBG(cerr << prolog << "SignedUrlCache::TheCache()->d_signed_urls.size(): " - << SignedUrlCache::TheCache()->d_signed_urls.size() << endl); - CPPUNIT_ASSERT(SignedUrlCache::TheCache()->d_signed_urls.size() == 1); - CPPUNIT_ASSERT(result_url->str() == result_url_str); - CPPUNIT_ASSERT(result_url->is_trusted()); - - result_url = SignedUrlCache::TheCache()->get_signed_url(untrusted_src_url); - DBG(cerr << prolog << "source_url: " << untrusted_src_url->str() << " is " - << (untrusted_src_url->is_trusted() ? "" : "NOT ") << "trusted." << endl); - DBG(cerr << prolog << "result_url: " << result_url->str() << " is " - << (result_url->is_trusted() ? "" : "NOT ") << "trusted." << endl); - DBG(cerr << prolog << "SignedUrlCache::TheCache()->d_signed_urls.size(): " - << SignedUrlCache::TheCache()->d_signed_urls.size() << endl); - CPPUNIT_ASSERT(SignedUrlCache::TheCache()->d_signed_urls.size() == 1); - CPPUNIT_ASSERT(result_url->str() == result_url_str); - CPPUNIT_ASSERT(!result_url->is_trusted()); - - } - catch (const BESError &be) { - stringstream msg; - msg << prolog << "ERROR! Caught BESError. Message: " << be.get_message() << endl; - CPPUNIT_FAIL(msg.str()); - } - DBG(cerr << prolog << "END" << endl); - } -*/ - /* TESTS END */ /*##################################################################################################*/ @@ -567,11 +346,6 @@ CPPUNIT_TEST_SUITE(SignedUrlCacheTest); // TODO-future: will add/update these tests once signing behavior is implemented! // - sign_url // - get_signed_url - // CPPUNIT_TEST(cache_test_00); - // CPPUNIT_TEST(cache_test_01); - // CPPUNIT_TEST(euc_ghrc_tea_url_test); - // CPPUNIT_TEST(euc_harmony_url_test); - // CPPUNIT_TEST(trusted_url_test_01); CPPUNIT_TEST_SUITE_END(); }; From 1df848559690bcc4f86e49f83fd46ee88cbc16bd Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Fri, 31 Oct 2025 14:33:45 -0400 Subject: [PATCH 19/77] Clean up --- http/SignedUrlCache.cc | 5 +++-- http/SignedUrlCache.h | 5 +++-- http/unit-tests/SignedUrlCacheTest.cc | 16 +++++++--------- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/http/SignedUrlCache.cc b/http/SignedUrlCache.cc index b08770bfb1..ed691d07e2 100644 --- a/http/SignedUrlCache.cc +++ b/http/SignedUrlCache.cc @@ -2,8 +2,8 @@ // This file is part of the BES http package, part of the Hyrax data server. -// Copyright (c) 2020 OPeNDAP, Inc. -// Author: Nathan Potter +// Copyright (c) 2025 OPeNDAP, Inc. +// Authors: Nathan Potter , Hannah Robertson // // This library is free software; you can redistribute it and/or // modify it under the terms of the GNU Lesser General Public @@ -23,6 +23,7 @@ // Authors: // ndp Nathan Potter +// Hannah Robertson #include "config.h" diff --git a/http/SignedUrlCache.h b/http/SignedUrlCache.h index 746ebc097a..cdd2bee50a 100644 --- a/http/SignedUrlCache.h +++ b/http/SignedUrlCache.h @@ -2,8 +2,8 @@ // This file is part of the BES http package, part of the Hyrax data server. -// Copyright (c) 2020 OPeNDAP, Inc. -// Author: Nathan Potter +// Copyright (c) 2025 OPeNDAP, Inc. +// Authors: Nathan Potter , Hannah Robertson // // This library is free software; you can redistribute it and/or // modify it under the terms of the GNU Lesser General Public @@ -23,6 +23,7 @@ // Authors: // ndp Nathan Potter +// Hannah Robertson #ifndef _bes_http_SignedUrlCache_h_ #define _bes_http_SignedUrlCache_h_ 1 diff --git a/http/unit-tests/SignedUrlCacheTest.cc b/http/unit-tests/SignedUrlCacheTest.cc index 10be69a3a3..0c1db4ccfb 100644 --- a/http/unit-tests/SignedUrlCacheTest.cc +++ b/http/unit-tests/SignedUrlCacheTest.cc @@ -2,8 +2,8 @@ // This file is part of the BES component of the Hyrax Data Server. -// Copyright (c) 2018 OPeNDAP, Inc. -// Author: Nathan Potter +// Copyright (c) 2025 OPeNDAP, Inc. +// Authors: Nathan Potter , Hannah Robertson // // This library is free software; you can redistribute it and/or // modify it under the terms of the GNU Lesser General Public @@ -314,12 +314,10 @@ class SignedUrlCacheTest : public CppUnit::TestFixture { CPPUNIT_ASSERT_MESSAGE("After attempting fetch from invalid url, credentials cache is still empty", theCache->d_s3credentials_cache.empty()); CPPUNIT_ASSERT_MESSAGE("Fetch from invalid url returns nullptr", result_bad == nullptr); - // TODO: think about whether it's worth mocking out a test url with fake credentials... - // auto result_good = theCache->get_s3credentials_from_endpoint("http://badurl"); - // CPPUNIT_ASSERT_MESSAGE("Fetch from valid url returns credentials", result_good != nullptr); - - // auto result_good_from_cache = theCache->retrieve_cached_signed_url_components(key); - // CPPUNIT_ASSERT_MESSAGE("After attempting fetch from valid url, credentials cache contains the result", result_good == result_good_from_cache); + // Note: we do not test a successful endpoint here, as that would require either + // relying on TEA or setting up a test url with fake credentials, + // both of which are brittle in their own ways. + // The inputs/outputs for good and bad retrieval are covered by other tests } /* TESTS END */ @@ -343,7 +341,7 @@ CPPUNIT_TEST_SUITE(SignedUrlCacheTest); CPPUNIT_TEST(get_s3credentials_from_endpoint_test); // ...and, specifically, the signing itself: - // TODO-future: will add/update these tests once signing behavior is implemented! + // TODO-future: will add/update these tests once signing behavior is implemented // - sign_url // - get_signed_url From 05d79c8e84f4e7f22decdb85f9a07378f587b1fd Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Fri, 31 Oct 2025 15:04:50 -0400 Subject: [PATCH 20/77] handle timestamp wrangling differently --- http/SignedUrlCache.cc | 19 ++++++++++++------- http/SignedUrlCache.h | 2 +- .../dmrpp_module/ngap_container/NgapApi.cc | 2 +- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/http/SignedUrlCache.cc b/http/SignedUrlCache.cc index ed691d07e2..591d4ee39a 100644 --- a/http/SignedUrlCache.cc +++ b/http/SignedUrlCache.cc @@ -71,28 +71,33 @@ shared_ptr SignedUrlCache::get_cached_signed_url(string const &ur // TODO-docstring; note if timestamp isn't valid, we return that we are expired // We intentionally copy here because we need to delete a character // Could alternatively convert to datetime when we originally cache it.... -bool SignedUrlCache::is_timestamp_after_now(std::string ×tamp_str) { +/** + * @brief Return true if the input occurred before the current time, false otherwise + * @param timestamp_str Timestamp must be formatted as returned by AWS endpoint, YYYY-MM-DD HH:mm:dd+timezone, where timezone is formatted `HH:MM`, e.g. `1980-07-16 18:40:58+00:00` + * @note Returns false if `timestamp_str` is not a valid timestamp string. + * @note Input string may be mutated. + */ +bool SignedUrlCache::is_timestamp_after_now(std::string const ×tamp_str) { auto now = std::chrono::system_clock::now(); auto now_secs = std::chrono::time_point_cast(now); + auto effective_timestamp_str = timestamp_str; if (timestamp_str.size() == 25) { // Hack to handle fact that s3credentials from aws include an // extra colon in their timezone field // This changes "1980-07-16 18:40:58+00:00" to "1980-07-16 18:40:58+0000" - timestamp_str.erase(22, 1); + effective_timestamp_str = string(timestamp_str).erase(22, 1); } + std::tm timestamp_time = {}; - auto time_parse_result = strptime(timestamp_str.c_str(), "%F %T%z", ×tamp_time); + auto time_parse_result = strptime(effective_timestamp_str.c_str(), "%F %T%z", ×tamp_time); if (time_parse_result == nullptr) { - // TODO: do we want to warn or anything here?? We wouldn't expect to be - // here unless we truly thought we had a valid timestamp str... + INFO_LOG(prolog + string("PRE-DEPRECATION WARNING - Retrieved s3 credentials timestamp was not able to be parsed - " + timestamp_str)); return false; } auto timestamp_time_point = std::chrono::system_clock::from_time_t(std::mktime(×tamp_time)); auto timestamp_secs = std::chrono::time_point_cast(timestamp_time_point); - - // TODO: do we want to add a buffer of a second or two here? return timestamp_secs > now_secs; } diff --git a/http/SignedUrlCache.h b/http/SignedUrlCache.h index cdd2bee50a..7adcbaec27 100644 --- a/http/SignedUrlCache.h +++ b/http/SignedUrlCache.h @@ -68,7 +68,7 @@ class SignedUrlCache : public BESObj { std::map> d_s3credentials_cache; std::shared_ptr retrieve_cached_s3credentials(std::string const &url_key); - static bool is_timestamp_after_now(std::string ×tamp); + static bool is_timestamp_after_now(std::string const ×tamp); // URLs that match are not cached. std::unique_ptr d_skip_regex = nullptr; diff --git a/modules/dmrpp_module/ngap_container/NgapApi.cc b/modules/dmrpp_module/ngap_container/NgapApi.cc index 3285e6ed03..a4e6c48d90 100644 --- a/modules/dmrpp_module/ngap_container/NgapApi.cc +++ b/modules/dmrpp_module/ngap_container/NgapApi.cc @@ -535,7 +535,7 @@ NgapApi::DataAccessUrls NgapApi::convert_ngap_resty_path_to_data_access_urls(con if (data_s3_url.empty() || s3credentials_url.empty()) { // Eventually we'll be removing the non-s3 access; we need to know about any unsupported cases before that happens. // Add a log warning that can be searched. - BES_PROFILE_TIMING(string("PRE-DEPRECATION WARNING - Data s3 url or s3credentials not found - ") + cmr_query_url); + INFO_LOG(prolog + string("PRE-DEPRECATION WARNING - Data s3 url or s3credentials not found - ") + cmr_query_url); } // Check for existing .dmrpp and remove it if found at the end of the url. - kln 6/6/25 From c440ee8bf6e61b97715aa89e7ed1168d1ff31ea7 Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Fri, 31 Oct 2025 15:15:04 -0400 Subject: [PATCH 21/77] update docstrings --- http/SignedUrlCache.cc | 47 +++++++++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 19 deletions(-) diff --git a/http/SignedUrlCache.cc b/http/SignedUrlCache.cc index 591d4ee39a..12e859742c 100644 --- a/http/SignedUrlCache.cc +++ b/http/SignedUrlCache.cc @@ -68,10 +68,6 @@ shared_ptr SignedUrlCache::get_cached_signed_url(string const &ur return signed_url; } -// TODO-docstring; note if timestamp isn't valid, we return that we are expired -// We intentionally copy here because we need to delete a character -// Could alternatively convert to datetime when we originally cache it.... - /** * @brief Return true if the input occurred before the current time, false otherwise * @param timestamp_str Timestamp must be formatted as returned by AWS endpoint, YYYY-MM-DD HH:mm:dd+timezone, where timezone is formatted `HH:MM`, e.g. `1980-07-16 18:40:58+00:00` @@ -172,7 +168,6 @@ shared_ptr SignedUrlCache::get_signed_url(shared_ptr source shared_ptr signed_url = get_cached_signed_url(source_url->str()); bool retrieve_and_cache = !signed_url || signed_url->is_expired(); - // TODO-H: make sure that `signed_url->is_expired();` handles correctly for the signed urls // It not found or expired, (re)load. if (retrieve_and_cache) { @@ -232,7 +227,10 @@ shared_ptr SignedUrlCache::get_signed_url(shared_ptr source return signed_url; } -// TODO: docstring +/** + * @brief Store each `s3_url` and `s3credentials_url` for key `key_href_url` + * @note Does not cache anything if any of the three inputs are empty + */ void SignedUrlCache::cache_signed_url_components(const std::string &key_href_url, const std::string &s3_url, const std::string &s3credentials_url) { if (key_href_url.empty() || s3_url.empty() || s3credentials_url.empty() ) { // Don't cache either if any is empty. @@ -242,7 +240,10 @@ void SignedUrlCache::cache_signed_url_components(const std::string &key_href_url d_href_to_s3credentials_cache[key_href_url] = s3credentials_url; } -// TODO: docstring +/** + * @brief Return pair of (s3_url, s3credentials_url) cached for key_href_url + * @note If key_href_url not in cache, returns pair of empty strings + */ std::pair SignedUrlCache::retrieve_cached_signed_url_components(const std::string &key_href_url) const { auto it_s3_url = d_href_to_s3_cache.find(key_href_url); auto it_s3credentials_url = d_href_to_s3credentials_cache.find(key_href_url); @@ -253,7 +254,10 @@ std::pair SignedUrlCache::retrieve_cached_signed_url_c return std::pair(it_s3_url->second, it_s3credentials_url->second); } -// TODO: docstring +/** + * @brief Return credentials tuple for given endpoint url `s3credentials_url`, stores credentials for endpoint in `d_s3credentials_cache` + * @note If credential retrieval fails at any point, returns nullptr and does not cache results + */ shared_ptr SignedUrlCache::get_s3credentials_from_endpoint(std::string const &s3credentials_url) { // 1. Get the credentials from TEA std::string s3credentials_json_string; @@ -283,7 +287,12 @@ shared_ptr SignedUrlCache::get_s3credentials_f } -// Lightly adapted from get_urls_from_granules_umm_json_v1_4 +/** + * @brief Extract credentials tuple from json response returned from an s3credentials endpoint + * @note Returns nullptr if input is not valid json or does not contain one of the four requisite + * strings: `accessKeyId`, `secretAccessKey`, `sessionToken`, or `expiration` + * @note Lightly adapted from get_urls_from_granules_umm_json_v1_4 + */ std::shared_ptr SignedUrlCache::extract_s3_credentials_from_response_json(std::string const &s3credentials_json_string) { rapidjson::Document s3credentials_response; s3credentials_response.Parse(s3credentials_json_string.c_str()); @@ -326,21 +335,18 @@ std::shared_ptr SignedUrlCache::extract_s3_cre expiration); } - /** - * TODO - * - * @param source_url - * @returns The effective (signed) URL, which may be a nullptr if signing credentials are not found -*/ + * @brief Sign `s3_url` with aws credentials in `s3_access_key_tuple`, or nullptr if any part of signing process fails + * @note Not yet implemented! + */ std::shared_ptr SignedUrlCache::sign_url(std::string const &s3_url, std::shared_ptr const s3_access_key_tuple) { - // TODO-fill in details! :) + // TODO-future: fill in implementation! return nullptr; } /** - * @return Is the cache enabled (set in the bes.conf file)? - * Follows the same settings as the EffectiveUrlsCache + * @brief Return if the cache is enabled, which is set in the bes.conf file + * @note Follows the same settings (and relies on the same bes.conf key) as the EffectiveUrlsCache */ bool SignedUrlCache::is_enabled() { // The first time here, the value of d_enabled is -1. Once we check for it in TheBESKeys @@ -353,7 +359,10 @@ bool SignedUrlCache::is_enabled() { return d_enabled; } -// * Follows the same settings as the EffectiveUrlsCache +/** + * @return Set the regex used to skip cache keys, which is set in the bes.conf file + * @note Follows the same settings (and relies on the same bes.conf key) as the EffectiveUrlsCache + */ void SignedUrlCache::set_skip_regex() { if (!d_skip_regex) { string pattern = TheBESKeys::TheKeys()->read_string_key(HTTP_CACHE_EFFECTIVE_URLS_SKIP_REGEX_KEY, ""); From aa65428795fd91743bab1de1143af74f0e72d504 Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Fri, 31 Oct 2025 16:08:03 -0400 Subject: [PATCH 22/77] clean up dump to include all cache maps --- http/SignedUrlCache.cc | 35 +++++++++++++++++++++++++-- http/unit-tests/SignedUrlCacheTest.cc | 29 ++++++++++++++++++---- 2 files changed, 57 insertions(+), 7 deletions(-) diff --git a/http/SignedUrlCache.cc b/http/SignedUrlCache.cc index 12e859742c..d8620c2eb9 100644 --- a/http/SignedUrlCache.cc +++ b/http/SignedUrlCache.cc @@ -386,14 +386,45 @@ void SignedUrlCache::dump(ostream &strm) const { strm << BESIndent::LMarg << "signed url list:" << endl; BESIndent::Indent(); for (auto const &i: d_signed_urls) { - strm << BESIndent::LMarg << i.first << " --> " << i.second->str(); + strm << BESIndent::LMarg << i.first << " --> " << i.second->str() << "\n"; } BESIndent::UnIndent(); } else { strm << BESIndent::LMarg << "signed url list: EMPTY" << endl; } - //TODO: add other three caches here: + if (!d_href_to_s3credentials_cache.empty()) { + strm << BESIndent::LMarg << "href-to-s3credentials list:" << endl; + BESIndent::Indent(); + for (auto const &i: d_href_to_s3credentials_cache) { + strm << BESIndent::LMarg << i.first << " --> " << i.second << "\n"; + } + BESIndent::UnIndent(); + } else { + strm << BESIndent::LMarg << "href-to-s3credentials list: EMPTY" << endl; + } + + if (!d_href_to_s3_cache.empty()) { + strm << BESIndent::LMarg << "href-to-s3url list:" << endl; + BESIndent::Indent(); + for (auto const &i: d_href_to_s3_cache) { + strm << BESIndent::LMarg << i.first << " --> " << i.second << "\n"; + } + BESIndent::UnIndent(); + } else { + strm << BESIndent::LMarg << "href-to-s3url list: EMPTY" << endl; + } + + if (!d_s3credentials_cache.empty()) { + strm << BESIndent::LMarg << "s3 credentials list:" << endl; + BESIndent::Indent(); + for (auto const &i: d_s3credentials_cache) { + strm << BESIndent::LMarg << i.first << " --> Expires: " << get<3>(*(i.second)) << "\n"; + } + BESIndent::UnIndent(); + } else { + strm << BESIndent::LMarg << "s3 credentials list: EMPTY" << endl; + } BESIndent::UnIndent(); } diff --git a/http/unit-tests/SignedUrlCacheTest.cc b/http/unit-tests/SignedUrlCacheTest.cc index 0c1db4ccfb..0f8189de1f 100644 --- a/http/unit-tests/SignedUrlCacheTest.cc +++ b/http/unit-tests/SignedUrlCacheTest.cc @@ -193,19 +193,38 @@ class SignedUrlCacheTest : public CppUnit::TestFixture { } void dump_test() { + SignedUrlCache *theCache = SignedUrlCache::TheCache(); + // Add values to each type of subcache - SignedUrlCache::TheCache()->d_signed_urls.insert( - pair>("www.foo.com", make_shared("http://www.bar.com"))); + theCache->d_signed_urls.insert( + pair>("www.once.com", make_shared("http://www.upon.com"))); + theCache->d_signed_urls.insert( + pair>("www.a.com", make_shared("http://www.time.com"))); + + auto value = make_shared("a man", "a plan", "a canal", "3035-07-16 02:20:33+00:00"); + theCache->d_s3credentials_cache.insert(pair>("palindrome", value)); + + theCache->d_href_to_s3credentials_cache.insert(pair("foo", "whee")); + theCache->d_href_to_s3_cache.insert(pair("yee", "haw")); + // Check to make sure dump includes them auto strm = std::ostringstream(); - SignedUrlCache::TheCache()->dump(strm); + theCache->dump(strm); // Remove start of string to skip address that varies auto result = strm.str().substr(49); std::string expected_str = string("d_skip_regex: ") + "\n signed url list:" + - "\n www.foo.com --> http://www.bar.com"; - CPPUNIT_ASSERT_MESSAGE("The dump should be `" + expected_str + "`; was `" + result + "`", expected_str == result); + // "\n www.foo.com --> http://www.bar.com"; + "\n www.a.com --> http://www.time.com" + + "\n www.once.com --> http://www.upon.com" + + "\n href-to-s3credentials list:" + + "\n foo --> whee" + + "\n href-to-s3url list:" + + "\n yee --> haw" + + "\n s3 credentials list:" + + "\n palindrome --> Expires: 3035-07-16 02:20:33+00:00\n"; + CPPUNIT_ASSERT_MESSAGE("The dump should be:\n" + expected_str + "\n\nINSTEAD was\n" + result, expected_str == result); } void is_timestamp_after_now_test() { From 01b566928d230f5f70b5491f0620d5782ec61fb4 Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Thu, 30 Oct 2025 12:27:26 -0400 Subject: [PATCH 23/77] aws wip --- http/SignedUrlCache.cc | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/http/SignedUrlCache.cc b/http/SignedUrlCache.cc index d8620c2eb9..9d63549419 100644 --- a/http/SignedUrlCache.cc +++ b/http/SignedUrlCache.cc @@ -32,6 +32,7 @@ #include #include +// #include "AWS_SDK.h" // TODO-include aws appropriately #include "TheBESKeys.h" #include "BESDebug.h" #include "BESStopWatch.h" @@ -340,7 +341,21 @@ std::shared_ptr SignedUrlCache::extract_s3_cre * @note Not yet implemented! */ std::shared_ptr SignedUrlCache::sign_url(std::string const &s3_url, std::shared_ptr const s3_access_key_tuple) { - // TODO-future: fill in implementation! + + // AWS_SDK aws_sdk; + // string id = get<0>(*s3_access_key_tuple); + // string secret = get<1>(*s3_access_key_tuple); + // aws_sdk.initialize_s3_client("us-east-1", id, secret); // TODO: uhhhhh where was the region from?????? can we get it from s3_url, I hope???? + + // std::string bucket = s3_url.substr(5, s3_url.find("/")); // Split off "s3://" which we know is here or the path wouldn't have been cached in the first place + // std::string object = s3_url.substr(s3_url.find(bucket) + 1); // Add 1 to cover the "/" after the bucket + // const uint64_t expiration_seconds = 60; // TODO: get from doing math on s3_access_key_tuple ! + + // const Aws::String url = aws_sdk.s3_generate_presigned_object_url(bucket, object, expiration_seconds); + + // // TODO: how might that fail? is it ever null or bad or...does it throw? if so, return nullptr first.... + + // return make_shared(url->str()); // TODO: get string out of AWS url??? or decide to return it..... return nullptr; } From c2657811eaa2c1d1952b68fc91c25dba46a39a69 Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Mon, 3 Nov 2025 16:11:22 -0500 Subject: [PATCH 24/77] wip sadness --- http/Makefile.am | 17 +++++++++++++++-- http/SignedUrlCache.cc | 32 ++++++++++++++++++++------------ 2 files changed, 35 insertions(+), 14 deletions(-) diff --git a/http/Makefile.am b/http/Makefile.am index 94789462d7..eb34ae67f4 100644 --- a/http/Makefile.am +++ b/http/Makefile.am @@ -5,14 +5,27 @@ AUTOMAKE_OPTIONS = foreign -AM_CPPFLAGS = $(OPENSSL_INC) -I$(top_srcdir) -I$(top_srcdir)/dispatch -I$(top_srcdir)/rapidjson $(DAP_CFLAGS) +AM_CPPFLAGS = $(OPENSSL_INC) -I$(top_srcdir) -I$(top_srcdir)/dispatch -I$(top_srcdir)/rapidjson $(DAP_CFLAGS) -I$(top_srcdir)/aws if BES_DEVELOPER AM_CPPFLAGS += -DBES_DEVELOPER endif + +LDADD = $(top_builddir)/dispatch/libbes_dispatch.la $(top_builddir)/aws/libbes_aws.la $(aws_libs) + +# aws_libdir is set in configure.ac. + +# Help the link step find libs at build time too (not just run time) +AM_LDFLAGS = -L$(aws_libdir) + +if DARWIN +AM_LDFLAGS += -Wl,-rpath,$(aws_libdir) -Wl,-rpath,@loader_path +else +AM_LDFLAGS += -Wl,-rpath,$(aws_libdir) -Wl,-rpath,'$$ORIGIN' +endif + AM_CXXFLAGS= -AM_LDFLAGS = include $(top_srcdir)/coverage.mk SUBDIRS = . unit-tests tests diff --git a/http/SignedUrlCache.cc b/http/SignedUrlCache.cc index 9d63549419..3b6c52cf8c 100644 --- a/http/SignedUrlCache.cc +++ b/http/SignedUrlCache.cc @@ -32,7 +32,7 @@ #include #include -// #include "AWS_SDK.h" // TODO-include aws appropriately +#include "AWS_SDK.h" #include "TheBESKeys.h" #include "BESDebug.h" #include "BESStopWatch.h" @@ -342,21 +342,29 @@ std::shared_ptr SignedUrlCache::extract_s3_cre */ std::shared_ptr SignedUrlCache::sign_url(std::string const &s3_url, std::shared_ptr const s3_access_key_tuple) { - // AWS_SDK aws_sdk; - // string id = get<0>(*s3_access_key_tuple); - // string secret = get<1>(*s3_access_key_tuple); - // aws_sdk.initialize_s3_client("us-east-1", id, secret); // TODO: uhhhhh where was the region from?????? can we get it from s3_url, I hope???? + bes::AWS_SDK aws_sdk; + string id = get<0>(*s3_access_key_tuple); + string secret = get<1>(*s3_access_key_tuple); - // std::string bucket = s3_url.substr(5, s3_url.find("/")); // Split off "s3://" which we know is here or the path wouldn't have been cached in the first place - // std::string object = s3_url.substr(s3_url.find(bucket) + 1); // Add 1 to cover the "/" after the bucket - // const uint64_t expiration_seconds = 60; // TODO: get from doing math on s3_access_key_tuple ! + return nullptr; - // const Aws::String url = aws_sdk.s3_generate_presigned_object_url(bucket, object, expiration_seconds); + // TODO: uhhhhh where was the region from?????? can we get it from s3_url, I hope???? + // aws_sdk.initialize_s3_client("us-west-2", id, secret); - // // TODO: how might that fail? is it ever null or bad or...does it throw? if so, return nullptr first.... + // // Get the bucket name by removing prefix "s3://" (which must exist or the path + // // wouldn't have been extracted from cmr) and including everything up to the first slash + // std::string bucket = s3_url.substr(5, s3_url.find("/")); - // return make_shared(url->str()); // TODO: get string out of AWS url??? or decide to return it..... - return nullptr; + // // The object name is everything after the bucket name, not including the first "/" + // std::string object = s3_url.substr(s3_url.find(bucket) + 1); + + // // TODO: get from doing math on s3_access_key_tuple ! + // const uint64_t expiration_seconds = 60; + + // // TODO: how might this fail? is it ever null or bad or...does it throw? if so, return nullptr first.... + // const Aws::String url_str = aws_sdk.s3_generate_presigned_object_url(bucket, object, expiration_seconds); + + // return make_shared(url_str); } /** From 59ab7e04c255c3b784b7b766de4e7a649c691041 Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Wed, 5 Nov 2025 16:07:35 -0500 Subject: [PATCH 25/77] Move SignedUrlCache implementation into .cc --- http/SignedUrlCache.cc | 11 +++++++++++ http/SignedUrlCache.h | 6 +----- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/http/SignedUrlCache.cc b/http/SignedUrlCache.cc index 3b6c52cf8c..eca174eb67 100644 --- a/http/SignedUrlCache.cc +++ b/http/SignedUrlCache.cc @@ -336,6 +336,16 @@ std::shared_ptr SignedUrlCache::extract_s3_cre expiration); } +SignedUrlCache *SignedUrlCache::TheCache() { + // Create a local static object the first time the function is called + static SignedUrlCache instance; + + // Initialize the aws library (must only be once in application!) + bes::AWS_SDK::aws_library_initialize(); + + return &instance; +} + /** * @brief Sign `s3_url` with aws credentials in `s3_access_key_tuple`, or nullptr if any part of signing process fails * @note Not yet implemented! @@ -346,6 +356,7 @@ std::shared_ptr SignedUrlCache::sign_url(std::string const &s3_url string id = get<0>(*s3_access_key_tuple); string secret = get<1>(*s3_access_key_tuple); + // bes::AWS_SDK::aws_library_shutdown(); return nullptr; // TODO: uhhhhh where was the region from?????? can we get it from s3_url, I hope???? diff --git a/http/SignedUrlCache.h b/http/SignedUrlCache.h index 7adcbaec27..4722c2816a 100644 --- a/http/SignedUrlCache.h +++ b/http/SignedUrlCache.h @@ -96,11 +96,7 @@ class SignedUrlCache : public BESObj { * * @return A pointer to the SignedUrlCache singleton */ - static SignedUrlCache *TheCache() { - // Create a local static object the first time the function is called - static SignedUrlCache instance; - return &instance; - } + static SignedUrlCache *TheCache(); SignedUrlCache(const SignedUrlCache &src) = delete; SignedUrlCache &operator=(const SignedUrlCache &rhs) = delete; From cd869e707d62c1516f8a3aecff79ffa2f462f3f6 Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Thu, 6 Nov 2025 12:05:07 -0500 Subject: [PATCH 26/77] move SignedUrlCache from http to aws --- CMakeLists.txt | 13 +++++++------ aws/Makefile.am | 8 ++++++-- {http => aws}/SignedUrlCache.cc | 0 {http => aws}/SignedUrlCache.h | 0 aws/unit-tests/.gitignore | 3 ++- aws/unit-tests/Makefile.am | 3 ++- .../unit-tests/SignedUrlCacheTest.cc | 0 http/Makefile.am | 19 ++----------------- http/unit-tests/.gitignore | 1 - http/unit-tests/Makefile.am | 5 +---- modules/dmrpp_module/Chunk.cc | 2 +- .../ngap_container/NgapOwnedContainer.cc | 2 +- 12 files changed, 22 insertions(+), 34 deletions(-) rename {http => aws}/SignedUrlCache.cc (100%) rename {http => aws}/SignedUrlCache.h (100%) rename {http => aws}/unit-tests/SignedUrlCacheTest.cc (100%) diff --git a/CMakeLists.txt b/CMakeLists.txt index e6ee70d6d8..7618fdce4d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -262,14 +262,18 @@ add_executable( dap/ShowPathInfoResponseHandler.h dap/TempFile.cc dap/TempFile.h - dap/DapUtils.cc - dap/DapUtils.h + dap/DapUtils.cc + dap/DapUtils.h dapreader/DapModule.cc dapreader/DapModule.h dapreader/DapRequestHandler.cc dapreader/DapRequestHandler.h + aws/SignedUrlCache.cc + aws/SignedUrlCache.h + aws/unit-tests/SignedUrlCacheTest.cc + dispatch/PicoSHA2/example/hasher.cpp dispatch/PicoSHA2/example/interactive_hasher.cpp dispatch/PicoSHA2/test/test.cpp @@ -414,7 +418,7 @@ add_executable( dispatch/BESModuleApp.h dispatch/BESNames.h dispatch/BESNotFoundError.h - dispatch/BESObj.h + dispatch/BESObj.h dispatch/BESPlugin.h dispatch/BESPluginFactory.h dispatch/BESProcIdResponseHandler.cc @@ -528,10 +532,7 @@ add_executable( http/EffectiveUrl.h http/EffectiveUrlCache.cc http/EffectiveUrlCache.h - http/SignedUrlCache.cc - http/SignedUrlCache.h http/unit-tests/EffectiveUrlCacheTest.cc - http/unit-tests/SignedUrlCacheTest.cc http/unit-tests/HttpUrlTest.cc http/unit-tests/RemoteResourceTest.cc http/unit-tests/AllowedHostsTest.cc diff --git a/aws/Makefile.am b/aws/Makefile.am index cab5de4f2a..3f39406a8d 100644 --- a/aws/Makefile.am +++ b/aws/Makefile.am @@ -47,5 +47,9 @@ cccc: -mkdir $(C4_DIR) cccc --outdir=$(C4_DIR) $(SRCS) $(HDRS) -SRCS = AWS_SDK.cc -HDRS = AWS_SDK.h IAWS_SDK.h +SRCS = AWS_SDK.cc \ + SignedUrlCache.cc + + +HDRS = AWS_SDK.h IAWS_SDK.h \ + SignedUrlCache.h diff --git a/http/SignedUrlCache.cc b/aws/SignedUrlCache.cc similarity index 100% rename from http/SignedUrlCache.cc rename to aws/SignedUrlCache.cc diff --git a/http/SignedUrlCache.h b/aws/SignedUrlCache.h similarity index 100% rename from http/SignedUrlCache.h rename to aws/SignedUrlCache.h diff --git a/aws/unit-tests/.gitignore b/aws/unit-tests/.gitignore index 3d08aff9e5..a267a42dc9 100644 --- a/aws/unit-tests/.gitignore +++ b/aws/unit-tests/.gitignore @@ -1 +1,2 @@ -AWS_SDK_Test \ No newline at end of file +AWS_SDK_Test +/SignedUrlCacheTest \ No newline at end of file diff --git a/aws/unit-tests/Makefile.am b/aws/unit-tests/Makefile.am index 53f1d701d0..8fd1cb1969 100644 --- a/aws/unit-tests/Makefile.am +++ b/aws/unit-tests/Makefile.am @@ -100,5 +100,6 @@ clean-local: test ! -d $(builddir)/static-cache || rm -rf $(builddir)/static-cache test ! -d $(builddir)/cache || rm -rf $(builddir)/cache -AWS_SDK_Test_SOURCES = AWS_SDK_Test.cc +AWS_SDK_Test_SOURCES = AWS_SDK_Test.cc \ + SignedUrlCacheTest.cc AWS_SDK_Test_LDADD = $(LDADD) diff --git a/http/unit-tests/SignedUrlCacheTest.cc b/aws/unit-tests/SignedUrlCacheTest.cc similarity index 100% rename from http/unit-tests/SignedUrlCacheTest.cc rename to aws/unit-tests/SignedUrlCacheTest.cc diff --git a/http/Makefile.am b/http/Makefile.am index eb34ae67f4..631c4c6d3a 100644 --- a/http/Makefile.am +++ b/http/Makefile.am @@ -5,27 +5,14 @@ AUTOMAKE_OPTIONS = foreign -AM_CPPFLAGS = $(OPENSSL_INC) -I$(top_srcdir) -I$(top_srcdir)/dispatch -I$(top_srcdir)/rapidjson $(DAP_CFLAGS) -I$(top_srcdir)/aws +AM_CPPFLAGS = $(OPENSSL_INC) -I$(top_srcdir) -I$(top_srcdir)/dispatch -I$(top_srcdir)/rapidjson $(DAP_CFLAGS) if BES_DEVELOPER AM_CPPFLAGS += -DBES_DEVELOPER endif - -LDADD = $(top_builddir)/dispatch/libbes_dispatch.la $(top_builddir)/aws/libbes_aws.la $(aws_libs) - -# aws_libdir is set in configure.ac. - -# Help the link step find libs at build time too (not just run time) -AM_LDFLAGS = -L$(aws_libdir) - -if DARWIN -AM_LDFLAGS += -Wl,-rpath,$(aws_libdir) -Wl,-rpath,@loader_path -else -AM_LDFLAGS += -Wl,-rpath,$(aws_libdir) -Wl,-rpath,'$$ORIGIN' -endif - AM_CXXFLAGS= +AM_LDFLAGS = include $(top_srcdir)/coverage.mk SUBDIRS = . unit-tests tests @@ -54,7 +41,6 @@ SRCS = CurlUtils.cc \ HttpUtils.cc \ ProxyConfig.cc \ EffectiveUrlCache.cc \ - SignedUrlCache.cc \ url_impl.cc \ EffectiveUrl.cc \ AllowedHosts.cc \ @@ -69,7 +55,6 @@ HDRS = CurlUtils.h \ ProxyConfig.h \ HttpNames.h \ EffectiveUrlCache.h \ - SignedUrlCache.h \ url_impl.h \ EffectiveUrl.h \ AllowedHosts.h \ diff --git a/http/unit-tests/.gitignore b/http/unit-tests/.gitignore index 6e7bb2da4d..b623b0b70c 100644 --- a/http/unit-tests/.gitignore +++ b/http/unit-tests/.gitignore @@ -3,7 +3,6 @@ /CurlUtilsTest /CurlSListTest /EffectiveUrlCacheTest -/SignedUrlCacheTest /HttpUrlTest /HttpErrorTest /AllowedHostsTest diff --git a/http/unit-tests/Makefile.am b/http/unit-tests/Makefile.am index eca33225a2..a63447c538 100644 --- a/http/unit-tests/Makefile.am +++ b/http/unit-tests/Makefile.am @@ -74,7 +74,7 @@ bes_ngap_s3_creds.conf: bes_ngap_s3_creds.conf.in $(top_srcdir)/configure.ac if CPPUNIT -UNIT_TESTS = HttpUtilsTest HttpErrorTest RemoteResourceTest EffectiveUrlCacheTest HttpUrlTest SignedUrlCacheTest \ +UNIT_TESTS = HttpUtilsTest HttpErrorTest RemoteResourceTest EffectiveUrlCacheTest HttpUrlTest \ AllowedHostsTest awsv4_test CurlUtilsTest CurlSListTest # CredentialsManagerTest was removed because it was testing our S3 signing code and @@ -122,9 +122,6 @@ CurlUtilsTest_LDADD = $(LIBADD) EffectiveUrlCacheTest_SOURCES = EffectiveUrlCacheTest.cc EffectiveUrlCacheTest_LDADD = $(LIBADD) -SignedUrlCacheTest_SOURCES = SignedUrlCacheTest.cc -SignedUrlCacheTest_LDADD = $(LIBADD) - HttpUrlTest_SOURCES = HttpUrlTest.cc HttpUrlTest_LDADD = $(LIBADD) diff --git a/modules/dmrpp_module/Chunk.cc b/modules/dmrpp_module/Chunk.cc index 1fcc1407bf..d6c32f79c3 100644 --- a/modules/dmrpp_module/Chunk.cc +++ b/modules/dmrpp_module/Chunk.cc @@ -52,7 +52,7 @@ using namespace std; using http::EffectiveUrlCache; -using http::SignedUrlCache; +using aws::SignedUrlCache; #define prolog std::string("Chunk::").append(__func__).append("() - ") diff --git a/modules/dmrpp_module/ngap_container/NgapOwnedContainer.cc b/modules/dmrpp_module/ngap_container/NgapOwnedContainer.cc index 4953fbe74f..e68e316da8 100644 --- a/modules/dmrpp_module/ngap_container/NgapOwnedContainer.cc +++ b/modules/dmrpp_module/ngap_container/NgapOwnedContainer.cc @@ -68,7 +68,7 @@ using namespace std; using namespace bes; using http::EffectiveUrlCache; -using http::SignedUrlCache; +using aws::SignedUrlCache; using namespace pugi; namespace ngap { From c6b2c9dfa309a22b43baf8b4821fe6d6c355e65f Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Thu, 13 Nov 2025 11:25:53 -0500 Subject: [PATCH 27/77] enable aws signedurlcachetest tests --- aws/Makefile.am | 4 +- aws/SignedUrlCache.cc | 20 ++-- aws/SignedUrlCache.h | 22 +++-- aws/unit-tests/AWS_SDK_Test.cc | 2 +- aws/unit-tests/Makefile.am | 12 ++- aws/unit-tests/SignedUrlCacheTest.cc | 92 ++----------------- modules/dmrpp_module/Chunk.cc | 2 +- modules/dmrpp_module/Makefile.am | 14 ++- .../dmrpp_module/build_dmrpp_h4/Makefile.am | 12 ++- .../dmrpp_module/ngap_container/Makefile.am | 2 +- .../ngap_container/NgapOwnedContainer.cc | 2 +- 11 files changed, 63 insertions(+), 121 deletions(-) diff --git a/aws/Makefile.am b/aws/Makefile.am index 3f39406a8d..d48819989f 100644 --- a/aws/Makefile.am +++ b/aws/Makefile.am @@ -5,7 +5,7 @@ AUTOMAKE_OPTIONS = foreign -AM_CPPFLAGS = -I$(top_srcdir) -I$(top_srcdir)/dispatch +AM_CPPFLAGS = -I$(top_srcdir) -I$(top_srcdir)/dispatch -I$(top_srcdir)/http if BES_DEVELOPER AM_CPPFLAGS += -DBES_DEVELOPER @@ -15,6 +15,8 @@ AM_CXXFLAGS= AM_LDFLAGS = include $(top_srcdir)/coverage.mk +LDADD = $(BES_HTTP_LIB) + SUBDIRS = . unit-tests noinst_LTLIBRARIES = libbes_aws.la diff --git a/aws/SignedUrlCache.cc b/aws/SignedUrlCache.cc index eca174eb67..4a2c9d1ed7 100644 --- a/aws/SignedUrlCache.cc +++ b/aws/SignedUrlCache.cc @@ -53,15 +53,15 @@ constexpr auto MODULE_DUMPER = "euc:dump"; #define prolog std::string("SignedUrlCache::").append(__func__).append("() - ") -namespace http { +namespace bes { /** * @brief Get the cached signed URL. * @param url_key Key to a cached signed URL. * @note This method is not, itself, thread safe. */ -shared_ptr SignedUrlCache::get_cached_signed_url(string const &url_key) { - shared_ptr signed_url(nullptr); +shared_ptr SignedUrlCache::get_cached_signed_url(string const &url_key) { + shared_ptr signed_url(nullptr); auto it = d_signed_urls.find(url_key); if (it != d_signed_urls.end()) { signed_url = (*it).second; @@ -128,7 +128,7 @@ shared_ptr SignedUrlCache::retrieve_cached_s3c * @param source_url * @returns The signed effective URL, nullptr if none able to be created */ -shared_ptr SignedUrlCache::get_signed_url(shared_ptr source_url) { +shared_ptr SignedUrlCache::get_signed_url(shared_ptr source_url) { BESDEBUG(MODULE, prolog << "BEGIN url: " << source_url->str() << endl); BESDEBUG(MODULE_DUMPER, prolog << "dump: " << endl << dump() << endl); @@ -167,7 +167,7 @@ shared_ptr SignedUrlCache::get_signed_url(shared_ptr source BESDEBUG(MODULE, prolog << "The cache_effective_urls_skip_regex() was NOT SET " << endl); } - shared_ptr signed_url = get_cached_signed_url(source_url->str()); + shared_ptr signed_url = get_cached_signed_url(source_url->str()); bool retrieve_and_cache = !signed_url || signed_url->is_expired(); // It not found or expired, (re)load. @@ -215,11 +215,11 @@ shared_ptr SignedUrlCache::get_signed_url(shared_ptr source // the instance we placed in the cache - it can be modified and the one in the cache // is unchanged. Trusted state was established from source_url when signed_url was // created in sign_url() - signed_url = make_shared(signed_url); + signed_url = make_shared(signed_url); } else { // Here we have a !expired instance of a shared_ptr retrieved from the cache. // Now we need to make a copy to return, inheriting trust from the requesting URL. - signed_url = make_shared(signed_url, source_url->is_trusted()); + signed_url = make_shared(signed_url, source_url->is_trusted()); } BESDEBUG(MODULE_DUMPER, prolog << "dump: " << endl << dump() << endl); @@ -350,7 +350,7 @@ SignedUrlCache *SignedUrlCache::TheCache() { * @brief Sign `s3_url` with aws credentials in `s3_access_key_tuple`, or nullptr if any part of signing process fails * @note Not yet implemented! */ -std::shared_ptr SignedUrlCache::sign_url(std::string const &s3_url, std::shared_ptr const s3_access_key_tuple) { +std::shared_ptr SignedUrlCache::sign_url(std::string const &s3_url, std::shared_ptr const s3_access_key_tuple) { bes::AWS_SDK aws_sdk; string id = get<0>(*s3_access_key_tuple); @@ -375,7 +375,7 @@ std::shared_ptr SignedUrlCache::sign_url(std::string const &s3_url // // TODO: how might this fail? is it ever null or bad or...does it throw? if so, return nullptr first.... // const Aws::String url_str = aws_sdk.s3_generate_presigned_object_url(bucket, object, expiration_seconds); - // return make_shared(url_str); + // return make_shared(url_str); } /** @@ -463,4 +463,4 @@ void SignedUrlCache::dump(ostream &strm) const { BESIndent::UnIndent(); } -} // namespace http +} // namespace bes diff --git a/aws/SignedUrlCache.h b/aws/SignedUrlCache.h index 4722c2816a..a7628f5d09 100644 --- a/aws/SignedUrlCache.h +++ b/aws/SignedUrlCache.h @@ -1,6 +1,6 @@ // -*- mode: c++; c-basic-offset:4 -*- -// This file is part of the BES http package, part of the Hyrax data server. +// This file is part of the BES aws package, part of the Hyrax data server. // Copyright (c) 2025 OPeNDAP, Inc. // Authors: Nathan Potter , Hannah Robertson @@ -25,8 +25,8 @@ // ndp Nathan Potter // Hannah Robertson -#ifndef _bes_http_SignedUrlCache_h_ -#define _bes_http_SignedUrlCache_h_ 1 +#ifndef _bes_aws_SignedUrlCache_h_ +#define _bes_aws_SignedUrlCache_h_ 1 #include #include @@ -38,9 +38,11 @@ #include "BESRegex.h" // for std::unique_ptr namespace http { + class EffectiveUrl; + class url; +} -class EffectiveUrl; -class url; +namespace bes { /** * This is a singleton class. It is used to associate a URL with its "pre-signed" AWS s3 URL. This means that @@ -75,9 +77,9 @@ class SignedUrlCache : public BESObj { int d_enabled = -1; - std::shared_ptr sign_url(std::string const &s3_url, + std::shared_ptr sign_url(std::string const &s3_url, std::shared_ptr const s3_access_key_tuple); - std::shared_ptr get_cached_signed_url(std::string const &url_key); + std::shared_ptr get_cached_signed_url(std::string const &url_key); void set_skip_regex(); @@ -105,7 +107,7 @@ class SignedUrlCache : public BESObj { void cache_signed_url_components(const std::string &key_href_url, const std::string &s3_url, const std::string &s3credentials_url); std::pair retrieve_cached_signed_url_components(const std::string &key_href_url) const; - std::shared_ptr get_signed_url(std::shared_ptr source_url); + std::shared_ptr get_signed_url(std::shared_ptr source_url); void dump(std::ostream &strm) const override; @@ -116,7 +118,7 @@ class SignedUrlCache : public BESObj { } }; -} // namespace http +} // namespace bes -#endif // _bes_http_SignedUrlCache_h_ +#endif // _bes_aws_SignedUrlCache_h_ diff --git a/aws/unit-tests/AWS_SDK_Test.cc b/aws/unit-tests/AWS_SDK_Test.cc index af55673486..120df96f0b 100644 --- a/aws/unit-tests/AWS_SDK_Test.cc +++ b/aws/unit-tests/AWS_SDK_Test.cc @@ -357,7 +357,7 @@ class AWS_SDK_Test : public CppUnit::TestFixture { }; CPPUNIT_TEST_SUITE_REGISTRATION(AWS_SDK_Test); -} // namespace http +} // namespace bes int main(int argc, char *argv[]) { return bes_run_tests(argc, argv, "cerr,bes,http") ? 0 : 1; diff --git a/aws/unit-tests/Makefile.am b/aws/unit-tests/Makefile.am index 8fd1cb1969..d3a101d6e2 100644 --- a/aws/unit-tests/Makefile.am +++ b/aws/unit-tests/Makefile.am @@ -2,7 +2,7 @@ AUTOMAKE_OPTIONS = foreign -AM_CPPFLAGS = -I$(top_srcdir) -I$(top_srcdir)/aws -I$(top_srcdir)/dispatch +AM_CPPFLAGS = -I$(top_srcdir) -I$(top_srcdir)/aws -I$(top_srcdir)/dispatch -I$(top_srcdir)/http # Help the link step find libs at build time too (not just run time) AM_LDFLAGS = -L$(aws_libdir) @@ -11,7 +11,7 @@ AM_LDFLAGS = -L$(aws_libdir) # like bin_PROGRAMS. # NB: aws_libs is set in configure.ac. jhrg 10/16/25 -LDADD = $(top_builddir)/dispatch/libbes_dispatch.la $(top_builddir)/aws/libbes_aws.la $(aws_libs) +LDADD = $(top_builddir)/dispatch/libbes_dispatch.la $(top_builddir)/aws/libbes_aws.la $(aws_libs) $(BES_HTTP_LIB) # aws_libdir is set in configure.ac. jhrg 10/16/25 @@ -80,7 +80,7 @@ test_config.h: $(srcdir)/test_config.h.in Makefile if CPPUNIT -UNIT_TESTS = AWS_SDK_Test +UNIT_TESTS = AWS_SDK_Test SignedUrlCacheTest else @@ -100,6 +100,8 @@ clean-local: test ! -d $(builddir)/static-cache || rm -rf $(builddir)/static-cache test ! -d $(builddir)/cache || rm -rf $(builddir)/cache -AWS_SDK_Test_SOURCES = AWS_SDK_Test.cc \ - SignedUrlCacheTest.cc +AWS_SDK_Test_SOURCES = AWS_SDK_Test.cc AWS_SDK_Test_LDADD = $(LDADD) + +SignedUrlCacheTest_SOURCES = SignedUrlCacheTest.cc +SignedUrlCacheTest_LDADD = $(LDADD) diff --git a/aws/unit-tests/SignedUrlCacheTest.cc b/aws/unit-tests/SignedUrlCacheTest.cc index 0f8189de1f..d42201d139 100644 --- a/aws/unit-tests/SignedUrlCacheTest.cc +++ b/aws/unit-tests/SignedUrlCacheTest.cc @@ -25,10 +25,7 @@ #include #include - -#include -#include -#include +#include #include @@ -46,40 +43,17 @@ #include "test_config.h" +#include "modules/common/run_tests_cppunit.h" + using namespace std; -static bool debug = false; -static bool Debug = false; -static bool bes_debug = false; -static bool ngap_tests = false; static std::string token; -#undef DBG -#define DBG(x) do { if (debug) x; } while(false) #define prolog std::string("SignedUrlCacheTest::").append(__func__).append("() - ") -namespace http { +namespace bes { class SignedUrlCacheTest : public CppUnit::TestFixture { -private: - string d_data_dir = TEST_DATA_DIR; - - void show_file(string filename) { - ifstream t(filename.c_str()); - - if (t.is_open()) { - string file_content((istreambuf_iterator(t)), istreambuf_iterator()); - t.close(); - cerr << endl << "#############################################################################" << endl; - cerr << "file: " << filename << endl; - cerr << ". . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . " << endl; - cerr << file_content << endl; - cerr << "#############################################################################" << endl; - } else { - cerr << "FAILED TO OPEN FILE: " << filename << endl; - } - } - public: // Called once before everything gets tested SignedUrlCacheTest() = default; @@ -91,14 +65,10 @@ class SignedUrlCacheTest : public CppUnit::TestFixture { void setUp() override { DBG(cerr << endl); DBG(cerr << prolog << "BEGIN" << endl); - DBG(cerr << prolog << "data_dir: " << d_data_dir << endl); string bes_conf = BESUtil::assemblePath(TEST_BUILD_DIR, "bes.conf"); DBG(cerr << prolog << "Using BES configuration: " << bes_conf << endl); - if (Debug) show_file(bes_conf); TheBESKeys::ConfigFile = bes_conf; - if (bes_debug) BESDebug::SetUp("cerr,bes,euc,http,curl"); - // Reset to same starting point every time // (It's a singleton so resetting it is important for test determinism) SignedUrlCache *theCache = SignedUrlCache::TheCache(); @@ -369,56 +339,8 @@ CPPUNIT_TEST_SUITE(SignedUrlCacheTest); CPPUNIT_TEST_SUITE_REGISTRATION(SignedUrlCacheTest); -} // namespace httpd_catalog +} // namespace bes int main(int argc, char *argv[]) { - CppUnit::TextTestRunner runner; - runner.addTest(CppUnit::TestFactoryRegistry::getRegistry().makeTest()); - - int option_char; - while ((option_char = getopt(argc, argv, "dbDPt:N")) != -1) - switch (option_char) { - case 'd': - debug = true; // debug is a static global - cerr << "debug enabled" << endl; - break; - case 'D': - Debug = true; // Debug is a static global - cerr << "Debug enabled" << endl; - break; - case 'b': - bes_debug = true; // debug is a static global - cerr << "bes_debug enabled" << endl; - break; - case 'N': - ngap_tests = true; // ngap_tests is a static global - cerr << "NGAP Tests Enabled." << token << endl; - break; - case 't': - token = optarg; // token is a static global - cerr << "Authorization header value: " << token << endl; - break; - default: - break; - } - - argc -= optind; - argv += optind; - - bool wasSuccessful = true; - string test; - if (0 == argc) { - // run them all - wasSuccessful = runner.run(""); - } else { - int i = 0; - while (i < argc) { - if (debug) cerr << "Running " << argv[i] << endl; - test = http::SignedUrlCacheTest::suite()->getName().append("::").append(argv[i]); - wasSuccessful = wasSuccessful && runner.run(test); - ++i; - } - } - - return wasSuccessful ? 0 : 1; -} + return bes_run_tests(argc, argv, "cerr,bes,http") ? 0 : 1; +} \ No newline at end of file diff --git a/modules/dmrpp_module/Chunk.cc b/modules/dmrpp_module/Chunk.cc index d6c32f79c3..3306aa573b 100644 --- a/modules/dmrpp_module/Chunk.cc +++ b/modules/dmrpp_module/Chunk.cc @@ -52,7 +52,7 @@ using namespace std; using http::EffectiveUrlCache; -using aws::SignedUrlCache; +using bes::SignedUrlCache; #define prolog std::string("Chunk::").append(__func__).append("() - ") diff --git a/modules/dmrpp_module/Makefile.am b/modules/dmrpp_module/Makefile.am index 4ee7d73031..c810e44319 100644 --- a/modules/dmrpp_module/Makefile.am +++ b/modules/dmrpp_module/Makefile.am @@ -4,7 +4,8 @@ AUTOMAKE_OPTIONS = foreign subdir-objects ACLOCAL_AMFLAGS = -I conf AM_CPPFLAGS = -I$(top_srcdir) -I$(top_srcdir)/dispatch -I$(top_srcdir)/dap -I$(top_srcdir)/xmlcommand \ - -I$(top_srcdir)/http -I$(top_srcdir)/modules/dmrpp_module/ngap_container -I$(top_srcdir)/pugixml/src $(DAP_CFLAGS) + -I$(top_srcdir)/http -I$(top_srcdir)/modules/dmrpp_module/ngap_container -I$(top_srcdir)/pugixml/src $(DAP_CFLAGS) \ + -I$(top_srcdir)/aws AM_CXXFLAGS = -Wno-vla-extension -Wno-inconsistent-missing-override # FIXME Remove this hack. Set these with configure. jhrg 11/25/19 @@ -27,7 +28,7 @@ AM_CPPFLAGS += -DMODULE_NAME=\"$(M_NAME)\" -DMODULE_VERSION=\"$(M_VER)\" # the --disable-shared is not required, but it seems to help with debuggers. CXXFLAGS_DEBUG = -g3 -O0 -Wall -W -Wcast-align -AM_LDFLAGS = +AM_LDFLAGS = -L$(aws_libdir) include $(top_srcdir)/coverage.mk SUBDIRS = ngap_container dmrpp_transmitter build_dmrpp_h4 . unit-tests tests data tests_build_dmrpp get_dmrpp @@ -76,7 +77,14 @@ build_dmrpp_SOURCES = $(BES_SRCS) $(BES_HDRS) DmrppRequestHandler.cc DmrppReques build_dmrpp_LDFLAGS = $(top_builddir)/dap/.libs/libdap_module.a build_dmrpp_LDADD = $(BES_DISPATCH_LIB) $(BES_HTTP_LIB) $(H5_LDFLAGS) \ $(H5_LIBS) $(DAP_SERVER_LIBS) $(DAP_CLIENT_LIBS) $(OPENSSL_LDFLAGS) $(OPENSSL_LIBS) \ - $(XML2_LIBS) $(BYTESWAP_LIBS) -lz + $(top_builddir)/aws/libbes_aws.la $(aws_libs) \ + $(XML2_LIBS) $(BYTESWAP_LIBS) -lz + +if DARWIN +AM_LDFLAGS += -Wl,-rpath,$(aws_libdir) -Wl,-rpath,@loader_path +else +AM_LDFLAGS += -Wl,-rpath,$(aws_libdir) -Wl,-rpath,'$$ORIGIN' +endif # jhrg 6/2/23 $(BES_EXTRA_LIBS) diff --git a/modules/dmrpp_module/build_dmrpp_h4/Makefile.am b/modules/dmrpp_module/build_dmrpp_h4/Makefile.am index 6b52380ee3..7ae46f0202 100644 --- a/modules/dmrpp_module/build_dmrpp_h4/Makefile.am +++ b/modules/dmrpp_module/build_dmrpp_h4/Makefile.am @@ -14,12 +14,12 @@ M_VER=0.1.0 AM_CPPFLAGS = $(HDF4_CFLAGS) $(HDFEOS2_CPPFLAGS) -I$(top_srcdir) -I$(top_srcdir)/dispatch -I$(top_srcdir)/modules/dmrpp_module \ -I$(top_srcdir)/pugixml/src -I$(top_srcdir)/http -I$(top_srcdir)/modules/dmrpp_module/ngap_container -I$(top_srcdir)/dap \ - $(DAP_CFLAGS) + $(DAP_CFLAGS) -I$(top_srcdir)/aws AM_CPPFLAGS += -DMODULE_NAME=\"$(M_NAME)\" -DMODULE_VERSION=\"$(M_VER)\" AM_CXXFLAGS= -AM_LDFLAGS = +AM_LDFLAGS = -L$(aws_libdir) include $(top_srcdir)/coverage.mk # SUBDIRS = . unit-tests tests @@ -51,7 +51,13 @@ build_dmrpp_h4_LDFLAGS = $(HDFEOS2_LDFLAGS) $(HDF4_LDFLAGS) $(BES_DAP_LIB_LDFLAG # jhrg 12/18/23 $(top_builddir)/dap/.libs/libdap_module.a build_dmrpp_h4_LDADD = $(BES_DISPATCH_LIB) $(BES_HTTP_LIB) $(DAP_SERVER_LIBS) $(DAP_CLIENT_LIBS) \ - $(OPENSSL_LDFLAGS) $(OPENSSL_LIBS) $(XML2_LIBS) $(BYTESWAP_LIBS) $(HDFEOS2_LIBS) $(HDF4_LIBS) + $(OPENSSL_LDFLAGS) $(OPENSSL_LIBS) $(XML2_LIBS) $(BYTESWAP_LIBS) $(HDFEOS2_LIBS) $(HDF4_LIBS) $(top_builddir)/aws/libbes_aws.la $(aws_libs) + +if DARWIN +AM_LDFLAGS += -Wl,-rpath,$(aws_libdir) -Wl,-rpath,@loader_path +else +AM_LDFLAGS += -Wl,-rpath,$(aws_libdir) -Wl,-rpath,'$$ORIGIN' +endif lib_besdir=$(libdir)/bes diff --git a/modules/dmrpp_module/ngap_container/Makefile.am b/modules/dmrpp_module/ngap_container/Makefile.am index 20cb182370..983a4ec8f3 100644 --- a/modules/dmrpp_module/ngap_container/Makefile.am +++ b/modules/dmrpp_module/ngap_container/Makefile.am @@ -7,7 +7,7 @@ AUTOMAKE_OPTIONS = foreign ACLOCAL_AMFLAGS = -I conf AM_CPPFLAGS = -I$(top_srcdir) -I$(top_srcdir)/dispatch -I$(top_srcdir)/dap -I$(top_srcdir)/xmlcommand \ --I$(top_srcdir)/http -I$(top_srcdir)/pugixml/src $(DAP_CFLAGS) +-I$(top_srcdir)/http -I$(top_srcdir)/aws -I$(top_srcdir)/pugixml/src $(DAP_CFLAGS) LIBADD = diff --git a/modules/dmrpp_module/ngap_container/NgapOwnedContainer.cc b/modules/dmrpp_module/ngap_container/NgapOwnedContainer.cc index e68e316da8..44602606a4 100644 --- a/modules/dmrpp_module/ngap_container/NgapOwnedContainer.cc +++ b/modules/dmrpp_module/ngap_container/NgapOwnedContainer.cc @@ -68,7 +68,7 @@ using namespace std; using namespace bes; using http::EffectiveUrlCache; -using aws::SignedUrlCache; +using bes::SignedUrlCache; using namespace pugi; namespace ngap { From fd688d3da07f392ade60ded922e5bbf7563588be Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Thu, 13 Nov 2025 16:23:51 -0500 Subject: [PATCH 28/77] add helper function for splitting s3 paths --- aws/Makefile.am | 2 - aws/SignedUrlCache.cc | 68 ++++++++++++++++++++-------- aws/SignedUrlCache.h | 5 +- aws/unit-tests/SignedUrlCacheTest.cc | 20 ++++++++ 4 files changed, 72 insertions(+), 23 deletions(-) diff --git a/aws/Makefile.am b/aws/Makefile.am index d48819989f..a715586fba 100644 --- a/aws/Makefile.am +++ b/aws/Makefile.am @@ -51,7 +51,5 @@ cccc: SRCS = AWS_SDK.cc \ SignedUrlCache.cc - - HDRS = AWS_SDK.h IAWS_SDK.h \ SignedUrlCache.h diff --git a/aws/SignedUrlCache.cc b/aws/SignedUrlCache.cc index 4a2c9d1ed7..8a72e4365c 100644 --- a/aws/SignedUrlCache.cc +++ b/aws/SignedUrlCache.cc @@ -343,39 +343,67 @@ SignedUrlCache *SignedUrlCache::TheCache() { // Initialize the aws library (must only be once in application!) bes::AWS_SDK::aws_library_initialize(); + // TODO: do i need a corresponding + // bes::AWS_SDK::aws_library_shutdown(); + // in the destructor?? + return &instance; } + /** - * @brief Sign `s3_url` with aws credentials in `s3_access_key_tuple`, or nullptr if any part of signing process fails - * @note Not yet implemented! + * @brief Split `s3_url` into bucket, object strings */ -std::shared_ptr SignedUrlCache::sign_url(std::string const &s3_url, std::shared_ptr const s3_access_key_tuple) { +std::pair SignedUrlCache::split_s3_url(std::string const &s3_url) { - bes::AWS_SDK aws_sdk; - string id = get<0>(*s3_access_key_tuple); - string secret = get<1>(*s3_access_key_tuple); + // Safety first (even though if we were missing the s3:// prefix, s3_url wouldn't have been extracted from the cmr result in the first place....) + if (s3_url.size() < 6 || s3_url.find("s3://") != 0) { + return std::pair("", ""); + } - // bes::AWS_SDK::aws_library_shutdown(); - return nullptr; + // Get the bucket name by removing prefix "s3://" (which must exist or the path + // wouldn't have been extracted from cmr) and including everything up to the first slash + std::string bucket = s3_url.substr(5, s3_url.find("/")); + + // The object name is everything after the bucket name, not including the first "/" + std::string object = s3_url.substr(s3_url.find(bucket) + 1 + bucket.size()); - // TODO: uhhhhh where was the region from?????? can we get it from s3_url, I hope???? - // aws_sdk.initialize_s3_client("us-west-2", id, secret); + return std::pair(bucket, object); +} - // // Get the bucket name by removing prefix "s3://" (which must exist or the path - // // wouldn't have been extracted from cmr) and including everything up to the first slash - // std::string bucket = s3_url.substr(5, s3_url.find("/")); +uint64_t SignedUrlCache::get_expiration_seconds(const string &credentials_expiration_datetime) { + uint64_t expiration_seconds(60); //TODO-actually do math here!! + return expiration_seconds; +} - // // The object name is everything after the bucket name, not including the first "/" - // std::string object = s3_url.substr(s3_url.find(bucket) + 1); +/** + * @brief Sign `s3_url` with aws credentials in `s3_access_key_tuple`, or nullptr if any part of signing process fails + */ +std::shared_ptr SignedUrlCache::sign_url(std::string const &s3_url, + std::shared_ptr const s3_access_key_tuple, + std::string aws_region) { + bes::AWS_SDK aws_sdk; + string id = get<0>(*s3_access_key_tuple); + string secret = get<1>(*s3_access_key_tuple); + aws_sdk.initialize_s3_client(aws_region, id, secret); + + string bucket; + string object; + tie(bucket, object) = split_s3_url(s3_url); + if (bucket.empty() || object.empty()) { + return nullptr; + } - // // TODO: get from doing math on s3_access_key_tuple ! - // const uint64_t expiration_seconds = 60; + auto expiration_seconds = get_expiration_seconds(get<3>(*s3_access_key_tuple)); - // // TODO: how might this fail? is it ever null or bad or...does it throw? if so, return nullptr first.... - // const Aws::String url_str = aws_sdk.s3_generate_presigned_object_url(bucket, object, expiration_seconds); + // Can this call fail? If the aws library isn't initialized, it could through a + // BESInternalFatalError, but the library is initialized in the constructor of SignedUrlCache, + // so if we're here it MUST be initialized. + // As far as I can tell, the internal signing function + // doesn't have a chance to throw.... + const Aws::String url_str = aws_sdk.s3_generate_presigned_object_url(bucket, object, expiration_seconds); - // return make_shared(url_str); + return make_shared(url_str); } /** diff --git a/aws/SignedUrlCache.h b/aws/SignedUrlCache.h index a7628f5d09..3c286d9943 100644 --- a/aws/SignedUrlCache.h +++ b/aws/SignedUrlCache.h @@ -77,8 +77,11 @@ class SignedUrlCache : public BESObj { int d_enabled = -1; + static std::pair split_s3_url(std::string const &s3_url); + static uint64_t get_expiration_seconds(const std::string &credentials_expiration_datetime); std::shared_ptr sign_url(std::string const &s3_url, - std::shared_ptr const s3_access_key_tuple); + std::shared_ptr const s3_access_key_tuple, + std::string aws_region="us-west-2"); std::shared_ptr get_cached_signed_url(std::string const &url_key); void set_skip_regex(); diff --git a/aws/unit-tests/SignedUrlCacheTest.cc b/aws/unit-tests/SignedUrlCacheTest.cc index d42201d139..ce6bb4abfe 100644 --- a/aws/unit-tests/SignedUrlCacheTest.cc +++ b/aws/unit-tests/SignedUrlCacheTest.cc @@ -309,6 +309,22 @@ class SignedUrlCacheTest : public CppUnit::TestFixture { // The inputs/outputs for good and bad retrieval are covered by other tests } + void split_s3_url_test() { + auto out1 = SignedUrlCache::split_s3_url("s3://foo/bar"); + auto expected1 = std::pair("foo", "bar"); + CPPUNIT_ASSERT_MESSAGE("Valid input is split into `" + get<0>(expected1) + "," + get<1>(expected1) + "`, got `" + get<0>(out1) + "," + get<1>(out1) + "`", expected1 == out1); + + auto expected_empty = std::pair("", ""); + auto out2 = SignedUrlCache::split_s3_url("foo/bar"); + CPPUNIT_ASSERT_MESSAGE("Missing s3:// prefix should return empty strings; returned `" + get<0>(out2) + "," + get<1>(out2) + "`", expected_empty == out2); + auto out3 = SignedUrlCache::split_s3_url("x"); + CPPUNIT_ASSERT_MESSAGE("Missing s3:// prefix should return empty strings; returned `" + get<0>(out3) + "," + get<1>(out3) + "`", expected_empty == out2); + + auto out4 = SignedUrlCache::split_s3_url("s3://foo/bar/wheeyay.txt"); + auto expected4 = std::pair("foo", "bar/wheeyay.txt"); + CPPUNIT_ASSERT_MESSAGE("Valid input is split into `" + get<0>(expected4) + "," + get<1>(expected4) + "`, got `" + get<0>(out4) + "," + get<1>(out4) + "`", expected4 == out4); + } + /* TESTS END */ /*##################################################################################################*/ @@ -334,6 +350,10 @@ CPPUNIT_TEST_SUITE(SignedUrlCacheTest); // - sign_url // - get_signed_url + // Last but not least, test those helper functions + CPPUNIT_TEST(split_s3_url_test); + // CPPUNIT_TEST(get_expiration_seconds); + CPPUNIT_TEST_SUITE_END(); }; From a45935bc1968a00db509959d7ecf0cc9a0259cad Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Fri, 14 Nov 2025 16:49:33 -0500 Subject: [PATCH 29/77] add expiration calculation --- aws/SignedUrlCache.cc | 42 +++++++++++++++++++++++----- aws/SignedUrlCache.h | 2 +- aws/unit-tests/SignedUrlCacheTest.cc | 41 ++++++++++++++++++++++++++- 3 files changed, 76 insertions(+), 9 deletions(-) diff --git a/aws/SignedUrlCache.cc b/aws/SignedUrlCache.cc index 8a72e4365c..29fdd04dbf 100644 --- a/aws/SignedUrlCache.cc +++ b/aws/SignedUrlCache.cc @@ -72,11 +72,10 @@ shared_ptr SignedUrlCache::get_cached_signed_url(string con /** * @brief Return true if the input occurred before the current time, false otherwise * @param timestamp_str Timestamp must be formatted as returned by AWS endpoint, YYYY-MM-DD HH:mm:dd+timezone, where timezone is formatted `HH:MM`, e.g. `1980-07-16 18:40:58+00:00` - * @note Returns false if `timestamp_str` is not a valid timestamp string. - * @note Input string may be mutated. + * @note Return false if `timestamp_str` is not a valid timestamp string. */ bool SignedUrlCache::is_timestamp_after_now(std::string const ×tamp_str) { - auto now = std::chrono::system_clock::now(); + auto now = std::chrono::system_clock::now(); auto now_secs = std::chrono::time_point_cast(now); auto effective_timestamp_str = timestamp_str; @@ -371,9 +370,34 @@ std::pair SignedUrlCache::split_s3_url(std::string con return std::pair(bucket, object); } -uint64_t SignedUrlCache::get_expiration_seconds(const string &credentials_expiration_datetime) { - uint64_t expiration_seconds(60); //TODO-actually do math here!! - return expiration_seconds; + +/** + * @brief Return difference between expiration time and now, if expiration is in future; 0 otherwise + * @param credentials_expiration_datetime Timestamp must be formatted as returned by AWS endpoint, YYYY-MM-DD HH:mm:dd+timezone, where timezone is formatted `HH:MM`, e.g. `1980-07-16 18:40:58+00:00` + * @note Return 0 if `timestamp_str` is not a valid timestamp string. + * @note Shares code with `is_timestamp_after_now`, could maybe converge + * @param current_time Exposed as parameter to aid in testing; defaults to now() + */ +uint64_t SignedUrlCache::num_seconds_until_expiration(const string &credentials_expiration_datetime, const chrono::system_clock::time_point current_time) { + auto now_secs = std::chrono::time_point_cast(current_time); + + auto effective_timestamp_str = credentials_expiration_datetime; + if (credentials_expiration_datetime.size() == 25) { + // Hack to handle fact that s3credentials from aws include an + // extra colon in their timezone field + // This changes "1980-07-16 18:40:58+00:00" to "1980-07-16 18:40:58+0000" + effective_timestamp_str = string(credentials_expiration_datetime).erase(22, 1); + } + + std::tm timestamp_time = {}; + auto time_parse_result = strptime(effective_timestamp_str.c_str(), "%F %T%z", ×tamp_time); + if (time_parse_result == nullptr) { + INFO_LOG(prolog + string("PRE-DEPRECATION WARNING - Retrieved s3 credentials timestamp was not able to be parsed - " + credentials_expiration_datetime)); + return 0; + } + auto timestamp_time_point = std::chrono::system_clock::from_time_t(std::mktime(×tamp_time)); + auto timestamp_secs = std::chrono::time_point_cast(timestamp_time_point); + return timestamp_secs > now_secs ? (timestamp_secs - now_secs).count() : 0; } /** @@ -394,7 +418,11 @@ std::shared_ptr SignedUrlCache::sign_url(std::string const & return nullptr; } - auto expiration_seconds = get_expiration_seconds(get<3>(*s3_access_key_tuple)); + auto expiration_seconds = num_seconds_until_expiration(get<3>(*s3_access_key_tuple)); + if (expiration_seconds == 0) { + // No point in creating a url that is already expired!! + return nullptr; + } // Can this call fail? If the aws library isn't initialized, it could through a // BESInternalFatalError, but the library is initialized in the constructor of SignedUrlCache, diff --git a/aws/SignedUrlCache.h b/aws/SignedUrlCache.h index 3c286d9943..a41ddfa86b 100644 --- a/aws/SignedUrlCache.h +++ b/aws/SignedUrlCache.h @@ -78,7 +78,7 @@ class SignedUrlCache : public BESObj { int d_enabled = -1; static std::pair split_s3_url(std::string const &s3_url); - static uint64_t get_expiration_seconds(const std::string &credentials_expiration_datetime); + static uint64_t num_seconds_until_expiration(const std::string &credentials_expiration_datetime, const std::chrono::system_clock::time_point current_time=std::chrono::system_clock::now()); std::shared_ptr sign_url(std::string const &s3_url, std::shared_ptr const s3_access_key_tuple, std::string aws_region="us-west-2"); diff --git a/aws/unit-tests/SignedUrlCacheTest.cc b/aws/unit-tests/SignedUrlCacheTest.cc index ce6bb4abfe..6b5cdf4192 100644 --- a/aws/unit-tests/SignedUrlCacheTest.cc +++ b/aws/unit-tests/SignedUrlCacheTest.cc @@ -325,6 +325,45 @@ class SignedUrlCacheTest : public CppUnit::TestFixture { CPPUNIT_ASSERT_MESSAGE("Valid input is split into `" + get<0>(expected4) + "," + get<1>(expected4) + "`, got `" + get<0>(out4) + "," + get<1>(out4) + "`", expected4 == out4); } + std::chrono::system_clock::time_point parse_as_time_point(const std::string& datetime_string) { + std::tm timestamp_time = {}; + auto time_parse_result = strptime(datetime_string.c_str(), "%F %T%z", ×tamp_time); + + // auto foo = std::mktime(×tamp_time); + // auto bar = std::chrono::system_clock::from_time_t(foo); + // return bar; + + return std::chrono::system_clock::from_time_t(std::mktime(×tamp_time)); + } + + void num_seconds_until_expiration_test() { + std::string current_time_str = "2025-04-01 09:30:00+0400"; + std::string current_time_aws_str = "2025-04-01 09:30:00+04:00"; + std::chrono::system_clock::time_point current_time = parse_as_time_point(current_time_str); + + auto out = SignedUrlCache::num_seconds_until_expiration(current_time_aws_str, current_time); + CPPUNIT_ASSERT_MESSAGE("Expiration time `now` should return 0s` for `" + current_time_aws_str + "`: `" + to_string(out) + "`", out == 0); + + auto time_plus_five_seconds_aws_str = "2025-04-01 09:30:05+04:00"; + auto out1 = SignedUrlCache::num_seconds_until_expiration(time_plus_five_seconds_aws_str, current_time); + CPPUNIT_ASSERT_MESSAGE("Expiration date should be 5s from now `" + to_string(out1) + "`", out1 == 5); + + auto time_minus_five_seconds_aws_str = "2025-04-01 09:29:55+04:00"; + auto out2 = SignedUrlCache::num_seconds_until_expiration(time_minus_five_seconds_aws_str, current_time); + CPPUNIT_ASSERT_MESSAGE("Expiration date in the past should return 0s `" + to_string(out2) + "`", out2 == 0); + + std::string expiration_time_past("1980-07-16 18:40:58+04:00"); + auto out3 = SignedUrlCache::num_seconds_until_expiration(expiration_time_past); + CPPUNIT_ASSERT_MESSAGE("Expiration time in past should return 0s `" + to_string(out3) + "`", out3 == 0); + + std::string expiration_time_future("3026-07-16 18:40:58+04:00"); + auto out4 = SignedUrlCache::num_seconds_until_expiration(expiration_time_future); + CPPUNIT_ASSERT_MESSAGE("Expiration time in future should be > 0s `" + to_string(out4) + "`", out4 > 0); + + auto out5 = SignedUrlCache::num_seconds_until_expiration("lil_date"); + CPPUNIT_ASSERT_MESSAGE("Invalid date string should return 0s `" + to_string(out5) + "`", out5 == 0); + } + /* TESTS END */ /*##################################################################################################*/ @@ -352,7 +391,7 @@ CPPUNIT_TEST_SUITE(SignedUrlCacheTest); // Last but not least, test those helper functions CPPUNIT_TEST(split_s3_url_test); - // CPPUNIT_TEST(get_expiration_seconds); + CPPUNIT_TEST(num_seconds_until_expiration_test); CPPUNIT_TEST_SUITE_END(); }; From d56a9cd9aafcb41d71e7e9fe7ce41fd0099650bc Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Fri, 14 Nov 2025 17:34:54 -0500 Subject: [PATCH 30/77] safety first --- .../dmrpp_module/ngap_container/NgapOwnedContainer.cc | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/modules/dmrpp_module/ngap_container/NgapOwnedContainer.cc b/modules/dmrpp_module/ngap_container/NgapOwnedContainer.cc index 44602606a4..16ce4e19ee 100644 --- a/modules/dmrpp_module/ngap_container/NgapOwnedContainer.cc +++ b/modules/dmrpp_module/ngap_container/NgapOwnedContainer.cc @@ -577,16 +577,20 @@ static inline bool is_eq(const char *value, const char *key) { NgapApi::DataAccessUrls NgapOwnedContainer::extract_s3_data_urls_from_dmrpp(const string &dmrpp_string) { // If the dmrpp is invalid, we will have hit a failure before now---so we can assume // it's generally safe---so do basically no additional safety checking + string href_attr; + string s3_attr; + string s3credentials_attr; // Load the xml document pugi::xml_document result_xml_doc; pugi::xml_parse_result result = result_xml_doc.load_string(dmrpp_string.c_str(), pugi::parse_default | pugi::parse_ws_pcdata_single); + if (!result) { + // It would be SO surprising to end up here! Nonetheless, if we do, handle it gracefully. + return tie(href_attr, s3_attr, s3credentials_attr); + } auto xml_root_node = result_xml_doc.first_child(); // Pull the expected data values from the xml - string href_attr; - string s3_attr; - string s3credentials_attr; for (xml_attribute attr = xml_root_node.first_attribute(); attr; attr = attr.next_attribute()) { if (is_eq(attr.name(), "dmrpp:href")) { href_attr = attr.value(); From 487d098753ca3d79d7fa1a6022c5bed738dc684b Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Mon, 17 Nov 2025 11:03:15 -0500 Subject: [PATCH 31/77] add docstring remove todo --- .../dmrpp_module/ngap_container/NgapOwnedContainer.cc | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/modules/dmrpp_module/ngap_container/NgapOwnedContainer.cc b/modules/dmrpp_module/ngap_container/NgapOwnedContainer.cc index 16ce4e19ee..37f45485a9 100644 --- a/modules/dmrpp_module/ngap_container/NgapOwnedContainer.cc +++ b/modules/dmrpp_module/ngap_container/NgapOwnedContainer.cc @@ -572,8 +572,13 @@ static inline bool is_eq(const char *value, const char *key) { return strcmp(value, key) == 0; } -// TODO-docstring -// Implementation borrowed from DMZ::process_dataset +/** + * @brief parse a DMR++ to retrieve the tuple of urls required for NGAP's + * S3 data access. + * + * Implementation adapted from `DMZ::process_dataset`. + * @param dmrpp_string DMR++ + */ NgapApi::DataAccessUrls NgapOwnedContainer::extract_s3_data_urls_from_dmrpp(const string &dmrpp_string) { // If the dmrpp is invalid, we will have hit a failure before now---so we can assume // it's generally safe---so do basically no additional safety checking From 96a2a10210b7c1f6b68f137ba60346a93a253ca6 Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Mon, 17 Nov 2025 11:30:08 -0500 Subject: [PATCH 32/77] move warning --- aws/unit-tests/SignedUrlCacheTest.cc | 4 ---- http/EffectiveUrlCache.cc | 1 + modules/dmrpp_module/Chunk.cc | 1 - 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/aws/unit-tests/SignedUrlCacheTest.cc b/aws/unit-tests/SignedUrlCacheTest.cc index 6b5cdf4192..1182912b2c 100644 --- a/aws/unit-tests/SignedUrlCacheTest.cc +++ b/aws/unit-tests/SignedUrlCacheTest.cc @@ -329,10 +329,6 @@ class SignedUrlCacheTest : public CppUnit::TestFixture { std::tm timestamp_time = {}; auto time_parse_result = strptime(datetime_string.c_str(), "%F %T%z", ×tamp_time); - // auto foo = std::mktime(×tamp_time); - // auto bar = std::chrono::system_clock::from_time_t(foo); - // return bar; - return std::chrono::system_clock::from_time_t(std::mktime(×tamp_time)); } diff --git a/http/EffectiveUrlCache.cc b/http/EffectiveUrlCache.cc index ef318a442d..8a7d6643f4 100644 --- a/http/EffectiveUrlCache.cc +++ b/http/EffectiveUrlCache.cc @@ -116,6 +116,7 @@ shared_ptr EffectiveUrlCache::get_effective_url(shared_ptr // It not found or expired, (re)load. if (retrieve_and_cache) { BESDEBUG(MODULE, prolog << "Acquiring effective URL for " << source_url->str() << endl); + INFO_LOG(prolog + "DEPRECATION WARNING: Acquiring effective URL via redirects."); { BES_STOPWATCH_START(MODULE_TIMER, prolog + "Retrieve and cache effective url for source url: " + source_url->str()); try { diff --git a/modules/dmrpp_module/Chunk.cc b/modules/dmrpp_module/Chunk.cc index 3306aa573b..822f0d888a 100644 --- a/modules/dmrpp_module/Chunk.cc +++ b/modules/dmrpp_module/Chunk.cc @@ -1366,7 +1366,6 @@ std::shared_ptr Chunk::get_data_url() const { std::shared_ptr url = SignedUrlCache::TheCache()->get_signed_url(d_data_url); if (url == nullptr) { - INFO_LOG(prolog + "Failed to locally sign url; constructing effective_url via redirects."); url = EffectiveUrlCache::TheCache()->get_effective_url(d_data_url); } BESDEBUG(MODULE, prolog << "Using data_url: " << url->str() << endl); From 14c01e30cf25eee9badc5559eb6c4eabe91fae24 Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Mon, 17 Nov 2025 13:22:00 -0500 Subject: [PATCH 33/77] temp disable tests to understand travis failure --- aws/SignedUrlCache.cc | 2 ++ aws/unit-tests/SignedUrlCacheTest.cc | 16 ++++++++-------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/aws/SignedUrlCache.cc b/aws/SignedUrlCache.cc index 29fdd04dbf..4c34311790 100644 --- a/aws/SignedUrlCache.cc +++ b/aws/SignedUrlCache.cc @@ -263,6 +263,8 @@ shared_ptr SignedUrlCache::get_s3credentials_f std::string s3credentials_json_string; try { BES_PROFILE_TIMING(string("Request s3 credentials from TEA - ") + s3credentials_url); + + // Note: this http_get call internally adds edl auth headers, if available curl::http_get(s3credentials_url, s3credentials_json_string); } catch (http::HttpError &http_error) { diff --git a/aws/unit-tests/SignedUrlCacheTest.cc b/aws/unit-tests/SignedUrlCacheTest.cc index 1182912b2c..e8168028a0 100644 --- a/aws/unit-tests/SignedUrlCacheTest.cc +++ b/aws/unit-tests/SignedUrlCacheTest.cc @@ -70,7 +70,7 @@ class SignedUrlCacheTest : public CppUnit::TestFixture { TheBESKeys::ConfigFile = bes_conf; // Reset to same starting point every time - // (It's a singleton so resetting it is important for test determinism) + // (It's a singleton so resetting it is important for testing determinism) SignedUrlCache *theCache = SignedUrlCache::TheCache(); theCache->d_enabled = -1; @@ -372,13 +372,13 @@ CPPUNIT_TEST_SUITE(SignedUrlCacheTest); CPPUNIT_TEST(dump_test); // Test behavior specific to SignedUrlCache: - CPPUNIT_TEST(is_timestamp_after_now_test); - CPPUNIT_TEST(retrieve_cached_s3credentials_test); - CPPUNIT_TEST(retrieve_cached_s3credentials_expired_credentials_test); - CPPUNIT_TEST(extract_s3_credentials_from_response_json_test); - CPPUNIT_TEST(cache_signed_url_components_test); - CPPUNIT_TEST(retrieve_cached_signed_url_components_test); - CPPUNIT_TEST(get_s3credentials_from_endpoint_test); + // CPPUNIT_TEST(is_timestamp_after_now_test); + // CPPUNIT_TEST(retrieve_cached_s3credentials_test); + // CPPUNIT_TEST(retrieve_cached_s3credentials_expired_credentials_test); + // CPPUNIT_TEST(extract_s3_credentials_from_response_json_test); + // CPPUNIT_TEST(cache_signed_url_components_test); + // CPPUNIT_TEST(retrieve_cached_signed_url_components_test); + // CPPUNIT_TEST(get_s3credentials_from_endpoint_test); // ...and, specifically, the signing itself: // TODO-future: will add/update these tests once signing behavior is implemented From 939c1f2e0fec997ba5eab8925c5872452e5a314e Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Mon, 17 Nov 2025 13:39:04 -0500 Subject: [PATCH 34/77] commit out all signedurlcachetests to see if it still fails --- aws/unit-tests/SignedUrlCacheTest.cc | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/aws/unit-tests/SignedUrlCacheTest.cc b/aws/unit-tests/SignedUrlCacheTest.cc index e8168028a0..a035e4937a 100644 --- a/aws/unit-tests/SignedUrlCacheTest.cc +++ b/aws/unit-tests/SignedUrlCacheTest.cc @@ -327,8 +327,7 @@ class SignedUrlCacheTest : public CppUnit::TestFixture { std::chrono::system_clock::time_point parse_as_time_point(const std::string& datetime_string) { std::tm timestamp_time = {}; - auto time_parse_result = strptime(datetime_string.c_str(), "%F %T%z", ×tamp_time); - + strptime(datetime_string.c_str(), "%F %T%z", ×tamp_time); return std::chrono::system_clock::from_time_t(std::mktime(×tamp_time)); } @@ -366,10 +365,10 @@ class SignedUrlCacheTest : public CppUnit::TestFixture { CPPUNIT_TEST_SUITE(SignedUrlCacheTest); // Test behavior analogous to that of the EffectiveUrlCache: - CPPUNIT_TEST(get_cached_signed_url_test); - CPPUNIT_TEST(is_cache_disabled_test); - CPPUNIT_TEST(set_skip_regex_test); - CPPUNIT_TEST(dump_test); + // CPPUNIT_TEST(get_cached_signed_url_test); + // CPPUNIT_TEST(is_cache_disabled_test); + // CPPUNIT_TEST(set_skip_regex_test); + // CPPUNIT_TEST(dump_test); // Test behavior specific to SignedUrlCache: // CPPUNIT_TEST(is_timestamp_after_now_test); @@ -386,8 +385,8 @@ CPPUNIT_TEST_SUITE(SignedUrlCacheTest); // - get_signed_url // Last but not least, test those helper functions - CPPUNIT_TEST(split_s3_url_test); - CPPUNIT_TEST(num_seconds_until_expiration_test); + // CPPUNIT_TEST(split_s3_url_test); + // CPPUNIT_TEST(num_seconds_until_expiration_test); CPPUNIT_TEST_SUITE_END(); }; From 08057a9bee687a0cd76cb3182ae0684eab79f102 Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Mon, 17 Nov 2025 14:06:32 -0500 Subject: [PATCH 35/77] fix linking in ngap_container --- modules/dmrpp_module/ngap_container/unit-tests/Makefile.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/dmrpp_module/ngap_container/unit-tests/Makefile.am b/modules/dmrpp_module/ngap_container/unit-tests/Makefile.am index 29673eed03..376d4e2c45 100644 --- a/modules/dmrpp_module/ngap_container/unit-tests/Makefile.am +++ b/modules/dmrpp_module/ngap_container/unit-tests/Makefile.am @@ -4,7 +4,7 @@ AUTOMAKE_OPTIONS = foreign AM_CPPFLAGS = -I$(top_srcdir) -I$(top_srcdir)/modules/common \ -I$(top_srcdir)/modules/dmrpp_module/ngap_container -I$(top_srcdir)/dispatch \ - -I$(top_srcdir)/dap -I$(top_srcdir)/http $(DAP_CFLAGS) + -I$(top_srcdir)/dap -I$(top_srcdir)/http -I$(top_srcdir)/aws $(DAP_CFLAGS) LIBADD = -L$(top_builddir)/modules/common -lmodules_common -L$(builddir)/../ -lngap \ $(BES_DISPATCH_LIB) $(top_builddir)/dap/.libs/libdap_module.a $(BES_HTTP_LIB) \ From 8d77254fdaf51a8e744e26f8393eb22129bf228c Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Mon, 17 Nov 2025 16:15:48 -0500 Subject: [PATCH 36/77] wip linker --- modules/dmrpp_module/Makefile.am | 10 +++++++- .../dmrpp_module/ngap_container/Makefile.am | 16 +++++++++---- .../ngap_container/unit-tests/Makefile.am | 23 +++++++++++++++---- 3 files changed, 40 insertions(+), 9 deletions(-) diff --git a/modules/dmrpp_module/Makefile.am b/modules/dmrpp_module/Makefile.am index c810e44319..9e36abfbb4 100644 --- a/modules/dmrpp_module/Makefile.am +++ b/modules/dmrpp_module/Makefile.am @@ -31,6 +31,8 @@ CXXFLAGS_DEBUG = -g3 -O0 -Wall -W -Wcast-align AM_LDFLAGS = -L$(aws_libdir) include $(top_srcdir)/coverage.mk +LDADD = $(top_builddir)/aws/libbes_aws.la $(aws_libs) + SUBDIRS = ngap_container dmrpp_transmitter build_dmrpp_h4 . unit-tests tests data tests_build_dmrpp get_dmrpp lib_besdir=$(libdir)/bes @@ -77,13 +79,19 @@ build_dmrpp_SOURCES = $(BES_SRCS) $(BES_HDRS) DmrppRequestHandler.cc DmrppReques build_dmrpp_LDFLAGS = $(top_builddir)/dap/.libs/libdap_module.a build_dmrpp_LDADD = $(BES_DISPATCH_LIB) $(BES_HTTP_LIB) $(H5_LDFLAGS) \ $(H5_LIBS) $(DAP_SERVER_LIBS) $(DAP_CLIENT_LIBS) $(OPENSSL_LDFLAGS) $(OPENSSL_LIBS) \ - $(top_builddir)/aws/libbes_aws.la $(aws_libs) \ + $(LDADD) \ $(XML2_LIBS) $(BYTESWAP_LIBS) -lz if DARWIN AM_LDFLAGS += -Wl,-rpath,$(aws_libdir) -Wl,-rpath,@loader_path else AM_LDFLAGS += -Wl,-rpath,$(aws_libdir) -Wl,-rpath,'$$ORIGIN' + ifeq ($(strip $(aws_libdir64)),) + $(info "aws_libdir64 not set, skipping") + else + AM_LDFLAGS += -L$(aws_libdir64) + AM_LDFLAGS += -Wl,-rpath,$(aws_libdir64) + endif endif # jhrg 6/2/23 $(BES_EXTRA_LIBS) diff --git a/modules/dmrpp_module/ngap_container/Makefile.am b/modules/dmrpp_module/ngap_container/Makefile.am index 983a4ec8f3..256b21ae9a 100644 --- a/modules/dmrpp_module/ngap_container/Makefile.am +++ b/modules/dmrpp_module/ngap_container/Makefile.am @@ -7,21 +7,29 @@ AUTOMAKE_OPTIONS = foreign ACLOCAL_AMFLAGS = -I conf AM_CPPFLAGS = -I$(top_srcdir) -I$(top_srcdir)/dispatch -I$(top_srcdir)/dap -I$(top_srcdir)/xmlcommand \ --I$(top_srcdir)/http -I$(top_srcdir)/aws -I$(top_srcdir)/pugixml/src $(DAP_CFLAGS) +-I$(top_srcdir)/http -I$(top_srcdir)/pugixml/src -I$(top_srcdir)/aws $(DAP_CFLAGS) -LIBADD = +LIBADD = $(top_builddir)/aws/libbes_aws.la AM_CXXFLAGS= -AM_LDFLAGS = +AM_LDFLAGS = -L$(aws_libdir) include $(top_srcdir)/coverage.mk +if DARWIN +AM_LDFLAGS += -Wl,-rpath,$(aws_libdir) -Wl,-rpath,@loader_path +else +AM_LDFLAGS += -Wl,-rpath,$(aws_libdir) -Wl,-rpath,'$$ORIGIN' +endif + SUBDIRS = . unit-tests noinst_LTLIBRARIES = libngap.la -libngap_la_SOURCES = $(NGAP_SRC) $(NGAP_HDR) +libngap_la_SOURCES = $(NGAP_SRC) $(NGAP_HDR) libngap_la_LDFLAGS = libngap_la_LIBADD = $(LIBADD) +LDADD = $(aws_libs) + NGAP_SRC = NgapOwnedContainer.cc NgapOwnedContainerStorage.cc NgapApi.cc NGAP_HDR = NgapNames.h NgapOwnedContainer.h NgapOwnedContainerStorage.h \ diff --git a/modules/dmrpp_module/ngap_container/unit-tests/Makefile.am b/modules/dmrpp_module/ngap_container/unit-tests/Makefile.am index 376d4e2c45..56177f5979 100644 --- a/modules/dmrpp_module/ngap_container/unit-tests/Makefile.am +++ b/modules/dmrpp_module/ngap_container/unit-tests/Makefile.am @@ -3,12 +3,12 @@ AUTOMAKE_OPTIONS = foreign AM_CPPFLAGS = -I$(top_srcdir) -I$(top_srcdir)/modules/common \ - -I$(top_srcdir)/modules/dmrpp_module/ngap_container -I$(top_srcdir)/dispatch \ - -I$(top_srcdir)/dap -I$(top_srcdir)/http -I$(top_srcdir)/aws $(DAP_CFLAGS) + -I$(top_srcdir)/modules/dmrpp_module/ngap_container -I$(top_srcdir)/aws -I$(top_srcdir)/dispatch \ + -I$(top_srcdir)/dap -I$(top_srcdir)/http $(DAP_CFLAGS) LIBADD = -L$(top_builddir)/modules/common -lmodules_common -L$(builddir)/../ -lngap \ $(BES_DISPATCH_LIB) $(top_builddir)/dap/.libs/libdap_module.a $(BES_HTTP_LIB) \ - $(DAP_SERVER_LIBS) $(DAP_CLIENT_LIBS) + $(DAP_SERVER_LIBS) $(DAP_CLIENT_LIBS) -L$(top_builddir)/aws/libbes_aws.la # $(BES_DAP_LIB) Replace with static library explicitly referenced. jhrg 4/29/24 @@ -22,9 +22,24 @@ endif CXXFLAGS_DEBUG = -g3 -O0 -Wall -W -Wcast-align -Werror AM_CXXFLAGS= -AM_LDFLAGS = +AM_LDFLAGS = -L$(aws_libdir) include $(top_srcdir)/coverage.mk +LDADD = $(top_builddir)/aws/libbes_aws.la $(aws_libs) + +if DARWIN +AM_LDFLAGS += -Wl,-rpath,$(aws_libdir) -Wl,-rpath,@loader_path +else +AM_LDFLAGS += -Wl,-rpath,$(aws_libdir) -Wl,-rpath,'$$ORIGIN' + ifeq ($(strip $(aws_libdir64)),) + $(info "aws_libdir64 not set, skipping") + else + AM_LDFLAGS += -L$(aws_libdir64) + AM_LDFLAGS += -Wl,-rpath,$(aws_libdir64) + endif +endif + + DISTCLEANFILES = test_config.h *.Po CLEANFILES = *.dbg *.log From 39cbd6a268616c6dc9707cf7b2c456f9e9b8f253 Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Mon, 17 Nov 2025 17:51:55 -0500 Subject: [PATCH 37/77] Update mkchk script to show errors where build failed --- mkchk | 2 +- mkdchk | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mkchk b/mkchk index 86589ecb19..8df2170637 100755 --- a/mkchk +++ b/mkchk @@ -25,4 +25,4 @@ then fi -make -k -j20 check 2>&1 | tee mk.log | grep -e "Making check in" -e "FAILED" -e "^FAIL: " +make -k -j20 check 2>&1 | tee mk.log | grep -e "Making check in" -e "FAILED" -e "^FAIL: " -e " Error 2" diff --git a/mkdchk b/mkdchk index 089d95aa7d..101523f965 100755 --- a/mkdchk +++ b/mkdchk @@ -1,3 +1,3 @@ #!/bin/bash -make -k -j20 GZIP_ENV=--fast distcheck 2>&1 | tee mk.log | grep -e "Making check in" -e "FAILED" +make -k -j20 GZIP_ENV=--fast distcheck 2>&1 | tee mk.log | grep -e "Making check in" -e "FAILED" -e "^FAIL: " -e " Error 2" From 1b55513e1cf7fcb6e3f4504aadf4158c9ed0913c Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Mon, 17 Nov 2025 17:10:06 -0500 Subject: [PATCH 38/77] add test output to .gitignore --- dispatch/unit-tests/.gitignore | 1 + modules/dmrpp_module/.gitignore | 12 ++++++++++++ 2 files changed, 13 insertions(+) diff --git a/dispatch/unit-tests/.gitignore b/dispatch/unit-tests/.gitignore index 2fa2daafc6..fb13d06091 100644 --- a/dispatch/unit-tests/.gitignore +++ b/dispatch/unit-tests/.gitignore @@ -2,6 +2,7 @@ cacheT containerT lockT uncompressT +cache /RemoteAccessTest /AllowedHostsTest diff --git a/modules/dmrpp_module/.gitignore b/modules/dmrpp_module/.gitignore index 6910d08f30..8548c56472 100644 --- a/modules/dmrpp_module/.gitignore +++ b/modules/dmrpp_module/.gitignore @@ -20,9 +20,21 @@ /unit-tests/Chunk.Tpo /unit-tests/CredentialsManager.Tpo /unit-tests/DmrppRequestHandler.Tpo +get_dmrpp/unit-tests/gen_dmrpp_side_car +get_dmrpp/unit-tests/get_dmrpp_h5 +get_dmrpp/unit-tests/get_hdf_side_car +get_dmrpp/unit-tests/grid_1_2d.h5.dmr +get_dmrpp/unit-tests/grid_1_2d.h5.dmr.bescmd +get_dmrpp/unit-tests/grid_1_2d.hdf.dmr +get_dmrpp/unit-tests/grid_1_2d.hdf.dmr.bescmd +get_dmrpp/unit-tests/grid_2_2d_ps.hdf.dmr +get_dmrpp/unit-tests/grid_2_2d_ps.hdf.dmr.bescmd +get_dmrpp/unit-tests/grid_2_2d_sin.h5.dmr +get_dmrpp/unit-tests/grid_2_2d_sin.h5.dmr.bescmd /tests_build_dmrpp/testsuite_dmrpp_tests /tests_build_dmrpp/test_bes.conf +/tests/testsuite_return_as_dmrpp /writer/h5common.cc /writer/h5common.h From 77085765b2c780c52b544ee0702d6c7e867e0d5a Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Mon, 17 Nov 2025 18:42:10 -0500 Subject: [PATCH 39/77] fix ngap_container linker --- aws/unit-tests/Makefile.am | 2 +- modules/dmrpp_module/Makefile.am | 2 +- modules/dmrpp_module/build_dmrpp_h4/Makefile.am | 2 +- modules/dmrpp_module/ngap_container/Makefile.am | 6 ++---- .../dmrpp_module/ngap_container/unit-tests/Makefile.am | 10 +++++----- 5 files changed, 10 insertions(+), 12 deletions(-) diff --git a/aws/unit-tests/Makefile.am b/aws/unit-tests/Makefile.am index ccc781e3ac..a335e2f31e 100644 --- a/aws/unit-tests/Makefile.am +++ b/aws/unit-tests/Makefile.am @@ -11,7 +11,7 @@ AM_LDFLAGS = -L$(aws_libdir) # like bin_PROGRAMS. # NB: aws_libs is set in configure.ac. jhrg 10/16/25 -LDADD = $(top_builddir)/dispatch/libbes_dispatch.la $(top_builddir)/aws/libbes_aws.la $(aws_libs) $(BES_HTTP_LIB) +LDADD = $(top_builddir)/dispatch/libbes_dispatch.la $(abs_top_builddir)/aws/libbes_aws.la $(aws_libs) $(BES_HTTP_LIB) # aws_libdir is set in configure.ac. jhrg 10/16/25 # aws_libdir64 is also set in configure.ac sbl 11/14/25 diff --git a/modules/dmrpp_module/Makefile.am b/modules/dmrpp_module/Makefile.am index 9e36abfbb4..86dbb6858c 100644 --- a/modules/dmrpp_module/Makefile.am +++ b/modules/dmrpp_module/Makefile.am @@ -31,7 +31,7 @@ CXXFLAGS_DEBUG = -g3 -O0 -Wall -W -Wcast-align AM_LDFLAGS = -L$(aws_libdir) include $(top_srcdir)/coverage.mk -LDADD = $(top_builddir)/aws/libbes_aws.la $(aws_libs) +LDADD = $(abs_top_builddir)/aws/libbes_aws.la $(aws_libs) SUBDIRS = ngap_container dmrpp_transmitter build_dmrpp_h4 . unit-tests tests data tests_build_dmrpp get_dmrpp diff --git a/modules/dmrpp_module/build_dmrpp_h4/Makefile.am b/modules/dmrpp_module/build_dmrpp_h4/Makefile.am index 7ae46f0202..dd0c82da00 100644 --- a/modules/dmrpp_module/build_dmrpp_h4/Makefile.am +++ b/modules/dmrpp_module/build_dmrpp_h4/Makefile.am @@ -51,7 +51,7 @@ build_dmrpp_h4_LDFLAGS = $(HDFEOS2_LDFLAGS) $(HDF4_LDFLAGS) $(BES_DAP_LIB_LDFLAG # jhrg 12/18/23 $(top_builddir)/dap/.libs/libdap_module.a build_dmrpp_h4_LDADD = $(BES_DISPATCH_LIB) $(BES_HTTP_LIB) $(DAP_SERVER_LIBS) $(DAP_CLIENT_LIBS) \ - $(OPENSSL_LDFLAGS) $(OPENSSL_LIBS) $(XML2_LIBS) $(BYTESWAP_LIBS) $(HDFEOS2_LIBS) $(HDF4_LIBS) $(top_builddir)/aws/libbes_aws.la $(aws_libs) + $(OPENSSL_LDFLAGS) $(OPENSSL_LIBS) $(XML2_LIBS) $(BYTESWAP_LIBS) $(HDFEOS2_LIBS) $(HDF4_LIBS) $(abs_top_builddir)/aws/libbes_aws.la $(aws_libs) if DARWIN AM_LDFLAGS += -Wl,-rpath,$(aws_libdir) -Wl,-rpath,@loader_path diff --git a/modules/dmrpp_module/ngap_container/Makefile.am b/modules/dmrpp_module/ngap_container/Makefile.am index 256b21ae9a..9f378d6f22 100644 --- a/modules/dmrpp_module/ngap_container/Makefile.am +++ b/modules/dmrpp_module/ngap_container/Makefile.am @@ -9,7 +9,7 @@ ACLOCAL_AMFLAGS = -I conf AM_CPPFLAGS = -I$(top_srcdir) -I$(top_srcdir)/dispatch -I$(top_srcdir)/dap -I$(top_srcdir)/xmlcommand \ -I$(top_srcdir)/http -I$(top_srcdir)/pugixml/src -I$(top_srcdir)/aws $(DAP_CFLAGS) -LIBADD = $(top_builddir)/aws/libbes_aws.la +LDADD = $(abs_top_builddir)/aws/libbes_aws.la $(aws_libs) AM_CXXFLAGS= AM_LDFLAGS = -L$(aws_libdir) @@ -26,9 +26,7 @@ SUBDIRS = . unit-tests noinst_LTLIBRARIES = libngap.la libngap_la_SOURCES = $(NGAP_SRC) $(NGAP_HDR) libngap_la_LDFLAGS = -libngap_la_LIBADD = $(LIBADD) - -LDADD = $(aws_libs) +libngap_la_LIBADD = $(LDADD) NGAP_SRC = NgapOwnedContainer.cc NgapOwnedContainerStorage.cc NgapApi.cc diff --git a/modules/dmrpp_module/ngap_container/unit-tests/Makefile.am b/modules/dmrpp_module/ngap_container/unit-tests/Makefile.am index 56177f5979..36bee2bcfb 100644 --- a/modules/dmrpp_module/ngap_container/unit-tests/Makefile.am +++ b/modules/dmrpp_module/ngap_container/unit-tests/Makefile.am @@ -8,7 +8,7 @@ AM_CPPFLAGS = -I$(top_srcdir) -I$(top_srcdir)/modules/common \ LIBADD = -L$(top_builddir)/modules/common -lmodules_common -L$(builddir)/../ -lngap \ $(BES_DISPATCH_LIB) $(top_builddir)/dap/.libs/libdap_module.a $(BES_HTTP_LIB) \ - $(DAP_SERVER_LIBS) $(DAP_CLIENT_LIBS) -L$(top_builddir)/aws/libbes_aws.la + $(DAP_SERVER_LIBS) $(DAP_CLIENT_LIBS) # $(BES_DAP_LIB) Replace with static library explicitly referenced. jhrg 4/29/24 @@ -25,7 +25,7 @@ AM_CXXFLAGS= AM_LDFLAGS = -L$(aws_libdir) include $(top_srcdir)/coverage.mk -LDADD = $(top_builddir)/aws/libbes_aws.la $(aws_libs) +LDADD = $(abs_top_builddir)/aws/libbes_aws.la $(aws_libs) if DARWIN AM_LDFLAGS += -Wl,-rpath,$(aws_libdir) -Wl,-rpath,@loader_path @@ -115,11 +115,11 @@ clean-local: test ! -d $(builddir)/owned-cache || rm -rf $(builddir)/owned-cache MemoryCacheTest_SOURCES = MemoryCacheTest.cc -MemoryCacheTest_LDADD = $(LIBADD) +MemoryCacheTest_LDADD = $(LIBADD) $(LDADD) NgapApiTest_SOURCES = NgapApiTest.cc -NgapApiTest_LDADD = $(LIBADD) +NgapApiTest_LDADD = $(LIBADD) $(LDADD) NgapOwnedContainerTest_SOURCES = NgapOwnedContainerTest.cc -NgapOwnedContainerTest_LDADD = $(LIBADD) +NgapOwnedContainerTest_LDADD = $(LIBADD) $(LDADD) From a1de61afeb56a535a6bd787a59215432a9b284d4 Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Mon, 17 Nov 2025 19:32:23 -0500 Subject: [PATCH 40/77] Fix dmrpp_module/unit tests linking --- modules/dmrpp_module/Makefile.am | 4 +++- modules/dmrpp_module/unit-tests/Makefile.am | 17 +++++++++++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/modules/dmrpp_module/Makefile.am b/modules/dmrpp_module/Makefile.am index 86dbb6858c..13974c4686 100644 --- a/modules/dmrpp_module/Makefile.am +++ b/modules/dmrpp_module/Makefile.am @@ -99,15 +99,17 @@ endif # check_dmrpp config check_dmrpp_CPPFLAGS = $(AM_CPPFLAGS) $(H5_CPPFLAGS) check_dmrpp_SOURCES = check_dmrpp.cc +check_dmrpp_LDADD = $(LDADD) # merge_dmrpp config merge_dmrpp_CPPFLAGS = $(AM_CPPFLAGS) merge_dmrpp_SOURCES = merge_dmrpp.cc +merge_dmrpp_LDADD = $(LDADD) # reduce_mdf config reduce_mdf_CPPFLAGS = $(AM_CPPFLAGS) reduce_mdf_SOURCES = reduce_mdf.cc -reduce_mdf_LDADD = $(OPENSSL_LDFLAGS) $(OPENSSL_LIBS) -lz +reduce_mdf_LDADD = $(OPENSSL_LDFLAGS) $(LDADD) $(OPENSSL_LIBS) -lz EXTRA_PROGRAMS = diff --git a/modules/dmrpp_module/unit-tests/Makefile.am b/modules/dmrpp_module/unit-tests/Makefile.am index b9cad3c4af..ab69ae0418 100644 --- a/modules/dmrpp_module/unit-tests/Makefile.am +++ b/modules/dmrpp_module/unit-tests/Makefile.am @@ -10,11 +10,12 @@ AUTOMAKE_OPTIONS = foreign subdir-objects AM_CPPFLAGS = $(H5_CPPFLAGS) -I$(top_srcdir) -I$(top_srcdir)/dispatch -I$(top_srcdir)/dap \ -I$(top_srcdir)/http -I$(top_srcdir)/pugixml/src -I$(top_srcdir)/modules/common \ - -I$(top_srcdir)/modules/dmrpp_module $(DAP_CFLAGS) + -I$(top_srcdir)/modules/dmrpp_module -I$(top_srcdir)/aws $(DAP_CFLAGS) # Added -lz for ubuntu LIBADD = $(BES_DISPATCH_LIB) $(top_builddir)/dap/.libs/libdap_module.a $(BES_HTTP_LIB) \ -L$(top_builddir)/modules/common -lmodules_common $(H5_LDFLAGS) \ + $(abs_top_builddir)/aws/libbes_aws.la $(aws_libs) \ $(H5_LIBS) $(DAP_SERVER_LIBS) $(DAP_CLIENT_LIBS) $(OPENSSL_LIBS) $(XML2_LIBS) -lz # jhrg 6/2/23 $(BES_EXTRA_LIBS) @@ -29,9 +30,21 @@ endif CXXFLAGS_DEBUG = -g3 -O0 -Wall -Wcast-align AM_CXXFLAGS = -Wno-vla-extension -Wno-inconsistent-missing-override -Wno-unused-variable -AM_LDFLAGS = +AM_LDFLAGS = -L$(aws_libdir) include $(top_srcdir)/coverage.mk +if DARWIN +AM_LDFLAGS += -Wl,-rpath,$(aws_libdir) -Wl,-rpath,@loader_path +else +AM_LDFLAGS += -Wl,-rpath,$(aws_libdir) -Wl,-rpath,'$$ORIGIN' + ifeq ($(strip $(aws_libdir64)),) + $(info "aws_libdir64 not set, skipping") + else + AM_LDFLAGS += -L$(aws_libdir64) + AM_LDFLAGS += -Wl,-rpath,$(aws_libdir64) + endif +endif + # This determines what gets built by make check check_PROGRAMS = $(UNIT_TESTS) From 14261687ca39821dcdcfb5a1cf0f1d001604934f Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Tue, 18 Nov 2025 13:06:55 -0500 Subject: [PATCH 41/77] linker changes; can now make bes, pass all unit tests w/out linker failure --- modules/dmrpp_module/Makefile.am | 29 +++++++++---------- .../dmrpp_module/build_dmrpp_h4/Makefile.am | 16 +++++----- .../dmrpp_module/ngap_container/Makefile.am | 3 +- 3 files changed, 25 insertions(+), 23 deletions(-) diff --git a/modules/dmrpp_module/Makefile.am b/modules/dmrpp_module/Makefile.am index 13974c4686..a2219a1c63 100644 --- a/modules/dmrpp_module/Makefile.am +++ b/modules/dmrpp_module/Makefile.am @@ -33,6 +33,18 @@ include $(top_srcdir)/coverage.mk LDADD = $(abs_top_builddir)/aws/libbes_aws.la $(aws_libs) +if DARWIN +AM_LDFLAGS += -Wl,-rpath,$(aws_libdir) -Wl,-rpath,@loader_path +else +AM_LDFLAGS += -Wl,-rpath,$(aws_libdir) -Wl,-rpath,'$$ORIGIN' + ifeq ($(strip $(aws_libdir64)),) + $(info "aws_libdir64 not set, skipping") + else + AM_LDFLAGS += -L$(aws_libdir64) + AM_LDFLAGS += -Wl,-rpath,$(aws_libdir64) + endif +endif + SUBDIRS = ngap_container dmrpp_transmitter build_dmrpp_h4 . unit-tests tests data tests_build_dmrpp get_dmrpp lib_besdir=$(libdir)/bes @@ -59,7 +71,7 @@ DMRPP_MODULE = DmrppModule.cc DmrppRequestHandler.cc DmrppModule.h DmrppRequestH libdmrpp_module_la_SOURCES = $(BES_HDRS) $(BES_SRCS) $(DMRPP_MODULE) libdmrpp_module_la_LDFLAGS = -avoid-version -module libdmrpp_module_la_LIBADD = -L$(builddir)/ngap_container -lngap $(BES_DISPATCH_LIB) \ - $(BES_HTTP_LIB) $(DAP_SERVER_LIBS) $(DAP_CLIENT_LIBS) \ + $(BES_HTTP_LIB) $(DAP_SERVER_LIBS) $(DAP_CLIENT_LIBS) $(LDADD) \ $(H5_LDFLAGS) $(H5_LIBS) $(OPENSSL_LDFLAGS) $(OPENSSL_LIBS) -ltest-types \ -Ldmrpp_transmitter -ldmrpp_return_as @@ -79,20 +91,7 @@ build_dmrpp_SOURCES = $(BES_SRCS) $(BES_HDRS) DmrppRequestHandler.cc DmrppReques build_dmrpp_LDFLAGS = $(top_builddir)/dap/.libs/libdap_module.a build_dmrpp_LDADD = $(BES_DISPATCH_LIB) $(BES_HTTP_LIB) $(H5_LDFLAGS) \ $(H5_LIBS) $(DAP_SERVER_LIBS) $(DAP_CLIENT_LIBS) $(OPENSSL_LDFLAGS) $(OPENSSL_LIBS) \ - $(LDADD) \ - $(XML2_LIBS) $(BYTESWAP_LIBS) -lz - -if DARWIN -AM_LDFLAGS += -Wl,-rpath,$(aws_libdir) -Wl,-rpath,@loader_path -else -AM_LDFLAGS += -Wl,-rpath,$(aws_libdir) -Wl,-rpath,'$$ORIGIN' - ifeq ($(strip $(aws_libdir64)),) - $(info "aws_libdir64 not set, skipping") - else - AM_LDFLAGS += -L$(aws_libdir64) - AM_LDFLAGS += -Wl,-rpath,$(aws_libdir64) - endif -endif + $(LDADD) $(XML2_LIBS) $(BYTESWAP_LIBS) -lz # jhrg 6/2/23 $(BES_EXTRA_LIBS) diff --git a/modules/dmrpp_module/build_dmrpp_h4/Makefile.am b/modules/dmrpp_module/build_dmrpp_h4/Makefile.am index dd0c82da00..1881d19cce 100644 --- a/modules/dmrpp_module/build_dmrpp_h4/Makefile.am +++ b/modules/dmrpp_module/build_dmrpp_h4/Makefile.am @@ -22,6 +22,14 @@ AM_CXXFLAGS= AM_LDFLAGS = -L$(aws_libdir) include $(top_srcdir)/coverage.mk +if DARWIN +AM_LDFLAGS += -Wl,-rpath,$(aws_libdir) -Wl,-rpath,@loader_path +else +AM_LDFLAGS += -Wl,-rpath,$(aws_libdir) -Wl,-rpath,'$$ORIGIN' +endif + +LDADD = $(abs_top_builddir)/aws/libbes_aws.la $(aws_libs) + # SUBDIRS = . unit-tests tests # These tests will fail if the DmrApiTest also fails: tests jhrg 6/29/23 @@ -51,13 +59,7 @@ build_dmrpp_h4_LDFLAGS = $(HDFEOS2_LDFLAGS) $(HDF4_LDFLAGS) $(BES_DAP_LIB_LDFLAG # jhrg 12/18/23 $(top_builddir)/dap/.libs/libdap_module.a build_dmrpp_h4_LDADD = $(BES_DISPATCH_LIB) $(BES_HTTP_LIB) $(DAP_SERVER_LIBS) $(DAP_CLIENT_LIBS) \ - $(OPENSSL_LDFLAGS) $(OPENSSL_LIBS) $(XML2_LIBS) $(BYTESWAP_LIBS) $(HDFEOS2_LIBS) $(HDF4_LIBS) $(abs_top_builddir)/aws/libbes_aws.la $(aws_libs) - -if DARWIN -AM_LDFLAGS += -Wl,-rpath,$(aws_libdir) -Wl,-rpath,@loader_path -else -AM_LDFLAGS += -Wl,-rpath,$(aws_libdir) -Wl,-rpath,'$$ORIGIN' -endif + $(OPENSSL_LDFLAGS) $(OPENSSL_LIBS) $(XML2_LIBS) $(BYTESWAP_LIBS) $(HDFEOS2_LIBS) $(HDF4_LIBS) $(LDADD) lib_besdir=$(libdir)/bes diff --git a/modules/dmrpp_module/ngap_container/Makefile.am b/modules/dmrpp_module/ngap_container/Makefile.am index 9f378d6f22..267206691a 100644 --- a/modules/dmrpp_module/ngap_container/Makefile.am +++ b/modules/dmrpp_module/ngap_container/Makefile.am @@ -9,7 +9,8 @@ ACLOCAL_AMFLAGS = -I conf AM_CPPFLAGS = -I$(top_srcdir) -I$(top_srcdir)/dispatch -I$(top_srcdir)/dap -I$(top_srcdir)/xmlcommand \ -I$(top_srcdir)/http -I$(top_srcdir)/pugixml/src -I$(top_srcdir)/aws $(DAP_CFLAGS) -LDADD = $(abs_top_builddir)/aws/libbes_aws.la $(aws_libs) +# LDADD = $(abs_top_builddir)/aws/libbes_aws.la $(aws_libs) #TODO-delete this line +LDADD = $(aws_libs) AM_CXXFLAGS= AM_LDFLAGS = -L$(aws_libdir) From 91cd9d585d765c725b32b9f09e888cea42a2b06f Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Tue, 18 Nov 2025 15:10:29 -0500 Subject: [PATCH 42/77] Link aws to the standalone --- Makefile.am | 6 +----- .../dmrpp_module/ngap_container/Makefile.am | 1 - standalone/Makefile.am | 18 +++++++++++++++--- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/Makefile.am b/Makefile.am index 605519aaae..65720e12ec 100644 --- a/Makefile.am +++ b/Makefile.am @@ -29,11 +29,7 @@ AM_CXXFLAGS= AM_LDFLAGS = include $(top_srcdir)/coverage.mk -SUBDIRS = dispatch xmlcommand ppt server http cmdln standalone docs bin templates hello_world - -if USE_AWS_SDK -SUBDIRS += aws -endif +SUBDIRS = dispatch xmlcommand ppt server http cmdln aws standalone docs bin templates hello_world if LIBDAP SUBDIRS += dap dapreader diff --git a/modules/dmrpp_module/ngap_container/Makefile.am b/modules/dmrpp_module/ngap_container/Makefile.am index 267206691a..6dc27b58bf 100644 --- a/modules/dmrpp_module/ngap_container/Makefile.am +++ b/modules/dmrpp_module/ngap_container/Makefile.am @@ -9,7 +9,6 @@ ACLOCAL_AMFLAGS = -I conf AM_CPPFLAGS = -I$(top_srcdir) -I$(top_srcdir)/dispatch -I$(top_srcdir)/dap -I$(top_srcdir)/xmlcommand \ -I$(top_srcdir)/http -I$(top_srcdir)/pugixml/src -I$(top_srcdir)/aws $(DAP_CFLAGS) -# LDADD = $(abs_top_builddir)/aws/libbes_aws.la $(aws_libs) #TODO-delete this line LDADD = $(aws_libs) AM_CXXFLAGS= diff --git a/standalone/Makefile.am b/standalone/Makefile.am index 400e2bec87..1d1a649d96 100644 --- a/standalone/Makefile.am +++ b/standalone/Makefile.am @@ -3,16 +3,28 @@ AUTOMAKE_OPTIONS = foreign subdir-objects -AM_CPPFLAGS = -I$(top_srcdir)/xmlcommand -I$(top_srcdir)/cmdln -I$(top_srcdir)/dispatch $(XML2_CFLAGS) +AM_CPPFLAGS = -I$(top_srcdir)/xmlcommand -I$(top_srcdir)/cmdln -I$(top_srcdir)/dispatch -I$(top_srcdir)/aws $(XML2_CFLAGS) if BES_DEVELOPER AM_CPPFLAGS += -DBES_DEVELOPER endif AM_CXXFLAGS= -AM_LDFLAGS = +AM_LDFLAGS = -L$(aws_libdir) include $(top_srcdir)/coverage.mk +if DARWIN +AM_LDFLAGS += -Wl,-rpath,$(aws_libdir) -Wl,-rpath,@loader_path +else +AM_LDFLAGS += -Wl,-rpath,$(aws_libdir) -Wl,-rpath,'$$ORIGIN' + ifeq ($(strip $(aws_libdir64)),) + $(info "aws_libdir64 not set, skipping") + else + AM_LDFLAGS += -L$(aws_libdir64) + AM_LDFLAGS += -Wl,-rpath,$(aws_libdir64) + endif +endif + EXTRA_DIST = response-split.py # This installs the script @@ -30,4 +42,4 @@ besstandalone_SOURCES = besstandalone.cc StandAloneApp.cc StandAloneClient.cc \ # This depends on building in bes/cmdln first. It's not a good way to write # teh Makefile.am, but I cannot get distcheck to work otherwise. jhrg 9/8/23 besstandalone_LDADD = $(top_builddir)/cmdln/CmdTranslation.o $(top_builddir)/dispatch/libbes_dispatch.la \ - $(top_builddir)/xmlcommand/libbes_xml_command.la $(READLINE_LIBS) $(XML2_LIBS) + $(top_builddir)/xmlcommand/libbes_xml_command.la $(READLINE_LIBS) $(XML2_LIBS) $(abs_top_builddir)/aws/libbes_aws.la $(aws_libs) From d0a5c6f10c06e8f142d6da673e212cdd1676c633 Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Wed, 19 Nov 2025 12:52:04 -0500 Subject: [PATCH 43/77] fix dmrpp module linking --- modules/dmrpp_module/Makefile.am | 2 +- modules/dmrpp_module/ngap_container/Makefile.am | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/dmrpp_module/Makefile.am b/modules/dmrpp_module/Makefile.am index a2219a1c63..12b3611c38 100644 --- a/modules/dmrpp_module/Makefile.am +++ b/modules/dmrpp_module/Makefile.am @@ -71,7 +71,7 @@ DMRPP_MODULE = DmrppModule.cc DmrppRequestHandler.cc DmrppModule.h DmrppRequestH libdmrpp_module_la_SOURCES = $(BES_HDRS) $(BES_SRCS) $(DMRPP_MODULE) libdmrpp_module_la_LDFLAGS = -avoid-version -module libdmrpp_module_la_LIBADD = -L$(builddir)/ngap_container -lngap $(BES_DISPATCH_LIB) \ - $(BES_HTTP_LIB) $(DAP_SERVER_LIBS) $(DAP_CLIENT_LIBS) $(LDADD) \ + $(BES_HTTP_LIB) $(DAP_SERVER_LIBS) $(DAP_CLIENT_LIBS) \ $(H5_LDFLAGS) $(H5_LIBS) $(OPENSSL_LDFLAGS) $(OPENSSL_LIBS) -ltest-types \ -Ldmrpp_transmitter -ldmrpp_return_as diff --git a/modules/dmrpp_module/ngap_container/Makefile.am b/modules/dmrpp_module/ngap_container/Makefile.am index 6dc27b58bf..9f378d6f22 100644 --- a/modules/dmrpp_module/ngap_container/Makefile.am +++ b/modules/dmrpp_module/ngap_container/Makefile.am @@ -9,7 +9,7 @@ ACLOCAL_AMFLAGS = -I conf AM_CPPFLAGS = -I$(top_srcdir) -I$(top_srcdir)/dispatch -I$(top_srcdir)/dap -I$(top_srcdir)/xmlcommand \ -I$(top_srcdir)/http -I$(top_srcdir)/pugixml/src -I$(top_srcdir)/aws $(DAP_CFLAGS) -LDADD = $(aws_libs) +LDADD = $(abs_top_builddir)/aws/libbes_aws.la $(aws_libs) AM_CXXFLAGS= AM_LDFLAGS = -L$(aws_libdir) From 6b54d8ae4672429929d7ed72a4cc246549b53aee Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Wed, 19 Nov 2025 12:52:40 -0500 Subject: [PATCH 44/77] re-enable signedurlcachetest --- aws/unit-tests/SignedUrlCacheTest.cc | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/aws/unit-tests/SignedUrlCacheTest.cc b/aws/unit-tests/SignedUrlCacheTest.cc index a035e4937a..d2502cb6b1 100644 --- a/aws/unit-tests/SignedUrlCacheTest.cc +++ b/aws/unit-tests/SignedUrlCacheTest.cc @@ -365,19 +365,19 @@ class SignedUrlCacheTest : public CppUnit::TestFixture { CPPUNIT_TEST_SUITE(SignedUrlCacheTest); // Test behavior analogous to that of the EffectiveUrlCache: - // CPPUNIT_TEST(get_cached_signed_url_test); - // CPPUNIT_TEST(is_cache_disabled_test); - // CPPUNIT_TEST(set_skip_regex_test); - // CPPUNIT_TEST(dump_test); + CPPUNIT_TEST(get_cached_signed_url_test); + CPPUNIT_TEST(is_cache_disabled_test); + CPPUNIT_TEST(set_skip_regex_test); + CPPUNIT_TEST(dump_test); // Test behavior specific to SignedUrlCache: - // CPPUNIT_TEST(is_timestamp_after_now_test); - // CPPUNIT_TEST(retrieve_cached_s3credentials_test); - // CPPUNIT_TEST(retrieve_cached_s3credentials_expired_credentials_test); - // CPPUNIT_TEST(extract_s3_credentials_from_response_json_test); - // CPPUNIT_TEST(cache_signed_url_components_test); - // CPPUNIT_TEST(retrieve_cached_signed_url_components_test); - // CPPUNIT_TEST(get_s3credentials_from_endpoint_test); + CPPUNIT_TEST(is_timestamp_after_now_test); + CPPUNIT_TEST(retrieve_cached_s3credentials_test); + CPPUNIT_TEST(retrieve_cached_s3credentials_expired_credentials_test); + CPPUNIT_TEST(extract_s3_credentials_from_response_json_test); + CPPUNIT_TEST(cache_signed_url_components_test); + CPPUNIT_TEST(retrieve_cached_signed_url_components_test); + CPPUNIT_TEST(get_s3credentials_from_endpoint_test); // ...and, specifically, the signing itself: // TODO-future: will add/update these tests once signing behavior is implemented @@ -385,8 +385,8 @@ CPPUNIT_TEST_SUITE(SignedUrlCacheTest); // - get_signed_url // Last but not least, test those helper functions - // CPPUNIT_TEST(split_s3_url_test); - // CPPUNIT_TEST(num_seconds_until_expiration_test); + CPPUNIT_TEST(split_s3_url_test); + CPPUNIT_TEST(num_seconds_until_expiration_test); CPPUNIT_TEST_SUITE_END(); }; From 2f6f2261066828d3364d4ac0a383566ca8010d21 Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Wed, 19 Nov 2025 13:38:54 -0500 Subject: [PATCH 45/77] revert unnecessary abspath change --- aws/unit-tests/Makefile.am | 2 +- modules/dmrpp_module/Makefile.am | 2 +- modules/dmrpp_module/build_dmrpp_h4/Makefile.am | 8 +++++++- modules/dmrpp_module/ngap_container/Makefile.am | 2 +- .../dmrpp_module/ngap_container/unit-tests/Makefile.am | 2 +- modules/dmrpp_module/unit-tests/Makefile.am | 2 +- standalone/Makefile.am | 2 +- 7 files changed, 13 insertions(+), 7 deletions(-) diff --git a/aws/unit-tests/Makefile.am b/aws/unit-tests/Makefile.am index a335e2f31e..ccc781e3ac 100644 --- a/aws/unit-tests/Makefile.am +++ b/aws/unit-tests/Makefile.am @@ -11,7 +11,7 @@ AM_LDFLAGS = -L$(aws_libdir) # like bin_PROGRAMS. # NB: aws_libs is set in configure.ac. jhrg 10/16/25 -LDADD = $(top_builddir)/dispatch/libbes_dispatch.la $(abs_top_builddir)/aws/libbes_aws.la $(aws_libs) $(BES_HTTP_LIB) +LDADD = $(top_builddir)/dispatch/libbes_dispatch.la $(top_builddir)/aws/libbes_aws.la $(aws_libs) $(BES_HTTP_LIB) # aws_libdir is set in configure.ac. jhrg 10/16/25 # aws_libdir64 is also set in configure.ac sbl 11/14/25 diff --git a/modules/dmrpp_module/Makefile.am b/modules/dmrpp_module/Makefile.am index 12b3611c38..0d9debe3d9 100644 --- a/modules/dmrpp_module/Makefile.am +++ b/modules/dmrpp_module/Makefile.am @@ -31,7 +31,7 @@ CXXFLAGS_DEBUG = -g3 -O0 -Wall -W -Wcast-align AM_LDFLAGS = -L$(aws_libdir) include $(top_srcdir)/coverage.mk -LDADD = $(abs_top_builddir)/aws/libbes_aws.la $(aws_libs) +LDADD = $(top_builddir)/aws/libbes_aws.la $(aws_libs) if DARWIN AM_LDFLAGS += -Wl,-rpath,$(aws_libdir) -Wl,-rpath,@loader_path diff --git a/modules/dmrpp_module/build_dmrpp_h4/Makefile.am b/modules/dmrpp_module/build_dmrpp_h4/Makefile.am index 1881d19cce..ad9dd9fd31 100644 --- a/modules/dmrpp_module/build_dmrpp_h4/Makefile.am +++ b/modules/dmrpp_module/build_dmrpp_h4/Makefile.am @@ -26,9 +26,15 @@ if DARWIN AM_LDFLAGS += -Wl,-rpath,$(aws_libdir) -Wl,-rpath,@loader_path else AM_LDFLAGS += -Wl,-rpath,$(aws_libdir) -Wl,-rpath,'$$ORIGIN' + ifeq ($(strip $(aws_libdir64)),) + $(info "aws_libdir64 not set, skipping") + else + AM_LDFLAGS += -L$(aws_libdir64) + AM_LDFLAGS += -Wl,-rpath,$(aws_libdir64) + endif endif -LDADD = $(abs_top_builddir)/aws/libbes_aws.la $(aws_libs) +LDADD = $(top_builddir)/aws/libbes_aws.la $(aws_libs) # SUBDIRS = . unit-tests tests diff --git a/modules/dmrpp_module/ngap_container/Makefile.am b/modules/dmrpp_module/ngap_container/Makefile.am index 9f378d6f22..64b3bc2527 100644 --- a/modules/dmrpp_module/ngap_container/Makefile.am +++ b/modules/dmrpp_module/ngap_container/Makefile.am @@ -9,7 +9,7 @@ ACLOCAL_AMFLAGS = -I conf AM_CPPFLAGS = -I$(top_srcdir) -I$(top_srcdir)/dispatch -I$(top_srcdir)/dap -I$(top_srcdir)/xmlcommand \ -I$(top_srcdir)/http -I$(top_srcdir)/pugixml/src -I$(top_srcdir)/aws $(DAP_CFLAGS) -LDADD = $(abs_top_builddir)/aws/libbes_aws.la $(aws_libs) +LDADD = $(top_builddir)/aws/libbes_aws.la $(aws_libs) AM_CXXFLAGS= AM_LDFLAGS = -L$(aws_libdir) diff --git a/modules/dmrpp_module/ngap_container/unit-tests/Makefile.am b/modules/dmrpp_module/ngap_container/unit-tests/Makefile.am index 36bee2bcfb..5380f2f9f5 100644 --- a/modules/dmrpp_module/ngap_container/unit-tests/Makefile.am +++ b/modules/dmrpp_module/ngap_container/unit-tests/Makefile.am @@ -25,7 +25,7 @@ AM_CXXFLAGS= AM_LDFLAGS = -L$(aws_libdir) include $(top_srcdir)/coverage.mk -LDADD = $(abs_top_builddir)/aws/libbes_aws.la $(aws_libs) +LDADD = $(top_builddir)/aws/libbes_aws.la $(aws_libs) if DARWIN AM_LDFLAGS += -Wl,-rpath,$(aws_libdir) -Wl,-rpath,@loader_path diff --git a/modules/dmrpp_module/unit-tests/Makefile.am b/modules/dmrpp_module/unit-tests/Makefile.am index ab69ae0418..e8481eb55e 100644 --- a/modules/dmrpp_module/unit-tests/Makefile.am +++ b/modules/dmrpp_module/unit-tests/Makefile.am @@ -15,7 +15,7 @@ AM_CPPFLAGS = $(H5_CPPFLAGS) -I$(top_srcdir) -I$(top_srcdir)/dispatch -I$(top_sr # Added -lz for ubuntu LIBADD = $(BES_DISPATCH_LIB) $(top_builddir)/dap/.libs/libdap_module.a $(BES_HTTP_LIB) \ -L$(top_builddir)/modules/common -lmodules_common $(H5_LDFLAGS) \ - $(abs_top_builddir)/aws/libbes_aws.la $(aws_libs) \ + $(top_builddir)/aws/libbes_aws.la $(aws_libs) \ $(H5_LIBS) $(DAP_SERVER_LIBS) $(DAP_CLIENT_LIBS) $(OPENSSL_LIBS) $(XML2_LIBS) -lz # jhrg 6/2/23 $(BES_EXTRA_LIBS) diff --git a/standalone/Makefile.am b/standalone/Makefile.am index 1d1a649d96..44203b5c7a 100644 --- a/standalone/Makefile.am +++ b/standalone/Makefile.am @@ -42,4 +42,4 @@ besstandalone_SOURCES = besstandalone.cc StandAloneApp.cc StandAloneClient.cc \ # This depends on building in bes/cmdln first. It's not a good way to write # teh Makefile.am, but I cannot get distcheck to work otherwise. jhrg 9/8/23 besstandalone_LDADD = $(top_builddir)/cmdln/CmdTranslation.o $(top_builddir)/dispatch/libbes_dispatch.la \ - $(top_builddir)/xmlcommand/libbes_xml_command.la $(READLINE_LIBS) $(XML2_LIBS) $(abs_top_builddir)/aws/libbes_aws.la $(aws_libs) + $(top_builddir)/xmlcommand/libbes_xml_command.la $(READLINE_LIBS) $(XML2_LIBS) $(top_builddir)/aws/libbes_aws.la $(aws_libs) From 27352b6fa83bada956971c8160edea2d51809e08 Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Wed, 19 Nov 2025 14:57:11 -0500 Subject: [PATCH 46/77] clean up --- modules/dmrpp_module/Makefile.am | 4 +--- modules/dmrpp_module/ngap_container/Makefile.am | 8 +++++++- .../dmrpp_module/ngap_container/unit-tests/Makefile.am | 8 ++++---- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/modules/dmrpp_module/Makefile.am b/modules/dmrpp_module/Makefile.am index 0d9debe3d9..bcbea7dbfd 100644 --- a/modules/dmrpp_module/Makefile.am +++ b/modules/dmrpp_module/Makefile.am @@ -98,17 +98,15 @@ build_dmrpp_LDADD = $(BES_DISPATCH_LIB) $(BES_HTTP_LIB) $(H5_LDFLAGS) \ # check_dmrpp config check_dmrpp_CPPFLAGS = $(AM_CPPFLAGS) $(H5_CPPFLAGS) check_dmrpp_SOURCES = check_dmrpp.cc -check_dmrpp_LDADD = $(LDADD) # merge_dmrpp config merge_dmrpp_CPPFLAGS = $(AM_CPPFLAGS) merge_dmrpp_SOURCES = merge_dmrpp.cc -merge_dmrpp_LDADD = $(LDADD) # reduce_mdf config reduce_mdf_CPPFLAGS = $(AM_CPPFLAGS) reduce_mdf_SOURCES = reduce_mdf.cc -reduce_mdf_LDADD = $(OPENSSL_LDFLAGS) $(LDADD) $(OPENSSL_LIBS) -lz +reduce_mdf_LDADD = $(OPENSSL_LDFLAGS) $(OPENSSL_LIBS) -lz EXTRA_PROGRAMS = diff --git a/modules/dmrpp_module/ngap_container/Makefile.am b/modules/dmrpp_module/ngap_container/Makefile.am index 64b3bc2527..202522efa2 100644 --- a/modules/dmrpp_module/ngap_container/Makefile.am +++ b/modules/dmrpp_module/ngap_container/Makefile.am @@ -19,12 +19,18 @@ if DARWIN AM_LDFLAGS += -Wl,-rpath,$(aws_libdir) -Wl,-rpath,@loader_path else AM_LDFLAGS += -Wl,-rpath,$(aws_libdir) -Wl,-rpath,'$$ORIGIN' + ifeq ($(strip $(aws_libdir64)),) + $(info "aws_libdir64 not set, skipping") + else + AM_LDFLAGS += -L$(aws_libdir64) + AM_LDFLAGS += -Wl,-rpath,$(aws_libdir64) + endif endif SUBDIRS = . unit-tests noinst_LTLIBRARIES = libngap.la -libngap_la_SOURCES = $(NGAP_SRC) $(NGAP_HDR) +libngap_la_SOURCES = $(NGAP_SRC) $(NGAP_HDR) libngap_la_LDFLAGS = libngap_la_LIBADD = $(LDADD) diff --git a/modules/dmrpp_module/ngap_container/unit-tests/Makefile.am b/modules/dmrpp_module/ngap_container/unit-tests/Makefile.am index 5380f2f9f5..889b75f23b 100644 --- a/modules/dmrpp_module/ngap_container/unit-tests/Makefile.am +++ b/modules/dmrpp_module/ngap_container/unit-tests/Makefile.am @@ -25,7 +25,7 @@ AM_CXXFLAGS= AM_LDFLAGS = -L$(aws_libdir) include $(top_srcdir)/coverage.mk -LDADD = $(top_builddir)/aws/libbes_aws.la $(aws_libs) +LDADD_AWS = $(top_builddir)/aws/libbes_aws.la $(aws_libs) if DARWIN AM_LDFLAGS += -Wl,-rpath,$(aws_libdir) -Wl,-rpath,@loader_path @@ -115,11 +115,11 @@ clean-local: test ! -d $(builddir)/owned-cache || rm -rf $(builddir)/owned-cache MemoryCacheTest_SOURCES = MemoryCacheTest.cc -MemoryCacheTest_LDADD = $(LIBADD) $(LDADD) +MemoryCacheTest_LDADD = $(LIBADD) NgapApiTest_SOURCES = NgapApiTest.cc -NgapApiTest_LDADD = $(LIBADD) $(LDADD) +NgapApiTest_LDADD = $(LIBADD) $(LDADD_AWS) NgapOwnedContainerTest_SOURCES = NgapOwnedContainerTest.cc -NgapOwnedContainerTest_LDADD = $(LIBADD) $(LDADD) +NgapOwnedContainerTest_LDADD = $(LIBADD) $(LDADD_AWS) From 9dd8da59f8bece0d749382a990f51ffebe4eeb04 Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Wed, 19 Nov 2025 15:47:57 -0500 Subject: [PATCH 47/77] more clean-up --- modules/dmrpp_module/ngap_container/Makefile.am | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/dmrpp_module/ngap_container/Makefile.am b/modules/dmrpp_module/ngap_container/Makefile.am index 202522efa2..44fe73d59a 100644 --- a/modules/dmrpp_module/ngap_container/Makefile.am +++ b/modules/dmrpp_module/ngap_container/Makefile.am @@ -9,7 +9,7 @@ ACLOCAL_AMFLAGS = -I conf AM_CPPFLAGS = -I$(top_srcdir) -I$(top_srcdir)/dispatch -I$(top_srcdir)/dap -I$(top_srcdir)/xmlcommand \ -I$(top_srcdir)/http -I$(top_srcdir)/pugixml/src -I$(top_srcdir)/aws $(DAP_CFLAGS) -LDADD = $(top_builddir)/aws/libbes_aws.la $(aws_libs) +LIBADD = $(top_builddir)/aws/libbes_aws.la $(aws_libs) AM_CXXFLAGS= AM_LDFLAGS = -L$(aws_libdir) @@ -32,7 +32,7 @@ SUBDIRS = . unit-tests noinst_LTLIBRARIES = libngap.la libngap_la_SOURCES = $(NGAP_SRC) $(NGAP_HDR) libngap_la_LDFLAGS = -libngap_la_LIBADD = $(LDADD) +libngap_la_LIBADD = $(LIBADD) NGAP_SRC = NgapOwnedContainer.cc NgapOwnedContainerStorage.cc NgapApi.cc From d0d57d6de47ac3e874e723c2d868eca546dc1730 Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Wed, 19 Nov 2025 17:05:12 -0500 Subject: [PATCH 48/77] comment out signedurlcachetest to see if everything else will build on travis --- aws/unit-tests/SignedUrlCacheTest.cc | 44 ++++++++++++++-------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/aws/unit-tests/SignedUrlCacheTest.cc b/aws/unit-tests/SignedUrlCacheTest.cc index d2502cb6b1..36e009bc91 100644 --- a/aws/unit-tests/SignedUrlCacheTest.cc +++ b/aws/unit-tests/SignedUrlCacheTest.cc @@ -365,28 +365,28 @@ class SignedUrlCacheTest : public CppUnit::TestFixture { CPPUNIT_TEST_SUITE(SignedUrlCacheTest); // Test behavior analogous to that of the EffectiveUrlCache: - CPPUNIT_TEST(get_cached_signed_url_test); - CPPUNIT_TEST(is_cache_disabled_test); - CPPUNIT_TEST(set_skip_regex_test); - CPPUNIT_TEST(dump_test); - - // Test behavior specific to SignedUrlCache: - CPPUNIT_TEST(is_timestamp_after_now_test); - CPPUNIT_TEST(retrieve_cached_s3credentials_test); - CPPUNIT_TEST(retrieve_cached_s3credentials_expired_credentials_test); - CPPUNIT_TEST(extract_s3_credentials_from_response_json_test); - CPPUNIT_TEST(cache_signed_url_components_test); - CPPUNIT_TEST(retrieve_cached_signed_url_components_test); - CPPUNIT_TEST(get_s3credentials_from_endpoint_test); - - // ...and, specifically, the signing itself: - // TODO-future: will add/update these tests once signing behavior is implemented - // - sign_url - // - get_signed_url - - // Last but not least, test those helper functions - CPPUNIT_TEST(split_s3_url_test); - CPPUNIT_TEST(num_seconds_until_expiration_test); + // CPPUNIT_TEST(get_cached_signed_url_test); + // CPPUNIT_TEST(is_cache_disabled_test); + // CPPUNIT_TEST(set_skip_regex_test); + // CPPUNIT_TEST(dump_test); + + // // Test behavior specific to SignedUrlCache: + // CPPUNIT_TEST(is_timestamp_after_now_test); + // CPPUNIT_TEST(retrieve_cached_s3credentials_test); + // CPPUNIT_TEST(retrieve_cached_s3credentials_expired_credentials_test); + // CPPUNIT_TEST(extract_s3_credentials_from_response_json_test); + // CPPUNIT_TEST(cache_signed_url_components_test); + // CPPUNIT_TEST(retrieve_cached_signed_url_components_test); + // CPPUNIT_TEST(get_s3credentials_from_endpoint_test); + + // // ...and, specifically, the signing itself: + // // TODO-future: will add/update these tests once signing behavior is implemented + // // - sign_url + // // - get_signed_url + + // // Last but not least, test those helper functions + // CPPUNIT_TEST(split_s3_url_test); + // CPPUNIT_TEST(num_seconds_until_expiration_test); CPPUNIT_TEST_SUITE_END(); }; From 9f5f155fca99e487d7dae89bd93b5119fc0d104c Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Thu, 20 Nov 2025 13:35:00 -0500 Subject: [PATCH 49/77] fix tests link, remove aws link from standalone --- modules/dmrpp_module/Makefile.am | 4 ++-- modules/dmrpp_module/build_dmrpp_h4/Makefile.am | 2 +- modules/dmrpp_module/ngap_container/Makefile.am | 17 +---------------- standalone/Makefile.am | 17 +++-------------- 4 files changed, 7 insertions(+), 33 deletions(-) diff --git a/modules/dmrpp_module/Makefile.am b/modules/dmrpp_module/Makefile.am index bcbea7dbfd..c881f06f51 100644 --- a/modules/dmrpp_module/Makefile.am +++ b/modules/dmrpp_module/Makefile.am @@ -69,9 +69,9 @@ vlsa_util.h float_byteswap.h DMRPP_MODULE = DmrppModule.cc DmrppRequestHandler.cc DmrppModule.h DmrppRequestHandler.h libdmrpp_module_la_SOURCES = $(BES_HDRS) $(BES_SRCS) $(DMRPP_MODULE) -libdmrpp_module_la_LDFLAGS = -avoid-version -module +libdmrpp_module_la_LDFLAGS = -avoid-version -module $(AM_LDFLAGS) libdmrpp_module_la_LIBADD = -L$(builddir)/ngap_container -lngap $(BES_DISPATCH_LIB) \ - $(BES_HTTP_LIB) $(DAP_SERVER_LIBS) $(DAP_CLIENT_LIBS) \ + $(BES_HTTP_LIB) $(DAP_SERVER_LIBS) $(DAP_CLIENT_LIBS) $(LDADD) \ $(H5_LDFLAGS) $(H5_LIBS) $(OPENSSL_LDFLAGS) $(OPENSSL_LIBS) -ltest-types \ -Ldmrpp_transmitter -ldmrpp_return_as diff --git a/modules/dmrpp_module/build_dmrpp_h4/Makefile.am b/modules/dmrpp_module/build_dmrpp_h4/Makefile.am index ad9dd9fd31..3574d528d4 100644 --- a/modules/dmrpp_module/build_dmrpp_h4/Makefile.am +++ b/modules/dmrpp_module/build_dmrpp_h4/Makefile.am @@ -60,7 +60,7 @@ HDR = build_dmrpp_util_h4.h ../Chunk.h ../DMRpp.h ../DMZ.h ../DmrppArray.h ../Dm build_dmrpp_h4_CPPFLAGS = $(AM_CPPFLAGS) #build_dmrpp_h4_LDFLAGS = $(HDFEOS2_LDFLAGS) $(HDFEOS2_LIBS) $(HDF4_LDFLAGS) $(BES_DAP_LIB_LDFLAGS) -build_dmrpp_h4_LDFLAGS = $(HDFEOS2_LDFLAGS) $(HDF4_LDFLAGS) $(BES_DAP_LIB_LDFLAGS) +build_dmrpp_h4_LDFLAGS = $(HDFEOS2_LDFLAGS) $(HDF4_LDFLAGS) $(BES_DAP_LIB_LDFLAGS) $(AM_LDFLAGS) # jhrg 12/18/23 $(top_builddir)/dap/.libs/libdap_module.a diff --git a/modules/dmrpp_module/ngap_container/Makefile.am b/modules/dmrpp_module/ngap_container/Makefile.am index 44fe73d59a..e44c45e964 100644 --- a/modules/dmrpp_module/ngap_container/Makefile.am +++ b/modules/dmrpp_module/ngap_container/Makefile.am @@ -9,30 +9,15 @@ ACLOCAL_AMFLAGS = -I conf AM_CPPFLAGS = -I$(top_srcdir) -I$(top_srcdir)/dispatch -I$(top_srcdir)/dap -I$(top_srcdir)/xmlcommand \ -I$(top_srcdir)/http -I$(top_srcdir)/pugixml/src -I$(top_srcdir)/aws $(DAP_CFLAGS) -LIBADD = $(top_builddir)/aws/libbes_aws.la $(aws_libs) - AM_CXXFLAGS= -AM_LDFLAGS = -L$(aws_libdir) +AM_LDFLAGS = include $(top_srcdir)/coverage.mk -if DARWIN -AM_LDFLAGS += -Wl,-rpath,$(aws_libdir) -Wl,-rpath,@loader_path -else -AM_LDFLAGS += -Wl,-rpath,$(aws_libdir) -Wl,-rpath,'$$ORIGIN' - ifeq ($(strip $(aws_libdir64)),) - $(info "aws_libdir64 not set, skipping") - else - AM_LDFLAGS += -L$(aws_libdir64) - AM_LDFLAGS += -Wl,-rpath,$(aws_libdir64) - endif -endif - SUBDIRS = . unit-tests noinst_LTLIBRARIES = libngap.la libngap_la_SOURCES = $(NGAP_SRC) $(NGAP_HDR) libngap_la_LDFLAGS = -libngap_la_LIBADD = $(LIBADD) NGAP_SRC = NgapOwnedContainer.cc NgapOwnedContainerStorage.cc NgapApi.cc diff --git a/standalone/Makefile.am b/standalone/Makefile.am index 44203b5c7a..e49162b5e5 100644 --- a/standalone/Makefile.am +++ b/standalone/Makefile.am @@ -3,27 +3,16 @@ AUTOMAKE_OPTIONS = foreign subdir-objects -AM_CPPFLAGS = -I$(top_srcdir)/xmlcommand -I$(top_srcdir)/cmdln -I$(top_srcdir)/dispatch -I$(top_srcdir)/aws $(XML2_CFLAGS) +AM_CPPFLAGS = -I$(top_srcdir)/xmlcommand -I$(top_srcdir)/cmdln -I$(top_srcdir)/dispatch $(XML2_CFLAGS) if BES_DEVELOPER AM_CPPFLAGS += -DBES_DEVELOPER endif AM_CXXFLAGS= -AM_LDFLAGS = -L$(aws_libdir) +AM_LDFLAGS = include $(top_srcdir)/coverage.mk -if DARWIN -AM_LDFLAGS += -Wl,-rpath,$(aws_libdir) -Wl,-rpath,@loader_path -else -AM_LDFLAGS += -Wl,-rpath,$(aws_libdir) -Wl,-rpath,'$$ORIGIN' - ifeq ($(strip $(aws_libdir64)),) - $(info "aws_libdir64 not set, skipping") - else - AM_LDFLAGS += -L$(aws_libdir64) - AM_LDFLAGS += -Wl,-rpath,$(aws_libdir64) - endif -endif EXTRA_DIST = response-split.py @@ -42,4 +31,4 @@ besstandalone_SOURCES = besstandalone.cc StandAloneApp.cc StandAloneClient.cc \ # This depends on building in bes/cmdln first. It's not a good way to write # teh Makefile.am, but I cannot get distcheck to work otherwise. jhrg 9/8/23 besstandalone_LDADD = $(top_builddir)/cmdln/CmdTranslation.o $(top_builddir)/dispatch/libbes_dispatch.la \ - $(top_builddir)/xmlcommand/libbes_xml_command.la $(READLINE_LIBS) $(XML2_LIBS) $(top_builddir)/aws/libbes_aws.la $(aws_libs) + $(top_builddir)/xmlcommand/libbes_xml_command.la $(READLINE_LIBS) $(XML2_LIBS) From 0e4b2048b84eac59fed310df66dcdd93c7e8888f Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Thu, 20 Nov 2025 14:09:12 -0500 Subject: [PATCH 50/77] revert top makefile.am changes --- Makefile.am | 6 +++++- modules/dmrpp_module/ngap_container/Makefile.am | 3 +++ standalone/Makefile.am | 3 +-- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/Makefile.am b/Makefile.am index 65720e12ec..605519aaae 100644 --- a/Makefile.am +++ b/Makefile.am @@ -29,7 +29,11 @@ AM_CXXFLAGS= AM_LDFLAGS = include $(top_srcdir)/coverage.mk -SUBDIRS = dispatch xmlcommand ppt server http cmdln aws standalone docs bin templates hello_world +SUBDIRS = dispatch xmlcommand ppt server http cmdln standalone docs bin templates hello_world + +if USE_AWS_SDK +SUBDIRS += aws +endif if LIBDAP SUBDIRS += dap dapreader diff --git a/modules/dmrpp_module/ngap_container/Makefile.am b/modules/dmrpp_module/ngap_container/Makefile.am index e44c45e964..dc2080c185 100644 --- a/modules/dmrpp_module/ngap_container/Makefile.am +++ b/modules/dmrpp_module/ngap_container/Makefile.am @@ -9,6 +9,8 @@ ACLOCAL_AMFLAGS = -I conf AM_CPPFLAGS = -I$(top_srcdir) -I$(top_srcdir)/dispatch -I$(top_srcdir)/dap -I$(top_srcdir)/xmlcommand \ -I$(top_srcdir)/http -I$(top_srcdir)/pugixml/src -I$(top_srcdir)/aws $(DAP_CFLAGS) +LIBADD = + AM_CXXFLAGS= AM_LDFLAGS = include $(top_srcdir)/coverage.mk @@ -18,6 +20,7 @@ SUBDIRS = . unit-tests noinst_LTLIBRARIES = libngap.la libngap_la_SOURCES = $(NGAP_SRC) $(NGAP_HDR) libngap_la_LDFLAGS = +libngap_la_LIBADD = $(LIBADD) NGAP_SRC = NgapOwnedContainer.cc NgapOwnedContainerStorage.cc NgapApi.cc diff --git a/standalone/Makefile.am b/standalone/Makefile.am index e49162b5e5..400e2bec87 100644 --- a/standalone/Makefile.am +++ b/standalone/Makefile.am @@ -10,10 +10,9 @@ AM_CPPFLAGS += -DBES_DEVELOPER endif AM_CXXFLAGS= -AM_LDFLAGS = +AM_LDFLAGS = include $(top_srcdir)/coverage.mk - EXTRA_DIST = response-split.py # This installs the script From ad58fb76058b8ac276fc2739e962cf425bf38a7e Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Thu, 20 Nov 2025 15:38:47 -0500 Subject: [PATCH 51/77] Add conf to aws unit-tests and reenable aws tests --- aws/unit-tests/Makefile.am | 12 ++++++----- aws/unit-tests/SignedUrlCacheTest.cc | 32 ++++++++++++++-------------- aws/unit-tests/bes.conf.in | 21 ++++++++++++++++++ 3 files changed, 44 insertions(+), 21 deletions(-) create mode 100644 aws/unit-tests/bes.conf.in diff --git a/aws/unit-tests/Makefile.am b/aws/unit-tests/Makefile.am index ccc781e3ac..fa5f94ae68 100644 --- a/aws/unit-tests/Makefile.am +++ b/aws/unit-tests/Makefile.am @@ -50,19 +50,19 @@ AM_CXXFLAGS = # See above. jhrg 10/10/25 AM_LDFLAGS = include $(top_srcdir)/coverage.mk -DISTCLEANFILES = test_config.h +DISTCLEANFILES = bes.conf test_config.h -CLEANFILES = *.dbg *.log +CLEANFILES = bes.conf *.dbg *.log -EXTRA_DIST = test_config.h.in +EXTRA_DIST = test_config.h.in bes.conf.in check_PROGRAMS = $(UNIT_TESTS) TESTS = $(UNIT_TESTS) -noinst_DATA = +noinst_DATA = bes.conf -BUILT_SOURCES = test_config.h +BUILT_SOURCES = test_config.h bes.conf noinst_HEADERS = test_config.h @@ -77,6 +77,8 @@ test_config.h: $(srcdir)/test_config.h.in Makefile -e "s%[@]abs_top_srcdir[@]%$${mod_abs_top_srcdir}%" \ -e "s%[@]abs_builddir[@]%$${mod_abs_builddir}%" $< > test_config.h +bes.conf: bes.conf.in $(top_srcdir)/configure.ac + %.conf: %.conf.in @clean_abs_top_srcdir=`${PYTHON} -c "import os.path; print(os.path.abspath('${abs_top_srcdir}'))"`; \ sed -e "s%[@]abs_top_srcdir[@]%$$clean_abs_top_srcdir%" \ diff --git a/aws/unit-tests/SignedUrlCacheTest.cc b/aws/unit-tests/SignedUrlCacheTest.cc index 36e009bc91..041eb14df9 100644 --- a/aws/unit-tests/SignedUrlCacheTest.cc +++ b/aws/unit-tests/SignedUrlCacheTest.cc @@ -365,28 +365,28 @@ class SignedUrlCacheTest : public CppUnit::TestFixture { CPPUNIT_TEST_SUITE(SignedUrlCacheTest); // Test behavior analogous to that of the EffectiveUrlCache: - // CPPUNIT_TEST(get_cached_signed_url_test); - // CPPUNIT_TEST(is_cache_disabled_test); - // CPPUNIT_TEST(set_skip_regex_test); - // CPPUNIT_TEST(dump_test); - - // // Test behavior specific to SignedUrlCache: - // CPPUNIT_TEST(is_timestamp_after_now_test); - // CPPUNIT_TEST(retrieve_cached_s3credentials_test); - // CPPUNIT_TEST(retrieve_cached_s3credentials_expired_credentials_test); - // CPPUNIT_TEST(extract_s3_credentials_from_response_json_test); - // CPPUNIT_TEST(cache_signed_url_components_test); - // CPPUNIT_TEST(retrieve_cached_signed_url_components_test); - // CPPUNIT_TEST(get_s3credentials_from_endpoint_test); + CPPUNIT_TEST(get_cached_signed_url_test); + CPPUNIT_TEST(is_cache_disabled_test); + CPPUNIT_TEST(set_skip_regex_test); + CPPUNIT_TEST(dump_test); + + // Test behavior specific to SignedUrlCache: + CPPUNIT_TEST(is_timestamp_after_now_test); + CPPUNIT_TEST(retrieve_cached_s3credentials_test); + CPPUNIT_TEST(retrieve_cached_s3credentials_expired_credentials_test); + CPPUNIT_TEST(extract_s3_credentials_from_response_json_test); + CPPUNIT_TEST(cache_signed_url_components_test); + CPPUNIT_TEST(retrieve_cached_signed_url_components_test); + CPPUNIT_TEST(get_s3credentials_from_endpoint_test); // // ...and, specifically, the signing itself: // // TODO-future: will add/update these tests once signing behavior is implemented // // - sign_url // // - get_signed_url - // // Last but not least, test those helper functions - // CPPUNIT_TEST(split_s3_url_test); - // CPPUNIT_TEST(num_seconds_until_expiration_test); + // Last but not least, test those helper functions + CPPUNIT_TEST(split_s3_url_test); + CPPUNIT_TEST(num_seconds_until_expiration_test); CPPUNIT_TEST_SUITE_END(); }; diff --git a/aws/unit-tests/bes.conf.in b/aws/unit-tests/bes.conf.in new file mode 100644 index 0000000000..fc749610fc --- /dev/null +++ b/aws/unit-tests/bes.conf.in @@ -0,0 +1,21 @@ + + +BES.Catalog.catalog.RootDirectory=@abs_top_srcdir@/aws +BES.Catalog.catalog.TypeMatch+=nc:.*\.nc(4)?(\.bz2|\.gz|\.Z)?$ + +BES.LogName = ./bes.log + +############################################################################### +# +# EffectiveUrlCache(Test) configuration. + +# Since many URLs result in a number of time consuming redirects (ex: OAuth2) +# We cache the "effective URL" for each to improve speed and reduce +# authentication churn +Http.cache.effective.urls = false + +# But we also know that many URLs (ex: AWS S3) will never redirect so we can +# skip the caching for those destinations. Any URL matching these patterns will +# not have ensuing redirects followed and cached. +Http.cache.effective.urls.skip.regex.pattern = ^https://foobar\.com/.*$ + From 385632714ebab6f838ed08ea7b80eabdb0479c74 Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Thu, 20 Nov 2025 16:14:36 -0500 Subject: [PATCH 52/77] more test fixes --- aws/unit-tests/SignedUrlCacheTest.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/aws/unit-tests/SignedUrlCacheTest.cc b/aws/unit-tests/SignedUrlCacheTest.cc index 041eb14df9..9cb141b12e 100644 --- a/aws/unit-tests/SignedUrlCacheTest.cc +++ b/aws/unit-tests/SignedUrlCacheTest.cc @@ -194,14 +194,14 @@ class SignedUrlCacheTest : public CppUnit::TestFixture { "\n yee --> haw" + "\n s3 credentials list:" + "\n palindrome --> Expires: 3035-07-16 02:20:33+00:00\n"; - CPPUNIT_ASSERT_MESSAGE("The dump should be:\n" + expected_str + "\n\nINSTEAD was\n" + result, expected_str == result); + CPPUNIT_ASSERT_MESSAGE("The dump should contain:\n" + expected_str + "\n\nbut did not; INSTEAD was\n" + result, result.find(expected_str) != std::string::npos); } void is_timestamp_after_now_test() { std::string str_old("1980-07-16 18:40:58+00:00"); CPPUNIT_ASSERT_MESSAGE("Ancient timestamp is before now", !SignedUrlCache::is_timestamp_after_now(str_old)); - std::string str_future("3035-07-16 02:20:33+00:00"); + std::string str_future("2135-07-16 02:20:33+00:00"); CPPUNIT_ASSERT_MESSAGE("Future timestamp is after now", SignedUrlCache::is_timestamp_after_now(str_future)); std::string str_invalid("invalid timestamp woo hooray huzzah"); @@ -213,7 +213,7 @@ class SignedUrlCacheTest : public CppUnit::TestFixture { auto result_not_in_cache = SignedUrlCache::TheCache()->retrieve_cached_s3credentials(key); CPPUNIT_ASSERT_MESSAGE("Cache miss should return null", result_not_in_cache == nullptr); - auto value = make_shared("a man", "a plan", "a canal", "3035-07-16 02:20:33+00:00"); + auto value = make_shared("a man", "a plan", "a canal", "2135-07-16 02:20:33+00:00"); SignedUrlCache::TheCache()->d_s3credentials_cache.insert(pair>(key, value)); auto result_in_cache = SignedUrlCache::TheCache()->retrieve_cached_s3credentials(key); CPPUNIT_ASSERT_MESSAGE("Cache hit should successfully retrieve result", result_in_cache == value); @@ -351,7 +351,7 @@ class SignedUrlCacheTest : public CppUnit::TestFixture { auto out3 = SignedUrlCache::num_seconds_until_expiration(expiration_time_past); CPPUNIT_ASSERT_MESSAGE("Expiration time in past should return 0s `" + to_string(out3) + "`", out3 == 0); - std::string expiration_time_future("3026-07-16 18:40:58+04:00"); + std::string expiration_time_future("2126-07-16 18:40:58+04:00"); auto out4 = SignedUrlCache::num_seconds_until_expiration(expiration_time_future); CPPUNIT_ASSERT_MESSAGE("Expiration time in future should be > 0s `" + to_string(out4) + "`", out4 > 0); From 6d768a49a272fa9d620b7af0d150914cee9ba11a Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Thu, 20 Nov 2025 18:44:04 -0500 Subject: [PATCH 53/77] add cmdln test that uses dmrpp --- .../tests/dmrpp/dmrpp-return-as-test-2.bescmd | 14 + .../dmrpp-return-as-test-2.bescmd.baseline | 27713 ++++++++++++++++ cmdln/tests/testsuite.at | 8 + 3 files changed, 27735 insertions(+) create mode 100644 cmdln/tests/dmrpp/dmrpp-return-as-test-2.bescmd create mode 100644 cmdln/tests/dmrpp/dmrpp-return-as-test-2.bescmd.baseline diff --git a/cmdln/tests/dmrpp/dmrpp-return-as-test-2.bescmd b/cmdln/tests/dmrpp/dmrpp-return-as-test-2.bescmd new file mode 100644 index 0000000000..aea7d0b5a6 --- /dev/null +++ b/cmdln/tests/dmrpp/dmrpp-return-as-test-2.bescmd @@ -0,0 +1,14 @@ + + + 300 + no + xml + http://localhost:8080/opendap/ngap/collections/C2532426483-ORNL_CLOUD/granules/Daymet_Daily_V4R1.daymet_v4_daily_na_tmax_2010.nc + 0 + 0 + collections/C2532426483-ORNL_CLOUD/granules/Daymet_Daily_V4R1.daymet_v4_daily_na_tmax_2010.nc + + + + + diff --git a/cmdln/tests/dmrpp/dmrpp-return-as-test-2.bescmd.baseline b/cmdln/tests/dmrpp/dmrpp-return-as-test-2.bescmd.baseline new file mode 100644 index 0000000000..949e1e8888 --- /dev/null +++ b/cmdln/tests/dmrpp/dmrpp-return-as-test-2.bescmd.baseline @@ -0,0 +1,27713 @@ + + + + + + + + + + m + + + y coordinate of projection + + + projection_y_coordinate + + + 8075 + + + + + + + + degrees_east + + + longitude coordinate + + + longitude + + + 1010 977 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + degrees_north + + + latitude coordinate + + + latitude + + + 1010 977 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + time + + + standard + + + days since 1950-01-01 00:00:00 + + + time_bnds + + + 24-hour day based on local time + + + 1 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + m + + + x coordinate of projection + + + projection_x_coordinate + + + 7814 + + + + + + + + + -9999. + + + daily maximum temperature + + + degrees C + + + -9999. + + + lat lon + + + lambert_conformal_conic + + + area: mean time: maximum + + + + + + 1 1000 1000 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + lambert_conformal_conic + + + -100. + + + 42.5 + + + 0. + + + 0. + + + 25. + 60. + + + 6378137. + + + 298.25722356300003 + + + + + + + + + + + 1 2 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + day of year (DOY) starting with day 1 on Januaray 1st + + + + 1 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2010 + + + Daymet Software Version 4.0 + + + Daymet Software Version 4.0 + + + Daymet Data Version 4.0 + + + CF-1.6 + + + Please see http://daymet.ornl.gov/ for current Daymet data citation information + + + Please see http://daymet.ornl.gov/ for current information on Daymet references + + + + 2025-09-15T19:37:22Z + + + 3.21.1-367 + + + 3.21.1-367 + + + libdap-3.21.1-99 + + + build_dmrpp -f /tmp/tmp0tv4iczy/daymet_v4_daily_na_tmax_2010.nc -r daymet_v4_daily_na_tmax_2010.nc.dmr -u OPeNDAP_DMRpp_DATA_ACCESS_URL -M + + + diff --git a/cmdln/tests/testsuite.at b/cmdln/tests/testsuite.at index 8763c78145..db77b5c731 100644 --- a/cmdln/tests/testsuite.at +++ b/cmdln/tests/testsuite.at @@ -101,6 +101,13 @@ _AT_BESCMD_TEST($abs_srcdir/$1, $abs_srcdir/$1.baseline, $2) AT_CLEANUP ]) +m4_define([AT_BESCMD_DMRPP_RESPONSE_TEST], +[AT_SETUP([BESCMD $1]) +AT_KEYWORDS([dmrpp]) +_AT_BESCMD_TEST($abs_srcdir/$1, $abs_srcdir/$1.baseline, $2) +AT_CLEANUP +]) + m4_define([AT_BESCMD_ERROR_RESPONSE_TEST], [ AT_SETUP([BESCMD $1]) @@ -233,3 +240,4 @@ AT_BESCMD_BINARYDATA_RESPONSE_TEST([ssfunc/geogrid_func_wholearray.3.bescmd]) #AT_BESCMD_SHOW_RESPONSE_TEST([show/show_02.bescmd], [xfail]) # Uses showInfo which was deprecated and dropped. AT_BESCMD_SHOW_RESPONSE_TEST([show/show_03.bescmd], [xfail]) +AT_BESCMD_DMRPP_RESPONSE_TEST([dmrpp/dmrpp-return-as-test-2.bescmd], [pass]) \ No newline at end of file From b55605f1e7a44a4186d15178f7093c5d1b044fd6 Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Fri, 21 Nov 2025 10:42:10 -0500 Subject: [PATCH 54/77] print testsuite log from travis --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 26e12f266e..93be3e9353 100644 --- a/.travis.yml +++ b/.travis.yml @@ -148,6 +148,8 @@ jobs: - autoreconf --force --install --verbose - ./configure --disable-dependency-tracking --prefix=$prefix --with-dependencies=$prefix/deps --enable-developer - make -j16 && make install && besctl start && make check -j16 && besctl stop + after_failure: + - cat /home/travis/build/OPENDAP/bes/cmdln/tests/testsuite.log - stage: build-and-package name: "distcheck" From 438b6c2c12165be62d21f5e4e034f37244658df6 Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Mon, 24 Nov 2025 11:26:37 -0500 Subject: [PATCH 55/77] add additional aws dep lib --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 807decdce6..9ff4020d59 100644 --- a/configure.ac +++ b/configure.ac @@ -600,7 +600,7 @@ AM_COND_IF([LOOK_FOR_AWS], aws_libs="-laws-crt-cpp -laws-cpp-sdk-s3 -laws-cpp-sdk-core \ -laws-c-http -laws-c-io -laws-c-auth -laws-c-cal -laws-c-compression \ -laws-c-event-stream -laws-c-mqtt -laws-c-s3 -laws-c-sdkutils -laws-c-common \ - -lxml2 -lz -lbz2 -lcurl" + -lxml2 -lz -lbz2 -lcurl -laws-checksums" dnl AC_CANONICAL_HOST is needed to access the 'host_os' variable AC_CANONICAL_HOST From 8f93f330c58fa95277b143f337452196e3198f72 Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Mon, 24 Nov 2025 14:13:35 -0500 Subject: [PATCH 56/77] temp turn off .travis jobs --- .travis.yml | 118 ++++++++++++++++++++++++++-------------------------- 1 file changed, 59 insertions(+), 59 deletions(-) diff --git a/.travis.yml b/.travis.yml index 93be3e9353..06a18e7a9a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -151,13 +151,13 @@ jobs: after_failure: - cat /home/travis/build/OPENDAP/bes/cmdln/tests/testsuite.log - - stage: build-and-package - name: "distcheck" - script: - - export BES_BUILD=distcheck - - autoreconf --force --install --verbose - - ./configure --disable-dependency-tracking --prefix=$prefix --with-dependencies=$prefix/deps --enable-developer - - make distcheck -j16 GZIP_ENV=--fast + # - stage: build-and-package + # name: "distcheck" + # script: + # - export BES_BUILD=distcheck + # - autoreconf --force --install --verbose + # - ./configure --disable-dependency-tracking --prefix=$prefix --with-dependencies=$prefix/deps --enable-developer + # - make distcheck -j16 GZIP_ENV=--fast - stage: build-and-package name: "Enterprise Linux 8 RPMs (via Rocky8)" @@ -175,63 +175,63 @@ jobs: --env AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY opendap/rocky8_hyrax_builder:latest /root/travis/travis/build-rpm.sh - - stage: build-and-package - name: "dist" - script: - - export BES_BUILD=srcdist - - autoreconf --force --install --verbose - - ./configure --disable-dependency-tracking --prefix=$prefix --with-dependencies=$prefix/deps --with-build=$BES_BUILD_NUMBER - - make dist -j7 - # Make both a bes-- and bes-snapshot tar.gz. This will simplify - # other operations that use the bes source code. Note that the VERSION file holds a - # string that is the version number that is set by the configure script and the build - # number passed into configure when it is run. jhrg 3/23/21 - - SOURCE_VERSION=$(cat bes_VERSION) - - mv bes-*.tar.gz bes-$SOURCE_VERSION.tar.gz - - cp bes-$SOURCE_VERSION.tar.gz bes-snapshot.tar.gz +# - stage: build-and-package +# name: "dist" +# script: +# - export BES_BUILD=srcdist +# - autoreconf --force --install --verbose +# - ./configure --disable-dependency-tracking --prefix=$prefix --with-dependencies=$prefix/deps --with-build=$BES_BUILD_NUMBER +# - make dist -j7 +# # Make both a bes-- and bes-snapshot tar.gz. This will simplify +# # other operations that use the bes source code. Note that the VERSION file holds a +# # string that is the version number that is set by the configure script and the build +# # number passed into configure when it is run. jhrg 3/23/21 +# - SOURCE_VERSION=$(cat bes_VERSION) +# - mv bes-*.tar.gz bes-$SOURCE_VERSION.tar.gz +# - cp bes-$SOURCE_VERSION.tar.gz bes-snapshot.tar.gz - - stage: scan - name: "scan bes" - script: - - export BES_BUILD=sonar-bes-framework - - autoreconf --force --install --verbose - - ./configure --disable-dependency-tracking --prefix=$prefix --with-dependencies=$prefix/deps --enable-developer --enable-coverage - - build-wrapper-linux-x86-64 --out-dir bw-output make -j16 - - sonar-scanner -Dproject.settings=sonar-bes-framework.properties -Dsonar.login=$SONAR_LOGIN - - curl -s https://sonarcloud.io/api/project_badges/quality_gate?project=opendap-bes | grep "QUALITY GATE PASS" +# - stage: scan +# name: "scan bes" +# script: +# - export BES_BUILD=sonar-bes-framework +# - autoreconf --force --install --verbose +# - ./configure --disable-dependency-tracking --prefix=$prefix --with-dependencies=$prefix/deps --enable-developer --enable-coverage +# - build-wrapper-linux-x86-64 --out-dir bw-output make -j16 +# - sonar-scanner -Dproject.settings=sonar-bes-framework.properties -Dsonar.login=$SONAR_LOGIN +# - curl -s https://sonarcloud.io/api/project_badges/quality_gate?project=opendap-bes | grep "QUALITY GATE PASS" - - stage: scan - name: "scan bes-modules-1" - script: - - export BES_BUILD=sonar-bes-modules - - autoreconf --force --install --verbose - - ./configure --disable-dependency-tracking --prefix=$prefix --with-dependencies=$prefix/deps --enable-developer --enable-coverage - - build-wrapper-linux-x86-64 --out-dir bw-output make -j16 - - sonar-scanner -Dproject.settings=sonar-bes-modules-1.properties -Dsonar.login=$SONAR_MODULES_LOGIN - - curl -s https://sonarcloud.io/api/project_badges/quality_gate?project=opendap-bes-modules | grep "QUALITY GATE PASS" +# - stage: scan +# name: "scan bes-modules-1" +# script: +# - export BES_BUILD=sonar-bes-modules +# - autoreconf --force --install --verbose +# - ./configure --disable-dependency-tracking --prefix=$prefix --with-dependencies=$prefix/deps --enable-developer --enable-coverage +# - build-wrapper-linux-x86-64 --out-dir bw-output make -j16 +# - sonar-scanner -Dproject.settings=sonar-bes-modules-1.properties -Dsonar.login=$SONAR_MODULES_LOGIN +# - curl -s https://sonarcloud.io/api/project_badges/quality_gate?project=opendap-bes-modules | grep "QUALITY GATE PASS" - - stage: scan - name: "scan bes-hdf-handlers" - script: - - export BES_BUILD=sonar-bes-hdf-handlers - - autoreconf --force --install --verbose - - ./configure --disable-dependency-tracking --prefix=$prefix --with-dependencies=$prefix/deps --enable-developer --enable-coverage - - build-wrapper-linux-x86-64 --out-dir bw-output make -j16 - - sonar-scanner -Dproject.settings=sonar-bes-hdf-handlers.properties -Dsonar.login=$SONAR_SUBMODULES_LOGIN - # We call the hdf4/5 handlers scan opendap-bes-submodules for historical reasons. jhrg 1/13/22 - - curl -s https://sonarcloud.io/api/project_badges/quality_gate?project=opendap-bes-submodules | grep "QUALITY GATE PASS" +# - stage: scan +# name: "scan bes-hdf-handlers" +# script: +# - export BES_BUILD=sonar-bes-hdf-handlers +# - autoreconf --force --install --verbose +# - ./configure --disable-dependency-tracking --prefix=$prefix --with-dependencies=$prefix/deps --enable-developer --enable-coverage +# - build-wrapper-linux-x86-64 --out-dir bw-output make -j16 +# - sonar-scanner -Dproject.settings=sonar-bes-hdf-handlers.properties -Dsonar.login=$SONAR_SUBMODULES_LOGIN +# # We call the hdf4/5 handlers scan opendap-bes-submodules for historical reasons. jhrg 1/13/22 +# - curl -s https://sonarcloud.io/api/project_badges/quality_gate?project=opendap-bes-submodules | grep "QUALITY GATE PASS" - - stage: hyrax-olfs-trigger - name: "Hyrax OLFS Trigger" - script: - - export STAGE=hyrax-olfs - - echo $STAGE - - autoreconf --force --install --verbose - - ./configure --disable-dependency-tracking --prefix=$prefix --with-dependencies=$prefix/deps --with-build=$BES_BUILD_NUMBER - - ./travis/trigger-olfs-build.sh +# - stage: hyrax-olfs-trigger +# name: "Hyrax OLFS Trigger" +# script: +# - export STAGE=hyrax-olfs +# - echo $STAGE +# - autoreconf --force --install --verbose +# - ./configure --disable-dependency-tracking --prefix=$prefix --with-dependencies=$prefix/deps --with-build=$BES_BUILD_NUMBER +# - ./travis/trigger-olfs-build.sh -after_script: - - ./travis/upload-test-results.sh +# after_script: +# - ./travis/upload-test-results.sh before_deploy: - export DEPLOY="S3" From 70cf422de20109f90917e43b685490107b9b5ac2 Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Tue, 25 Nov 2025 13:50:59 -0500 Subject: [PATCH 57/77] rearrange aws libs to attempt to fix static build --- configure.ac | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/configure.ac b/configure.ac index 9ff4020d59..1b44ff9bb1 100644 --- a/configure.ac +++ b/configure.ac @@ -597,10 +597,11 @@ AM_COND_IF([LOOK_FOR_AWS], AS_IF([$found_aws_h && $found_s3client_h], [aws_prefix=$ac_bes_dependencies_prefix - aws_libs="-laws-crt-cpp -laws-cpp-sdk-s3 -laws-cpp-sdk-core \ - -laws-c-http -laws-c-io -laws-c-auth -laws-c-cal -laws-c-compression \ - -laws-c-event-stream -laws-c-mqtt -laws-c-s3 -laws-c-sdkutils -laws-c-common \ - -lxml2 -lz -lbz2 -lcurl -laws-checksums" + aws_libs=" -laws-c-compression -laws-c-event-stream -laws-crt-cpp \ + -laws-cpp-sdk-s3 -laws-cpp-sdk-core \ + -laws-c-http -laws-c-io -laws-c-auth -laws-c-cal \ + -laws-c-mqtt -laws-c-s3 -laws-c-sdkutils -laws-c-common \ + -lxml2 -lz -lbz2 -lcurl -laws-checksums" dnl AC_CANONICAL_HOST is needed to access the 'host_os' variable AC_CANONICAL_HOST From 5b6a07d7cbb43f8fc02f461c500c2a1db1af760a Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Tue, 25 Nov 2025 14:29:17 -0500 Subject: [PATCH 58/77] rearrange more --- configure.ac | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/configure.ac b/configure.ac index 1b44ff9bb1..1b992970ea 100644 --- a/configure.ac +++ b/configure.ac @@ -597,11 +597,11 @@ AM_COND_IF([LOOK_FOR_AWS], AS_IF([$found_aws_h && $found_s3client_h], [aws_prefix=$ac_bes_dependencies_prefix - aws_libs=" -laws-c-compression -laws-c-event-stream -laws-crt-cpp \ - -laws-cpp-sdk-s3 -laws-cpp-sdk-core \ - -laws-c-http -laws-c-io -laws-c-auth -laws-c-cal \ - -laws-c-mqtt -laws-c-s3 -laws-c-sdkutils -laws-c-common \ - -lxml2 -lz -lbz2 -lcurl -laws-checksums" + aws_libs=" -laws-c-common -laws-c-compression -laws-c-event-stream -laws-crt-cpp \ + -laws-cpp-sdk-s3 -laws-cpp-sdk-core \ + -laws-c-http -laws-c-io -laws-c-auth -laws-c-cal \ + -laws-c-mqtt -laws-c-s3 -laws-c-sdkutils \ + -lxml2 -lz -lbz2 -lcurl -laws-checksums" dnl AC_CANONICAL_HOST is needed to access the 'host_os' variable AC_CANONICAL_HOST From c8211d88ea77fa456fc21327f3bb170c7e833712 Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Tue, 25 Nov 2025 14:46:46 -0500 Subject: [PATCH 59/77] one more switcharoo... --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 1b992970ea..3357ad86e9 100644 --- a/configure.ac +++ b/configure.ac @@ -598,7 +598,7 @@ AM_COND_IF([LOOK_FOR_AWS], [aws_prefix=$ac_bes_dependencies_prefix aws_libs=" -laws-c-common -laws-c-compression -laws-c-event-stream -laws-crt-cpp \ - -laws-cpp-sdk-s3 -laws-cpp-sdk-core \ + -laws-cpp-sdk-core -laws-cpp-sdk-s3 \ -laws-c-http -laws-c-io -laws-c-auth -laws-c-cal \ -laws-c-mqtt -laws-c-s3 -laws-c-sdkutils \ -lxml2 -lz -lbz2 -lcurl -laws-checksums" From 7991d058737ee65418b39a8f30341a6632b52475 Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Tue, 25 Nov 2025 15:04:08 -0500 Subject: [PATCH 60/77] more rearrange --- configure.ac | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index 3357ad86e9..caba564aee 100644 --- a/configure.ac +++ b/configure.ac @@ -597,9 +597,9 @@ AM_COND_IF([LOOK_FOR_AWS], AS_IF([$found_aws_h && $found_s3client_h], [aws_prefix=$ac_bes_dependencies_prefix - aws_libs=" -laws-c-common -laws-c-compression -laws-c-event-stream -laws-crt-cpp \ + aws_libs=" -laws-c-compression -laws-c-event-stream -laws-c-common -laws-c-io -laws-c-http -laws-crt-cpp \ -laws-cpp-sdk-core -laws-cpp-sdk-s3 \ - -laws-c-http -laws-c-io -laws-c-auth -laws-c-cal \ + -laws-c-auth -laws-c-cal \ -laws-c-mqtt -laws-c-s3 -laws-c-sdkutils \ -lxml2 -lz -lbz2 -lcurl -laws-checksums" From 7590c693cc0f012b503d298bdea98deaa854b2e7 Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Tue, 25 Nov 2025 15:53:37 -0500 Subject: [PATCH 61/77] welp was making it worse --- configure.ac | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/configure.ac b/configure.ac index caba564aee..cb97faa627 100644 --- a/configure.ac +++ b/configure.ac @@ -597,10 +597,10 @@ AM_COND_IF([LOOK_FOR_AWS], AS_IF([$found_aws_h && $found_s3client_h], [aws_prefix=$ac_bes_dependencies_prefix - aws_libs=" -laws-c-compression -laws-c-event-stream -laws-c-common -laws-c-io -laws-c-http -laws-crt-cpp \ - -laws-cpp-sdk-core -laws-cpp-sdk-s3 \ + aws_libs=" -laws-cpp-sdk-core -laws-crt-cpp -laws-cpp-sdk-s3 \ + -laws-c-io -laws-c-http -laws-c-compression -laws-c-event-stream \ -laws-c-auth -laws-c-cal \ - -laws-c-mqtt -laws-c-s3 -laws-c-sdkutils \ + -laws-c-mqtt -laws-c-s3 -laws-c-sdkutils -laws-c-common \ -lxml2 -lz -lbz2 -lcurl -laws-checksums" dnl AC_CANONICAL_HOST is needed to access the 'host_os' variable From 55f7a77b3fb0ebec78c353319b780733e6764565 Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Tue, 25 Nov 2025 22:45:49 -0500 Subject: [PATCH 62/77] add dmrpp check that doesn't fail if unauthorized --- ...dmrpp-return-as-test-2.bescmd.baseline_alt | 20 +++++++++++++ cmdln/tests/testsuite.at | 29 +++++++++++++++++-- 2 files changed, 46 insertions(+), 3 deletions(-) create mode 100644 cmdln/tests/dmrpp/dmrpp-return-as-test-2.bescmd.baseline_alt diff --git a/cmdln/tests/dmrpp/dmrpp-return-as-test-2.bescmd.baseline_alt b/cmdln/tests/dmrpp/dmrpp-return-as-test-2.bescmd.baseline_alt new file mode 100644 index 0000000000..7bc1e7d756 --- /dev/null +++ b/cmdln/tests/dmrpp/dmrpp-return-as-test-2.bescmd.baseline_alt @@ -0,0 +1,20 @@ + + + + + 7 + + support@opendap.org + 0 + 401 + https://data.ornldaac.earthdata.nasa.gov/protected/daymet/Daymet_Daily_V4R1/data/daymet_v4_daily_na_tmax_2010.nc.dmrpp + + + + + CurlUtils.cc + 856 + + + + \ No newline at end of file diff --git a/cmdln/tests/testsuite.at b/cmdln/tests/testsuite.at index db77b5c731..b59324b643 100644 --- a/cmdln/tests/testsuite.at +++ b/cmdln/tests/testsuite.at @@ -101,11 +101,34 @@ _AT_BESCMD_TEST($abs_srcdir/$1, $abs_srcdir/$1.baseline, $2) AT_CLEANUP ]) -m4_define([AT_BESCMD_DMRPP_RESPONSE_TEST], +dnl Either match the expected dmrpp result (if EDL credentials present) +dnl or match with the error type returned, without any urls +dnl that could leak credentials or be unmatchable due to unique IDs +m4_define([AT_BESCMD_DMRPP_RESPONSE_TEST], [AT_SETUP([BESCMD $1]) AT_KEYWORDS([dmrpp]) -_AT_BESCMD_TEST($abs_srcdir/$1, $abs_srcdir/$1.baseline, $2) -AT_CLEANUP + + input=$abs_srcdir/$1 + baseline=$abs_srcdir/$1.baseline + baseline_alt=$abs_srcdir/$1.baseline_alt + pass=$2 + + AS_IF([test -n "$baselines" -a x$baselines = xyes], + [ + AT_CHECK([bescmdln -i $input], [], [stdout]) + AT_CHECK([mv stdout $baseline.tmp]) + ], + [ + AT_CHECK([bescmdln -i $input], [], [stdout]) + AS_IF(diff -b -B $baseline stdout, + [AT_CHECK([diff -b -B $baseline stdout])], + [ + AT_CHECK([cat stdout | sed 's/.*//g' | sed 's/.*//g' | sed 's/ Date: Tue, 25 Nov 2025 23:01:26 -0500 Subject: [PATCH 63/77] temp disable another travis job --- .travis.yml | 20 +++++++++---------- ...t-2.bescmd => dmrpp-return-as-test.bescmd} | 0 ...e => dmrpp-return-as-test.bescmd.baseline} | 0 ... dmrpp-return-as-test.bescmd.baseline_alt} | 0 cmdln/tests/testsuite.at | 2 +- 5 files changed, 11 insertions(+), 11 deletions(-) rename cmdln/tests/dmrpp/{dmrpp-return-as-test-2.bescmd => dmrpp-return-as-test.bescmd} (100%) rename cmdln/tests/dmrpp/{dmrpp-return-as-test-2.bescmd.baseline => dmrpp-return-as-test.bescmd.baseline} (100%) rename cmdln/tests/dmrpp/{dmrpp-return-as-test-2.bescmd.baseline_alt => dmrpp-return-as-test.bescmd.baseline_alt} (100%) diff --git a/.travis.yml b/.travis.yml index 06a18e7a9a..8bf3dd7950 100644 --- a/.travis.yml +++ b/.travis.yml @@ -140,16 +140,16 @@ stages: jobs: include: - - stage: build-and-package - name: "check" - script: - - export BES_BUILD=main - # - export RUNTESTFLAGS="-v" - - autoreconf --force --install --verbose - - ./configure --disable-dependency-tracking --prefix=$prefix --with-dependencies=$prefix/deps --enable-developer - - make -j16 && make install && besctl start && make check -j16 && besctl stop - after_failure: - - cat /home/travis/build/OPENDAP/bes/cmdln/tests/testsuite.log + # - stage: build-and-package + # name: "check" + # script: + # - export BES_BUILD=main + # # - export RUNTESTFLAGS="-v" + # - autoreconf --force --install --verbose + # - ./configure --disable-dependency-tracking --prefix=$prefix --with-dependencies=$prefix/deps --enable-developer + # - make -j16 && make install && besctl start && make check -j16 && besctl stop + # after_failure: + # - cat /home/travis/build/OPENDAP/bes/cmdln/tests/testsuite.log # - stage: build-and-package # name: "distcheck" diff --git a/cmdln/tests/dmrpp/dmrpp-return-as-test-2.bescmd b/cmdln/tests/dmrpp/dmrpp-return-as-test.bescmd similarity index 100% rename from cmdln/tests/dmrpp/dmrpp-return-as-test-2.bescmd rename to cmdln/tests/dmrpp/dmrpp-return-as-test.bescmd diff --git a/cmdln/tests/dmrpp/dmrpp-return-as-test-2.bescmd.baseline b/cmdln/tests/dmrpp/dmrpp-return-as-test.bescmd.baseline similarity index 100% rename from cmdln/tests/dmrpp/dmrpp-return-as-test-2.bescmd.baseline rename to cmdln/tests/dmrpp/dmrpp-return-as-test.bescmd.baseline diff --git a/cmdln/tests/dmrpp/dmrpp-return-as-test-2.bescmd.baseline_alt b/cmdln/tests/dmrpp/dmrpp-return-as-test.bescmd.baseline_alt similarity index 100% rename from cmdln/tests/dmrpp/dmrpp-return-as-test-2.bescmd.baseline_alt rename to cmdln/tests/dmrpp/dmrpp-return-as-test.bescmd.baseline_alt diff --git a/cmdln/tests/testsuite.at b/cmdln/tests/testsuite.at index b59324b643..2864fe4efd 100644 --- a/cmdln/tests/testsuite.at +++ b/cmdln/tests/testsuite.at @@ -263,4 +263,4 @@ AT_BESCMD_BINARYDATA_RESPONSE_TEST([ssfunc/geogrid_func_wholearray.3.bescmd]) #AT_BESCMD_SHOW_RESPONSE_TEST([show/show_02.bescmd], [xfail]) # Uses showInfo which was deprecated and dropped. AT_BESCMD_SHOW_RESPONSE_TEST([show/show_03.bescmd], [xfail]) -AT_BESCMD_DMRPP_RESPONSE_TEST([dmrpp/dmrpp-return-as-test-2.bescmd], [pass]) \ No newline at end of file +AT_BESCMD_DMRPP_RESPONSE_TEST([dmrpp/dmrpp-return-as-test.bescmd], [pass]) \ No newline at end of file From b364550800f86b7c9619809f8f644492696fc33b Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Wed, 26 Nov 2025 09:39:52 -0500 Subject: [PATCH 64/77] fifteenth time's the charm --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index cb97faa627..f68dc4fe50 100644 --- a/configure.ac +++ b/configure.ac @@ -597,7 +597,7 @@ AM_COND_IF([LOOK_FOR_AWS], AS_IF([$found_aws_h && $found_s3client_h], [aws_prefix=$ac_bes_dependencies_prefix - aws_libs=" -laws-cpp-sdk-core -laws-crt-cpp -laws-cpp-sdk-s3 \ + aws_libs=" -laws-cpp-sdk-s3 -laws-crt-cpp -laws-cpp-sdk-core\ -laws-c-io -laws-c-http -laws-c-compression -laws-c-event-stream \ -laws-c-auth -laws-c-cal \ -laws-c-mqtt -laws-c-s3 -laws-c-sdkutils -laws-c-common \ From 5a552131df574d2ebeb9522f6a19e90511fa59be Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Wed, 26 Nov 2025 16:26:45 -0500 Subject: [PATCH 65/77] you put the right dep in you put the right dep out --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index f68dc4fe50..00aceaf446 100644 --- a/configure.ac +++ b/configure.ac @@ -597,7 +597,7 @@ AM_COND_IF([LOOK_FOR_AWS], AS_IF([$found_aws_h && $found_s3client_h], [aws_prefix=$ac_bes_dependencies_prefix - aws_libs=" -laws-cpp-sdk-s3 -laws-crt-cpp -laws-cpp-sdk-core\ + aws_libs=" -laws-cpp-sdk-s3 -laws-cpp-sdk-core -laws-crt-cpp \ -laws-c-io -laws-c-http -laws-c-compression -laws-c-event-stream \ -laws-c-auth -laws-c-cal \ -laws-c-mqtt -laws-c-s3 -laws-c-sdkutils -laws-c-common \ From 41a0155033d0118f974724bd519355b1a40054ec Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Wed, 26 Nov 2025 17:08:18 -0500 Subject: [PATCH 66/77] you put the right dep in and you shake it all about --- configure.ac | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index 00aceaf446..832ef34ec3 100644 --- a/configure.ac +++ b/configure.ac @@ -597,8 +597,8 @@ AM_COND_IF([LOOK_FOR_AWS], AS_IF([$found_aws_h && $found_s3client_h], [aws_prefix=$ac_bes_dependencies_prefix - aws_libs=" -laws-cpp-sdk-s3 -laws-cpp-sdk-core -laws-crt-cpp \ - -laws-c-io -laws-c-http -laws-c-compression -laws-c-event-stream \ + aws_libs=" -laws-cpp-sdk-s3 -laws-cpp-sdk-core -laws-c-io -laws-crt-cpp \ + -laws-c-http -laws-c-compression -laws-c-event-stream \ -laws-c-auth -laws-c-cal \ -laws-c-mqtt -laws-c-s3 -laws-c-sdkutils -laws-c-common \ -lxml2 -lz -lbz2 -lcurl -laws-checksums" From 719a7d1e879f98c1c3ee5c5f0f73ba816484fd0f Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Wed, 26 Nov 2025 17:29:19 -0500 Subject: [PATCH 67/77] you do the linky pokey --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 832ef34ec3..09d319dcbb 100644 --- a/configure.ac +++ b/configure.ac @@ -597,7 +597,7 @@ AM_COND_IF([LOOK_FOR_AWS], AS_IF([$found_aws_h && $found_s3client_h], [aws_prefix=$ac_bes_dependencies_prefix - aws_libs=" -laws-cpp-sdk-s3 -laws-cpp-sdk-core -laws-c-io -laws-crt-cpp \ + aws_libs=" -laws-cpp-sdk-s3 -laws-cpp-sdk-core -laws-crt-cpp -laws-c-io \ -laws-c-http -laws-c-compression -laws-c-event-stream \ -laws-c-auth -laws-c-cal \ -laws-c-mqtt -laws-c-s3 -laws-c-sdkutils -laws-c-common \ From 610e3e6007504c6c3bc1846def92bc6671810232 Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Wed, 26 Nov 2025 17:46:16 -0500 Subject: [PATCH 68/77] and you link it all around --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 09d319dcbb..621cdd867b 100644 --- a/configure.ac +++ b/configure.ac @@ -601,7 +601,7 @@ AM_COND_IF([LOOK_FOR_AWS], -laws-c-http -laws-c-compression -laws-c-event-stream \ -laws-c-auth -laws-c-cal \ -laws-c-mqtt -laws-c-s3 -laws-c-sdkutils -laws-c-common \ - -lxml2 -lz -lbz2 -lcurl -laws-checksums" + -lxml2 -lz -lbz2 -lcurl -laws-checksums -ls2n-tls -aws-lc" dnl AC_CANONICAL_HOST is needed to access the 'host_os' variable AC_CANONICAL_HOST From ea45f4a69d0db6d29a5b4388b0fc2e7e11ab3fdd Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Wed, 26 Nov 2025 18:12:24 -0500 Subject: [PATCH 69/77] but that's not what it's all about --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 621cdd867b..1197b66226 100644 --- a/configure.ac +++ b/configure.ac @@ -601,7 +601,7 @@ AM_COND_IF([LOOK_FOR_AWS], -laws-c-http -laws-c-compression -laws-c-event-stream \ -laws-c-auth -laws-c-cal \ -laws-c-mqtt -laws-c-s3 -laws-c-sdkutils -laws-c-common \ - -lxml2 -lz -lbz2 -lcurl -laws-checksums -ls2n-tls -aws-lc" + -lxml2 -lz -lbz2 -lcurl -laws-checksums -ls2n-tls -laws-lc -lcrypto" dnl AC_CANONICAL_HOST is needed to access the 'host_os' variable AC_CANONICAL_HOST From 9f9d3d81fc0aa8e6b36e0af09f0f918ea78ff7f6 Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Mon, 1 Dec 2025 09:46:55 -0500 Subject: [PATCH 70/77] simplify --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 1197b66226..2caddebbd5 100644 --- a/configure.ac +++ b/configure.ac @@ -601,7 +601,7 @@ AM_COND_IF([LOOK_FOR_AWS], -laws-c-http -laws-c-compression -laws-c-event-stream \ -laws-c-auth -laws-c-cal \ -laws-c-mqtt -laws-c-s3 -laws-c-sdkutils -laws-c-common \ - -lxml2 -lz -lbz2 -lcurl -laws-checksums -ls2n-tls -laws-lc -lcrypto" + -lxml2 -lz -lbz2 -lcurl -laws-checksums -lcrypto" dnl AC_CANONICAL_HOST is needed to access the 'host_os' variable AC_CANONICAL_HOST From 6a82907ba76ce7377ffebe1c4d74d9b1171a5d55 Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Mon, 1 Dec 2025 12:30:39 -0500 Subject: [PATCH 71/77] Add s2n now that it is available in hyrax-dependencies --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 2caddebbd5..38fbe13877 100644 --- a/configure.ac +++ b/configure.ac @@ -601,7 +601,7 @@ AM_COND_IF([LOOK_FOR_AWS], -laws-c-http -laws-c-compression -laws-c-event-stream \ -laws-c-auth -laws-c-cal \ -laws-c-mqtt -laws-c-s3 -laws-c-sdkutils -laws-c-common \ - -lxml2 -lz -lbz2 -lcurl -laws-checksums -lcrypto" + -lxml2 -lz -lbz2 -lcurl -laws-checksums -lcrypto -ls2n" dnl AC_CANONICAL_HOST is needed to access the 'host_os' variable AC_CANONICAL_HOST From fa822697b68668dff14542cef313d624e15cdcd0 Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Mon, 8 Dec 2025 15:02:41 -0500 Subject: [PATCH 72/77] shot in dark: maybe we don't need aws-crt-cpp --- configure.ac | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index 38fbe13877..92f235f275 100644 --- a/configure.ac +++ b/configure.ac @@ -597,11 +597,11 @@ AM_COND_IF([LOOK_FOR_AWS], AS_IF([$found_aws_h && $found_s3client_h], [aws_prefix=$ac_bes_dependencies_prefix - aws_libs=" -laws-cpp-sdk-s3 -laws-cpp-sdk-core -laws-crt-cpp -laws-c-io \ + aws_libs=" -laws-cpp-sdk-s3 -laws-cpp-sdk-core -laws-c-io \ -laws-c-http -laws-c-compression -laws-c-event-stream \ -laws-c-auth -laws-c-cal \ -laws-c-mqtt -laws-c-s3 -laws-c-sdkutils -laws-c-common \ - -lxml2 -lz -lbz2 -lcurl -laws-checksums -lcrypto -ls2n" + -lxml2 -lz -lbz2 -lcurl -laws-checksums" dnl AC_CANONICAL_HOST is needed to access the 'host_os' variable AC_CANONICAL_HOST From 7029be6b69f4ffcc8802f971f4747551e9954990 Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Mon, 8 Dec 2025 16:25:29 -0500 Subject: [PATCH 73/77] comment out travis again for failing fast --- .travis.yml | 102 ++++++++++++++++++++++++++-------------------------- 1 file changed, 51 insertions(+), 51 deletions(-) diff --git a/.travis.yml b/.travis.yml index e5a97df107..27de31830a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -144,22 +144,22 @@ stages: jobs: include: - - stage: build-and-package - name: "check" - script: - - export BES_BUILD=main - # - export RUNTESTFLAGS="-v" - - autoreconf --force --install --verbose - - ./configure --disable-dependency-tracking --prefix=$prefix --with-dependencies=$prefix/deps $GDAL_OPTION --enable-developer - - make -j16 && make install && besctl start && make check -j16 && besctl stop + # - stage: build-and-package + # name: "check" + # script: + # - export BES_BUILD=main + # # - export RUNTESTFLAGS="-v" + # - autoreconf --force --install --verbose + # - ./configure --disable-dependency-tracking --prefix=$prefix --with-dependencies=$prefix/deps $GDAL_OPTION --enable-developer + # - make -j16 && make install && besctl start && make check -j16 && besctl stop - - stage: build-and-package - name: "distcheck" - script: - - export BES_BUILD=distcheck - - autoreconf --force --install --verbose - - ./configure --disable-dependency-tracking --prefix=$prefix --with-dependencies=$prefix/deps $GDAL_OPTION--enable-developer - - make distcheck -j16 GZIP_ENV=--fast + # - stage: build-and-package + # name: "distcheck" + # script: + # - export BES_BUILD=distcheck + # - autoreconf --force --install --verbose + # - ./configure --disable-dependency-tracking --prefix=$prefix --with-dependencies=$prefix/deps $GDAL_OPTION--enable-developer + # - make distcheck -j16 GZIP_ENV=--fast - stage: build-and-package name: "Enterprise Linux 8 RPMs (via Rocky8)" @@ -178,20 +178,20 @@ jobs: --env AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY opendap/rocky8_hyrax_builder:latest /root/travis/travis/build-rpm.sh - - stage: build-and-package - name: "dist" - script: - - export BES_BUILD=srcdist - - autoreconf --force --install --verbose - - ./configure --disable-dependency-tracking --prefix=$prefix --with-dependencies=$prefix/deps $GDAL_OPTION --with-build=$BES_BUILD_NUMBER - - make dist -j7 - # Make both a bes-- and bes-snapshot tar.gz. This will simplify - # other operations that use the bes source code. Note that the VERSION file holds a - # string that is the version number that is set by the configure script and the build - # number passed into configure when it is run. jhrg 3/23/21 - - SOURCE_VERSION=$(cat bes_VERSION) - - mv bes-*.tar.gz bes-$SOURCE_VERSION.tar.gz - - cp bes-$SOURCE_VERSION.tar.gz bes-snapshot.tar.gz +# - stage: build-and-package +# name: "dist" +# script: +# - export BES_BUILD=srcdist +# - autoreconf --force --install --verbose +# - ./configure --disable-dependency-tracking --prefix=$prefix --with-dependencies=$prefix/deps $GDAL_OPTION --with-build=$BES_BUILD_NUMBER +# - make dist -j7 +# # Make both a bes-- and bes-snapshot tar.gz. This will simplify +# # other operations that use the bes source code. Note that the VERSION file holds a +# # string that is the version number that is set by the configure script and the build +# # number passed into configure when it is run. jhrg 3/23/21 +# - SOURCE_VERSION=$(cat bes_VERSION) +# - mv bes-*.tar.gz bes-$SOURCE_VERSION.tar.gz +# - cp bes-$SOURCE_VERSION.tar.gz bes-snapshot.tar.gz # - stage: scan # name: "scan bes" @@ -236,26 +236,26 @@ jobs: # after_script: # - ./travis/upload-test-results.sh -before_deploy: - - export DEPLOY="S3" - # Make sure that we have the target dir... - - mkdir -p $TRAVIS_BUILD_DIR/package; - # Source distribution prep (copies both the 'version' and 'snapshot') - - if test "$BES_BUILD" = "srcdist"; then cp bes-*.tar.gz $TRAVIS_BUILD_DIR/package; fi - # Rocky8 distribution prep - - if test "$BES_BUILD" = "rocky8"; then ./travis/rpm-to-package-dir.sh "el8"; fi - # Check for the stuff... - - ls -l $TRAVIS_BUILD_DIR/package +# before_deploy: +# - export DEPLOY="S3" +# # Make sure that we have the target dir... +# - mkdir -p $TRAVIS_BUILD_DIR/package; +# # Source distribution prep (copies both the 'version' and 'snapshot') +# - if test "$BES_BUILD" = "srcdist"; then cp bes-*.tar.gz $TRAVIS_BUILD_DIR/package; fi +# # Rocky8 distribution prep +# - if test "$BES_BUILD" = "rocky8"; then ./travis/rpm-to-package-dir.sh "el8"; fi +# # Check for the stuff... +# - ls -l $TRAVIS_BUILD_DIR/package -# The deploy section copies the snapshot build product our S3 bucket and to www.opendap.org -deploy: - # Push all build results to our S3 bucket - - provider: s3 - access_key_id: $AWS_ACCESS_KEY_ID - secret_access_key: $AWS_SECRET_ACCESS_KEY - bucket: opendap.travis.build - skip_cleanup: true - local_dir: $TRAVIS_BUILD_DIR/package - on: - all_branches: true - condition: $BES_BUILD =~ ^rocky8|srcdist$ +# # The deploy section copies the snapshot build product our S3 bucket and to www.opendap.org +# deploy: +# # Push all build results to our S3 bucket +# - provider: s3 +# access_key_id: $AWS_ACCESS_KEY_ID +# secret_access_key: $AWS_SECRET_ACCESS_KEY +# bucket: opendap.travis.build +# skip_cleanup: true +# local_dir: $TRAVIS_BUILD_DIR/package +# on: +# all_branches: true +# condition: $BES_BUILD =~ ^rocky8|srcdist$ From d31f1d886ca50a0f90a9e6edefbf912ace88c1d7 Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Mon, 8 Dec 2025 16:35:42 -0500 Subject: [PATCH 74/77] add crt back in --- configure.ac | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index 92f235f275..23325d9242 100644 --- a/configure.ac +++ b/configure.ac @@ -597,11 +597,11 @@ AM_COND_IF([LOOK_FOR_AWS], AS_IF([$found_aws_h && $found_s3client_h], [aws_prefix=$ac_bes_dependencies_prefix - aws_libs=" -laws-cpp-sdk-s3 -laws-cpp-sdk-core -laws-c-io \ + aws_libs=" -laws-cpp-sdk-s3 -laws-cpp-sdk-core -laws-crt-cpp -laws-c-io \ -laws-c-http -laws-c-compression -laws-c-event-stream \ -laws-c-auth -laws-c-cal \ -laws-c-mqtt -laws-c-s3 -laws-c-sdkutils -laws-c-common \ - -lxml2 -lz -lbz2 -lcurl -laws-checksums" + -lxml2 -lz -lbz2 -lcurl -laws-checksums -ls2n" dnl AC_CANONICAL_HOST is needed to access the 'host_os' variable AC_CANONICAL_HOST From 8e1ef1059edc691ad54d09c787eaa6fca4b0b899 Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Fri, 13 Feb 2026 09:54:51 -0500 Subject: [PATCH 75/77] fix build_dmrpp linking --- modules/dmrpp_module/Makefile.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/dmrpp_module/Makefile.am b/modules/dmrpp_module/Makefile.am index c881f06f51..bb8c2c2492 100644 --- a/modules/dmrpp_module/Makefile.am +++ b/modules/dmrpp_module/Makefile.am @@ -88,7 +88,7 @@ h5common.h: Makefile build_dmrpp_CPPFLAGS = $(AM_CPPFLAGS) $(H5_CPPFLAGS) -I$(top_srcdir)/modules/hdf5_handler build_dmrpp_SOURCES = $(BES_SRCS) $(BES_HDRS) DmrppRequestHandler.cc DmrppRequestHandler.h \ build_dmrpp.cc build_dmrpp_util.cc build_dmrpp_util.h h5common.cc h5common.h -build_dmrpp_LDFLAGS = $(top_builddir)/dap/.libs/libdap_module.a +build_dmrpp_LDFLAGS = $(top_builddir)/dap/.libs/libdap_module.a $(AM_LDFLAGS) build_dmrpp_LDADD = $(BES_DISPATCH_LIB) $(BES_HTTP_LIB) $(H5_LDFLAGS) \ $(H5_LIBS) $(DAP_SERVER_LIBS) $(DAP_CLIENT_LIBS) $(OPENSSL_LDFLAGS) $(OPENSSL_LIBS) \ $(LDADD) $(XML2_LIBS) $(BYTESWAP_LIBS) -lz From 20d56b0852853a64641ed266ea0bb99b42f3425e Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Fri, 13 Feb 2026 09:57:53 -0500 Subject: [PATCH 76/77] uncomment travis --- .travis.yml | 138 ++++++++++++++++++++++++++-------------------------- 1 file changed, 69 insertions(+), 69 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5d3e7e6546..1e5d80ecbc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -140,24 +140,24 @@ stages: jobs: include: - # - stage: build-and-package - # name: "check" - # script: - # - export BES_BUILD=main - # # - export RUNTESTFLAGS="-v" - # - autoreconf --force --install --verbose - # - ./configure --disable-dependency-tracking --prefix=$prefix --with-dependencies=$prefix/deps $GDAL_OPTION --enable-developer - # - export LD_LIBRARY_PATH="/home/travis/install/deps/lib:$LD_LIBRARY_PATH" - # - echo "LD_LIBRARY_PATH - $LD_LIBRARY_PATH" >&2 - # - make -j16 && make install && besctl start && make check -j16 && besctl stop + - stage: build-and-package + name: "check" + script: + - export BES_BUILD=main + # - export RUNTESTFLAGS="-v" + - autoreconf --force --install --verbose + - ./configure --disable-dependency-tracking --prefix=$prefix --with-dependencies=$prefix/deps $GDAL_OPTION --enable-developer + - export LD_LIBRARY_PATH="/home/travis/install/deps/lib:$LD_LIBRARY_PATH" + - echo "LD_LIBRARY_PATH - $LD_LIBRARY_PATH" >&2 + - make -j16 && make install && besctl start && make check -j16 && besctl stop - # - stage: build-and-package - # name: "distcheck" - # script: - # - export BES_BUILD=distcheck - # - autoreconf --force --install --verbose - # - ./configure --disable-dependency-tracking --prefix=$prefix --with-dependencies=$prefix/deps $GDAL_OPTION--enable-developer - # - make distcheck -j16 GZIP_ENV=--fast + - stage: build-and-package + name: "distcheck" + script: + - export BES_BUILD=distcheck + - autoreconf --force --install --verbose + - ./configure --disable-dependency-tracking --prefix=$prefix --with-dependencies=$prefix/deps $GDAL_OPTION--enable-developer + - make distcheck -j16 GZIP_ENV=--fast - stage: build-and-package name: "Enterprise Linux 8 RPMs (via Rocky8)" @@ -218,63 +218,63 @@ jobs: echo "Tagging image with BES version: ${BUILD_VERSION_TAG}" docker tag ${SNAPSHOT_IMAGE_TAG} ${BUILD_VERSION_TAG} -# - stage: build-and-package -# name: "dist" -# script: -# - export BES_BUILD=srcdist -# - autoreconf --force --install --verbose -# - ./configure --disable-dependency-tracking --prefix=$prefix --with-dependencies=$prefix/deps $GDAL_OPTION --with-build=$BES_BUILD_NUMBER -# - make dist -j7 -# # Make both a bes-- and bes-snapshot tar.gz. This will simplify -# # other operations that use the bes source code. Note that the VERSION file holds a -# # string that is the version number that is set by the configure script and the build -# # number passed into configure when it is run. jhrg 3/23/21 -# - SOURCE_VERSION=$(cat bes_VERSION) -# - mv bes-*.tar.gz bes-$SOURCE_VERSION.tar.gz -# - cp bes-$SOURCE_VERSION.tar.gz bes-snapshot.tar.gz + - stage: build-and-package + name: "dist" + script: + - export BES_BUILD=srcdist + - autoreconf --force --install --verbose + - ./configure --disable-dependency-tracking --prefix=$prefix --with-dependencies=$prefix/deps $GDAL_OPTION --with-build=$BES_BUILD_NUMBER + - make dist -j7 + # Make both a bes-- and bes-snapshot tar.gz. This will simplify + # other operations that use the bes source code. Note that the VERSION file holds a + # string that is the version number that is set by the configure script and the build + # number passed into configure when it is run. jhrg 3/23/21 + - SOURCE_VERSION=$(cat bes_VERSION) + - mv bes-*.tar.gz bes-$SOURCE_VERSION.tar.gz + - cp bes-$SOURCE_VERSION.tar.gz bes-snapshot.tar.gz -# - stage: scan -# name: "scan bes" -# script: -# - export BES_BUILD=sonar-bes-framework -# - autoreconf --force --install --verbose -# - ./configure --disable-dependency-tracking --prefix=$prefix --with-dependencies=$prefix/deps --enable-developer --enable-coverage -# - build-wrapper-linux-x86-64 --out-dir bw-output make -j16 -# - sonar-scanner -Dproject.settings=sonar-bes-framework.properties -Dsonar.login=$SONAR_LOGIN -# - curl -s https://sonarcloud.io/api/project_badges/quality_gate?project=opendap-bes | grep "QUALITY GATE PASS" + - stage: scan + name: "scan bes" + script: + - export BES_BUILD=sonar-bes-framework + - autoreconf --force --install --verbose + - ./configure --disable-dependency-tracking --prefix=$prefix --with-dependencies=$prefix/deps --enable-developer --enable-coverage + - build-wrapper-linux-x86-64 --out-dir bw-output make -j16 + - sonar-scanner -Dproject.settings=sonar-bes-framework.properties -Dsonar.login=$SONAR_LOGIN + - curl -s https://sonarcloud.io/api/project_badges/quality_gate?project=opendap-bes | grep "QUALITY GATE PASS" -# - stage: scan -# name: "scan bes-modules-1" -# script: -# - export BES_BUILD=sonar-bes-modules -# - autoreconf --force --install --verbose -# - ./configure --disable-dependency-tracking --prefix=$prefix --with-dependencies=$prefix/deps --enable-developer --enable-coverage -# - build-wrapper-linux-x86-64 --out-dir bw-output make -j16 -# - sonar-scanner -Dproject.settings=sonar-bes-modules-1.properties -Dsonar.login=$SONAR_MODULES_LOGIN -# - curl -s https://sonarcloud.io/api/project_badges/quality_gate?project=opendap-bes-modules | grep "QUALITY GATE PASS" + - stage: scan + name: "scan bes-modules-1" + script: + - export BES_BUILD=sonar-bes-modules + - autoreconf --force --install --verbose + - ./configure --disable-dependency-tracking --prefix=$prefix --with-dependencies=$prefix/deps --enable-developer --enable-coverage + - build-wrapper-linux-x86-64 --out-dir bw-output make -j16 + - sonar-scanner -Dproject.settings=sonar-bes-modules-1.properties -Dsonar.login=$SONAR_MODULES_LOGIN + - curl -s https://sonarcloud.io/api/project_badges/quality_gate?project=opendap-bes-modules | grep "QUALITY GATE PASS" -# - stage: scan -# name: "scan bes-hdf-handlers" -# script: -# - export BES_BUILD=sonar-bes-hdf-handlers -# - autoreconf --force --install --verbose -# - ./configure --disable-dependency-tracking --prefix=$prefix --with-dependencies=$prefix/deps --enable-developer --enable-coverage -# - build-wrapper-linux-x86-64 --out-dir bw-output make -j16 -# - sonar-scanner -Dproject.settings=sonar-bes-hdf-handlers.properties -Dsonar.login=$SONAR_SUBMODULES_LOGIN -# # We call the hdf4/5 handlers scan opendap-bes-submodules for historical reasons. jhrg 1/13/22 -# - curl -s https://sonarcloud.io/api/project_badges/quality_gate?project=opendap-bes-submodules | grep "QUALITY GATE PASS" + - stage: scan + name: "scan bes-hdf-handlers" + script: + - export BES_BUILD=sonar-bes-hdf-handlers + - autoreconf --force --install --verbose + - ./configure --disable-dependency-tracking --prefix=$prefix --with-dependencies=$prefix/deps --enable-developer --enable-coverage + - build-wrapper-linux-x86-64 --out-dir bw-output make -j16 + - sonar-scanner -Dproject.settings=sonar-bes-hdf-handlers.properties -Dsonar.login=$SONAR_SUBMODULES_LOGIN + # We call the hdf4/5 handlers scan opendap-bes-submodules for historical reasons. jhrg 1/13/22 + - curl -s https://sonarcloud.io/api/project_badges/quality_gate?project=opendap-bes-submodules | grep "QUALITY GATE PASS" -# - stage: hyrax-olfs-trigger -# name: "Hyrax OLFS Trigger" -# script: -# - export STAGE=hyrax-olfs -# - echo $STAGE -# - autoreconf --force --install --verbose -# - ./configure --disable-dependency-tracking --prefix=$prefix --with-dependencies=$prefix/deps --with-build=$BES_BUILD_NUMBER -# - ./travis/trigger-olfs-build.sh + - stage: hyrax-olfs-trigger + name: "Hyrax OLFS Trigger" + script: + - export STAGE=hyrax-olfs + - echo $STAGE + - autoreconf --force --install --verbose + - ./configure --disable-dependency-tracking --prefix=$prefix --with-dependencies=$prefix/deps --with-build=$BES_BUILD_NUMBER + - ./travis/trigger-olfs-build.sh -# after_script: -# - ./travis/upload-test-results.sh +after_script: + - ./travis/upload-test-results.sh before_deploy: - export DEPLOY="S3" From 175a7824e7456efcc7ca76bc7ebfa239f3ffbe27 Mon Sep 17 00:00:00 2001 From: Hannah Robertson <1076701+hannahilea@users.noreply.github.com> Date: Fri, 13 Feb 2026 14:29:41 -0500 Subject: [PATCH 77/77] remove invalid s2n dep --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 85cdb4c771..b7669d0051 100644 --- a/configure.ac +++ b/configure.ac @@ -635,7 +635,7 @@ AM_COND_IF([LOOK_FOR_AWS], -laws-c-http -laws-c-compression -laws-c-event-stream \ -laws-c-auth -laws-c-cal \ -laws-c-mqtt -laws-c-s3 -laws-c-sdkutils -laws-c-common \ - -lxml2 -lz -lbz2 -lcurl -laws-checksums -ls2n" + -lxml2 -lz -lbz2 -lcurl -laws-checksums" dnl AC_CANONICAL_HOST is needed to access the 'host_os' variable AC_CANONICAL_HOST