diff --git a/src/bk_download.cpp b/src/bk_download.cpp index 51e15a0d..c8a06b86 100644 --- a/src/bk_download.cpp +++ b/src/bk_download.cpp @@ -62,17 +62,13 @@ void BkDownload::switch_to_local_file() { } bool BkDownload::download_done() { - auto tracer = overlaybd_otel::get_tracer("overlaybd"); - auto parent_span = opentelemetry::trace::Tracer::GetCurrentSpan(); - opentelemetry::trace::StartSpanOptions options; - options.parent = parent_span->GetContext(); - auto span = tracer->StartSpan("overlaybd.download.verify_and_commit", {}, {}, options); + auto tracer = overlaybd_otel::GetTracer(); + auto span = tracer->StartSpan("download.verify_and_commit"); auto scope = tracer->WithActiveSpan(span); auto lfs = new_localfs_adaptor(); if (!lfs) { span->SetAttribute("error", "failed_to_create_fs_adaptor"); - span->End(); LOG_ERROR("new_localfs_adaptor() return NULL"); return false; } @@ -87,9 +83,7 @@ bool BkDownload::download_done() { // verify sha256 photon::semaphore done; std::string shares; - opentelemetry::trace::StartSpanOptions verify_options; - verify_options.parent = opentelemetry::trace::Tracer::GetCurrentSpan()->GetContext(); - auto verify_span = tracer->StartSpan("sha256_verification", {}, {}, verify_options); + auto verify_span = tracer->StartSpan("sha256_verification"); auto verify_scope = tracer->WithActiveSpan(verify_span); verify_span->SetAttribute("expected_digest", digest); @@ -109,81 +103,47 @@ bool BkDownload::download_done() { span->SetAttribute("error", "checksum_mismatch"); span->SetAttribute("expected_digest", digest); span->SetAttribute("actual_digest", shares); - span->End(); LOG_ERROR("verify checksum ` failed (expect: `, got: `)", old_name, digest, shares); force_download = true; // force redownload next time return false; } - opentelemetry::trace::StartSpanOptions rename_options; - rename_options.parent = opentelemetry::trace::Tracer::GetCurrentSpan()->GetContext(); - auto rename_span = tracer->StartSpan("rename_to_commit", {}, {}, rename_options); - auto rename_scope = tracer->WithActiveSpan(rename_span); int ret = lfs->rename(old_name.c_str(), new_name.c_str()); - rename_span->SetAttribute("success", ret == 0); - if (ret != 0) { span->SetAttribute("error", "rename_failed"); - span->End(); LOG_ERRNO_RETURN(0, false, "rename(`,`) failed", old_name, new_name); } span->SetAttribute("success", true); LOG_INFO("download verify done. rename(`,`) success", old_name, new_name); - span->End(); return true; } bool BkDownload::download() { - auto tracer = overlaybd_otel::get_tracer("overlaybd"); - // Get current span context from parent if it exists - auto parent_span = opentelemetry::trace::Tracer::GetCurrentSpan(); - opentelemetry::trace::StartSpanOptions options; - options.parent = parent_span->GetContext(); - auto span = tracer->StartSpan("overlaybd.download.lifecycle", {}, {}, options); + auto tracer = overlaybd_otel::GetTracer(); + auto span = tracer->StartSpan("download"); auto scope = tracer->WithActiveSpan(span); - span->SetAttribute("url", url); + span->SetAttribute("original_url", url); span->SetAttribute("dir", dir); span->SetAttribute("file_size", file_size); if (check_downloaded(dir)) { - opentelemetry::trace::StartSpanOptions local_options; - local_options.parent = opentelemetry::trace::Tracer::GetCurrentSpan()->GetContext(); - auto local_span = tracer->StartSpan("overlaybd.download.switch_to_local", {}, {}, local_options); - auto local_scope = tracer->WithActiveSpan(local_span); switch_to_local_file(); span->SetAttribute("from_cache", true); - span->End(); + span->SetAttribute("success", true); return true; } span->SetAttribute("from_cache", false); bool success = false; if (download_blob()) { - opentelemetry::trace::StartSpanOptions verify_options; - verify_options.parent = opentelemetry::trace::Tracer::GetCurrentSpan()->GetContext(); - auto verify_span = tracer->StartSpan("overlaybd.download.verify", {}, {}, verify_options); - auto verify_scope = tracer->WithActiveSpan(verify_span); - if (!download_done()) { - verify_span->SetAttribute("success", false); - span->SetAttribute("success", false); - span->End(); - return false; + if (download_done()) { + switch_to_local_file(); + success = true; } - verify_span->SetAttribute("success", true); - verify_span->End(); - - opentelemetry::trace::StartSpanOptions switch_options; - switch_options.parent = opentelemetry::trace::Tracer::GetCurrentSpan()->GetContext(); - auto switch_span = tracer->StartSpan("overlaybd.download.switch_to_local", {}, {}, switch_options); - auto switch_scope = tracer->WithActiveSpan(switch_span); - switch_to_local_file(); - success = true; } - span->SetAttribute("success", success); - span->End(); return success; } @@ -201,11 +161,8 @@ void BkDownload::unlock_file() { } bool BkDownload::download_blob() { - auto tracer = overlaybd_otel::get_tracer("overlaybd"); - auto parent_span = opentelemetry::trace::Tracer::GetCurrentSpan(); - opentelemetry::trace::StartSpanOptions options; - options.parent = parent_span->GetContext(); - auto span = tracer->StartSpan("overlaybd.download.blob", {}, {}, options); + auto tracer = overlaybd_otel::GetTracer(); + auto span = tracer->StartSpan("download.blob"); auto scope = tracer->WithActiveSpan(span); std::string dl_file_path = dir + "/" + DOWNLOAD_TMP_NAME; @@ -230,7 +187,6 @@ bool BkDownload::download_blob() { auto dst = open_localfile_adaptor(dl_file_path.c_str(), O_RDWR | O_CREAT, 0644); if (dst == nullptr) { span->SetAttribute("error", "failed_to_open_dst"); - span->End(); LOG_ERRNO_RETURN(0, false, "failed to open dst file `", dl_file_path.c_str()); } DEFER(delete dst;); @@ -243,7 +199,6 @@ bool BkDownload::download_blob() { ::posix_memalign(&buff, ALIGNMENT, bs); if (buff == nullptr) { span->SetAttribute("error", "failed_to_allocate_buffer"); - span->End(); LOG_ERRNO_RETURN(0, false, "failed to allocate buffer with ", VALUE(bs)); } DEFER(free(buff)); @@ -256,7 +211,6 @@ bool BkDownload::download_blob() { while (offset < (ssize_t)file_size) { if (running != 1) { span->SetAttribute("error", "download_interrupted"); - span->End(); LOG_INFO("image file exit when background downloading"); return false; } @@ -279,22 +233,15 @@ bool BkDownload::download_blob() { if (!(retry--)) { span->SetAttribute("error", "max_read_retries_exceeded"); span->SetAttribute("failed_offset", offset); - span->End(); LOG_ERROR_RETURN(EIO, false, "failed to read at ", VALUE(offset), VALUE(count)); } ssize_t rlen; { - auto read_span = tracer->StartSpan("overlaybd.download.read_block"); - auto read_scope = tracer->WithActiveSpan(read_span); - read_span->SetAttribute("offset", offset); - read_span->SetAttribute("size", count); SCOPE_AUDIT("bk_download", AU_FILEOP(url, offset, rlen)); rlen = src->pread(buff, bs, offset); if (rlen >= 0) { - read_span->SetAttribute("bytes_read", rlen); total_bytes_read += rlen; } - read_span->End(); } if (rlen < 0) { retries++; @@ -306,19 +253,12 @@ bool BkDownload::download_blob() { if (!(retry--)) { span->SetAttribute("error", "max_write_retries_exceeded"); span->SetAttribute("failed_offset", offset); - span->End(); LOG_ERROR_RETURN(EIO, false, "failed to write at ", VALUE(offset), VALUE(count)); } - auto write_span = tracer->StartSpan("overlaybd.download.write_block"); - auto write_scope = tracer->WithActiveSpan(write_span); - write_span->SetAttribute("offset", offset); - write_span->SetAttribute("size", count); auto wlen = dst->pwrite(buff, count, offset); if (wlen >= 0) { - write_span->SetAttribute("bytes_written", wlen); total_bytes_written += wlen; } - write_span->End(); // but once write lenth larger than read length treats as OK if (wlen < rlen) { retries++; @@ -333,12 +273,11 @@ bool BkDownload::download_blob() { span->SetAttribute("total_retries", retries); span->SetAttribute("success", true); LOG_INFO("download blob done. (`)", dl_file_path); - span->End(); return true; } void bk_download_proc(std::list &dl_list, uint64_t delay_sec, int &running) { - auto tracer = overlaybd_otel::get_tracer("overlaybd"); + auto tracer = overlaybd_otel::GetTracer(); auto span = tracer->StartSpan("background_download_process"); auto scope = tracer->WithActiveSpan(span); @@ -375,7 +314,6 @@ void bk_download_proc(std::list &dl_list, uint64_t delay_sec if (!dl_item->lock_file()) { dl_span->SetAttribute("status", "lock_failed"); - dl_span->End(); dl_list.push_back(dl_item); continue; } @@ -385,7 +323,6 @@ void bk_download_proc(std::list &dl_list, uint64_t delay_sec if (running != 1) { dl_span->SetAttribute("status", "interrupted"); - dl_span->End(); LOG_WARN("image exited, background download exit..."); delete dl_item; break; @@ -393,14 +330,12 @@ void bk_download_proc(std::list &dl_list, uint64_t delay_sec if (!succ && dl_item->try_cnt > 0) { dl_span->SetAttribute("status", "retry"); - dl_span->End(); dl_list.push_back(dl_item); LOG_WARN("download failed, push back to download queue and retry `", dl_item->dir); continue; } dl_span->SetAttribute("status", succ ? "success" : "failed"); - dl_span->End(); LOG_DEBUG("finish downloading or no retry any more: `, retry_cnt: `", dl_item->dir, dl_item->try_cnt); @@ -417,7 +352,6 @@ void bk_download_proc(std::list &dl_list, uint64_t delay_sec } } LOG_INFO("BACKGROUND DOWNLOAD THREAD EXIT."); - span->End(); } } // namespace BKDL diff --git a/src/overlaybd/otel/CMakeLists.txt b/src/overlaybd/otel/CMakeLists.txt index d90a40fb..f4021b7f 100644 --- a/src/overlaybd/otel/CMakeLists.txt +++ b/src/overlaybd/otel/CMakeLists.txt @@ -1,5 +1,6 @@ set(SOURCE_OTEL tracer_common.cpp + context_storage.cpp ) include(FetchContent) @@ -24,6 +25,7 @@ set_target_properties(otel_lib PROPERTIES LINKER_LANGUAGE CXX) target_include_directories(otel_lib PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} + ${PHOTON_INCLUDE_DIR} ${opentelemetry-cpp_SOURCE_DIR}/api/include ${opentelemetry-cpp_SOURCE_DIR}/sdk/include ${opentelemetry-cpp_SOURCE_DIR}/exporters/otlp/include diff --git a/src/overlaybd/otel/context_storage.cpp b/src/overlaybd/otel/context_storage.cpp new file mode 100644 index 00000000..beb2597d --- /dev/null +++ b/src/overlaybd/otel/context_storage.cpp @@ -0,0 +1,45 @@ +#include "context_storage.h" + +#include + +#include "photon/thread/thread-local.h" + +using opentelemetry::context::Context; +using opentelemetry::context::Token; + +static std::vector& GetStack() { + static photon::thread_local_ptr> stack; + return *stack; +} + +namespace overlaybd_otel { + +Context LibPhotonContextStorage::GetCurrent() noexcept { + std::vector& stack = GetStack(); + if (stack.empty()) { + return Context(); + } + return stack.back(); +} + +bool LibPhotonContextStorage::Detach(Token &token) noexcept { + std::vector& stack = GetStack(); + if (stack.empty()) { + return false; + } + for (auto it = stack.rbegin(); it != stack.rend(); ++it) { + if (token == *it) { + stack.erase(std::prev(it.base()), stack.end()); + return true; + } + } + return false; +} + +opentelemetry::nostd::unique_ptr LibPhotonContextStorage::Attach(const Context& context) noexcept { + std::vector& stack = GetStack(); + stack.push_back(context); + return CreateToken(context); +} + +} // overlaybd_otel diff --git a/src/overlaybd/otel/context_storage.h b/src/overlaybd/otel/context_storage.h new file mode 100644 index 00000000..3da94f47 --- /dev/null +++ b/src/overlaybd/otel/context_storage.h @@ -0,0 +1,21 @@ +#pragma once +#include +#include + +#include "opentelemetry/context/runtime_context.h" + +namespace overlaybd_otel { + +// ContextStorage implementation that uses photonlib's coroutine-aware thread local storage. +class LibPhotonContextStorage : public opentelemetry::context::RuntimeContextStorage { +public: + LibPhotonContextStorage() noexcept = default; + + opentelemetry::context::Context GetCurrent() noexcept override; + + bool Detach(opentelemetry::context::Token &token) noexcept override; + + opentelemetry::nostd::unique_ptr Attach(const opentelemetry::context::Context& context) noexcept override; +}; + +} // overlaybd_otel diff --git a/src/overlaybd/otel/tracer_common.cpp b/src/overlaybd/otel/tracer_common.cpp index 27d0234c..dab69679 100644 --- a/src/overlaybd/otel/tracer_common.cpp +++ b/src/overlaybd/otel/tracer_common.cpp @@ -1,37 +1,28 @@ +#include "opentelemetry/context/runtime_context.h" #include "tracer_common.h" +#include "context_storage.h" namespace overlaybd_otel { void InitTracer() { - // Create OTLP HTTP exporter configuration - opentelemetry::exporter::otlp::OtlpHttpExporterOptions opts; - + // Install a libphoton coroutine-aware context storage implementation, which uses libphoton's thread-local implementation. + opentelemetry::context::RuntimeContext::SetRuntimeContextStorage( + opentelemetry::nostd::unique_ptr( + new LibPhotonContextStorage())); + // Create OTLP/HTTP exporter using the factory - auto exporter = opentelemetry::exporter::otlp::OtlpHttpExporterFactory::Create(opts); - - opentelemetry::sdk::trace::BatchSpanProcessorOptions bspOpts{}; - auto processor = - opentelemetry::sdk::trace::BatchSpanProcessorFactory::Create(std::move(exporter), bspOpts); - - // Create a simple processor (we'll use simple instead of batch for now) - // auto processor = - // opentelemetry::sdk::trace::SimpleSpanProcessorFactory::Create(std::move(exporter)); std::vector> processors; - processors.push_back(std::move(processor)); + processors.push_back( + opentelemetry::sdk::trace::BatchSpanProcessorFactory::Create( + opentelemetry::exporter::otlp::OtlpHttpExporterFactory::Create(), {})); // Default is an always-on sampler - std::unique_ptr context = - opentelemetry::sdk::trace::TracerContextFactory::Create(std::move(processors)); std::shared_ptr provider = - opentelemetry::sdk::trace::TracerProviderFactory::Create(std::move(context)); + opentelemetry::sdk::trace::TracerProviderFactory::Create( + opentelemetry::sdk::trace::TracerContextFactory::Create(std::move(processors))); // Set the global trace provider opentelemetry::sdk::trace::Provider::SetTracerProvider(provider); - - // // set global propagator - // opentelemetry::context::propagation::GlobalTextMapPropagator::SetGlobalPropagator( - // opentelemetry::nostd::shared_ptr( - // new opentelemetry::trace::propagation::HttpTraceContext())); } void CleanupTracer() { @@ -39,9 +30,8 @@ void CleanupTracer() { opentelemetry::sdk::trace::Provider::SetTracerProvider(none); } -opentelemetry::nostd::shared_ptr get_tracer(std::string tracer_name) { - auto provider = opentelemetry::trace::Provider::GetTracerProvider(); - return provider->GetTracer(tracer_name); +opentelemetry::nostd::shared_ptr GetTracer(opentelemetry::nostd::string_view tracer_name) { + return opentelemetry::trace::Provider::GetTracerProvider()->GetTracer(tracer_name); } } // namespace overlaybd_otel diff --git a/src/overlaybd/otel/tracer_common.h b/src/overlaybd/otel/tracer_common.h index 31ffbdf6..2613056e 100644 --- a/src/overlaybd/otel/tracer_common.h +++ b/src/overlaybd/otel/tracer_common.h @@ -25,6 +25,6 @@ namespace overlaybd_otel { void InitTracer(); void CleanupTracer(); -opentelemetry::nostd::shared_ptr get_tracer(std::string tracer_name); +opentelemetry::nostd::shared_ptr GetTracer(opentelemetry::nostd::string_view tracer_name = "overlaybd"); } // namespace overlaybd_otel \ No newline at end of file diff --git a/src/overlaybd/registryfs/CMakeLists.txt b/src/overlaybd/registryfs/CMakeLists.txt index 7ce30abe..7a9b3fec 100644 --- a/src/overlaybd/registryfs/CMakeLists.txt +++ b/src/overlaybd/registryfs/CMakeLists.txt @@ -8,3 +8,4 @@ target_include_directories(registryfs_lib PUBLIC ${rapidjson_SOURCE_DIR}/include ${PHOTON_INCLUDE_DIR} ) +target_link_libraries(registryfs_lib PRIVATE otel_lib) diff --git a/src/overlaybd/registryfs/registryfs_v2.cpp b/src/overlaybd/registryfs/registryfs_v2.cpp index a9d59a05..de92778e 100644 --- a/src/overlaybd/registryfs/registryfs_v2.cpp +++ b/src/overlaybd/registryfs/registryfs_v2.cpp @@ -16,7 +16,6 @@ #include "../../version.h" #include "registryfs.h" - #include #include #include @@ -49,6 +48,8 @@ #include #include +#include "../otel/tracer_common.h" + using namespace photon::fs; using namespace photon::net::http; @@ -147,19 +148,19 @@ class RegistryFSImpl_v2 : public RegistryFS { if (m_tls_ctx) delete m_tls_ctx; } - long get_data(const estring &url, off_t offset, size_t count, uint64_t timeout, HTTP_OP &op) { + long get_data(const estring &url, off_t offset, size_t count, uint64_t timeout, HTTP_OP &op) { Timeout tmo(timeout); long ret = 0; UrlInfo *actual_info = m_url_info.acquire(url, [&]() -> UrlInfo * { return get_actual_url(url, tmo.timeout(), ret); }); - if (actual_info == nullptr) + if (actual_info == nullptr) { return ret; + } - estring *actual_url = (estring*)&url; - if (actual_info->mode == UrlMode::Redirect) - actual_url = &actual_info->info; + estring *actual_url = (estring*)&actual_info->info; + // use p2p proxy estring accelerate_url; if (m_accelerate.size() > 0) { @@ -168,6 +169,8 @@ class RegistryFSImpl_v2 : public RegistryFS { LOG_DEBUG("p2p_url: `", *actual_url); } + overlaybd_otel::GetTracer()->GetCurrentSpan()->SetAttribute("actual_url", *actual_url); + op.req.reset(Verb::GET, *actual_url); // set token if needed if (actual_info->mode == UrlMode::Self && !actual_info->info.empty()) { @@ -183,7 +186,6 @@ class RegistryFSImpl_v2 : public RegistryFS { m_url_info.release(url); return ret; } - m_url_info.release(url, true); LOG_ERROR_RETURN(0, ret, "Failed to fetch data ", VALUE(url), VALUE(op.status_code), VALUE(ret)); } @@ -287,7 +289,7 @@ class RegistryFSImpl_v2 : public RegistryFS { return new UrlInfo{UrlMode::Redirect, location}; } if (code == 200) { - UrlInfo *info = new UrlInfo{UrlMode::Self, ""}; + UrlInfo *info = new UrlInfo{UrlMode::Self, url}; if (authtype == AuthType::Bearer && token && !token->empty()) info->info = kBearerAuthPrefix + *token; else if (authtype == AuthType::Basic) { @@ -475,6 +477,13 @@ class RegistryFileImpl_v2 : public photon::fs::VirtualReadOnlyFile { } ssize_t preadv(const struct iovec *iov, int iovcnt, off_t offset) override { + auto tracer = overlaybd_otel::GetTracer(); + auto span = tracer->StartSpan("registryfs_v2.preadv"); + auto scope = tracer->WithActiveSpan(span); + span->SetAttribute("url", m_url); + span->SetAttribute("timeout_us", m_timeout); + span->SetAttribute("offset", offset); + if (m_filesize == 0) { struct stat stat; auto stret = fstat(&stat); @@ -492,25 +501,36 @@ class RegistryFileImpl_v2 : public photon::fs::VirtualReadOnlyFile { if (count + offset > filesize) count = filesize - offset; LOG_DEBUG("pulling blob from registry: ", VALUE(m_url), VALUE(offset), VALUE(count)); + span->SetAttribute("count", count); HTTP_OP op; auto code = m_fs->get_data(m_url, offset, count, tmo.timeout(), op); if (code != 200 && code != 206) { ERRNO eno; if (tmo.expire() < photon::now) { + span->SetAttribute("error", true); + span->SetAttribute("reason", "timeout"); LOG_ERROR_RETURN(ETIMEDOUT, -1, "timed out in preadv ", VALUE(m_url), VALUE(offset)); } if (retry--) { LOG_WARN("failed to perform HTTP GET, going to retry ", VALUE(code), VALUE(offset), VALUE(count), eno); + span->AddEvent("retry", { + { "http.status", code }, + { "attempts_left", retry }, + }); photon::thread_usleep(1000); goto again; } else { + span->SetAttribute("error", true); + span->SetAttribute("http.status", code); LOG_ERROR_RETURN(ENOENT, -1, "failed to perform HTTP GET ", VALUE(m_url), VALUE(offset)); } } - return op.resp.readv(iov, iovcnt); + ssize_t bytes_read = op.resp.readv(iov, iovcnt); + span->SetAttribute("bytes_read", bytes_read); + return bytes_read; } int64_t get_length(uint64_t timeout = -1) {