From edbe8ed40fdd57a1209af54d73f267a7e0378129 Mon Sep 17 00:00:00 2001 From: Valentin Seitz Date: Fri, 19 Jul 2024 12:29:50 +0200 Subject: [PATCH 1/7] Add NVTX --- CMakeLists.txt | 8 ++++++++ src/backends/CMakeLists.txt | 4 ++++ src/backends/default/default.hpp | 4 ++-- src/backends/detection/detection.hpp | 4 ++-- src/backends/nvtx/CMakeLists.txt | 1 + src/backends/nvtx/nvtx_pushpop.cpp | 26 ++++++++++++++++++++++++++ src/backends/nvtx/nvtx_pushpop.hpp | 18 ++++++++++++++++++ src/delegator.cpp | 14 ++++++++++++-- 8 files changed, 73 insertions(+), 6 deletions(-) create mode 100644 src/backends/nvtx/CMakeLists.txt create mode 100644 src/backends/nvtx/nvtx_pushpop.cpp create mode 100644 src/backends/nvtx/nvtx_pushpop.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 8105169..d5937e8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,6 +12,7 @@ list (APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake") # Available backend options option(ENABLE_EXTRAE "Enable Extrae" OFF) option(ENABLE_DLB "Enable DLB" OFF) +option(ENABLE_NVTX "Enable NVTX" OFF) # other options options option(BUILD_C_FORTRAN "Enable C and Fortran Interfaces" ON) @@ -86,6 +87,13 @@ if(ENABLE_DLB) endif() +if(ENABLE_NVTX) + find_package(CUDAToolkit REQUIRED COMPONENTS CUDA::nvtx3) + target_link_libraries(nesmik PRIVATE CUDA::nvtx3) + target_compile_definitions(nesmik PRIVATE "ENABLE_NVTX") +endif() + + include(FetchContent) FetchContent_Declare(json URL https://github.com/nlohmann/json/releases/download/v3.11.3/json.tar.xz) diff --git a/src/backends/CMakeLists.txt b/src/backends/CMakeLists.txt index ccb0564..f8b0022 100644 --- a/src/backends/CMakeLists.txt +++ b/src/backends/CMakeLists.txt @@ -7,3 +7,7 @@ endif() if(ENABLE_EXTRAE) add_subdirectory(extrae) endif() + +if(ENABLE_NVTX) + add_subdirectory(nvtx) +endif() diff --git a/src/backends/default/default.hpp b/src/backends/default/default.hpp index 7158c24..824c909 100644 --- a/src/backends/default/default.hpp +++ b/src/backends/default/default.hpp @@ -10,6 +10,6 @@ class NPNDefault : public NotProperlyNestedAnnotationStrategy { &info_to_start) noexcept override; virtual void RegionStop( const NotProperlyNestedRegionInformation &info_to_stop) noexcept override; - virtual void Init() noexcept; - virtual void Finalize() noexcept; + virtual void Init() noexcept override; + virtual void Finalize() noexcept override; }; diff --git a/src/backends/detection/detection.hpp b/src/backends/detection/detection.hpp index e1d8758..f077a6f 100644 --- a/src/backends/detection/detection.hpp +++ b/src/backends/detection/detection.hpp @@ -51,7 +51,7 @@ class DetectionStrategy : public ProperlyNestedAnnotationStrategy { const ProperlyNestedRegionInformation& region) noexcept override; virtual void RegionStopLast( const ProperlyNestedRegionInformation& region) noexcept override; - virtual void Init() noexcept; - virtual void Finalize() noexcept; + virtual void Init() noexcept override; + virtual void Finalize() noexcept override; }; #endif // NESMIK_DETECTION_H diff --git a/src/backends/nvtx/CMakeLists.txt b/src/backends/nvtx/CMakeLists.txt new file mode 100644 index 0000000..1fe7698 --- /dev/null +++ b/src/backends/nvtx/CMakeLists.txt @@ -0,0 +1 @@ +target_sources(nesmik PRIVATE nvtx_pushpop.cpp) diff --git a/src/backends/nvtx/nvtx_pushpop.cpp b/src/backends/nvtx/nvtx_pushpop.cpp new file mode 100644 index 0000000..63440f4 --- /dev/null +++ b/src/backends/nvtx/nvtx_pushpop.cpp @@ -0,0 +1,26 @@ +#include "nvtx_pushpop.hpp" + +#include +extern "C" { +#include +} +NVTXPushPopStrategy::NVTXPushPopStrategy() { + parallelism_descriptor_ = { + .mpi_descriptor_ = MPIDescriptor::DontCare, + .thread_descriptor_ = ThreadDescriptor::Unsupported}; +} + +void NVTXPushPopStrategy::RegionStart( + const ProperlyNestedRegionInformation ®ion) noexcept { + std::cout << "push" << region.name << std::endl; + nvtxRangePush( + std::string(region.name).c_str()); // Range around the whole function +} + +void NVTXPushPopStrategy::RegionStopLast( + const ProperlyNestedRegionInformation ®ion) noexcept { + std::cout << "pop" << std::endl; + nvtxRangePop(); // End the outer range +} +void NVTXPushPopStrategy::Init() noexcept { return; } +void NVTXPushPopStrategy::Finalize() noexcept { return; } diff --git a/src/backends/nvtx/nvtx_pushpop.hpp b/src/backends/nvtx/nvtx_pushpop.hpp new file mode 100644 index 0000000..b77b2d6 --- /dev/null +++ b/src/backends/nvtx/nvtx_pushpop.hpp @@ -0,0 +1,18 @@ +#ifndef NESMIK_NVTX_PUSHPOP_H +#define NESMIK_NVTX_PUSHPOP_H + +#include + +class NVTXPushPopStrategy : public ProperlyNestedAnnotationStrategy { + public: + NVTXPushPopStrategy(); + inline static const std::string name = "NVTX"; + void RegionStart( + const ProperlyNestedRegionInformation ®ion) noexcept override; + void RegionStopLast( + const ProperlyNestedRegionInformation ®ion) noexcept override; + virtual void Init() noexcept; + virtual void Finalize() noexcept; +}; + +#endif // NESMIK_NVTX_PUSHPOP_H diff --git a/src/delegator.cpp b/src/delegator.cpp index 60f68f5..fa0737b 100644 --- a/src/delegator.cpp +++ b/src/delegator.cpp @@ -19,6 +19,9 @@ #include "backends/extrae/extrae_type_stack.hpp" #endif +#ifdef ENABLE_NVTX +#include "backends/nvtx/nvtx_pushpop.hpp" +#endif void Delegator::InitDetectionBackend(const std::string &backend) { pn_annotation_strategy_ = std::make_unique(); } @@ -56,6 +59,12 @@ void Delegator::InitProperlyNestedBackends(const std::string &backend) { pn_annotation_strategy_ = std::make_unique(); } #endif +#ifdef ENABLE_NVTX + else if (backend.compare(NVTXPushPopStrategy::name) == 0) { + + pn_annotation_strategy_ = std::make_unique(); + } +#endif } void Delegator::Init(std::string_view nesting_mode, @@ -140,8 +149,9 @@ void Delegator::Init(std::string_view nesting_mode, << std::endl; nesting_mode_ = NestingMode::ProperlyNestedButOtherBackend; } - - InitNonProperlyNestedBackends(backend); + if (pn_annotation_strategy_ == nullptr) { + InitNonProperlyNestedBackends(backend); + } break; } -- GitLab From 4174dca11832c42c7f5a0ff50fb1478df02c3544 Mon Sep 17 00:00:00 2001 From: Valentin Seitz Date: Fri, 19 Jul 2024 13:00:50 +0200 Subject: [PATCH 2/7] added domain --- src/backends/nvtx/nvtx_pushpop.cpp | 14 +++++++++++--- src/backends/nvtx/nvtx_pushpop.hpp | 7 ++++++- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/backends/nvtx/nvtx_pushpop.cpp b/src/backends/nvtx/nvtx_pushpop.cpp index 63440f4..16de342 100644 --- a/src/backends/nvtx/nvtx_pushpop.cpp +++ b/src/backends/nvtx/nvtx_pushpop.cpp @@ -8,19 +8,27 @@ NVTXPushPopStrategy::NVTXPushPopStrategy() { parallelism_descriptor_ = { .mpi_descriptor_ = MPIDescriptor::DontCare, .thread_descriptor_ = ThreadDescriptor::Unsupported}; + domain_ = nvtxDomainCreateA("nesmik"); } void NVTXPushPopStrategy::RegionStart( const ProperlyNestedRegionInformation ®ion) noexcept { std::cout << "push" << region.name << std::endl; - nvtxRangePush( - std::string(region.name).c_str()); // Range around the whole function + + nvtxEventAttributes_t eventAttrib = {0}; + eventAttrib.version = NVTX_VERSION; + eventAttrib.size = NVTX_EVENT_ATTRIB_STRUCT_SIZE; + eventAttrib.messageType = NVTX_MESSAGE_TYPE_ASCII; + eventAttrib.message.ascii = std::string(region.name).c_str(); + nvtxDomainRangePushEx(domain_, &eventAttrib); + // nvtxDomainRangePushA(domain_, + // std::string(region.name).c_str()); // Range around the whole function } void NVTXPushPopStrategy::RegionStopLast( const ProperlyNestedRegionInformation ®ion) noexcept { std::cout << "pop" << std::endl; - nvtxRangePop(); // End the outer range + nvtxDomainRangePop(domain_); // End the outer range } void NVTXPushPopStrategy::Init() noexcept { return; } void NVTXPushPopStrategy::Finalize() noexcept { return; } diff --git a/src/backends/nvtx/nvtx_pushpop.hpp b/src/backends/nvtx/nvtx_pushpop.hpp index b77b2d6..1a6b9ab 100644 --- a/src/backends/nvtx/nvtx_pushpop.hpp +++ b/src/backends/nvtx/nvtx_pushpop.hpp @@ -2,8 +2,13 @@ #define NESMIK_NVTX_PUSHPOP_H #include - +extern "C" { +#include +} class NVTXPushPopStrategy : public ProperlyNestedAnnotationStrategy { + private: + nvtxDomainHandle_t domain_; + public: NVTXPushPopStrategy(); inline static const std::string name = "NVTX"; -- GitLab From cbe49b542d2151c2c8d54524d11efe392556e4bb Mon Sep 17 00:00:00 2001 From: Valentin Seitz Date: Fri, 19 Jul 2024 16:42:19 +0200 Subject: [PATCH 3/7] Create own domain --- src/backends/nvtx/nvtx_pushpop.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/backends/nvtx/nvtx_pushpop.cpp b/src/backends/nvtx/nvtx_pushpop.cpp index 16de342..f691064 100644 --- a/src/backends/nvtx/nvtx_pushpop.cpp +++ b/src/backends/nvtx/nvtx_pushpop.cpp @@ -8,21 +8,22 @@ NVTXPushPopStrategy::NVTXPushPopStrategy() { parallelism_descriptor_ = { .mpi_descriptor_ = MPIDescriptor::DontCare, .thread_descriptor_ = ThreadDescriptor::Unsupported}; - domain_ = nvtxDomainCreateA("nesmik"); + + domain_ = nvtxDomainCreateA("neSmiK"); } void NVTXPushPopStrategy::RegionStart( const ProperlyNestedRegionInformation ®ion) noexcept { std::cout << "push" << region.name << std::endl; - nvtxEventAttributes_t eventAttrib = {0}; + nvtxEventAttributes_t eventAttrib; eventAttrib.version = NVTX_VERSION; eventAttrib.size = NVTX_EVENT_ATTRIB_STRUCT_SIZE; + // Configure the Attributes eventAttrib.messageType = NVTX_MESSAGE_TYPE_ASCII; + eventAttrib.message.ascii = std::string(region.name).c_str(); nvtxDomainRangePushEx(domain_, &eventAttrib); - // nvtxDomainRangePushA(domain_, - // std::string(region.name).c_str()); // Range around the whole function } void NVTXPushPopStrategy::RegionStopLast( -- GitLab From d07c1d8a85560ecdfb950ea8a2c8e86c733442c1 Mon Sep 17 00:00:00 2001 From: Valentin Seitz Date: Fri, 19 Jul 2024 16:48:26 +0200 Subject: [PATCH 4/7] Add stack height as category --- src/backends/nvtx/nvtx_pushpop.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backends/nvtx/nvtx_pushpop.cpp b/src/backends/nvtx/nvtx_pushpop.cpp index f691064..c39825e 100644 --- a/src/backends/nvtx/nvtx_pushpop.cpp +++ b/src/backends/nvtx/nvtx_pushpop.cpp @@ -21,7 +21,7 @@ void NVTXPushPopStrategy::RegionStart( eventAttrib.size = NVTX_EVENT_ATTRIB_STRUCT_SIZE; // Configure the Attributes eventAttrib.messageType = NVTX_MESSAGE_TYPE_ASCII; - + eventAttrib.category = region.stack_.size(); eventAttrib.message.ascii = std::string(region.name).c_str(); nvtxDomainRangePushEx(domain_, &eventAttrib); } -- GitLab From 5fe7e1b9e730d6728052e4af5b2311a3487c4871 Mon Sep 17 00:00:00 2001 From: Valentin Seitz Date: Fri, 19 Jul 2024 16:58:05 +0200 Subject: [PATCH 5/7] add 0 category --- src/backends/nvtx/nvtx_pushpop.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backends/nvtx/nvtx_pushpop.cpp b/src/backends/nvtx/nvtx_pushpop.cpp index c39825e..da6cf05 100644 --- a/src/backends/nvtx/nvtx_pushpop.cpp +++ b/src/backends/nvtx/nvtx_pushpop.cpp @@ -21,7 +21,7 @@ void NVTXPushPopStrategy::RegionStart( eventAttrib.size = NVTX_EVENT_ATTRIB_STRUCT_SIZE; // Configure the Attributes eventAttrib.messageType = NVTX_MESSAGE_TYPE_ASCII; - eventAttrib.category = region.stack_.size(); + eventAttrib.category = 0; eventAttrib.message.ascii = std::string(region.name).c_str(); nvtxDomainRangePushEx(domain_, &eventAttrib); } -- GitLab From 04b65ebccf04e537257f7c10243fa31a6f8aa67e Mon Sep 17 00:00:00 2001 From: Valentin Seitz Date: Fri, 19 Jul 2024 17:10:45 +0200 Subject: [PATCH 6/7] Remove debug prints --- src/backends/nvtx/nvtx_pushpop.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/backends/nvtx/nvtx_pushpop.cpp b/src/backends/nvtx/nvtx_pushpop.cpp index da6cf05..91abf32 100644 --- a/src/backends/nvtx/nvtx_pushpop.cpp +++ b/src/backends/nvtx/nvtx_pushpop.cpp @@ -14,8 +14,6 @@ NVTXPushPopStrategy::NVTXPushPopStrategy() { void NVTXPushPopStrategy::RegionStart( const ProperlyNestedRegionInformation ®ion) noexcept { - std::cout << "push" << region.name << std::endl; - nvtxEventAttributes_t eventAttrib; eventAttrib.version = NVTX_VERSION; eventAttrib.size = NVTX_EVENT_ATTRIB_STRUCT_SIZE; @@ -28,8 +26,7 @@ void NVTXPushPopStrategy::RegionStart( void NVTXPushPopStrategy::RegionStopLast( const ProperlyNestedRegionInformation ®ion) noexcept { - std::cout << "pop" << std::endl; - nvtxDomainRangePop(domain_); // End the outer range + nvtxDomainRangePop(domain_); } void NVTXPushPopStrategy::Init() noexcept { return; } void NVTXPushPopStrategy::Finalize() noexcept { return; } -- GitLab From 2a2a03b3fb49bc93647767a75973c874d80b52a2 Mon Sep 17 00:00:00 2001 From: Valentin Seitz Date: Wed, 24 Jul 2024 15:05:06 +0200 Subject: [PATCH 7/7] Seperate concerns. removing changes to delegator logic --- src/delegator.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/delegator.cpp b/src/delegator.cpp index fa0737b..890eb06 100644 --- a/src/delegator.cpp +++ b/src/delegator.cpp @@ -149,9 +149,7 @@ void Delegator::Init(std::string_view nesting_mode, << std::endl; nesting_mode_ = NestingMode::ProperlyNestedButOtherBackend; } - if (pn_annotation_strategy_ == nullptr) { - InitNonProperlyNestedBackends(backend); - } + InitNonProperlyNestedBackends(backend); break; } -- GitLab