diff --git a/cfgs/nccl_regions.cfg b/cfgs/nccl_regions.cfg new file mode 100644 index 0000000000000000000000000000000000000000..6c60e81b3f5cad36116232afc3d9bc672f2519d1 --- /dev/null +++ b/cfgs/nccl_regions.cfg @@ -0,0 +1,46 @@ +#ParaverCFG +ConfigFile.Version: 3.4 +ConfigFile.NumWindows: 1 +ConfigFile.BeginDescription +NVTX regions corresponding to NCCL domain +ConfigFile.EndDescription + +################################################################################ +< NEW DISPLAYING WINDOW NCCL regions > +################################################################################ +window_name NCCL regions +window_type single +window_id 1 +window_position_x 816 +window_position_y 494 +window_width 922 +window_height 165 +window_comm_lines_enabled false +window_flags_enabled false +window_noncolor_mode true +window_custom_color_enabled false +window_semantic_scale_min_at_zero false +window_logical_filtered true +window_physical_filtered false +window_comm_fromto true +window_comm_tagsize true +window_comm_typeval true +window_units Nanoseconds +window_maximum_y 5.000000000000 +window_minimum_y 1.000000000000 +window_compute_y_max true +window_level thread +window_scale_relative 1.000000000000 +window_end_time_relative 1.000000000000 +window_object appl { 1, { All } } +window_begin_time_relative 0.000000000000 +window_open true +window_drawmode draw_maximum +window_drawmode_rows draw_last +window_pixel_size 1 +window_labels_to_draw 1 +window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } +window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, Stacked Val}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } +window_filter_module evt_type 1 9500 +window_filter_module evt_type_label 1 "NCCL regions" + diff --git a/cfgs/nvtx_ranges.cfg b/cfgs/nvtx_ranges.cfg index a0db6bfc637f7d44d55eb87dea52e5525b57db4f..67b81b978489cd4495bfeb1f3b4e1e0088137da9 100644 --- a/cfgs/nvtx_ranges.cfg +++ b/cfgs/nvtx_ranges.cfg @@ -2,17 +2,17 @@ ConfigFile.Version: 3.4 ConfigFile.NumWindows: 1 ConfigFile.BeginDescription - +NVTX pushpop ranges of default domain ConfigFile.EndDescription ################################################################################ -< NEW DISPLAYING WINDOW NVTX Ranges > +< NEW DISPLAYING WINDOW NVTX default domain > ################################################################################ -window_name NVTX Ranges +window_name NVTX default domain window_type single window_id 1 -window_position_x 612 -window_position_y 518 +window_position_x 816 +window_position_y 286 window_width 922 window_height 165 window_comm_lines_enabled false @@ -28,7 +28,7 @@ window_comm_typeval true window_units Nanoseconds window_maximum_y 52.000000000000 window_minimum_y 1.000000000000 -window_compute_y_max false +window_compute_y_max true window_level thread window_scale_relative 1.000000000000 window_end_time_relative 1.000000000000 @@ -41,6 +41,6 @@ window_pixel_size 1 window_labels_to_draw 1 window_selected_functions { 14, { {cpu, Active Thd}, {appl, Adding}, {task, Adding}, {thread, Last Evt Val}, {node, Adding}, {system, Adding}, {workload, Adding}, {from_obj, All}, {to_obj, All}, {tag_msg, All}, {size_msg, All}, {bw_msg, All}, {evt_type, =}, {evt_value, All} } } window_compose_functions { 9, { {compose_cpu, As Is}, {compose_appl, As Is}, {compose_task, As Is}, {compose_thread, Stacked Val}, {compose_node, As Is}, {compose_system, As Is}, {compose_workload, As Is}, {topcompose1, As Is}, {topcompose2, As Is} } } -window_filter_module evt_type 1 9003 -window_filter_module evt_type_label 1 "NVTX ranges" +window_filter_module evt_type 1 9600 +window_filter_module evt_type_label 1 "NVTX pushpop ranges default domain" diff --git a/nsys2prv/parse_nsys_stats.py b/nsys2prv/parse_nsys_stats.py index 8804d3ed95a42a75eb1a9d73eb9aa6f54d874ec6..a2dc4d3c0d67099a18004879ad96c3cd38806c21 100755 --- a/nsys2prv/parse_nsys_stats.py +++ b/nsys2prv/parse_nsys_stats.py @@ -99,6 +99,10 @@ def main(): event_type_mpi = 9300 event_type_metrics_base = 9400 + event_type_nvtx_base = 9600 + event_type_nvtx_nesmik = 81000 + event_type_nvtx_nccl = 9500 + event_type_openacc = 66000000 event_type_openacc_data = 66000001 event_type_openacc_launch = 66000002 @@ -175,6 +179,7 @@ def main(): if t_nvtx: nvtx_df = pd.read_csv(build_nsys_stats_name("nvtx_pushpop_trace")) + nvtx_df["domain"] = nvtx_df["Name"].str.split(":").str[0] else: nvtx_df = pd.DataFrame() @@ -468,11 +473,32 @@ def main(): #subset of df nvtx_df_subset = nvtx_df[(nvtx_df["Lvl"] >= nvtx_stack_top) & (nvtx_df["Lvl"] <= nvtx_stack_bottom)] + # split NCCL events + nvtx_nccl_df = nvtx_df_subset[nvtx_df_subset["domain"] == "NCCL"].copy() + nvtx_df_subset = nvtx_df_subset.drop(nvtx_df_subset[nvtx_df_subset["domain"] == "NCCL" ].index) + nvtx_nccl_df["event_type"] = event_type_nvtx_nccl + + # Now recurring domains, starting with nesmik + nvtx_df_subset.loc[nvtx_df_subset["domain"] == "neSmiK", "event_type"] = event_type_nvtx_nesmik + nvtx_df_subset["event_type"] = (nvtx_df_subset[nvtx_df_subset["domain"] != "neSmiK"].sort_values("domain").groupby(["domain"]).ngroup() * 100) + event_type_nvtx_base + nvtx_df_subset.loc[nvtx_df_subset["domain"] == "", "domain"] = "default" + nvtx_df_subset["event_value"] = nvtx_df_subset.groupby(["Name"]).ngroup() + 1 #nvtx_df_subset["event_value"] = nvtx_df_subset["RangeId"] + domain_names = nvtx_df_subset[["event_type", "domain"]].drop_duplicates() + + domains_dict = [] + for i, r in domain_names.iterrows(): + domains_dict.append({"name": r["domain"], "type": r["event_type"], "names": nvtx_df_subset.loc[nvtx_df_subset["domain"] == r["domain"], ['event_value', 'Name']].drop_duplicates().sort_values("event_value")}) + ranges_names = nvtx_df_subset[['event_value', 'Name']].drop_duplicates() ranges_names.sort_values("event_value", inplace=True) + # Now nccl treating + if not nvtx_nccl_df.empty: + nvtx_nccl_df["event_value"] = nvtx_nccl_df.groupby(["Name"]).ngroup() + 1 + nccl_names = nvtx_nccl_df[['event_value', 'Name']].drop_duplicates().sort_values("event_value") + if t_nvtx_startend: nvtx_startend_df["event_value"] = nvtx_startend_df.groupby(["tag"]).ngroup() + 1 nvtx_startend_names = nvtx_startend_df[['tag', 'event_value']].drop_duplicates() @@ -516,55 +542,55 @@ def main(): with open(trace_name+".pcf", "w") as pcf_file: CONFIG = """ - DEFAULT_OPTIONS - - LEVEL THREAD - UNITS NANOSEC - LOOK_BACK 100 - SPEED 1 - FLAG_ICONS ENABLED - NUM_OF_STATE_COLORS 1000 - YMAX_SCALE 37 - - - DEFAULT_SEMANTIC - - THREAD_FUNC State As Is - - GRADIENT_COLOR - 0 {0,255,2} - 1 {0,244,13} - 2 {0,232,25} - 3 {0,220,37} - 4 {0,209,48} - 5 {0,197,60} - 6 {0,185,72} - 7 {0,173,84} - 8 {0,162,95} - 9 {0,150,107} - 10 {0,138,119} - 11 {0,127,130} - 12 {0,115,142} - 13 {0,103,154} - 14 {0,91,166} - - - GRADIENT_NAMES - 0 Gradient 0 - 1 Grad. 1/MPI Events - 2 Grad. 2/OMP Events - 3 Grad. 3/OMP locks - 4 Grad. 4/User func - 5 Grad. 5/User Events - 6 Grad. 6/General Events - 7 Grad. 7/Hardware Counters - 8 Gradient 8 - 9 Gradient 9 - 10 Gradient 10 - 11 Gradient 11 - 12 Gradient 12 - 13 Gradient 13 - 14 Gradient 14 +DEFAULT_OPTIONS + +LEVEL THREAD +UNITS NANOSEC +LOOK_BACK 100 +SPEED 1 +FLAG_ICONS ENABLED +NUM_OF_STATE_COLORS 1000 +YMAX_SCALE 37 + + +DEFAULT_SEMANTIC + +THREAD_FUNC State As Is + +GRADIENT_COLOR +0 {0,255,2} +1 {0,244,13} +2 {0,232,25} +3 {0,220,37} +4 {0,209,48} +5 {0,197,60} +6 {0,185,72} +7 {0,173,84} +8 {0,162,95} +9 {0,150,107} +10 {0,138,119} +11 {0,127,130} +12 {0,115,142} +13 {0,103,154} +14 {0,91,166} + + +GRADIENT_NAMES +0 Gradient 0 +1 Grad. 1/MPI Events +2 Grad. 2/OMP Events +3 Grad. 3/OMP locks +4 Grad. 4/User func +5 Grad. 5/User Events +6 Grad. 6/General Events +7 Grad. 7/Hardware Counters +8 Gradient 8 +9 Gradient 9 +10 Gradient 10 +11 Gradient 11 +12 Gradient 12 +13 Gradient 13 +14 Gradient 14 """ @@ -643,13 +669,23 @@ def main(): pcf_file.write("\n") if t_nvtx: + for i, v in enumerate(domains_dict): + pcf_file.write("EVENT_TYPE\n") + pcf_file.write("0 {} NVTX pushpop ranges {} domain\n".format(v["type"], v["name"])) + pcf_file.write("VALUES\n") + pcf_file.write("0 End\n") + for index, row in v["names"].iterrows(): + pcf_file.write("{} {}\n".format(row["event_value"], row["Name"])) + pcf_file.write("\n") + + if not nvtx_nccl_df.empty: pcf_file.write("EVENT_TYPE\n") - pcf_file.write("0 {} NVTX pushpop ranges\n".format(event_type_nvtx)) + pcf_file.write("0 {} NCCL regions\n".format(event_type_nvtx_nccl)) pcf_file.write("VALUES\n") pcf_file.write("0 End\n") - for index, row in ranges_names.iterrows(): + for index, row in nccl_names.iterrows(): pcf_file.write("{} {}\n".format(row["event_value"], row["Name"])) - pcf_file.write("\n") + if t_nvtx_startend: pcf_file.write("EVENT_TYPE\n") @@ -823,7 +859,12 @@ def main(): if t_nvtx: ewr(prv_file, nvtx_df_subset, "NVTX pushpop ranges", lambda r: - (create_event_record(r.iloc[0], r.iloc[2], int(r["thread"]), int(r["task"]), event_type_nvtx, r["event_value"]))) + (create_event_record(r.iloc[0], r.iloc[2], int(r["thread"]), int(r["task"]), r["event_type"], r["event_value"]))) + + # NVTX NCCL regions, still missing nccl info + if not nvtx_nccl_df.empty: + ewr(prv_file, nvtx_nccl_df, "NVTX NCCL regions", lambda r: + (create_event_record(r.iloc[0], r.iloc[2], int(r["thread"]), int(r["task"]), r["event_type"], r["event_value"]))) if t_nvtx_startend: ewr(prv_file, nvtx_startend_df, "NVTX startend ranges", lambda r: diff --git a/parser-playground.ipynb b/parser-playground.ipynb index cb4dfd8771d4441555245aa3b06d0ab8a973d1da..1d5fabc569393cae9317e2c121704eb2d04dc8cd 100644 --- a/parser-playground.ipynb +++ b/parser-playground.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 7, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -28,7 +28,7 @@ "\n", "locale.setlocale(locale.LC_ALL, '')\n", "\n", - "trace_name = \"test-sod2d-openacc\"\n", + "trace_name = \"test-heka\"\n", "event_type_kernels = 63000006\n", "event_type_memcopy_size = 63000002\n", "event_type_api = 63000000\n", @@ -49,6 +49,10 @@ "event_type_name_openacc_data = 66100001\n", "event_type_name_openacc_launch = 66100002\n", "\n", + "event_type_nvtx_base = 9600\n", + "event_type_nvtx_nesmik = 81000\n", + "event_type_nvtx_nccl = 9500\n", + "\n", "event_type_func_openacc = 66200000\n", "event_type_func_openacc_data = 66200001\n", "event_type_func_openacc_launch = 66200002\n", @@ -70,6 +74,447 @@ " return os.path.join(REPORT_DIR, base_name+\"_{}.csv\".format(report_name))" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## NVTX domain separation" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Start (ns)End (ns)Duration (ns)DurChild (ns)DurNonChild (ns)NamePidTidLvlNumChildRangeIdParentIdRangeStackNameTreedomain
07434607234470089262343957431923438856816717503:gradient_range122269312226930217NaN:17:gradient_range
1743874723445717343234382785962343823512743469:gradient_range122269512226950218NaN:18:gradient_range
2744742323446499050234390516272343899421257415:gradient_range122269612226960219NaN:19:gradient_range
37465398117433591591173589376111520980930214912831:step10122269512226951202018.0:18:20--:step10
47481635117470433121173956167711515908837223652840:step10122269612226961202119.0:19:21--:step10
................................................
28415263609881792636101859230413030413NCCL:ncclGroupEnd122269312226930028432NaN:28432NCCL:ncclGroupEndNCCL
284162695589677031966183120501028635005010286350NCCL:ncclCommAbort122269512226950028433NaN:28433NCCL:ncclCommAbortNCCL
284172705260604731968162807491555676004915556760NCCL:ncclCommAbort122269412226940028434NaN:28434NCCL:ncclCommAbortNCCL
284182716530733031967011519480170418904801704189NCCL:ncclCommAbort122269612226960028435NaN:28435NCCL:ncclCommAbortNCCL
2841937089101875376821055815930037060593003706NCCL:ncclCommAbort122269312226930028436NaN:28436NCCL:ncclCommAbortNCCL
\n", + "

28420 rows × 15 columns

\n", + "
" + ], + "text/plain": [ + " Start (ns) End (ns) Duration (ns) DurChild (ns) \\\n", + "0 7434607 23447008926 23439574319 23438856816 \n", + "1 7438747 23445717343 23438278596 23438235127 \n", + "2 7447423 23446499050 23439051627 23438994212 \n", + "3 7465398 11743359159 11735893761 11520980930 \n", + "4 7481635 11747043312 11739561677 11515908837 \n", + "... ... ... ... ... \n", + "28415 26360988179 26361018592 30413 0 \n", + "28416 26955896770 31966183120 5010286350 0 \n", + "28417 27052606047 31968162807 4915556760 0 \n", + "28418 27165307330 31967011519 4801704189 0 \n", + "28419 37089101875 37682105581 593003706 0 \n", + "\n", + " DurNonChild (ns) Name Pid Tid Lvl NumChild \\\n", + "0 717503 :gradient_range 1222693 1222693 0 2 \n", + "1 43469 :gradient_range 1222695 1222695 0 2 \n", + "2 57415 :gradient_range 1222696 1222696 0 2 \n", + "3 214912831 :step10 1222695 1222695 1 20 \n", + "4 223652840 :step10 1222696 1222696 1 20 \n", + "... ... ... ... ... ... ... \n", + "28415 30413 NCCL:ncclGroupEnd 1222693 1222693 0 0 \n", + "28416 5010286350 NCCL:ncclCommAbort 1222695 1222695 0 0 \n", + "28417 4915556760 NCCL:ncclCommAbort 1222694 1222694 0 0 \n", + "28418 4801704189 NCCL:ncclCommAbort 1222696 1222696 0 0 \n", + "28419 593003706 NCCL:ncclCommAbort 1222693 1222693 0 0 \n", + "\n", + " RangeId ParentId RangeStack NameTree domain \n", + "0 17 NaN :17 :gradient_range \n", + "1 18 NaN :18 :gradient_range \n", + "2 19 NaN :19 :gradient_range \n", + "3 20 18.0 :18:20 --:step10 \n", + "4 21 19.0 :19:21 --:step10 \n", + "... ... ... ... ... ... \n", + "28415 28432 NaN :28432 NCCL:ncclGroupEnd NCCL \n", + "28416 28433 NaN :28433 NCCL:ncclCommAbort NCCL \n", + "28417 28434 NaN :28434 NCCL:ncclCommAbort NCCL \n", + "28418 28435 NaN :28435 NCCL:ncclCommAbort NCCL \n", + "28419 28436 NaN :28436 NCCL:ncclCommAbort NCCL \n", + "\n", + "[28420 rows x 15 columns]" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "t_nvtx = True\n", + "if t_nvtx:\n", + " nvtx_df = pd.read_csv(build_nsys_stats_name(\"nvtx_pushpop_trace\"))\n", + "else:\n", + " nvtx_df = pd.DataFrame()\n", + "\n", + "if t_nvtx: nvtx_df.rename(columns={\"PID\":\"Pid\", \"TID\":\"Tid\"}, inplace=True)\n", + "\n", + "nvtx_df[\"domain\"] = nvtx_df[\"Name\"].str.split(\":\").str[0]\n", + "nvtx_df" + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "metadata": {}, + "outputs": [ + { + "ename": "TypeError", + "evalue": "StringMethods.split() takes from 1 to 2 positional arguments but 3 were given", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[43], line 34\u001b[0m\n\u001b[1;32m 32\u001b[0m nvtx_nccl_df[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mevent_value\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m nvtx_nccl_df\u001b[38;5;241m.\u001b[39mgroupby([\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mName\u001b[39m\u001b[38;5;124m\"\u001b[39m])\u001b[38;5;241m.\u001b[39mngroup() \u001b[38;5;241m+\u001b[39m \u001b[38;5;241m1\u001b[39m\n\u001b[1;32m 33\u001b[0m nccl_names \u001b[38;5;241m=\u001b[39m nvtx_nccl_df[[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mevent_value\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mName\u001b[39m\u001b[38;5;124m'\u001b[39m]]\u001b[38;5;241m.\u001b[39mdrop_duplicates()\n\u001b[0;32m---> 34\u001b[0m nccl_names[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mName\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[43mnccl_names\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mName\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstr\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msplit\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m:\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m)\u001b[49m\u001b[38;5;241m.\u001b[39mstr[\u001b[38;5;241m1\u001b[39m]\n", + "File \u001b[0;32m~/Documents/BePPP/heka/tooling/env/lib/python3.12/site-packages/pandas/core/strings/accessor.py:137\u001b[0m, in \u001b[0;36mforbid_nonstring_types.._forbid_nonstring_types..wrapper\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 132\u001b[0m msg \u001b[38;5;241m=\u001b[39m (\n\u001b[1;32m 133\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCannot use .str.\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mfunc_name\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m with values of \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 134\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124minferred dtype \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_inferred_dtype\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 135\u001b[0m )\n\u001b[1;32m 136\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m(msg)\n\u001b[0;32m--> 137\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", + "\u001b[0;31mTypeError\u001b[0m: StringMethods.split() takes from 1 to 2 positional arguments but 3 were given" + ] + } + ], + "source": [ + "if t_nvtx:\n", + " nvtx_df_subset = nvtx_df\n", + " lower_level = max(nvtx_df[\"Lvl\"])\n", + "\n", + " if nvtx_select_frames:\n", + " #subset of df\n", + " nvtx_df_subset = nvtx_df[(nvtx_df[\"Lvl\"] >= nvtx_stack_top) & (nvtx_df[\"Lvl\"] <= nvtx_stack_bottom)]\n", + "\n", + " # split NCCL events\n", + " nvtx_nccl_df = nvtx_df_subset[nvtx_df_subset[\"domain\"] == \"NCCL\"].copy()\n", + " nvtx_df_subset = nvtx_df_subset.drop(nvtx_df_subset[nvtx_df_subset[\"domain\"] == \"NCCL\" ].index)\n", + " nvtx_nccl_df[\"event_type\"] = event_type_nvtx_nccl\n", + "\n", + " # Now recurring domains, starting with nesmik\n", + " nvtx_df_subset.loc[nvtx_df_subset[\"domain\"] == \"neSmiK\", \"event_type\"] = event_type_nvtx_nesmik\n", + " nvtx_df_subset[\"event_type\"] = (nvtx_df_subset[nvtx_df_subset[\"domain\"] != \"neSmiK\"].groupby([\"domain\"]).ngroup() * 100) + event_type_nvtx_base\n", + " nvtx_df_subset.loc[nvtx_df_subset[\"domain\"] == \"\", \"domain\"] = \"default\"\n", + "\n", + " nvtx_df_subset[\"event_value\"] = nvtx_df_subset.groupby([\"Name\"]).ngroup() + 1\n", + " #nvtx_df_subset[\"event_value\"] = nvtx_df_subset[\"RangeId\"]\n", + " domain_names = nvtx_df_subset[[\"event_type\", \"domain\"]].drop_duplicates()\n", + " \n", + " domains_dict = []\n", + " for i, r in domain_names.iterrows():\n", + " domains_dict.append({\"name\": r[\"domain\"], \"type\": r[\"event_type\"], \"names\": nvtx_df_subset.loc[nvtx_df_subset[\"domain\"] == r[\"domain\"], ['event_value', 'Name']].drop_duplicates().sort_values(\"event_value\")})\n", + "\n", + " ranges_names = nvtx_df_subset[['event_value', 'Name']].drop_duplicates()\n", + " ranges_names.sort_values(\"event_value\", inplace=True)\n", + "\n", + " # Now nccl treating\n", + " if not nvtx_nccl_df.empty:\n", + " nvtx_nccl_df[\"event_value\"] = nvtx_nccl_df.groupby([\"Name\"]).ngroup() + 1\n", + " nccl_names = nvtx_nccl_df[['event_value', 'Name']].drop_duplicates()\n" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
event_valueName
31014NCCL:ncclGroupStart
31022NCCL:ncclAllReduce
31033NCCL:ncclGroupEnd
125971NCCL:ncclAllGather
\n", + "
" + ], + "text/plain": [ + " event_value Name\n", + "3101 4 NCCL:ncclGroupStart\n", + "3102 2 NCCL:ncclAllReduce\n", + "3103 3 NCCL:ncclGroupEnd\n", + "12597 1 NCCL:ncclAllGather" + ] + }, + "execution_count": 39, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "nccl_names" + ] + }, { "cell_type": "code", "execution_count": 2,