From 0c85266e11ac1bb94849d5102484a5d1935700b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Clasc=C3=A0?= Date: Thu, 1 Aug 2024 18:44:06 +0200 Subject: [PATCH 1/4] This commit classifies MPI events with the same event types and event values as Extrae does. THis allows using same CFGs that paraver includes for MPI. It also adds Rank information (when available) to the process model building, making rank 0 first in the list. --- nsys2prv/parse_nsys_stats.py | 111 +++++++-- nsys2prv/scripts/mpi_coll.sql | 42 ++++ nsys2prv/scripts/mpi_io.sql | 57 +++++ nsys2prv/scripts/mpi_other.sql | 42 ++++ nsys2prv/scripts/mpi_p2p.sql | 57 +++++ nsys2prv/semantics/mpi_event_encoding.py | 275 +++++++++++++++++++++++ parser-playground.ipynb | 120 ++++++++++ pyproject.toml | 2 +- 8 files changed, 687 insertions(+), 19 deletions(-) create mode 100644 nsys2prv/scripts/mpi_coll.sql create mode 100644 nsys2prv/scripts/mpi_io.sql create mode 100644 nsys2prv/scripts/mpi_other.sql create mode 100644 nsys2prv/scripts/mpi_p2p.sql create mode 100644 nsys2prv/semantics/mpi_event_encoding.py diff --git a/nsys2prv/parse_nsys_stats.py b/nsys2prv/parse_nsys_stats.py index 2d3d75e..bebf3c5 100755 --- a/nsys2prv/parse_nsys_stats.py +++ b/nsys2prv/parse_nsys_stats.py @@ -8,8 +8,9 @@ import time import subprocess import os import locale -from sqlalchemy import create_engine +from sqlalchemy import create_engine, text from .EventWriter import event_writer as ewr +from .semantics.mpi_event_encoding import * def main(): @@ -65,7 +66,9 @@ def main(): if "nvtx_pushpop_trace" in reports: t_nvtx = True if "cuda_api_trace" in reports: t_apicalls = True - if "mpi_event_trace" in reports: t_mpi = True + if "mpi_event_trace" in reports: + t_mpi = True + reports.remove("mpi_event_trace") if "gpu_metrics" in reports: t_metrics = True reports.remove("gpu_metrics") @@ -155,6 +158,8 @@ def main(): kernels_df = pd.read_csv(build_nsys_stats_name("cuda_gpu_trace")) kernels_df.rename(columns={"CorrId": "CorrID"}, inplace=True) + engine = create_engine(f"sqlite:///{os.path.splitext(REPORT_FILE)[0]}.sqlite") + if t_apicalls: cuda_api_df = pd.read_csv(build_nsys_stats_name("cuda_api_trace")) else: @@ -166,20 +171,51 @@ def main(): nvtx_df = pd.DataFrame() if t_nvtx_startend: - engine = create_engine(f"sqlite:///{os.path.splitext(REPORT_FILE)[0]}.sqlite") with engine.connect() as conn, conn.begin(): with open(os.path.join(os.path.dirname(__file__), 'scripts/nvtx_startend_trace.sql'), 'r') as query: - nvtx_startend_df = pd.read_sql_query(query.read(), conn) + nvtx_startend_df = pd.read_sql_query(text(query.read()), conn) else: nvtx_startend_df = pd.DataFrame() if t_mpi: - mpi_df = pd.read_csv(build_nsys_stats_name("mpi_event_trace")) + with engine.connect() as conn, conn.begin(): + with open(os.path.join(os.path.dirname(__file__), 'scripts/mpi_p2p.sql'), 'r') as query: + mpi_p2p_df = pd.read_sql_query(text(query.read()), conn) + mpi_p2p_df["event_type"] = MPITYPE_PTOP + with open(os.path.join(os.path.dirname(__file__), 'scripts/mpi_coll.sql'), 'r') as query: + mpi_coll_df = pd.read_sql_query(text(query.read()), conn) + mpi_coll_df = mpi_coll_df.drop(mpi_coll_df[mpi_coll_df["Event"].str.contains("File") ].index) + mpi_coll_df["event_type"] = MPITYPE_COLLECTIVE + with open(os.path.join(os.path.dirname(__file__), 'scripts/mpi_other.sql'), 'r') as query: + mpi_other_df = pd.read_sql_query(text(query.read()), conn) + mpi_other_df = mpi_other_df.drop(mpi_other_df[mpi_other_df["Event"].str.contains("File") ].index) + mpi_other_df = mpi_other_df.drop(mpi_other_df[mpi_other_df["Event"].str.contains("Win|MPI_Get|MPI_Put|Accumulate") ].index) + mpi_other_df["event_type"] = MPITYPE_OTHER + with open(os.path.join(os.path.dirname(__file__), 'scripts/mpi_other.sql'), 'r') as query: + mpi_rma_df = pd.read_sql_query(text(query.read()), conn) + mpi_rma_df = mpi_rma_df[mpi_rma_df["Event"].str.contains("Win|MPI_Get|MPI_Put|Accumulate")] + mpi_rma_df["event_type"] = MPITYPE_RMA + with open(os.path.join(os.path.dirname(__file__), 'scripts/mpi_io.sql'), 'r') as query: + mpi_io_df = pd.read_sql_query(text(query.read()), conn) + mpi_io_df = mpi_io_df[mpi_io_df["Event"].str.contains("File")] + mpi_io_df["event_type"] = MPITYPE_IO + mpi_df = pd.concat([mpi_p2p_df, mpi_coll_df, mpi_other_df, mpi_rma_df, mpi_io_df]) + #mpi_df = pd.read_csv(build_nsys_stats_name("mpi_event_trace")) else: - mpi_df = pd.DataFrame() + #mpi_df = pd.DataFrame() + mpi_p2p_df = pd.DataFrame() + mpi_coll_df = pd.DataFrame() + mpi_other_df = pd.DataFrame() + mpi_rma_df = pd.DataFrame() + mpi_io_df = pd.DataFrame() # Obtain context Info context_info = pd.read_sql_table("TARGET_INFO_CUDA_CONTEXT_INFO", f"sqlite:///{os.path.splitext(REPORT_FILE)[0]}.sqlite") + if t_mpi: + mpi_query = "SELECT globalTid / 0x1000000 % 0x1000000 AS Pid, globalTid % 0x1000000 AS Tid, rank FROM MPI_RANKS;" + with engine.connect() as conn, conn.begin(): + rank_info = pd.read_sql_query(mpi_query, conn) + context_info.sort_values(["processId"], inplace=True) if t_metrics: @@ -196,14 +232,13 @@ def main(): if t_openacc: - engine = create_engine(f"sqlite:///{os.path.splitext(REPORT_FILE)[0]}.sqlite") with engine.connect() as conn, conn.begin(): with open(os.path.join(os.path.dirname(__file__), 'scripts/openacc_other.sql'), 'r') as query: - openacc_other_df = pd.read_sql_query(query, conn) + openacc_other_df = pd.read_sql_query(text(query.read()), conn) with open(os.path.join(os.path.dirname(__file__), 'scripts/openacc_launch.sql'), 'r') as query: - openacc_launch_df = pd.read_sql_query(query, conn) + openacc_launch_df = pd.read_sql_query(text(query.read()), conn) with open(os.path.join(os.path.dirname(__file__), 'scripts/openacc_data.sql'), 'r') as query: - openacc_data_df = pd.read_sql_query(query, conn) + openacc_data_df = pd.read_sql_query(text(query.read()), conn) openacc_event_kind = pd.read_sql_table("ENUM_OPENACC_EVENT_KIND", conn) @@ -229,7 +264,11 @@ def main(): threads = pd.concat(compute_threads_with).drop_duplicates() - threads.sort_values(["Pid"], inplace=True) + if t_mpi: + threads["Rank"] = threads["Pid"].map(rank_info.set_index("Pid")["rank"]) + threads.sort_values(["Rank"], inplace=True) + else: + threads.sort_values(["Pid"], inplace=True) threads["thread"] = threads.groupby(["Pid"]).cumcount() + 1 threads["task"] = threads.groupby(["Pid"]).ngroup() + 1 threads["device"] = threads["Pid"].map(context_info[context_info["contextId"] == 1].set_index("processId")["deviceId"]) @@ -381,10 +420,18 @@ def main(): api_call_names = cuda_api_df[['Name', 'event_value']].drop_duplicates() api_call_names.sort_values("event_value", inplace=True) + # if t_mpi: + # mpi_df["event_value"] = mpi_df.groupby(["Event"]).ngroup() + 1 + # mpi_names = mpi_df[['Event', 'event_value']].drop_duplicates() + # mpi_names.sort_values("event_value", inplace=True) + if t_mpi: - mpi_df["event_value"] = mpi_df.groupby(["Event"]).ngroup() + 1 - mpi_names = mpi_df[['Event', 'event_value']].drop_duplicates() - mpi_names.sort_values("event_value", inplace=True) + mpi_values = pd.DataFrame.from_dict(MPIVal, orient='index', columns=["event_value"]) + mpi_names = pd.DataFrame.from_dict(MPI_Val_Labels, orient='index', columns=["Name"]) + mpi_names = mpi_names.merge(mpi_values, left_index=True, right_index=True) + mpi_df["event_value"] = mpi_df["Event"].map(mpi_names.set_index('Name')["event_value"]) + + kernels_df["event_value"] = kernels_df.groupby(["Name"]).ngroup() + 1 + api_call_names.count().iloc[0] # Add padding to event values so CUDA calls and CUDA kernels can be added kernel_names = kernels_df[['event_value', 'Name']].drop_duplicates() @@ -513,11 +560,39 @@ def main(): if t_mpi: pcf_file.write("EVENT_TYPE\n") - pcf_file.write("0 {} MPI Calls\n".format(event_type_mpi)) + pcf_file.write("0 {} {}\n".format(MPITYPE_PTOP, MPI_Type_Labels["MPITYPE_PTOP"])) pcf_file.write("VALUES\n") pcf_file.write("0 End\n") for index, row in mpi_names.iterrows(): - pcf_file.write("{} {}\n".format(row["event_value"], row["Event"])) + pcf_file.write("{} {}\n".format(row["event_value"], row["Name"])) + pcf_file.write("\n") + pcf_file.write("EVENT_TYPE\n") + pcf_file.write("0 {} {}\n".format(MPITYPE_COLLECTIVE, MPI_Type_Labels["MPITYPE_COLLECTIVE"])) + pcf_file.write("VALUES\n") + pcf_file.write("0 End\n") + for index, row in mpi_names.iterrows(): + pcf_file.write("{} {}\n".format(row["event_value"], row["Name"])) + pcf_file.write("\n") + pcf_file.write("EVENT_TYPE\n") + pcf_file.write("0 {} {}\n".format(MPITYPE_OTHER, MPI_Type_Labels["MPITYPE_OTHER"])) + pcf_file.write("VALUES\n") + pcf_file.write("0 End\n") + for index, row in mpi_names.iterrows(): + pcf_file.write("{} {}\n".format(row["event_value"], row["Name"])) + pcf_file.write("\n") + pcf_file.write("EVENT_TYPE\n") + pcf_file.write("0 {} {}\n".format(MPITYPE_RMA, MPI_Type_Labels["MPITYPE_RMA"])) + pcf_file.write("VALUES\n") + pcf_file.write("0 End\n") + for index, row in mpi_names.iterrows(): + pcf_file.write("{} {}\n".format(row["event_value"], row["Name"])) + pcf_file.write("\n") + pcf_file.write("EVENT_TYPE\n") + pcf_file.write("0 {} {}\n".format(MPITYPE_IO, MPI_Type_Labels["MPITYPE_IO"])) + pcf_file.write("VALUES\n") + pcf_file.write("0 End\n") + for index, row in mpi_names.iterrows(): + pcf_file.write("{} {}\n".format(row["event_value"], row["Name"])) pcf_file.write("\n") pcf_file.write("EVENT_TYPE\n") @@ -697,7 +772,7 @@ def main(): if t_apicalls: compute_max_with.append((cuda_api_df["Start (ns)"] + cuda_api_df["Duration (ns)"]).max()) if t_nvtx: compute_max_with.append(nvtx_df["End (ns)"].max()) if t_nvtx_startend: compute_max_with.append(nvtx_startend_df["end"].max()) - if t_mpi: compute_max_with.append(mpi_df["End (ns)"].max()) + if t_mpi: compute_max_with.append(mpi_df["End:ts_ns"].max()) ftime = max(compute_max_with) header = "#Paraver ({}):{}_ns:0:1:{}\n".format(now, ftime, applist) @@ -734,7 +809,7 @@ def main(): if t_mpi: ewr(prv_file, mpi_df, "MPI events", lambda r: - (create_event_record(r.iloc[0], r.iloc[2], int(r["thread"]), int(r["task"]), event_type_mpi, r["event_value"]))) + (create_event_record(r.iloc[1], r.iloc[3], int(r["thread"]), int(r["task"]), r["event_type"], r["event_value"]))) if t_openacc: t_acc_d = [event_type_openacc_data, event_type_name_openacc_data, event_type_func_openacc_data, event_type_openacc_data_size] diff --git a/nsys2prv/scripts/mpi_coll.sql b/nsys2prv/scripts/mpi_coll.sql new file mode 100644 index 0000000..e9754d6 --- /dev/null +++ b/nsys2prv/scripts/mpi_coll.sql @@ -0,0 +1,42 @@ +WITH + evts AS ( + SELECT + 'collectives' AS source, + start AS start, + end AS end, + globalTid AS globalTid, + textId AS textId, + NULL AS size, + size AS collSendSize, + recvSize AS collRecvSize, + NULL AS tag, + NULL AS remoteRank, + rootRank AS rootRank + FROM + MPI_COLLECTIVES_EVENTS + ) +SELECT + e.source as "Kind", + e.start AS "Start:ts_ns", + e.end AS "End:ts_ns", + e.end - e.start AS "Duration:dur_ns", + s.value AS "Event", + (e.globalTid >> 24) & 0x00FFFFFF AS "Pid", + e.globalTid & 0x00FFFFFF AS "Tid", + e.tag AS "Tag", + r.rank AS "Rank", + e.remoteRank AS "PeerRank", + e.rootRank AS "RootRank", + e.size AS "Size:mem_b", + e.collSendSize AS "CollSendSize:mem_b", + e.collRecvSize AS "CollRecvSize:mem_b" +FROM + evts AS e +LEFT JOIN + StringIds AS s + ON e.textId == s.id +LEFT JOIN + MPI_RANKS AS r + ON e.globalTid == r.globalTid +ORDER BY 1 +; \ No newline at end of file diff --git a/nsys2prv/scripts/mpi_io.sql b/nsys2prv/scripts/mpi_io.sql new file mode 100644 index 0000000..288782e --- /dev/null +++ b/nsys2prv/scripts/mpi_io.sql @@ -0,0 +1,57 @@ +WITH + evts AS ( + SELECT + 'other' AS source, + start AS start, + end AS end, + globalTid AS globalTid, + textId AS textId, + NULL AS size, + NULL AS collSendSize, + NULL AS collRecvSize, + NULL AS tag, + NULL AS remoteRank, + NULL AS rootRank + FROM + MPI_OTHER_EVENTS + UNION ALL + SELECT + 'collectives' AS source, + start AS start, + end AS end, + globalTid AS globalTid, + textId AS textId, + NULL AS size, + size AS collSendSize, + recvSize AS collRecvSize, + NULL AS tag, + NULL AS remoteRank, + rootRank AS rootRank + FROM + MPI_COLLECTIVES_EVENTS + ) +SELECT + e.source as "Kind", + e.start AS "Start:ts_ns", + e.end AS "End:ts_ns", + e.end - e.start AS "Duration:dur_ns", + s.value AS "Event", + (e.globalTid >> 24) & 0x00FFFFFF AS "Pid", + e.globalTid & 0x00FFFFFF AS "Tid", + e.tag AS "Tag", + r.rank AS "Rank", + e.remoteRank AS "PeerRank", + e.rootRank AS "RootRank", + e.size AS "Size:mem_b", + e.collSendSize AS "CollSendSize:mem_b", + e.collRecvSize AS "CollRecvSize:mem_b" +FROM + evts AS e +LEFT JOIN + StringIds AS s + ON e.textId == s.id +LEFT JOIN + MPI_RANKS AS r + ON e.globalTid == r.globalTid +ORDER BY 1 +; \ No newline at end of file diff --git a/nsys2prv/scripts/mpi_other.sql b/nsys2prv/scripts/mpi_other.sql new file mode 100644 index 0000000..53173e8 --- /dev/null +++ b/nsys2prv/scripts/mpi_other.sql @@ -0,0 +1,42 @@ +WITH + evts AS ( + SELECT + 'other' AS source, + start AS start, + end AS end, + globalTid AS globalTid, + textId AS textId, + NULL AS size, + NULL AS collSendSize, + NULL AS collRecvSize, + NULL AS tag, + NULL AS remoteRank, + NULL AS rootRank + FROM + MPI_OTHER_EVENTS + ) +SELECT + e.source as "Kind", + e.start AS "Start:ts_ns", + e.end AS "End:ts_ns", + e.end - e.start AS "Duration:dur_ns", + s.value AS "Event", + (e.globalTid >> 24) & 0x00FFFFFF AS "Pid", + e.globalTid & 0x00FFFFFF AS "Tid", + e.tag AS "Tag", + r.rank AS "Rank", + e.remoteRank AS "PeerRank", + e.rootRank AS "RootRank", + e.size AS "Size:mem_b", + e.collSendSize AS "CollSendSize:mem_b", + e.collRecvSize AS "CollRecvSize:mem_b" +FROM + evts AS e +LEFT JOIN + StringIds AS s + ON e.textId == s.id +LEFT JOIN + MPI_RANKS AS r + ON e.globalTid == r.globalTid +ORDER BY 1 +; \ No newline at end of file diff --git a/nsys2prv/scripts/mpi_p2p.sql b/nsys2prv/scripts/mpi_p2p.sql new file mode 100644 index 0000000..a2cf516 --- /dev/null +++ b/nsys2prv/scripts/mpi_p2p.sql @@ -0,0 +1,57 @@ +WITH + evts AS ( + SELECT + 'p2p' AS source, + start AS start, + end AS end, + globalTid AS globalTid, + textId AS textId, + size AS size, + NULL AS collSendSize, + NULL AS collRecvSize, + tag AS tag, + remoteRank AS remoteRank, + NULL AS rootRank + FROM + MPI_P2P_EVENTS + UNION ALL + SELECT + 'p2p' AS source, + start AS start, + end AS end, + globalTid AS globalTid, + textId AS textId, + NULL AS size, + NULL AS collSendSize, + NULL AS collRecvSize, + NULL AS tag, + NULL AS remoteRank, + NULL AS rootRank + FROM + MPI_START_WAIT_EVENTS + ) +SELECT + e.source as "Kind", + e.start AS "Start:ts_ns", + e.end AS "End:ts_ns", + e.end - e.start AS "Duration:dur_ns", + s.value AS "Event", + (e.globalTid >> 24) & 0x00FFFFFF AS "Pid", + e.globalTid & 0x00FFFFFF AS "Tid", + e.tag AS "Tag", + r.rank AS "Rank", + e.remoteRank AS "PeerRank", + e.rootRank AS "RootRank", + e.size AS "Size:mem_b", + e.collSendSize AS "CollSendSize:mem_b", + e.collRecvSize AS "CollRecvSize:mem_b" +FROM + evts AS e +LEFT JOIN + StringIds AS s + ON e.textId == s.id +LEFT JOIN + MPI_RANKS AS r + ON e.globalTid == r.globalTid +ORDER BY 1 +; \ No newline at end of file diff --git a/nsys2prv/semantics/mpi_event_encoding.py b/nsys2prv/semantics/mpi_event_encoding.py new file mode 100644 index 0000000..b4b7d19 --- /dev/null +++ b/nsys2prv/semantics/mpi_event_encoding.py @@ -0,0 +1,275 @@ +# /*****************************************************************************\ +# * ANALYSIS PERFORMANCE TOOLS * +# * Extrae * +# * Instrumentation package for parallel applications * +# ***************************************************************************** +# * ___ This library is free software; you can redistribute it and/or * +# * / __ modify it under the terms of the GNU LGPL as published * +# * / / _____ by the Free Software Foundation; either version 2.1 * +# * / / / \ of the License, or (at your option) any later version. * +# * ( ( ( B S C ) * +# * \ \ \_____/ This library is distributed in hope that it will be * +# * \ \__ useful but WITHOUT ANY WARRANTY; without even the * +# * \___ implied warranty of MERCHANTABILITY or FITNESS FOR A * +# * PARTICULAR PURPOSE. See the GNU LGPL for more details. * +# * * +# * You should have received a copy of the GNU Lesser General Public License * +# * along with this library; if not, write to the Free Software Foundation, * +# * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * +# * The GNU LEsser General Public License is contained in the file COPYING. * +# * --------- * +# * Barcelona Supercomputing Center - Centro Nacional de Supercomputacion * +# \*****************************************************************************/ + + +MPITYPE_PTOP = 50000001 +MPITYPE_COLLECTIVE = 50000002 +MPITYPE_OTHER = 50000003 +MPITYPE_RMA = 50000004 +MPITYPE_RMA_SIZE = 50001000 +MPITYPE_RMA_TARGET_RANK = 50001001 +MPITYPE_RMA_ORIGIN_ADDR = 50001002 +MPITYPE_RMA_TARGET_DISP = 50001003 +MPITYPE_COMM = MPITYPE_OTHER +MPITYPE_GROUP = MPITYPE_OTHER +MPITYPE_TOPOLOGIES = MPITYPE_OTHER +MPITYPE_TYPE = MPITYPE_OTHER +MPITYPE_IO = 50000005 + + +MPIVal = {'MPI_END': 0, 'MPI_SEND': 1, 'MPI_RECV': 2, 'MPI_ISEND': 3, 'MPI_IRECV': 4, 'MPI_WAIT': 5, 'MPI_WAITALL': 6, 'MPI_BCAST': 7, 'MPI_BARRIER': 8, 'MPI_REDUCE': 9, 'MPI_ALLREDUCE': 10, 'MPI_ALLTOALL': 11, 'MPI_ALLTOALLV': 12, 'MPI_GATHER': 13, 'MPI_GATHERV': 14, 'MPI_SCATTER': 15, 'MPI_SCATTERV': 16, 'MPI_ALLGATHER': 17, 'MPI_ALLGATHERV': 18, 'MPI_COMM_RANK': 19, 'MPI_COMM_SIZE': 20, 'MPI_COMM_CREATE': 21, 'MPI_COMM_DUP': 22, 'MPI_COMM_SPLIT': 23, 'MPI_COMM_GROUP': 24, 'MPI_COMM_FREE': 25, 'MPI_COMM_REMOTE_GROUP': 26, 'MPI_COMM_REMOTE_SIZE': 27, 'MPI_COMM_TEST_INTER': 28, 'MPI_COMM_COMPARE': 29, 'MPI_SCAN': 30, 'MPI_INIT': 31, 'MPI_FINALIZE': 32, 'MPI_BSEND': 33, 'MPI_SSEND': 34, 'MPI_RSEND': 35, 'MPI_IBSEND': 36, 'MPI_ISSEND': 37, 'MPI_IRSEND': 38, 'MPI_TEST': 39, 'MPI_CANCEL': 40, 'MPI_SENDRECV': 41, 'MPI_SENDRECV_REPLACE': 42, 'MPI_CART_CREATE': 43, 'MPI_CART_SHIFT': 44, 'MPI_CART_COORDS': 45, 'MPI_CART_GET': 46, 'MPI_CART_MAP': 47, 'MPI_CART_RANK': 48, 'MPI_CART_SUB': 49, 'MPI_CARTDIM_GET': 50, 'MPI_DIMS_CREATE': 51, 'MPI_GRAPH_GET': 52, 'MPI_GRAPH_MAP': 53, 'MPI_GRAPH_CREATE': 54, 'MPI_GRAPH_NEIGHBORS': 55, 'MPI_GRAPHDIMS_GET': 56, 'MPI_GRAPH_NEIGHBORS_COUNT': 57, 'MPI_TOPO_TEST': 58, 'MPI_WAITANY': 59, 'MPI_WAITSOME': 60, 'MPI_PROBE': 61, 'MPI_IPROBE': 62, 'MPI_WIN_CREATE': 63, 'MPI_WIN_FREE': 64, 'MPI_PUT': 65, 'MPI_GET': 66, 'MPI_ACCUMULATE': 67, 'MPI_WIN_FENCE': 68, 'MPI_WIN_START': 69, 'MPI_WIN_COMPLETE': 70, 'MPI_WIN_POST': 71, 'MPI_WIN_WAIT': 72, 'MPI_WIN_TEST': 73, 'MPI_WIN_LOCK': 74, 'MPI_WIN_UNLOCK': 75, 'MPI_PACK': 76, 'MPI_UNPACK': 77, 'MPI_OP_CREATE': 78, 'MPI_OP_FREE': 79, 'MPI_REDUCE_SCATTER': 80, 'MPI_ATTR_DELETE': 81, 'MPI_ATTR_GET': 82, 'MPI_ATTR_PUT': 83, 'MPI_GROUP_DIFFERENCE': 84, 'MPI_GROUP_EXCL': 85, 'MPI_GROUP_FREE': 86, 'MPI_GROUP_INCL': 87, 'MPI_GROUP_INTERSECTION': 88, 'MPI_GROUP_RANK': 89, 'MPI_GROUP_RANGE_EXCL': 90, 'MPI_GROUP_RANGE_INCL': 91, 'MPI_GROUP_SIZE': 92, 'MPI_GROUP_TRANSLATE_RANKS': 93, 'MPI_GROUP_UNION': 94, 'MPI_GROUP_COMPARE': 95, 'MPI_INTERCOMM_CREATE': 96, 'MPI_INTERCOMM_MERGE': 97, 'MPI_KEYVAL_FREE': 98, 'MPI_KEYVAL_CREATE': 99, 'MPI_ABORT': 100, 'MPI_ERROR_CLASS': 101, 'MPI_ERRHANDLER_CREATE': 102, 'MPI_ERRHANDLER_FREE': 103, 'MPI_ERRHANDLER_GET': 104, 'MPI_ERROR_STRING': 105, 'MPI_ERRHANDLER_SET': 106, 'MPI_GET_PROCESSOR_NAME': 107, 'MPI_INITIALIZED': 108, 'MPI_WTICK': 109, 'MPI_WTIME': 110, 'MPI_ADDRESS': 111, 'MPI_BSEND_INIT': 112, 'MPI_BUFFER_ATTACH': 113, 'MPI_BUFFER_DETACH': 114, 'MPI_REQUEST_FREE': 115, 'MPI_RECV_INIT': 116, 'MPI_SEND_INIT': 117, 'MPI_GET_COUNT': 118, 'MPI_GET_ELEMENTS': 119, 'MPI_PACK_SIZE': 120, 'MPI_RSEND_INIT': 121, 'MPI_SSEND_INIT': 122, 'MPI_START': 123, 'MPI_STARTALL': 124, 'MPI_TESTALL': 125, 'MPI_TESTANY': 126, 'MPI_TEST_CANCELLED': 127, 'MPI_TESTSOME': 128, 'MPI_TYPE_COMMIT': 129, 'MPI_TYPE_CONTIGUOUS': 130, 'MPI_TYPE_EXTENT': 131, 'MPI_TYPE_FREE': 132, 'MPI_TYPE_HINDEXED': 133, 'MPI_TYPE_HVECTOR': 134, 'MPI_TYPE_INDEXED': 135, 'MPI_TYPE_LB': 136, 'MPI_TYPE_SIZE': 137, 'MPI_TYPE_STRUCT': 138, 'MPI_TYPE_UB': 139, 'MPI_TYPE_VECTOR': 140, 'MPI_FILE_OPEN': 141, 'MPI_FILE_CLOSE': 142, 'MPI_FILE_READ': 143, 'MPI_FILE_READ_ALL': 144, 'MPI_FILE_WRITE': 145, 'MPI_FILE_WRITE_ALL': 146, 'MPI_FILE_READ_AT': 147, 'MPI_FILE_READ_AT_ALL': 148, 'MPI_FILE_WRITE_AT': 149, 'MPI_FILE_WRITE_AT_ALL': 150, 'MPI_COMM_SPAWN': 151, 'MPI_COMM_SPAWN_MULTIPLE': 152, 'MPI_REQUEST_GET_STATUS': 153, 'MPI_IREDUCE': 154, 'MPI_IALLREDUCE': 155, 'MPI_IBARRIER': 156, 'MPI_IBCAST': 157, 'MPI_IALLTOALL': 158, 'MPI_IALLTOALLV': 159, 'MPI_IALLGATHER': 160, 'MPI_IALLGATHERV': 161, 'MPI_IGATHER': 162, 'MPI_IGATHERV': 163, 'MPI_ISCATTER': 164, 'MPI_ISCATTERV': 165, 'MPI_IREDUCESCAT': 166, 'MPI_ISCAN': 167, 'MPI_REDUCE_SCATTER_BLOCK': 168, 'MPI_IREDUCE_SCATTER_BLOCK': 169, 'MPI_ALLTOALLW': 170, 'MPI_IALLTOALLW': 171, 'MPI_GET_ACCUMULATE': 172, 'MPI_DIST_GRAPH_CREATE': 173, 'MPI_NEIGHBOR_ALLGATHER': 174, 'MPI_INEIGHBOR_ALLGATHER': 175, 'MPI_NEIGHBOR_ALLGATHERV': 176, 'MPI_INEIGHBOR_ALLGATHERV': 177, 'MPI_NEIGHBOR_ALLTOALL': 178, 'MPI_INEIGHBOR_ALLTOALL': 179, 'MPI_NEIGHBOR_ALLTOALLV': 180, 'MPI_INEIGHBOR_ALLTOALLV': 181, 'MPI_NEIGHBOR_ALLTOALLW': 182, 'MPI_INEIGHBOR_ALLTOALLW': 183, 'MPI_FETCH_AND_OP': 184, 'MPI_COMPARE_AND_SWAP': 185, 'MPI_WIN_FLUSH': 186, 'MPI_WIN_FLUSH_ALL': 187, 'MPI_WIN_FLUSH_LOCAL': 188, 'MPI_WIN_FLUSH_LOCAL_ALL': 189, 'MPI_MPROBE': 190, 'MPI_IMPROBE': 191, 'MPI_MRECV': 192, 'MPI_IMRECV': 193, 'MPI_COMM_SPLIT_TYPE': 194, 'MPI_FILE_WRITE_ALL_BEGIN': 195, 'MPI_FILE_WRITE_ALL_END': 196, 'MPI_FILE_READ_ALL_BEGIN': 197, 'MPI_FILE_READ_ALL_END': 198, 'MPI_FILE_WRITE_AT_ALL_BEGIN': 199, 'MPI_FILE_WRITE_AT_ALL_END': 200, 'MPI_FILE_READ_AT_ALL_BEGIN': 201, 'MPI_FILE_READ_AT_ALL_END': 202, 'MPI_FILE_READ_ORDERED': 203, 'MPI_FILE_READ_ORDERED_BEGIN': 204, 'MPI_FILE_READ_ORDERED_END': 205, 'MPI_FILE_READ_SHARED': 206, 'MPI_FILE_WRITE_ORDERED': 207, 'MPI_FILE_WRITE_ORDERED_BEGIN': 208, 'MPI_FILE_WRITE_ORDERED_END': 209, 'MPI_FILE_WRITE_SHARED': 210, 'MPI_COMM_DUP_WITH_INFO': 211, 'MPI_DIST_GRAPH_CREATE_ADJACENT': 212, 'MPI_COMM_CREATE_GROUP': 213, 'MPI_EXSCAN': 214, 'MPI_IEXSCAN_VAL': 215} + +MPI_Type_Labels = { +'MPITYPE_PTOP' : "MPI Point-to-point", +'MPITYPE_COLLECTIVE' : "MPI Collective Comm", +'MPITYPE_OTHER' : "MPI Other", +'MPITYPE_RMA' : "MPI One-sided", +'MPITYPE_RMA_SIZE' : "MPI One-sided size", +'MPITYPE_RMA_TARGET_RANK' : "MPI One,-sided target rank", +'MPITYPE_RMA_ORIGIN_ADDR' : "MPI One,-sided origin address", +'MPITYPE_RMA_TARGET_DISP' : "MPI One,-sided target displacement", +'MPITYPE_COMM' : "MPI Other", +'MPITYPE_GROUP' : "MPI Other", +'MPITYPE_TOPOLOGIES' : "MPI Other", +'MPITYPE_TYPE' : "MPI Other", +'MPITYPE_IO' : "MPI I/O" +} + +MPI_Val_Labels = { +"MPIEND_LABEL" : "End", +'MPI_SEND' : "MPI_Send", +'MPI_RECV' : "MPI_Recv", +'MPI_ISEND' : "MPI_Isend", +'MPI_IRECV' : "MPI_Irecv", +'MPI_WAIT' : "MPI_Wait", +'MPI_WAITALL' : "MPI_Waitall", +'MPI_REDUCE' : "MPI_Reduce", +'MPI_ALLREDUCE' : "MPI_Allreduce", +'MPI_BARRIER' : "MPI_Barrier", +'MPI_BCAST' : "MPI_Bcast", +'MPI_ALLTOALL' : "MPI_Alltoall", +'MPI_ALLTOALLV' : "MPI_Alltoallv", +'MPI_ALLGATHER' : "MPI_Allgather", +'MPI_ALLGATHERV' : "MPI_Allgatherv", +'MPI_GATHER' : "MPI_Gather", +'MPI_GATHERV' : "MPI_Gatherv", +'MPI_SCATTER' : "MPI_Scatter", +'MPI_SCATTERV' : "MPI_Scatterv", +'MPI_REDUCE_SCATTER' : "MPI_Reduce_scatter", +'MPI_SCAN' : "MPI_Scan", +'MPI_REDUCE_SCATTER_BLOCK' : "MPI_Reduce_scatter_block", +'MPI_ALLTOALLW' : "MPI_Alltoallw", +'MPI_EXSCAN' : "MPI_Exscan", +'MPI_IREDUCE' : "MPI_Ireduce", +'MPI_IALLREDUCE' : "MPI_Iallreduce", +'MPI_IBARRIER' : "MPI_Ibarrier", +'MPI_IBCAST' : "MPI_Ibcast", +'MPI_IALLTOALL' : "MPI_Ialltoall", +'MPI_IALLTOALLV' : "MPI_Ialltoallv", +'MPI_IALLGATHER' : "MPI_Iallgather", +'MPI_IALLGATHERV' : "MPI_Iallgatherv", +'MPI_IGATHER' : "MPI_Igather", +'MPI_IGATHERV' : "MPI_Igatherv", +'MPI_ISCATTER' : "MPI_Iscatter", +'MPI_ISCATTERV' : "MPI_Iscatterv", +'MPI_IREDUCESCAT' : "MPI_Ireduce_scatter", +'MPI_ISCAN' : "MPI_Iscan", +'MPI_IREDUCE_SCATTER_BLOCK' : "MPI_Ireduce_scatter_block", +'MPI_IALLTOALLW' : "MPI_Ialltoallw", +'MPI_IEXSCAN' : "MPI_Iexscan", +'MPI_NEIGHBOR_ALLGATHER' : "MPI_Neighbor_allgather", +'MPI_INEIGHBOR_ALLGATHER' : "MPI_Ineighbor_allgather", +'MPI_NEIGHBOR_ALLGATHERV' : "MPI_Neighbor_allgatherv", +'MPI_INEIGHBOR_ALLGATHERV' : "MPI_Ineighbor_allgatherv", +'MPI_NEIGHBOR_ALLTOALL' : "MPI_Neighbor_alltoall", +'MPI_INEIGHBOR_ALLTOALL' : "MPI_Ineighbor_alltoall", +'MPI_NEIGHBOR_ALLTOALLV' : "MPI_Neighbor_alltoallv", +'MPI_INEIGHBOR_ALLTOALLV' : "MPI_Ineighbor_alltoallv", +'MPI_NEIGHBOR_ALLTOALLW' : "MPI_Neighbor_alltoallw", +'MPI_INEIGHBOR_ALLTOALLW' : "MPI_Ineighbor_alltoallw", +'MPI_INIT' : "MPI_Init", +'MPI_FINALIZE' : "MPI_Finalize", +'MPI_BSEND' : "MPI_Bsend", +'MPI_SSEND' : "MPI_Ssend", +'MPI_RSEND' : "MPI_Rsend", +'MPI_IBSEND' : "MPI_Ibsend", +'MPI_ISSEND' : "MPI_Issend", +'MPI_IRSEND' : "MPI_Irsend", +'MPI_TEST' : "MPI_Test", +'MPI_CANCEL' : "MPI_Cancel", +'MPI_SENDRECV' : "MPI_Sendrecv", +'MPI_SENDRECV_REPLACE' : "MPI_Sendrecv_replace", +'MPI_CART_CREATE' : "MPI_Cart_create", +'MPI_CART_SHIFT' : "MPI_Cart_shift", +'MPI_CART_COORDS' : "MPI_Cart_coords", +'MPI_CART_GET' : "MPI_Cart_get", +'MPI_CART_MAP' : "MPI_Cart_map", +'MPI_CART_RANK' : "MPI_Cart_rank", +'MPI_CART_SUB' : "MPI_Cart_sub", +'MPI_CARTDIM_GET' : "MPI_Cartdim_get", +'MPI_DIMS_CREATE' : "MPI_Dims_create", +'MPI_GRAPH_GET' : "MPI_Graph_get", +'MPI_GRAPH_MAP' : "MPI_Graph_map", +'MPI_GRAPH_CREATE' : "MPI_Graph_create", +'MPI_DIST_GRAPH_CREATE' : "MPI_Dist_graph_create", +'MPI_DIST_GRAPH_CREATE_ADJACENT' : "MPI_Dist_graph_create_adjacent", +'MPI_GRAPH_NEIGHBORS' : "MPI_Graph_neighbors", +'MPI_GRAPHDIMS_GET' : "MPI_Graphdims_get", +'MPI_GRAPH_NEIGHBORS_COUNT' : "MPI_Graph_neighbors_count", +'MPI_WAITANY' : "MPI_Waitany", +'MPI_TOPO_TEST' : "MPI_Topo_test", +'MPI_WAITSOME' : "MPI_Waitsome", +'MPI_PROBE' : "MPI_Probe", +'MPI_IPROBE' : "MPI_Iprobe", +'MPI_MPROBE' : "MPI_Mprobe", +'MPI_IMPROBE' : "MPI_Improbe", +'MPI_MRECV' : "MPI_Mrecv", +'MPI_IMRECV' : "MPI_Imrecv", +'MPI_WIN_CREATE' : "MPI_Win_create", +'MPI_WIN_FREE' : "MPI_Win_free", +'MPI_PUT' : "MPI_Put", +'MPI_GET' : "MPI_Get", +'MPI_ACCUMULATE' : "MPI_Accumulate", +'MPI_WIN_FENCE' : "MPI_Win_fence", +'MPI_WIN_START' : "MPI_Win_complete", +'MPI_WIN_COMPLETE' : "MPI_Win_start", +'MPI_WIN_POST' : "MPI_Win_post", +'MPI_WIN_WAIT' : "MPI_Win_wait", +'MPI_WIN_TEST' : "MPI_Win_test", +'MPI_WIN_LOCK' : "MPI_Win_lock", +'MPI_WIN_UNLOCK' : "MPI_Win_unlock", +'MPI_GET_ACCUMULATE' : "MPI_Get_accumulate", +'MPI_FETCH_AND_OP' : "MPI_Fetch_and_op", +'MPI_COMPARE_AND_SWAP' : "MPI_Compare_and_swap", +'MPI_WIN_FLUSH' : "MPI_Win_flush", +'MPI_WIN_FLUSH_ALL' : "MPI_Win_flush_all", +'MPI_WIN_FLUSH_LOCAL' : "MPI_Win_flush_local", +'MPI_WIN_FLUSH_LOCAL_ALL' : "MPI_Win_flush_local_all", +'MPI_PACK' : "MPI_Pack", +'MPI_UNPACK' : "MPI_Unpack", +'MPI_OP_CREATE' : "MPI_Op_create", +'MPI_OP_FREE' : "MPI_Op_free", +'MPI_ATTR_DELETE' : "MPI_Attr_delete", +'MPI_ATTR_GET' : "MPI_Attr_get", +'MPI_ATTR_PUT' : "MPI_Attr_put", +'MPI_COMM_RANK' : "MPI_Comm_rank", +'MPI_COMM_SIZE' : "MPI_Comm_size", +'MPI_COMM_CREATE' : "MPI_Comm_create", +'MPI_COMM_CREATE_GROUP' : "MPI_Comm_create_group", +'MPI_COMM_DUP' : "MPI_Comm_dup", +'MPI_COMM_DUP_WITH_INFO' : "MPI_Comm_dup_with_info", +'MPI_COMM_SPLIT' : "MPI_Comm_split", +'MPI_COMM_SPLIT_TYPE' : "MPI_Comm_split_type", +'MPI_COMM_SPAWN' : "MPI_Comm_spawn", +'MPI_COMM_SPAWN_MULTIPLE' : "MPI_Comm_spawn_multiple", +'MPI_COMM_GROUP' : "MPI_Comm_group", +'MPI_COMM_FREE' : "MPI_Comm_free", +'MPI_COMM_REMOTE_GROUP' : "MPI_Comm_remote_group", +'MPI_COMM_REMOTE_SIZE' : "MPI_Comm_remote_size", +'MPI_COMM_TEST_INTER' : "MPI_Comm_test_inter", +'MPI_COMM_COMPARE' : "MPI_Comm_compare", +'MPI_GROUP_DIFFERENCE' : "MPI_Group_difference", +'MPI_GROUP_EXCL' : "MPI_Group_excl", +'MPI_GROUP_FREE' : "MPI_Group_free", +'MPI_GROUP_INCL' : "MPI_Group_incl", +'MPI_GROUP_INTERSECTION' : "MPI_Group_intersection", +'MPI_GROUP_RANK' : "MPI_Group_rank", +'MPI_GROUP_RANGE_EXCL' : "MPI_Group_range_excl", +'MPI_GROUP_RANGE_INCL' : "MPI_Group_range_incl", +'MPI_GROUP_SIZE' : "MPI_Group_size", +'MPI_GROUP_TRANSLATE_RANKS' : "MPI_Group_translate_ranks", +'MPI_GROUP_UNION' : "MPI_Group_union", +'MPI_GROUP_COMPARE' : "MPI_Group_compare", +'MPI_INTERCOMM_CREATE' : "MPI_Intercomm_create", +'MPI_INTERCOMM_MERGE' : "MPI_Intercomm_merge", +'MPI_KEYVAL_FREE' : "MPI_Keyval_free", +'MPI_KEYVAL_CREATE' : "MPI_Keyval_create", +'MPI_ABORT' : "MPI_Abort", +'MPI_ERROR_CLASS' : "MPI_Error_class", +'MPI_ERRHANDLER_CREATE' : "MPI_Errhandler_create", +'MPI_ERRHANDLER_FREE' : "MPI_Errhandler_free", +'MPI_ERRHANDLER_GET' : "MPI_Errhandler_get", +'MPI_ERROR_STRING' : "MPI_Error_string", +'MPI_ERRHANDLER_SET' : "MPI_Errhandler_set", +'MPI_GET_PROCESSOR_NAME' : "MPI_Get_processor_name", +'MPI_INITIALIZED' : "MPI_Initialized", +'MPI_WTICK' : "MPI_Wtick", +'MPI_WTIME' : "MPI_Wtime", +'MPI_ADDRESS' : "MPI_Address", +'MPI_BSEND_INIT' : "MPI_Bsend_init", +'MPI_BUFFER_ATTACH' : "MPI_Buffer_attach", +'MPI_BUFFER_DETACH' : "MPI_Buffer_detach", +'MPI_REQUEST_FREE' : "MPI_Request_free", +'MPI_RECV_INIT' : "MPI_Recv_init", +'MPI_SEND_INIT' : "MPI_Send_init", +'MPI_GET_COUNT' : "MPI_Get_count", +'MPI_GET_ELEMENTS' : "MPI_Get_elements", +'MPI_PACK_SIZE' : "MPI_Pack_size", +'MPI_RSEND_INIT' : "MPI_Rsend_init", +'MPI_SSEND_INIT' : "MPI_Ssend_init", +'MPI_START' : "MPI_Start", +'MPI_STARTALL' : "MPI_Startall", +'MPI_TESTALL' : "MPI_Testall", +'MPI_TESTANY' : "MPI_Testany", +'MPI_TEST_CANCELLED' : "MPI_Test_cancelled", +'MPI_TESTSOME' : "MPI_Testsome", +'MPI_TYPE_COMMIT' : "MPI_Type_commit", +'MPI_TYPE_CONTIGUOUS' : "MPI_Type_contiguous", +'MPI_TYPE_EXTENT' : "MPI_Type_extent", +'MPI_TYPE_FREE' : "MPI_Type_free", +'MPI_TYPE_HINDEXED' : "MPI_Type_hindexed", +'MPI_TYPE_HVECTOR' : "MPI_Type_hvector", +'MPI_TYPE_INDEXED' : "MPI_Type_indexed", +'MPI_TYPE_LB' : "MPI_Type_lb", +'MPI_TYPE_SIZE' : "MPI_Type_size", +'MPI_TYPE_STRUCT' : "MPI_Type_struct", +'MPI_TYPE_UB' : "MPI_Type_ub", +'MPI_TYPE_VECTOR' : "MPI_Type_vector", +'MPI_FILE_OPEN' : "MPI_File_open", +'MPI_FILE_CLOSE' : "MPI_File_close", +'MPI_FILE_READ' : "MPI_File_read", +'MPI_FILE_READ_ALL' : "MPI_File_read_all", +'MPI_FILE_READ_ALL_BEGIN' : "MPI_File_read_all_begin", +'MPI_FILE_READ_ALL_END' : "MPI_File_read_all_end", +'MPI_FILE_READ_AT' : "MPI_File_read_at", +'MPI_FILE_READ_AT_ALL' : "MPI_File_read_at_all", +'MPI_FILE_READ_AT_ALL_BEGIN' : "MPI_File_read_at_all_begin", +'MPI_FILE_READ_AT_ALL_END' : "MPI_File_read_at_all_end", +'MPI_FILE_READ_ORDERED' : "MPI_File_read_ordered", +'MPI_FILE_READ_ORDERED_BEGIN' : "MPI_File_read_ordered_begin", +'MPI_FILE_READ_ORDERED_END' : "MPI_File_read_ordered_end", +'MPI_FILE_READ_SHARED' : "MPI_File_read_shared", +'MPI_FILE_WRITE' : "MPI_File_write", +'MPI_FILE_WRITE_ALL' : "MPI_File_write_all", +'MPI_FILE_WRITE_ALL_BEGIN' : "MPI_File_write_all_begin", +'MPI_FILE_WRITE_ALL_END' : "MPI_File_write_all_end", +'MPI_FILE_WRITE_AT' : "MPI_File_write_at", +'MPI_FILE_WRITE_AT_ALL' : "MPI_File_write_at_all", +'MPI_FILE_WRITE_AT_ALL_BEGIN' : "MPI_File_write_at_all_begin", +'MPI_FILE_WRITE_AT_ALL_END' : "MPI_File_write_at_all_end", +'MPI_FILE_WRITE_ORDERED' : "MPI_File_write_ordered", +'MPI_FILE_WRITE_ORDERED_BEGIN' : "MPI_File_write_ordered_begin", +'MPI_FILE_WRITE_ORDERED_END' : "MPI_File_write_ordered_end", +'MPI_FILE_WRITE_SHARED' : "MPI_File_write_shared", +'MPI_REQUEST_GET_STATUS' : "MPI_Request_get_status" +} \ No newline at end of file diff --git a/parser-playground.ipynb b/parser-playground.ipynb index 3a2169c..370390e 100644 --- a/parser-playground.ipynb +++ b/parser-playground.ipynb @@ -70,6 +70,126 @@ " return os.path.join(REPORT_DIR, base_name+\"_{}.csv\".format(report_name))" ] }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
event_valueName
MPI_SEND1MPI_Send
MPI_RECV2MPI_Recv
MPI_ISEND3MPI_Isend
MPI_IRECV4MPI_Irecv
MPI_WAIT5MPI_Wait
.........
MPI_FILE_WRITE_SHARED210MPI_File_write_shared
MPI_COMM_DUP_WITH_INFO211MPI_Comm_dup_with_info
MPI_DIST_GRAPH_CREATE_ADJACENT212MPI_Dist_graph_create_adjacent
MPI_COMM_CREATE_GROUP213MPI_Comm_create_group
MPI_EXSCAN214MPI_Exscan
\n", + "

214 rows × 2 columns

\n", + "
" + ], + "text/plain": [ + " event_value Name\n", + "MPI_SEND 1 MPI_Send\n", + "MPI_RECV 2 MPI_Recv\n", + "MPI_ISEND 3 MPI_Isend\n", + "MPI_IRECV 4 MPI_Irecv\n", + "MPI_WAIT 5 MPI_Wait\n", + "... ... ...\n", + "MPI_FILE_WRITE_SHARED 210 MPI_File_write_shared\n", + "MPI_COMM_DUP_WITH_INFO 211 MPI_Comm_dup_with_info\n", + "MPI_DIST_GRAPH_CREATE_ADJACENT 212 MPI_Dist_graph_create_adjacent\n", + "MPI_COMM_CREATE_GROUP 213 MPI_Comm_create_group\n", + "MPI_EXSCAN 214 MPI_Exscan\n", + "\n", + "[214 rows x 2 columns]" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from nsys2prv.semantics.mpi_event_encoding import *\n", + "mpi_values = pd.DataFrame.from_dict(MPIVal, orient='index', columns=[\"event_value\"])\n", + "mpi_labels = pd.DataFrame.from_dict(MPI_Val_Labels, orient='index', columns=[\"Name\"])\n", + "mpi_values.merge(mpi_labels, left_index=True, right_index=True)" + ] + }, { "cell_type": "code", "execution_count": 4, diff --git a/pyproject.toml b/pyproject.toml index 8e9fd4c..62dd86b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ homepage = "https://pm.bsc.es/gitlab/beppp/nsys2prv" python = "^3.11" pandas = "^2.2.2" tqdm = "*" -sqlalchemy = "*" +sqlalchemy = "^2.0.0" [tool.poetry.scripts] nsys2prv = 'nsys2prv.parse_nsys_stats:main' -- GitLab From ce896e41cf8d994db36f6fd69afc619086eebf8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Clasc=C3=A0?= Date: Thu, 8 Aug 2024 13:12:59 +0200 Subject: [PATCH 2/4] remove useless statement --- nsys2prv/parse_nsys_stats.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/nsys2prv/parse_nsys_stats.py b/nsys2prv/parse_nsys_stats.py index bebf3c5..1524b11 100755 --- a/nsys2prv/parse_nsys_stats.py +++ b/nsys2prv/parse_nsys_stats.py @@ -286,8 +286,7 @@ def main(): nvtx_df["task"] = 0 nvtx_startend_df["thread"] = 0 nvtx_startend_df["task"] = 0 - mpi_df["thread"] = 0 - mpi_df["task"] = 0 + if t_openacc: openacc_other_df["thread"] = 0 openacc_other_df["task"] = 0 -- GitLab From a4fc2ce32c2964e26f687303f099fc2bfc677764 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Clasc=C3=A0?= Date: Thu, 8 Aug 2024 13:14:33 +0200 Subject: [PATCH 3/4] Adds debugging of MPI comms to playground --- parser-playground.ipynb | 351 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 346 insertions(+), 5 deletions(-) diff --git a/parser-playground.ipynb b/parser-playground.ipynb index 370390e..1447644 100644 --- a/parser-playground.ipynb +++ b/parser-playground.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ @@ -13,7 +13,7 @@ "import os\n", "import locale\n", "import sqlite3\n", - "from sqlalchemy import create_engine\n", + "from sqlalchemy import create_engine, text\n", "\n", "NSIGHT_HOME=\"/home/mclasca/Apps/nsight-system/2024.1\"\n", "#NSIGHT_HOME = os.getenv('NSIGHT_HOME')\n", @@ -63,7 +63,7 @@ "nvtx_stack_top = 1\n", "nvtx_stack_bottom = 4\n", "\n", - "reports = [\"nvtx_pushpop_trace\", \"cuda_api_trace\", \"cuda_gpu_trace\"]\n", + "reports = [\"cuda_api_trace\", \"cuda_gpu_trace\"]\n", "\n", "def build_nsys_stats_name(report_name):\n", " base_name = os.path.splitext(os.path.basename(REPORT_FILE))[0]\n", @@ -72,7 +72,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 2, "metadata": {}, "outputs": [ { @@ -178,7 +178,7 @@ "[214 rows x 2 columns]" ] }, - "execution_count": 7, + "execution_count": 2, "metadata": {}, "output_type": "execute_result" } @@ -614,6 +614,347 @@ "kernels_df" ] }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/tmp/ipykernel_184821/1351675269.py:25: FutureWarning: The behavior of DataFrame concatenation with empty or all-NA entries is deprecated. In a future version, this will no longer exclude empty or all-NA columns when determining the result dtypes. To retain the old behavior, exclude the relevant entries before the concat operation.\n", + " mpi_df = pd.concat([mpi_p2p_df, mpi_coll_df, mpi_other_df, mpi_rma_df, mpi_io_df])\n" + ] + } + ], + "source": [ + "engine = create_engine(f\"sqlite:///{os.path.splitext(REPORT_FILE)[0]}.sqlite\")\n", + "with engine.connect() as conn, conn.begin():\n", + " with open(os.path.join('nsys2prv/scripts/mpi_p2p.sql'), 'r') as query:\n", + " mpi_p2p_df = pd.read_sql_query(text(query.read()), conn)\n", + " mpi_p2p_df[\"event_type\"] = MPITYPE_PTOP\n", + " with open(os.path.join('nsys2prv/scripts/mpi_coll.sql'), 'r') as query:\n", + " mpi_coll_df = pd.read_sql_query(text(query.read()), conn)\n", + " mpi_coll_df = mpi_coll_df.drop(mpi_coll_df[mpi_coll_df[\"Event\"].str.contains(\"File\") ].index)\n", + " mpi_coll_df[\"event_type\"] = MPITYPE_COLLECTIVE\n", + " with open(os.path.join('nsys2prv/scripts/mpi_other.sql'), 'r') as query:\n", + " mpi_other_df = pd.read_sql_query(text(query.read()), conn)\n", + " mpi_other_df = mpi_other_df.drop(mpi_other_df[mpi_other_df[\"Event\"].str.contains(\"File\") ].index)\n", + " mpi_other_df = mpi_other_df.drop(mpi_other_df[mpi_other_df[\"Event\"].str.contains(\"Win|MPI_Get|MPI_Put|Accumulate\") ].index)\n", + " mpi_other_df[\"event_type\"] = MPITYPE_OTHER\n", + " with open(os.path.join('nsys2prv/scripts/mpi_other.sql'), 'r') as query:\n", + " mpi_rma_df = pd.read_sql_query(text(query.read()), conn)\n", + " mpi_rma_df = mpi_rma_df[mpi_rma_df[\"Event\"].str.contains(\"Win|MPI_Get|MPI_Put|Accumulate\")]\n", + " mpi_rma_df[\"event_type\"] = MPITYPE_RMA\n", + " mpi_rma_df[\"Kind\"] = \"rma\"\n", + " with open(os.path.join('nsys2prv/scripts/mpi_io.sql'), 'r') as query:\n", + " mpi_io_df = pd.read_sql_query(text(query.read()), conn)\n", + " mpi_io_df = mpi_io_df[mpi_io_df[\"Event\"].str.contains(\"File\")]\n", + " mpi_io_df[\"event_type\"] = MPITYPE_IO\n", + " mpi_io_df[\"Kind\"] = \"io\"\n", + "mpi_df = pd.concat([mpi_p2p_df, mpi_coll_df, mpi_other_df, mpi_rma_df, mpi_io_df])" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
KindStart:ts_nsEnd:ts_nsDuration:dur_nsEventPidTidTagRankPeerRankRootRankSize:mem_bCollSendSize:mem_bCollRecvSize:mem_bevent_type
0p2p842536969084253717462056MPI_Irecv5856115856110.001.0NaN755688.0NaNNaN50000001
1p2p842539570384283335282937825MPI_Isend5856115856110.001.0NaN755688.0NaNNaN50000001
2p2p842567867484256812572583MPI_Irecv5856135856130.020.0NaN8468.0NaNNaN50000001
3p2p842569310384290608673367764MPI_Isend5856135856130.020.0NaN8468.0NaNNaN50000001
4p2p842574568584257477842099MPI_Irecv5856125856120.010.0NaN755688.0NaNNaN50000001
................................................
59659p2p382722370113827226415827147MPI_Waitall585613585613NaN2NaNNaNNaNNaNNaN50000001
59660p2p382722370113827226415827147MPI_Waitall585613585613NaN2NaNNaNNaNNaNNaN50000001
59661p2p382722370113827226415827147MPI_Waitall585613585613NaN2NaNNaNNaNNaNNaN50000001
59662p2p382722370113827226415827147MPI_Waitall585613585613NaN2NaNNaNNaNNaNNaN50000001
59663p2p382722370113827226415827147MPI_Waitall585613585613NaN2NaNNaNNaNNaNNaN50000001
\n", + "

59664 rows × 15 columns

\n", + "
" + ], + "text/plain": [ + " Kind Start:ts_ns End:ts_ns Duration:dur_ns Event Pid \\\n", + "0 p2p 8425369690 8425371746 2056 MPI_Irecv 585611 \n", + "1 p2p 8425395703 8428333528 2937825 MPI_Isend 585611 \n", + "2 p2p 8425678674 8425681257 2583 MPI_Irecv 585613 \n", + "3 p2p 8425693103 8429060867 3367764 MPI_Isend 585613 \n", + "4 p2p 8425745685 8425747784 2099 MPI_Irecv 585612 \n", + "... ... ... ... ... ... ... \n", + "59659 p2p 38272237011 38272264158 27147 MPI_Waitall 585613 \n", + "59660 p2p 38272237011 38272264158 27147 MPI_Waitall 585613 \n", + "59661 p2p 38272237011 38272264158 27147 MPI_Waitall 585613 \n", + "59662 p2p 38272237011 38272264158 27147 MPI_Waitall 585613 \n", + "59663 p2p 38272237011 38272264158 27147 MPI_Waitall 585613 \n", + "\n", + " Tid Tag Rank PeerRank RootRank Size:mem_b CollSendSize:mem_b \\\n", + "0 585611 0.0 0 1.0 NaN 755688.0 NaN \n", + "1 585611 0.0 0 1.0 NaN 755688.0 NaN \n", + "2 585613 0.0 2 0.0 NaN 8468.0 NaN \n", + "3 585613 0.0 2 0.0 NaN 8468.0 NaN \n", + "4 585612 0.0 1 0.0 NaN 755688.0 NaN \n", + "... ... ... ... ... ... ... ... \n", + "59659 585613 NaN 2 NaN NaN NaN NaN \n", + "59660 585613 NaN 2 NaN NaN NaN NaN \n", + "59661 585613 NaN 2 NaN NaN NaN NaN \n", + "59662 585613 NaN 2 NaN NaN NaN NaN \n", + "59663 585613 NaN 2 NaN NaN NaN NaN \n", + "\n", + " CollRecvSize:mem_b event_type \n", + "0 NaN 50000001 \n", + "1 NaN 50000001 \n", + "2 NaN 50000001 \n", + "3 NaN 50000001 \n", + "4 NaN 50000001 \n", + "... ... ... \n", + "59659 NaN 50000001 \n", + "59660 NaN 50000001 \n", + "59661 NaN 50000001 \n", + "59662 NaN 50000001 \n", + "59663 NaN 50000001 \n", + "\n", + "[59664 rows x 15 columns]" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "mpi_df[mpi_df[\"Kind\"] == \"p2p\"]" + ] + }, { "cell_type": "code", "execution_count": 2, -- GitLab From 1cc7cf9ef08847ef7f64589143b7b68dfc67feba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20Clasc=C3=A0?= Date: Fri, 9 Aug 2024 15:13:05 +0200 Subject: [PATCH 4/4] Adds events for global MPI ops sizes --- nsys2prv/parse_nsys_stats.py | 73 +++-- nsys2prv/scripts/mpi_coll.sql | 4 +- nsys2prv/semantics/mpi_event_encoding.py | 2 + parser-playground.ipynb | 368 +++++++++++------------ 4 files changed, 237 insertions(+), 210 deletions(-) diff --git a/nsys2prv/parse_nsys_stats.py b/nsys2prv/parse_nsys_stats.py index 1524b11..6df3a63 100755 --- a/nsys2prv/parse_nsys_stats.py +++ b/nsys2prv/parse_nsys_stats.py @@ -8,7 +8,8 @@ import time import subprocess import os import locale -from sqlalchemy import create_engine, text +from sqlalchemy import create_engine, text, dialects +from sqlalchemy.exc import OperationalError from .EventWriter import event_writer as ewr from .semantics.mpi_event_encoding import * @@ -179,27 +180,42 @@ def main(): if t_mpi: with engine.connect() as conn, conn.begin(): - with open(os.path.join(os.path.dirname(__file__), 'scripts/mpi_p2p.sql'), 'r') as query: - mpi_p2p_df = pd.read_sql_query(text(query.read()), conn) - mpi_p2p_df["event_type"] = MPITYPE_PTOP - with open(os.path.join(os.path.dirname(__file__), 'scripts/mpi_coll.sql'), 'r') as query: - mpi_coll_df = pd.read_sql_query(text(query.read()), conn) - mpi_coll_df = mpi_coll_df.drop(mpi_coll_df[mpi_coll_df["Event"].str.contains("File") ].index) - mpi_coll_df["event_type"] = MPITYPE_COLLECTIVE - with open(os.path.join(os.path.dirname(__file__), 'scripts/mpi_other.sql'), 'r') as query: - mpi_other_df = pd.read_sql_query(text(query.read()), conn) - mpi_other_df = mpi_other_df.drop(mpi_other_df[mpi_other_df["Event"].str.contains("File") ].index) - mpi_other_df = mpi_other_df.drop(mpi_other_df[mpi_other_df["Event"].str.contains("Win|MPI_Get|MPI_Put|Accumulate") ].index) - mpi_other_df["event_type"] = MPITYPE_OTHER - with open(os.path.join(os.path.dirname(__file__), 'scripts/mpi_other.sql'), 'r') as query: - mpi_rma_df = pd.read_sql_query(text(query.read()), conn) - mpi_rma_df = mpi_rma_df[mpi_rma_df["Event"].str.contains("Win|MPI_Get|MPI_Put|Accumulate")] - mpi_rma_df["event_type"] = MPITYPE_RMA - with open(os.path.join(os.path.dirname(__file__), 'scripts/mpi_io.sql'), 'r') as query: - mpi_io_df = pd.read_sql_query(text(query.read()), conn) - mpi_io_df = mpi_io_df[mpi_io_df["Event"].str.contains("File")] - mpi_io_df["event_type"] = MPITYPE_IO - mpi_df = pd.concat([mpi_p2p_df, mpi_coll_df, mpi_other_df, mpi_rma_df, mpi_io_df]) + try: + with open(os.path.join(os.path.dirname(__file__), 'scripts/mpi_p2p.sql'), 'r') as query: + if conn.dialect.has_table(connection=conn, table_name='MPI_P2P_EVENTS') and conn.dialect.has_table(connection=conn, table_name='MPI_START_WAIT_EVENTS'): + mpi_p2p_df = pd.read_sql_query(text(query.read()), conn) + mpi_p2p_df["event_type"] = MPITYPE_PTOP + else: mpi_p2p_df = pd.DataFrame() + with open(os.path.join(os.path.dirname(__file__), 'scripts/mpi_coll.sql'), 'r') as query: + if conn.dialect.has_table(connection=conn, table_name='MPI_COLLECTIVES_EVENTS'): + mpi_coll_df = pd.read_sql_query(text(query.read()), conn) + mpi_coll_df = mpi_coll_df.drop(mpi_coll_df[mpi_coll_df["Event"].str.contains("File") ].index) + mpi_coll_df["event_type"] = MPITYPE_COLLECTIVE + else: mpi_coll_df = pd.DataFrame() + with open(os.path.join(os.path.dirname(__file__), 'scripts/mpi_other.sql'), 'r') as query: + if conn.dialect.has_table(connection=conn, table_name='MPI_OTHER_EVENTS'): + mpi_other_df = pd.read_sql_query(text(query.read()), conn) + mpi_other_df = mpi_other_df.drop(mpi_other_df[mpi_other_df["Event"].str.contains("File") ].index) + mpi_other_df = mpi_other_df.drop(mpi_other_df[mpi_other_df["Event"].str.contains("Win|MPI_Get|MPI_Put|Accumulate") ].index) + mpi_other_df["event_type"] = MPITYPE_OTHER + else: mpi_other_df = pd.DataFrame() + with open(os.path.join(os.path.dirname(__file__), 'scripts/mpi_other.sql'), 'r') as query: + if conn.dialect.has_table(connection=conn, table_name='MPI_OTHER_EVENTS'): + mpi_rma_df = pd.read_sql_query(text(query.read()), conn) + mpi_rma_df = mpi_rma_df[mpi_rma_df["Event"].str.contains("Win|MPI_Get|MPI_Put|Accumulate")] + mpi_rma_df["event_type"] = MPITYPE_RMA + else: mpi_rma_df = pd.DataFrame() + with open(os.path.join(os.path.dirname(__file__), 'scripts/mpi_io.sql'), 'r') as query: + if conn.dialect.has_table(connection=conn, table_name='MPI_OTHER_EVENTS') and conn.dialect.has_table(connection=conn, table_name='MPI_COLLECTIVES_EVENTS'): + mpi_io_df = pd.read_sql_query(text(query.read()), conn) + mpi_io_df = mpi_io_df[mpi_io_df["Event"].str.contains("File")] + mpi_io_df["event_type"] = MPITYPE_IO + else: mpi_io_df = pd.DataFrame() + mpi_df = pd.concat([mpi_p2p_df, mpi_coll_df, mpi_other_df, mpi_rma_df, mpi_io_df]) + except OperationalError as oe: + print("There has been a problem fetching MPI information. MPI data will be skipped.") + print(f"[ERROR]: {oe.detail}") + t_mpi = False #mpi_df = pd.read_csv(build_nsys_stats_name("mpi_event_trace")) else: #mpi_df = pd.DataFrame() @@ -593,6 +609,11 @@ def main(): for index, row in mpi_names.iterrows(): pcf_file.write("{} {}\n".format(row["event_value"], row["Name"])) pcf_file.write("\n") + pcf_file.write("EVENT_TYPE\n") + pcf_file.write("1 {} {}\n".format(MPITYPE_SEND_GLOBAL_SIZE, "Send Size in MPI Global OP")) + pcf_file.write("1 {} {}\n".format(MPITYPE_RECV_GLOBAL_SIZE, "Recv Size in MPI Global OP")) + pcf_file.write("\n") + pcf_file.write("EVENT_TYPE\n") pcf_file.write("0 {} CUDA kernel\n".format(event_type_kernels)) @@ -807,8 +828,12 @@ def main(): if t_mpi: - ewr(prv_file, mpi_df, "MPI events", lambda r: - (create_event_record(r.iloc[1], r.iloc[3], int(r["thread"]), int(r["task"]), r["event_type"], r["event_value"]))) + def serialize_mpi(r): + if r["Kind"] == "collectives": + return create_combined_events_record(r.iloc[1], r.iloc[3], int(r["thread"]), int(r["task"]), [r["event_type"], MPITYPE_SEND_GLOBAL_SIZE, MPITYPE_RECV_GLOBAL_SIZE], [r["event_value"], r["CollSendSize:mem_b"], r["CollRecvSize:mem_b"]]) + else: + return create_event_record(r.iloc[1], r.iloc[3], int(r["thread"]), int(r["task"]), r["event_type"], r["event_value"]) + ewr(prv_file, mpi_df, "MPI events", lambda r: serialize_mpi(r)) if t_openacc: t_acc_d = [event_type_openacc_data, event_type_name_openacc_data, event_type_func_openacc_data, event_type_openacc_data_size] diff --git a/nsys2prv/scripts/mpi_coll.sql b/nsys2prv/scripts/mpi_coll.sql index e9754d6..cdaa637 100644 --- a/nsys2prv/scripts/mpi_coll.sql +++ b/nsys2prv/scripts/mpi_coll.sql @@ -28,8 +28,8 @@ SELECT e.remoteRank AS "PeerRank", e.rootRank AS "RootRank", e.size AS "Size:mem_b", - e.collSendSize AS "CollSendSize:mem_b", - e.collRecvSize AS "CollRecvSize:mem_b" + IFNULL(e.collSendSize, 0) AS "CollSendSize:mem_b", + IFNULL(e.collRecvSize, 0) AS "CollRecvSize:mem_b" FROM evts AS e LEFT JOIN diff --git a/nsys2prv/semantics/mpi_event_encoding.py b/nsys2prv/semantics/mpi_event_encoding.py index b4b7d19..65acdf2 100644 --- a/nsys2prv/semantics/mpi_event_encoding.py +++ b/nsys2prv/semantics/mpi_event_encoding.py @@ -26,6 +26,8 @@ MPITYPE_PTOP = 50000001 MPITYPE_COLLECTIVE = 50000002 MPITYPE_OTHER = 50000003 MPITYPE_RMA = 50000004 +MPITYPE_SEND_GLOBAL_SIZE = 50100001 +MPITYPE_RECV_GLOBAL_SIZE = 50100002 MPITYPE_RMA_SIZE = 50001000 MPITYPE_RMA_TARGET_RANK = 50001001 MPITYPE_RMA_ORIGIN_ADDR = 50001002 diff --git a/parser-playground.ipynb b/parser-playground.ipynb index 1447644..b48c4e7 100644 --- a/parser-playground.ipynb +++ b/parser-playground.ipynb @@ -616,7 +616,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 36, "metadata": {}, "outputs": [ { @@ -658,7 +658,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 38, "metadata": {}, "outputs": [ { @@ -701,94 +701,94 @@ " \n", " \n", " \n", - " 0\n", - " p2p\n", - " 8425369690\n", - " 8425371746\n", - " 2056\n", - " MPI_Irecv\n", - " 585611\n", - " 585611\n", + " 4\n", + " collectives\n", + " 1956941376\n", + " 1956971021\n", + " 29645\n", + " MPI_Bcast\n", + " 585614\n", + " 585614\n", + " None\n", + " 3\n", + " None\n", " 0.0\n", + " None\n", " 0\n", - " 1.0\n", - " NaN\n", - " 755688.0\n", - " NaN\n", - " NaN\n", - " 50000001\n", + " 8\n", + " 50000002\n", " \n", " \n", - " 1\n", - " p2p\n", - " 8425395703\n", - " 8428333528\n", - " 2937825\n", - " MPI_Isend\n", - " 585611\n", - " 585611\n", + " 5\n", + " collectives\n", + " 1956946364\n", + " 1956970755\n", + " 24391\n", + " MPI_Bcast\n", + " 585612\n", + " 585612\n", + " None\n", + " 1\n", + " None\n", " 0.0\n", + " None\n", " 0\n", - " 1.0\n", - " NaN\n", - " 755688.0\n", - " NaN\n", - " NaN\n", - " 50000001\n", + " 8\n", + " 50000002\n", " \n", " \n", - " 2\n", - " p2p\n", - " 8425678674\n", - " 8425681257\n", - " 2583\n", - " MPI_Irecv\n", + " 6\n", + " collectives\n", + " 1956949643\n", + " 1956967650\n", + " 18007\n", + " MPI_Bcast\n", " 585613\n", " 585613\n", - " 0.0\n", + " None\n", " 2\n", + " None\n", " 0.0\n", - " NaN\n", - " 8468.0\n", - " NaN\n", - " NaN\n", - " 50000001\n", + " None\n", + " 0\n", + " 8\n", + " 50000002\n", " \n", " \n", - " 3\n", - " p2p\n", - " 8425693103\n", - " 8429060867\n", - " 3367764\n", - " MPI_Isend\n", - " 585613\n", - " 585613\n", - " 0.0\n", - " 2\n", + " 7\n", + " collectives\n", + " 1956965684\n", + " 1956967384\n", + " 1700\n", + " MPI_Bcast\n", + " 585611\n", + " 585611\n", + " None\n", + " 0\n", + " None\n", " 0.0\n", - " NaN\n", - " 8468.0\n", - " NaN\n", - " NaN\n", - " 50000001\n", + " None\n", + " 32\n", + " 8\n", + " 50000002\n", " \n", " \n", - " 4\n", - " p2p\n", - " 8425745685\n", - " 8425747784\n", - " 2099\n", - " MPI_Irecv\n", - " 585612\n", - " 585612\n", - " 0.0\n", - " 1\n", + " 8\n", + " collectives\n", + " 1957587415\n", + " 1962742830\n", + " 5155415\n", + " MPI_Bcast\n", + " 585614\n", + " 585614\n", + " None\n", + " 3\n", + " None\n", " 0.0\n", - " NaN\n", - " 755688.0\n", - " NaN\n", - " NaN\n", - " 50000001\n", + " None\n", + " 0\n", + " 8\n", + " 50000002\n", " \n", " \n", " ...\n", @@ -809,150 +809,150 @@ " ...\n", " \n", " \n", - " 59659\n", - " p2p\n", - " 38272237011\n", - " 38272264158\n", - " 27147\n", - " MPI_Waitall\n", - " 585613\n", - " 585613\n", - " NaN\n", - " 2\n", - " NaN\n", - " NaN\n", - " NaN\n", - " NaN\n", + " 1663\n", + " collectives\n", + " 38877340595\n", + " 38877344920\n", + " 4325\n", + " MPI_Allreduce\n", + " 585611\n", + " 585611\n", + " None\n", + " 0\n", + " None\n", " NaN\n", - " 50000001\n", + " None\n", + " 16\n", + " 16\n", + " 50000002\n", " \n", " \n", - " 59660\n", - " p2p\n", - " 38272237011\n", - " 38272264158\n", - " 27147\n", - " MPI_Waitall\n", - " 585613\n", - " 585613\n", - " NaN\n", - " 2\n", - " NaN\n", - " NaN\n", - " NaN\n", - " NaN\n", + " 1664\n", + " collectives\n", + " 38878902093\n", + " 38879094208\n", + " 192115\n", + " MPI_Allreduce\n", + " 585612\n", + " 585612\n", + " None\n", + " 1\n", + " None\n", " NaN\n", - " 50000001\n", + " None\n", + " 16\n", + " 16\n", + " 50000002\n", " \n", " \n", - " 59661\n", - " p2p\n", - " 38272237011\n", - " 38272264158\n", - " 27147\n", - " MPI_Waitall\n", - " 585613\n", - " 585613\n", - " NaN\n", - " 2\n", - " NaN\n", - " NaN\n", - " NaN\n", - " NaN\n", + " 1665\n", + " collectives\n", + " 38878902634\n", + " 38879095108\n", + " 192474\n", + " MPI_Allreduce\n", + " 585614\n", + " 585614\n", + " None\n", + " 3\n", + " None\n", " NaN\n", - " 50000001\n", + " None\n", + " 16\n", + " 16\n", + " 50000002\n", " \n", " \n", - " 59662\n", - " p2p\n", - " 38272237011\n", - " 38272264158\n", - " 27147\n", - " MPI_Waitall\n", + " 1666\n", + " collectives\n", + " 38878906283\n", + " 38879094490\n", + " 188207\n", + " MPI_Allreduce\n", " 585613\n", " 585613\n", - " NaN\n", + " None\n", " 2\n", + " None\n", " NaN\n", - " NaN\n", - " NaN\n", - " NaN\n", - " NaN\n", - " 50000001\n", + " None\n", + " 16\n", + " 16\n", + " 50000002\n", " \n", " \n", - " 59663\n", - " p2p\n", - " 38272237011\n", - " 38272264158\n", - " 27147\n", - " MPI_Waitall\n", - " 585613\n", - " 585613\n", - " NaN\n", - " 2\n", - " NaN\n", - " NaN\n", - " NaN\n", - " NaN\n", + " 1667\n", + " collectives\n", + " 38879090831\n", + " 38879094772\n", + " 3941\n", + " MPI_Allreduce\n", + " 585611\n", + " 585611\n", + " None\n", + " 0\n", + " None\n", " NaN\n", - " 50000001\n", + " None\n", + " 16\n", + " 16\n", + " 50000002\n", " \n", " \n", "\n", - "

59664 rows × 15 columns

\n", + "

1652 rows × 15 columns

\n", "" ], "text/plain": [ - " Kind Start:ts_ns End:ts_ns Duration:dur_ns Event Pid \\\n", - "0 p2p 8425369690 8425371746 2056 MPI_Irecv 585611 \n", - "1 p2p 8425395703 8428333528 2937825 MPI_Isend 585611 \n", - "2 p2p 8425678674 8425681257 2583 MPI_Irecv 585613 \n", - "3 p2p 8425693103 8429060867 3367764 MPI_Isend 585613 \n", - "4 p2p 8425745685 8425747784 2099 MPI_Irecv 585612 \n", - "... ... ... ... ... ... ... \n", - "59659 p2p 38272237011 38272264158 27147 MPI_Waitall 585613 \n", - "59660 p2p 38272237011 38272264158 27147 MPI_Waitall 585613 \n", - "59661 p2p 38272237011 38272264158 27147 MPI_Waitall 585613 \n", - "59662 p2p 38272237011 38272264158 27147 MPI_Waitall 585613 \n", - "59663 p2p 38272237011 38272264158 27147 MPI_Waitall 585613 \n", + " Kind Start:ts_ns End:ts_ns Duration:dur_ns Event \\\n", + "4 collectives 1956941376 1956971021 29645 MPI_Bcast \n", + "5 collectives 1956946364 1956970755 24391 MPI_Bcast \n", + "6 collectives 1956949643 1956967650 18007 MPI_Bcast \n", + "7 collectives 1956965684 1956967384 1700 MPI_Bcast \n", + "8 collectives 1957587415 1962742830 5155415 MPI_Bcast \n", + "... ... ... ... ... ... \n", + "1663 collectives 38877340595 38877344920 4325 MPI_Allreduce \n", + "1664 collectives 38878902093 38879094208 192115 MPI_Allreduce \n", + "1665 collectives 38878902634 38879095108 192474 MPI_Allreduce \n", + "1666 collectives 38878906283 38879094490 188207 MPI_Allreduce \n", + "1667 collectives 38879090831 38879094772 3941 MPI_Allreduce \n", "\n", - " Tid Tag Rank PeerRank RootRank Size:mem_b CollSendSize:mem_b \\\n", - "0 585611 0.0 0 1.0 NaN 755688.0 NaN \n", - "1 585611 0.0 0 1.0 NaN 755688.0 NaN \n", - "2 585613 0.0 2 0.0 NaN 8468.0 NaN \n", - "3 585613 0.0 2 0.0 NaN 8468.0 NaN \n", - "4 585612 0.0 1 0.0 NaN 755688.0 NaN \n", - "... ... ... ... ... ... ... ... \n", - "59659 585613 NaN 2 NaN NaN NaN NaN \n", - "59660 585613 NaN 2 NaN NaN NaN NaN \n", - "59661 585613 NaN 2 NaN NaN NaN NaN \n", - "59662 585613 NaN 2 NaN NaN NaN NaN \n", - "59663 585613 NaN 2 NaN NaN NaN NaN \n", + " Pid Tid Tag Rank PeerRank RootRank Size:mem_b \\\n", + "4 585614 585614 None 3 None 0.0 None \n", + "5 585612 585612 None 1 None 0.0 None \n", + "6 585613 585613 None 2 None 0.0 None \n", + "7 585611 585611 None 0 None 0.0 None \n", + "8 585614 585614 None 3 None 0.0 None \n", + "... ... ... ... ... ... ... ... \n", + "1663 585611 585611 None 0 None NaN None \n", + "1664 585612 585612 None 1 None NaN None \n", + "1665 585614 585614 None 3 None NaN None \n", + "1666 585613 585613 None 2 None NaN None \n", + "1667 585611 585611 None 0 None NaN None \n", "\n", - " CollRecvSize:mem_b event_type \n", - "0 NaN 50000001 \n", - "1 NaN 50000001 \n", - "2 NaN 50000001 \n", - "3 NaN 50000001 \n", - "4 NaN 50000001 \n", - "... ... ... \n", - "59659 NaN 50000001 \n", - "59660 NaN 50000001 \n", - "59661 NaN 50000001 \n", - "59662 NaN 50000001 \n", - "59663 NaN 50000001 \n", + " CollSendSize:mem_b CollRecvSize:mem_b event_type \n", + "4 0 8 50000002 \n", + "5 0 8 50000002 \n", + "6 0 8 50000002 \n", + "7 32 8 50000002 \n", + "8 0 8 50000002 \n", + "... ... ... ... \n", + "1663 16 16 50000002 \n", + "1664 16 16 50000002 \n", + "1665 16 16 50000002 \n", + "1666 16 16 50000002 \n", + "1667 16 16 50000002 \n", "\n", - "[59664 rows x 15 columns]" + "[1652 rows x 15 columns]" ] }, - "execution_count": 17, + "execution_count": 38, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "mpi_df[mpi_df[\"Kind\"] == \"p2p\"]" + "mpi_coll_df" ] }, { -- GitLab