GCC Code Coverage Report

Directory: src/
File: src/talp/talp_record.c
Date: 2026-09-15 07:37:49
Exec Total Coverage
Lines: 30 30 100.0%
Functions: 1 1 100.0%
Branches: 14 16 87.5%
Line Branch Exec Source
1 /*********************************************************************************/
2 /* Copyright 2009-2026 Barcelona Supercomputing Center */
3 /* */
4 /* This file is part of the DLB library. */
5 /* */
6 /* DLB is free software: you can redistribute it and/or modify */
7 /* it under the terms of the GNU Lesser General Public License as published by */
8 /* the Free Software Foundation, either version 3 of the License, or */
9 /* (at your option) any later version. */
10 /* */
11 /* DLB is distributed in the hope that it will be useful, */
12 /* but WITHOUT ANY WARRANTY; without even the implied warranty of */
13 /* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the */
14 /* GNU Lesser General Public License for more details. */
15 /* */
16 /* You should have received a copy of the GNU Lesser General Public License */
17 /* along with DLB. If not, see <https://www.gnu.org/licenses/>. */
18 /*********************************************************************************/
19
20 #include "talp/talp_record.h"
21
22 #include "LB_comm/shmem_talp.h"
23 #include "LB_core/node_barrier.h"
24 #include "LB_core/spd.h"
25 #include "apis/dlb_talp.h"
26 #include "support/debug.h"
27 #include "support/mask_utils.h"
28 #include "support/options.h"
29 #include "talp/perf_metrics.h"
30 #include "talp/regions.h"
31 #include "talp/talp_output.h"
32 #include "talp/talp_types.h"
33 #ifdef MPI_LIB
34 #include "mpi/mpi_core.h"
35 #endif
36
37 #include <stddef.h>
38 #include <stdio.h>
39 #include <unistd.h>
40
41
42 /*********************************************************************************/
43 /* TALP Record in serial mode (non-MPI / partial output) */
44 /*********************************************************************************/
45
46 /* For any given monitor, record metrics considering only this (sub-)process */
47 1233 void talp_record_monitor(const subprocess_descriptor_t *spd,
48 const dlb_monitor_t *monitor) {
49
50 1233 talp_info_t *talp_info = spd->talp_info;
51
52 1233 int rank = 0;
53 1233 int num_ranks = 0;
54 1233 int num_nodes = 1;
55 #if MPI_LIB
56 if (talp_info->flags.have_mpi) {
57 rank = _mpi_rank;
58 num_ranks = _mpi_size;
59 num_nodes = _num_nodes;
60 }
61 #endif
62
63
2/2
✓ Branch 0 taken 30 times.
✓ Branch 1 taken 1203 times.
 1233 if (spd->options.talp_summary != SUMMARY_NONE) {
64 30 talp_output_record_process_info();
65 }
66
67
2/2
✓ Branch 0 taken 2 times.
✓ Branch 1 taken 1231 times.
 1233 if (spd->options.talp_summary & SUMMARY_PROCESS) {
68
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2 times.
 2 verbose(VB_TALP, "TALP process summary: recording region %s", monitor->name);
69
70 2 process_record_t process_record = {
71 .rank = rank,
72 2 .pid = spd->id,
73 .monitor = *monitor,
74 };
75
76 2 monitor_data_t *monitor_data = monitor->_data;
77
78 /* Fill hostname and CPU mask strings in process_record */
79 2 gethostname(process_record.hostname, HOST_NAME_MAX);
80 4 snprintf(process_record.cpuset, TALP_OUTPUT_CPUSET_MAX, "%s",
81 2 mu_to_str(&monitor_data->cpu_mask));
82 2 mu_get_quoted_mask(&monitor_data->cpu_mask,
83 process_record.cpuset_quoted, TALP_OUTPUT_CPUSET_MAX);
84
85 /* Add record */
86 2 talp_output_record_process(monitor->name, &process_record, -1);
87 }
88
89
2/2
✓ Branch 0 taken 30 times.
✓ Branch 1 taken 1203 times.
 1233 if (spd->options.talp_summary & SUMMARY_POP_METRICS) {
90
2/2
✓ Branch 0 taken 28 times.
✓ Branch 1 taken 2 times.
 30 if (monitor->elapsed_time > 0) {
91
2/2
✓ Branch 0 taken 4 times.
✓ Branch 1 taken 24 times.
 28 verbose(VB_TALP, "TALP summary: recording region %s", monitor->name);
92
93 pop_base_metrics_t base_metrics;
94 28 perf_metrics__local_monitor_into_base_metrics(&base_metrics, monitor, talp_info->flags);
95
96 dlb_pop_metrics_t pop_metrics;
97 28 perf_metrics__base_to_pop_metrics(monitor->name, &base_metrics, &pop_metrics);
98 28 talp_output_record_pop_metrics(&pop_metrics);
99
100
2/2
✓ Branch 0 taken 22 times.
✓ Branch 1 taken 6 times.
 28 if (monitor == talp_info->monitor) {
101 22 talp_output_record_resources(monitor->num_cpus, mu_get_system_count(),
102 num_nodes, num_ranks, base_metrics.num_gpus);
103 }
104
105 } else {
106
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2 times.
 2 verbose(VB_TALP, "TALP summary: recording empty region %s", monitor->name);
107 2 dlb_pop_metrics_t pop_metrics = {0};
108 2 snprintf(pop_metrics.name, DLB_MONITOR_NAME_MAX, "%s", monitor->name);
109 2 talp_output_record_pop_metrics(&pop_metrics);
110 }
111 }
112 1233 }
113
114
115 /*********************************************************************************/
116 /* TALP Record in MPI mode */
117 /*********************************************************************************/
118
119 #if MPI_LIB
120
121 /* Compute Node summary of all Global Monitors and record data */
122 void talp_record_node_summary(const subprocess_descriptor_t *spd) {
123 node_record_t *node_summary = NULL;
124 size_t node_summary_size = 0;
125
126 /* Perform a barrier so that all processes in the node have arrived at the
127 * MPI_Finalize */
128 node_barrier(spd, NULL);
129
130 /* Node process 0 reduces all global regions from all processes in the node */
131 if (_process_id == 0) {
132 /* Obtain a list of regions associated with the Global Region Name, sorted by PID */
133 int max_procs = mu_get_system_size();
134 talp_region_list_t *region_list = malloc(max_procs * sizeof(talp_region_list_t));
135 int nelems;
136 shmem_talp__get_regionlist(region_list, &nelems, max_procs, region_get_global_name());
137
138 /* Allocate and initialize node summary structure */
139 node_summary_size = sizeof(node_record_t) + sizeof(process_in_node_record_t) * nelems;
140 node_summary = malloc(node_summary_size);
141 *node_summary = (const node_record_t) {
142 .node_id = _node_id,
143 .nelems = nelems,
144 };
145
146 /* Iterate the PID list and gather times of every process */
147 for (int i = 0; i < nelems; ++i) {
148 int64_t mpi_time = region_list[i].mpi_time;
149 int64_t useful_time = region_list[i].useful_time;
150
151 /* Save times in local structure */
152 node_summary->processes[i].pid = region_list[i].pid;
153 node_summary->processes[i].mpi_time = mpi_time;
154 node_summary->processes[i].useful_time = useful_time;
155
156 /* Accumulate total and max values */
157 node_summary->avg_useful_time += useful_time;
158 node_summary->avg_mpi_time += mpi_time;
159 node_summary->max_useful_time = max_int64(useful_time, node_summary->max_useful_time);
160 node_summary->max_mpi_time = max_int64(mpi_time, node_summary->max_mpi_time);
161 }
162 free(region_list);
163
164 /* Compute average values */
165 node_summary->avg_useful_time /= node_summary->nelems;
166 node_summary->avg_mpi_time /= node_summary->nelems;
167 }
168
169 /* Perform a final barrier so that all processes let the _process_id 0 to
170 * gather all the data */
171 node_barrier(spd, NULL);
172
173 /* All main processes from each node send data to rank 0 */
174 if (_process_id == 0) {
175 verbose(VB_TALP, "Node summary: gathering data");
176
177 /* MPI type: int64_t */
178 MPI_Datatype mpi_int64_type = get_mpi_int64_type();
179
180 /* MPI type: pid_t */
181 MPI_Datatype mpi_pid_type;
182 PMPI_Type_match_size(MPI_TYPECLASS_INTEGER, sizeof(pid_t), &mpi_pid_type);
183
184 /* MPI struct type: process_in_node_record_t */
185 MPI_Datatype mpi_process_info_type;
186 {
187 int count = 3;
188 int blocklengths[] = {1, 1, 1};
189 MPI_Aint displacements[] = {
190 offsetof(process_in_node_record_t, pid),
191 offsetof(process_in_node_record_t, mpi_time),
192 offsetof(process_in_node_record_t, useful_time)};
193 MPI_Datatype types[] = {mpi_pid_type, mpi_int64_type, mpi_int64_type};
194 MPI_Datatype tmp_type;
195 PMPI_Type_create_struct(count, blocklengths, displacements, types, &tmp_type);
196 PMPI_Type_create_resized(tmp_type, 0, sizeof(process_in_node_record_t),
197 &mpi_process_info_type);
198 PMPI_Type_commit(&mpi_process_info_type);
199 }
200
201 /* MPI struct type: node_record_t */
202 MPI_Datatype mpi_node_record_type;;
203 {
204 int count = 7;
205 int blocklengths[] = {1, 1, 1, 1, 1, 1, node_summary->nelems};
206 MPI_Aint displacements[] = {
207 offsetof(node_record_t, node_id),
208 offsetof(node_record_t, nelems),
209 offsetof(node_record_t, avg_useful_time),
210 offsetof(node_record_t, avg_mpi_time),
211 offsetof(node_record_t, max_useful_time),
212 offsetof(node_record_t, max_mpi_time),
213 offsetof(node_record_t, processes)};
214 MPI_Datatype types[] = {MPI_INT, MPI_INT, mpi_int64_type, mpi_int64_type,
215 mpi_int64_type, mpi_int64_type, mpi_process_info_type};
216 MPI_Datatype tmp_type;
217 PMPI_Type_create_struct(count, blocklengths, displacements, types, &tmp_type);
218 PMPI_Type_create_resized(tmp_type, 0, node_summary_size, &mpi_node_record_type);
219 PMPI_Type_commit(&mpi_node_record_type);
220 }
221
222 /* Gather data */
223 void *recvbuf = NULL;
224 if (_mpi_rank == 0) {
225 recvbuf = malloc(_num_nodes * node_summary_size);
226 }
227 PMPI_Gather(node_summary, 1, mpi_node_record_type,
228 recvbuf, 1, mpi_node_record_type,
229 0, getInterNodeComm());
230
231 /* Free send buffer and MPI Datatypes */
232 free(node_summary);
233 PMPI_Type_free(&mpi_process_info_type);
234 PMPI_Type_free(&mpi_node_record_type);
235
236 /* Add records */
237 if (_mpi_rank == 0) {
238 for (int node_id = 0; node_id < _num_nodes; ++node_id) {
239 verbose(VB_TALP, "Node summary: recording node %d", node_id);
240 node_record_t *node_record = (node_record_t*)(
241 (unsigned char *)recvbuf + node_summary_size * node_id);
242 ensure( node_id == node_record->node_id, "Node id error in %s", __func__ );
243 talp_output_record_node(node_record);
244 }
245 free(recvbuf);
246 }
247 }
248 }
249
250 /* Gather PROCESS data of a monitor among all ranks and record it in rank 0 */
251 void talp_record_process_summary(const subprocess_descriptor_t *spd,
252 const dlb_monitor_t *monitor) {
253
254 monitor_data_t *monitor_data = monitor->_data;
255
256 /* Internal monitors will not be recorded */
257 if (monitor_data->flags.internal) {
258 return;
259 }
260
261 if (_mpi_rank == 0) {
262 verbose(VB_TALP, "Process summary: gathering region %s", monitor->name);
263 }
264
265 process_record_t process_record_send = {
266 .rank = _mpi_rank,
267 .pid = spd->id,
268 .node_id = _node_id,
269 .monitor = *monitor,
270 };
271
272 /* Invalidate pointers of the copied monitor */
273 process_record_send.monitor.name = NULL;
274 process_record_send.monitor._data = NULL;
275
276 /* Fill hostname and CPU mask strings in process_record_send */
277 gethostname(process_record_send.hostname, HOST_NAME_MAX);
278 snprintf(process_record_send.cpuset, TALP_OUTPUT_CPUSET_MAX, "%s",
279 mu_to_str(&monitor_data->cpu_mask));
280 mu_get_quoted_mask(&monitor_data->cpu_mask,
281 process_record_send.cpuset_quoted, TALP_OUTPUT_CPUSET_MAX);
282
283 /* MPI type: int64_t */
284 MPI_Datatype mpi_int64_type = get_mpi_int64_type();
285
286 /* MPI type: pid_t */
287 MPI_Datatype mpi_pid_type;
288 PMPI_Type_match_size(MPI_TYPECLASS_INTEGER, sizeof(pid_t), &mpi_pid_type);
289
290 /* Note: obviously, it doesn't make sense to send addresses via MPI, but we
291 * are sending the whole dlb_monitor_t, so... Addresses are discarded
292 * either way. */
293
294 /* MPI type: void* */
295 MPI_Datatype address_type;
296 PMPI_Type_match_size(MPI_TYPECLASS_INTEGER, sizeof(void*), &address_type);
297
298 /* MPI struct type: dlb_monitor_t */
299 MPI_Datatype mpi_dlb_monitor_type;
300 {
301 int blocklengths[] = {
302 #define FIELD_BLOCKLENGTH(name, c_type, mpi_type) 1,
303 FOR_DLB_MONITOR_FIELDS(FIELD_BLOCKLENGTH)
304 #undef FIELD_BLOCKLENGTH
305 };
306
307 MPI_Aint displacements[] = {
308 #define FIELD_DISPLACEMENT(name, c_type, mpi_type) offsetof(dlb_monitor_t, name),
309 FOR_DLB_MONITOR_FIELDS(FIELD_DISPLACEMENT)
310 #undef FIELD_DISPLACEMENT
311 };
312
313 MPI_Datatype types[] = {
314 #define FIELD_MPI_TYPE(name, c_type, mpi_type) mpi_type,
315 FOR_DLB_MONITOR_FIELDS(FIELD_MPI_TYPE)
316 #undef FIELD_MPI_TYPE
317 };
318
319 enum {count = sizeof(blocklengths) / sizeof(blocklengths[0])};
320
321 MPI_Datatype tmp_type;
322 PMPI_Type_create_struct(count, blocklengths, displacements, types, &tmp_type);
323 PMPI_Type_create_resized(tmp_type, 0, sizeof(dlb_monitor_t), &mpi_dlb_monitor_type);
324 PMPI_Type_commit(&mpi_dlb_monitor_type);
325 }
326
327 /* MPI struct type: process_record_t */
328 MPI_Datatype mpi_process_record_type;
329 {
330 int count = 7;
331 int blocklengths[] = {1, 1, 1, HOST_NAME_MAX,
332 TALP_OUTPUT_CPUSET_MAX, TALP_OUTPUT_CPUSET_MAX, 1};
333 MPI_Aint displacements[] = {
334 offsetof(process_record_t, rank),
335 offsetof(process_record_t, pid),
336 offsetof(process_record_t, node_id),
337 offsetof(process_record_t, hostname),
338 offsetof(process_record_t, cpuset),
339 offsetof(process_record_t, cpuset_quoted),
340 offsetof(process_record_t, monitor)};
341 MPI_Datatype types[] = {MPI_INT, mpi_pid_type, MPI_INT, MPI_CHAR, MPI_CHAR,
342 MPI_CHAR, mpi_dlb_monitor_type};
343 MPI_Datatype tmp_type;
344 PMPI_Type_create_struct(count, blocklengths, displacements, types, &tmp_type);
345 PMPI_Type_create_resized(tmp_type, 0, sizeof(process_record_t),
346 &mpi_process_record_type);
347 PMPI_Type_commit(&mpi_process_record_type);
348 }
349
350 /* Gather data */
351 process_record_t *recvbuf = NULL;
352 if (_mpi_rank == 0) {
353 recvbuf = malloc(_mpi_size * sizeof(process_record_t));
354 }
355 PMPI_Gather(&process_record_send, 1, mpi_process_record_type,
356 recvbuf, 1, mpi_process_record_type,
357 0, getWorldComm());
358
359 /* Add records */
360 if (_mpi_rank == 0) {
361 for (int rank = 0; rank < _mpi_size; ++rank) {
362 verbose(VB_TALP, "Process summary: recording region %s on rank %d",
363 monitor->name, rank);
364 talp_output_record_process(monitor->name, &recvbuf[rank], _mpi_size);
365 }
366 free(recvbuf);
367 }
368
369 /* Free MPI types */
370 PMPI_Type_free(&mpi_dlb_monitor_type);
371 PMPI_Type_free(&mpi_process_record_type);
372 }
373
374 /* Gather POP METRICS data of a monitor among all ranks and record it in rank 0 */
375 void talp_record_pop_summary(const subprocess_descriptor_t *spd,
376 const dlb_monitor_t *monitor) {
377
378 /* Internal monitors will not be recorded */
379 if (((monitor_data_t*)monitor->_data)->flags.internal) {
380 return;
381 }
382
383 if (_mpi_rank == 0) {
384 verbose(VB_TALP, "TALP summary: gathering region %s", monitor->name);
385 }
386
387 talp_info_t *talp_info = spd->talp_info;
388
389 /* Reduce monitor among all MPI ranks into MPI rank 0 */
390 pop_base_metrics_t base_metrics;
391 perf_metrics__reduce_monitor_into_base_metrics(&base_metrics, monitor, false);
392
393 if (_mpi_rank == 0) {
394 if (base_metrics.elapsed_time > 0) {
395
396 /* Only the global region records the resources */
397 if (monitor == talp_info->monitor) {
398 talp_output_record_resources(base_metrics.num_cpus,
399 base_metrics.num_available_cpus,
400 base_metrics.num_nodes, base_metrics.num_mpi_ranks,
401 base_metrics.num_gpus);
402 }
403
404 /* Construct pop_metrics out of base metrics */
405 dlb_pop_metrics_t pop_metrics;
406 perf_metrics__base_to_pop_metrics(monitor->name, &base_metrics, &pop_metrics);
407
408 /* Record */
409 verbose(VB_TALP, "TALP summary: recording region %s", monitor->name);
410 talp_output_record_pop_metrics(&pop_metrics);
411
412 } else {
413 /* Record empty */
414 verbose(VB_TALP, "TALP summary: recording empty region %s", monitor->name);
415 dlb_pop_metrics_t pop_metrics = {0};
416 snprintf(pop_metrics.name, DLB_MONITOR_NAME_MAX, "%s", monitor->name);
417 talp_output_record_pop_metrics(&pop_metrics);
418 }
419 }
420 }
421
422 #endif /* MPI_LIB */
423