| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | /*********************************************************************************/ | ||
| 2 | /* Copyright 2009-2025 Barcelona Supercomputing Center */ | ||
| 3 | /* */ | ||
| 4 | /* This file is part of the DLB library. */ | ||
| 5 | /* */ | ||
| 6 | /* DLB is free software: you can redistribute it and/or modify */ | ||
| 7 | /* it under the terms of the GNU Lesser General Public License as published by */ | ||
| 8 | /* the Free Software Foundation, either version 3 of the License, or */ | ||
| 9 | /* (at your option) any later version. */ | ||
| 10 | /* */ | ||
| 11 | /* DLB is distributed in the hope that it will be useful, */ | ||
| 12 | /* but WITHOUT ANY WARRANTY; without even the implied warranty of */ | ||
| 13 | /* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the */ | ||
| 14 | /* GNU Lesser General Public License for more details. */ | ||
| 15 | /* */ | ||
| 16 | /* You should have received a copy of the GNU Lesser General Public License */ | ||
| 17 | /* along with DLB. If not, see <https://www.gnu.org/licenses/>. */ | ||
| 18 | /*********************************************************************************/ | ||
| 19 | |||
| 20 | #include "talp/perf_metrics.h" | ||
| 21 | |||
| 22 | #include "LB_core/spd.h" | ||
| 23 | #include "apis/dlb_talp.h" | ||
| 24 | #include "support/debug.h" | ||
| 25 | #include "support/mask_utils.h" | ||
| 26 | #include "support/gpu_mask_utils.h" | ||
| 27 | #include "talp/talp_gpu.h" | ||
| 28 | #ifdef MPI_LIB | ||
| 29 | #include "mpi/mpi_core.h" | ||
| 30 | #endif | ||
| 31 | |||
| 32 | #include <stddef.h> | ||
| 33 | #include <stdio.h> | ||
| 34 | #include <string.h> | ||
| 35 | |||
| 36 | /*********************************************************************************/ | ||
| 37 | /* POP metrics - pure MPI model */ | ||
| 38 | /*********************************************************************************/ | ||
| 39 | |||
| 40 | /* Compute POP metrics for the MPI model | ||
| 41 | * (This funtion is actually not used anywhere) */ | ||
| 42 | static inline void perf_metrics__compute_mpi_model( | ||
| 43 | perf_metrics_mpi_t *metrics, | ||
| 44 | int num_cpus, | ||
| 45 | int num_nodes, | ||
| 46 | int64_t elapsed_time, | ||
| 47 | int64_t elapsed_useful, | ||
| 48 | int64_t app_sum_useful, | ||
| 49 | int64_t node_sum_useful) __attribute__((unused)); | ||
| 50 | static inline void perf_metrics__compute_mpi_model( | ||
| 51 | perf_metrics_mpi_t *metrics, | ||
| 52 | int num_cpus, | ||
| 53 | int num_nodes, | ||
| 54 | int64_t elapsed_time, | ||
| 55 | int64_t elapsed_useful, | ||
| 56 | int64_t app_sum_useful, | ||
| 57 | int64_t node_sum_useful) { | ||
| 58 | |||
| 59 | if (elapsed_time > 0) { | ||
| 60 | *metrics = (const perf_metrics_mpi_t) { | ||
| 61 | .parallel_efficiency = (float)app_sum_useful / (elapsed_time * num_cpus), | ||
| 62 | .communication_efficiency = (float)elapsed_useful / elapsed_time, | ||
| 63 | .load_balance = (float)app_sum_useful / (elapsed_useful * num_cpus), | ||
| 64 | .lb_in = (float)(node_sum_useful * num_nodes) / (elapsed_useful * num_cpus), | ||
| 65 | .lb_out = (float)app_sum_useful / (node_sum_useful * num_nodes), | ||
| 66 | }; | ||
| 67 | } else { | ||
| 68 | *metrics = (const perf_metrics_mpi_t) {}; | ||
| 69 | } | ||
| 70 | } | ||
| 71 | |||
| 72 | /* Compute POP metrics for the MPI model, but with some inferred values: | ||
| 73 | * (Only useful for node metrics) */ | ||
| 74 | 18 | void perf_metrics__infer_mpi_model( | |
| 75 | perf_metrics_mpi_t *metrics, | ||
| 76 | int processes_per_node, | ||
| 77 | int64_t node_sum_useful, | ||
| 78 | int64_t node_sum_mpi, | ||
| 79 | int64_t max_useful_time) { | ||
| 80 | |||
| 81 | 18 | int64_t elapsed_time = (node_sum_useful + node_sum_mpi) / processes_per_node; | |
| 82 |
1/2✓ Branch 0 taken 18 times.
✗ Branch 1 not taken.
|
18 | if (elapsed_time > 0) { |
| 83 | 18 | *metrics = (const perf_metrics_mpi_t) { | |
| 84 | 18 | .parallel_efficiency = (float)node_sum_useful / (node_sum_useful + node_sum_mpi), | |
| 85 | 18 | .communication_efficiency = (float)max_useful_time / elapsed_time, | |
| 86 | 18 | .load_balance = ((float)node_sum_useful / processes_per_node) / max_useful_time, | |
| 87 | }; | ||
| 88 | } else { | ||
| 89 | ✗ | *metrics = (const perf_metrics_mpi_t) {}; | |
| 90 | } | ||
| 91 | 18 | } | |
| 92 | |||
| 93 | |||
| 94 | /*********************************************************************************/ | ||
| 95 | /* POP metrics - hybrid MPI + OpenMP model */ | ||
| 96 | /*********************************************************************************/ | ||
| 97 | |||
| 98 | /* Computed efficiency metrics for the POP hybrid model */ | ||
| 99 | typedef struct perf_metrics_hybrid_t { | ||
| 100 | float parallel_efficiency; | ||
| 101 | float mpi_parallel_efficiency; | ||
| 102 | float mpi_communication_efficiency; | ||
| 103 | float mpi_load_balance; | ||
| 104 | float mpi_load_balance_in; | ||
| 105 | float mpi_load_balance_out; | ||
| 106 | float omp_parallel_efficiency; | ||
| 107 | float omp_load_balance; | ||
| 108 | float omp_scheduling_efficiency; | ||
| 109 | float omp_coverage_efficiency; | ||
| 110 | float device_offload_efficiency; | ||
| 111 | float gpu_parallel_efficiency; | ||
| 112 | float gpu_load_balance; | ||
| 113 | float gpu_communication_efficiency; | ||
| 114 | float gpu_orchestration_efficiency; | ||
| 115 | } perf_metrics_hybrid_t; | ||
| 116 | |||
| 117 | |||
| 118 | /* Compute POP metrics for the hybrid MPI + OpenMP model | ||
| 119 | * (Ver. 1: All metrics are multiplicative, but some of them are > 1) */ | ||
| 120 | ✗ | static inline void perf_metrics__compute_hybrid_model_v1( | |
| 121 | perf_metrics_hybrid_t *metrics, | ||
| 122 | const pop_base_metrics_t *base_metrics) { | ||
| 123 | |||
| 124 | ✗ | int num_cpus = base_metrics->num_cpus; | |
| 125 | ✗ | int num_gpus = base_metrics->num_gpus; | |
| 126 | ✗ | int64_t elapsed_time = base_metrics->elapsed_time; | |
| 127 | ✗ | int64_t useful_time = base_metrics->useful_time; | |
| 128 | ✗ | int64_t mpi_time = base_metrics->mpi_time; | |
| 129 | ✗ | int64_t omp_load_imbalance_time = base_metrics->omp_load_imbalance_time; | |
| 130 | ✗ | int64_t omp_scheduling_time = base_metrics->omp_scheduling_time; | |
| 131 | ✗ | int64_t omp_outside_parallel_time = base_metrics->omp_outside_parallel_time; | |
| 132 | ✗ | int64_t gpu_runtime_time = base_metrics->gpu_runtime_time; | |
| 133 | ✗ | double min_mpi_normd_proc = base_metrics->min_mpi_normd_proc; | |
| 134 | ✗ | double min_mpi_normd_node = base_metrics->min_mpi_normd_node; | |
| 135 | ✗ | int64_t gpu_useful_time = base_metrics->gpu_useful_time; | |
| 136 | ✗ | int64_t max_gpu_useful_time = base_metrics->max_gpu_useful_time; | |
| 137 | ✗ | int64_t max_gpu_active_time = base_metrics->max_gpu_active_time; | |
| 138 | |||
| 139 | /* Active is the union of all times (while CPU is not disabled) */ | ||
| 140 | ✗ | int64_t sum_active = useful_time + mpi_time + omp_load_imbalance_time + | |
| 141 | ✗ | omp_scheduling_time + omp_outside_parallel_time + gpu_runtime_time; | |
| 142 | |||
| 143 | /* Equivalent to all CPU time if OMP was not present */ | ||
| 144 | ✗ | int64_t sum_active_non_omp = useful_time + mpi_time + gpu_runtime_time; | |
| 145 | |||
| 146 | /* Equivalent to all CPU time if GPU was not present */ | ||
| 147 | ✗ | int64_t sum_active_non_gpu = sum_active - gpu_runtime_time; | |
| 148 | |||
| 149 | /* MPI time normalized at application level */ | ||
| 150 | ✗ | double mpi_normd_app = (double)mpi_time / num_cpus; | |
| 151 | |||
| 152 | /* Non-MPI time normalized at application level */ | ||
| 153 | ✗ | double non_mpi_normd_app = elapsed_time - mpi_normd_app; | |
| 154 | |||
| 155 | /* Max value of non-MPI times normalized at process level */ | ||
| 156 | ✗ | double max_non_mpi_normd_proc = elapsed_time - min_mpi_normd_proc; | |
| 157 | |||
| 158 | /* Max value of non-MPI times normalized at node level */ | ||
| 159 | ✗ | double max_non_mpi_normd_node = elapsed_time - min_mpi_normd_node; | |
| 160 | |||
| 161 | /* All Device time */ | ||
| 162 | ✗ | int64_t sum_device_time = elapsed_time * num_gpus; | |
| 163 | |||
| 164 | /* Compute output metrics */ | ||
| 165 | ✗ | *metrics = (const perf_metrics_hybrid_t) { | |
| 166 | ✗ | .parallel_efficiency = (float)useful_time / sum_active, | |
| 167 | ✗ | .mpi_parallel_efficiency = (float)useful_time / (useful_time + mpi_time), | |
| 168 | .mpi_communication_efficiency = | ||
| 169 | ✗ | max_non_mpi_normd_proc / (non_mpi_normd_app + mpi_normd_app), | |
| 170 | ✗ | .mpi_load_balance = non_mpi_normd_app / max_non_mpi_normd_proc, | |
| 171 | ✗ | .mpi_load_balance_in = max_non_mpi_normd_node / max_non_mpi_normd_proc, | |
| 172 | ✗ | .mpi_load_balance_out = non_mpi_normd_app / max_non_mpi_normd_node, | |
| 173 | ✗ | .omp_parallel_efficiency = (float)sum_active_non_omp / sum_active, | |
| 174 | ✗ | .omp_load_balance = (float)(sum_active_non_omp + omp_outside_parallel_time) | |
| 175 | ✗ | / (sum_active_non_omp + omp_outside_parallel_time + omp_load_imbalance_time), | |
| 176 | .omp_scheduling_efficiency = | ||
| 177 | ✗ | (float)(sum_active_non_omp + omp_outside_parallel_time + omp_load_imbalance_time) | |
| 178 | ✗ | / (sum_active_non_omp + omp_outside_parallel_time + omp_load_imbalance_time | |
| 179 | ✗ | + omp_scheduling_time), | |
| 180 | ✗ | .omp_coverage_efficiency = (float)sum_active_non_omp | |
| 181 | ✗ | / (sum_active_non_omp + omp_outside_parallel_time), | |
| 182 | ✗ | .device_offload_efficiency = (float)sum_active_non_gpu / sum_active, | |
| 183 | .gpu_parallel_efficiency = sum_device_time == 0 ? 0 | ||
| 184 | ✗ | : (float)gpu_useful_time / sum_device_time, | |
| 185 | ✗ | .gpu_load_balance = max_gpu_useful_time * num_gpus == 0 ? 0 | |
| 186 | ✗ | : (float)gpu_useful_time / (max_gpu_useful_time * num_gpus), | |
| 187 | .gpu_communication_efficiency = max_gpu_active_time == 0 ? 0 | ||
| 188 | ✗ | : (float)max_gpu_useful_time / max_gpu_active_time, | |
| 189 | .gpu_orchestration_efficiency = sum_device_time == 0 ? 0 | ||
| 190 | ✗ | : (float)max_gpu_active_time / elapsed_time, | |
| 191 | }; | ||
| 192 | } | ||
| 193 | |||
| 194 | /* Compute POP metrics for the hybrid MPI + OpenMP model (Ver. 2: PE != MPE * OPE) */ | ||
| 195 | 29 | static inline void perf_metrics__compute_hybrid_model_v2( | |
| 196 | perf_metrics_hybrid_t *metrics, | ||
| 197 | const pop_base_metrics_t *base_metrics) { | ||
| 198 | |||
| 199 | 29 | int num_cpus = base_metrics->num_cpus; | |
| 200 | 29 | int num_gpus = base_metrics->num_gpus; | |
| 201 | 29 | int64_t elapsed_time = base_metrics->elapsed_time; | |
| 202 | 29 | int64_t useful_time = base_metrics->useful_time; | |
| 203 | 29 | int64_t mpi_time = base_metrics->mpi_time; | |
| 204 | 29 | int64_t mpi_worker_idle_time = base_metrics->mpi_worker_idle_time; | |
| 205 | 29 | int64_t omp_load_imbalance_time = base_metrics->omp_load_imbalance_time; | |
| 206 | 29 | int64_t omp_scheduling_time = base_metrics->omp_scheduling_time; | |
| 207 | 29 | int64_t omp_outside_parallel_time = base_metrics->omp_outside_parallel_time; | |
| 208 | 29 | int64_t gpu_runtime_time = base_metrics->gpu_runtime_time; | |
| 209 | 29 | double min_mpi_normd_proc = base_metrics->min_mpi_normd_proc; | |
| 210 | 29 | double min_mpi_normd_node = base_metrics->min_mpi_normd_node; | |
| 211 | 29 | int64_t gpu_useful_time = base_metrics->gpu_useful_time; | |
| 212 | 29 | int64_t max_gpu_useful_time = base_metrics->max_gpu_useful_time; | |
| 213 | 29 | int64_t max_gpu_active_time = base_metrics->max_gpu_active_time; | |
| 214 | |||
| 215 | /* Active is the union of all times (CPU not disabled) */ | ||
| 216 | 29 | int64_t sum_active = useful_time + mpi_time + omp_load_imbalance_time + | |
| 217 | 29 | omp_scheduling_time + omp_outside_parallel_time + gpu_runtime_time; | |
| 218 | |||
| 219 | /* Equivalent to all CPU time if OMP was not present */ | ||
| 220 | 29 | int64_t sum_active_non_omp = useful_time + mpi_time + gpu_runtime_time; | |
| 221 | |||
| 222 | /* CPU time of OpenMP not useful */ | ||
| 223 | 29 | int64_t sum_omp_not_useful = omp_load_imbalance_time + omp_scheduling_time + | |
| 224 | omp_outside_parallel_time; | ||
| 225 | |||
| 226 | /* MPI time normalized at application level */ | ||
| 227 | 29 | double mpi_normd_app = (double)(mpi_time + mpi_worker_idle_time) / num_cpus; | |
| 228 | |||
| 229 | /* Non-MPI time normalized at application level */ | ||
| 230 | 29 | double non_mpi_normd_app = elapsed_time - mpi_normd_app; | |
| 231 | |||
| 232 | /* Max value of non-MPI times normalized at process level */ | ||
| 233 | 29 | double max_non_mpi_normd_proc = elapsed_time - min_mpi_normd_proc; | |
| 234 | |||
| 235 | /* Max value of non-MPI times normalized at node level */ | ||
| 236 | 29 | double max_non_mpi_normd_node = elapsed_time - min_mpi_normd_node; | |
| 237 | |||
| 238 | /* All Device time */ | ||
| 239 | 29 | int64_t sum_device_time = elapsed_time * num_gpus; | |
| 240 | |||
| 241 | /* Compute output metrics */ | ||
| 242 | 29 | *metrics = (const perf_metrics_hybrid_t) { | |
| 243 | 29 | .parallel_efficiency = (float)useful_time / sum_active, | |
| 244 | 29 | .mpi_parallel_efficiency = non_mpi_normd_app / elapsed_time, | |
| 245 | 29 | .mpi_communication_efficiency = max_non_mpi_normd_proc / elapsed_time, | |
| 246 | 29 | .mpi_load_balance = non_mpi_normd_app / max_non_mpi_normd_proc, | |
| 247 | 29 | .mpi_load_balance_in = max_non_mpi_normd_node / max_non_mpi_normd_proc, | |
| 248 | 29 | .mpi_load_balance_out = non_mpi_normd_app / max_non_mpi_normd_node, | |
| 249 | 29 | .omp_parallel_efficiency = (float)sum_active_non_omp / sum_active, | |
| 250 | 29 | .omp_load_balance = (float)(sum_active_non_omp + omp_outside_parallel_time) | |
| 251 | 29 | / (sum_active_non_omp + omp_outside_parallel_time + omp_load_imbalance_time), | |
| 252 | .omp_scheduling_efficiency = | ||
| 253 | 29 | (float)(sum_active_non_omp + omp_outside_parallel_time + omp_load_imbalance_time) | |
| 254 | 29 | / (sum_active_non_omp + omp_outside_parallel_time + omp_load_imbalance_time | |
| 255 | 29 | + omp_scheduling_time), | |
| 256 | 29 | .omp_coverage_efficiency = (float)sum_active_non_omp | |
| 257 | 29 | / (sum_active_non_omp + omp_outside_parallel_time), | |
| 258 | 29 | .device_offload_efficiency = (float)(useful_time + sum_omp_not_useful) | |
| 259 | 29 | / (useful_time + sum_omp_not_useful + gpu_runtime_time), | |
| 260 | .gpu_parallel_efficiency = sum_device_time == 0 ? 0 | ||
| 261 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 29 times.
|
29 | : (float)gpu_useful_time / sum_device_time, |
| 262 | 29 | .gpu_load_balance = max_gpu_useful_time * num_gpus == 0 ? 0 | |
| 263 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 29 times.
|
29 | : (float)gpu_useful_time / (max_gpu_useful_time * num_gpus), |
| 264 | .gpu_communication_efficiency = max_gpu_active_time == 0 ? 0 | ||
| 265 |
2/2✓ Branch 0 taken 1 times.
✓ Branch 1 taken 28 times.
|
29 | : (float)max_gpu_useful_time / max_gpu_active_time, |
| 266 | .gpu_orchestration_efficiency = sum_device_time == 0 ? 0 | ||
| 267 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 29 times.
|
29 | : (float)max_gpu_active_time / elapsed_time, |
| 268 | }; | ||
| 269 | 29 | } | |
| 270 | |||
| 271 | #ifdef MPI_LIB | ||
| 272 | |||
| 273 | /* The following node and app reductions are needed to compute POP metrics: */ | ||
| 274 | |||
| 275 | /*** Node reduction ***/ | ||
| 276 | |||
| 277 | /* Data type to reduce among processes in node */ | ||
| 278 | typedef struct node_reduction_t { | ||
| 279 | bool node_used; | ||
| 280 | int cpus_node; | ||
| 281 | int64_t mpi_time; | ||
| 282 | int64_t mpi_worker_idle_time; | ||
| 283 | uint64_t gpu_ids[MAX_NODE_GPUS]; | ||
| 284 | int64_t gpu_useful[MAX_NODE_GPUS]; | ||
| 285 | int64_t gpu_communication[MAX_NODE_GPUS]; | ||
| 286 | int num_gpu_ids; | ||
| 287 | } node_reduction_t; | ||
| 288 | |||
| 289 | static void merge_gpu_metrics(node_reduction_t *inout, const node_reduction_t *in) { | ||
| 290 | |||
| 291 | for (int i = 0; i < in->num_gpu_ids; ++i) { | ||
| 292 | uint64_t id = in->gpu_ids[i]; | ||
| 293 | |||
| 294 | int idx = -1; | ||
| 295 | for (int j = 0; j < inout->num_gpu_ids; ++j) { | ||
| 296 | if (inout->gpu_ids[j] == id) { idx = j; break; } | ||
| 297 | } | ||
| 298 | |||
| 299 | if (idx < 0) { | ||
| 300 | if (inout->num_gpu_ids >= MAX_NODE_GPUS) { | ||
| 301 | warning("MAX_NODE_GPUS exceeded, gpu metrics will be inaccurate"); | ||
| 302 | continue; | ||
| 303 | } | ||
| 304 | idx = inout->num_gpu_ids++; | ||
| 305 | inout->gpu_ids[idx] = id; | ||
| 306 | inout->gpu_useful[idx] = 0; | ||
| 307 | inout->gpu_communication[idx] = 0; | ||
| 308 | } | ||
| 309 | |||
| 310 | inout->gpu_useful[idx] = max_int64(inout->gpu_useful[idx], in->gpu_useful[i]); | ||
| 311 | inout->gpu_communication[idx] = max_int64( | ||
| 312 | inout->gpu_communication[idx], in->gpu_communication[i]); | ||
| 313 | } | ||
| 314 | } | ||
| 315 | |||
| 316 | /* Function called in the MPI node reduction */ | ||
| 317 | static void mpi_node_reduction_fn(void *invec, void *inoutvec, int *len, | ||
| 318 | MPI_Datatype *datatype) { | ||
| 319 | |||
| 320 | const node_reduction_t *in = invec; | ||
| 321 | node_reduction_t *inout = inoutvec; | ||
| 322 | |||
| 323 | int _len = *len; | ||
| 324 | for (int i = 0; i < _len; ++i) { | ||
| 325 | if (in[i].node_used) { | ||
| 326 | inout[i].node_used = true; | ||
| 327 | inout[i].cpus_node += in[i].cpus_node; | ||
| 328 | inout[i].mpi_time += in[i].mpi_time; | ||
| 329 | inout[i].mpi_worker_idle_time += in[i].mpi_worker_idle_time; | ||
| 330 | merge_gpu_metrics(&inout[i], &in[i]); | ||
| 331 | } | ||
| 332 | } | ||
| 333 | } | ||
| 334 | |||
| 335 | /* Function to perform the reduction at node level */ | ||
| 336 | static void reduce_pop_metrics_node_reduction(node_reduction_t *node_reduction, | ||
| 337 | const dlb_monitor_t *monitor) { | ||
| 338 | |||
| 339 | node_reduction_t node_reduction_send = { | ||
| 340 | .node_used = monitor->num_measurements > 0, | ||
| 341 | .cpus_node = monitor->num_cpus, | ||
| 342 | .mpi_time = monitor->mpi_time, | ||
| 343 | .mpi_worker_idle_time = monitor->mpi_worker_idle_time, | ||
| 344 | }; | ||
| 345 | |||
| 346 | /* Construct a contiguous output array of the GPUs times and their unique id */ | ||
| 347 | int num_gpus = 0; | ||
| 348 | const monitor_data_t *monitor_data = monitor->_data; | ||
| 349 | uint64_t mask = monitor_data->gpu_mask; | ||
| 350 | while(mask) { | ||
| 351 | int gpu = gm_ctz(mask); | ||
| 352 | if (unlikely(gpu < 0)) { | ||
| 353 | warning("Unmatched gpu %d found in mask 0x%" PRIx64 "." | ||
| 354 | " Please report bug", gpu, mask); | ||
| 355 | continue; | ||
| 356 | } | ||
| 357 | |||
| 358 | node_reduction_send.gpu_ids[num_gpus] = talp_gpu_local_to_unique_id((uint32_t)gpu); | ||
| 359 | node_reduction_send.gpu_useful[num_gpus] = monitor_data->gpu_timers[gpu].useful; | ||
| 360 | node_reduction_send.gpu_communication[num_gpus] = | ||
| 361 | monitor_data->gpu_timers[gpu].communication; | ||
| 362 | |||
| 363 | mask = gm_clear_lsb(mask); | ||
| 364 | ++num_gpus; | ||
| 365 | } | ||
| 366 | node_reduction_send.num_gpu_ids = num_gpus; | ||
| 367 | |||
| 368 | /* MPI types: int64_t and uint64_t */ | ||
| 369 | MPI_Datatype mpi_int64_type = get_mpi_int64_type(); | ||
| 370 | MPI_Datatype mpi_uint64_type = get_mpi_uint64_type(); | ||
| 371 | |||
| 372 | /* MPI struct type: node_reduction_t */ | ||
| 373 | MPI_Datatype mpi_node_reduction_type; | ||
| 374 | { | ||
| 375 | int blocklengths[] = {1, 1, 1, 1, MAX_NODE_GPUS, MAX_NODE_GPUS, MAX_NODE_GPUS, 1}; | ||
| 376 | MPI_Aint displacements[] = { | ||
| 377 | offsetof(node_reduction_t, node_used), | ||
| 378 | offsetof(node_reduction_t, cpus_node), | ||
| 379 | offsetof(node_reduction_t, mpi_time), | ||
| 380 | offsetof(node_reduction_t, mpi_worker_idle_time), | ||
| 381 | offsetof(node_reduction_t, gpu_ids), | ||
| 382 | offsetof(node_reduction_t, gpu_useful), | ||
| 383 | offsetof(node_reduction_t, gpu_communication), | ||
| 384 | offsetof(node_reduction_t, num_gpu_ids)}; | ||
| 385 | MPI_Datatype types[] = {MPI_C_BOOL, MPI_INT, mpi_int64_type, mpi_int64_type, | ||
| 386 | mpi_uint64_type, mpi_int64_type, mpi_int64_type, MPI_INT}; | ||
| 387 | |||
| 388 | enum {count = sizeof(blocklengths) / sizeof(blocklengths[0])}; | ||
| 389 | static_ensure(sizeof(displacements)/sizeof(displacements[0]) == count, | ||
| 390 | "displacements size mismatch"); | ||
| 391 | static_ensure(sizeof(types)/sizeof(types[0]) == count, | ||
| 392 | "types size mismatch"); | ||
| 393 | |||
| 394 | MPI_Datatype tmp_type; | ||
| 395 | PMPI_Type_create_struct(count, blocklengths, displacements, types, &tmp_type); | ||
| 396 | PMPI_Type_create_resized(tmp_type, 0, sizeof(node_reduction_t), | ||
| 397 | &mpi_node_reduction_type); | ||
| 398 | PMPI_Type_commit(&mpi_node_reduction_type); | ||
| 399 | |||
| 400 | } | ||
| 401 | |||
| 402 | /* Define MPI operation (the GPUs array merging makes this op a non-commutative) */ | ||
| 403 | MPI_Op node_reduction_op; | ||
| 404 | PMPI_Op_create(mpi_node_reduction_fn, false, &node_reduction_op); | ||
| 405 | |||
| 406 | /* MPI reduction */ | ||
| 407 | PMPI_Reduce(&node_reduction_send, node_reduction, 1, | ||
| 408 | mpi_node_reduction_type, node_reduction_op, | ||
| 409 | 0, getNodeComm()); | ||
| 410 | |||
| 411 | /* Check that we have not summed more CPUs that the node count */ | ||
| 412 | int system_count = mu_get_system_count(); | ||
| 413 | if (node_reduction->cpus_node > system_count) { | ||
| 414 | verbose(VB_TALP, "Warning: Number of CPUs after node reduction (%d) is greater" | ||
| 415 | " than the node CPU count (%d). Reverting value.", | ||
| 416 | node_reduction->cpus_node, system_count); | ||
| 417 | node_reduction->cpus_node = system_count; | ||
| 418 | } | ||
| 419 | |||
| 420 | /* Free MPI types */ | ||
| 421 | PMPI_Type_free(&mpi_node_reduction_type); | ||
| 422 | PMPI_Op_free(&node_reduction_op); | ||
| 423 | } | ||
| 424 | |||
| 425 | /** App reduction ***/ | ||
| 426 | |||
| 427 | /* Function called in the MPI app reduction */ | ||
| 428 | static void mpi_reduction_fn(void *invec, void *inoutvec, int *len, | ||
| 429 | MPI_Datatype *datatype) { | ||
| 430 | |||
| 431 | const pop_base_metrics_t *in = invec; | ||
| 432 | pop_base_metrics_t *inout = inoutvec; | ||
| 433 | |||
| 434 | int _len = *len; | ||
| 435 | for (int i = 0; i < _len; ++i) { | ||
| 436 | /* Resources */ | ||
| 437 | inout[i].num_cpus += in[i].num_cpus; | ||
| 438 | inout[i].num_available_cpus += in[i].num_available_cpus; | ||
| 439 | inout[i].num_omp_threads += in[i].num_omp_threads; | ||
| 440 | inout[i].num_mpi_ranks += in[i].num_mpi_ranks; | ||
| 441 | inout[i].num_nodes += in[i].num_nodes; | ||
| 442 | inout[i].avg_cpus += in[i].avg_cpus; | ||
| 443 | inout[i].num_gpus += in[i].num_gpus; | ||
| 444 | /* Hardware Counters */ | ||
| 445 | inout[i].cycles += in[i].cycles; | ||
| 446 | inout[i].instructions += in[i].instructions; | ||
| 447 | /* Statistics */ | ||
| 448 | inout[i].num_measurements += in[i].num_measurements; | ||
| 449 | inout[i].num_mpi_calls += in[i].num_mpi_calls; | ||
| 450 | inout[i].num_omp_parallels += in[i].num_omp_parallels; | ||
| 451 | inout[i].num_omp_tasks += in[i].num_omp_tasks; | ||
| 452 | inout[i].num_gpu_runtime_calls += in[i].num_gpu_runtime_calls; | ||
| 453 | /* Host Times */ | ||
| 454 | inout[i].elapsed_time = max_int64(inout[i].elapsed_time, in[i].elapsed_time); | ||
| 455 | inout[i].useful_time += in[i].useful_time; | ||
| 456 | inout[i].mpi_time += in[i].mpi_time; | ||
| 457 | inout[i].mpi_worker_idle_time += in[i].mpi_worker_idle_time; | ||
| 458 | inout[i].omp_load_imbalance_time += in[i].omp_load_imbalance_time; | ||
| 459 | inout[i].omp_scheduling_time += in[i].omp_scheduling_time; | ||
| 460 | inout[i].omp_outside_parallel_time += in[i].omp_outside_parallel_time; | ||
| 461 | inout[i].gpu_runtime_time += in[i].gpu_runtime_time; | ||
| 462 | |||
| 463 | /* Host Normalized Times */ | ||
| 464 | inout[i].min_mpi_normd_proc = | ||
| 465 | min_double_non_zero(inout[i].min_mpi_normd_proc, in[i].min_mpi_normd_proc); | ||
| 466 | inout[i].min_mpi_normd_node = | ||
| 467 | min_double_non_zero(inout[i].min_mpi_normd_node, in[i].min_mpi_normd_node); | ||
| 468 | |||
| 469 | /* Device Times */ | ||
| 470 | inout[i].gpu_useful_time += in[i].gpu_useful_time; | ||
| 471 | inout[i].gpu_communication_time += in[i].gpu_communication_time; | ||
| 472 | inout[i].gpu_inactive_time += in[i].gpu_inactive_time; | ||
| 473 | |||
| 474 | /* Device Max Times */ | ||
| 475 | inout[i].max_gpu_useful_time = | ||
| 476 | max_int64(inout[i].max_gpu_useful_time, in[i].max_gpu_useful_time); | ||
| 477 | inout[i].max_gpu_active_time = | ||
| 478 | max_int64(inout[i].max_gpu_active_time, in[i].max_gpu_active_time); | ||
| 479 | } | ||
| 480 | } | ||
| 481 | |||
| 482 | /* Function to perform the reduction at application level */ | ||
| 483 | static void reduce_pop_metrics_app_reduction(pop_base_metrics_t *base_metrics, | ||
| 484 | const node_reduction_t *node_reduction, const dlb_monitor_t *monitor, | ||
| 485 | bool all_to_all) { | ||
| 486 | |||
| 487 | double min_mpi_normd_proc = monitor->num_cpus == 0 ? 0.0 | ||
| 488 | : (double)(monitor->mpi_time + monitor->mpi_worker_idle_time) / monitor->num_cpus; | ||
| 489 | double min_mpi_normd_node = _process_id != 0 ? 0.0 | ||
| 490 | : node_reduction->cpus_node == 0 ? 0.0 | ||
| 491 | : (double)(node_reduction->mpi_time + node_reduction->mpi_worker_idle_time) | ||
| 492 | / node_reduction->cpus_node; | ||
| 493 | |||
| 494 | /* The number of GPUs and their times need to be aggregated by node, since | ||
| 495 | * devices can be shared among processes in the node. */ | ||
| 496 | int num_gpus = 0; | ||
| 497 | int64_t gpu_useful_time = 0; | ||
| 498 | int64_t gpu_communication_time = 0; | ||
| 499 | if (_process_id == 0 && node_reduction->node_used) { | ||
| 500 | num_gpus = node_reduction->num_gpu_ids; | ||
| 501 | for (int i = 0; i < num_gpus; ++i) { | ||
| 502 | gpu_useful_time += node_reduction->gpu_useful[i]; | ||
| 503 | gpu_communication_time += node_reduction->gpu_communication[i]; | ||
| 504 | } | ||
| 505 | } | ||
| 506 | |||
| 507 | const pop_base_metrics_t app_reduction_send = { | ||
| 508 | /* Resources */ | ||
| 509 | .num_cpus = monitor->num_cpus, | ||
| 510 | .num_available_cpus = _process_id == 0 && node_reduction->node_used | ||
| 511 | ? mu_get_system_count() : 0, | ||
| 512 | .num_omp_threads = monitor->num_omp_threads, | ||
| 513 | .num_mpi_ranks = 1, | ||
| 514 | .num_nodes = _process_id == 0 && node_reduction->node_used ? 1 : 0, | ||
| 515 | .avg_cpus = monitor->avg_cpus, | ||
| 516 | .num_gpus = num_gpus, | ||
| 517 | /* Hardware Counters */ | ||
| 518 | .cycles = (double)monitor->cycles, | ||
| 519 | .instructions = (double)monitor->instructions, | ||
| 520 | /* Statistics */ | ||
| 521 | .num_measurements = monitor->num_measurements, | ||
| 522 | .num_mpi_calls = monitor->num_mpi_calls, | ||
| 523 | .num_omp_parallels = monitor->num_omp_parallels, | ||
| 524 | .num_omp_tasks = monitor->num_omp_tasks, | ||
| 525 | .num_gpu_runtime_calls = monitor->num_gpu_runtime_calls, | ||
| 526 | /* Host Times */ | ||
| 527 | .elapsed_time = monitor->elapsed_time, | ||
| 528 | .useful_time = monitor->useful_time, | ||
| 529 | .mpi_time = monitor->mpi_time, | ||
| 530 | .mpi_worker_idle_time = monitor->mpi_worker_idle_time, | ||
| 531 | .omp_load_imbalance_time = monitor->omp_load_imbalance_time, | ||
| 532 | .omp_scheduling_time = monitor->omp_scheduling_time, | ||
| 533 | .omp_outside_parallel_time = monitor->omp_outside_parallel_time, | ||
| 534 | .gpu_runtime_time = monitor->gpu_runtime_time, | ||
| 535 | /* Host Normalized Times */ | ||
| 536 | .min_mpi_normd_proc = min_mpi_normd_proc, | ||
| 537 | .min_mpi_normd_node = min_mpi_normd_node, | ||
| 538 | /* Device Times */ | ||
| 539 | .gpu_useful_time = gpu_useful_time, | ||
| 540 | .gpu_communication_time = gpu_communication_time, | ||
| 541 | .gpu_inactive_time = monitor->gpu_inactive_time, | ||
| 542 | /* Device Max Times */ | ||
| 543 | .max_gpu_useful_time = monitor->gpu_useful_time, | ||
| 544 | .max_gpu_active_time = monitor->gpu_useful_time + monitor->gpu_communication_time, | ||
| 545 | }; | ||
| 546 | |||
| 547 | /* MPI type: int64_t */ | ||
| 548 | MPI_Datatype mpi_int64_type = get_mpi_int64_type(); | ||
| 549 | |||
| 550 | /* MPI struct type: app_reduction_t */ | ||
| 551 | MPI_Datatype mpi_app_reduction_type; | ||
| 552 | { | ||
| 553 | |||
| 554 | int blocklengths[] = { | ||
| 555 | #define FIELD_BLOCKLENGTH(name, c_type, mpi_type) 1, | ||
| 556 | FOR_POP_BASE_METRICS_FIELDS(FIELD_BLOCKLENGTH) | ||
| 557 | #undef FIELD_BLOCKLENGTH | ||
| 558 | }; | ||
| 559 | |||
| 560 | MPI_Aint displacements[] = { | ||
| 561 | #define FIELD_DISPLACEMENT(name, c_type, mpi_type) offsetof(pop_base_metrics_t, name), | ||
| 562 | FOR_POP_BASE_METRICS_FIELDS(FIELD_DISPLACEMENT) | ||
| 563 | #undef FIELD_DISPLACEMENT | ||
| 564 | }; | ||
| 565 | |||
| 566 | MPI_Datatype types[] = { | ||
| 567 | #define FIELD_MPI_TYPE(name, c_type, mpi_type) mpi_type, | ||
| 568 | FOR_POP_BASE_METRICS_FIELDS(FIELD_MPI_TYPE) | ||
| 569 | #undef FIELD_MPI_TYPE | ||
| 570 | }; | ||
| 571 | |||
| 572 | enum {count = sizeof(blocklengths) / sizeof(blocklengths[0])}; | ||
| 573 | |||
| 574 | MPI_Datatype tmp_type; | ||
| 575 | PMPI_Type_create_struct(count, blocklengths, displacements, types, &tmp_type); | ||
| 576 | PMPI_Type_create_resized(tmp_type, 0, sizeof(pop_base_metrics_t), | ||
| 577 | &mpi_app_reduction_type); | ||
| 578 | PMPI_Type_commit(&mpi_app_reduction_type); | ||
| 579 | } | ||
| 580 | |||
| 581 | /* Define MPI operation */ | ||
| 582 | MPI_Op app_reduction_op; | ||
| 583 | PMPI_Op_create(mpi_reduction_fn, true, &app_reduction_op); | ||
| 584 | |||
| 585 | /* MPI reduction */ | ||
| 586 | if (!all_to_all) { | ||
| 587 | PMPI_Reduce(&app_reduction_send, base_metrics, 1, | ||
| 588 | mpi_app_reduction_type, app_reduction_op, | ||
| 589 | 0, getWorldComm()); | ||
| 590 | } else { | ||
| 591 | PMPI_Allreduce(&app_reduction_send, base_metrics, 1, | ||
| 592 | mpi_app_reduction_type, app_reduction_op, | ||
| 593 | getWorldComm()); | ||
| 594 | } | ||
| 595 | |||
| 596 | /* Free MPI types */ | ||
| 597 | PMPI_Type_free(&mpi_app_reduction_type); | ||
| 598 | PMPI_Op_free(&app_reduction_op); | ||
| 599 | } | ||
| 600 | |||
| 601 | #endif | ||
| 602 | |||
| 603 | |||
| 604 | |||
| 605 | #if MPI_LIB | ||
| 606 | /* Construct a base metrics struct out of a monitor reduced via MPI */ | ||
| 607 | void perf_metrics__reduce_monitor_into_base_metrics(pop_base_metrics_t *base_metrics, | ||
| 608 | const dlb_monitor_t *monitor, bool all_to_all) { | ||
| 609 | |||
| 610 | /* First, reduce some values among processes in the node, | ||
| 611 | * needed to compute pop metrics */ | ||
| 612 | node_reduction_t node_reduction = {0}; | ||
| 613 | reduce_pop_metrics_node_reduction(&node_reduction, monitor); | ||
| 614 | |||
| 615 | /* With the node reduction, reduce again among all process */ | ||
| 616 | *base_metrics = (pop_base_metrics_t){0}; | ||
| 617 | reduce_pop_metrics_app_reduction(base_metrics, &node_reduction, | ||
| 618 | monitor, all_to_all); | ||
| 619 | } | ||
| 620 | #endif | ||
| 621 | |||
| 622 | |||
| 623 | /* Construct a base metrics struct out of a single monitor */ | ||
| 624 | 29 | void perf_metrics__local_monitor_into_base_metrics(pop_base_metrics_t *base_metrics, | |
| 625 | const dlb_monitor_t *monitor, talp_flags_t talp_flags) { | ||
| 626 | |||
| 627 | 29 | double mpi_normd = | |
| 628 | 29 | (double)(monitor->mpi_time + monitor->mpi_worker_idle_time) / monitor->num_cpus; | |
| 629 | |||
| 630 | 29 | int num_mpi_ranks = 0; | |
| 631 | 29 | int num_nodes = 1; | |
| 632 | #if MPI_LIB | ||
| 633 | if (talp_flags.have_mpi) { | ||
| 634 | num_mpi_ranks = _mpi_size; | ||
| 635 | num_nodes = _num_nodes; | ||
| 636 | } | ||
| 637 | #endif | ||
| 638 | |||
| 639 | 29 | *base_metrics = (const pop_base_metrics_t){ | |
| 640 | 29 | .num_cpus = monitor->num_cpus, | |
| 641 | 29 | .num_available_cpus = mu_get_system_count(), | |
| 642 | 29 | .num_omp_threads = monitor->num_omp_threads, | |
| 643 | .num_mpi_ranks = num_mpi_ranks, | ||
| 644 | .num_nodes = num_nodes, | ||
| 645 | 29 | .avg_cpus = monitor->avg_cpus, | |
| 646 | 29 | .num_gpus = monitor->num_gpus, | |
| 647 | 29 | .cycles = (double)monitor->cycles, | |
| 648 | 29 | .instructions = (double)monitor->instructions, | |
| 649 | 29 | .num_measurements = monitor->num_measurements, | |
| 650 | 29 | .num_mpi_calls = monitor->num_mpi_calls, | |
| 651 | 29 | .num_omp_parallels = monitor->num_omp_parallels, | |
| 652 | 29 | .num_omp_tasks = monitor->num_omp_tasks, | |
| 653 | 29 | .num_gpu_runtime_calls = monitor->num_gpu_runtime_calls, | |
| 654 | 29 | .elapsed_time = monitor->elapsed_time, | |
| 655 | 29 | .useful_time = monitor->useful_time, | |
| 656 | 29 | .mpi_time = monitor->mpi_time, | |
| 657 | 29 | .mpi_worker_idle_time = monitor->mpi_worker_idle_time, | |
| 658 | 29 | .omp_load_imbalance_time = monitor->omp_load_imbalance_time, | |
| 659 | 29 | .omp_scheduling_time = monitor->omp_scheduling_time, | |
| 660 | 29 | .omp_outside_parallel_time = monitor->omp_outside_parallel_time, | |
| 661 | 29 | .gpu_runtime_time = monitor->gpu_runtime_time, | |
| 662 | .min_mpi_normd_proc = mpi_normd, | ||
| 663 | .min_mpi_normd_node = mpi_normd, | ||
| 664 | 29 | .gpu_useful_time = monitor->gpu_useful_time, | |
| 665 | 29 | .gpu_communication_time = monitor->gpu_communication_time, | |
| 666 | 29 | .gpu_inactive_time = monitor->gpu_inactive_time, | |
| 667 | 29 | .max_gpu_useful_time = monitor->gpu_useful_time, | |
| 668 | 29 | .max_gpu_active_time = monitor->gpu_useful_time + monitor->gpu_communication_time, | |
| 669 | }; | ||
| 670 | 29 | } | |
| 671 | |||
| 672 | /* Compute POP metrics out of a base metrics struct */ | ||
| 673 | 29 | void perf_metrics__base_to_pop_metrics(const char *monitor_name, | |
| 674 | const pop_base_metrics_t *base_metrics, dlb_pop_metrics_t *pop_metrics) { | ||
| 675 | |||
| 676 | /* Compute POP metrics */ | ||
| 677 | 29 | perf_metrics_hybrid_t metrics = {0}; | |
| 678 | |||
| 679 |
1/2✓ Branch 0 taken 29 times.
✗ Branch 1 not taken.
|
29 | if (base_metrics->useful_time > 0) { |
| 680 | |||
| 681 |
1/3✗ Branch 0 not taken.
✓ Branch 1 taken 29 times.
✗ Branch 2 not taken.
|
29 | switch(thread_spd->options.talp_model) { |
| 682 | ✗ | case TALP_MODEL_HYBRID_V1: | |
| 683 | ✗ | perf_metrics__compute_hybrid_model_v1(&metrics, base_metrics); | |
| 684 | ✗ | break; | |
| 685 | 29 | case TALP_MODEL_HYBRID_V2: | |
| 686 | 29 | perf_metrics__compute_hybrid_model_v2(&metrics, base_metrics); | |
| 687 | 29 | break; | |
| 688 | }; | ||
| 689 | } | ||
| 690 | |||
| 691 | /* Initialize structure */ | ||
| 692 | 29 | *pop_metrics = (const dlb_pop_metrics_t) { | |
| 693 | 29 | .num_cpus = base_metrics->num_cpus, | |
| 694 | 29 | .num_omp_threads = base_metrics->num_omp_threads, | |
| 695 | 29 | .num_mpi_ranks = base_metrics->num_mpi_ranks, | |
| 696 | 29 | .num_nodes = base_metrics->num_nodes, | |
| 697 | 29 | .avg_cpus = base_metrics->avg_cpus, | |
| 698 | 29 | .num_gpus = base_metrics->num_gpus, | |
| 699 | 29 | .cycles = base_metrics->cycles, | |
| 700 | 29 | .instructions = base_metrics->instructions, | |
| 701 | 29 | .num_measurements = base_metrics->num_measurements, | |
| 702 | 29 | .num_mpi_calls = base_metrics->num_mpi_calls, | |
| 703 | 29 | .num_omp_parallels = base_metrics->num_omp_parallels, | |
| 704 | 29 | .num_omp_tasks = base_metrics->num_omp_tasks, | |
| 705 | 29 | .num_gpu_runtime_calls = base_metrics->num_gpu_runtime_calls, | |
| 706 | 29 | .elapsed_time = base_metrics->elapsed_time, | |
| 707 | 29 | .useful_time = base_metrics->useful_time, | |
| 708 | 29 | .mpi_time = base_metrics->mpi_time, | |
| 709 | 29 | .mpi_worker_idle_time = base_metrics->mpi_worker_idle_time, | |
| 710 | 29 | .omp_load_imbalance_time = base_metrics->omp_load_imbalance_time, | |
| 711 | 29 | .omp_scheduling_time = base_metrics->omp_scheduling_time, | |
| 712 | 29 | .omp_outside_parallel_time = base_metrics->omp_outside_parallel_time, | |
| 713 | 29 | .gpu_runtime_time = base_metrics->gpu_runtime_time, | |
| 714 | 29 | .min_mpi_normd_proc = base_metrics->min_mpi_normd_proc, | |
| 715 | 29 | .min_mpi_normd_node = base_metrics->min_mpi_normd_node, | |
| 716 | 29 | .gpu_useful_time = base_metrics->gpu_useful_time, | |
| 717 | 29 | .gpu_communication_time = base_metrics->gpu_communication_time, | |
| 718 | 29 | .gpu_inactive_time = base_metrics->gpu_inactive_time, | |
| 719 | 29 | .max_gpu_useful_time = base_metrics->max_gpu_useful_time, | |
| 720 | 29 | .max_gpu_active_time = base_metrics->max_gpu_active_time, | |
| 721 | 29 | .parallel_efficiency = metrics.parallel_efficiency, | |
| 722 | 29 | .mpi_parallel_efficiency = metrics.mpi_parallel_efficiency, | |
| 723 | 29 | .mpi_communication_efficiency = metrics.mpi_communication_efficiency, | |
| 724 | 29 | .mpi_load_balance = metrics.mpi_load_balance, | |
| 725 | 29 | .mpi_load_balance_in = metrics.mpi_load_balance_in, | |
| 726 | 29 | .mpi_load_balance_out = metrics.mpi_load_balance_out, | |
| 727 | 29 | .omp_parallel_efficiency = metrics.omp_parallel_efficiency, | |
| 728 | 29 | .omp_load_balance = metrics.omp_load_balance, | |
| 729 | 29 | .omp_scheduling_efficiency = metrics.omp_scheduling_efficiency, | |
| 730 | 29 | .omp_coverage_efficiency = metrics.omp_coverage_efficiency, | |
| 731 | 29 | .device_offload_efficiency = metrics.device_offload_efficiency, | |
| 732 | 29 | .gpu_parallel_efficiency = metrics.gpu_parallel_efficiency, | |
| 733 | 29 | .gpu_load_balance = metrics.gpu_load_balance, | |
| 734 | 29 | .gpu_communication_efficiency = metrics.gpu_communication_efficiency, | |
| 735 | 29 | .gpu_orchestration_efficiency = metrics.gpu_orchestration_efficiency, | |
| 736 | }; | ||
| 737 | 29 | snprintf(pop_metrics->name, DLB_MONITOR_NAME_MAX, "%s", monitor_name); | |
| 738 | 29 | } | |
| 739 |