GCC Code Coverage Report

Directory: src/
File: src/LB_policies/lewi_mask.c
Date: 2024-11-22 17:07:10
Exec Total Coverage
Lines: 306 376 81.4%
Functions: 31 36 86.1%
Branches: 126 201 62.7%
Line Branch Exec Source
1 /*********************************************************************************/
2 /* Copyright 2009-2024 Barcelona Supercomputing Center */
3 /* */
4 /* This file is part of the DLB library. */
5 /* */
6 /* DLB is free software: you can redistribute it and/or modify */
7 /* it under the terms of the GNU Lesser General Public License as published by */
8 /* the Free Software Foundation, either version 3 of the License, or */
9 /* (at your option) any later version. */
10 /* */
11 /* DLB is distributed in the hope that it will be useful, */
12 /* but WITHOUT ANY WARRANTY; without even the implied warranty of */
13 /* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the */
14 /* GNU Lesser General Public License for more details. */
15 /* */
16 /* You should have received a copy of the GNU Lesser General Public License */
17 /* along with DLB. If not, see <https://www.gnu.org/licenses/>. */
18 /*********************************************************************************/
19
20 #include "LB_policies/lewi_mask.h"
21
22 #include "LB_core/spd.h"
23 #include "LB_comm/shmem_cpuinfo.h"
24 #include "LB_comm/shmem_async.h"
25 #include "apis/dlb_errors.h"
26 #include "support/debug.h"
27 #include "support/gslist.h"
28 #include "support/mask_utils.h"
29 #include "support/small_array.h"
30 #include "support/types.h"
31
32 #include <sched.h>
33 #include <stdlib.h>
34 #include <string.h>
35 #include <pthread.h>
36
37 /* array_cpuid_t */
38 #define ARRAY_T cpuid_t
39 #include "support/array_template.h"
40
41 /* array_cpuinfo_task_t */
42 #define ARRAY_T cpuinfo_task_t
43 #define ARRAY_KEY_T pid_t
44 #include "support/array_template.h"
45
46
47 /* Node size will be the same for all processes in the node,
48 * it is safe to be out of the shared memory */
49 static int node_size = -1;
50
51 /* LeWI_mask data is private for each process */
52 typedef struct LeWI_mask_info {
53 int64_t last_borrow;
54 int max_parallelism;
55 array_cpuid_t cpus_priority_array;
56 cpu_set_t pending_reclaimed_cpus; /* CPUs that become reclaimed after an MPI */
57 cpu_set_t in_mpi_cpus; /* CPUs inside an MPI call */
58 GSList *cpuid_arrays; /* thread-private pointers to free at finalize */
59 GSList *cpuinfo_task_arrays; /* thread-private pointers to free at finalize */
60 pthread_mutex_t mutex; /* Mutex to protect lewi_info */
61 } lewi_info_t;
62
63
64 /* Compute the common elements between a cpuid array a cpu_set:
65 * cpuid_t *result = cpuid_t *op1 AND cpu_set_t *op2
66 * (cpu_set_t* may be NULL, meaning neutral value)
67 */
68 72 static inline void cpu_array_and(array_cpuid_t *result,
69 const array_cpuid_t *op1, const cpu_set_t *op2) {
70
2/2
✓ Branch 0 taken 612 times.
✓ Branch 1 taken 72 times.
 684 for (unsigned int i = 0; i < op1->count; ++i) {
71
7/8
✓ Branch 0 taken 144 times.
✓ Branch 1 taken 468 times.
✓ Branch 2 taken 144 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 88 times.
✓ Branch 5 taken 56 times.
✓ Branch 6 taken 88 times.
✓ Branch 7 taken 56 times.
 612 if (op2 == NULL || CPU_ISSET(op1->items[i], op2)) {
72 556 array_cpuid_t_push(result, op1->items[i]);
73 }
74 }
75 72 }
76
77 /* Construct a cpuid array from a cpu_set_t */
78 4 static inline void cpu_array_from_cpuset(array_cpuid_t *result,
79 const cpu_set_t *cpu_set) {
80 4 for (int cpuid = mu_get_first_cpu(cpu_set);
81
2/2
✓ Branch 0 taken 4 times.
✓ Branch 1 taken 4 times.
 8 cpuid >= 0;
82 4 cpuid = mu_get_next_cpu(cpu_set, cpuid)) {
83 4 array_cpuid_t_push(result, cpuid);
84 }
85 4 }
86
87
88 /* Construct a priority list of CPUs considering the topology and the affinity mask */
89 23 static void lewi_mask_UpdateOwnershipInfo(const subprocess_descriptor_t *spd,
90 const cpu_set_t *process_mask) {
91
92 23 lewi_info_t *lewi_info = spd->lewi_info;
93 23 lewi_affinity_t lewi_affinity = spd->options.lewi_affinity;
94
95 cpu_set_t affinity_mask;
96 23 mu_get_nodes_intersecting_with_cpuset(&affinity_mask, process_mask);
97
98 /* Reset current priority array */
99 23 array_cpuid_t *cpus_priority_array = &lewi_info->cpus_priority_array;
100 23 array_cpuid_t_clear(cpus_priority_array);
101
102 /* First, construct a list of potentially available CPUs
103 * (owned + nearby, depending on the affinity policy) */
104 cpu_set_t system_mask;
105 23 mu_get_system_mask(&system_mask);
106 23 for (int cpuid = mu_get_first_cpu(&system_mask);
107
2/2
✓ Branch 0 taken 368 times.
✓ Branch 1 taken 23 times.
 391 cpuid >= 0;
108 368 cpuid = mu_get_next_cpu(&system_mask, cpuid)) {
109
5/6
✓ Branch 0 taken 368 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 140 times.
✓ Branch 3 taken 228 times.
✓ Branch 4 taken 140 times.
✓ Branch 5 taken 228 times.
 368 if (CPU_ISSET(cpuid, process_mask)) {
110 140 array_cpuid_t_push(cpus_priority_array, cpuid);
111 } else {
112
1/5
✓ Branch 0 taken 228 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
 228 switch (lewi_affinity) {
113 228 case LEWI_AFFINITY_AUTO:
114 case LEWI_AFFINITY_MASK:
115 case LEWI_AFFINITY_NEARBY_FIRST:
116 228 array_cpuid_t_push(cpus_priority_array, cpuid);
117 228 break;
118 ✗ case LEWI_AFFINITY_NEARBY_ONLY:
119 ✗ if (CPU_ISSET(cpuid, &affinity_mask)) {
120 ✗ array_cpuid_t_push(cpus_priority_array, cpuid);
121 }
122 ✗ break;
123 ✗ case LEWI_AFFINITY_SPREAD_IFEMPTY:
124 // This case cannot be pre-computed
125 ✗ break;
126 ✗ case LEWI_AFFINITY_NONE:
127 /* LEWI_AFFINITY_NONE should force LeWI without mask support */
128 ✗ fatal("Unhandled LEWI_AFFINITY_NONE in LeWI mask. "
129 "Please report bug");
130 }
131 }
132 }
133
134 /* Sort available CPUs according to the affinity */
135 cpu_set_t affinity[3];
136 23 memcpy(&affinity[0], process_mask, sizeof(cpu_set_t));
137 23 memcpy(&affinity[1], &affinity_mask, sizeof(cpu_set_t));
138 23 CPU_ZERO(&affinity[2]);
139 23 qsort_r(cpus_priority_array->items, cpus_priority_array->count,
140 sizeof(cpuid_t), mu_cmp_cpuids_by_affinity, &affinity);
141 23 }
142
143
144 /*********************************************************************************/
145 /* Thread-local pointers for temporary storage during LeWI calls */
146 /*********************************************************************************/
147
148 /* Array of eligible CPUs. Usually a subset of the CPUs priority array. */
149 static __thread array_cpuid_t _cpu_subset = {};
150
151 76 static inline array_cpuid_t* get_cpu_subset(const subprocess_descriptor_t *spd) {
152 /* Thread already has an allocated array, return it */
153
2/2
✓ Branch 0 taken 67 times.
✓ Branch 1 taken 9 times.
 76 if (likely(_cpu_subset.items != NULL)) {
154 67 array_cpuid_t_clear(&_cpu_subset);
155 67 return &_cpu_subset;
156 }
157
158 /* Otherwise, allocate */
159 9 array_cpuid_t_init(&_cpu_subset, node_size);
160
161 /* Add pointer to lewi_info to deallocate later */
162 9 lewi_info_t *lewi_info = spd->lewi_info;
163 9 pthread_mutex_lock(&lewi_info->mutex);
164 {
165 9 lewi_info->cpuid_arrays =
166 9 g_slist_prepend(lewi_info->cpuid_arrays, &_cpu_subset);
167 }
168 9 pthread_mutex_unlock(&lewi_info->mutex);
169
170 9 return &_cpu_subset;
171 }
172
173 /* Array of cpuinfo tasks. For enabling or disabling CPUs. */
174 static __thread array_cpuinfo_task_t _tasks = {};
175
176 218 static inline array_cpuinfo_task_t* get_tasks(const subprocess_descriptor_t *spd) {
177 /* Thread already has an allocated array, return it */
178
2/2
✓ Branch 0 taken 195 times.
✓ Branch 1 taken 23 times.
 218 if (likely(_tasks.items != NULL)) {
179 195 array_cpuinfo_task_t_clear(&_tasks);
180 195 return &_tasks;
181 }
182
183 /* Otherwise, allocate */
184 23 array_cpuinfo_task_t_init(&_tasks, node_size*2);
185
186 /* Add pointer to lewi_info to deallocate later */
187 23 lewi_info_t *lewi_info = spd->lewi_info;
188 23 pthread_mutex_lock(&lewi_info->mutex);
189 {
190 23 lewi_info->cpuinfo_task_arrays =
191 23 g_slist_prepend(lewi_info->cpuinfo_task_arrays, &_tasks);
192 }
193 23 pthread_mutex_unlock(&lewi_info->mutex);
194
195 23 return &_tasks;
196 }
197
198
199 /*********************************************************************************/
200 /* Resolve cpuinfo tasks */
201 /*********************************************************************************/
202
203 122 static void resolve_cpuinfo_tasks(const subprocess_descriptor_t *restrict spd,
204 array_cpuinfo_task_t *restrict tasks) {
205
206 122 size_t tasks_count = tasks->count;
207
208 /* We don't need it strictly sorted, but if there are 3 or more tasks
209 * we need to group them by pid */
210
2/2
✓ Branch 0 taken 17 times.
✓ Branch 1 taken 105 times.
 122 if (tasks_count > 2 ) {
211 17 array_cpuinfo_task_t_sort(tasks);
212 }
213
214 /* Iterate tasks by group of PIDs */
215
2/2
✓ Branch 0 taken 114 times.
✓ Branch 1 taken 122 times.
 236 for (size_t i=0; i<tasks_count; ) {
216
217 /* count how many times this PID is repeated */
218 114 size_t j = i+1;
219 114 while (j < tasks_count
220
4/4
✓ Branch 0 taken 68 times.
✓ Branch 1 taken 98 times.
✓ Branch 2 taken 52 times.
✓ Branch 3 taken 16 times.
 166 && tasks->items[j].pid == tasks->items[i].pid) {
221 52 ++j;
222 }
223 114 size_t num_tasks = j - i;
224
225
2/2
✓ Branch 0 taken 77 times.
✓ Branch 1 taken 37 times.
 114 if (num_tasks == 1) {
226 /* resolve single task */
227 77 const cpuinfo_task_t *task = &tasks->items[i];
228
2/2
✓ Branch 0 taken 70 times.
✓ Branch 1 taken 7 times.
 77 if (task->pid == spd->id) {
229
2/2
✓ Branch 0 taken 55 times.
✓ Branch 1 taken 15 times.
 70 if (task->action == ENABLE_CPU) {
230 55 enable_cpu(&spd->pm, task->cpuid);
231 }
232
1/2
✓ Branch 0 taken 15 times.
✗ Branch 1 not taken.
 15 else if (task->action == DISABLE_CPU) {
233 15 disable_cpu(&spd->pm, task->cpuid);
234 }
235 }
236
2/2
✓ Branch 0 taken 1 times.
✓ Branch 1 taken 6 times.
 7 else if (spd->options.mode == MODE_ASYNC) {
237
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1 times.
 1 if (task->action == ENABLE_CPU) {
238 ✗ shmem_async_enable_cpu(task->pid, task->cpuid);
239 }
240
1/2
✓ Branch 0 taken 1 times.
✗ Branch 1 not taken.
 1 else if (task->action == DISABLE_CPU) {
241 1 shmem_async_disable_cpu(task->pid, task->cpuid);
242 1 shmem_cpuinfo__return_async_cpu(task->pid, task->cpuid);
243 }
244 }
245 } else {
246 /* group tasks */
247 37 cpu_set_t cpus_to_enable = {};
248 37 cpu_set_t cpus_to_disable = {};
249
2/2
✓ Branch 0 taken 89 times.
✓ Branch 1 taken 37 times.
 126 for (size_t k = i; k < i+num_tasks; ++k) {
250 89 const cpuinfo_task_t *task = &tasks->items[k];
251
2/2
✓ Branch 0 taken 65 times.
✓ Branch 1 taken 24 times.
 89 if (task->action == ENABLE_CPU) {
252
1/2
✓ Branch 0 taken 65 times.
✗ Branch 1 not taken.
 65 CPU_SET(task->cpuid, &cpus_to_enable);
253 }
254
1/2
✓ Branch 0 taken 24 times.
✗ Branch 1 not taken.
 24 else if (task->action == DISABLE_CPU) {
255
1/2
✓ Branch 0 taken 24 times.
✗ Branch 1 not taken.
 24 CPU_SET(task->cpuid, &cpus_to_disable);
256 }
257 }
258
259 /* resolve group of tasks for the same PID */
260 37 const cpuinfo_task_t *task = &tasks->items[i];
261
2/2
✓ Branch 0 taken 28 times.
✓ Branch 1 taken 9 times.
 37 if (task->pid == spd->id) {
262
2/2
✓ Branch 1 taken 25 times.
✓ Branch 2 taken 3 times.
 28 if (CPU_COUNT(&cpus_to_enable) > 0) {
263 25 enable_cpu_set(&spd->pm, &cpus_to_enable);
264 }
265
2/2
✓ Branch 1 taken 3 times.
✓ Branch 2 taken 25 times.
 28 if (CPU_COUNT(&cpus_to_disable) > 0) {
266 3 disable_cpu_set(&spd->pm, &cpus_to_disable);
267 }
268 }
269
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 9 times.
 9 else if (spd->options.mode == MODE_ASYNC) {
270 ✗ if (CPU_COUNT(&cpus_to_enable) > 0) {
271 ✗ shmem_async_enable_cpu_set(task->pid, &cpus_to_enable);
272 }
273 ✗ if (CPU_COUNT(&cpus_to_disable) > 0) {
274 ✗ shmem_async_disable_cpu_set(task->pid, &cpus_to_disable);
275 ✗ shmem_cpuinfo__return_async_cpu_mask(task->pid, &cpus_to_disable);
276 }
277 }
278 }
279
280 114 i += num_tasks;
281 }
282 122 }
283
284
285 /*********************************************************************************/
286 /* Init / Finalize */
287 /*********************************************************************************/
288
289 23 int lewi_mask_Init(subprocess_descriptor_t *spd) {
290 /* Value is always updated to allow testing different node sizes */
291 23 node_size = mu_get_system_size();
292
293 /* Allocate and initialize private structure */
294 23 spd->lewi_info = malloc(sizeof(lewi_info_t));
295 23 lewi_info_t *lewi_info = spd->lewi_info;
296 23 *lewi_info = (const lewi_info_t) {
297 23 .max_parallelism = spd->options.lewi_max_parallelism,
298 .mutex = PTHREAD_MUTEX_INITIALIZER,
299 };
300 23 array_cpuid_t_init(&lewi_info->cpus_priority_array, node_size);
301 23 lewi_mask_UpdateOwnershipInfo(spd, &spd->process_mask);
302
303 /* Enable request queues only in async mode */
304
2/2
✓ Branch 0 taken 5 times.
✓ Branch 1 taken 18 times.
 23 if (spd->options.mode == MODE_ASYNC) {
305 5 shmem_cpuinfo__enable_request_queues();
306 }
307
308 23 return DLB_SUCCESS;
309 }
310
311 23 int lewi_mask_Finalize(subprocess_descriptor_t *spd) {
312 /* De-register subprocess from the shared memory */
313 23 array_cpuinfo_task_t *tasks = get_tasks(spd);
314 23 int error = shmem_cpuinfo__deregister(spd->id, tasks);
315
1/2
✓ Branch 0 taken 23 times.
✗ Branch 1 not taken.
 23 if (error == DLB_SUCCESS) {
316 23 resolve_cpuinfo_tasks(spd, tasks);
317 }
318
319 /* Deallocate thread private arrays */
320 23 lewi_info_t *lewi_info = spd->lewi_info;
321 23 pthread_mutex_lock(&lewi_info->mutex);
322 {
323 23 for (GSList *node = lewi_info->cpuid_arrays;
324
2/2
✓ Branch 0 taken 9 times.
✓ Branch 1 taken 23 times.
 32 node != NULL;
325 9 node = node->next) {
326 9 array_cpuid_t *array = node->data;
327 9 array_cpuid_t_destroy(array);
328 }
329 23 g_slist_free(lewi_info->cpuid_arrays);
330
331 23 for (GSList *node = lewi_info->cpuinfo_task_arrays;
332
2/2
✓ Branch 0 taken 23 times.
✓ Branch 1 taken 23 times.
 46 node != NULL;
333 23 node = node->next) {
334 23 array_cpuid_t *array = node->data;
335 23 array_cpuid_t_destroy(array);
336 }
337 23 g_slist_free(lewi_info->cpuinfo_task_arrays);
338 }
339 23 pthread_mutex_unlock(&lewi_info->mutex);
340
341
342 /* Deallocate private structure */
343 23 array_cpuid_t_destroy(&lewi_info->cpus_priority_array);
344 23 free(lewi_info);
345 23 lewi_info = NULL;
346
347 23 return (error >= 0) ? DLB_SUCCESS : error;
348 }
349
350
351 /*********************************************************************************/
352 /* LeWI Modes (enable/disable, max_parallelism, ...) */
353 /*********************************************************************************/
354
355 2 int lewi_mask_EnableDLB(const subprocess_descriptor_t *spd) {
356 /* Reset value of last_borrow */
357 2 ((lewi_info_t*)spd->lewi_info)->last_borrow = 0;
358 2 return DLB_SUCCESS;
359 }
360
361 2 int lewi_mask_DisableDLB(const subprocess_descriptor_t *spd) {
362 2 array_cpuinfo_task_t *tasks = get_tasks(spd);
363 2 int error = shmem_cpuinfo__reset(spd->id, tasks);
364
1/2
✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
 2 if (error == DLB_SUCCESS) {
365 2 resolve_cpuinfo_tasks(spd, tasks);
366 }
367 2 return error;
368 }
369
370 2 int lewi_mask_SetMaxParallelism(const subprocess_descriptor_t *spd, int max) {
371 2 int error = DLB_SUCCESS;
372
1/2
✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
 2 if (max > 0) {
373 2 lewi_info_t *lewi_info = spd->lewi_info;
374 2 lewi_info->max_parallelism = max;
375 2 array_cpuinfo_task_t *tasks = get_tasks(spd);
376 2 error = shmem_cpuinfo__update_max_parallelism(spd->id, max, tasks);
377
1/2
✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
 2 if (error == DLB_SUCCESS) {
378 2 resolve_cpuinfo_tasks(spd, tasks);
379 }
380 }
381 2 return error;
382 }
383
384 2 int lewi_mask_UnsetMaxParallelism(const subprocess_descriptor_t *spd) {
385 2 lewi_info_t *lewi_info = spd->lewi_info;
386 2 lewi_info->max_parallelism = 0;
387 2 return DLB_SUCCESS;
388 }
389
390
391 /*********************************************************************************/
392 /* MPI */
393 /*********************************************************************************/
394
395 /* Obtain thread mask and remove first core if keep_cpu_on_blocking_call */
396 10 static inline void get_mask_for_blocking_call(
397 cpu_set_t *cpu_set, bool keep_cpu_on_blocking_call) {
398
399 /* Obtain current thread's affinity mask */
400 10 pthread_getaffinity_np(pthread_self(), sizeof(cpu_set_t), cpu_set);
401
402
2/2
✓ Branch 0 taken 2 times.
✓ Branch 1 taken 8 times.
 10 if (keep_cpu_on_blocking_call) {
403 /* Remove the first core */
404 2 int first_cpuid = mu_get_first_cpu(cpu_set);
405 2 const mu_cpuset_t *first_core = mu_get_core_mask(first_cpuid);
406 2 mu_substract(cpu_set, cpu_set, first_core->set);
407 }
408 10 }
409
410 /* Lend the CPUs of the thread encountering the blocking call */
411 5 int lewi_mask_IntoBlockingCall(const subprocess_descriptor_t *spd) {
412 5 int error = DLB_NOUPDT;
413
414 /* Obtain affinity mask to lend */
415 cpu_set_t cpu_set;
416 5 get_mask_for_blocking_call(&cpu_set,
417 5 spd->options.lewi_keep_cpu_on_blocking_call);
418
419
2/2
✓ Branch 1 taken 4 times.
✓ Branch 2 taken 1 times.
 5 if (mu_count(&cpu_set) > 0) {
420 #ifdef DEBUG_VERSION
421 /* Add cpu_set to in_mpi_cpus */
422 4 lewi_info_t *lewi_info = spd->lewi_info;
423
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 4 times.
 4 fatal_cond(mu_intersects(&lewi_info->in_mpi_cpus, &cpu_set),
424 "Some CPU in %s already into blocking call", mu_to_str(&cpu_set));
425 4 mu_or(&lewi_info->in_mpi_cpus, &lewi_info->in_mpi_cpus, &cpu_set);
426 #endif
427
1/2
✓ Branch 0 taken 4 times.
✗ Branch 1 not taken.
 4 verbose(VB_MICROLB, "In blocking call, lending %s", mu_to_str(&cpu_set));
428
429 /* Finally, lend mask */
430 4 error = lewi_mask_LendCpuMask(spd, &cpu_set);
431 }
432 5 return error;
433 }
434
435 /* Reclaim the CPUs that were lent when encountering the blocking call.
436 * The thread must have not change its affinity mask since then. */
437 5 int lewi_mask_OutOfBlockingCall(const subprocess_descriptor_t *spd) {
438 5 int error = DLB_NOUPDT;
439
440 /* Obtain affinity mask to reclaim */
441 cpu_set_t cpu_set;
442 5 get_mask_for_blocking_call(&cpu_set,
443 5 spd->options.lewi_keep_cpu_on_blocking_call);
444
445
2/2
✓ Branch 1 taken 4 times.
✓ Branch 2 taken 1 times.
 5 if (mu_count(&cpu_set) > 0) {
446 4 lewi_info_t *lewi_info = spd->lewi_info;
447 #ifdef DEBUG_VERSION
448 /* Clear cpu_set from in_mpi_cpus */
449
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 4 times.
 4 fatal_cond(!mu_is_subset(&lewi_info->in_mpi_cpus, &cpu_set),
450 "Some CPU in %s is not into blocking call", mu_to_str(&cpu_set));
451 4 mu_substract(&lewi_info->in_mpi_cpus, &lewi_info->in_mpi_cpus, &cpu_set);
452 #endif
453
1/2
✓ Branch 0 taken 4 times.
✗ Branch 1 not taken.
 4 verbose(VB_MICROLB, "Out of blocking call, acquiring %s", mu_to_str(&cpu_set));
454
455 /* If there are owned CPUs to reclaim: */
456 cpu_set_t owned_cpus;
457 4 mu_and(&owned_cpus, &cpu_set, &spd->process_mask);
458
2/2
✓ Branch 1 taken 2 times.
✓ Branch 2 taken 2 times.
 4 if (mu_count(&owned_cpus) > 0) {
459
460 /* Construct a CPU array from owned_cpus */
461 2 array_cpuid_t *cpu_subset = get_cpu_subset(spd);
462 2 cpu_array_from_cpuset(cpu_subset, &owned_cpus);
463
464 /* Acquire CPUs, but do not call any enable CPU callback */
465 2 array_cpuinfo_task_t *tasks = get_tasks(spd);
466 2 error = shmem_cpuinfo__acquire_from_cpu_subset(spd->id, cpu_subset, tasks);
467
468 /* Once CPUs are acquired from the shmem, we don't need to call
469 * any ENABLE callback because they weren't disabled in IntoBlockingCall.
470 * Only if asynchronous mode, keep DISABLE tasks, otherwise remove all. */
471
2/2
✓ Branch 0 taken 1 times.
✓ Branch 1 taken 1 times.
 2 if (error == DLB_NOTED
472
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1 times.
 1 && spd->options.mode == MODE_ASYNC) {
473 ✗ for (size_t i = 0, i_write = 0; i < tasks->count; ++i) {
474 ✗ const cpuinfo_task_t *task = &tasks->items[i];
475 ✗ if (task->action == DISABLE_CPU) {
476 ✗ if (i != i_write) {
477 ✗ tasks->items[i_write] = tasks->items[i];
478 }
479 ✗ ++i_write;
480 }
481 }
482 /* Async: resolve only DISABLE tasks */
483 ✗ resolve_cpuinfo_tasks(spd, tasks);
484 } else {
485 /* Polling: clear tasks */
486 2 array_cpuinfo_task_t_clear(tasks);
487 }
488 }
489
490 /* If there are ONLY non-owned CPUs to reclaim:
491 * (if some owned CPU was already reclaimed, we forget about non-owned) */
492 cpu_set_t non_owned_cpus;
493 4 mu_substract(&non_owned_cpus, &cpu_set, &spd->process_mask);
494
2/2
✓ Branch 1 taken 2 times.
✓ Branch 2 taken 2 times.
 4 if (mu_count(&non_owned_cpus) > 0
495
1/2
✓ Branch 1 taken 2 times.
✗ Branch 2 not taken.
 2 && mu_count(&owned_cpus) == 0) {
496
497 /* Construct a CPU array from non_owned_cpus */
498 2 array_cpuid_t *cpu_subset = get_cpu_subset(spd);
499 2 cpu_array_from_cpuset(cpu_subset, &non_owned_cpus);
500
501 /* Borrow CPUs, but also ignoring the enable callbacks */
502 2 array_cpuinfo_task_t *tasks = get_tasks(spd);
503 2 error = shmem_cpuinfo__borrow_from_cpu_subset(spd->id, cpu_subset, tasks);
504
505
2/2
✓ Branch 0 taken 1 times.
✓ Branch 1 taken 1 times.
 2 if (error != DLB_SUCCESS) {
506 /* Annotate the CPU as pending to bypass the shared memory for the next query */
507 1 mu_or(&lewi_info->pending_reclaimed_cpus, &lewi_info->pending_reclaimed_cpus,
508 &non_owned_cpus);
509
510 /* We lost the CPU while in MPI.
511 * In async mode, we can just disable the CPU.
512 * In polling mode, the process needs to call Return */
513
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1 times.
 1 if (spd->options.mode == MODE_ASYNC) {
514 ✗ disable_cpu_set(&spd->pm, &non_owned_cpus);
515 }
516 }
517 }
518 }
519
520 5 return error;
521 }
522
523
524 /*********************************************************************************/
525 /* Lend */
526 /*********************************************************************************/
527
528 ✗ int lewi_mask_Lend(const subprocess_descriptor_t *spd) {
529 cpu_set_t mask;
530 ✗ mu_get_system_mask(&mask);
531 ✗ return lewi_mask_LendCpuMask(spd, &mask);
532 }
533
534 43 int lewi_mask_LendCpu(const subprocess_descriptor_t *spd, int cpuid) {
535 43 array_cpuinfo_task_t *tasks = get_tasks(spd);
536 43 int error = shmem_cpuinfo__lend_cpu(spd->id, cpuid, tasks);
537
538
1/2
✓ Branch 0 taken 43 times.
✗ Branch 1 not taken.
 43 if (error == DLB_SUCCESS) {
539
2/2
✓ Branch 0 taken 4 times.
✓ Branch 1 taken 39 times.
 43 if (spd->options.mode == MODE_ASYNC) {
540 4 resolve_cpuinfo_tasks(spd, tasks);
541 }
542
543 /* Clear possible pending reclaimed CPUs */
544 43 lewi_info_t *lewi_info = spd->lewi_info;
545
1/2
✓ Branch 0 taken 43 times.
✗ Branch 1 not taken.
 43 CPU_CLR(cpuid, &lewi_info->pending_reclaimed_cpus);
546 }
547 43 return error;
548 }
549
550 21 int lewi_mask_LendCpuMask(const subprocess_descriptor_t *spd, const cpu_set_t *mask) {
551 21 array_cpuinfo_task_t *tasks = get_tasks(spd);
552 21 int error = shmem_cpuinfo__lend_cpu_mask(spd->id, mask, tasks);
553
1/2
✓ Branch 0 taken 21 times.
✗ Branch 1 not taken.
 21 if (error == DLB_SUCCESS) {
554
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 21 times.
 21 if (spd->options.mode == MODE_ASYNC) {
555 ✗ resolve_cpuinfo_tasks(spd, tasks);
556 }
557
558 /* Clear possible pending reclaimed CPUs */
559 21 lewi_info_t *lewi_info = spd->lewi_info;
560 21 mu_substract(&lewi_info->pending_reclaimed_cpus,
561 21 &lewi_info->pending_reclaimed_cpus, mask);
562 }
563 21 return error;
564 }
565
566
567 /*********************************************************************************/
568 /* Reclaim */
569 /*********************************************************************************/
570
571 2 int lewi_mask_Reclaim(const subprocess_descriptor_t *spd) {
572 /* Even though shmem_cpuinfo__reclaim_all exists,
573 * iterating the mask should be more efficient */
574 2 return lewi_mask_ReclaimCpuMask(spd, &spd->process_mask);
575 }
576
577 5 int lewi_mask_ReclaimCpu(const subprocess_descriptor_t *spd, int cpuid) {
578 5 array_cpuinfo_task_t *tasks = get_tasks(spd);
579 5 int error = shmem_cpuinfo__reclaim_cpu(spd->id, cpuid, tasks);
580
4/4
✓ Branch 0 taken 4 times.
✓ Branch 1 taken 1 times.
✓ Branch 2 taken 3 times.
✓ Branch 3 taken 1 times.
 5 if (error == DLB_SUCCESS || error == DLB_NOTED) {
581 4 resolve_cpuinfo_tasks(spd, tasks);
582 }
583 5 return error;
584 }
585
586 ✗ int lewi_mask_ReclaimCpus(const subprocess_descriptor_t *spd, int ncpus) {
587 ✗ array_cpuinfo_task_t *tasks = get_tasks(spd);
588 ✗ int error = shmem_cpuinfo__reclaim_cpus(spd->id, ncpus, tasks);
589 ✗ if (error == DLB_SUCCESS || error == DLB_NOTED) {
590 ✗ resolve_cpuinfo_tasks(spd, tasks);
591 }
592 ✗ return error;
593 }
594
595 8 int lewi_mask_ReclaimCpuMask(const subprocess_descriptor_t *spd, const cpu_set_t *mask) {
596 8 array_cpuinfo_task_t *tasks = get_tasks(spd);
597 8 int error = shmem_cpuinfo__reclaim_cpu_mask(spd->id, mask, tasks);
598 8 resolve_cpuinfo_tasks(spd, tasks);
599 8 return error;
600 }
601
602
603 /*********************************************************************************/
604 /* Acquire */
605 /*********************************************************************************/
606
607 18 int lewi_mask_AcquireCpu(const subprocess_descriptor_t *spd, int cpuid) {
608 18 array_cpuinfo_task_t *tasks = get_tasks(spd);
609 18 int error = shmem_cpuinfo__acquire_cpu(spd->id, cpuid, tasks);
610
4/4
✓ Branch 0 taken 7 times.
✓ Branch 1 taken 11 times.
✓ Branch 2 taken 3 times.
✓ Branch 3 taken 4 times.
 18 if (error == DLB_SUCCESS || error == DLB_NOTED) {
611 14 resolve_cpuinfo_tasks(spd, tasks);
612 }
613 18 return error;
614 }
615
616 38 int lewi_mask_AcquireCpus(const subprocess_descriptor_t *spd, int ncpus) {
617 38 int error = DLB_NOUPDT;
618
2/2
✓ Branch 0 taken 2 times.
✓ Branch 1 taken 36 times.
 38 if (ncpus == 0) {
619 /* AcquireCPUs(0) has a special meaning of removing any previous request */
620 2 shmem_cpuinfo__remove_requests(spd->id);
621 2 error = DLB_SUCCESS;
622
1/2
✓ Branch 0 taken 36 times.
✗ Branch 1 not taken.
 36 } else if (ncpus > 0) {
623 36 error = lewi_mask_AcquireCpusInMask(spd, ncpus, NULL);
624 }
625 38 return error;
626 }
627
628 10 int lewi_mask_AcquireCpuMask(const subprocess_descriptor_t *spd, const cpu_set_t *mask) {
629 10 return lewi_mask_AcquireCpusInMask(spd, 0, mask);
630 }
631
632 57 int lewi_mask_AcquireCpusInMask(const subprocess_descriptor_t *spd, int ncpus,
633 const cpu_set_t *mask) {
634 57 lewi_info_t *lewi_info = spd->lewi_info;
635 57 bool async = spd->options.mode == MODE_ASYNC;
636
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 57 times.
 57 int64_t *last_borrow = async ? NULL : &lewi_info->last_borrow;
637
638 /* Construct a CPU array based on cpus_priority_array and mask (if present) */
639 57 array_cpuid_t *cpu_subset = get_cpu_subset(spd);
640 57 cpu_array_and(cpu_subset, &lewi_info->cpus_priority_array, mask);
641
642 /* Provide a number of requested CPUs only if needed */
643
2/2
✓ Branch 0 taken 47 times.
✓ Branch 1 taken 10 times.
 57 int *requested_ncpus = ncpus > 0 ? &ncpus : NULL;
644
645 57 array_cpuinfo_task_t *tasks = get_tasks(spd);
646 57 int error = shmem_cpuinfo__acquire_ncpus_from_cpu_subset(spd->id,
647 57 requested_ncpus, cpu_subset, spd->options.lewi_affinity,
648 lewi_info->max_parallelism, last_borrow, tasks);
649
650
2/2
✓ Branch 0 taken 37 times.
✓ Branch 1 taken 20 times.
 57 if (error != DLB_NOUPDT) {
651 37 resolve_cpuinfo_tasks(spd, tasks);
652 }
653 57 return error;
654 }
655
656
657 /*********************************************************************************/
658 /* Borrow */
659 /*********************************************************************************/
660
661 4 int lewi_mask_Borrow(const subprocess_descriptor_t *spd) {
662 cpu_set_t system_mask;
663 4 mu_get_system_mask(&system_mask);
664 4 return lewi_mask_BorrowCpusInMask(spd, 0, &system_mask);
665 }
666
667 5 int lewi_mask_BorrowCpu(const subprocess_descriptor_t *spd, int cpuid) {
668 5 array_cpuinfo_task_t *tasks = get_tasks(spd);
669 5 int error = shmem_cpuinfo__borrow_cpu(spd->id, cpuid, tasks);
670
1/2
✓ Branch 0 taken 5 times.
✗ Branch 1 not taken.
 5 if (error == DLB_SUCCESS) {
671 5 resolve_cpuinfo_tasks(spd, tasks);
672 }
673 5 return error;
674 }
675
676 7 int lewi_mask_BorrowCpus(const subprocess_descriptor_t *spd, int ncpus) {
677 7 return lewi_mask_BorrowCpusInMask(spd, ncpus, NULL);
678 }
679
680 ✗ int lewi_mask_BorrowCpuMask(const subprocess_descriptor_t *spd, const cpu_set_t *mask) {
681 ✗ return lewi_mask_BorrowCpusInMask(spd, 0, mask);
682 }
683
684 15 int lewi_mask_BorrowCpusInMask(const subprocess_descriptor_t *spd, int ncpus, const cpu_set_t *mask) {
685 15 lewi_info_t *lewi_info = spd->lewi_info;
686 15 bool async = spd->options.mode == MODE_ASYNC;
687
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 15 times.
 15 int64_t *last_borrow = async ? NULL : &lewi_info->last_borrow;
688
689 /* Construct a CPU array based on cpus_priority_array and mask (if present) */
690 15 array_cpuid_t *cpu_subset = get_cpu_subset(spd);
691 15 cpu_array_and(cpu_subset, &lewi_info->cpus_priority_array, mask);
692
693 /* Provide a number of requested CPUs only if needed */
694
2/2
✓ Branch 0 taken 11 times.
✓ Branch 1 taken 4 times.
 15 int *requested_ncpus = ncpus > 0 ? &ncpus : NULL;
695
696 15 array_cpuinfo_task_t *tasks = get_tasks(spd);
697 15 int error = shmem_cpuinfo__borrow_ncpus_from_cpu_subset(spd->id,
698 15 requested_ncpus, cpu_subset, spd->options.lewi_affinity,
699 lewi_info->max_parallelism, last_borrow, tasks);
700
701
2/2
✓ Branch 0 taken 9 times.
✓ Branch 1 taken 6 times.
 15 if (error == DLB_SUCCESS) {
702 9 resolve_cpuinfo_tasks(spd, tasks);
703 }
704
705 15 return error;
706 }
707
708
709 /*********************************************************************************/
710 /* Return */
711 /*********************************************************************************/
712
713 1 int lewi_mask_Return(const subprocess_descriptor_t *spd) {
714
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1 times.
 1 if (spd->options.mode == MODE_ASYNC) {
715 // Return should not be called in async mode
716 ✗ return DLB_ERR_NOCOMP;
717 }
718
719 1 array_cpuinfo_task_t *tasks = get_tasks(spd);
720 1 int error = shmem_cpuinfo__return_all(spd->id, tasks);
721 1 resolve_cpuinfo_tasks(spd, tasks);
722
723 /* Check possible pending reclaimed CPUs */
724 1 lewi_info_t *lewi_info = spd->lewi_info;
725 1 int cpuid = mu_get_first_cpu(&lewi_info->pending_reclaimed_cpus);
726
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1 times.
 1 if (cpuid >= 0) {
727 do {
728 ✗ disable_cpu(&spd->pm, cpuid);
729 ✗ cpuid = mu_get_next_cpu(&lewi_info->pending_reclaimed_cpus, cpuid);
730 ✗ } while (cpuid >= 0);
731 ✗ CPU_ZERO(&lewi_info->pending_reclaimed_cpus);
732 ✗ error = DLB_SUCCESS;
733 }
734
735 1 return error;
736 }
737
738 14 int lewi_mask_ReturnCpu(const subprocess_descriptor_t *spd, int cpuid) {
739
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 14 times.
 14 if (spd->options.mode == MODE_ASYNC) {
740 // Return should not be called in async mode
741 ✗ return DLB_ERR_NOCOMP;
742 }
743
744 14 array_cpuinfo_task_t *tasks = get_tasks(spd);
745 14 int error = shmem_cpuinfo__return_cpu(spd->id, cpuid, tasks);
746
3/4
✓ Branch 0 taken 1 times.
✓ Branch 1 taken 13 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 1 times.
 14 if (error == DLB_SUCCESS || error == DLB_ERR_REQST) {
747 13 resolve_cpuinfo_tasks(spd, tasks);
748
1/2
✓ Branch 0 taken 1 times.
✗ Branch 1 not taken.
 1 } else if (error == DLB_ERR_PERM) {
749 /* Check possible pending reclaimed CPUs */
750 1 lewi_info_t *lewi_info = spd->lewi_info;
751
3/6
✓ Branch 0 taken 1 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 1 times.
✗ Branch 5 not taken.
 1 if (CPU_ISSET(cpuid, &lewi_info->pending_reclaimed_cpus)) {
752
1/2
✓ Branch 0 taken 1 times.
✗ Branch 1 not taken.
 1 CPU_CLR(cpuid, &lewi_info->pending_reclaimed_cpus);
753 1 disable_cpu(&spd->pm, cpuid);
754 1 error = DLB_SUCCESS;
755 }
756 }
757 14 return error;
758 }
759
760 ✗ int lewi_mask_ReturnCpuMask(const subprocess_descriptor_t *spd, const cpu_set_t *mask) {
761 ✗ if (spd->options.mode == MODE_ASYNC) {
762 // Return should not be called in async mode
763 ✗ return DLB_ERR_NOCOMP;
764 }
765
766 ✗ array_cpuinfo_task_t *tasks = get_tasks(spd);
767 ✗ int error = shmem_cpuinfo__return_cpu_mask(spd->id, mask, tasks);
768 ✗ resolve_cpuinfo_tasks(spd, tasks);
769
770 /* Check possible pending reclaimed CPUs */
771 ✗ lewi_info_t *lewi_info = spd->lewi_info;
772 ✗ if (CPU_COUNT(&lewi_info->pending_reclaimed_cpus) > 0) {
773 cpu_set_t cpus_to_return;
774 ✗ CPU_AND(&cpus_to_return, &lewi_info->pending_reclaimed_cpus, mask);
775 ✗ for (int cpuid = mu_get_first_cpu(&cpus_to_return); cpuid >= 0;
776 ✗ cpuid = mu_get_next_cpu(&cpus_to_return, cpuid)) {
777 ✗ disable_cpu(&spd->pm, cpuid);
778 }
779 ✗ mu_substract(&lewi_info->pending_reclaimed_cpus,
780 ✗ &lewi_info->pending_reclaimed_cpus, &cpus_to_return);
781 ✗ error = DLB_SUCCESS;
782 }
783
784 ✗ return error;
785 }
786
787
788 // Others
789
790 32 int lewi_mask_CheckCpuAvailability(const subprocess_descriptor_t *spd, int cpuid) {
791 32 return shmem_cpuinfo__check_cpu_availability(spd->id, cpuid);
792 }
793
794 ✗ int lewi_mask_UpdateOwnership(const subprocess_descriptor_t *spd,
795 const cpu_set_t *process_mask) {
796 /* Update priority array */
797 ✗ lewi_mask_UpdateOwnershipInfo(spd, process_mask);
798
799 /* Update cpuinfo data and return reclaimed CPUs */
800 ✗ array_cpuinfo_task_t *tasks = get_tasks(spd);
801 ✗ shmem_cpuinfo__update_ownership(spd->id, process_mask, tasks);
802 ✗ resolve_cpuinfo_tasks(spd, tasks);
803
804 /* Check possible pending reclaimed CPUs */
805 ✗ lewi_info_t *lewi_info = spd->lewi_info;
806 ✗ int cpuid = mu_get_first_cpu(&lewi_info->pending_reclaimed_cpus);
807 ✗ if (cpuid >= 0) {
808 do {
809 ✗ disable_cpu(&spd->pm, cpuid);
810 ✗ cpuid = mu_get_next_cpu(&lewi_info->pending_reclaimed_cpus, cpuid);
811 ✗ } while (cpuid >= 0);
812 ✗ CPU_ZERO(&lewi_info->pending_reclaimed_cpus);
813 }
814
815 ✗ return DLB_SUCCESS;
816 }
817