GCC Code Coverage Report

Directory: src/
File: src/LB_policies/lewi_mask.c
Date: 2025-11-21 10:34:40
Exec Total Coverage
Lines: 332 386 86.0%
Functions: 36 40 90.0%
Branches: 162 233 69.5%
Line Branch Exec Source
1 /*********************************************************************************/
2 /* Copyright 2009-2024 Barcelona Supercomputing Center */
3 /* */
4 /* This file is part of the DLB library. */
5 /* */
6 /* DLB is free software: you can redistribute it and/or modify */
7 /* it under the terms of the GNU Lesser General Public License as published by */
8 /* the Free Software Foundation, either version 3 of the License, or */
9 /* (at your option) any later version. */
10 /* */
11 /* DLB is distributed in the hope that it will be useful, */
12 /* but WITHOUT ANY WARRANTY; without even the implied warranty of */
13 /* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the */
14 /* GNU Lesser General Public License for more details. */
15 /* */
16 /* You should have received a copy of the GNU Lesser General Public License */
17 /* along with DLB. If not, see <https://www.gnu.org/licenses/>. */
18 /*********************************************************************************/
19
20 #include "LB_policies/lewi_mask.h"
21
22 #include "LB_core/spd.h"
23 #include "LB_comm/shmem_cpuinfo.h"
24 #include "LB_comm/shmem_async.h"
25 #include "apis/dlb_errors.h"
26 #include "support/debug.h"
27 #include "support/mask_utils.h"
28 #include "support/small_array.h"
29 #include "support/types.h"
30
31 #include <sched.h>
32 #include <stdlib.h>
33 #include <string.h>
34 #include <pthread.h>
35
36 /* array_cpuid_t */
37 #define ARRAY_T cpuid_t
38 #include "support/array_template.h"
39
40 /* array_cpuinfo_task_t */
41 #define ARRAY_T cpuinfo_task_t
42 #define ARRAY_KEY_T pid_t
43 #include "support/array_template.h"
44
45
46 /* Node size will be the same for all processes in the node,
47 * it is safe to be out of the shared memory */
48 static int node_size = -1;
49
50 /* LeWI_mask data is private for each process */
51 typedef struct LeWI_mask_info {
52 int64_t last_borrow;
53 int max_parallelism;
54 array_cpuid_t cpus_priority_array;
55 cpu_set_t pending_reclaimed_cpus; /* CPUs that become reclaimed after an MPI */
56 cpu_set_t in_mpi_cpus; /* CPUs inside an MPI call */
57 pthread_mutex_t mutex; /* Mutex to protect lewi_info */
58 } lewi_info_t;
59
60
61 /* Compute the common elements between a cpuid array a cpu_set:
62 * cpuid_t *result = cpuid_t *op1 AND cpu_set_t *op2
63 * (cpu_set_t* may be NULL, meaning neutral value)
64 */
65 123 static inline void cpu_array_and(array_cpuid_t *result,
66 const array_cpuid_t *op1, const cpu_set_t *op2) {
67
2/2
✓ Branch 0 taken 984 times.
✓ Branch 1 taken 123 times.
 1107 for (unsigned int i = 0; i < op1->count; ++i) {
68
7/8
✓ Branch 0 taken 240 times.
✓ Branch 1 taken 744 times.
✓ Branch 2 taken 240 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 158 times.
✓ Branch 5 taken 82 times.
✓ Branch 6 taken 158 times.
✓ Branch 7 taken 82 times.
 984 if (op2 == NULL || CPU_ISSET(op1->items[i], op2)) {
69 902 array_cpuid_t_push(result, op1->items[i]);
70 }
71 }
72 123 }
73
74 /* Construct a cpuid array from a cpu_set_t */
75 12 static inline void cpu_array_from_cpuset(array_cpuid_t *result,
76 const cpu_set_t *cpu_set) {
77 12 for (int cpuid = mu_get_first_cpu(cpu_set);
78
3/4
✓ Branch 0 taken 24 times.
✓ Branch 1 taken 12 times.
✓ Branch 2 taken 24 times.
✗ Branch 3 not taken.
 36 cpuid >= 0 && cpuid != DLB_CPUID_INVALID;
79 24 cpuid = mu_get_next_cpu(cpu_set, cpuid)) {
80 24 array_cpuid_t_push(result, cpuid);
81 }
82 12 }
83
84
85 /* Construct a priority list of CPUs considering the topology and the affinity mask */
86 40 static void lewi_mask_UpdateOwnershipInfo(const subprocess_descriptor_t *spd,
87 const cpu_set_t *process_mask) {
88
89 40 lewi_info_t *lewi_info = spd->lewi_info;
90 40 lewi_affinity_t lewi_affinity = spd->options.lewi_affinity;
91
92 cpu_set_t affinity_mask;
93 40 mu_get_nodes_intersecting_with_cpuset(&affinity_mask, process_mask);
94
95 /* Reset current priority array */
96 40 array_cpuid_t *cpus_priority_array = &lewi_info->cpus_priority_array;
97 40 array_cpuid_t_clear(cpus_priority_array);
98
99 /* First, construct a list of potentially available CPUs
100 * (owned + nearby, depending on the affinity policy) */
101 cpu_set_t system_mask;
102 40 mu_get_system_mask(&system_mask);
103 40 for (int cpuid = mu_get_first_cpu(&system_mask);
104
3/4
✓ Branch 0 taken 584 times.
✓ Branch 1 taken 40 times.
✓ Branch 2 taken 584 times.
✗ Branch 3 not taken.
 624 cpuid >= 0 && cpuid != DLB_CPUID_INVALID;
105 584 cpuid = mu_get_next_cpu(&system_mask, cpuid)) {
106
5/6
✓ Branch 0 taken 584 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 192 times.
✓ Branch 3 taken 392 times.
✓ Branch 4 taken 192 times.
✓ Branch 5 taken 392 times.
 584 if (CPU_ISSET(cpuid, process_mask)) {
107 192 array_cpuid_t_push(cpus_priority_array, cpuid);
108 } else {
109
1/5
✓ Branch 0 taken 392 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
 392 switch (lewi_affinity) {
110 392 case LEWI_AFFINITY_AUTO:
111 case LEWI_AFFINITY_MASK:
112 case LEWI_AFFINITY_NEARBY_FIRST:
113 392 array_cpuid_t_push(cpus_priority_array, cpuid);
114 392 break;
115 ✗ case LEWI_AFFINITY_NEARBY_ONLY:
116 ✗ if (CPU_ISSET(cpuid, &affinity_mask)) {
117 ✗ array_cpuid_t_push(cpus_priority_array, cpuid);
118 }
119 ✗ break;
120 ✗ case LEWI_AFFINITY_SPREAD_IFEMPTY:
121 // This case cannot be pre-computed
122 ✗ break;
123 ✗ case LEWI_AFFINITY_NONE:
124 /* LEWI_AFFINITY_NONE should force LeWI without mask support */
125 ✗ fatal("Unhandled LEWI_AFFINITY_NONE in LeWI mask. "
126 "Please report bug");
127 }
128 }
129 }
130
131 /* Sort available CPUs according to the affinity */
132 cpu_set_t affinity[3];
133 40 memcpy(&affinity[0], process_mask, sizeof(cpu_set_t));
134 40 memcpy(&affinity[1], &affinity_mask, sizeof(cpu_set_t));
135 40 CPU_ZERO(&affinity[2]);
136 40 qsort_r(cpus_priority_array->items, cpus_priority_array->count,
137 sizeof(cpuid_t), mu_cmp_cpuids_by_affinity, &affinity);
138 40 }
139
140
141 /*********************************************************************************/
142 /* Thread-local pointers for temporary storage during LeWI calls */
143 /*********************************************************************************/
144
145 /* Array of eligible CPUs. Usually a subset of the CPUs priority array. */
146 static __thread array_cpuid_t *_cpu_subset = NULL;
147
148 static pthread_key_t cpu_subset_key;
149 static pthread_once_t cpu_subset_once = PTHREAD_ONCE_INIT;
150
151 17 static void cpu_subset_destructor(void *p) {
152
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 17 times.
 17 ensure(p == _cpu_subset, "pthread key does not match _cpu_subset");
153 17 array_cpuid_t_destroy(_cpu_subset);
154 17 free(_cpu_subset);
155 17 _cpu_subset = NULL;
156 17 }
157
158 10 static void make_cpu_subset_key(void) {
159 /* Associate key with destructor, all worker threads will call it on pthread_exit */
160 10 pthread_key_create(&cpu_subset_key, cpu_subset_destructor);
161 10 }
162
163 135 static inline array_cpuid_t* get_cpu_subset(const subprocess_descriptor_t *spd) {
164 /* Thread already has an allocated array, return it */
165
2/2
✓ Branch 0 taken 118 times.
✓ Branch 1 taken 17 times.
 135 if (likely(_cpu_subset != NULL)) {
166 118 array_cpuid_t_clear(_cpu_subset);
167 118 return _cpu_subset;
168 }
169
170 /* Otherwise, allocate */
171 17 _cpu_subset = malloc(sizeof(array_cpuid_t));
172 17 array_cpuid_t_init(_cpu_subset, node_size);
173
174 /* Associate pointer to key to clean up array on thread destruction */
175 17 pthread_once(&cpu_subset_once, make_cpu_subset_key);
176 17 pthread_setspecific(cpu_subset_key, _cpu_subset);
177
178 17 return _cpu_subset;
179 }
180
181 /* Array of cpuinfo tasks. For enabling or disabling CPUs. */
182 static __thread array_cpuinfo_task_t *_tasks = NULL;
183
184 static pthread_key_t tasks_key;
185 static pthread_once_t tasks_once = PTHREAD_ONCE_INIT;
186
187 40 static void tasks_destructor(void *p) {
188
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 40 times.
 40 ensure(p == _tasks, "pthread key does not match _tasks");
189 40 array_cpuinfo_task_t_destroy(_tasks);
190 40 free(_tasks);
191 40 _tasks = NULL;
192 40 }
193
194 15 static void make_tasks_key(void) {
195 /* Associate key with destructor, all worker threads will call it on pthread_exit */
196 15 pthread_key_create(&tasks_key, tasks_destructor);
197 15 }
198
199 397 static inline array_cpuinfo_task_t* get_tasks(const subprocess_descriptor_t *spd) {
200 /* Thread already has an allocated array, return it */
201
2/2
✓ Branch 0 taken 357 times.
✓ Branch 1 taken 40 times.
 397 if (likely(_tasks != NULL)) {
202 357 array_cpuinfo_task_t_clear(_tasks);
203 357 return _tasks;
204 }
205
206 /* Otherwise, allocate */
207 40 _tasks = malloc(sizeof(array_cpuinfo_task_t));
208 40 array_cpuinfo_task_t_init(_tasks, node_size*2);
209
210 /* Associate pointer to key to clean up structure on thread destruction */
211 40 pthread_once(&tasks_once, make_tasks_key);
212 40 pthread_setspecific(tasks_key, _tasks);
213
214 40 return _tasks;
215 }
216
217
218 /*********************************************************************************/
219 /* Resolve cpuinfo tasks */
220 /*********************************************************************************/
221
222 290 static void resolve_cpuinfo_tasks(const subprocess_descriptor_t *restrict spd,
223 array_cpuinfo_task_t *restrict tasks) {
224
225 290 size_t tasks_count = tasks->count;
226
227 /* We don't need it strictly sorted, but if there are 3 or more tasks
228 * we need to group them by pid */
229
2/2
✓ Branch 0 taken 31 times.
✓ Branch 1 taken 259 times.
 290 if (tasks_count > 2 ) {
230 31 array_cpuinfo_task_t_sort(tasks);
231 }
232
233 /* Iterate tasks by group of PIDs */
234
2/2
✓ Branch 0 taken 210 times.
✓ Branch 1 taken 290 times.
 500 for (size_t i=0; i<tasks_count; ) {
235
236 /* count how many times this PID is repeated */
237 210 size_t j = i+1;
238 210 while (j < tasks_count
239
4/4
✓ Branch 0 taken 148 times.
✓ Branch 1 taken 178 times.
✓ Branch 2 taken 116 times.
✓ Branch 3 taken 32 times.
 326 && tasks->items[j].pid == tasks->items[i].pid) {
240 116 ++j;
241 }
242 210 size_t num_tasks = j - i;
243
244
2/2
✓ Branch 0 taken 139 times.
✓ Branch 1 taken 71 times.
 210 if (num_tasks == 1) {
245 /* resolve single task */
246 139 const cpuinfo_task_t *task = &tasks->items[i];
247
2/2
✓ Branch 0 taken 115 times.
✓ Branch 1 taken 24 times.
 139 if (task->pid == spd->id) {
248
2/2
✓ Branch 0 taken 98 times.
✓ Branch 1 taken 17 times.
 115 if (task->action == ENABLE_CPU) {
249
2/2
✓ Branch 0 taken 56 times.
✓ Branch 1 taken 42 times.
 98 verbose(VB_MICROLB, "Enabling CPU %d", task->cpuid);
250 98 enable_cpu(&spd->pm, task->cpuid);
251 }
252
1/2
✓ Branch 0 taken 17 times.
✗ Branch 1 not taken.
 17 else if (task->action == DISABLE_CPU) {
253
2/2
✓ Branch 0 taken 6 times.
✓ Branch 1 taken 11 times.
 17 verbose(VB_MICROLB, "Disabling CPU %d", task->cpuid);
254 17 disable_cpu(&spd->pm, task->cpuid);
255 }
256 }
257
2/2
✓ Branch 0 taken 17 times.
✓ Branch 1 taken 7 times.
 24 else if (spd->options.mode == MODE_ASYNC) {
258
2/2
✓ Branch 0 taken 8 times.
✓ Branch 1 taken 9 times.
 17 if (task->action == ENABLE_CPU) {
259
2/2
✓ Branch 0 taken 2 times.
✓ Branch 1 taken 6 times.
 8 verbose(VB_MICROLB, "Requesting process %d to enable CPU %d",
260 task->pid, task->cpuid);
261 8 shmem_async_enable_cpu(task->pid, task->cpuid);
262 }
263
1/2
✓ Branch 0 taken 9 times.
✗ Branch 1 not taken.
 9 else if (task->action == DISABLE_CPU) {
264
2/2
✓ Branch 0 taken 6 times.
✓ Branch 1 taken 3 times.
 9 verbose(VB_MICROLB, "Requesting process %d to disable CPU %d",
265 task->pid, task->cpuid);
266 9 shmem_async_disable_cpu(task->pid, task->cpuid);
267 9 shmem_cpuinfo__return_async_cpu(task->pid, task->cpuid);
268 }
269 }
270 } else {
271 /* group tasks */
272 71 cpu_set_t cpus_to_enable = {};
273 71 cpu_set_t cpus_to_disable = {};
274
2/2
✓ Branch 0 taken 187 times.
✓ Branch 1 taken 71 times.
 258 for (size_t k = i; k < i+num_tasks; ++k) {
275 187 const cpuinfo_task_t *task = &tasks->items[k];
276
2/2
✓ Branch 0 taken 143 times.
✓ Branch 1 taken 44 times.
 187 if (task->action == ENABLE_CPU) {
277
1/2
✓ Branch 0 taken 143 times.
✗ Branch 1 not taken.
 143 CPU_SET(task->cpuid, &cpus_to_enable);
278 }
279
1/2
✓ Branch 0 taken 44 times.
✗ Branch 1 not taken.
 44 else if (task->action == DISABLE_CPU) {
280
1/2
✓ Branch 0 taken 44 times.
✗ Branch 1 not taken.
 44 CPU_SET(task->cpuid, &cpus_to_disable);
281 }
282 }
283
284 /* resolve group of tasks for the same PID */
285 71 const cpuinfo_task_t *task = &tasks->items[i];
286
2/2
✓ Branch 0 taken 51 times.
✓ Branch 1 taken 20 times.
 71 if (task->pid == spd->id) {
287
2/2
✓ Branch 1 taken 48 times.
✓ Branch 2 taken 3 times.
 51 if (CPU_COUNT(&cpus_to_enable) > 0) {
288
2/2
✓ Branch 0 taken 27 times.
✓ Branch 1 taken 21 times.
 48 verbose(VB_MICROLB, "Enabling CPUs %s", mu_to_str(&cpus_to_enable));
289 48 enable_cpu_set(&spd->pm, &cpus_to_enable);
290 }
291
2/2
✓ Branch 1 taken 3 times.
✓ Branch 2 taken 48 times.
 51 if (CPU_COUNT(&cpus_to_disable) > 0) {
292
2/2
✓ Branch 0 taken 2 times.
✓ Branch 1 taken 1 times.
 3 verbose(VB_MICROLB, "Disabling CPUs %s", mu_to_str(&cpus_to_disable));
293 3 disable_cpu_set(&spd->pm, &cpus_to_disable);
294 }
295 }
296
2/2
✓ Branch 0 taken 11 times.
✓ Branch 1 taken 9 times.
 20 else if (spd->options.mode == MODE_ASYNC) {
297
2/2
✓ Branch 1 taken 4 times.
✓ Branch 2 taken 7 times.
 11 if (CPU_COUNT(&cpus_to_enable) > 0) {
298
2/2
✓ Branch 0 taken 2 times.
✓ Branch 1 taken 2 times.
 4 verbose(VB_MICROLB, "Requesting process %d to enable CPUs %s",
299 task->pid, mu_to_str(&cpus_to_enable));
300 4 shmem_async_enable_cpu_set(task->pid, &cpus_to_enable);
301 }
302
2/2
✓ Branch 1 taken 7 times.
✓ Branch 2 taken 4 times.
 11 if (CPU_COUNT(&cpus_to_disable) > 0) {
303
2/2
✓ Branch 0 taken 5 times.
✓ Branch 1 taken 2 times.
 7 verbose(VB_MICROLB, "Requesting process %d to disable CPUs %s",
304 task->pid, mu_to_str(&cpus_to_disable));
305 7 shmem_async_disable_cpu_set(task->pid, &cpus_to_disable);
306 7 shmem_cpuinfo__return_async_cpu_mask(task->pid, &cpus_to_disable);
307 }
308 }
309 }
310
311 210 i += num_tasks;
312 }
313 290 }
314
315
316 /*********************************************************************************/
317 /* Init / Finalize */
318 /*********************************************************************************/
319
320 40 int lewi_mask_Init(subprocess_descriptor_t *spd) {
321 /* Value is always updated to allow testing different node sizes */
322 40 node_size = mu_get_system_size();
323
324 /* Allocate and initialize private structure */
325 40 spd->lewi_info = malloc(sizeof(lewi_info_t));
326 40 lewi_info_t *lewi_info = spd->lewi_info;
327 40 *lewi_info = (const lewi_info_t) {
328 40 .max_parallelism = spd->options.lewi_max_parallelism,
329 .mutex = PTHREAD_MUTEX_INITIALIZER,
330 };
331 40 array_cpuid_t_init(&lewi_info->cpus_priority_array, node_size);
332 40 lewi_mask_UpdateOwnershipInfo(spd, &spd->process_mask);
333
334 /* Enable request queues only in async mode */
335
2/2
✓ Branch 0 taken 16 times.
✓ Branch 1 taken 24 times.
 40 if (spd->options.mode == MODE_ASYNC) {
336 16 shmem_cpuinfo__enable_request_queues();
337 }
338
339 40 return DLB_SUCCESS;
340 }
341
342 40 int lewi_mask_Finalize(subprocess_descriptor_t *spd) {
343 /* De-register subprocess from the shared memory */
344 40 array_cpuinfo_task_t *tasks = get_tasks(spd);
345 40 int error = shmem_cpuinfo__deregister(spd->id, tasks);
346
1/2
✓ Branch 0 taken 40 times.
✗ Branch 1 not taken.
 40 if (error == DLB_SUCCESS) {
347 40 resolve_cpuinfo_tasks(spd, tasks);
348 }
349
350 /* Deallocate main thread arrays (worker threads do it on thread_exit) */
351
2/2
✓ Branch 0 taken 17 times.
✓ Branch 1 taken 23 times.
 40 if (_cpu_subset != NULL) cpu_subset_destructor(_cpu_subset);
352
1/2
✓ Branch 0 taken 40 times.
✗ Branch 1 not taken.
 40 if (_tasks != NULL) tasks_destructor(_tasks);
353
354 /* Deallocate private structure */
355 40 lewi_info_t *lewi_info = spd->lewi_info;
356 40 array_cpuid_t_destroy(&lewi_info->cpus_priority_array);
357 40 free(lewi_info);
358 40 lewi_info = NULL;
359
360 40 return (error >= 0) ? DLB_SUCCESS : error;
361 }
362
363
364 /*********************************************************************************/
365 /* LeWI Modes (enable/disable, max_parallelism, ...) */
366 /*********************************************************************************/
367
368 5 int lewi_mask_EnableDLB(const subprocess_descriptor_t *spd) {
369 /* Reset value of last_borrow */
370 5 ((lewi_info_t*)spd->lewi_info)->last_borrow = 0;
371 5 return DLB_SUCCESS;
372 }
373
374 5 int lewi_mask_DisableDLB(const subprocess_descriptor_t *spd) {
375 5 array_cpuinfo_task_t *tasks = get_tasks(spd);
376 5 int error = shmem_cpuinfo__reset(spd->id, tasks);
377
1/2
✓ Branch 0 taken 5 times.
✗ Branch 1 not taken.
 5 if (error == DLB_SUCCESS) {
378 5 resolve_cpuinfo_tasks(spd, tasks);
379 }
380 5 return error;
381 }
382
383 4 int lewi_mask_SetMaxParallelism(const subprocess_descriptor_t *spd, int max) {
384 4 int error = DLB_SUCCESS;
385
1/2
✓ Branch 0 taken 4 times.
✗ Branch 1 not taken.
 4 if (max > 0) {
386 4 lewi_info_t *lewi_info = spd->lewi_info;
387 4 lewi_info->max_parallelism = max;
388 4 array_cpuinfo_task_t *tasks = get_tasks(spd);
389 4 error = shmem_cpuinfo__update_max_parallelism(spd->id, max, tasks);
390
1/2
✓ Branch 0 taken 4 times.
✗ Branch 1 not taken.
 4 if (error == DLB_SUCCESS) {
391 4 resolve_cpuinfo_tasks(spd, tasks);
392 }
393 }
394 4 return error;
395 }
396
397 4 int lewi_mask_UnsetMaxParallelism(const subprocess_descriptor_t *spd) {
398 4 lewi_info_t *lewi_info = spd->lewi_info;
399 4 lewi_info->max_parallelism = 0;
400 4 return DLB_SUCCESS;
401 }
402
403
404 /*********************************************************************************/
405 /* MPI */
406 /*********************************************************************************/
407
408 /* Obtain thread mask and remove first core if keep_cpu_on_blocking_call */
409 28 static inline void get_mask_for_blocking_call(
410 cpu_set_t *cpu_set, bool keep_cpu_on_blocking_call) {
411
412 /* Obtain current thread's affinity mask */
413 28 pthread_getaffinity_np(pthread_self(), sizeof(cpu_set_t), cpu_set);
414
415
2/2
✓ Branch 0 taken 4 times.
✓ Branch 1 taken 24 times.
 28 if (keep_cpu_on_blocking_call) {
416 /* Remove the first core */
417 4 int first_cpuid = mu_get_first_cpu(cpu_set);
418 4 const mu_cpuset_t *first_core = mu_get_core_mask(first_cpuid);
419 4 mu_subtract(cpu_set, cpu_set, first_core->set);
420 }
421 28 }
422
423 /* Lend the CPUs of the thread encountering the blocking call */
424 14 int lewi_mask_IntoBlockingCall(const subprocess_descriptor_t *spd) {
425
426 14 lewi_info_t *lewi_info = spd->lewi_info;
427
428 /* Obtain affinity mask to lend */
429 cpu_set_t cpu_set;
430 14 get_mask_for_blocking_call(&cpu_set,
431 14 spd->options.lewi_keep_cpu_on_blocking_call);
432
433
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 14 times.
 14 if (unlikely(mu_intersects(&lewi_info->in_mpi_cpus, &cpu_set))) {
434 #ifdef DEBUG_VERSION
435 ✗ fatal("Some CPU in %s already into blocking call", mu_to_str(&cpu_set));
436 #else
437 /* This scenario is not expected to occur; if it does, simply ignore. */
438 return DLB_NOUPDT;
439 #endif
440 }
441
442 14 int error = DLB_NOUPDT;
443
444
2/2
✓ Branch 1 taken 12 times.
✓ Branch 2 taken 2 times.
 14 if (mu_count(&cpu_set) > 0) {
445
446 /* Add cpu_set to in_mpi_cpus */
447 12 mu_or(&lewi_info->in_mpi_cpus, &lewi_info->in_mpi_cpus, &cpu_set);
448
449
1/2
✓ Branch 0 taken 12 times.
✗ Branch 1 not taken.
 12 verbose(VB_MICROLB, "In blocking call, lending %s", mu_to_str(&cpu_set));
450
451 /* Finally, lend mask */
452 12 error = lewi_mask_LendCpuMask(spd, &cpu_set);
453 }
454 14 return error;
455 }
456
457 /* Reclaim the CPUs that were lent when encountering the blocking call.
458 * The thread must have not change its affinity mask since then. */
459 14 int lewi_mask_OutOfBlockingCall(const subprocess_descriptor_t *spd) {
460
461 14 int error = DLB_NOUPDT;
462 14 lewi_info_t *lewi_info = spd->lewi_info;
463
464 /* Obtain affinity mask to lend */
465 cpu_set_t cpu_set;
466 14 get_mask_for_blocking_call(&cpu_set,
467 14 spd->options.lewi_keep_cpu_on_blocking_call);
468
469
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 14 times.
 14 if (unlikely(!mu_is_subset(&lewi_info->in_mpi_cpus, &cpu_set))) {
470 #ifdef DEBUG_VERSION
471 ✗ fatal("Some CPU in %s is not into blocking call", mu_to_str(&cpu_set));
472 #else
473 /* This scenario is not expected to occur; if it does, simply ignore. */
474 return DLB_NOUPDT;
475 #endif
476 }
477
478
2/2
✓ Branch 1 taken 12 times.
✓ Branch 2 taken 2 times.
 14 if (mu_count(&lewi_info->in_mpi_cpus) > 0) {
479
480 /* Clear cpu_set from in_mpi_cpus */
481 12 mu_subtract(&lewi_info->in_mpi_cpus, &lewi_info->in_mpi_cpus, &cpu_set);
482
483
1/2
✓ Branch 0 taken 12 times.
✗ Branch 1 not taken.
 12 verbose(VB_MICROLB, "Out of blocking call, acquiring %s", mu_to_str(&cpu_set));
484
485 /* If there are owned CPUs to reclaim: */
486 cpu_set_t owned_cpus;
487 12 mu_and(&owned_cpus, &cpu_set, &spd->process_mask);
488
2/2
✓ Branch 1 taken 6 times.
✓ Branch 2 taken 6 times.
 12 if (mu_count(&owned_cpus) > 0) {
489
490 /* Construct a CPU array from owned_cpus */
491 6 array_cpuid_t *cpu_subset = get_cpu_subset(spd);
492 6 cpu_array_from_cpuset(cpu_subset, &owned_cpus);
493
494 /* Acquire CPUs */
495 6 array_cpuinfo_task_t *tasks = get_tasks(spd);
496 6 error = shmem_cpuinfo__acquire_from_cpu_subset(spd->id, cpu_subset, tasks);
497
498 /* Resolve tasks: Even if callbacks weren't called during IntoBlockingCall,
499 * other LeWI actions may have been called after it so we cannot ignore
500 * pending actions */
501
3/4
✓ Branch 0 taken 3 times.
✓ Branch 1 taken 3 times.
✓ Branch 2 taken 3 times.
✗ Branch 3 not taken.
 6 if (error == DLB_SUCCESS || error == DLB_NOTED) {
502 6 resolve_cpuinfo_tasks(spd, tasks);
503 }
504 }
505
506 /* If there are ONLY non-owned CPUs to reclaim:
507 * (if some owned CPU was already reclaimed, we forget about non-owned) */
508 cpu_set_t non_owned_cpus;
509 12 mu_subtract(&non_owned_cpus, &cpu_set, &spd->process_mask);
510
2/2
✓ Branch 1 taken 6 times.
✓ Branch 2 taken 6 times.
 12 if (mu_count(&non_owned_cpus) > 0
511
1/2
✓ Branch 1 taken 6 times.
✗ Branch 2 not taken.
 6 && mu_count(&owned_cpus) == 0) {
512
513 /* Remove CPUs that are disabled in the shmem */
514 6 for (int cpuid = mu_get_first_cpu(&non_owned_cpus);
515
3/4
✓ Branch 0 taken 6 times.
✓ Branch 1 taken 6 times.
✓ Branch 2 taken 6 times.
✗ Branch 3 not taken.
 12 cpuid >= 0 && cpuid != DLB_CPUID_INVALID;
516 6 cpuid = mu_get_next_cpu(&non_owned_cpus, cpuid)) {
517
2/2
✓ Branch 1 taken 2 times.
✓ Branch 2 taken 4 times.
 6 if (!shmem_cpuinfo__is_cpu_enabled(cpuid)) {
518
1/2
✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
 2 CPU_CLR(cpuid, &non_owned_cpus);
519 }
520 }
521
522 /* Construct a CPU array from non_owned_cpus */
523 6 array_cpuid_t *cpu_subset = get_cpu_subset(spd);
524 6 cpu_array_from_cpuset(cpu_subset, &non_owned_cpus);
525
526 /* Borrow CPUs, but also ignoring the enable callbacks */
527 6 array_cpuinfo_task_t *tasks = get_tasks(spd);
528 6 error = shmem_cpuinfo__borrow_from_cpu_subset(spd->id, cpu_subset, tasks);
529
530 /* A non-success error means that not all CPUs could be borrowed */
531
2/2
✓ Branch 0 taken 4 times.
✓ Branch 1 taken 2 times.
 6 if (error != DLB_SUCCESS) {
532 /* Annotate the CPU as pending to bypass the shared memory for the next query */
533 4 mu_or(&lewi_info->pending_reclaimed_cpus, &lewi_info->pending_reclaimed_cpus,
534 &non_owned_cpus);
535
536 /* We lost the CPU while in MPI.
537 * In async mode, we can just disable the CPU.
538 * In polling mode, the process needs to call Return */
539
2/2
✓ Branch 0 taken 2 times.
✓ Branch 1 taken 2 times.
 4 if (spd->options.mode == MODE_ASYNC) {
540 2 disable_cpu_set(&spd->pm, &non_owned_cpus);
541 }
542 }
543 }
544 }
545
546 14 return error;
547 }
548
549
550 /*********************************************************************************/
551 /* Lend */
552 /*********************************************************************************/
553
554 2 int lewi_mask_Lend(const subprocess_descriptor_t *spd) {
555 cpu_set_t mask;
556 2 mu_get_system_mask(&mask);
557 2 return lewi_mask_LendCpuMask(spd, &mask);
558 }
559
560 83 int lewi_mask_LendCpu(const subprocess_descriptor_t *spd, int cpuid) {
561 83 array_cpuinfo_task_t *tasks = get_tasks(spd);
562 83 int error = shmem_cpuinfo__lend_cpu(spd->id, cpuid, tasks);
563
564
1/2
✓ Branch 0 taken 83 times.
✗ Branch 1 not taken.
 83 if (error == DLB_SUCCESS) {
565
2/2
✓ Branch 0 taken 44 times.
✓ Branch 1 taken 39 times.
 83 if (spd->options.mode == MODE_ASYNC) {
566 44 resolve_cpuinfo_tasks(spd, tasks);
567 }
568
569 /* Clear possible pending reclaimed CPUs */
570 83 lewi_info_t *lewi_info = spd->lewi_info;
571
1/2
✓ Branch 0 taken 83 times.
✗ Branch 1 not taken.
 83 CPU_CLR(cpuid, &lewi_info->pending_reclaimed_cpus);
572 }
573 83 return error;
574 }
575
576 51 int lewi_mask_LendCpuMask(const subprocess_descriptor_t *spd, const cpu_set_t *mask) {
577 51 array_cpuinfo_task_t *tasks = get_tasks(spd);
578 51 int error = shmem_cpuinfo__lend_cpu_mask(spd->id, mask, tasks);
579
1/2
✓ Branch 0 taken 51 times.
✗ Branch 1 not taken.
 51 if (error == DLB_SUCCESS) {
580
2/2
✓ Branch 0 taken 26 times.
✓ Branch 1 taken 25 times.
 51 if (spd->options.mode == MODE_ASYNC) {
581 26 resolve_cpuinfo_tasks(spd, tasks);
582 }
583
584 /* Clear possible pending reclaimed CPUs */
585 51 lewi_info_t *lewi_info = spd->lewi_info;
586 51 mu_subtract(&lewi_info->pending_reclaimed_cpus,
587 51 &lewi_info->pending_reclaimed_cpus, mask);
588 }
589 51 return error;
590 }
591
592
593 /*********************************************************************************/
594 /* Reclaim */
595 /*********************************************************************************/
596
597 6 int lewi_mask_Reclaim(const subprocess_descriptor_t *spd) {
598 /* Even though shmem_cpuinfo__reclaim_all exists,
599 * iterating the mask should be more efficient */
600 6 return lewi_mask_ReclaimCpuMask(spd, &spd->process_mask);
601 }
602
603 8 int lewi_mask_ReclaimCpu(const subprocess_descriptor_t *spd, int cpuid) {
604 8 array_cpuinfo_task_t *tasks = get_tasks(spd);
605 8 int error = shmem_cpuinfo__reclaim_cpu(spd->id, cpuid, tasks);
606
4/4
✓ Branch 0 taken 7 times.
✓ Branch 1 taken 1 times.
✓ Branch 2 taken 6 times.
✓ Branch 3 taken 1 times.
 8 if (error == DLB_SUCCESS || error == DLB_NOTED) {
607 7 resolve_cpuinfo_tasks(spd, tasks);
608 }
609 8 return error;
610 }
611
612 ✗ int lewi_mask_ReclaimCpus(const subprocess_descriptor_t *spd, int ncpus) {
613 ✗ array_cpuinfo_task_t *tasks = get_tasks(spd);
614 ✗ int error = shmem_cpuinfo__reclaim_cpus(spd->id, ncpus, tasks);
615 ✗ if (error == DLB_SUCCESS || error == DLB_NOTED) {
616 ✗ resolve_cpuinfo_tasks(spd, tasks);
617 }
618 ✗ return error;
619 }
620
621 17 int lewi_mask_ReclaimCpuMask(const subprocess_descriptor_t *spd, const cpu_set_t *mask) {
622 17 array_cpuinfo_task_t *tasks = get_tasks(spd);
623 17 int error = shmem_cpuinfo__reclaim_cpu_mask(spd->id, mask, tasks);
624 17 resolve_cpuinfo_tasks(spd, tasks);
625 17 return error;
626 }
627
628
629 /*********************************************************************************/
630 /* Acquire */
631 /*********************************************************************************/
632
633 29 int lewi_mask_AcquireCpu(const subprocess_descriptor_t *spd, int cpuid) {
634 29 array_cpuinfo_task_t *tasks = get_tasks(spd);
635 29 int error = shmem_cpuinfo__acquire_cpu(spd->id, cpuid, tasks);
636
4/4
✓ Branch 0 taken 12 times.
✓ Branch 1 taken 17 times.
✓ Branch 2 taken 9 times.
✓ Branch 3 taken 3 times.
 29 if (error == DLB_SUCCESS || error == DLB_NOTED) {
637 26 resolve_cpuinfo_tasks(spd, tasks);
638 }
639 29 return error;
640 }
641
642 65 int lewi_mask_AcquireCpus(const subprocess_descriptor_t *spd, int ncpus) {
643 65 int error = DLB_NOUPDT;
644
2/2
✓ Branch 0 taken 9 times.
✓ Branch 1 taken 56 times.
 65 if (ncpus == 0) {
645 /* AcquireCPUs(0) has a special meaning of removing any previous request */
646 9 shmem_cpuinfo__remove_requests(spd->id);
647 9 error = DLB_SUCCESS;
648
1/2
✓ Branch 0 taken 56 times.
✗ Branch 1 not taken.
 56 } else if (ncpus > 0) {
649 56 error = lewi_mask_AcquireCpusInMask(spd, ncpus, NULL);
650 }
651 65 return error;
652 }
653
654 22 int lewi_mask_AcquireCpuMask(const subprocess_descriptor_t *spd, const cpu_set_t *mask) {
655 22 return lewi_mask_AcquireCpusInMask(spd, 0, mask);
656 }
657
658 93 int lewi_mask_AcquireCpusInMask(const subprocess_descriptor_t *spd, int ncpus,
659 const cpu_set_t *mask) {
660 93 lewi_info_t *lewi_info = spd->lewi_info;
661 93 bool async = spd->options.mode == MODE_ASYNC;
662
2/2
✓ Branch 0 taken 36 times.
✓ Branch 1 taken 57 times.
 93 int64_t *last_borrow = async ? NULL : &lewi_info->last_borrow;
663
664 /* Construct a CPU array based on cpus_priority_array and mask (if present) */
665 93 array_cpuid_t *cpu_subset = get_cpu_subset(spd);
666 93 cpu_array_and(cpu_subset, &lewi_info->cpus_priority_array, mask);
667
668 /* Provide a number of requested CPUs only if needed */
669
2/2
✓ Branch 0 taken 71 times.
✓ Branch 1 taken 22 times.
 93 int *requested_ncpus = ncpus > 0 ? &ncpus : NULL;
670
671 93 array_cpuinfo_task_t *tasks = get_tasks(spd);
672 93 int error = shmem_cpuinfo__acquire_ncpus_from_cpu_subset(spd->id,
673 93 requested_ncpus, cpu_subset, spd->options.lewi_affinity,
674 lewi_info->max_parallelism, last_borrow, tasks);
675
676
2/2
✓ Branch 0 taken 73 times.
✓ Branch 1 taken 20 times.
 93 if (error != DLB_NOUPDT) {
677 73 resolve_cpuinfo_tasks(spd, tasks);
678 }
679 93 return error;
680 }
681
682
683 /*********************************************************************************/
684 /* Borrow */
685 /*********************************************************************************/
686
687 8 int lewi_mask_Borrow(const subprocess_descriptor_t *spd) {
688 cpu_set_t system_mask;
689 8 mu_get_system_mask(&system_mask);
690 8 return lewi_mask_BorrowCpusInMask(spd, 0, &system_mask);
691 }
692
693 10 int lewi_mask_BorrowCpu(const subprocess_descriptor_t *spd, int cpuid) {
694 10 array_cpuinfo_task_t *tasks = get_tasks(spd);
695 10 int error = shmem_cpuinfo__borrow_cpu(spd->id, cpuid, tasks);
696
1/2
✓ Branch 0 taken 10 times.
✗ Branch 1 not taken.
 10 if (error == DLB_SUCCESS) {
697 10 resolve_cpuinfo_tasks(spd, tasks);
698 }
699 10 return error;
700 }
701
702 14 int lewi_mask_BorrowCpus(const subprocess_descriptor_t *spd, int ncpus) {
703 14 return lewi_mask_BorrowCpusInMask(spd, ncpus, NULL);
704 }
705
706 ✗ int lewi_mask_BorrowCpuMask(const subprocess_descriptor_t *spd, const cpu_set_t *mask) {
707 ✗ return lewi_mask_BorrowCpusInMask(spd, 0, mask);
708 }
709
710 30 int lewi_mask_BorrowCpusInMask(const subprocess_descriptor_t *spd, int ncpus, const cpu_set_t *mask) {
711 30 lewi_info_t *lewi_info = spd->lewi_info;
712 30 bool async = spd->options.mode == MODE_ASYNC;
713
2/2
✓ Branch 0 taken 15 times.
✓ Branch 1 taken 15 times.
 30 int64_t *last_borrow = async ? NULL : &lewi_info->last_borrow;
714
715 /* Construct a CPU array based on cpus_priority_array and mask (if present) */
716 30 array_cpuid_t *cpu_subset = get_cpu_subset(spd);
717 30 cpu_array_and(cpu_subset, &lewi_info->cpus_priority_array, mask);
718
719 /* Provide a number of requested CPUs only if needed */
720
2/2
✓ Branch 0 taken 22 times.
✓ Branch 1 taken 8 times.
 30 int *requested_ncpus = ncpus > 0 ? &ncpus : NULL;
721
722 30 array_cpuinfo_task_t *tasks = get_tasks(spd);
723 30 int error = shmem_cpuinfo__borrow_ncpus_from_cpu_subset(spd->id,
724 30 requested_ncpus, cpu_subset, spd->options.lewi_affinity,
725 lewi_info->max_parallelism, last_borrow, tasks);
726
727
2/2
✓ Branch 0 taken 18 times.
✓ Branch 1 taken 12 times.
 30 if (error == DLB_SUCCESS) {
728 18 resolve_cpuinfo_tasks(spd, tasks);
729 }
730
731 30 return error;
732 }
733
734
735 /*********************************************************************************/
736 /* Return */
737 /*********************************************************************************/
738
739 1 int lewi_mask_Return(const subprocess_descriptor_t *spd) {
740
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1 times.
 1 if (spd->options.mode == MODE_ASYNC) {
741 // Return should not be called in async mode
742 ✗ return DLB_ERR_NOCOMP;
743 }
744
745 1 array_cpuinfo_task_t *tasks = get_tasks(spd);
746 1 int error = shmem_cpuinfo__return_all(spd->id, tasks);
747 1 resolve_cpuinfo_tasks(spd, tasks);
748
749 /* Check possible pending reclaimed CPUs */
750 1 lewi_info_t *lewi_info = spd->lewi_info;
751 1 int cpuid = mu_get_first_cpu(&lewi_info->pending_reclaimed_cpus);
752
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1 times.
 1 if (cpuid >= 0) {
753 do {
754 ✗ disable_cpu(&spd->pm, cpuid);
755 ✗ cpuid = mu_get_next_cpu(&lewi_info->pending_reclaimed_cpus, cpuid);
756 ✗ } while (cpuid >= 0 && cpuid != DLB_CPUID_INVALID);
757 ✗ CPU_ZERO(&lewi_info->pending_reclaimed_cpus);
758 ✗ error = DLB_SUCCESS;
759 }
760
761 1 return error;
762 }
763
764 21 int lewi_mask_ReturnCpu(const subprocess_descriptor_t *spd, int cpuid) {
765
2/2
✓ Branch 0 taken 7 times.
✓ Branch 1 taken 14 times.
 21 if (spd->options.mode == MODE_ASYNC) {
766 // Return should not be called in async mode
767 7 return DLB_ERR_NOCOMP;
768 }
769
770 14 array_cpuinfo_task_t *tasks = get_tasks(spd);
771 14 int error = shmem_cpuinfo__return_cpu(spd->id, cpuid, tasks);
772
3/4
✓ Branch 0 taken 1 times.
✓ Branch 1 taken 13 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 1 times.
 14 if (error == DLB_SUCCESS || error == DLB_ERR_REQST) {
773 13 resolve_cpuinfo_tasks(spd, tasks);
774
1/2
✓ Branch 0 taken 1 times.
✗ Branch 1 not taken.
 1 } else if (error == DLB_ERR_PERM) {
775 /* Check possible pending reclaimed CPUs */
776 1 lewi_info_t *lewi_info = spd->lewi_info;
777
3/6
✓ Branch 0 taken 1 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 1 times.
✗ Branch 5 not taken.
 1 if (CPU_ISSET(cpuid, &lewi_info->pending_reclaimed_cpus)) {
778
1/2
✓ Branch 0 taken 1 times.
✗ Branch 1 not taken.
 1 CPU_CLR(cpuid, &lewi_info->pending_reclaimed_cpus);
779 1 disable_cpu(&spd->pm, cpuid);
780 1 error = DLB_SUCCESS;
781 }
782 }
783 14 return error;
784 }
785
786 ✗ int lewi_mask_ReturnCpuMask(const subprocess_descriptor_t *spd, const cpu_set_t *mask) {
787 ✗ if (spd->options.mode == MODE_ASYNC) {
788 // Return should not be called in async mode
789 ✗ return DLB_ERR_NOCOMP;
790 }
791
792 ✗ array_cpuinfo_task_t *tasks = get_tasks(spd);
793 ✗ int error = shmem_cpuinfo__return_cpu_mask(spd->id, mask, tasks);
794 ✗ resolve_cpuinfo_tasks(spd, tasks);
795
796 /* Check possible pending reclaimed CPUs */
797 ✗ lewi_info_t *lewi_info = spd->lewi_info;
798 ✗ if (CPU_COUNT(&lewi_info->pending_reclaimed_cpus) > 0) {
799 cpu_set_t cpus_to_return;
800 ✗ CPU_AND(&cpus_to_return, &lewi_info->pending_reclaimed_cpus, mask);
801 ✗ for (int cpuid = mu_get_first_cpu(&cpus_to_return);
802 ✗ cpuid >= 0 && cpuid != DLB_CPUID_INVALID;
803 ✗ cpuid = mu_get_next_cpu(&cpus_to_return, cpuid)) {
804 ✗ disable_cpu(&spd->pm, cpuid);
805 }
806 ✗ mu_subtract(&lewi_info->pending_reclaimed_cpus,
807 ✗ &lewi_info->pending_reclaimed_cpus, &cpus_to_return);
808 ✗ error = DLB_SUCCESS;
809 }
810
811 ✗ return error;
812 }
813
814
815 // Others
816
817 32 int lewi_mask_CheckCpuAvailability(const subprocess_descriptor_t *spd, int cpuid) {
818 32 return shmem_cpuinfo__check_cpu_availability(spd->id, cpuid);
819 }
820
821 ✗ int lewi_mask_UpdateOwnership(const subprocess_descriptor_t *spd,
822 const cpu_set_t *process_mask) {
823 /* Update priority array */
824 ✗ lewi_mask_UpdateOwnershipInfo(spd, process_mask);
825
826 /* Update cpuinfo data and return reclaimed CPUs */
827 ✗ array_cpuinfo_task_t *tasks = get_tasks(spd);
828 ✗ shmem_cpuinfo__update_ownership(spd->id, process_mask, tasks);
829 ✗ resolve_cpuinfo_tasks(spd, tasks);
830
831 /* Check possible pending reclaimed CPUs */
832 ✗ lewi_info_t *lewi_info = spd->lewi_info;
833 ✗ int cpuid = mu_get_first_cpu(&lewi_info->pending_reclaimed_cpus);
834 ✗ if (cpuid >= 0) {
835 do {
836 ✗ disable_cpu(&spd->pm, cpuid);
837 ✗ cpuid = mu_get_next_cpu(&lewi_info->pending_reclaimed_cpus, cpuid);
838 ✗ } while (cpuid >= 0 && cpuid != DLB_CPUID_INVALID);
839 ✗ CPU_ZERO(&lewi_info->pending_reclaimed_cpus);
840 }
841
842 ✗ return DLB_SUCCESS;
843 }
844