From 8638d9f1228af6e95edb683163949e69519f8a8f Mon Sep 17 00:00:00 2001 From: Victor Lopez Date: Mon, 4 Jun 2018 15:21:22 +0200 Subject: [PATCH] Add SMPProcessor::_bindingList to allow multiple bindings per thread --- src/arch/smp/smpplugin.cpp | 24 ++++++++++++++++-------- src/arch/smp/smpprocessor.cpp | 7 ++++--- src/arch/smp/smpprocessor.hpp | 5 ++++- src/arch/unix-os/pthread.cpp | 11 ++++------- 4 files changed, 28 insertions(+), 19 deletions(-) diff --git a/src/arch/smp/smpplugin.cpp b/src/arch/smp/smpplugin.cpp index d8eae9d82..535caf988 100644 --- a/src/arch/smp/smpplugin.cpp +++ b/src/arch/smp/smpplugin.cpp @@ -193,15 +193,18 @@ nanos::PE * smpProcessorFactory ( int id, int uid ) verbose0("requested cpus: " << _requestedCPUs << " available: " << _availableCPUs << " to be used: " << _currentCPUs); //! \note Fill _bindings vector with the active CPUs first, then the not active + std::map bindings_list; _bindings.reserve( _availableCPUs ); for ( int i = 0; i < _availableCPUs; i++ ) { if ( _cpuProcessMask.isSet(i) ) { _bindings.push_back(i); + bindings_list.insert( std::pair(i, CpuSet(i)) ); } } for ( int i = 0; i < _availableCPUs; i++ ) { if ( !_cpuProcessMask.isSet(i) ) { _bindings.push_back(i); + bindings_list.insert( std::pair(i, CpuSet(i)) ); } } @@ -234,37 +237,42 @@ nanos::PE * smpProcessorFactory ( int id, int uid ) //! \note Create the SMPProcessors in _cpus array int count = 0; for ( std::vector::iterator it = _bindings.begin(); it != _bindings.end(); it++ ) { + int cpuid = *it; SMPProcessor *cpu; - bool active = ( (count < _currentCPUs) && _cpuProcessMask.isSet(*it) ); + bool active = (count < _currentCPUs) && _cpuProcessMask.isSet(cpuid); unsigned numaNode; // If this PE can't be seen by hwloc (weird case in Altix 2, for instance) - if ( !sys._hwloc.isCpuAvailable( *it ) ) { + if ( !sys._hwloc.isCpuAvailable( cpuid ) ) { /* There's a problem: we can't query it's numa node. Let's give it 0 (ticket #1090), consider throwing a warning */ numaNode = 0; } else - numaNode = getNodeOfPE( *it ); + numaNode = getNodeOfPE( cpuid ); unsigned socket = numaNode; /* FIXME: socket */ + memory_space_id_t id; if ( _smpPrivateMemory && count >= _smpHostCpus && !_memkindSupport ) { OSAllocator a; - memory_space_id_t id = sys.addSeparateMemoryAddressSpace( ext::getSMPDevice(), _smpAllocWide, sys.getRegionCacheSlabSize() ); + id = sys.addSeparateMemoryAddressSpace( ext::getSMPDevice(), + _smpAllocWide, sys.getRegionCacheSlabSize() ); SeparateMemoryAddressSpace &numaMem = sys.getSeparateMemory( id ); numaMem.setSpecificData( NEW SimpleAllocator( ( uintptr_t ) a.allocate(_smpPrivateMemorySize), _smpPrivateMemorySize ) ); numaMem.setAcceleratorNumber( sys.getNewAcceleratorId() ); - cpu = NEW SMPProcessor( *it, id, active, numaNode, socket ); } else { - - cpu = NEW SMPProcessor( *it, mem_id, active, numaNode, socket ); + id = mem_id; } + + // Create SMPProcessor object + cpu = NEW SMPProcessor( cpuid, bindings_list[cpuid], id, active, numaNode, socket ); + if ( active ) { _cpuActiveMask.set( cpu->getBindingId() ); } //cpu->setNUMANode( getNodeOfPE( cpu->getId() ) ); (*_cpus)[count] = cpu; - (*_cpusByCpuId)[ *it ] = cpu; + (*_cpusByCpuId)[cpuid] = cpu; count += 1; } diff --git a/src/arch/smp/smpprocessor.cpp b/src/arch/smp/smpprocessor.cpp index 84f2163f1..c68a9bc0d 100644 --- a/src/arch/smp/smpprocessor.cpp +++ b/src/arch/smp/smpprocessor.cpp @@ -35,9 +35,11 @@ size_t SMPProcessor::_threadsStackSize = 0; System::CachePolicyType SMPProcessor::_cachePolicy = System::DEFAULT; size_t SMPProcessor::_cacheDefaultSize = 1048580; -SMPProcessor::SMPProcessor( int bindingId, memory_space_id_t memId, bool active, unsigned int numaNode, unsigned int socket ) : +SMPProcessor::SMPProcessor( int bindingId, const CpuSet& bindingList, + memory_space_id_t memId, bool active, unsigned int numaNode, unsigned int socket ) : PE( &getSMPDevice(), memId, 0 /* always local node */, numaNode, true, socket, true ), - _bindingId( bindingId ), _reserved( false ), _active( active ), _futureThreads( 0 ) {} + _bindingId( bindingId ), _bindingList( bindingList ), + _reserved( false ), _active( active ), _futureThreads( 0 ) {} void SMPProcessor::prepareConfig ( Config &config ) { @@ -134,4 +136,3 @@ void SMPProcessor::setNumFutureThreads( unsigned int nthreads ) { unsigned int SMPProcessor::getNumFutureThreads() const { return _futureThreads; } - diff --git a/src/arch/smp/smpprocessor.hpp b/src/arch/smp/smpprocessor.hpp index 212666ab3..91ab7628f 100644 --- a/src/arch/smp/smpprocessor.hpp +++ b/src/arch/smp/smpprocessor.hpp @@ -43,6 +43,7 @@ namespace ext { static size_t _cacheDefaultSize; static System::CachePolicyType _cachePolicy; unsigned int _bindingId; + CpuSet _bindingList; bool _reserved; bool _active; unsigned int _futureThreads; @@ -54,9 +55,11 @@ namespace ext { public: // constructors - SMPProcessor( int bindingId, memory_space_id_t numMemId, bool active, unsigned int numaNode, unsigned int socket ); + SMPProcessor( int bindingId, const CpuSet& bindingList, memory_space_id_t numMemId, + bool active, unsigned int numaNode, unsigned int socket ); unsigned int getBindingId() const { return _bindingId; } + const CpuSet& getBindingList() const { return _bindingList; } virtual ~SMPProcessor() {} diff --git a/src/arch/unix-os/pthread.cpp b/src/arch/unix-os/pthread.cpp index 46d0ff6e0..af277ebc5 100644 --- a/src/arch/unix-os/pthread.cpp +++ b/src/arch/unix-os/pthread.cpp @@ -113,12 +113,9 @@ void PThread::join () void PThread::bind() { - int cpu_id = _core->getBindingId(); - cpu_set_t cpu_set; - CPU_ZERO( &cpu_set ); - CPU_SET( cpu_id, &cpu_set ); - verbose( "Binding thread " << getMyThreadSafe()->getId() << " to cpu " << cpu_id ); - pthread_setaffinity_np( _pth, sizeof(cpu_set_t), &cpu_set ); + const CpuSet& binding = _core->getBindingList(); + verbose( "Binding thread " << getMyThreadSafe()->getId() << " to cpus [" << binding << "]" ); + pthread_setaffinity_np( _pth, sizeof(cpu_set_t), binding.get_cpu_set_pointer() ); NANOS_INSTRUMENT ( static InstrumentationDictionary *ID = sys.getInstrumentation()->getInstrumentationDictionary(); ) NANOS_INSTRUMENT ( static nanos_event_key_t cpuid_key = ID->getEventKey("cpuid"); ) @@ -129,7 +126,7 @@ void PThread::bind() NANOS_INSTRUMENT ( keys[1] = numa_key ) NANOS_INSTRUMENT ( nanos_event_value_t values[2]; ) - NANOS_INSTRUMENT ( values[0] = (nanos_event_value_t) cpu_id + 1; ) + NANOS_INSTRUMENT ( values[0] = (nanos_event_value_t) _core->getBindingId() + 1; ) NANOS_INSTRUMENT ( values[1] = (nanos_event_value_t) _core->getNumaNode() + 1; ) NANOS_INSTRUMENT ( sys.getInstrumentation()->raisePointEvents(2, keys, values); ) } -- GitLab