score/src/smp.c

0001 /* SPDX-License-Identifier: BSD-2-Clause */
0002
0003 /**
0004  * @file
0005  *
0006  * @ingroup RTEMSScoreSMP
0007  *
0008  * @brief This source file contains the definition of ::_SMP_Online_processors
0009  *   and ::_SMP_Processor_maximum and the implementation of
0010  *   _SMP_Handler_initialize(),  _SMP_Process_message(),
0011  *   _SMP_Request_shutdown(), _SMP_Request_start_multitasking(),
0012  *   _SMP_Send_message(), _SMP_Should_start_processor(),
0013  *   _SMP_Start_multitasking_on_secondary_processor(), and
0014  *   _SMP_Try_to_process_message().
0015  */
0016
0017 /*
0018  *  COPYRIGHT (c) 1989-2011.
0019  *  On-Line Applications Research Corporation (OAR).
0020  *
0021  * Redistribution and use in source and binary forms, with or without
0022  * modification, are permitted provided that the following conditions
0023  * are met:
0024  * 1. Redistributions of source code must retain the above copyright
0025  *    notice, this list of conditions and the following disclaimer.
0026  * 2. Redistributions in binary form must reproduce the above copyright
0027  *    notice, this list of conditions and the following disclaimer in the
0028  *    documentation and/or other materials provided with the distribution.
0029  *
0030  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
0031  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
0032  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
0033  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
0034  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
0035  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
0036  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
0037  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
0038  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
0039  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
0040  * POSSIBILITY OF SUCH DAMAGE.
0041  */
0042
0043 #ifdef HAVE_CONFIG_H
0044 #include "config.h"
0045 #endif
0046
0047 #include <rtems/score/smpimpl.h>
0048 #include <rtems/score/assert.h>
0049 #include <rtems/score/schedulerimpl.h>
0050 #include <rtems/score/threadimpl.h>
0051 #include <rtems/config.h>
0052
0053 #if CPU_USE_DEFERRED_FP_SWITCH == TRUE
0054   #error "deferred FP switch not implemented for SMP"
0055 #endif
0056
0057 /**
0058  * @brief Indicates if the system is ready to start multitasking.
0059  *
0060  * Only the boot processor is allowed to change this object.  If the object has
0061  * a non-zero value and no fatal error occurred, then secondary processors
0062  * should call _Thread_Start_multitasking() to start multiprocessing.
0063  */
0064 static Atomic_Uint _SMP_Ready_to_start_multitasking;
0065
0066 Processor_mask _SMP_Online_processors;
0067
0068 uint32_t _SMP_Processor_maximum;
0069
0070 static const Scheduler_Assignment *_Scheduler_Get_initial_assignment(
0071   uint32_t cpu_index
0072 )
0073 {
0074   return &_Scheduler_Initial_assignments[ cpu_index ];
0075 }
0076
0077 static bool _Scheduler_Is_mandatory_processor(
0078   const Scheduler_Assignment *assignment
0079 )
0080 {
0081   return (assignment->attributes & SCHEDULER_ASSIGN_PROCESSOR_MANDATORY) != 0;
0082 }
0083
0084 static bool _Scheduler_Should_start_processor(
0085   const Scheduler_Assignment *assignment
0086 )
0087 {
0088   return assignment->scheduler != NULL;
0089 }
0090
0091 static void _SMP_Start_processors( uint32_t cpu_max )
0092 {
0093   Per_CPU_Control *cpu_self;
0094   uint32_t         cpu_index;
0095
0096   cpu_self = _Per_CPU_Get();
0097   cpu_self->boot = true;
0098   cpu_self->online = true;
0099
0100   for ( cpu_index = 0 ; cpu_index < cpu_max; ++cpu_index ) {
0101     const Scheduler_Assignment *assignment;
0102     Per_CPU_Control            *cpu;
0103     bool                        started;
0104
0105     assignment = _Scheduler_Get_initial_assignment( cpu_index );
0106     cpu = _Per_CPU_Get_by_index( cpu_index );
0107
0108     if ( cpu != cpu_self ) {
0109       if ( _Scheduler_Should_start_processor( assignment ) ) {
0110         started = _CPU_SMP_Start_processor( cpu_index );
0111
0112         if ( !started && _Scheduler_Is_mandatory_processor( assignment ) ) {
0113           _SMP_Fatal( SMP_FATAL_START_OF_MANDATORY_PROCESSOR_FAILED );
0114         }
0115       } else {
0116         started = false;
0117       }
0118
0119       cpu->online = started;
0120     } else {
0121       if ( !_Scheduler_Should_start_processor( assignment ) ) {
0122         _SMP_Fatal( SMP_FATAL_BOOT_PROCESSOR_NOT_ASSIGNED_TO_SCHEDULER );
0123       }
0124
0125       started = true;
0126     }
0127
0128     if ( started ) {
0129       const Scheduler_Control *scheduler;
0130       Scheduler_Context       *context;
0131
0132       scheduler = assignment->scheduler;
0133       context = _Scheduler_Get_context( scheduler );
0134
0135       _Processor_mask_Set( &_SMP_Online_processors, cpu_index );
0136       _Processor_mask_Set( &context->Processors, cpu_index );
0137       cpu->Scheduler.control = scheduler;
0138       cpu->Scheduler.context = context;
0139     }
0140   }
0141 }
0142
0143 void _SMP_Handler_initialize( void )
0144 {
0145   uint32_t cpu_config_max;
0146   uint32_t cpu_max;
0147   uint32_t cpu_index;
0148
0149   cpu_config_max = rtems_configuration_get_maximum_processors();
0150
0151   for ( cpu_index = 0 ; cpu_index < cpu_config_max; ++cpu_index ) {
0152     Per_CPU_Control *cpu;
0153
0154     cpu = _Per_CPU_Get_by_index( cpu_index );
0155     _ISR_lock_Set_name( &cpu->Lock, "Per-CPU" );
0156     _ISR_lock_Set_name( &cpu->Jobs.Lock, "Per-CPU Jobs" );
0157     _ISR_lock_Set_name( &cpu->Watchdog.Lock, "Per-CPU Watchdog" );
0158     _Chain_Initialize_empty( &cpu->Threads_in_need_for_help );
0159   }
0160
0161   /*
0162    * Discover and initialize the secondary cores in an SMP system.
0163    */
0164
0165   cpu_max = _CPU_SMP_Initialize();
0166   cpu_max = cpu_max < cpu_config_max ? cpu_max : cpu_config_max;
0167   _SMP_Processor_maximum = cpu_max;
0168
0169   for ( cpu_index = cpu_max ; cpu_index < cpu_config_max; ++cpu_index ) {
0170     const Scheduler_Assignment *assignment;
0171
0172     assignment = _Scheduler_Get_initial_assignment( cpu_index );
0173
0174     if ( _Scheduler_Is_mandatory_processor( assignment ) ) {
0175       _SMP_Fatal( SMP_FATAL_MANDATORY_PROCESSOR_NOT_PRESENT );
0176     }
0177   }
0178
0179   _SMP_Start_processors( cpu_max );
0180
0181   _CPU_SMP_Finalize_initialization( cpu_max );
0182 }
0183
0184 void _SMP_Wait_for_ready_to_start_multitasking( void )
0185 {
0186   Per_CPU_Control *cpu_self;
0187   uint32_t         cpu_max;
0188   uint32_t         cpu_index;
0189
0190   cpu_max = _SMP_Get_processor_maximum();
0191   cpu_self = _Per_CPU_Get();
0192
0193   for ( cpu_index = 0 ; cpu_index < cpu_max ; ++cpu_index ) {
0194     Per_CPU_Control *cpu;
0195
0196     cpu = _Per_CPU_Get_by_index( cpu_index );
0197
0198     if ( cpu != cpu_self && _Per_CPU_Is_processor_online( cpu ) ) {
0199       while (
0200         _Per_CPU_Get_state( cpu ) != PER_CPU_STATE_READY_TO_START_MULTITASKING
0201       ) {
0202         _SMP_Try_to_process_message(
0203           cpu_self,
0204           _Atomic_Load_ulong( &cpu_self->message, ATOMIC_ORDER_RELAXED )
0205         );
0206       }
0207     }
0208   }
0209 }
0210
0211 void _SMP_Request_start_multitasking( void )
0212 {
0213   _SMP_Wait_for_ready_to_start_multitasking();
0214   _Atomic_Store_uint(
0215     &_SMP_Ready_to_start_multitasking,
0216     1U,
0217     ATOMIC_ORDER_RELEASE
0218   );
0219 }
0220
0221 bool _SMP_Should_start_processor( uint32_t cpu_index )
0222 {
0223   const Scheduler_Assignment *assignment;
0224
0225   assignment = _Scheduler_Get_initial_assignment( cpu_index );
0226   return _Scheduler_Should_start_processor( assignment );
0227 }
0228
0229 static void _SMP_Wait_for_start_multitasking( Per_CPU_Control *cpu_self )
0230 {
0231   unsigned int ready;
0232
0233   do {
0234     _SMP_Try_to_process_message(
0235       cpu_self,
0236       _Atomic_Load_ulong( &cpu_self->message, ATOMIC_ORDER_RELAXED )
0237     );
0238     ready = _Atomic_Load_uint(
0239       &_SMP_Ready_to_start_multitasking,
0240       ATOMIC_ORDER_ACQUIRE
0241     );
0242   } while ( ready == 0U );
0243 }
0244
0245 void _SMP_Start_multitasking_on_secondary_processor(
0246   Per_CPU_Control *cpu_self
0247 )
0248 {
0249   uint32_t cpu_index_self;
0250
0251   cpu_index_self = _Per_CPU_Get_index( cpu_self );
0252
0253   /*
0254    * Call fatal error and per-CPU job handlers with thread dispatching
0255    * disabled.
0256    */
0257   cpu_self->thread_dispatch_disable_level = 1;
0258
0259   if ( cpu_index_self >= rtems_configuration_get_maximum_processors() ) {
0260     _SMP_Fatal( SMP_FATAL_MULTITASKING_START_ON_INVALID_PROCESSOR );
0261   }
0262
0263   if ( !_SMP_Should_start_processor( cpu_index_self ) ) {
0264     _SMP_Fatal( SMP_FATAL_MULTITASKING_START_ON_UNASSIGNED_PROCESSOR );
0265   }
0266
0267   _Per_CPU_Set_state( cpu_self, PER_CPU_STATE_READY_TO_START_MULTITASKING );
0268   _SMP_Wait_for_start_multitasking( cpu_self );
0269
0270   if ( !_Per_CPU_Is_processor_online( cpu_self ) ) {
0271     _SMP_Fatal( SMP_FATAL_MULTITASKING_START_ON_NOT_ONLINE_PROCESSOR );
0272   }
0273
0274   _Thread_Start_multitasking();
0275 }
0276
0277 void _SMP_Request_shutdown( void )
0278 {
0279   ISR_Level              level;
0280   const Per_CPU_Control *cpu_self;
0281   uint32_t               cpu_max;
0282   uint32_t               cpu_index;
0283
0284   _ISR_Local_disable( level );
0285   (void) level;
0286
0287   cpu_max = _SMP_Processor_configured_maximum;
0288   cpu_self = _Per_CPU_Get();
0289
0290   for ( cpu_index = 0 ; cpu_index < cpu_max ; ++cpu_index ) {
0291     Per_CPU_Control *cpu;
0292
0293     cpu = _Per_CPU_Get_by_index( cpu_index );
0294
0295     if ( cpu == cpu_self ) {
0296       _Per_CPU_Set_state( cpu, PER_CPU_STATE_SHUTDOWN );
0297     } else {
0298       _Atomic_Fetch_or_ulong(
0299         &cpu->message,
0300         SMP_MESSAGE_SHUTDOWN,
0301         ATOMIC_ORDER_RELEASE
0302       );
0303
0304       if ( _Per_CPU_Get_state( cpu ) == PER_CPU_STATE_UP ) {
0305         _CPU_SMP_Send_interrupt( cpu_index );
0306       }
0307     }
0308   }
0309 }
0310
0311 void _SMP_Process_message( Per_CPU_Control *cpu_self, long unsigned message )
0312 {
0313   if ( ( message & SMP_MESSAGE_SHUTDOWN ) != 0 ) {
0314     ISR_Level level;
0315
0316     _CPU_ISR_Disable( level );
0317     (void) level;
0318
0319     /* Check the state to prevent recursive shutdowns */
0320     if ( _Per_CPU_Get_state( cpu_self ) != PER_CPU_STATE_SHUTDOWN ) {
0321       _Per_CPU_Set_state( cpu_self, PER_CPU_STATE_SHUTDOWN );
0322       _SMP_Fatal( SMP_FATAL_SHUTDOWN_RESPONSE );
0323     }
0324   }
0325
0326   if ( ( message & SMP_MESSAGE_PERFORM_JOBS ) != 0 ) {
0327     _Per_CPU_Perform_jobs( cpu_self );
0328   }
0329 }
0330
0331 void _SMP_Try_to_process_message(
0332   Per_CPU_Control *cpu_self,
0333   unsigned long    message
0334 )
0335 {
0336   if ( message != 0 ) {
0337     /*
0338      * Fetch the current message.  Only a read-modify-write operation
0339      * guarantees that we get an up to date message.  This is especially
0340      * important if the function was called using SMP_MESSAGE_FORCE_PROCESSING.
0341      */
0342     message = _Atomic_Exchange_ulong(
0343       &cpu_self->message,
0344       0,
0345       ATOMIC_ORDER_ACQUIRE
0346     );
0347
0348     _SMP_Process_message( cpu_self, message );
0349   }
0350 }
0351
0352 void _SMP_Send_message( Per_CPU_Control *cpu, unsigned long message )
0353 {
0354   (void) _Atomic_Fetch_or_ulong(
0355     &cpu->message, message,
0356     ATOMIC_ORDER_RELEASE
0357   );
0358
0359   if ( _Per_CPU_Get_state( cpu ) == PER_CPU_STATE_UP ) {
0360     _CPU_SMP_Send_interrupt( _Per_CPU_Get_index( cpu ) );
0361   }
0362 }