shared/exceptions/ppc_exc_asm_macros.h

0001 /*
0002  * (c) 1999, Eric Valette valette@crf.canon.fr
0003  *
0004  * Modified and partially rewritten by Till Straumann, 2007-2008
0005  *
0006  * Modified by Sebastian Huber <sebastian.huber@embedded-brains.de>, 2008-2012.
0007  *
0008  * Low-level assembly code for PPC exceptions (macros).
0009  *
0010  * This file was written with the goal to eliminate
0011  * ALL #ifdef <cpu_flavor> conditionals -- please do not
0012  * reintroduce such statements.
0013  */
0014
0015 #include <bspopts.h>
0016 #include <bsp/vectors.h>
0017 #include <libcpu/powerpc-utility.h>
0018
0019 #define LT(cr) ((cr)*4+0)
0020 #define GT(cr) ((cr)*4+1)
0021 #define EQ(cr) ((cr)*4+2)
0022
0023 /* Opcode of 'stw r1, off(r13)' */
0024 #define STW_R1_R13(off) ((((36<<10)|(r1<<5)|(r13))<<16) | ((off)&0xffff))
0025
0026 #define FRAME_REGISTER r14
0027 #define VECTOR_REGISTER r4
0028 #define SCRATCH_REGISTER_0 r5
0029 #define SCRATCH_REGISTER_1 r6
0030 #define SCRATCH_REGISTER_2 r7
0031
0032 #define FRAME_OFFSET( r) GPR14_OFFSET( r)
0033 #define VECTOR_OFFSET( r) GPR4_OFFSET( r)
0034 #define SCRATCH_REGISTER_0_OFFSET( r) GPR5_OFFSET( r)
0035 #define SCRATCH_REGISTER_1_OFFSET( r) GPR6_OFFSET( r)
0036 #define SCRATCH_REGISTER_2_OFFSET( r) GPR7_OFFSET( r)
0037
0038 #define CR_TYPE 2
0039 #define CR_MSR 3
0040 #define CR_LOCK 4
0041
0042     /*
0043      * Minimal prologue snippets:
0044      *
0045      * Rationale: on some PPCs the vector offsets are spaced
0046      * as closely as 16 bytes.
0047      *
0048      * If we deal with asynchronous exceptions ('interrupts')
0049      * then we can use 4 instructions to
0050      *   1. atomically write lock to indicate ISR is in progress
0051      *      (we cannot atomically increase the Thread_Dispatch_disable_level,
0052      *      see README)
0053      *   2. save a register in special area
0054      *   3. load register with vector info
0055      *   4. branch
0056      *
0057      * If we deal with a synchronous exception (no stack switch
0058      * nor dispatch-disabling necessary) then it's easier:
0059      *   1. push stack frame
0060      *   2. save register on stack
0061      *   3. load register with vector info
0062      *   4. branch
0063      *
0064      */
0065
0066 /*
0067  *****************************************************************************
0068  * MACRO: PPC_EXC_MIN_PROLOG_ASYNC
0069  *****************************************************************************
0070  * USES:    VECTOR_REGISTER
0071  * ON EXIT: Vector in VECTOR_REGISTER
0072  *
0073  * NOTES:   VECTOR_REGISTER saved in special variable
0074  *          'ppc_exc_vector_register_\_PRI'.
0075  *
0076  */
0077     .macro  PPC_EXC_MIN_PROLOG_ASYNC _NAME _VEC _PRI _FLVR
0078
0079     .global ppc_exc_min_prolog_async_\_NAME
0080 ppc_exc_min_prolog_async_\_NAME:
0081     /* Atomically write lock variable in 1st instruction with non-zero
0082      * value (r1 is always nonzero; r13 could also be used)
0083      *
0084      * NOTE: raising an exception and executing this first instruction
0085      *       of the exception handler is apparently NOT atomic, i.e., a
0086      *       low-priority IRQ could set the PC to this location and a
0087      *       critical IRQ could intervene just at this point.
0088      *
0089      *       We check against this pathological case by checking the
0090      *       opcode/instruction at the interrupted PC for matching
0091      *
0092      *         stw r1, ppc_exc_lock_XXX@sdarel(r13)
0093      *
0094      *       ASSUMPTION:
0095      *          1) ALL 'asynchronous' exceptions (which disable thread-
0096      *             dispatching) execute THIS 'magical' instruction
0097      *             FIRST.
0098      *          2) This instruction (including the address offset)
0099      *             is not used anywhere else (probably a safe assumption).
0100      */
0101     stw r1, ppc_exc_lock_\_PRI@sdarel(r13)
0102     /*  We have no stack frame yet; store VECTOR_REGISTER in special area;
0103      * a higher-priority (critical) interrupt uses a different area
0104      * (hence the different prologue snippets) (\PRI)
0105      */
0106     stw VECTOR_REGISTER, ppc_exc_vector_register_\_PRI@sdarel(r13)
0107     /*  Load vector.
0108      */
0109     li  VECTOR_REGISTER, ( \_VEC | 0xffff8000 )
0110
0111     /*
0112      * We store the absolute branch target address here.  It will be used
0113      * to generate the branch operation in ppc_exc_make_prologue().
0114      */
0115     .int    ppc_exc_wrap_\_FLVR
0116
0117     .endm
0118
0119 /*
0120  *****************************************************************************
0121  * MACRO: PPC_EXC_MIN_PROLOG_SYNC
0122  *****************************************************************************
0123  * USES:    VECTOR_REGISTER
0124  * ON EXIT: vector in VECTOR_REGISTER
0125  *
0126  * NOTES:   exception stack frame pushed; VECTOR_REGISTER saved in frame
0127  *
0128  */
0129     .macro  PPC_EXC_MIN_PROLOG_SYNC _NAME _VEC _PRI _FLVR
0130
0131     .global ppc_exc_min_prolog_sync_\_NAME
0132 ppc_exc_min_prolog_sync_\_NAME:
0133     stwu    r1, -EXCEPTION_FRAME_END(r1)
0134     stw VECTOR_REGISTER, VECTOR_OFFSET(r1)
0135     li  VECTOR_REGISTER, \_VEC
0136
0137     /*
0138      * We store the absolute branch target address here.  It will be used
0139      * to generate the branch operation in ppc_exc_make_prologue().
0140      */
0141     .int    ppc_exc_wrap_nopush_\_FLVR
0142
0143     .endm
0144
0145 /*
0146  *****************************************************************************
0147  * MACRO: TEST_1ST_OPCODE_crit
0148  *****************************************************************************
0149  *
0150  * USES:    REG, cr0
0151  * ON EXIT: REG available (contains *pc - STW_R1_R13(0)),
0152  *          return value in cr0.
0153  *
0154  * test opcode interrupted by critical (asynchronous) exception; set CR_LOCK if
0155  *
0156  *   *SRR0 == 'stw r1, ppc_exc_lock_std@sdarel(r13)'
0157  *
0158  */
0159     .macro  TEST_1ST_OPCODE_crit _REG
0160
0161     lwz \_REG, SRR0_FRAME_OFFSET(FRAME_REGISTER)
0162     lwz \_REG, 0(\_REG)
0163     /*  opcode now in REG */
0164
0165     /*  subtract upper 16bits of 'stw r1, 0(r13)' instruction */
0166     subis   \_REG, \_REG, STW_R1_R13(0)@h
0167     /*
0168      * if what's left compares against the 'ppc_exc_lock_std@sdarel'
0169      * address offset then we have a match...
0170      */
0171     cmplwi  cr0, \_REG, ppc_exc_lock_std@sdarel
0172
0173     .endm
0174
0175 /*
0176  *****************************************************************************
0177  * MACRO: TEST_LOCK_std
0178  *****************************************************************************
0179  *
0180  * USES:    CR_LOCK
0181  * ON EXIT: CR_LOCK is set (indicates no lower-priority locks are engaged)
0182  *
0183  */
0184     .macro  TEST_LOCK_std _FLVR
0185     /* 'std' is lowest level, i.e., can not be locked -> EQ(CR_LOCK) = 1 */
0186     creqv   EQ(CR_LOCK), EQ(CR_LOCK), EQ(CR_LOCK)
0187     .endm
0188
0189 /*
0190  ******************************************************************************
0191  * MACRO: TEST_LOCK_crit
0192  ******************************************************************************
0193  *
0194  * USES:    CR_LOCK, cr0, SCRATCH_REGISTER_0, SCRATCH_REGISTER_1
0195  * ON EXIT: cr0, SCRATCH_REGISTER_0, SCRATCH_REGISTER_1 available,
0196  *          returns result in CR_LOCK.
0197  *
0198  * critical-exception wrapper has to check 'std' lock:
0199  *
0200  * Return CR_LOCK = (   (interrupt_mask & MSR_CE) != 0
0201                  &&                  ppc_lock_std == 0
0202  *               && * SRR0 != <write std lock instruction> )
0203  *
0204  */
0205     .macro  TEST_LOCK_crit _FLVR
0206     /* If MSR_CE is not in the IRQ mask then we must never allow
0207      * thread-dispatching!
0208      */
0209     GET_INTERRUPT_MASK mask=SCRATCH_REGISTER_1
0210     /* EQ(cr0) = ((interrupt_mask & MSR_CE) == 0) */
0211     andis.  SCRATCH_REGISTER_1, SCRATCH_REGISTER_1, MSR_CE@h
0212     beq TEST_LOCK_crit_done_\_FLVR
0213
0214     /* STD interrupt could have been interrupted before executing the 1st
0215      * instruction which sets the lock; check this case by looking at the
0216      * opcode present at the interrupted PC location.
0217      */
0218     TEST_1ST_OPCODE_crit    _REG=SCRATCH_REGISTER_0
0219     /*
0220      * At this point cr0 is set if
0221      *
0222      *   *(PC) == 'stw r1, ppc_exc_lock_std@sdarel(r13)'
0223      *
0224      */
0225
0226     /* check lock */
0227     lwz SCRATCH_REGISTER_1, ppc_exc_lock_std@sdarel(r13)
0228     cmplwi  CR_LOCK, SCRATCH_REGISTER_1, 0
0229
0230     /* set EQ(CR_LOCK) to result */
0231 TEST_LOCK_crit_done_\_FLVR:
0232     /* If we end up here because the interrupt mask did not contain
0233      * MSR_CE then cr0 is set and therefore the value of CR_LOCK
0234      * does not matter since   x && !1 == 0:
0235      *
0236      *  if ( (interrupt_mask & MSR_CE) == 0 ) {
0237      *      EQ(CR_LOCK) = EQ(CR_LOCK) && ! ((interrupt_mask & MSR_CE) == 0)
0238      *  } else {
0239      *      EQ(CR_LOCK) = (ppc_exc_lock_std == 0) && ! (*pc == <write std lock instruction>)
0240      *  }
0241      */
0242     crandc  EQ(CR_LOCK), EQ(CR_LOCK), EQ(cr0)
0243
0244     .endm
0245
0246 /*
0247  ******************************************************************************
0248  * MACRO: TEST_LOCK_mchk
0249  ******************************************************************************
0250  *
0251  * USES:    CR_LOCK
0252  * ON EXIT: CR_LOCK is cleared.
0253  *
0254  * We never want to disable machine-check exceptions to avoid a checkstop. This
0255  * means that we cannot use enabling/disabling this type of exception for
0256  * protection of critical OS data structures.  Therefore, calling OS primitives
0257  * from a machine-check handler is ILLEGAL. Since machine-checks can happen
0258  * anytime it is not legal to perform a context switch (since the exception
0259  * could hit a IRQ protected section of code).  We simply let this test return
0260  * 0 so that ppc_exc_wrapup is never called after handling a machine-check.
0261  */
0262     .macro  TEST_LOCK_mchk _SRR0 _FLVR
0263
0264     crxor   EQ(CR_LOCK), EQ(CR_LOCK), EQ(CR_LOCK)
0265
0266     .endm
0267
0268 /*
0269  ******************************************************************************
0270  * MACRO: RECOVER_CHECK_\PRI
0271  ******************************************************************************
0272  *
0273  * USES:    cr0, SCRATCH_REGISTER_0, SCRATCH_REGISTER_1
0274  * ON EXIT: cr0, SCRATCH_REGISTER_0, SCRATCH_REGISTER_1 available
0275  *
0276  * Checks if the exception is recoverable for exceptions which need such a
0277  * test.
0278  */
0279
0280 /* Standard*/
0281     .macro  RECOVER_CHECK_std _FLVR
0282
0283 #ifndef PPC_EXC_CONFIG_BOOKE_ONLY
0284
0285     /* Check if exception is recoverable */
0286     lwz SCRATCH_REGISTER_0, SRR1_FRAME_OFFSET(FRAME_REGISTER)
0287     lwz SCRATCH_REGISTER_1, ppc_exc_msr_bits@sdarel(r13)
0288     xor SCRATCH_REGISTER_1, SCRATCH_REGISTER_1, SCRATCH_REGISTER_0
0289     andi.   SCRATCH_REGISTER_0, SCRATCH_REGISTER_1, MSR_RI
0290
0291 recover_check_twiddle_std_\_FLVR:
0292
0293     /* Not recoverable? */
0294     bne recover_check_twiddle_std_\_FLVR
0295
0296 #endif /* PPC_EXC_CONFIG_BOOKE_ONLY */
0297
0298     .endm
0299
0300 /* Critical */
0301     .macro  RECOVER_CHECK_crit _FLVR
0302
0303     /* Nothing to do */
0304
0305     .endm
0306
0307 /* Machine check */
0308     .macro  RECOVER_CHECK_mchk _FLVR
0309
0310 #ifndef PPC_EXC_CONFIG_BOOKE_ONLY
0311
0312     /* Check if exception is recoverable */
0313     lwz SCRATCH_REGISTER_0, SRR1_FRAME_OFFSET(FRAME_REGISTER)
0314     lwz SCRATCH_REGISTER_1, ppc_exc_msr_bits@sdarel(r13)
0315     xor SCRATCH_REGISTER_1, SCRATCH_REGISTER_1, SCRATCH_REGISTER_0
0316     andi.   SCRATCH_REGISTER_0, SCRATCH_REGISTER_1, MSR_RI
0317
0318 recover_check_twiddle_mchk_\_FLVR:
0319
0320     /* Not recoverable? */
0321     bne recover_check_twiddle_mchk_\_FLVR
0322
0323 #endif /* PPC_EXC_CONFIG_BOOKE_ONLY */
0324
0325     .endm
0326
0327 /*
0328  ******************************************************************************
0329  * MACRO: WRAP
0330  ******************************************************************************
0331  *
0332  * Minimal prologue snippets jump into WRAP which calls the high level
0333  * exception handler.  We must have this macro  instantiated for each possible
0334  * flavor of exception so that we use the proper lock variable, SRR register
0335  * pair and RFI instruction.
0336  *
0337  * We have two types of exceptions: synchronous and asynchronous (= interrupt
0338  * like).  The type is encoded in the vector register (= VECTOR_REGISTER).  For
0339  * interrupt like exceptions the MSB in the vector register is set.  The
0340  * exception type is kept in the comparison register CR_TYPE.  Normal
0341  * exceptions (MSB is clear) use the task stack and a context switch may happen
0342  * at any time.  The interrupt like exceptions disable thread dispatching and
0343  * switch to the interrupt stack (base address is in SPRG1).
0344  *
0345  *                                      +
0346  *                                      |
0347  *                                      | Minimal prologue
0348  *                                      |
0349  *                                      +
0350  *                                      |
0351  *                                      | o Setup frame pointer
0352  *                                      | o Save basic registers
0353  *                                      | o Determine exception type:
0354  *                                      |   synchronous or asynchronous
0355  *                                      |
0356  *                                +-----+
0357  * Synchronous exceptions:        |     | Asynchronous exceptions:
0358  *                                |     |
0359  * Save non-volatile registers    |     | o Increment thread dispatch
0360  *                                |     |   disable level
0361  *                                |     | o Increment ISR nest level
0362  *                                |     | o Clear lock
0363  *                                |     | o Switch stack if necessary
0364  *                                |     |
0365  *                                +---->+
0366  *                                      |
0367  *                                      | o Save volatile registers
0368  *                                      | o Change MSR if necessary
0369  *                                      | o Call high level handler
0370  *                                      | o Call global handler if necessary
0371  *                                      | o Check if exception is recoverable
0372  *                                      |
0373  *                                +-----+
0374  * Synchronous exceptions:        |     | Asynchronous exceptions:
0375  *                                |     |
0376  * Restore non-volatile registers |     | o Decrement ISR nest level
0377  *                                |     | o Switch stack
0378  *                                |     | o Decrement thread dispatch
0379  *                                |     |   disable level
0380  *                                |     | o Test lock
0381  *                                |     | o May do a context switch
0382  *                                |     |
0383  *                                +---->+
0384  *                                      |
0385  *                                      | o Restore MSR if necessary
0386  *                                      | o Restore volatile registers
0387  *                                      | o Restore frame pointer
0388  *                                      | o Return
0389  *                                      |
0390  *                                      +
0391  */
0392     .macro  WRAP _FLVR _PRI _SRR0 _SRR1 _RFI
0393
0394     .global ppc_exc_wrap_\_FLVR
0395 ppc_exc_wrap_\_FLVR:
0396
0397     /* Push exception frame */
0398     stwu    r1, -EXCEPTION_FRAME_END(r1)
0399
0400     .global ppc_exc_wrap_nopush_\_FLVR
0401 ppc_exc_wrap_nopush_\_FLVR:
0402
0403     /* Save frame register */
0404     stw FRAME_REGISTER, FRAME_OFFSET(r1)
0405
0406 wrap_no_save_frame_register_\_FLVR:
0407
0408     /*
0409      * We save at first only some scratch registers
0410      * and the CR.  We use a non-volatile register
0411      * for the exception frame pointer (= FRAME_REGISTER).
0412      */
0413
0414     /* Move frame address in non-volatile FRAME_REGISTER */
0415     mr  FRAME_REGISTER, r1
0416
0417     /* Save scratch registers */
0418     stw SCRATCH_REGISTER_0, SCRATCH_REGISTER_0_OFFSET(FRAME_REGISTER)
0419     stw SCRATCH_REGISTER_1, SCRATCH_REGISTER_1_OFFSET(FRAME_REGISTER)
0420     stw SCRATCH_REGISTER_2, SCRATCH_REGISTER_2_OFFSET(FRAME_REGISTER)
0421
0422     /* Save CR */
0423     mfcr    SCRATCH_REGISTER_0
0424     stw SCRATCH_REGISTER_0, EXC_CR_OFFSET(FRAME_REGISTER)
0425
0426     /* Check exception type and remember it in non-volatile CR_TYPE */
0427     cmpwi   CR_TYPE, VECTOR_REGISTER, 0
0428
0429 #if defined(PPC_MULTILIB_FPU) || defined(PPC_MULTILIB_ALTIVEC)
0430     /* Enable FPU and/or AltiVec */
0431     mfmsr   SCRATCH_REGISTER_0
0432 #ifdef PPC_MULTILIB_FPU
0433     ori SCRATCH_REGISTER_0, SCRATCH_REGISTER_0, MSR_FP
0434 #endif
0435 #ifdef PPC_MULTILIB_ALTIVEC
0436     oris    SCRATCH_REGISTER_0, SCRATCH_REGISTER_0, MSR_VE >> 16
0437 #endif
0438     mtmsr   SCRATCH_REGISTER_0
0439     isync
0440 #endif
0441
0442     /*
0443      * Depending on the exception type we do now save the non-volatile
0444      * registers or disable thread dispatching and switch to the ISR stack.
0445      */
0446
0447     /* Branch for synchronous exceptions */
0448     bge CR_TYPE, wrap_save_non_volatile_regs_\_FLVR
0449
0450     /*
0451      * Increment the thread dispatch disable level in case a higher
0452      * priority exception occurs we don't want it to run the scheduler.  It
0453      * is safe to increment this without disabling higher priority
0454      * exceptions since those will see that we wrote the lock anyways.
0455      */
0456
0457     /* Increment ISR nest level and thread dispatch disable level */
0458     GET_SELF_CPU_CONTROL    SCRATCH_REGISTER_2
0459     lwz SCRATCH_REGISTER_0, PER_CPU_ISR_NEST_LEVEL@l(SCRATCH_REGISTER_2)
0460     lwz SCRATCH_REGISTER_1, PER_CPU_THREAD_DISPATCH_DISABLE_LEVEL(SCRATCH_REGISTER_2)
0461     addi    SCRATCH_REGISTER_0, SCRATCH_REGISTER_0, 1
0462     addi    SCRATCH_REGISTER_1, SCRATCH_REGISTER_1, 1
0463     stw SCRATCH_REGISTER_0, PER_CPU_ISR_NEST_LEVEL@l(SCRATCH_REGISTER_2)
0464     stw SCRATCH_REGISTER_1, PER_CPU_THREAD_DISPATCH_DISABLE_LEVEL(SCRATCH_REGISTER_2)
0465
0466     /*
0467      * No higher-priority exception occurring after this point
0468      * can cause a context switch.
0469      */
0470
0471     /* Clear lock */
0472     li  SCRATCH_REGISTER_0, 0
0473     stw SCRATCH_REGISTER_0, ppc_exc_lock_\_PRI@sdarel(r13)
0474
0475     /* Switch stack if necessary */
0476     mfspr   SCRATCH_REGISTER_0, SPRG1
0477     cmpw    SCRATCH_REGISTER_0, r1
0478     blt wrap_stack_switch_\_FLVR
0479     mfspr   SCRATCH_REGISTER_1, SPRG2
0480     cmpw    SCRATCH_REGISTER_1, r1
0481     blt wrap_stack_switch_done_\_FLVR
0482
0483 wrap_stack_switch_\_FLVR:
0484
0485     mr  r1, SCRATCH_REGISTER_0
0486
0487 wrap_stack_switch_done_\_FLVR:
0488
0489     /*
0490      * Load the pristine VECTOR_REGISTER from a special location for
0491      * asynchronous exceptions.  The synchronous exceptions save the
0492      * VECTOR_REGISTER in their minimal prologue.
0493      */
0494     lwz SCRATCH_REGISTER_2, ppc_exc_vector_register_\_PRI@sdarel(r13)
0495
0496     /* Save pristine vector register */
0497     stw SCRATCH_REGISTER_2, VECTOR_OFFSET(FRAME_REGISTER)
0498
0499 wrap_disable_thread_dispatching_done_\_FLVR:
0500
0501     /*
0502      * We now have SCRATCH_REGISTER_0, SCRATCH_REGISTER_1,
0503      * SCRATCH_REGISTER_2 and CR available.  VECTOR_REGISTER still holds
0504      * the vector (and exception type).  FRAME_REGISTER is a pointer to the
0505      * exception frame (always on the stack of the interrupted context).
0506      * r1 is the stack pointer, either on the task stack or on the ISR
0507      * stack.  CR_TYPE holds the exception type.
0508      */
0509
0510     /* Save SRR0 */
0511     mfspr   SCRATCH_REGISTER_0, \_SRR0
0512     stw SCRATCH_REGISTER_0, SRR0_FRAME_OFFSET(FRAME_REGISTER)
0513
0514     /* Save SRR1 */
0515     mfspr   SCRATCH_REGISTER_0, \_SRR1
0516     stw SCRATCH_REGISTER_0, SRR1_FRAME_OFFSET(FRAME_REGISTER)
0517
0518     /* Save CTR */
0519     mfctr   SCRATCH_REGISTER_0
0520     stw SCRATCH_REGISTER_0, EXC_CTR_OFFSET(FRAME_REGISTER)
0521
0522     /* Save XER */
0523     mfxer   SCRATCH_REGISTER_0
0524     stw SCRATCH_REGISTER_0, EXC_XER_OFFSET(FRAME_REGISTER)
0525
0526     /* Save LR */
0527     mflr    SCRATCH_REGISTER_0
0528     stw SCRATCH_REGISTER_0, EXC_LR_OFFSET(FRAME_REGISTER)
0529
0530     /* Save volatile registers */
0531     stw r0, GPR0_OFFSET(FRAME_REGISTER)
0532     stw r3, GPR3_OFFSET(FRAME_REGISTER)
0533     stw r8, GPR8_OFFSET(FRAME_REGISTER)
0534     stw r9, GPR9_OFFSET(FRAME_REGISTER)
0535     stw r10, GPR10_OFFSET(FRAME_REGISTER)
0536     stw r11, GPR11_OFFSET(FRAME_REGISTER)
0537     stw r12, GPR12_OFFSET(FRAME_REGISTER)
0538
0539     /* Save read-only small data area anchor (EABI) */
0540     stw r2, GPR2_OFFSET(FRAME_REGISTER)
0541
0542     /* Save vector number and exception type */
0543     stw VECTOR_REGISTER, EXCEPTION_NUMBER_OFFSET(FRAME_REGISTER)
0544
0545 #ifndef PPC_EXC_CONFIG_BOOKE_ONLY
0546
0547     /* Load MSR bit mask */
0548     lwz SCRATCH_REGISTER_0, ppc_exc_msr_bits@sdarel(r13)
0549
0550     /*
0551      * Change the MSR if necessary (MMU, RI),
0552      * remember decision in non-volatile CR_MSR
0553      */
0554     cmpwi   CR_MSR, SCRATCH_REGISTER_0, 0
0555     bne CR_MSR, wrap_change_msr_\_FLVR
0556
0557 wrap_change_msr_done_\_FLVR:
0558
0559 #endif /* PPC_EXC_CONFIG_BOOKE_ONLY */
0560
0561 #if defined(__ALTIVEC__) && !defined(PPC_MULTILIB_ALTIVEC)
0562     LA  SCRATCH_REGISTER_0, _CPU_save_altivec_volatile
0563     mtctr SCRATCH_REGISTER_0
0564     addi r3, FRAME_REGISTER, EXC_VEC_OFFSET
0565     bctrl
0566     /*
0567      * Establish defaults for vrsave and vscr
0568      */
0569     li       SCRATCH_REGISTER_0, 0
0570     mtvrsave SCRATCH_REGISTER_0
0571     /*
0572      * Use java/c9x mode; clear saturation bit
0573      */
0574     vxor     0, 0, 0
0575     mtvscr   0
0576     /*
0577      * Reload VECTOR_REGISTER
0578      */
0579     lwz      VECTOR_REGISTER, EXCEPTION_NUMBER_OFFSET(FRAME_REGISTER)
0580 #endif
0581
0582 #ifdef PPC_MULTILIB_ALTIVEC
0583     li  SCRATCH_REGISTER_0, PPC_EXC_VR_OFFSET(0)
0584     stvx    v0, FRAME_REGISTER, SCRATCH_REGISTER_0
0585     mfvscr  v0
0586     li  SCRATCH_REGISTER_0, PPC_EXC_VR_OFFSET(1)
0587     stvx    v1, FRAME_REGISTER, SCRATCH_REGISTER_0
0588     li  SCRATCH_REGISTER_0, PPC_EXC_VR_OFFSET(2)
0589     stvx    v2, FRAME_REGISTER, SCRATCH_REGISTER_0
0590     li  SCRATCH_REGISTER_0, PPC_EXC_VR_OFFSET(3)
0591     stvx    v3, FRAME_REGISTER, SCRATCH_REGISTER_0
0592     li  SCRATCH_REGISTER_0, PPC_EXC_VR_OFFSET(4)
0593     stvx    v4, FRAME_REGISTER, SCRATCH_REGISTER_0
0594     li  SCRATCH_REGISTER_0, PPC_EXC_VR_OFFSET(5)
0595     stvx    v5, FRAME_REGISTER, SCRATCH_REGISTER_0
0596     li  SCRATCH_REGISTER_0, PPC_EXC_VR_OFFSET(6)
0597     stvx    v6, FRAME_REGISTER, SCRATCH_REGISTER_0
0598     li  SCRATCH_REGISTER_0, PPC_EXC_VR_OFFSET(7)
0599     stvx    v7, FRAME_REGISTER, SCRATCH_REGISTER_0
0600     li  SCRATCH_REGISTER_0, PPC_EXC_VR_OFFSET(8)
0601     stvx    v8, FRAME_REGISTER, SCRATCH_REGISTER_0
0602     li  SCRATCH_REGISTER_0, PPC_EXC_VR_OFFSET(9)
0603     stvx    v9, FRAME_REGISTER, SCRATCH_REGISTER_0
0604     li  SCRATCH_REGISTER_0, PPC_EXC_VR_OFFSET(0)
0605     stvx    v10, FRAME_REGISTER, SCRATCH_REGISTER_0
0606     li  SCRATCH_REGISTER_0, PPC_EXC_VR_OFFSET(11)
0607     stvx    v11, FRAME_REGISTER, SCRATCH_REGISTER_0
0608     li  SCRATCH_REGISTER_0, PPC_EXC_VR_OFFSET(12)
0609     stvx    v12, FRAME_REGISTER, SCRATCH_REGISTER_0
0610     li  SCRATCH_REGISTER_0, PPC_EXC_VR_OFFSET(13)
0611     stvx    v13, FRAME_REGISTER, SCRATCH_REGISTER_0
0612     li  SCRATCH_REGISTER_0, PPC_EXC_VR_OFFSET(14)
0613     stvx    v14, FRAME_REGISTER, SCRATCH_REGISTER_0
0614     li  SCRATCH_REGISTER_0, PPC_EXC_VR_OFFSET(15)
0615     stvx    v15, FRAME_REGISTER, SCRATCH_REGISTER_0
0616     li  SCRATCH_REGISTER_0, PPC_EXC_VR_OFFSET(16)
0617     stvx    v16, FRAME_REGISTER, SCRATCH_REGISTER_0
0618     li  SCRATCH_REGISTER_0, PPC_EXC_VR_OFFSET(17)
0619     stvx    v17, FRAME_REGISTER, SCRATCH_REGISTER_0
0620     li  SCRATCH_REGISTER_0, PPC_EXC_VR_OFFSET(18)
0621     stvx    v18, FRAME_REGISTER, SCRATCH_REGISTER_0
0622     li  SCRATCH_REGISTER_0, PPC_EXC_VR_OFFSET(19)
0623     stvx    v19, FRAME_REGISTER, SCRATCH_REGISTER_0
0624     li  SCRATCH_REGISTER_0, PPC_EXC_VSCR_OFFSET
0625     stvewx  v0, r1, SCRATCH_REGISTER_0
0626 #endif
0627
0628 #ifdef PPC_MULTILIB_FPU
0629     stfd    f0, PPC_EXC_FR_OFFSET(0)(FRAME_REGISTER)
0630     mffs    f0
0631     stfd    f1, PPC_EXC_FR_OFFSET(1)(FRAME_REGISTER)
0632     stfd    f2, PPC_EXC_FR_OFFSET(2)(FRAME_REGISTER)
0633     stfd    f3, PPC_EXC_FR_OFFSET(3)(FRAME_REGISTER)
0634     stfd    f4, PPC_EXC_FR_OFFSET(4)(FRAME_REGISTER)
0635     stfd    f5, PPC_EXC_FR_OFFSET(5)(FRAME_REGISTER)
0636     stfd    f6, PPC_EXC_FR_OFFSET(6)(FRAME_REGISTER)
0637     stfd    f7, PPC_EXC_FR_OFFSET(7)(FRAME_REGISTER)
0638     stfd    f8, PPC_EXC_FR_OFFSET(8)(FRAME_REGISTER)
0639     stfd    f9, PPC_EXC_FR_OFFSET(9)(FRAME_REGISTER)
0640     stfd    f10, PPC_EXC_FR_OFFSET(10)(FRAME_REGISTER)
0641     stfd    f11, PPC_EXC_FR_OFFSET(11)(FRAME_REGISTER)
0642     stfd    f12, PPC_EXC_FR_OFFSET(12)(FRAME_REGISTER)
0643     stfd    f13, PPC_EXC_FR_OFFSET(13)(FRAME_REGISTER)
0644     stfd    f0, PPC_EXC_FPSCR_OFFSET(FRAME_REGISTER)
0645 #endif
0646
0647     /*
0648      * Call high level exception handler
0649      */
0650
0651     /*
0652      * Get the handler table index from the vector number.  We have to
0653      * discard the exception type.  Take only the least significant five
0654      * bits (= LAST_VALID_EXC + 1) from the vector register.  Multiply by
0655      * four (= size of function pointer).
0656      */
0657     rlwinm  SCRATCH_REGISTER_1, VECTOR_REGISTER, 2, 25, 29
0658
0659     /* Load handler table address */
0660     LA  SCRATCH_REGISTER_0, ppc_exc_handler_table
0661
0662     /* Load handler address */
0663     lwzx    SCRATCH_REGISTER_0, SCRATCH_REGISTER_0, SCRATCH_REGISTER_1
0664
0665     /*
0666      * First parameter = exception frame pointer + FRAME_LINK_SPACE
0667      *
0668      * We add FRAME_LINK_SPACE to the frame pointer because the high level
0669      * handler expects a BSP_Exception_frame structure.
0670      */
0671     addi    r3, FRAME_REGISTER, FRAME_LINK_SPACE
0672
0673     /*
0674      * Second parameter = vector number (r4 is the VECTOR_REGISTER)
0675      *
0676      * Discard the exception type and store the vector number
0677      * in the vector register.  Take only the least significant
0678      * five bits (= LAST_VALID_EXC + 1).
0679      */
0680     rlwinm  VECTOR_REGISTER, VECTOR_REGISTER, 0, 27, 31
0681
0682     /* Call handler */
0683     mtctr   SCRATCH_REGISTER_0
0684     bctrl
0685
0686     /* Check return value and call global handler if necessary */
0687     cmpwi   r3, 0
0688     bne wrap_call_global_handler_\_FLVR
0689
0690 wrap_handler_done_\_FLVR:
0691
0692     /* Check if exception is recoverable */
0693     RECOVER_CHECK_\_PRI _FLVR=\_FLVR
0694
0695     /*
0696      * Depending on the exception type we do now restore the non-volatile
0697      * registers or enable thread dispatching and switch back from the ISR
0698      * stack.
0699      */
0700
0701     /* Branch for synchronous exceptions */
0702     bge CR_TYPE, wrap_restore_non_volatile_regs_\_FLVR
0703
0704     /*
0705      * Switch back to original stack (FRAME_REGISTER == r1 if we are still
0706      * on the IRQ stack).
0707      */
0708     mr  r1, FRAME_REGISTER
0709
0710     /*
0711      * Check thread dispatch disable level AND lower priority locks (in
0712      * CR_LOCK): ONLY if the thread dispatch disable level == 0 AND no lock
0713      * is set then call ppc_exc_wrapup() which may do a context switch.  We
0714      * can skip TEST_LOCK, because it has no side effects.
0715      */
0716
0717     /* Decrement ISR nest level and thread dispatch disable level */
0718     GET_SELF_CPU_CONTROL    SCRATCH_REGISTER_2
0719     lwz SCRATCH_REGISTER_0, PER_CPU_ISR_NEST_LEVEL@l(SCRATCH_REGISTER_2)
0720     lwz SCRATCH_REGISTER_1, PER_CPU_THREAD_DISPATCH_DISABLE_LEVEL(SCRATCH_REGISTER_2)
0721     subi    SCRATCH_REGISTER_0, SCRATCH_REGISTER_0, 1
0722     subic.  SCRATCH_REGISTER_1, SCRATCH_REGISTER_1, 1
0723     stw SCRATCH_REGISTER_0, PER_CPU_ISR_NEST_LEVEL@l(SCRATCH_REGISTER_2)
0724     stw SCRATCH_REGISTER_1, PER_CPU_THREAD_DISPATCH_DISABLE_LEVEL(SCRATCH_REGISTER_2)
0725
0726     /* Branch to skip thread dispatching */
0727     bne wrap_thread_dispatching_done_\_FLVR
0728
0729     /* Test lower-priority locks (result in non-volatile CR_LOCK) */
0730     TEST_LOCK_\_PRI _FLVR=\_FLVR
0731
0732     /* Branch to skip thread dispatching */
0733     bne CR_LOCK, wrap_thread_dispatching_done_\_FLVR
0734
0735     /* Load address of ppc_exc_wrapup() */
0736     LA  SCRATCH_REGISTER_0, ppc_exc_wrapup
0737
0738     /* First parameter = exception frame pointer + FRAME_LINK_SPACE */
0739     addi    r3, FRAME_REGISTER, FRAME_LINK_SPACE
0740
0741     /* Call ppc_exc_wrapup() */
0742     mtctr   SCRATCH_REGISTER_0
0743     bctrl
0744
0745 wrap_thread_dispatching_done_\_FLVR:
0746
0747 #if defined(__ALTIVEC__) && !defined(PPC_MULTILIB_ALTIVEC)
0748     LA  SCRATCH_REGISTER_0, _CPU_load_altivec_volatile
0749     mtctr   SCRATCH_REGISTER_0
0750     addi    r3, FRAME_REGISTER, EXC_VEC_OFFSET
0751     bctrl
0752 #endif
0753
0754 #ifdef PPC_MULTILIB_ALTIVEC
0755     li  SCRATCH_REGISTER_0, PPC_EXC_MIN_VSCR_OFFSET
0756     lvewx   v0, r1, SCRATCH_REGISTER_0
0757     mtvscr  v0
0758     li  SCRATCH_REGISTER_0, PPC_EXC_VR_OFFSET(0)
0759     lvx v0, FRAME_REGISTER, SCRATCH_REGISTER_0
0760     li  SCRATCH_REGISTER_0, PPC_EXC_VR_OFFSET(1)
0761     lvx v1, FRAME_REGISTER, SCRATCH_REGISTER_0
0762     li  SCRATCH_REGISTER_0, PPC_EXC_VR_OFFSET(2)
0763     lvx v2, FRAME_REGISTER, SCRATCH_REGISTER_0
0764     li  SCRATCH_REGISTER_0, PPC_EXC_VR_OFFSET(3)
0765     lvx v3, FRAME_REGISTER, SCRATCH_REGISTER_0
0766     li  SCRATCH_REGISTER_0, PPC_EXC_VR_OFFSET(4)
0767     lvx v4, FRAME_REGISTER, SCRATCH_REGISTER_0
0768     li  SCRATCH_REGISTER_0, PPC_EXC_VR_OFFSET(5)
0769     lvx v5, FRAME_REGISTER, SCRATCH_REGISTER_0
0770     li  SCRATCH_REGISTER_0, PPC_EXC_VR_OFFSET(6)
0771     lvx v6, FRAME_REGISTER, SCRATCH_REGISTER_0
0772     li  SCRATCH_REGISTER_0, PPC_EXC_VR_OFFSET(7)
0773     lvx v7, FRAME_REGISTER, SCRATCH_REGISTER_0
0774     li  SCRATCH_REGISTER_0, PPC_EXC_VR_OFFSET(8)
0775     lvx v8, FRAME_REGISTER, SCRATCH_REGISTER_0
0776     li  SCRATCH_REGISTER_0, PPC_EXC_VR_OFFSET(9)
0777     lvx v9, FRAME_REGISTER, SCRATCH_REGISTER_0
0778     li  SCRATCH_REGISTER_0, PPC_EXC_VR_OFFSET(0)
0779     lvx v10, FRAME_REGISTER, SCRATCH_REGISTER_0
0780     li  SCRATCH_REGISTER_0, PPC_EXC_VR_OFFSET(11)
0781     lvx v11, FRAME_REGISTER, SCRATCH_REGISTER_0
0782     li  SCRATCH_REGISTER_0, PPC_EXC_VR_OFFSET(12)
0783     lvx v12, FRAME_REGISTER, SCRATCH_REGISTER_0
0784     li  SCRATCH_REGISTER_0, PPC_EXC_VR_OFFSET(13)
0785     lvx v13, FRAME_REGISTER, SCRATCH_REGISTER_0
0786     li  SCRATCH_REGISTER_0, PPC_EXC_VR_OFFSET(14)
0787     lvx v14, FRAME_REGISTER, SCRATCH_REGISTER_0
0788     li  SCRATCH_REGISTER_0, PPC_EXC_VR_OFFSET(15)
0789     lvx v15, FRAME_REGISTER, SCRATCH_REGISTER_0
0790     li  SCRATCH_REGISTER_0, PPC_EXC_VR_OFFSET(16)
0791     lvx v16, FRAME_REGISTER, SCRATCH_REGISTER_0
0792     li  SCRATCH_REGISTER_0, PPC_EXC_VR_OFFSET(17)
0793     lvx v17, FRAME_REGISTER, SCRATCH_REGISTER_0
0794     li  SCRATCH_REGISTER_0, PPC_EXC_VR_OFFSET(18)
0795     lvx v18, FRAME_REGISTER, SCRATCH_REGISTER_0
0796     li  SCRATCH_REGISTER_0, PPC_EXC_VR_OFFSET(19)
0797     lvx v19, FRAME_REGISTER, SCRATCH_REGISTER_0
0798 #endif
0799
0800 #ifdef PPC_MULTILIB_FPU
0801     lfd f0, PPC_EXC_FPSCR_OFFSET(FRAME_REGISTER)
0802     mtfsf   0xff, f0
0803     lfd f0, PPC_EXC_FR_OFFSET(0)(FRAME_REGISTER)
0804     lfd f1, PPC_EXC_FR_OFFSET(1)(FRAME_REGISTER)
0805     lfd f2, PPC_EXC_FR_OFFSET(2)(FRAME_REGISTER)
0806     lfd f3, PPC_EXC_FR_OFFSET(3)(FRAME_REGISTER)
0807     lfd f4, PPC_EXC_FR_OFFSET(4)(FRAME_REGISTER)
0808     lfd f5, PPC_EXC_FR_OFFSET(5)(FRAME_REGISTER)
0809     lfd f6, PPC_EXC_FR_OFFSET(6)(FRAME_REGISTER)
0810     lfd f7, PPC_EXC_FR_OFFSET(7)(FRAME_REGISTER)
0811     lfd f8, PPC_EXC_FR_OFFSET(8)(FRAME_REGISTER)
0812     lfd f9, PPC_EXC_FR_OFFSET(9)(FRAME_REGISTER)
0813     lfd f10, PPC_EXC_FR_OFFSET(10)(FRAME_REGISTER)
0814     lfd f11, PPC_EXC_FR_OFFSET(11)(FRAME_REGISTER)
0815     lfd f12, PPC_EXC_FR_OFFSET(12)(FRAME_REGISTER)
0816     lfd f13, PPC_EXC_FR_OFFSET(13)(FRAME_REGISTER)
0817 #endif
0818
0819 #ifndef PPC_EXC_CONFIG_BOOKE_ONLY
0820
0821     /* Restore MSR? */
0822     bne CR_MSR, wrap_restore_msr_\_FLVR
0823
0824 wrap_restore_msr_done_\_FLVR:
0825
0826 #endif /* PPC_EXC_CONFIG_BOOKE_ONLY */
0827
0828     /*
0829      * At this point r1 is a valid exception frame pointer and
0830      * FRAME_REGISTER is no longer needed.
0831      */
0832
0833     /* Restore frame register */
0834     lwz FRAME_REGISTER, FRAME_OFFSET(r1)
0835
0836     /* Restore XER and CTR */
0837     lwz SCRATCH_REGISTER_0, EXC_XER_OFFSET(r1)
0838     lwz SCRATCH_REGISTER_1, EXC_CTR_OFFSET(r1)
0839     mtxer   SCRATCH_REGISTER_0
0840     mtctr   SCRATCH_REGISTER_1
0841
0842     /* Restore CR and LR */
0843     lwz SCRATCH_REGISTER_0, EXC_CR_OFFSET(r1)
0844     lwz SCRATCH_REGISTER_1, EXC_LR_OFFSET(r1)
0845     mtcr    SCRATCH_REGISTER_0
0846     mtlr    SCRATCH_REGISTER_1
0847
0848     /* Restore volatile registers */
0849     lwz r0, GPR0_OFFSET(r1)
0850     lwz r3, GPR3_OFFSET(r1)
0851     lwz r8, GPR8_OFFSET(r1)
0852     lwz r9, GPR9_OFFSET(r1)
0853     lwz r10, GPR10_OFFSET(r1)
0854     lwz r11, GPR11_OFFSET(r1)
0855     lwz r12, GPR12_OFFSET(r1)
0856
0857     /* Restore read-only small data area anchor (EABI) */
0858     lwz r2, GPR2_OFFSET(r1)
0859
0860     /* Restore vector register */
0861     lwz VECTOR_REGISTER, VECTOR_OFFSET(r1)
0862
0863     /*
0864      * Disable all asynchronous exceptions which can do a thread dispatch.
0865      * See README.
0866      */
0867     INTERRUPT_DISABLE   SCRATCH_REGISTER_1, SCRATCH_REGISTER_0
0868
0869     /* Restore scratch registers and SRRs */
0870     lwz SCRATCH_REGISTER_0, SRR0_FRAME_OFFSET(r1)
0871     lwz SCRATCH_REGISTER_1, SRR1_FRAME_OFFSET(r1)
0872     lwz SCRATCH_REGISTER_2, SCRATCH_REGISTER_2_OFFSET(r1)
0873     mtspr   \_SRR0, SCRATCH_REGISTER_0
0874     lwz SCRATCH_REGISTER_0, SCRATCH_REGISTER_0_OFFSET(r1)
0875     mtspr   \_SRR1, SCRATCH_REGISTER_1
0876     lwz SCRATCH_REGISTER_1, SCRATCH_REGISTER_1_OFFSET(r1)
0877
0878     /*
0879      * We restore r1 from the frame rather than just popping (adding to
0880      * current r1) since the exception handler might have done strange
0881      * things (e.g. a debugger moving and relocating the stack).
0882      */
0883     lwz r1, 0(r1)
0884
0885     /* Return */
0886     \_RFI
0887
0888 #ifndef PPC_EXC_CONFIG_BOOKE_ONLY
0889
0890 wrap_change_msr_\_FLVR:
0891
0892     mfmsr   SCRATCH_REGISTER_1
0893     or  SCRATCH_REGISTER_1, SCRATCH_REGISTER_1, SCRATCH_REGISTER_0
0894     mtmsr   SCRATCH_REGISTER_1
0895     msync
0896     isync
0897     b   wrap_change_msr_done_\_FLVR
0898
0899 wrap_restore_msr_\_FLVR:
0900
0901     lwz SCRATCH_REGISTER_0, ppc_exc_msr_bits@sdarel(r13)
0902     mfmsr   SCRATCH_REGISTER_1
0903     andc    SCRATCH_REGISTER_1, SCRATCH_REGISTER_1, SCRATCH_REGISTER_0
0904     mtmsr   SCRATCH_REGISTER_1
0905     msync
0906     isync
0907     b   wrap_restore_msr_done_\_FLVR
0908
0909 #endif /* PPC_EXC_CONFIG_BOOKE_ONLY */
0910
0911 wrap_save_non_volatile_regs_\_FLVR:
0912
0913     /* Load pristine stack pointer */
0914     lwz SCRATCH_REGISTER_1, 0(FRAME_REGISTER)
0915
0916     /* Save small data area anchor (SYSV) */
0917     stw r13, GPR13_OFFSET(FRAME_REGISTER)
0918
0919     /* Save pristine stack pointer */
0920     stw SCRATCH_REGISTER_1, GPR1_OFFSET(FRAME_REGISTER)
0921
0922     /* r14 is the FRAME_REGISTER and will be saved elsewhere */
0923
0924     /* Save non-volatile registers r15 .. r31 */
0925 #ifndef __SPE__
0926     stmw    r15, GPR15_OFFSET(FRAME_REGISTER)
0927 #else
0928     stw r15, GPR15_OFFSET(FRAME_REGISTER)
0929     stw r16, GPR16_OFFSET(FRAME_REGISTER)
0930     stw r17, GPR17_OFFSET(FRAME_REGISTER)
0931     stw r18, GPR18_OFFSET(FRAME_REGISTER)
0932     stw r19, GPR19_OFFSET(FRAME_REGISTER)
0933     stw r20, GPR20_OFFSET(FRAME_REGISTER)
0934     stw r21, GPR21_OFFSET(FRAME_REGISTER)
0935     stw r22, GPR22_OFFSET(FRAME_REGISTER)
0936     stw r23, GPR23_OFFSET(FRAME_REGISTER)
0937     stw r24, GPR24_OFFSET(FRAME_REGISTER)
0938     stw r25, GPR25_OFFSET(FRAME_REGISTER)
0939     stw r26, GPR26_OFFSET(FRAME_REGISTER)
0940     stw r27, GPR27_OFFSET(FRAME_REGISTER)
0941     stw r28, GPR28_OFFSET(FRAME_REGISTER)
0942     stw r29, GPR29_OFFSET(FRAME_REGISTER)
0943     stw r30, GPR30_OFFSET(FRAME_REGISTER)
0944     stw r31, GPR31_OFFSET(FRAME_REGISTER)
0945 #endif
0946
0947 #ifdef PPC_MULTILIB_ALTIVEC
0948     li  SCRATCH_REGISTER_1, PPC_EXC_VR_OFFSET(20)
0949     stvx    v20, FRAME_REGISTER, SCRATCH_REGISTER_1
0950     li  SCRATCH_REGISTER_1, PPC_EXC_VR_OFFSET(21)
0951     stvx    v21, FRAME_REGISTER, SCRATCH_REGISTER_1
0952     li  SCRATCH_REGISTER_1, PPC_EXC_VR_OFFSET(22)
0953     stvx    v22, FRAME_REGISTER, SCRATCH_REGISTER_1
0954     li  SCRATCH_REGISTER_1, PPC_EXC_VR_OFFSET(23)
0955     stvx    v23, FRAME_REGISTER, SCRATCH_REGISTER_1
0956     li  SCRATCH_REGISTER_1, PPC_EXC_VR_OFFSET(24)
0957     stvx    v24, FRAME_REGISTER, SCRATCH_REGISTER_1
0958     li  SCRATCH_REGISTER_1, PPC_EXC_VR_OFFSET(25)
0959     stvx    v25, FRAME_REGISTER, SCRATCH_REGISTER_1
0960     li  SCRATCH_REGISTER_1, PPC_EXC_VR_OFFSET(26)
0961     stvx    v26, FRAME_REGISTER, SCRATCH_REGISTER_1
0962     li  SCRATCH_REGISTER_1, PPC_EXC_VR_OFFSET(27)
0963     stvx    v27, FRAME_REGISTER, SCRATCH_REGISTER_1
0964     li  SCRATCH_REGISTER_1, PPC_EXC_VR_OFFSET(28)
0965     stvx    v28, FRAME_REGISTER, SCRATCH_REGISTER_1
0966     li  SCRATCH_REGISTER_1, PPC_EXC_VR_OFFSET(29)
0967     stvx    v29, FRAME_REGISTER, SCRATCH_REGISTER_1
0968     li  SCRATCH_REGISTER_1, PPC_EXC_VR_OFFSET(30)
0969     stvx    v30, FRAME_REGISTER, SCRATCH_REGISTER_1
0970     li  SCRATCH_REGISTER_1, PPC_EXC_VR_OFFSET(31)
0971     stvx    v31, FRAME_REGISTER, SCRATCH_REGISTER_1
0972     mfvrsave    SCRATCH_REGISTER_1
0973     stw SCRATCH_REGISTER_1, PPC_EXC_VRSAVE_OFFSET(FRAME_REGISTER)
0974 #endif
0975
0976 #ifdef PPC_MULTILIB_FPU
0977     stfd    f14, PPC_EXC_FR_OFFSET(14)(FRAME_REGISTER)
0978     stfd    f15, PPC_EXC_FR_OFFSET(15)(FRAME_REGISTER)
0979     stfd    f16, PPC_EXC_FR_OFFSET(16)(FRAME_REGISTER)
0980     stfd    f17, PPC_EXC_FR_OFFSET(17)(FRAME_REGISTER)
0981     stfd    f18, PPC_EXC_FR_OFFSET(18)(FRAME_REGISTER)
0982     stfd    f19, PPC_EXC_FR_OFFSET(19)(FRAME_REGISTER)
0983     stfd    f20, PPC_EXC_FR_OFFSET(20)(FRAME_REGISTER)
0984     stfd    f21, PPC_EXC_FR_OFFSET(21)(FRAME_REGISTER)
0985     stfd    f22, PPC_EXC_FR_OFFSET(22)(FRAME_REGISTER)
0986     stfd    f23, PPC_EXC_FR_OFFSET(23)(FRAME_REGISTER)
0987     stfd    f24, PPC_EXC_FR_OFFSET(24)(FRAME_REGISTER)
0988     stfd    f25, PPC_EXC_FR_OFFSET(25)(FRAME_REGISTER)
0989     stfd    f26, PPC_EXC_FR_OFFSET(26)(FRAME_REGISTER)
0990     stfd    f27, PPC_EXC_FR_OFFSET(27)(FRAME_REGISTER)
0991     stfd    f28, PPC_EXC_FR_OFFSET(28)(FRAME_REGISTER)
0992     stfd    f29, PPC_EXC_FR_OFFSET(29)(FRAME_REGISTER)
0993     stfd    f30, PPC_EXC_FR_OFFSET(30)(FRAME_REGISTER)
0994     stfd    f31, PPC_EXC_FR_OFFSET(31)(FRAME_REGISTER)
0995 #endif
0996
0997     b   wrap_disable_thread_dispatching_done_\_FLVR
0998
0999 wrap_restore_non_volatile_regs_\_FLVR:
1000
1001     /* Load stack pointer */
1002     lwz SCRATCH_REGISTER_0, GPR1_OFFSET(r1)
1003
1004     /* Restore small data area anchor (SYSV) */
1005     lwz r13, GPR13_OFFSET(r1)
1006
1007     /* r14 is the FRAME_REGISTER and will be restored elsewhere */
1008
1009     /* Restore non-volatile registers r15 .. r31 */
1010 #ifndef __SPE__
1011     lmw r15, GPR15_OFFSET(r1)
1012 #else
1013     lwz r15, GPR15_OFFSET(FRAME_REGISTER)
1014     lwz r16, GPR16_OFFSET(FRAME_REGISTER)
1015     lwz r17, GPR17_OFFSET(FRAME_REGISTER)
1016     lwz r18, GPR18_OFFSET(FRAME_REGISTER)
1017     lwz r19, GPR19_OFFSET(FRAME_REGISTER)
1018     lwz r20, GPR20_OFFSET(FRAME_REGISTER)
1019     lwz r21, GPR21_OFFSET(FRAME_REGISTER)
1020     lwz r22, GPR22_OFFSET(FRAME_REGISTER)
1021     lwz r23, GPR23_OFFSET(FRAME_REGISTER)
1022     lwz r24, GPR24_OFFSET(FRAME_REGISTER)
1023     lwz r25, GPR25_OFFSET(FRAME_REGISTER)
1024     lwz r26, GPR26_OFFSET(FRAME_REGISTER)
1025     lwz r27, GPR27_OFFSET(FRAME_REGISTER)
1026     lwz r28, GPR28_OFFSET(FRAME_REGISTER)
1027     lwz r29, GPR29_OFFSET(FRAME_REGISTER)
1028     lwz r30, GPR30_OFFSET(FRAME_REGISTER)
1029     lwz r31, GPR31_OFFSET(FRAME_REGISTER)
1030 #endif
1031
1032     /* Restore stack pointer */
1033     stw SCRATCH_REGISTER_0, 0(r1)
1034
1035 #ifdef PPC_MULTILIB_ALTIVEC
1036     li  SCRATCH_REGISTER_1, PPC_EXC_VR_OFFSET(20)
1037     lvx v20, FRAME_REGISTER, SCRATCH_REGISTER_1
1038     li  SCRATCH_REGISTER_1, PPC_EXC_VR_OFFSET(21)
1039     lvx v21, FRAME_REGISTER, SCRATCH_REGISTER_1
1040     li  SCRATCH_REGISTER_1, PPC_EXC_VR_OFFSET(22)
1041     lvx v22, FRAME_REGISTER, SCRATCH_REGISTER_1
1042     li  SCRATCH_REGISTER_1, PPC_EXC_VR_OFFSET(23)
1043     lvx v23, FRAME_REGISTER, SCRATCH_REGISTER_1
1044     li  SCRATCH_REGISTER_1, PPC_EXC_VR_OFFSET(24)
1045     lvx v24, FRAME_REGISTER, SCRATCH_REGISTER_1
1046     li  SCRATCH_REGISTER_1, PPC_EXC_VR_OFFSET(25)
1047     lvx v25, FRAME_REGISTER, SCRATCH_REGISTER_1
1048     li  SCRATCH_REGISTER_1, PPC_EXC_VR_OFFSET(26)
1049     lvx v26, FRAME_REGISTER, SCRATCH_REGISTER_1
1050     li  SCRATCH_REGISTER_1, PPC_EXC_VR_OFFSET(27)
1051     lvx v27, FRAME_REGISTER, SCRATCH_REGISTER_1
1052     li  SCRATCH_REGISTER_1, PPC_EXC_VR_OFFSET(28)
1053     lvx v28, FRAME_REGISTER, SCRATCH_REGISTER_1
1054     li  SCRATCH_REGISTER_1, PPC_EXC_VR_OFFSET(29)
1055     lvx v29, FRAME_REGISTER, SCRATCH_REGISTER_1
1056     li  SCRATCH_REGISTER_1, PPC_EXC_VR_OFFSET(30)
1057     lvx v30, FRAME_REGISTER, SCRATCH_REGISTER_1
1058     li  SCRATCH_REGISTER_1, PPC_EXC_VR_OFFSET(31)
1059     lvx v31, FRAME_REGISTER, SCRATCH_REGISTER_1
1060     lwz SCRATCH_REGISTER_1, PPC_EXC_VRSAVE_OFFSET(FRAME_REGISTER)
1061     mtvrsave    SCRATCH_REGISTER_1
1062 #endif
1063
1064 #ifdef PPC_MULTILIB_FPU
1065     lfd f14, PPC_EXC_FR_OFFSET(14)(FRAME_REGISTER)
1066     lfd f15, PPC_EXC_FR_OFFSET(15)(FRAME_REGISTER)
1067     lfd f16, PPC_EXC_FR_OFFSET(16)(FRAME_REGISTER)
1068     lfd f17, PPC_EXC_FR_OFFSET(17)(FRAME_REGISTER)
1069     lfd f18, PPC_EXC_FR_OFFSET(18)(FRAME_REGISTER)
1070     lfd f19, PPC_EXC_FR_OFFSET(19)(FRAME_REGISTER)
1071     lfd f20, PPC_EXC_FR_OFFSET(20)(FRAME_REGISTER)
1072     lfd f21, PPC_EXC_FR_OFFSET(21)(FRAME_REGISTER)
1073     lfd f22, PPC_EXC_FR_OFFSET(22)(FRAME_REGISTER)
1074     lfd f23, PPC_EXC_FR_OFFSET(23)(FRAME_REGISTER)
1075     lfd f24, PPC_EXC_FR_OFFSET(24)(FRAME_REGISTER)
1076     lfd f25, PPC_EXC_FR_OFFSET(25)(FRAME_REGISTER)
1077     lfd f26, PPC_EXC_FR_OFFSET(26)(FRAME_REGISTER)
1078     lfd f27, PPC_EXC_FR_OFFSET(27)(FRAME_REGISTER)
1079     lfd f28, PPC_EXC_FR_OFFSET(28)(FRAME_REGISTER)
1080     lfd f29, PPC_EXC_FR_OFFSET(29)(FRAME_REGISTER)
1081     lfd f30, PPC_EXC_FR_OFFSET(30)(FRAME_REGISTER)
1082     lfd f31, PPC_EXC_FR_OFFSET(31)(FRAME_REGISTER)
1083 #endif
1084
1085     b   wrap_thread_dispatching_done_\_FLVR
1086
1087 wrap_call_global_handler_\_FLVR:
1088
1089     /* First parameter = exception frame pointer + FRAME_LINK_SPACE */
1090     addi    r3, FRAME_REGISTER, FRAME_LINK_SPACE
1091
1092 #ifndef PPC_EXC_CONFIG_USE_FIXED_HANDLER
1093
1094     /* Load global handler address */
1095     LW  SCRATCH_REGISTER_0, globalExceptHdl
1096
1097     /* Check address */
1098     cmpwi   SCRATCH_REGISTER_0, 0
1099     beq wrap_handler_done_\_FLVR
1100
1101     /* Call global handler */
1102     mtctr   SCRATCH_REGISTER_0
1103     bctrl
1104
1105 #else /* PPC_EXC_CONFIG_USE_FIXED_HANDLER */
1106
1107     /* Call fixed global handler */
1108     bl  C_exception_handler
1109
1110 #endif /* PPC_EXC_CONFIG_USE_FIXED_HANDLER */
1111
1112     b   wrap_handler_done_\_FLVR
1113
1114     .endm