dev/dma/fsl-edma.c

0001 /**
0002  * @file
0003  *
0004  * @ingroup RTEMSBSPsSharedFslEDMA
0005  *
0006  * @brief Freescale / NXP Enhanced Direct Memory Access (eDMA).
0007  */
0008
0009 /*
0010  * Copyright (C) 2008, 2020 embedded brains GmbH & Co. KG
0011  *
0012  * Redistribution and use in source and binary forms, with or without
0013  * modification, are permitted provided that the following conditions
0014  * are met:
0015  * 1. Redistributions of source code must retain the above copyright
0016  *    notice, this list of conditions and the following disclaimer.
0017  * 2. Redistributions in binary form must reproduce the above copyright
0018  *    notice, this list of conditions and the following disclaimer in the
0019  *    documentation and/or other materials provided with the distribution.
0020  *
0021  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
0022  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
0023  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
0024  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
0025  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
0026  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
0027  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
0028  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
0029  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
0030  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
0031  * POSSIBILITY OF SUCH DAMAGE.
0032  */
0033
0034 #include <fsl/edma.h>
0035 #include <fsl/regs-edma.h>
0036
0037 #include <assert.h>
0038
0039 #include <bsp.h>
0040 #include <bsp/fatal.h>
0041 #include <bsp/irq.h>
0042 #ifdef LIBBSP_ARM_IMXRT_BSP_H
0043 #include <fsl_device_registers.h>
0044 #endif
0045
0046 #define EDMA_CHANNELS_PER_GROUP 32U
0047
0048 #define EDMA_GROUP_COUNT ((EDMA_CHANNEL_COUNT + 31U) / 32U)
0049
0050 #define EDMA_GROUP_INDEX(channel) ((channel) / EDMA_CHANNELS_PER_GROUP)
0051
0052 #define EDMA_GROUP_BIT(channel) (1U << ((channel) % EDMA_CHANNELS_PER_GROUP))
0053
0054 #define EDMA_MODULE_INDEX(channel) ((channel) / EDMA_CHANNELS_PER_MODULE)
0055
0056 static uint32_t edma_channel_occupation [EDMA_GROUP_COUNT];
0057
0058 static RTEMS_CHAIN_DEFINE_EMPTY(edma_channel_chain);
0059
0060 volatile struct fsl_edma *edma_inst[EDMA_MODULE_COUNT] = {
0061 #ifdef LIBBSP_ARM_IMXRT_BSP_H
0062   (volatile struct fsl_edma *) DMA0,
0063 #else /* ! LIBBSP_ARM_IMXRT_BSP_H */
0064   #if EDMA_MODULE_COUNT == 1
0065     (volatile struct fsl_edma *) &EDMA,
0066   #elif EDMA_MODULE_COUNT == 2
0067     (volatile struct fsl_edma *) &EDMA_A,
0068     (volatile struct fsl_edma *) &EDMA_B,
0069   #else
0070     #error "unsupported module count"
0071   #endif
0072 #endif /* LIBBSP_ARM_IMXRT_BSP_H */
0073 };
0074
0075 unsigned fsl_edma_channel_index_of_tcd(
0076   volatile struct fsl_edma_tcd *edma_tcd
0077 )
0078 {
0079   volatile struct fsl_edma *edma = fsl_edma_by_tcd(edma_tcd);
0080   unsigned channel = edma_tcd - &edma->TCD[0];
0081
0082 #if EDMA_MODULE_COUNT == 2
0083   if (edma_inst[1] == edma) {
0084     channel += EDMA_CHANNELS_PER_MODULE;
0085   }
0086 #elif EDMA_MODULE_COUNT > 2
0087   #error "unsupported module count"
0088 #endif
0089
0090   return channel;
0091 }
0092
0093 volatile struct fsl_edma_tcd *fsl_edma_tcd_of_channel_index(unsigned index)
0094 {
0095   unsigned module = index / EDMA_CHANNELS_PER_MODULE;
0096   unsigned channel = index % EDMA_CHANNELS_PER_MODULE;
0097
0098   return &edma_inst[module]->TCD[channel];
0099 }
0100
0101 static volatile struct fsl_edma *fsl_edma_get_regs_by_module(unsigned module)
0102 {
0103   return edma_inst[module];
0104 }
0105
0106 static uint32_t fsl_edma_bit_array_get_lowest_0(uint32_t *bit_array)
0107 {
0108   unsigned array;
0109
0110   for (array = 0; array < EDMA_MODULE_COUNT; ++array) {
0111     uint32_t first_0 = __builtin_ffs(~(bit_array[array]));
0112     if (first_0 > 0) {
0113       return (first_0 - 1) + array * 32;
0114     }
0115   }
0116
0117   return UINT32_MAX;
0118 }
0119
0120 static uint32_t fsl_edma_bit_array_set(unsigned channel, uint32_t *bit_array)
0121 {
0122   unsigned array = channel / 32;
0123   uint32_t bit = 1U << (channel % 32);
0124   uint32_t previous = bit_array [array];
0125
0126   bit_array [array] = previous | bit;
0127
0128   return previous;
0129 }
0130
0131 static uint32_t fsl_edma_bit_array_clear(unsigned channel, uint32_t *bit_array)
0132 {
0133   unsigned array = channel / 32;
0134   uint32_t bit = 1U << (channel % 32);
0135   uint32_t previous = bit_array [array];
0136
0137   bit_array [array] = previous & ~bit;
0138
0139   return previous;
0140 }
0141
0142 static void fsl_edma_interrupt_handler(void *arg)
0143 {
0144   fsl_edma_channel_context *ctx = arg;
0145
0146   fsl_edma_clear_interrupts(ctx->edma_tcd);
0147
0148   (*ctx->done)(ctx, 0);
0149 }
0150
0151 static void edma_interrupt_error_handler(void *arg)
0152 {
0153   rtems_chain_control *chain = &edma_channel_chain;
0154   rtems_chain_node *node = rtems_chain_first(chain);
0155
0156   uint32_t error_channels [] = {
0157 #if EDMA_GROUP_COUNT >= 1
0158     edma_inst[0]->ERL,
0159 #endif
0160 #if EDMA_GROUP_COUNT >= 2
0161     edma_inst[0]->ERH,
0162 #endif
0163 #if EDMA_GROUP_COUNT >= 3
0164     edma_inst[1]->ERL,
0165 #endif
0166   };
0167   uint32_t error_status [] = {
0168 #if EDMA_GROUP_COUNT >= 1
0169     edma_inst[0]->ESR,
0170 #endif
0171 #if EDMA_GROUP_COUNT >= 3
0172     edma_inst[1]->ESR,
0173 #endif
0174   };
0175
0176 #if EDMA_GROUP_COUNT >= 1
0177   edma_inst[0]->ERL = error_channels [0];
0178 #endif
0179 #if EDMA_GROUP_COUNT >= 2
0180   edma_inst[0]->ERH = error_channels [1];
0181 #endif
0182 #if EDMA_GROUP_COUNT >= 3
0183   edma_inst[1]->ERL = error_channels [2];
0184 #endif
0185
0186   while (!rtems_chain_is_tail(chain, node)) {
0187     fsl_edma_channel_context *ctx = (fsl_edma_channel_context *) node;
0188     unsigned channel_index = fsl_edma_channel_index_of_tcd(ctx->edma_tcd);
0189     unsigned group_index = EDMA_GROUP_INDEX(channel_index);
0190     unsigned group_bit = EDMA_GROUP_BIT(channel_index);
0191
0192     if ((error_channels [group_index] & group_bit) != 0) {
0193       unsigned module_index = EDMA_MODULE_INDEX(channel_index);
0194
0195       (*ctx->done)(ctx, error_status [module_index]);
0196     }
0197
0198     node = rtems_chain_next(node);
0199   }
0200 }
0201
0202 void fsl_edma_init(void)
0203 {
0204   rtems_status_code sc = RTEMS_SUCCESSFUL;
0205   unsigned channel_remaining = EDMA_CHANNEL_COUNT;
0206   unsigned module = 0;
0207   unsigned group = 0;
0208
0209   for (module = 0; module < EDMA_MODULE_COUNT; ++module) {
0210     volatile struct fsl_edma *edma = fsl_edma_get_regs_by_module(module);
0211     unsigned channel_count = channel_remaining < EDMA_CHANNELS_PER_MODULE ?
0212       channel_remaining : EDMA_CHANNELS_PER_MODULE;
0213     unsigned channel = 0;
0214
0215     channel_remaining -= channel_count;
0216
0217     /* Disable requests */
0218     edma->CERQR = EDMA_CERQR_CAER;
0219
0220     /* Arbitration mode: group round robin, channel fixed */
0221     edma->CR |= EDMA_CR_ERGA;
0222     edma->CR &= ~EDMA_CR_ERCA;
0223 #if defined(BSP_FSL_EDMA_EMLM)
0224     edma->CR |= EDMA_CR_EMLM;
0225 #endif
0226     for (channel = 0; channel < channel_count; ++channel) {
0227       volatile struct fsl_edma_tcd *tcd = &edma->TCD [channel];
0228       edma->CPR [channel] = EDMA_CPR_ECP | EDMA_CPR_CHPRI(channel);
0229
0230       /* Initialize TCD, stop channel first */
0231       tcd->BMF = 0;
0232       tcd->SADDR = 0;
0233       tcd->SDF = 0;
0234       tcd->NBYTES = 0;
0235       tcd->SLAST = 0;
0236       tcd->DADDR = 0;
0237       tcd->CDF = 0;
0238       tcd->DLAST_SGA = 0;
0239     }
0240
0241     /* Clear interrupt requests */
0242     edma->CIRQR = EDMA_CIRQR_CAIR;
0243     edma->CER = EDMA_CER_CAEI;
0244   }
0245
0246   for (group = 0; group < EDMA_GROUP_COUNT; ++group) {
0247 #if defined(LIBBSP_POWERPC_MPC55XXEVB_BSP_H)
0248     sc = mpc55xx_interrupt_handler_install(
0249       MPC55XX_IRQ_EDMA_ERROR(group),
0250       "eDMA Error",
0251       RTEMS_INTERRUPT_UNIQUE,
0252       MPC55XX_INTC_DEFAULT_PRIORITY,
0253       edma_interrupt_error_handler,
0254       NULL
0255     );
0256 #elif defined(LIBBSP_ARM_IMXRT_BSP_H)
0257     sc = rtems_interrupt_handler_install(
0258       DMA_ERROR_IRQn,
0259       "eDMA Error",
0260       RTEMS_INTERRUPT_UNIQUE,
0261       edma_interrupt_error_handler,
0262       NULL
0263     );
0264 #else
0265   #error "Unknown chip"
0266 #endif
0267     if (sc != RTEMS_SUCCESSFUL) {
0268       bsp_fatal(MPC55XX_FATAL_EDMA_IRQ_INSTALL);
0269     }
0270   }
0271 }
0272
0273 static rtems_status_code fsl_edma_obtain_next_free_channel_and_get_tcd(
0274   volatile struct fsl_edma_tcd **edma_tcd
0275 )
0276 {
0277   rtems_status_code sc = RTEMS_SUCCESSFUL;
0278   unsigned channel_index;
0279   rtems_interrupt_level level;
0280
0281   rtems_interrupt_disable(level);
0282   channel_index = fsl_edma_bit_array_get_lowest_0(&edma_channel_occupation [0]);
0283   if (channel_index > EDMA_CHANNEL_COUNT) {
0284     sc = RTEMS_RESOURCE_IN_USE;
0285   } else {
0286     fsl_edma_bit_array_set(
0287       channel_index,
0288       &edma_channel_occupation [0]
0289     );
0290   }
0291   rtems_interrupt_enable(level);
0292
0293   if (sc == RTEMS_SUCCESSFUL) {
0294     *edma_tcd = fsl_edma_tcd_of_channel_index(channel_index);
0295   }
0296
0297   return sc;
0298 }
0299
0300 rtems_status_code fsl_edma_obtain_channel_by_tcd(
0301   volatile struct fsl_edma_tcd *edma_tcd
0302 )
0303 {
0304   rtems_status_code sc = RTEMS_SUCCESSFUL;
0305   unsigned channel_index = fsl_edma_channel_index_of_tcd(edma_tcd);
0306   rtems_interrupt_level level;
0307   uint32_t channel_occupation;
0308
0309   rtems_interrupt_disable(level);
0310   channel_occupation = fsl_edma_bit_array_set(
0311     channel_index,
0312     &edma_channel_occupation [0]
0313   );
0314   rtems_interrupt_enable(level);
0315
0316   if ((channel_occupation & EDMA_GROUP_BIT(channel_index)) != 0) {
0317     sc = RTEMS_RESOURCE_IN_USE;
0318   }
0319
0320   return sc;
0321 }
0322
0323 void fsl_edma_release_channel_by_tcd(volatile struct fsl_edma_tcd *edma_tcd)
0324 {
0325   unsigned channel_index = fsl_edma_channel_index_of_tcd(edma_tcd);
0326   rtems_interrupt_level level;
0327
0328   rtems_interrupt_disable(level);
0329   fsl_edma_bit_array_clear(channel_index, &edma_channel_occupation [0]);
0330   rtems_interrupt_enable(level);
0331
0332   fsl_edma_disable_hardware_requests(edma_tcd);
0333   fsl_edma_disable_error_interrupts(edma_tcd);
0334 }
0335
0336 static rtems_status_code fsl_edma_install_obtained_channel(
0337   fsl_edma_channel_context *ctx,
0338   unsigned irq_priority
0339 )
0340 {
0341   unsigned channel_index = fsl_edma_channel_index_of_tcd(ctx->edma_tcd);
0342   rtems_status_code sc;
0343
0344 #if defined(LIBBSP_POWERPC_MPC55XXEVB_BSP_H)
0345   sc = fsl_interrupt_handler_install(
0346     MPC55XX_IRQ_EDMA(channel_index),
0347     "eDMA Channel",
0348     RTEMS_INTERRUPT_SHARED,
0349     irq_priority,
0350     fsl_edma_interrupt_handler,
0351     ctx
0352   );
0353 #elif defined(LIBBSP_ARM_IMXRT_BSP_H)
0354   sc = rtems_interrupt_handler_install(
0355     DMA0_DMA16_IRQn + (channel_index % 16),
0356     "eDMA Channel",
0357     RTEMS_INTERRUPT_SHARED,
0358     fsl_edma_interrupt_handler,
0359     ctx
0360   );
0361 #else
0362   #error "Unknown chip"
0363 #endif
0364   if (sc == RTEMS_SUCCESSFUL) {
0365     rtems_chain_prepend(&edma_channel_chain, &ctx->node);
0366     fsl_edma_enable_error_interrupts(ctx->edma_tcd);
0367   } else {
0368     fsl_edma_release_channel_by_tcd(ctx->edma_tcd);
0369     sc = RTEMS_IO_ERROR;
0370   }
0371
0372   return sc;
0373 }
0374
0375 rtems_status_code fsl_edma_obtain_channel(
0376   fsl_edma_channel_context *ctx,
0377   unsigned irq_priority
0378 )
0379 {
0380   rtems_status_code sc = fsl_edma_obtain_channel_by_tcd(ctx->edma_tcd);
0381   if (sc == RTEMS_SUCCESSFUL) {
0382     sc = fsl_edma_install_obtained_channel(ctx, irq_priority);
0383   }
0384
0385   return sc;
0386 }
0387
0388 rtems_status_code fsl_edma_obtain_next_free_channel(
0389   fsl_edma_channel_context *ctx
0390 )
0391 {
0392   rtems_status_code sc;
0393
0394   sc = fsl_edma_obtain_next_free_channel_and_get_tcd(&ctx->edma_tcd);
0395   if (sc == RTEMS_SUCCESSFUL) {
0396     sc = fsl_edma_install_obtained_channel(ctx,
0397 #ifdef LIBBSP_POWERPC_MPC55XXEVB_BSP_H
0398       MPC55XX_INTC_DEFAULT_PRIORITY
0399 #else
0400       0
0401 #endif
0402     );
0403   }
0404
0405   return sc;
0406 }
0407
0408 void fsl_edma_release_channel(fsl_edma_channel_context *ctx)
0409 {
0410   rtems_status_code sc = RTEMS_SUCCESSFUL;
0411   unsigned channel_index = fsl_edma_channel_index_of_tcd(ctx->edma_tcd);
0412
0413   fsl_edma_release_channel_by_tcd(ctx->edma_tcd);
0414   rtems_chain_extract(&ctx->node);
0415
0416   sc = rtems_interrupt_handler_remove(
0417 #if defined(LIBBSP_POWERPC_MPC55XXEVB_BSP_H)
0418     MPC55XX_IRQ_EDMA(channel_index),
0419 #elif defined(LIBBSP_ARM_IMXRT_BSP_H)
0420     DMA0_DMA16_IRQn + (channel_index % 16),
0421 #else
0422   #error "Unknown chip"
0423 #endif
0424     fsl_edma_interrupt_handler,
0425     ctx
0426   );
0427   if (sc != RTEMS_SUCCESSFUL) {
0428     bsp_fatal(MPC55XX_FATAL_EDMA_IRQ_REMOVE);
0429   }
0430 }
0431
0432 void fsl_edma_copy(
0433   volatile struct fsl_edma_tcd *edma_tcd,
0434   const struct fsl_edma_tcd *source_tcd
0435 )
0436 {
0437   /* Clear DONE flag */
0438   edma_tcd->BMF = 0;
0439
0440   edma_tcd->SADDR = source_tcd->SADDR;
0441   edma_tcd->SDF = source_tcd->SDF;
0442   edma_tcd->NBYTES = source_tcd->NBYTES;
0443   edma_tcd->SLAST = source_tcd->SLAST;
0444   edma_tcd->DADDR = source_tcd->DADDR;
0445   edma_tcd->CDF = source_tcd->CDF;
0446   edma_tcd->DLAST_SGA = source_tcd->DLAST_SGA;
0447   edma_tcd->BMF = source_tcd->BMF;
0448 }
0449
0450 void fsl_edma_copy_and_enable_hardware_requests(
0451   volatile struct fsl_edma_tcd *edma_tcd,
0452   const struct fsl_edma_tcd *source_tcd
0453 )
0454 {
0455   fsl_edma_copy(edma_tcd, source_tcd);
0456   fsl_edma_enable_hardware_requests(edma_tcd);
0457 }
0458
0459 void fsl_edma_sg_link(
0460   volatile struct fsl_edma_tcd *edma_tcd,
0461   const struct fsl_edma_tcd *source_tcd
0462 )
0463 {
0464   edma_tcd->DLAST_SGA = (int32_t) source_tcd;
0465   edma_tcd->BMF |= EDMA_TCD_BMF_E_SG;
0466
0467   if ((edma_tcd->BMF & EDMA_TCD_BMF_E_SG) == 0) {
0468     fsl_edma_copy(edma_tcd, source_tcd);
0469   }
0470 }