diff --git a/arch/arm/core/aarch32/mpu/arm_mpu.c b/arch/arm/core/aarch32/mpu/arm_mpu.c index f41f497e3a..b0965798df 100644 --- a/arch/arm/core/aarch32/mpu/arm_mpu.c +++ b/arch/arm/core/aarch32/mpu/arm_mpu.c @@ -7,6 +7,7 @@ #include #include #include +#include #include "arm_core_mpu_dev.h" #include #include @@ -199,7 +200,7 @@ void arm_core_mpu_enable(void) void arm_core_mpu_disable(void) { /* Force any outstanding transfers to complete before disabling MPU */ - __DMB(); + barrier_dmem_fence_full(); /* Disable MPU */ MPU->CTRL = 0; diff --git a/arch/arm/core/aarch32/mpu/nxp_mpu.c b/arch/arm/core/aarch32/mpu/nxp_mpu.c index 5469433132..bb55c0f7ab 100644 --- a/arch/arm/core/aarch32/mpu/nxp_mpu.c +++ b/arch/arm/core/aarch32/mpu/nxp_mpu.c @@ -11,6 +11,7 @@ #include "arm_core_mpu_dev.h" #include #include +#include #include #define LOG_LEVEL CONFIG_MPU_LOG_LEVEL @@ -408,7 +409,7 @@ void arm_core_mpu_enable(void) void arm_core_mpu_disable(void) { /* Force any outstanding transfers to complete before disabling MPU */ - __DMB(); + barrier_dmem_fence_full(); /* Disable MPU */ SYSMPU->CESR &= ~SYSMPU_CESR_VLD_MASK; diff --git a/arch/arm64/core/cortex_r/arm_mpu.c b/arch/arm64/core/cortex_r/arm_mpu.c index 58b7dcd1eb..63a368c8cd 100644 --- a/arch/arm64/core/cortex_r/arm_mpu.c +++ b/arch/arm64/core/cortex_r/arm_mpu.c @@ -13,6 +13,7 @@ #include #include #include +#include LOG_MODULE_REGISTER(mpu, CONFIG_MPU_LOG_LEVEL); @@ -87,7 +88,7 @@ void arm_core_mpu_disable(void) uint64_t val; /* Force any outstanding transfers to complete before disabling MPU */ - dmb(); + barrier_dmem_fence_full(); val = read_sctlr_el1(); val &= ~SCTLR_M_BIT; diff --git a/drivers/ethernet/eth_dwmac.c b/drivers/ethernet/eth_dwmac.c index 41bef3288c..498f6b17ca 100644 --- a/drivers/ethernet/eth_dwmac.c +++ b/drivers/ethernet/eth_dwmac.c @@ -16,6 +16,7 @@ LOG_MODULE_REGISTER(LOG_MODULE_NAME); #include #include #include +#include #include #include "eth_dwmac_priv.h" @@ -188,7 +189,7 @@ static int dwmac_send(const struct device *dev, struct net_pkt *pkt) } while (frag); /* make sure all the above made it to memory */ - __DMB(); + barrier_dmem_fence_full(); /* update the descriptor index head */ p->tx_desc_head = d_idx; @@ -380,7 +381,7 @@ static void dwmac_rx_refill_thread(void *arg1, void *unused1, void *unused2) d->des3 = RDES3_BUF1V | RDES3_IOC | RDES3_OWN; /* commit the above to memory */ - __DMB(); + barrier_dmem_fence_full(); /* advance to the next descriptor */ p->rx_desc_head = INC_WRAP(d_idx, NB_RX_DESCS); diff --git a/drivers/ethernet/eth_sam_gmac.c b/drivers/ethernet/eth_sam_gmac.c index 3caa9f368a..05aa7e10d1 100644 --- a/drivers/ethernet/eth_sam_gmac.c +++ b/drivers/ethernet/eth_sam_gmac.c @@ -34,6 +34,7 @@ LOG_MODULE_REGISTER(LOG_MODULE_NAME); #include #include #include +#include #include #include #include @@ -1338,7 +1339,7 @@ static struct net_pkt *frame_get(struct gmac_queue *queue) /* Guarantee that status word is written before the address * word to avoid race condition. */ - __DMB(); /* data memory barrier */ + barrier_dmem_fence_full(); /* Update buffer descriptor address word */ wrap = (tail == rx_desc_list->len-1U ? GMAC_RXW0_WRAP : 0); rx_desc->w0 = ((uint32_t)frag->data & GMAC_RXW0_ADDR) | wrap; @@ -1583,7 +1584,7 @@ static int eth_tx(const struct device *dev, struct net_pkt *pkt) /* Guarantee that all the fragments have been written before removing * the used bit to avoid race condition. */ - __DMB(); /* data memory barrier */ + barrier_dmem_fence_full(); /* Remove the used bit of the first fragment to allow the controller * to process it and the following fragments. @@ -1605,7 +1606,7 @@ static int eth_tx(const struct device *dev, struct net_pkt *pkt) /* Guarantee that the first fragment got its bit removed before starting * sending packets to avoid packets getting stuck. */ - __DMB(); /* data memory barrier */ + barrier_dmem_fence_full(); /* Start transmission */ gmac->GMAC_NCR |= GMAC_NCR_TSTART; diff --git a/drivers/serial/uart_pl011.c b/drivers/serial/uart_pl011.c index 6b93ae9802..ee009c4d17 100644 --- a/drivers/serial/uart_pl011.c +++ b/drivers/serial/uart_pl011.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #if defined(CONFIG_PINCTRL) #include @@ -212,7 +213,7 @@ static int pl011_set_baudrate(const struct device *dev, get_uart(dev)->ibrd = bauddiv >> PL011_FBRD_WIDTH; get_uart(dev)->fbrd = bauddiv & ((1u << PL011_FBRD_WIDTH) - 1u); - __DMB(); + barrier_dmem_fence_full(); /* In order to internally update the contents of ibrd or fbrd, a * lcr_h write must always be performed at the end diff --git a/drivers/timer/nrf_rtc_timer.c b/drivers/timer/nrf_rtc_timer.c index 724ee3f058..dcfd1dde11 100644 --- a/drivers/timer/nrf_rtc_timer.c +++ b/drivers/timer/nrf_rtc_timer.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -152,7 +153,7 @@ static bool compare_int_lock(int32_t chan) nrf_rtc_int_disable(RTC, NRF_RTC_CHANNEL_INT_MASK(chan)); - __DMB(); + barrier_dmem_fence_full(); __ISB(); return prev & BIT(chan); @@ -387,7 +388,7 @@ uint64_t z_nrf_rtc_timer_read(void) { uint64_t val = ((uint64_t)overflow_cnt) << COUNTER_BIT_WIDTH; - __DMB(); + barrier_dmem_fence_full(); uint32_t cntr = counter(); diff --git a/drivers/usb/device/usb_dc_sam_usbc.c b/drivers/usb/device/usb_dc_sam_usbc.c index 4ad70024ff..b0c9b6d417 100644 --- a/drivers/usb/device/usb_dc_sam_usbc.c +++ b/drivers/usb/device/usb_dc_sam_usbc.c @@ -14,6 +14,7 @@ LOG_MODULE_REGISTER(usb_dc_sam_usbc, CONFIG_USB_DRIVER_LOG_LEVEL); #include #include #include +#include #include #include @@ -654,7 +655,7 @@ static void usb_dc_sam_usbc_isr(void) } usb_dc_sam_usbc_isr_barrier: - __DMB(); + barrier_dmem_fence_full(); } int usb_dc_attach(void) diff --git a/drivers/xen/events.c b/drivers/xen/events.c index 7229dd46d8..e077b0f6cf 100644 --- a/drivers/xen/events.c +++ b/drivers/xen/events.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -219,7 +220,7 @@ static void events_isr(void *data) */ vcpu->evtchn_upcall_pending = 0; - dmb(); + barrier_dmem_fence_full(); /* Can not use system atomic_t/atomic_set() due to 32-bit casting */ pos_selector = __atomic_exchange_n(&vcpu->evtchn_pending_sel, diff --git a/drivers/xen/gnttab.c b/drivers/xen/gnttab.c index 8cb0fe299e..d871eb62d2 100644 --- a/drivers/xen/gnttab.c +++ b/drivers/xen/gnttab.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include @@ -86,7 +87,7 @@ static void gnttab_grant_permit_access(grant_ref_t gref, domid_t domid, gnttab.table[gref].frame = gfn; gnttab.table[gref].domid = domid; /* Need to be sure that gfn and domid will be set before flags */ - __DMB(); + barrier_dmem_fence_full(); gnttab.table[gref].flags = flags; } diff --git a/include/zephyr/arch/arm/aarch32/cortex_a_r/sys_io.h b/include/zephyr/arch/arm/aarch32/cortex_a_r/sys_io.h index 68ede510ac..c1615c7a15 100644 --- a/include/zephyr/arch/arm/aarch32/cortex_a_r/sys_io.h +++ b/include/zephyr/arch/arm/aarch32/cortex_a_r/sys_io.h @@ -17,6 +17,7 @@ #include #include #include +#include #ifdef __cplusplus extern "C" { @@ -30,13 +31,13 @@ static ALWAYS_INLINE uint8_t sys_read8(mem_addr_t addr) __asm__ volatile("ldrb %0, [%1]" : "=r" (val) : "r" (addr)); - __DMB(); + barrier_dmem_fence_full(); return val; } static ALWAYS_INLINE void sys_write8(uint8_t data, mem_addr_t addr) { - __DMB(); + barrier_dmem_fence_full(); __asm__ volatile("strb %0, [%1]" : : "r" (data), "r" (addr)); } @@ -46,13 +47,13 @@ static ALWAYS_INLINE uint16_t sys_read16(mem_addr_t addr) __asm__ volatile("ldrh %0, [%1]" : "=r" (val) : "r" (addr)); - __DMB(); + barrier_dmem_fence_full(); return val; } static ALWAYS_INLINE void sys_write16(uint16_t data, mem_addr_t addr) { - __DMB(); + barrier_dmem_fence_full(); __asm__ volatile("strh %0, [%1]" : : "r" (data), "r" (addr)); } @@ -62,13 +63,13 @@ static ALWAYS_INLINE uint32_t sys_read32(mem_addr_t addr) __asm__ volatile("ldr %0, [%1]" : "=r" (val) : "r" (addr)); - __DMB(); + barrier_dmem_fence_full(); return val; } static ALWAYS_INLINE void sys_write32(uint32_t data, mem_addr_t addr) { - __DMB(); + barrier_dmem_fence_full(); __asm__ volatile("str %0, [%1]" : : "r" (data), "r" (addr)); } @@ -78,7 +79,7 @@ static ALWAYS_INLINE uint64_t sys_read64(mem_addr_t addr) __asm__ volatile("ldrd %Q0, %R0, [%1]" : "=r" (val) : "r" (addr)); - __DMB(); + barrier_dmem_fence_full(); return val; } diff --git a/include/zephyr/arch/arm64/lib_helpers.h b/include/zephyr/arch/arm64/lib_helpers.h index f133f4b468..1c4abb4fc6 100644 --- a/include/zephyr/arch/arm64/lib_helpers.h +++ b/include/zephyr/arch/arm64/lib_helpers.h @@ -155,12 +155,10 @@ static ALWAYS_INLINE void disable_fiq(void) #define wfi() __asm__ volatile("wfi" : : : "memory") #define dsb() __asm__ volatile ("dsb sy" ::: "memory") -#define dmb() __asm__ volatile ("dmb sy" ::: "memory") #define isb() __asm__ volatile ("isb" ::: "memory") /* Zephyr needs these as well */ #define __ISB() isb() -#define __DMB() dmb() #define __DSB() dsb() static inline bool is_el_implemented(unsigned int el) diff --git a/include/zephyr/arch/arm64/sys_io.h b/include/zephyr/arch/arm64/sys_io.h index 08e62874a7..d78be72305 100644 --- a/include/zephyr/arch/arm64/sys_io.h +++ b/include/zephyr/arch/arm64/sys_io.h @@ -16,6 +16,7 @@ #include #include +#include #ifdef __cplusplus extern "C" { @@ -39,13 +40,13 @@ static ALWAYS_INLINE uint8_t sys_read8(mem_addr_t addr) __asm__ volatile("ldrb %w0, [%1]" : "=r" (val) : "r" (addr)); - __DMB(); + barrier_dmem_fence_full(); return val; } static ALWAYS_INLINE void sys_write8(uint8_t data, mem_addr_t addr) { - __DMB(); + barrier_dmem_fence_full(); __asm__ volatile("strb %w0, [%1]" : : "r" (data), "r" (addr)); } @@ -55,13 +56,13 @@ static ALWAYS_INLINE uint16_t sys_read16(mem_addr_t addr) __asm__ volatile("ldrh %w0, [%1]" : "=r" (val) : "r" (addr)); - __DMB(); + barrier_dmem_fence_full(); return val; } static ALWAYS_INLINE void sys_write16(uint16_t data, mem_addr_t addr) { - __DMB(); + barrier_dmem_fence_full(); __asm__ volatile("strh %w0, [%1]" : : "r" (data), "r" (addr)); } @@ -71,13 +72,13 @@ static ALWAYS_INLINE uint32_t sys_read32(mem_addr_t addr) __asm__ volatile("ldr %w0, [%1]" : "=r" (val) : "r" (addr)); - __DMB(); + barrier_dmem_fence_full(); return val; } static ALWAYS_INLINE void sys_write32(uint32_t data, mem_addr_t addr) { - __DMB(); + barrier_dmem_fence_full(); __asm__ volatile("str %w0, [%1]" : : "r" (data), "r" (addr)); } @@ -87,13 +88,13 @@ static ALWAYS_INLINE uint64_t sys_read64(mem_addr_t addr) __asm__ volatile("ldr %x0, [%1]" : "=r" (val) : "r" (addr)); - __DMB(); + barrier_dmem_fence_full(); return val; } static ALWAYS_INLINE void sys_write64(uint64_t data, mem_addr_t addr) { - __DMB(); + barrier_dmem_fence_full(); __asm__ volatile("str %x0, [%1]" : : "r" (data), "r" (addr)); } diff --git a/tests/arch/arm/arm_thread_swap/src/arm_thread_arch.c b/tests/arch/arm/arm_thread_swap/src/arm_thread_arch.c index d7b058c07a..d18161c850 100644 --- a/tests/arch/arm/arm_thread_swap/src/arm_thread_arch.c +++ b/tests/arch/arm/arm_thread_swap/src/arm_thread_arch.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -392,7 +393,7 @@ static void alt_thread_entry(void) /* Manually trigger a context-switch, to swap-out * the alternative test thread. */ - __DMB(); + barrier_dmem_fence_full(); SCB->ICSR |= SCB_ICSR_PENDSVSET_Msk; irq_unlock(0); @@ -593,7 +594,7 @@ ZTEST(arm_thread_swap, test_arm_thread_swap) /* Manually trigger a context-switch to swap-out the current thread. * Request a return to a different interrupt lock state. */ - __DMB(); + barrier_dmem_fence_full(); #if defined(CONFIG_NO_OPTIMIZATIONS) SCB->ICSR |= SCB_ICSR_PENDSVSET_Msk;