/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * Generic barrier definitions. * * It should be possible to use these on really simple architectures, * but it serves more as a starting point for new ports. * * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved. * Written by David Howells (dhowells@redhat.com)
*/ #ifndef __ASM_GENERIC_BARRIER_H #define __ASM_GENERIC_BARRIER_H
/* * Architectures that want generic instrumentation can define __ prefixed * variants of all barriers.
*/
#ifdef __mb #define mb() do { kcsan_mb(); __mb(); } while (0) #endif
#ifdef __rmb #define rmb() do { kcsan_rmb(); __rmb(); } while (0) #endif
#ifdef __wmb #define wmb() do { kcsan_wmb(); __wmb(); } while (0) #endif
#ifdef __dma_mb #define dma_mb() do { kcsan_mb(); __dma_mb(); } while (0) #endif
#ifdef __dma_rmb #define dma_rmb() do { kcsan_rmb(); __dma_rmb(); } while (0) #endif
#ifdef __dma_wmb #define dma_wmb() do { kcsan_wmb(); __dma_wmb(); } while (0) #endif
/* * Force strict CPU ordering. And yes, this is required on UP too when we're * talking to devices. * * Fall back to compiler barriers if nothing better is provided.
*/
/* Barriers for virtual machine guests when talking to an SMP host */ #define virt_mb() do { kcsan_mb(); __smp_mb(); } while (0) #define virt_rmb() do { kcsan_rmb(); __smp_rmb(); } while (0) #define virt_wmb() do { kcsan_wmb(); __smp_wmb(); } while (0) #define virt_store_mb(var, value) do { kcsan_mb(); __smp_store_mb(var, value); } while (0) #define virt_mb__before_atomic() do { kcsan_mb(); __smp_mb__before_atomic(); } while (0) #define virt_mb__after_atomic() do { kcsan_mb(); __smp_mb__after_atomic(); } while (0) #define virt_store_release(p, v) do { kcsan_release(); __smp_store_release(p, v); } while(0) #define virt_load_acquire(p) __smp_load_acquire(p)
/** * smp_acquire__after_ctrl_dep() - Provide ACQUIRE ordering after a control dependency * * A control dependency provides a LOAD->STORE order, the additional RMB * provides LOAD->LOAD order, together they provide LOAD->{LOAD,STORE} order, * aka. (load)-ACQUIRE. * * Architectures that do not do load speculation can have this be barrier().
*/ #ifndef smp_acquire__after_ctrl_dep #define smp_acquire__after_ctrl_dep() smp_rmb() #endif
/** * smp_cond_load_relaxed() - (Spin) wait for cond with no ordering guarantees * @ptr: pointer to the variable to wait on * @cond: boolean expression to wait for * * Equivalent to using READ_ONCE() on the condition variable. * * Due to C lacking lambda expressions we load the value of *ptr into a * pre-named variable @VAL to be used in @cond.
*/ #ifndef smp_cond_load_relaxed #define smp_cond_load_relaxed(ptr, cond_expr) ({ \
typeof(ptr) __PTR = (ptr); \
__unqual_scalar_typeof(*ptr) VAL; \ for (;;) { \
VAL = READ_ONCE(*__PTR); \ if (cond_expr) \ break; \
cpu_relax(); \
} \
(typeof(*ptr))VAL; \
}) #endif
/** * smp_cond_load_acquire() - (Spin) wait for cond with ACQUIRE ordering * @ptr: pointer to the variable to wait on * @cond: boolean expression to wait for * * Equivalent to using smp_load_acquire() on the condition variable but employs * the control dependency of the wait to reduce the barrier on many platforms.
*/ #ifndef smp_cond_load_acquire #define smp_cond_load_acquire(ptr, cond_expr) ({ \
__unqual_scalar_typeof(*ptr) _val; \
_val = smp_cond_load_relaxed(ptr, cond_expr); \
smp_acquire__after_ctrl_dep(); \
(typeof(*ptr))_val; \
}) #endif
/* * pmem_wmb() ensures that all stores for which the modification * are written to persistent storage by preceding instructions have * updated persistent storage before any data access or data transfer * caused by subsequent instructions is initiated.
*/ #ifndef pmem_wmb #define pmem_wmb() wmb() #endif
/* * ioremap_wc() maps I/O memory as memory with write-combining attributes. For * this kind of memory accesses, the CPU may wait for prior accesses to be * merged with subsequent ones. In some situation, such wait is bad for the * performance. io_stop_wc() can be used to prevent the merging of * write-combining memory accesses before this macro with those after it.
*/ #ifndef io_stop_wc #define io_stop_wc() do { } while (0) #endif
/* * Architectures that guarantee an implicit smp_mb() in switch_mm() * can override smp_mb__after_switch_mm.
*/ #ifndef smp_mb__after_switch_mm # define smp_mb__after_switch_mm() smp_mb() #endif
