mirror of
https://github.com/BobTheBlinker/android_kernel_motorola_sm6375.git
synced 2026-10-11 07:03:09 -04:00
Merge branch 'locking-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull locking and atomic updates from Ingo Molnar:
"Main changes in this cycle are:
- Extend atomic primitives with coherent logic op primitives
(atomic_{or,and,xor}()) and deprecate the old partial APIs
(atomic_{set,clear}_mask())
The old ops were incoherent with incompatible signatures across
architectures and with incomplete support. Now every architecture
supports the primitives consistently (by Peter Zijlstra)
- Generic support for 'relaxed atomics':
- _acquire/release/relaxed() flavours of xchg(), cmpxchg() and {add,sub}_return()
- atomic_read_acquire()
- atomic_set_release()
This came out of porting qwrlock code to arm64 (by Will Deacon)
- Clean up the fragile static_key APIs that were causing repeat bugs,
by introducing a new one:
DEFINE_STATIC_KEY_TRUE(name);
DEFINE_STATIC_KEY_FALSE(name);
which define a key of different types with an initial true/false
value.
Then allow:
static_branch_likely()
static_branch_unlikely()
to take a key of either type and emit the right instruction for the
case. To be able to know the 'type' of the static key we encode it
in the jump entry (by Peter Zijlstra)
- Static key self-tests (by Jason Baron)
- qrwlock optimizations (by Waiman Long)
- small futex enhancements (by Davidlohr Bueso)
- ... and misc other changes"
* 'locking-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (63 commits)
jump_label/x86: Work around asm build bug on older/backported GCCs
locking, ARM, atomics: Define our SMP atomics in terms of _relaxed() operations
locking, include/llist: Use linux/atomic.h instead of asm/cmpxchg.h
locking/qrwlock: Make use of _{acquire|release|relaxed}() atomics
locking/qrwlock: Implement queue_write_unlock() using smp_store_release()
locking/lockref: Remove homebrew cmpxchg64_relaxed() macro definition
locking, asm-generic: Add _{relaxed|acquire|release}() variants for 'atomic_long_t'
locking, asm-generic: Rework atomic-long.h to avoid bulk code duplication
locking/atomics: Add _{acquire|release|relaxed}() variants of some atomic operations
locking, compiler.h: Cast away attributes in the WRITE_ONCE() magic
locking/static_keys: Make verify_keys() static
jump label, locking/static_keys: Update docs
locking/static_keys: Provide a selftest
jump_label: Provide a self-test
s390/uaccess, locking/static_keys: employ static_branch_likely()
x86, tsc, locking/static_keys: Employ static_branch_likely()
locking/static_keys: Add selftest
locking/static_keys: Add a new static_key interface
locking/static_keys: Rework update logic
locking/static_keys: Add static_key_{en,dis}able() helpers
...
This commit is contained in:
Linus Torvalds
commit
ca520cab25
139 changed files with 2425 additions and 3585 deletions
|
|
@ -23,236 +23,159 @@
|
|||
typedef atomic64_t atomic_long_t;
|
||||
|
||||
#define ATOMIC_LONG_INIT(i) ATOMIC64_INIT(i)
|
||||
#define ATOMIC_LONG_PFX(x) atomic64 ## x
|
||||
|
||||
static inline long atomic_long_read(atomic_long_t *l)
|
||||
{
|
||||
atomic64_t *v = (atomic64_t *)l;
|
||||
|
||||
return (long)atomic64_read(v);
|
||||
}
|
||||
|
||||
static inline void atomic_long_set(atomic_long_t *l, long i)
|
||||
{
|
||||
atomic64_t *v = (atomic64_t *)l;
|
||||
|
||||
atomic64_set(v, i);
|
||||
}
|
||||
|
||||
static inline void atomic_long_inc(atomic_long_t *l)
|
||||
{
|
||||
atomic64_t *v = (atomic64_t *)l;
|
||||
|
||||
atomic64_inc(v);
|
||||
}
|
||||
|
||||
static inline void atomic_long_dec(atomic_long_t *l)
|
||||
{
|
||||
atomic64_t *v = (atomic64_t *)l;
|
||||
|
||||
atomic64_dec(v);
|
||||
}
|
||||
|
||||
static inline void atomic_long_add(long i, atomic_long_t *l)
|
||||
{
|
||||
atomic64_t *v = (atomic64_t *)l;
|
||||
|
||||
atomic64_add(i, v);
|
||||
}
|
||||
|
||||
static inline void atomic_long_sub(long i, atomic_long_t *l)
|
||||
{
|
||||
atomic64_t *v = (atomic64_t *)l;
|
||||
|
||||
atomic64_sub(i, v);
|
||||
}
|
||||
|
||||
static inline int atomic_long_sub_and_test(long i, atomic_long_t *l)
|
||||
{
|
||||
atomic64_t *v = (atomic64_t *)l;
|
||||
|
||||
return atomic64_sub_and_test(i, v);
|
||||
}
|
||||
|
||||
static inline int atomic_long_dec_and_test(atomic_long_t *l)
|
||||
{
|
||||
atomic64_t *v = (atomic64_t *)l;
|
||||
|
||||
return atomic64_dec_and_test(v);
|
||||
}
|
||||
|
||||
static inline int atomic_long_inc_and_test(atomic_long_t *l)
|
||||
{
|
||||
atomic64_t *v = (atomic64_t *)l;
|
||||
|
||||
return atomic64_inc_and_test(v);
|
||||
}
|
||||
|
||||
static inline int atomic_long_add_negative(long i, atomic_long_t *l)
|
||||
{
|
||||
atomic64_t *v = (atomic64_t *)l;
|
||||
|
||||
return atomic64_add_negative(i, v);
|
||||
}
|
||||
|
||||
static inline long atomic_long_add_return(long i, atomic_long_t *l)
|
||||
{
|
||||
atomic64_t *v = (atomic64_t *)l;
|
||||
|
||||
return (long)atomic64_add_return(i, v);
|
||||
}
|
||||
|
||||
static inline long atomic_long_sub_return(long i, atomic_long_t *l)
|
||||
{
|
||||
atomic64_t *v = (atomic64_t *)l;
|
||||
|
||||
return (long)atomic64_sub_return(i, v);
|
||||
}
|
||||
|
||||
static inline long atomic_long_inc_return(atomic_long_t *l)
|
||||
{
|
||||
atomic64_t *v = (atomic64_t *)l;
|
||||
|
||||
return (long)atomic64_inc_return(v);
|
||||
}
|
||||
|
||||
static inline long atomic_long_dec_return(atomic_long_t *l)
|
||||
{
|
||||
atomic64_t *v = (atomic64_t *)l;
|
||||
|
||||
return (long)atomic64_dec_return(v);
|
||||
}
|
||||
|
||||
static inline long atomic_long_add_unless(atomic_long_t *l, long a, long u)
|
||||
{
|
||||
atomic64_t *v = (atomic64_t *)l;
|
||||
|
||||
return (long)atomic64_add_unless(v, a, u);
|
||||
}
|
||||
|
||||
#define atomic_long_inc_not_zero(l) atomic64_inc_not_zero((atomic64_t *)(l))
|
||||
|
||||
#define atomic_long_cmpxchg(l, old, new) \
|
||||
(atomic64_cmpxchg((atomic64_t *)(l), (old), (new)))
|
||||
#define atomic_long_xchg(v, new) \
|
||||
(atomic64_xchg((atomic64_t *)(v), (new)))
|
||||
|
||||
#else /* BITS_PER_LONG == 64 */
|
||||
#else
|
||||
|
||||
typedef atomic_t atomic_long_t;
|
||||
|
||||
#define ATOMIC_LONG_INIT(i) ATOMIC_INIT(i)
|
||||
static inline long atomic_long_read(atomic_long_t *l)
|
||||
{
|
||||
atomic_t *v = (atomic_t *)l;
|
||||
#define ATOMIC_LONG_PFX(x) atomic ## x
|
||||
|
||||
return (long)atomic_read(v);
|
||||
#endif
|
||||
|
||||
#define ATOMIC_LONG_READ_OP(mo) \
|
||||
static inline long atomic_long_read##mo(atomic_long_t *l) \
|
||||
{ \
|
||||
ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l; \
|
||||
\
|
||||
return (long)ATOMIC_LONG_PFX(_read##mo)(v); \
|
||||
}
|
||||
ATOMIC_LONG_READ_OP()
|
||||
ATOMIC_LONG_READ_OP(_acquire)
|
||||
|
||||
static inline void atomic_long_set(atomic_long_t *l, long i)
|
||||
{
|
||||
atomic_t *v = (atomic_t *)l;
|
||||
#undef ATOMIC_LONG_READ_OP
|
||||
|
||||
atomic_set(v, i);
|
||||
#define ATOMIC_LONG_SET_OP(mo) \
|
||||
static inline void atomic_long_set##mo(atomic_long_t *l, long i) \
|
||||
{ \
|
||||
ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l; \
|
||||
\
|
||||
ATOMIC_LONG_PFX(_set##mo)(v, i); \
|
||||
}
|
||||
ATOMIC_LONG_SET_OP()
|
||||
ATOMIC_LONG_SET_OP(_release)
|
||||
|
||||
#undef ATOMIC_LONG_SET_OP
|
||||
|
||||
#define ATOMIC_LONG_ADD_SUB_OP(op, mo) \
|
||||
static inline long \
|
||||
atomic_long_##op##_return##mo(long i, atomic_long_t *l) \
|
||||
{ \
|
||||
ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l; \
|
||||
\
|
||||
return (long)ATOMIC_LONG_PFX(_##op##_return##mo)(i, v); \
|
||||
}
|
||||
ATOMIC_LONG_ADD_SUB_OP(add,)
|
||||
ATOMIC_LONG_ADD_SUB_OP(add, _relaxed)
|
||||
ATOMIC_LONG_ADD_SUB_OP(add, _acquire)
|
||||
ATOMIC_LONG_ADD_SUB_OP(add, _release)
|
||||
ATOMIC_LONG_ADD_SUB_OP(sub,)
|
||||
ATOMIC_LONG_ADD_SUB_OP(sub, _relaxed)
|
||||
ATOMIC_LONG_ADD_SUB_OP(sub, _acquire)
|
||||
ATOMIC_LONG_ADD_SUB_OP(sub, _release)
|
||||
|
||||
#undef ATOMIC_LONG_ADD_SUB_OP
|
||||
|
||||
#define atomic_long_cmpxchg_relaxed(l, old, new) \
|
||||
(ATOMIC_LONG_PFX(_cmpxchg_relaxed)((ATOMIC_LONG_PFX(_t) *)(l), \
|
||||
(old), (new)))
|
||||
#define atomic_long_cmpxchg_acquire(l, old, new) \
|
||||
(ATOMIC_LONG_PFX(_cmpxchg_acquire)((ATOMIC_LONG_PFX(_t) *)(l), \
|
||||
(old), (new)))
|
||||
#define atomic_long_cmpxchg_release(l, old, new) \
|
||||
(ATOMIC_LONG_PFX(_cmpxchg_release)((ATOMIC_LONG_PFX(_t) *)(l), \
|
||||
(old), (new)))
|
||||
#define atomic_long_cmpxchg(l, old, new) \
|
||||
(ATOMIC_LONG_PFX(_cmpxchg)((ATOMIC_LONG_PFX(_t) *)(l), (old), (new)))
|
||||
|
||||
#define atomic_long_xchg_relaxed(v, new) \
|
||||
(ATOMIC_LONG_PFX(_xchg_relaxed)((ATOMIC_LONG_PFX(_t) *)(v), (new)))
|
||||
#define atomic_long_xchg_acquire(v, new) \
|
||||
(ATOMIC_LONG_PFX(_xchg_acquire)((ATOMIC_LONG_PFX(_t) *)(v), (new)))
|
||||
#define atomic_long_xchg_release(v, new) \
|
||||
(ATOMIC_LONG_PFX(_xchg_release)((ATOMIC_LONG_PFX(_t) *)(v), (new)))
|
||||
#define atomic_long_xchg(v, new) \
|
||||
(ATOMIC_LONG_PFX(_xchg)((ATOMIC_LONG_PFX(_t) *)(v), (new)))
|
||||
|
||||
static inline void atomic_long_inc(atomic_long_t *l)
|
||||
{
|
||||
atomic_t *v = (atomic_t *)l;
|
||||
ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l;
|
||||
|
||||
atomic_inc(v);
|
||||
ATOMIC_LONG_PFX(_inc)(v);
|
||||
}
|
||||
|
||||
static inline void atomic_long_dec(atomic_long_t *l)
|
||||
{
|
||||
atomic_t *v = (atomic_t *)l;
|
||||
ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l;
|
||||
|
||||
atomic_dec(v);
|
||||
ATOMIC_LONG_PFX(_dec)(v);
|
||||
}
|
||||
|
||||
static inline void atomic_long_add(long i, atomic_long_t *l)
|
||||
{
|
||||
atomic_t *v = (atomic_t *)l;
|
||||
ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l;
|
||||
|
||||
atomic_add(i, v);
|
||||
ATOMIC_LONG_PFX(_add)(i, v);
|
||||
}
|
||||
|
||||
static inline void atomic_long_sub(long i, atomic_long_t *l)
|
||||
{
|
||||
atomic_t *v = (atomic_t *)l;
|
||||
ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l;
|
||||
|
||||
atomic_sub(i, v);
|
||||
ATOMIC_LONG_PFX(_sub)(i, v);
|
||||
}
|
||||
|
||||
static inline int atomic_long_sub_and_test(long i, atomic_long_t *l)
|
||||
{
|
||||
atomic_t *v = (atomic_t *)l;
|
||||
ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l;
|
||||
|
||||
return atomic_sub_and_test(i, v);
|
||||
return ATOMIC_LONG_PFX(_sub_and_test)(i, v);
|
||||
}
|
||||
|
||||
static inline int atomic_long_dec_and_test(atomic_long_t *l)
|
||||
{
|
||||
atomic_t *v = (atomic_t *)l;
|
||||
ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l;
|
||||
|
||||
return atomic_dec_and_test(v);
|
||||
return ATOMIC_LONG_PFX(_dec_and_test)(v);
|
||||
}
|
||||
|
||||
static inline int atomic_long_inc_and_test(atomic_long_t *l)
|
||||
{
|
||||
atomic_t *v = (atomic_t *)l;
|
||||
ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l;
|
||||
|
||||
return atomic_inc_and_test(v);
|
||||
return ATOMIC_LONG_PFX(_inc_and_test)(v);
|
||||
}
|
||||
|
||||
static inline int atomic_long_add_negative(long i, atomic_long_t *l)
|
||||
{
|
||||
atomic_t *v = (atomic_t *)l;
|
||||
ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l;
|
||||
|
||||
return atomic_add_negative(i, v);
|
||||
}
|
||||
|
||||
static inline long atomic_long_add_return(long i, atomic_long_t *l)
|
||||
{
|
||||
atomic_t *v = (atomic_t *)l;
|
||||
|
||||
return (long)atomic_add_return(i, v);
|
||||
}
|
||||
|
||||
static inline long atomic_long_sub_return(long i, atomic_long_t *l)
|
||||
{
|
||||
atomic_t *v = (atomic_t *)l;
|
||||
|
||||
return (long)atomic_sub_return(i, v);
|
||||
return ATOMIC_LONG_PFX(_add_negative)(i, v);
|
||||
}
|
||||
|
||||
static inline long atomic_long_inc_return(atomic_long_t *l)
|
||||
{
|
||||
atomic_t *v = (atomic_t *)l;
|
||||
ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l;
|
||||
|
||||
return (long)atomic_inc_return(v);
|
||||
return (long)ATOMIC_LONG_PFX(_inc_return)(v);
|
||||
}
|
||||
|
||||
static inline long atomic_long_dec_return(atomic_long_t *l)
|
||||
{
|
||||
atomic_t *v = (atomic_t *)l;
|
||||
ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l;
|
||||
|
||||
return (long)atomic_dec_return(v);
|
||||
return (long)ATOMIC_LONG_PFX(_dec_return)(v);
|
||||
}
|
||||
|
||||
static inline long atomic_long_add_unless(atomic_long_t *l, long a, long u)
|
||||
{
|
||||
atomic_t *v = (atomic_t *)l;
|
||||
ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l;
|
||||
|
||||
return (long)atomic_add_unless(v, a, u);
|
||||
return (long)ATOMIC_LONG_PFX(_add_unless)(v, a, u);
|
||||
}
|
||||
|
||||
#define atomic_long_inc_not_zero(l) atomic_inc_not_zero((atomic_t *)(l))
|
||||
|
||||
#define atomic_long_cmpxchg(l, old, new) \
|
||||
(atomic_cmpxchg((atomic_t *)(l), (old), (new)))
|
||||
#define atomic_long_xchg(v, new) \
|
||||
(atomic_xchg((atomic_t *)(v), (new)))
|
||||
|
||||
#endif /* BITS_PER_LONG == 64 */
|
||||
#define atomic_long_inc_not_zero(l) \
|
||||
ATOMIC_LONG_PFX(_inc_not_zero)((ATOMIC_LONG_PFX(_t) *)(l))
|
||||
|
||||
#endif /* _ASM_GENERIC_ATOMIC_LONG_H */
|
||||
|
|
|
|||
|
|
@ -98,15 +98,16 @@ ATOMIC_OP_RETURN(add, +)
|
|||
ATOMIC_OP_RETURN(sub, -)
|
||||
#endif
|
||||
|
||||
#ifndef atomic_clear_mask
|
||||
#ifndef atomic_and
|
||||
ATOMIC_OP(and, &)
|
||||
#define atomic_clear_mask(i, v) atomic_and(~(i), (v))
|
||||
#endif
|
||||
|
||||
#ifndef atomic_set_mask
|
||||
#define CONFIG_ARCH_HAS_ATOMIC_OR
|
||||
#ifndef atomic_or
|
||||
ATOMIC_OP(or, |)
|
||||
#define atomic_set_mask(i, v) atomic_or((i), (v))
|
||||
#endif
|
||||
|
||||
#ifndef atomic_xor
|
||||
ATOMIC_OP(xor, ^)
|
||||
#endif
|
||||
|
||||
#undef ATOMIC_OP_RETURN
|
||||
|
|
|
|||
|
|
@ -32,6 +32,10 @@ extern long long atomic64_##op##_return(long long a, atomic64_t *v);
|
|||
ATOMIC64_OPS(add)
|
||||
ATOMIC64_OPS(sub)
|
||||
|
||||
ATOMIC64_OP(and)
|
||||
ATOMIC64_OP(or)
|
||||
ATOMIC64_OP(xor)
|
||||
|
||||
#undef ATOMIC64_OPS
|
||||
#undef ATOMIC64_OP_RETURN
|
||||
#undef ATOMIC64_OP
|
||||
|
|
|
|||
|
|
@ -108,12 +108,12 @@
|
|||
do { \
|
||||
compiletime_assert_atomic_type(*p); \
|
||||
smp_mb(); \
|
||||
ACCESS_ONCE(*p) = (v); \
|
||||
WRITE_ONCE(*p, v); \
|
||||
} while (0)
|
||||
|
||||
#define smp_load_acquire(p) \
|
||||
({ \
|
||||
typeof(*p) ___p1 = ACCESS_ONCE(*p); \
|
||||
typeof(*p) ___p1 = READ_ONCE(*p); \
|
||||
compiletime_assert_atomic_type(*p); \
|
||||
smp_mb(); \
|
||||
___p1; \
|
||||
|
|
|
|||
|
|
@ -36,39 +36,39 @@
|
|||
/*
|
||||
* External function declarations
|
||||
*/
|
||||
extern void queue_read_lock_slowpath(struct qrwlock *lock);
|
||||
extern void queue_write_lock_slowpath(struct qrwlock *lock);
|
||||
extern void queued_read_lock_slowpath(struct qrwlock *lock, u32 cnts);
|
||||
extern void queued_write_lock_slowpath(struct qrwlock *lock);
|
||||
|
||||
/**
|
||||
* queue_read_can_lock- would read_trylock() succeed?
|
||||
* queued_read_can_lock- would read_trylock() succeed?
|
||||
* @lock: Pointer to queue rwlock structure
|
||||
*/
|
||||
static inline int queue_read_can_lock(struct qrwlock *lock)
|
||||
static inline int queued_read_can_lock(struct qrwlock *lock)
|
||||
{
|
||||
return !(atomic_read(&lock->cnts) & _QW_WMASK);
|
||||
}
|
||||
|
||||
/**
|
||||
* queue_write_can_lock- would write_trylock() succeed?
|
||||
* queued_write_can_lock- would write_trylock() succeed?
|
||||
* @lock: Pointer to queue rwlock structure
|
||||
*/
|
||||
static inline int queue_write_can_lock(struct qrwlock *lock)
|
||||
static inline int queued_write_can_lock(struct qrwlock *lock)
|
||||
{
|
||||
return !atomic_read(&lock->cnts);
|
||||
}
|
||||
|
||||
/**
|
||||
* queue_read_trylock - try to acquire read lock of a queue rwlock
|
||||
* queued_read_trylock - try to acquire read lock of a queue rwlock
|
||||
* @lock : Pointer to queue rwlock structure
|
||||
* Return: 1 if lock acquired, 0 if failed
|
||||
*/
|
||||
static inline int queue_read_trylock(struct qrwlock *lock)
|
||||
static inline int queued_read_trylock(struct qrwlock *lock)
|
||||
{
|
||||
u32 cnts;
|
||||
|
||||
cnts = atomic_read(&lock->cnts);
|
||||
if (likely(!(cnts & _QW_WMASK))) {
|
||||
cnts = (u32)atomic_add_return(_QR_BIAS, &lock->cnts);
|
||||
cnts = (u32)atomic_add_return_acquire(_QR_BIAS, &lock->cnts);
|
||||
if (likely(!(cnts & _QW_WMASK)))
|
||||
return 1;
|
||||
atomic_sub(_QR_BIAS, &lock->cnts);
|
||||
|
|
@ -77,11 +77,11 @@ static inline int queue_read_trylock(struct qrwlock *lock)
|
|||
}
|
||||
|
||||
/**
|
||||
* queue_write_trylock - try to acquire write lock of a queue rwlock
|
||||
* queued_write_trylock - try to acquire write lock of a queue rwlock
|
||||
* @lock : Pointer to queue rwlock structure
|
||||
* Return: 1 if lock acquired, 0 if failed
|
||||
*/
|
||||
static inline int queue_write_trylock(struct qrwlock *lock)
|
||||
static inline int queued_write_trylock(struct qrwlock *lock)
|
||||
{
|
||||
u32 cnts;
|
||||
|
||||
|
|
@ -89,78 +89,70 @@ static inline int queue_write_trylock(struct qrwlock *lock)
|
|||
if (unlikely(cnts))
|
||||
return 0;
|
||||
|
||||
return likely(atomic_cmpxchg(&lock->cnts,
|
||||
cnts, cnts | _QW_LOCKED) == cnts);
|
||||
return likely(atomic_cmpxchg_acquire(&lock->cnts,
|
||||
cnts, cnts | _QW_LOCKED) == cnts);
|
||||
}
|
||||
/**
|
||||
* queue_read_lock - acquire read lock of a queue rwlock
|
||||
* queued_read_lock - acquire read lock of a queue rwlock
|
||||
* @lock: Pointer to queue rwlock structure
|
||||
*/
|
||||
static inline void queue_read_lock(struct qrwlock *lock)
|
||||
static inline void queued_read_lock(struct qrwlock *lock)
|
||||
{
|
||||
u32 cnts;
|
||||
|
||||
cnts = atomic_add_return(_QR_BIAS, &lock->cnts);
|
||||
cnts = atomic_add_return_acquire(_QR_BIAS, &lock->cnts);
|
||||
if (likely(!(cnts & _QW_WMASK)))
|
||||
return;
|
||||
|
||||
/* The slowpath will decrement the reader count, if necessary. */
|
||||
queue_read_lock_slowpath(lock);
|
||||
queued_read_lock_slowpath(lock, cnts);
|
||||
}
|
||||
|
||||
/**
|
||||
* queue_write_lock - acquire write lock of a queue rwlock
|
||||
* queued_write_lock - acquire write lock of a queue rwlock
|
||||
* @lock : Pointer to queue rwlock structure
|
||||
*/
|
||||
static inline void queue_write_lock(struct qrwlock *lock)
|
||||
static inline void queued_write_lock(struct qrwlock *lock)
|
||||
{
|
||||
/* Optimize for the unfair lock case where the fair flag is 0. */
|
||||
if (atomic_cmpxchg(&lock->cnts, 0, _QW_LOCKED) == 0)
|
||||
if (atomic_cmpxchg_acquire(&lock->cnts, 0, _QW_LOCKED) == 0)
|
||||
return;
|
||||
|
||||
queue_write_lock_slowpath(lock);
|
||||
queued_write_lock_slowpath(lock);
|
||||
}
|
||||
|
||||
/**
|
||||
* queue_read_unlock - release read lock of a queue rwlock
|
||||
* queued_read_unlock - release read lock of a queue rwlock
|
||||
* @lock : Pointer to queue rwlock structure
|
||||
*/
|
||||
static inline void queue_read_unlock(struct qrwlock *lock)
|
||||
static inline void queued_read_unlock(struct qrwlock *lock)
|
||||
{
|
||||
/*
|
||||
* Atomically decrement the reader count
|
||||
*/
|
||||
smp_mb__before_atomic();
|
||||
atomic_sub(_QR_BIAS, &lock->cnts);
|
||||
(void)atomic_sub_return_release(_QR_BIAS, &lock->cnts);
|
||||
}
|
||||
|
||||
#ifndef queue_write_unlock
|
||||
/**
|
||||
* queue_write_unlock - release write lock of a queue rwlock
|
||||
* queued_write_unlock - release write lock of a queue rwlock
|
||||
* @lock : Pointer to queue rwlock structure
|
||||
*/
|
||||
static inline void queue_write_unlock(struct qrwlock *lock)
|
||||
static inline void queued_write_unlock(struct qrwlock *lock)
|
||||
{
|
||||
/*
|
||||
* If the writer field is atomic, it can be cleared directly.
|
||||
* Otherwise, an atomic subtraction will be used to clear it.
|
||||
*/
|
||||
smp_mb__before_atomic();
|
||||
atomic_sub(_QW_LOCKED, &lock->cnts);
|
||||
smp_store_release((u8 *)&lock->cnts, 0);
|
||||
}
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Remapping rwlock architecture specific functions to the corresponding
|
||||
* queue rwlock functions.
|
||||
*/
|
||||
#define arch_read_can_lock(l) queue_read_can_lock(l)
|
||||
#define arch_write_can_lock(l) queue_write_can_lock(l)
|
||||
#define arch_read_lock(l) queue_read_lock(l)
|
||||
#define arch_write_lock(l) queue_write_lock(l)
|
||||
#define arch_read_trylock(l) queue_read_trylock(l)
|
||||
#define arch_write_trylock(l) queue_write_trylock(l)
|
||||
#define arch_read_unlock(l) queue_read_unlock(l)
|
||||
#define arch_write_unlock(l) queue_write_unlock(l)
|
||||
#define arch_read_can_lock(l) queued_read_can_lock(l)
|
||||
#define arch_write_can_lock(l) queued_write_can_lock(l)
|
||||
#define arch_read_lock(l) queued_read_lock(l)
|
||||
#define arch_write_lock(l) queued_write_lock(l)
|
||||
#define arch_read_trylock(l) queued_read_trylock(l)
|
||||
#define arch_write_trylock(l) queued_write_trylock(l)
|
||||
#define arch_read_unlock(l) queued_read_unlock(l)
|
||||
#define arch_write_unlock(l) queued_write_unlock(l)
|
||||
|
||||
#endif /* __ASM_GENERIC_QRWLOCK_H */
|
||||
|
|
|
|||
Loading…
Reference in a new issue