mirror of
https://github.com/AuxXxilium/linux_dsm_epyc7002.git
synced 2024-11-24 21:30:53 +07:00
b519b56e37
The qrwlock slowpaths involve spinning when either a prospective reader is waiting for a concurrent writer to drain, or a prospective writer is waiting for concurrent readers to drain. In both of these situations, atomic_cond_read_acquire() can be used to avoid busy-waiting and make use of any backoff functionality provided by the architecture. This patch replaces the open-code loops and rspin_until_writer_unlock() implementation with atomic_cond_read_acquire(). The write mode transition zero to _QW_WAITING is left alone, since (a) this doesn't need acquire semantics and (b) should be fast. Tested-by: Waiman Long <longman@redhat.com> Tested-by: Jeremy Linton <jeremy.linton@arm.com> Tested-by: Adam Wallis <awallis@codeaurora.org> Tested-by: Jan Glauber <jglauber@cavium.com> Signed-off-by: Will Deacon <will.deacon@arm.com> Acked-by: Peter Zijlstra <peterz@infradead.org> Cc: Boqun Feng <boqun.feng@gmail.com> Cc: Jeremy.Linton@arm.com Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com> Cc: Thomas Gleixner <tglx@linutronix.de> Cc: linux-arm-kernel@lists.infradead.org Link: http://lkml.kernel.org/r/1507810851-306-4-git-send-email-will.deacon@arm.com Signed-off-by: Ingo Molnar <mingo@kernel.org>
103 lines
3.0 KiB
C
103 lines
3.0 KiB
C
/*
|
|
* Queued read/write locks
|
|
*
|
|
* This program is free software; you can redistribute it and/or modify
|
|
* it under the terms of the GNU General Public License as published by
|
|
* the Free Software Foundation; either version 2 of the License, or
|
|
* (at your option) any later version.
|
|
*
|
|
* This program is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU General Public License for more details.
|
|
*
|
|
* (C) Copyright 2013-2014 Hewlett-Packard Development Company, L.P.
|
|
*
|
|
* Authors: Waiman Long <waiman.long@hp.com>
|
|
*/
|
|
#include <linux/smp.h>
|
|
#include <linux/bug.h>
|
|
#include <linux/cpumask.h>
|
|
#include <linux/percpu.h>
|
|
#include <linux/hardirq.h>
|
|
#include <linux/spinlock.h>
|
|
#include <asm/qrwlock.h>
|
|
|
|
/**
|
|
* queued_read_lock_slowpath - acquire read lock of a queue rwlock
|
|
* @lock: Pointer to queue rwlock structure
|
|
*/
|
|
void queued_read_lock_slowpath(struct qrwlock *lock)
|
|
{
|
|
/*
|
|
* Readers come here when they cannot get the lock without waiting
|
|
*/
|
|
if (unlikely(in_interrupt())) {
|
|
/*
|
|
* Readers in interrupt context will get the lock immediately
|
|
* if the writer is just waiting (not holding the lock yet),
|
|
* so spin with ACQUIRE semantics until the lock is available
|
|
* without waiting in the queue.
|
|
*/
|
|
atomic_cond_read_acquire(&lock->cnts, (VAL & _QW_WMASK)
|
|
!= _QW_LOCKED);
|
|
return;
|
|
}
|
|
atomic_sub(_QR_BIAS, &lock->cnts);
|
|
|
|
/*
|
|
* Put the reader into the wait queue
|
|
*/
|
|
arch_spin_lock(&lock->wait_lock);
|
|
atomic_add(_QR_BIAS, &lock->cnts);
|
|
|
|
/*
|
|
* The ACQUIRE semantics of the following spinning code ensure
|
|
* that accesses can't leak upwards out of our subsequent critical
|
|
* section in the case that the lock is currently held for write.
|
|
*/
|
|
atomic_cond_read_acquire(&lock->cnts, (VAL & _QW_WMASK) != _QW_LOCKED);
|
|
|
|
/*
|
|
* Signal the next one in queue to become queue head
|
|
*/
|
|
arch_spin_unlock(&lock->wait_lock);
|
|
}
|
|
EXPORT_SYMBOL(queued_read_lock_slowpath);
|
|
|
|
/**
|
|
* queued_write_lock_slowpath - acquire write lock of a queue rwlock
|
|
* @lock : Pointer to queue rwlock structure
|
|
*/
|
|
void queued_write_lock_slowpath(struct qrwlock *lock)
|
|
{
|
|
/* Put the writer into the wait queue */
|
|
arch_spin_lock(&lock->wait_lock);
|
|
|
|
/* Try to acquire the lock directly if no reader is present */
|
|
if (!atomic_read(&lock->cnts) &&
|
|
(atomic_cmpxchg_acquire(&lock->cnts, 0, _QW_LOCKED) == 0))
|
|
goto unlock;
|
|
|
|
/*
|
|
* Set the waiting flag to notify readers that a writer is pending,
|
|
* or wait for a previous writer to go away.
|
|
*/
|
|
for (;;) {
|
|
if (!READ_ONCE(lock->wmode) &&
|
|
(cmpxchg_relaxed(&lock->wmode, 0, _QW_WAITING) == 0))
|
|
break;
|
|
|
|
cpu_relax();
|
|
}
|
|
|
|
/* When no more readers, set the locked flag */
|
|
do {
|
|
atomic_cond_read_acquire(&lock->cnts, VAL == _QW_WAITING);
|
|
} while (atomic_cmpxchg_relaxed(&lock->cnts, _QW_WAITING,
|
|
_QW_LOCKED) != _QW_WAITING);
|
|
unlock:
|
|
arch_spin_unlock(&lock->wait_lock);
|
|
}
|
|
EXPORT_SYMBOL(queued_write_lock_slowpath);
|