AuxXxilium/linux_dsm_epyc7002
1
0
Fork 0
mirror of https://github.com/AuxXxilium/linux_dsm_epyc7002.git synced 2024-11-29 20:56:41 +07:00
Code Issues Actions Packages Projects Releases Wiki Activity

rcu: Make synchronize_rcu_expedited() use sequence-counter scheme

Browse Source 
Although synchronize_rcu_expedited() uses a sequence-counter scheme, it
is based on a single increment per grace period, which means that tasks
piggybacking off of concurrent grace periods may be forced to wait longer
than necessary.  This commit therefore applies the new sequence-count
functions developed for synchronize_sched_expedited() to speed things
up a bit and to consolidate the sequence-counter implementation.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
This commit is contained in:
Paul E. McKenney 2015-06-25 15:52:50 -07:00
parent 28f00767e3
commit 543c6158f6
1 changed files with 6 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
kernel/rcu/tree_plugin.h
View File

@ -536,7 +536,6 @@ void synchronize_rcu(void)
EXPORT_SYMBOL_GPL(synchronize_rcu);
static DECLARE_WAIT_QUEUE_HEAD(sync_rcu_preempt_exp_wq);
static unsigned long sync_rcu_preempt_exp_count;
static DEFINE_MUTEX(sync_rcu_preempt_exp_mutex);
/*
@ -704,12 +703,10 @@ void synchronize_rcu_expedited(void)
{
struct rcu_node *rnp;
struct rcu_state *rsp = rcu_state_p;
unsigned long snap;
unsigned long s;
int trycount = 0;
smp_mb(); /* Caller's modifications seen first by other CPUs. */
snap = READ_ONCE(sync_rcu_preempt_exp_count) + 1;
smp_mb(); /* Above access cannot bleed into critical section. */
s = rcu_exp_gp_seq_snap(rsp);
/*
* Acquire lock, falling back to synchronize_rcu() if too many
@ -717,8 +714,7 @@ void synchronize_rcu_expedited(void)
* expedited grace period for us, just leave.
*/
while (!mutex_trylock(&sync_rcu_preempt_exp_mutex)) {
if (ULONG_CMP_LT(snap,
READ_ONCE(sync_rcu_preempt_exp_count)))
if (rcu_exp_gp_seq_done(rsp, s))
goto mb_ret; /* Others did our work for us. */
if (trycount++ < 10) {
udelay(trycount * num_online_cpus());
@ -727,8 +723,9 @@ void synchronize_rcu_expedited(void)
return;
}
}
if (ULONG_CMP_LT(snap, READ_ONCE(sync_rcu_preempt_exp_count)))
if (rcu_exp_gp_seq_done(rsp, s))
goto unlock_mb_ret; /* Others did our work for us. */
rcu_exp_gp_seq_start(rsp);
/* force all RCU readers onto ->blkd_tasks lists. */
synchronize_sched_expedited();
@ -750,8 +747,7 @@ void synchronize_rcu_expedited(void)
sync_rcu_preempt_exp_done(rnp));
/* Clean up and exit. */
smp_mb(); /* ensure expedited GP seen before counter increment. */
WRITE_ONCE(sync_rcu_preempt_exp_count, sync_rcu_preempt_exp_count + 1);
rcu_exp_gp_seq_end(rsp);
unlock_mb_ret:
mutex_unlock(&sync_rcu_preempt_exp_mutex);
mb_ret: