diff options
author | Paul E. McKenney <paulmck@linux.vnet.ibm.com> | 2017-04-25 14:03:11 -0700 |
---|---|---|
committer | Paul E. McKenney <paulmck@linux.vnet.ibm.com> | 2017-04-26 16:32:17 -0700 |
commit | 22607d66bbc3e81140d3bcf08894f4378eb36428 (patch) | |
tree | 88b0caa1cd8bfa66e6a4b00ad21108f03369eb03 /kernel | |
parent | 2da4b2a7fd8de52c44fa83c59f00d38a0d90caae (diff) | |
download | op-kernel-dev-22607d66bbc3e81140d3bcf08894f4378eb36428.zip op-kernel-dev-22607d66bbc3e81140d3bcf08894f4378eb36428.tar.gz |
srcu: Specify auto-expedite holdoff time
On small systems, in the absence of readers, expedited SRCU grace
periods can complete in less than a microsecond. This means that an
eight-CPU system can have all CPUs doing synchronize_srcu() in a tight
loop and almost always expedite. This might actually be desirable in
some situations, but in general it is a good way to needlessly burn
CPU cycles. And in those situations where it is desirable, your friend
is the function synchronize_srcu_expedited().
For other situations, this commit adds a kernel parameter that specifies
a holdoff between completing the last SRCU grace period and auto-expediting
the next. If the next grace period starts before the holdoff expires,
auto-expediting is disabled. The holdoff is 50 microseconds by default,
and can be tuned to the desired number of nanoseconds. A value of zero
disables auto-expediting.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Tested-by: Mike Galbraith <efault@gmx.de>
Diffstat (limited to 'kernel')
-rw-r--r-- | kernel/rcu/srcutree.c | 18 |
1 files changed, 17 insertions, 1 deletions
diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c index 2286e06..74c283f 100644 --- a/kernel/rcu/srcutree.c +++ b/kernel/rcu/srcutree.c @@ -34,10 +34,14 @@ #include <linux/sched.h> #include <linux/smp.h> #include <linux/delay.h> +#include <linux/module.h> #include <linux/srcu.h> #include "rcu.h" +ulong exp_holdoff = 50 * 1000; /* Holdoff (ns) for auto-expediting. */ +module_param(exp_holdoff, ulong, 0444); + static void srcu_invoke_callbacks(struct work_struct *work); static void srcu_reschedule(struct srcu_struct *sp, unsigned long delay); @@ -145,6 +149,7 @@ static int init_srcu_struct_fields(struct srcu_struct *sp, bool is_static) sp->sda = alloc_percpu(struct srcu_data); init_srcu_struct_nodes(sp, is_static); sp->srcu_gp_seq_needed_exp = 0; + sp->srcu_last_gp_end = ktime_get_mono_fast_ns(); smp_store_release(&sp->srcu_gp_seq_needed, 0); /* Init done. */ return sp->sda ? 0 : -ENOMEM; } @@ -498,6 +503,7 @@ static void srcu_gp_end(struct srcu_struct *sp) idx = rcu_seq_state(sp->srcu_gp_seq); WARN_ON_ONCE(idx != SRCU_STATE_SCAN2); cbdelay = srcu_get_delay(sp); + sp->srcu_last_gp_end = ktime_get_mono_fast_ns(); rcu_seq_end(&sp->srcu_gp_seq); gpseq = rcu_seq_current(&sp->srcu_gp_seq); if (ULONG_CMP_LT(sp->srcu_gp_seq_needed_exp, gpseq)) @@ -700,9 +706,10 @@ static void srcu_flip(struct srcu_struct *sp) */ static bool srcu_might_be_idle(struct srcu_struct *sp) { + unsigned long curseq; unsigned long flags; struct srcu_data *sdp; - unsigned long curseq; + unsigned long t; /* If the local srcu_data structure has callbacks, not idle. */ local_irq_save(flags); @@ -718,6 +725,15 @@ static bool srcu_might_be_idle(struct srcu_struct *sp) * Exact information would require acquiring locks, which would * kill scalability, hence the probabalistic nature of the probe. */ + + /* First, see if enough time has passed since the last GP. */ + t = ktime_get_mono_fast_ns(); + if (exp_holdoff == 0 || + time_in_range_open(t, sp->srcu_last_gp_end, + sp->srcu_last_gp_end + exp_holdoff)) + return false; /* Too soon after last GP. */ + + /* Next, check for probable idleness. */ curseq = rcu_seq_current(&sp->srcu_gp_seq); smp_mb(); /* Order ->srcu_gp_seq with ->srcu_gp_seq_needed. */ if (ULONG_CMP_LT(curseq, READ_ONCE(sp->srcu_gp_seq_needed))) |