diff options
| author | Frederic Weisbecker <frederic@kernel.org> | 2022-10-16 16:22:53 +0000 |
|---|---|---|
| committer | Treehugger Robot <android-test-infra-autosubmit@system.gserviceaccount.com> | 2024-04-29 22:43:39 +0000 |
| commit | 24e6758060b8a5bf7366892080d968962a5420e2 (patch) | |
| tree | aa3bdf2a90cd9044a012eb61e0c5cf0bceb4e8cb | |
| parent | fb310d468a41c61f9dc9c0be165b7e021a5d2ca9 (diff) | |
| download | common-24e6758060b8a5bf7366892080d968962a5420e2.tar.gz | |
BACKPORT: rcu: Fix missing nocb gp wake on rcu_barrier()
In preparation for RCU lazy changes, wake up the RCU nocb gp thread if
needed after an entrain. This change prevents the RCU barrier callback
from waiting in the queue for several seconds before the lazy callbacks
in front of it are serviced.
Reported-by: Joel Fernandes (Google) <joel@joelfernandes.org>
Signed-off-by: Frederic Weisbecker <frederic@kernel.org>
Signed-off-by: Joel Fernandes (Google) <joel@joelfernandes.org>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
(cherry picked from commit b8f7aca3f0e0e6223094ba2662bac90353674b04
https://git.kernel.org/pub/scm/linux/kernel/git/paulmck/linux-rcu.git rcu/next)
(Backport:
Conflicts:
kernel/rcu/tree.c
Due to missing 'rcu: Rework rcu_barrier() and callback-migration logic'
Chose not to backport that.)
Bug: 258241771
Bug: 222463781
Test: CQ
Change-Id: Ib55c5886764b74df22531eca35f076ef7acc08dd
Signed-off-by: Joel Fernandes <joelaf@google.com>
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/kernel/+/4062165
Reviewed-by: Vineeth Pillai <vineethrp@google.com>
(cherry picked from commit fc6e55ea65dca9cc52bda6081341f3fcc87f6ee7)
[Cherry picked from chromeos-5.15 tree. Minor tweaks to commit message
to match Android style]
Signed-off-by: Qais Yousef <qyousef@google.com>
| -rw-r--r-- | kernel/rcu/tree.c | 11 | ||||
| -rw-r--r-- | kernel/rcu/tree.h | 1 | ||||
| -rw-r--r-- | kernel/rcu/tree_nocb.h | 5 |
3 files changed, 17 insertions, 0 deletions
diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index be627fb32a91..a0989afc9980 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -4003,12 +4003,21 @@ static void rcu_barrier_func(void *cpu_in) { uintptr_t cpu = (uintptr_t)cpu_in; struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu); + bool wake_nocb = false; + bool was_alldone = false; rcu_barrier_trace(TPS("IRQ"), -1, rcu_state.barrier_sequence); rdp->barrier_head.func = rcu_barrier_callback; debug_rcu_head_queue(&rdp->barrier_head); rcu_nocb_lock(rdp); + /* + * Flush bypass and wakeup rcuog if we add callbacks to an empty regular + * queue. This way we don't wait for bypass timer that can reach seconds + * if it's fully lazy. + */ + was_alldone = rcu_rdp_is_offloaded(rdp) && !rcu_segcblist_pend_cbs(&rdp->cblist); WARN_ON_ONCE(!rcu_nocb_flush_bypass(rdp, NULL, jiffies)); + wake_nocb = was_alldone && rcu_segcblist_pend_cbs(&rdp->cblist); if (rcu_segcblist_entrain(&rdp->cblist, &rdp->barrier_head)) { atomic_inc(&rcu_state.barrier_cpu_count); } else { @@ -4017,6 +4026,8 @@ static void rcu_barrier_func(void *cpu_in) rcu_state.barrier_sequence); } rcu_nocb_unlock(rdp); + if (wake_nocb) + wake_nocb_gp(rdp, false); } /** diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h index 222a5a59f535..168b1b84b138 100644 --- a/kernel/rcu/tree.h +++ b/kernel/rcu/tree.h @@ -437,6 +437,7 @@ static void zero_cpu_stall_ticks(struct rcu_data *rdp); static struct swait_queue_head *rcu_nocb_gp_get(struct rcu_node *rnp); static void rcu_nocb_gp_cleanup(struct swait_queue_head *sq); static void rcu_init_one_nocb(struct rcu_node *rnp); +static bool wake_nocb_gp(struct rcu_data *rdp, bool force); static bool rcu_nocb_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, unsigned long j); static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp, diff --git a/kernel/rcu/tree_nocb.h b/kernel/rcu/tree_nocb.h index 8fdf44f8523f..1b74e65399e0 100644 --- a/kernel/rcu/tree_nocb.h +++ b/kernel/rcu/tree_nocb.h @@ -1449,6 +1449,11 @@ static void rcu_init_one_nocb(struct rcu_node *rnp) { } +static bool wake_nocb_gp(struct rcu_data *rdp, bool force) +{ + return false; +} + static bool rcu_nocb_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp, unsigned long j) { |