malhuda/msm-4.14
1
0
Fork 0
mirror of https://github.com/rd-stuffs/msm-4.14.git synced 2025-02-20 11:45:48 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

sched/smp: Optimize send_call_function_single_ipi()

Browse Source 
Just like the ttwu_queue_remote() IPI, make use of _TIF_POLLING_NRFLAG
to avoid sending IPIs to idle CPUs.

[ mingo: Fix UP build bug. ]

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Link: https://lore.kernel.org/r/20200526161907.953304789@infradead.org
Signed-off-by: Vishalcj17 <vishalcj@aospa.co>
Change-Id: Ife9ead8594aec91756d40381cc0fff1d187ebded
This commit is contained in:
Peter Zijlstra 2020-05-26 18:11:01 +02:00 committed by Adithya R
parent 128ae49439
commit af19a6a3bb
4 changed files with 34 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
kernel/sched/core.c
View File

@ -1890,6 +1890,16 @@ void scheduler_ipi(void)
irq_exit();
}
void send_call_function_single_ipi(int cpu)
{
struct rq *rq = cpu_rq(cpu);
if (!set_nr_if_polling(rq->idle))
arch_send_call_function_single_ipi(cpu);
else
trace_sched_wake_idle_without_ipi(cpu);
}
static void ttwu_queue_remote(struct task_struct *p, int cpu, int wake_flags)
{
struct rq *rq = cpu_rq(cpu);

5
kernel/sched/idle.c
View File

@ -291,6 +291,11 @@ static void do_idle(void)
*/
smp_mb__after_atomic();
/*
* RCU relies on this call to be done outside of an RCU read-side
* critical section.
*/
flush_smp_call_function_from_idle();
sched_ttwu_pending();
schedule_idle();

7
kernel/sched/sched.h
View File

@ -1385,11 +1385,12 @@ static inline void unregister_sched_domain_sysctl(void)
}
#endif
#else
extern void flush_smp_call_function_from_idle(void);
#else /* !CONFIG_SMP: */
static inline void flush_smp_call_function_from_idle(void) { }
static inline void sched_ttwu_pending(void) { }
#endif /* CONFIG_SMP */
#endif
#include "stats.h"
#include "autogroup.h"

16
kernel/smp.c
View File

@ -137,6 +137,8 @@ static __always_inline void csd_unlock(call_single_data_t *csd)
static DEFINE_PER_CPU_SHARED_ALIGNED(call_single_data_t, csd_data);
extern void send_call_function_single_ipi(int cpu);
/*
* Insert a previously allocated call_single_data_t element
* for execution on the given CPU. data must already have
@ -180,7 +182,7 @@ static int generic_exec_single(int cpu, call_single_data_t *csd,
* equipped to do the right thing...
*/
if (llist_add(&csd->llist, &per_cpu(call_single_queue, cpu)))
arch_send_call_function_single_ipi(cpu);
send_call_function_single_ipi(cpu);
return 0;
}
@ -261,6 +263,18 @@ static void flush_smp_call_function_queue(bool warn_cpu_offline)
irq_work_run();
}
void flush_smp_call_function_from_idle(void)
{
unsigned long flags;
if (llist_empty(this_cpu_ptr(&call_single_queue)))
return;
local_irq_save(flags);
flush_smp_call_function_queue(true);
local_irq_restore(flags);
}
/*
* smp_call_function_single - Run a function on a specific CPU
* @func: The function to run. This must be fast and non-blocking.