cpufreq: Introduce driver for event-based CPU boosting

This is a simple CPU input boost driver that boosts all online CPUs for
a fixed amount of time. Additionally, there is an API for other drivers
to request a boost kick (or a max-boost kick), so boosting can be done
on any custom event. This API is mainly intended for the framebuffer
driver to send a boost kick whenever there is a new frame ready to be
rendered to the display.

This driver also boosts all online CPUs to their maximum frequencies
when the display is powered on (this is the wake boost).

Since this driver requires careful tuning for optimal performance, there
are no user-exposed knobs to configure it. All necessary configuration
is done via the supplied Kconfig options.

This driver is designed for heterogeneous multi-processor systems with
two CPU clusters.

Change-Id: I4ca8e6a9233c875d07d6471af68bad64a3addbba
Signed-off-by: Sultan Alsawaf <sultan@kerneltoast.com>
Signed-off-by: Richard Raya <rdxzv.dev@gmail.com>

drivers/cpufreq/Kconfig

@@ -250,6 +250,67 @@ config CPU_FREQ_GOV_INTERACTIVE
 
 	  If in doubt, say N.
 
+config CPU_INPUT_BOOST
+	bool "CPU Input Boost"
+	help
+	  Boosts the CPU on touchscreen and touchpad input, and allows for
+	  boosting on other custom events, mainly which is intended to be for
+	  boosting when there is a new frame ready to be rendered to the
+	  display. The boost frequencies for this driver should be set so that
+	  frame drops are near-zero at the boosted frequencies and power
+	  consumption is minimized at said frequency combination.
+
+if CPU_INPUT_BOOST
+
+config INPUT_BOOST_DURATION_MS
+	int "Input boost duration"
+	default "100"
+	help
+	  Input boost duration in milliseconds.
+
+config WAKE_BOOST_DURATION_MS
+	int "Wake boost duration"
+	default "1000"
+	help
+	  Wake boost duration in milliseconds.
+
+config INPUT_BOOST_FREQ_LP
+	int "Low-power cluster boost freq"
+	default "0"
+	help
+	  Input boost frequency for the low-power CPU cluster.
+
+config INPUT_BOOST_FREQ_PERF
+	int "Performance cluster boost freq"
+	default "0"
+	help
+	  Input boost frequency for the performance CPU cluster.
+
+config MAX_BOOST_FREQ_LP
+	int "Low-power cluster max-boost freq"
+	default "0"
+	help
+	  Max-boost frequency for the low-power CPU cluster.
+
+config MAX_BOOST_FREQ_PERF
+	int "Performance cluster max-boost freq"
+	default "0"
+	help
+	  Max-boost frequency for the performance CPU cluster.
+
+config MIN_FREQ_LP
+	int "LP cluster min boost to set after boost"
+	default "0"
+	help
+	  Min freq to set for lp cluster when boost is over.
+
+config MIN_FREQ_PERF
+	int "HP cluster min boost to set after boost"
+	default "0"
+	help
+	  Min freq to set for hp cluster when boost is over.
+endif
+
 comment "CPU frequency scaling drivers"
 
 config CPUFREQ_DT

drivers/cpufreq/Makefile

@@ -24,6 +24,9 @@ obj-$(CONFIG_CPU_FREQ_GOV_COMMON)		+= cpufreq_governor.o
 obj-$(CONFIG_CPU_FREQ_GOV_ATTR_SET)	+= cpufreq_governor_attr_set.o
 obj-$(CONFIG_CPU_BOOST)			+= cpu-boost.o
 
+# CPU Input Boost
+obj-$(CONFIG_CPU_INPUT_BOOST)		+= cpu_input_boost.o
+
 obj-$(CONFIG_CPUFREQ_DT)		+= cpufreq-dt.o
 obj-$(CONFIG_CPUFREQ_DT_PLATDEV)	+= cpufreq-dt-platdev.o
 

drivers/cpufreq/cpu_input_boost.c

@@ -0,0 +1,366 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2018-2019 Sultan Alsawaf <sultan@kerneltoast.com>.
+ */
+
+#define pr_fmt(fmt) "cpu_input_boost: " fmt
+
+#include <linux/cpu.h>
+#include <linux/cpufreq.h>
+#include <linux/msm_drm_notify.h>
+#include <linux/input.h>
+#include <linux/kthread.h>
+#include <linux/version.h>
+#include <linux/slab.h>
+
+/* The sched_param struct is located elsewhere in newer kernels */
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0)
+#include <uapi/linux/sched/types.h>
+#endif
+
+enum {
+	SCREEN_OFF,
+	INPUT_BOOST,
+	MAX_BOOST
+};
+
+struct boost_drv {
+	struct delayed_work input_unboost;
+	struct delayed_work max_unboost;
+	struct notifier_block cpu_notif;
+	struct notifier_block msm_drm_notif;
+	wait_queue_head_t boost_waitq;
+	atomic_long_t max_boost_expires;
+	unsigned long state;
+};
+
+static void input_unboost_worker(struct work_struct *work);
+static void max_unboost_worker(struct work_struct *work);
+
+static struct boost_drv boost_drv_g __read_mostly = {
+	.input_unboost = __DELAYED_WORK_INITIALIZER(boost_drv_g.input_unboost,
+						    input_unboost_worker, 0),
+	.max_unboost = __DELAYED_WORK_INITIALIZER(boost_drv_g.max_unboost,
+						  max_unboost_worker, 0),
+	.boost_waitq = __WAIT_QUEUE_HEAD_INITIALIZER(boost_drv_g.boost_waitq)
+};
+
+static unsigned int get_input_boost_freq(struct cpufreq_policy *policy)
+{
+	unsigned int freq;
+
+	if (cpumask_test_cpu(policy->cpu, cpu_lp_mask))
+		freq = CONFIG_INPUT_BOOST_FREQ_LP;
+	else
+		freq = CONFIG_INPUT_BOOST_FREQ_PERF;
+
+	return min(freq, policy->max);
+}
+
+static unsigned int get_max_boost_freq(struct cpufreq_policy *policy)
+{
+	unsigned int freq;
+
+	if (cpumask_test_cpu(policy->cpu, cpu_lp_mask))
+		freq = CONFIG_MAX_BOOST_FREQ_LP;
+	else
+		freq = CONFIG_MAX_BOOST_FREQ_PERF;
+
+	return min(freq, policy->max);
+}
+
+static void update_online_cpu_policy(void)
+{
+	unsigned int cpu;
+
+	/* Only one CPU from each cluster needs to be updated */
+	get_online_cpus();
+	cpu = cpumask_first_and(cpu_lp_mask, cpu_online_mask);
+	cpufreq_update_policy(cpu);
+	cpu = cpumask_first_and(cpu_perf_mask, cpu_online_mask);
+	cpufreq_update_policy(cpu);
+	put_online_cpus();
+}
+
+static void __cpu_input_boost_kick(struct boost_drv *b)
+{
+	if (test_bit(SCREEN_OFF, &b->state))
+		return;
+
+	set_bit(INPUT_BOOST, &b->state);
+	if (!mod_delayed_work(system_unbound_wq, &b->input_unboost,
+			      msecs_to_jiffies(CONFIG_INPUT_BOOST_DURATION_MS)))
+		wake_up(&b->boost_waitq);
+}
+
+void cpu_input_boost_kick(void)
+{
+	struct boost_drv *b = &boost_drv_g;
+
+	__cpu_input_boost_kick(b);
+}
+
+static void __cpu_input_boost_kick_max(struct boost_drv *b,
+				       unsigned int duration_ms)
+{
+	unsigned long boost_jiffies = msecs_to_jiffies(duration_ms);
+	unsigned long curr_expires, new_expires;
+
+	if (test_bit(SCREEN_OFF, &b->state))
+		return;
+
+	do {
+		curr_expires = atomic_long_read(&b->max_boost_expires);
+		new_expires = jiffies + boost_jiffies;
+
+		/* Skip this boost if there's a longer boost in effect */
+		if (time_after(curr_expires, new_expires))
+			return;
+	} while (atomic_long_cmpxchg(&b->max_boost_expires, curr_expires,
+				     new_expires) != curr_expires);
+
+	set_bit(MAX_BOOST, &b->state);
+	if (!mod_delayed_work(system_unbound_wq, &b->max_unboost,
+			      boost_jiffies))
+		wake_up(&b->boost_waitq);
+}
+
+void cpu_input_boost_kick_max(unsigned int duration_ms)
+{
+	struct boost_drv *b = &boost_drv_g;
+
+	__cpu_input_boost_kick_max(b, duration_ms);
+}
+
+static void input_unboost_worker(struct work_struct *work)
+{
+	struct boost_drv *b = container_of(to_delayed_work(work),
+					   typeof(*b), input_unboost);
+
+	clear_bit(INPUT_BOOST, &b->state);
+	wake_up(&b->boost_waitq);
+}
+
+static void max_unboost_worker(struct work_struct *work)
+{
+	struct boost_drv *b = container_of(to_delayed_work(work),
+					   typeof(*b), max_unboost);
+
+	clear_bit(MAX_BOOST, &b->state);
+	wake_up(&b->boost_waitq);
+}
+
+static int cpu_boost_thread(void *data)
+{
+	static const struct sched_param sched_max_rt_prio = {
+		.sched_priority = MAX_RT_PRIO - 1
+	};
+	struct boost_drv *b = data;
+	unsigned long old_state = 0;
+
+	sched_setscheduler_nocheck(current, SCHED_FIFO, &sched_max_rt_prio);
+
+	while (1) {
+		bool should_stop = false;
+		unsigned long curr_state;
+
+		wait_event(b->boost_waitq,
+			(curr_state = READ_ONCE(b->state)) != old_state ||
+			(should_stop = kthread_should_stop()));
+
+		if (should_stop)
+			break;
+
+		old_state = curr_state;
+		update_online_cpu_policy();
+	}
+
+	return 0;
+}
+
+static int cpu_notifier_cb(struct notifier_block *nb, unsigned long action,
+			   void *data)
+{
+	struct boost_drv *b = container_of(nb, typeof(*b), cpu_notif);
+	struct cpufreq_policy *policy = data;
+
+	if (action != CPUFREQ_ADJUST)
+		return NOTIFY_OK;
+
+	/* Unboost when the screen is off */
+	if (test_bit(SCREEN_OFF, &b->state)) {
+		policy->min = policy->cpuinfo.min_freq;
+		return NOTIFY_OK;
+	}
+
+	/* Boost CPU to max frequency for max boost */
+	if (test_bit(MAX_BOOST, &b->state)) {
+		policy->min = get_max_boost_freq(policy);
+		return NOTIFY_OK;
+	}
+
+	/*
+	 * Boost to policy->max if the boost frequency is higher. When
+	 * unboosting, set policy->min to the absolute min freq for the CPU.
+	 */
+	if (test_bit(INPUT_BOOST, &b->state))
+		policy->min = get_input_boost_freq(policy);
+	else if (cpumask_test_cpu(policy->cpu, cpu_lp_mask))
+		policy->min = CONFIG_MIN_FREQ_LP;
+	else
+		policy->min = CONFIG_MIN_FREQ_PERF;
+
+	return NOTIFY_OK;
+}
+
+static int msm_drm_notifier_cb(struct notifier_block *nb, unsigned long action,
+			  void *data)
+{
+	struct boost_drv *b = container_of(nb, typeof(*b), msm_drm_notif);
+	struct msm_drm_notifier *evdata = data;
+	int *blank = evdata->data;
+
+	/* Parse framebuffer blank events as soon as they occur */
+	if (action != MSM_DRM_EARLY_EVENT_BLANK)
+		return NOTIFY_OK;
+
+	/* Boost when the screen turns on and unboost when it turns off */
+	if (*blank == MSM_DRM_BLANK_UNBLANK) {
+		clear_bit(SCREEN_OFF, &b->state);
+		__cpu_input_boost_kick_max(b, CONFIG_WAKE_BOOST_DURATION_MS);
+	} else {
+		set_bit(SCREEN_OFF, &b->state);
+		wake_up(&b->boost_waitq);
+	}
+
+	return NOTIFY_OK;
+}
+
+static void cpu_input_boost_input_event(struct input_handle *handle,
+					unsigned int type, unsigned int code,
+					int value)
+{
+	struct boost_drv *b = handle->handler->private;
+
+	__cpu_input_boost_kick(b);
+}
+
+static int cpu_input_boost_input_connect(struct input_handler *handler,
+					 struct input_dev *dev,
+					 const struct input_device_id *id)
+{
+	struct input_handle *handle;
+	int ret;
+
+	handle = kzalloc(sizeof(*handle), GFP_KERNEL);
+	if (!handle)
+		return -ENOMEM;
+
+	handle->dev = dev;
+	handle->handler = handler;
+	handle->name = "cpu_input_boost_handle";
+
+	ret = input_register_handle(handle);
+	if (ret)
+		goto free_handle;
+
+	ret = input_open_device(handle);
+	if (ret)
+		goto unregister_handle;
+
+	return 0;
+
+unregister_handle:
+	input_unregister_handle(handle);
+free_handle:
+	kfree(handle);
+	return ret;
+}
+
+static void cpu_input_boost_input_disconnect(struct input_handle *handle)
+{
+	input_close_device(handle);
+	input_unregister_handle(handle);
+	kfree(handle);
+}
+
+static const struct input_device_id cpu_input_boost_ids[] = {
+	/* Multi-touch touchscreen */
+	{
+		.flags = INPUT_DEVICE_ID_MATCH_EVBIT |
+			INPUT_DEVICE_ID_MATCH_ABSBIT,
+		.evbit = { BIT_MASK(EV_ABS) },
+		.absbit = { [BIT_WORD(ABS_MT_POSITION_X)] =
+			BIT_MASK(ABS_MT_POSITION_X) |
+			BIT_MASK(ABS_MT_POSITION_Y) }
+	},
+	/* Touchpad */
+	{
+		.flags = INPUT_DEVICE_ID_MATCH_KEYBIT |
+			INPUT_DEVICE_ID_MATCH_ABSBIT,
+		.keybit = { [BIT_WORD(BTN_TOUCH)] = BIT_MASK(BTN_TOUCH) },
+		.absbit = { [BIT_WORD(ABS_X)] =
+			BIT_MASK(ABS_X) | BIT_MASK(ABS_Y) }
+	},
+	/* Keypad */
+	{
+		.flags = INPUT_DEVICE_ID_MATCH_EVBIT,
+		.evbit = { BIT_MASK(EV_KEY) }
+	},
+	{ }
+};
+
+static struct input_handler cpu_input_boost_input_handler = {
+	.event		= cpu_input_boost_input_event,
+	.connect	= cpu_input_boost_input_connect,
+	.disconnect	= cpu_input_boost_input_disconnect,
+	.name		= "cpu_input_boost_handler",
+	.id_table	= cpu_input_boost_ids
+};
+
+static int __init cpu_input_boost_init(void)
+{
+	struct boost_drv *b = &boost_drv_g;
+	struct task_struct *thread;
+	int ret;
+
+	b->cpu_notif.notifier_call = cpu_notifier_cb;
+	ret = cpufreq_register_notifier(&b->cpu_notif, CPUFREQ_POLICY_NOTIFIER);
+	if (ret) {
+		pr_err("Failed to register cpufreq notifier, err: %d\n", ret);
+		return ret;
+	}
+
+	cpu_input_boost_input_handler.private = b;
+	ret = input_register_handler(&cpu_input_boost_input_handler);
+	if (ret) {
+		pr_err("Failed to register input handler, err: %d\n", ret);
+		goto unregister_cpu_notif;
+	}
+
+	b->msm_drm_notif.notifier_call = msm_drm_notifier_cb;
+	b->msm_drm_notif.priority = INT_MAX;
+	ret = msm_drm_register_client(&b->msm_drm_notif);
+	if (ret) {
+		pr_err("Failed to register msm_drm notifier, err: %d\n", ret);
+		goto unregister_handler;
+	}
+
+	thread = kthread_run(cpu_boost_thread, b, "cpu_boostd");
+	if (IS_ERR(thread)) {
+		ret = PTR_ERR(thread);
+		pr_err("Failed to start CPU boost thread, err: %d\n", ret);
+		goto unregister_fb_notif;
+	}
+
+	return 0;
+
+unregister_fb_notif:
+	msm_drm_unregister_client(&b->msm_drm_notif);
+unregister_handler:
+	input_unregister_handler(&cpu_input_boost_input_handler);
+unregister_cpu_notif:
+	cpufreq_unregister_notifier(&b->cpu_notif, CPUFREQ_POLICY_NOTIFIER);
+	return ret;
+}
+subsys_initcall(cpu_input_boost_init);

include/linux/cpu_input_boost.h

@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2018-2019 Sultan Alsawaf <sultan@kerneltoast.com>.
+ */
+#ifndef _CPU_INPUT_BOOST_H_
+#define _CPU_INPUT_BOOST_H_
+
+#ifdef CONFIG_CPU_INPUT_BOOST
+void cpu_input_boost_kick(void);
+void cpu_input_boost_kick_max(unsigned int duration_ms);
+#else
+static inline void cpu_input_boost_kick(void)
+{
+}
+static inline void cpu_input_boost_kick_max(unsigned int duration_ms)
+{
+}
+#endif
+
+#endif /* _CPU_INPUT_BOOST_H_ */