cpuidle: Mark CPUs idle as late as possible to avoid unneeded IPIs

It isn't guaranteed a CPU will idle upon calling lpm_cpuidle_enter(),
since it could abort early at the need_resched() check. In this case,
it's possible for an IPI to be sent to this "idle" CPU needlessly, thus
wasting power. For the same reason, it's also wasteful to keep a CPU
marked idle even after it's woken up.

Reduce the window that CPUs are marked idle to as small as it can be in
order to improve power consumption.

Signed-off-by: Sultan Alsawaf <[email protected]>

Author: kerneltoast

drivers/cpuidle/cpuidle.c

@@ -42,18 +42,15 @@ static atomic_t idled = ATOMIC_INIT(0);
 #error idled CPU mask not big enough for NR_CPUS
 #endif
 
-static void cpuidle_set_idle_cpu(unsigned int cpu)
+void cpuidle_set_idle_cpu(unsigned int cpu)
 {
 	atomic_or(BIT(cpu), &idled);
 }
 
-static void cpuidle_clear_idle_cpu(unsigned int cpu)
+void cpuidle_clear_idle_cpu(unsigned int cpu)
 {
 	atomic_andnot(BIT(cpu), &idled);
 }
-#else
-static inline void cpuidle_set_idle_cpu(unsigned int cpu) { }
-static inline void cpuidle_clear_idle_cpu(unsigned int cpu) { }
 #endif
 
 int cpuidle_disabled(void)
@@ -220,9 +217,7 @@ int cpuidle_enter_state(struct cpuidle_device *dev, struct cpuidle_driver *drv,
 	time_start = ktime_get();
 
 	stop_critical_timings();
-	cpuidle_set_idle_cpu(dev->cpu);
 	entered_state = target_state->enter(dev, drv, index);
-	cpuidle_clear_idle_cpu(dev->cpu);
 	start_critical_timings();
 
 	time_end = ktime_get();

drivers/cpuidle/lpm-levels.c

@@ -1506,7 +1506,9 @@ static int lpm_cpuidle_enter(struct cpuidle_device *dev,
 		goto exit;
 
 	BUG_ON(!use_psci);
+	cpuidle_set_idle_cpu(dev->cpu);
 	success = psci_enter_sleep(cluster, idx, true);
+	cpuidle_clear_idle_cpu(dev->cpu);
 
 exit:
 	end_time = ktime_to_ns(ktime_get());

include/linux/cpuidle.h

@@ -252,4 +252,12 @@ static inline int cpuidle_register_governor(struct cpuidle_governor *gov)
 #define CPUIDLE_DRIVER_STATE_START	0
 #endif
 
+#ifdef CONFIG_SMP
+void cpuidle_set_idle_cpu(unsigned int cpu);
+void cpuidle_clear_idle_cpu(unsigned int cpu);
+#else
+static inline void cpuidle_set_idle_cpu(unsigned int cpu) { }
+static inline void cpuidle_clear_idle_cpu(unsigned int cpu) { }
+#endif
+
 #endif /* _LINUX_CPUIDLE_H */