From c2d429419506e3efc5356ba8beae3c1a38971140 Mon Sep 17 00:00:00 2001 From: Jianyong Wu Date: Mon, 23 Mar 2026 14:14:15 +0000 Subject: [PATCH 01/11] sched/topo: Add helper to get node and node distance for llc Add helpers to get the NUMA node containing an LLC and the NUMA distance between two LLCs for the following patches. Signed-off-by: Jianyong Wu --- include/linux/topology.h | 4 ++++ kernel/sched/topology.c | 31 +++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/include/linux/topology.h b/include/linux/topology.h index 6575af39fd10f..11232dd6a1a3f 100644 --- a/include/linux/topology.h +++ b/include/linux/topology.h @@ -265,6 +265,10 @@ static inline const struct cpumask *cpu_node_mask(int cpu) return cpumask_of_node(cpu_to_node(cpu)); } +#define NR_LLCS NR_CPUS +int llc_to_node(int llc); +int llc_distance(int llc0, int llc1); + #ifdef CONFIG_NUMA int sched_numa_find_nth_cpu(const struct cpumask *cpus, int cpu, int node); extern const struct cpumask *sched_numa_hop_mask(unsigned int node, unsigned int hops); diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index e86dea1b9e86c..943dc461ad9e6 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -675,6 +675,8 @@ DEFINE_PER_CPU(struct sched_domain __rcu *, sd_asym_cpucapacity); DEFINE_STATIC_KEY_FALSE(sched_asym_cpucapacity); DEFINE_STATIC_KEY_FALSE(sched_cluster_active); +static int llc_to_node_map[NR_LLCS] = { [0 ... NR_LLCS-1] = -1 }; + static void update_top_cache_domain(int cpu) { struct sched_domain_shared *sds = NULL; @@ -811,6 +813,31 @@ DEFINE_STATIC_KEY_FALSE(sched_cache_active); /* user wants cache aware scheduling [0 or 1] */ int sysctl_sched_cache_user = 1; +/* Return the NUMA node containing the llc */ +int llc_to_node(int llc) +{ + if (llc < 0) + return -1; + + if (llc >= NR_LLCS) + return -1; + + return llc_to_node_map[llc]; +} + +/* Return the NUMA distance between the node containing LLC1 and the node containing LLC2 */ +int llc_distance(int llc1, int llc2) +{ + int numa1, numa2; + + numa1 = llc_to_node(llc1); + numa2 = llc_to_node(llc2); + if (numa1 < 0 || numa2 < 0) + return -1; + + return node_distance(numa1, numa2); +} + static bool alloc_sd_pref(const struct cpumask *cpu_map, struct s_data *d) { @@ -902,6 +929,9 @@ static bool alloc_sd_pref(const struct cpumask *cpu_map, { return false; } + +int llc_to_node(int llc) { return -1; } +int llc_distance(int llc1, int llc2) { return -1; } #endif /* @@ -2709,6 +2739,7 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att } lid = per_cpu(sd_llc_id, i); + llc_to_node_map[tl_max_llcs] = cpu_to_node(i); if (lid == -1) { int j; From f3c4ade733aa4d4c526cfd30444383bdb927d5cd Mon Sep 17 00:00:00 2001 From: Jianyong Wu Date: Mon, 23 Mar 2026 14:38:14 +0000 Subject: [PATCH 02/11] sched/topo: Add a easy way to get sched domain for NODE There is no straightforward way to retrieve the NODE-level sched domain, which will be used in subsequent patches. To simplify this, add a per-CPU variable `sd_node` for convenience. Signed-off-by: Jianyong Wu --- kernel/sched/sched.h | 1 + kernel/sched/topology.c | 17 +++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index f4785f84b1f12..d847c1c12639c 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2104,6 +2104,7 @@ DECLARE_PER_CPU(int, sd_llc_size); DECLARE_PER_CPU(int, sd_llc_id); DECLARE_PER_CPU(int, sd_share_id); DECLARE_PER_CPU(struct sched_domain_shared __rcu *, sd_llc_shared); +DECLARE_PER_CPU(struct sched_domain __rcu *, sd_node); DECLARE_PER_CPU(struct sched_domain __rcu *, sd_numa); DECLARE_PER_CPU(struct sched_domain __rcu *, sd_asym_packing); DECLARE_PER_CPU(struct sched_domain __rcu *, sd_asym_cpucapacity); diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index 943dc461ad9e6..99037ca8430ec 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -668,6 +668,7 @@ DEFINE_PER_CPU(int, sd_llc_size); DEFINE_PER_CPU(int, sd_llc_id) = -1; DEFINE_PER_CPU(int, sd_share_id); DEFINE_PER_CPU(struct sched_domain_shared __rcu *, sd_llc_shared); +DEFINE_PER_CPU(struct sched_domain __rcu *, sd_node); DEFINE_PER_CPU(struct sched_domain __rcu *, sd_numa); DEFINE_PER_CPU(struct sched_domain __rcu *, sd_asym_packing); DEFINE_PER_CPU(struct sched_domain __rcu *, sd_asym_cpucapacity); @@ -677,6 +678,19 @@ DEFINE_STATIC_KEY_FALSE(sched_cluster_active); static int llc_to_node_map[NR_LLCS] = { [0 ... NR_LLCS-1] = -1 }; +/* Get sched domain by its name */ +static struct sched_domain *get_sched_domain_by_name(int cpu, char *name) +{ + struct sched_domain *sd = NULL; + + for_each_domain(cpu, sd) { + if (!strcmp(sd->name, name)) + break; + } + + return sd; +} + static void update_top_cache_domain(int cpu) { struct sched_domain_shared *sds = NULL; @@ -706,6 +720,9 @@ static void update_top_cache_domain(int cpu) */ per_cpu(sd_share_id, cpu) = id; + sd = get_sched_domain_by_name(cpu, "NODE"); + rcu_assign_pointer(per_cpu(sd_node, cpu), sd); + sd = lowest_flag_domain(cpu, SD_NUMA); rcu_assign_pointer(per_cpu(sd_numa, cpu), sd); From 2b61ffd39992dd2c1cd751f9b5a12540715dbfbb Mon Sep 17 00:00:00 2001 From: Jianyong Wu Date: Mon, 23 Mar 2026 15:21:26 +0000 Subject: [PATCH 03/11] sched/fair: select preferred llc inside preferred NUMA MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In the current implementation, the preferred LLC is selected based on the LLC with the largest running time of the thread group. However, the preferred LLC may be prone to frequent migration when the workload spreads across the entire system—especially when the number of CPUs sharing an LLC is small. A better approach is to first select a preferred NUMA node in the same way, and then select the preferred LLC within that preferred NUMA node. Signed-off-by: Jianyong Wu --- kernel/sched/fair.c | 72 +++++++++++++++++++++++++++------------------ 1 file changed, 43 insertions(+), 29 deletions(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 977091fd0e499..5353d684072d9 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -1599,8 +1599,9 @@ static void task_cache_work(struct callback_head *work) struct task_struct *p = current, *cur; struct mm_struct *mm = p->mm; unsigned long m_a_occ = 0; - unsigned long curr_m_a_occ = 0; - int cpu, m_a_cpu = -1, nr_running = 0, curr_cpu; + unsigned long m_a_n_occ = 0; + unsigned long curr_m_a_n_occ = 0; + int cpu, m_a_cpu = -1, m_a_n_cpu = -1, nr_running = 0, curr_cpu; cpumask_var_t cpus; WARN_ON_ONCE(work != &p->cache_work); @@ -1626,30 +1627,44 @@ static void task_cache_work(struct callback_head *work) cpumask_copy(cpus, cpu_online_mask); for_each_cpu(cpu, cpus) { - /* XXX sched_cluster_active */ - struct sched_domain *sd = per_cpu(sd_llc, cpu); - unsigned long occ, m_occ = 0, a_occ = 0; - int m_cpu = -1, i; + struct sched_domain *nsd = per_cpu(sd_node, cpu); + unsigned long occ, m_occ = 0, a_occ = 0, a_n_occ = 0; + int m_cpu = -1, i, k; - if (!sd) + if (!nsd) continue; - for_each_cpu(i, sched_domain_span(sd)) { - occ = fraction_mm_sched(cpu_rq(i), - per_cpu_ptr(mm->sc_stat.pcpu_sched, i)); - a_occ += occ; - if (occ > m_occ) { - m_occ = occ; - m_cpu = i; + for_each_cpu_and(k, sched_domain_span(nsd), cpus) { + a_occ = m_a_occ = m_occ = 0; + struct sched_domain *sd = per_cpu(sd_llc, k); + + if (!sd) + continue; + + for_each_cpu(i, sched_domain_span(sd)) { + occ = fraction_mm_sched(cpu_rq(i), + per_cpu_ptr(mm->sc_stat.pcpu_sched, i)); + a_occ += occ; + if (occ > m_occ) { + m_occ = occ; + m_cpu = i; + } + scoped_guard (rcu) { + cur = rcu_dereference(cpu_rq(i)->curr); + if (cur && !(cur->flags & (PF_EXITING | + PF_KTHREAD)) && cur->mm == mm) + nr_running++; + } } - scoped_guard (rcu) { - cur = rcu_dereference(cpu_rq(i)->curr); - if (cur && !(cur->flags & (PF_EXITING | PF_KTHREAD)) && - cur->mm == mm) - nr_running++; + + cpumask_andnot(cpus, cpus, sched_domain_span(sd)); + if (a_occ > m_a_occ) { + m_a_occ = a_occ; + m_a_cpu = m_cpu; } + /* record for numa node */ + a_n_occ += a_occ; } - /* * Compare the accumulated occupancy of each LLC. The * reason for using accumulated occupancy rather than average @@ -1665,19 +1680,18 @@ static void task_cache_work(struct callback_head *work) * the average number of faults per CPU. This strategy is also * followed here. */ - if (a_occ > m_a_occ) { - m_a_occ = a_occ; - m_a_cpu = m_cpu; + if (a_n_occ > m_a_n_occ) { + m_a_n_occ = a_n_occ; + m_a_n_cpu = m_a_cpu; } - if (llc_id(cpu) == llc_id(mm->sc_stat.cpu)) - curr_m_a_occ = a_occ; - - cpumask_andnot(cpus, cpus, sched_domain_span(sd)); + if (mm->sc_stat.cpu != -1 && cpu_to_node(cpu) == + cpu_to_node(mm->sc_stat.cpu)) + curr_m_a_n_occ = a_n_occ; } } - if (m_a_occ > (2 * curr_m_a_occ)) { + if (m_a_n_occ > (2 * curr_m_a_n_occ)) { /* * Avoid switching sc_stat.cpu too fast. * The reason to choose 2X is because: @@ -1688,7 +1702,7 @@ static void task_cache_work(struct callback_head *work) * 3. 2X is chosen based on test results, as it delivers * the optimal performance gain so far. */ - mm->sc_stat.cpu = m_a_cpu; + mm->sc_stat.cpu = m_a_n_cpu; } update_avg_scale(&mm->sc_stat.nr_running_avg, nr_running); From 1239530a38607e19f7d5a049135e1a0c7fd6f8a9 Mon Sep 17 00:00:00 2001 From: Jianyong Wu Date: Mon, 23 Mar 2026 16:34:46 +0000 Subject: [PATCH 04/11] sched/fair: decide if migrate for a generic node sched/fair: Decide whether to migrate for a generic node Cache-aware scheduling focuses only on the LLC. It makes migration decisions based on a strict policy: whether the destination LLC is the preferred LLC. This loses other important information such as NUMA distance. To utilize NUMA distance information, we extend the concept of a node from the LLC to a generic one. This generic node can refer to an LLC, a NUMA node, or a group of NUMA nodes, and can be represented by a sched domain above the LLC level. Based on this, a helper function to decide whether migration can occur between generic nodes is introduced. Signed-off-by: Jianyong Wu --- kernel/sched/fair.c | 101 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 5353d684072d9..73d94c7cd99df 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -10061,6 +10061,7 @@ static __maybe_unused bool get_llc_stats(int cpu, unsigned long *util, return true; } + /* * Decision matrix according to the LLC utilization. To * decide whether we can do task aggregation across LLC. @@ -10162,6 +10163,106 @@ static enum llc_mig can_migrate_llc(int src_cpu, int dst_cpu, return mig_llc; } +/* + * Like get_llc_stats but for sched domain that above LLC level. + * Based on get_llc_stats, we can accumulate utility and cap for + * sched domain in the granularity of LLC. + */ +static bool get_sd_stats(struct sched_domain *sd, unsigned long *util_out, unsigned long *cap_out) +{ + struct cpumask mask; + int cpu; + unsigned long util_tmp, cap_tmp, util = 0, cap = 0; + struct sched_domain *sd_tmp; + + if (!sd || !util_out || !cap_out) + return false; + + cpumask_copy(&mask, sched_domain_span(sd)); + for_each_cpu(cpu, &mask) { + if (!get_llc_stats(cpu, &util_tmp, &cap_tmp)) + return false; + + sd_tmp = rcu_dereference(per_cpu(sd_llc, cpu)); + cpumask_andnot(&mask, &mask, sched_domain_span(sd_tmp)); + util += util_tmp; + cap += cap_tmp; + } + + *util_out = util; + *cap_out = cap; + + return true; +} + +/* Decide if a sched domain is overload. */ +static bool is_domain_overload(struct sched_domain *sd) +{ + int ret; + unsigned long util = 0, cap = 0; + + get_sd_stats(sd, &util, &cap); + + /* We are not a llc. Need change name? */ + ret = !fits_llc_capacity(util, cap); + + return ret; +} + +/* + * Decide if migration should happen on a specific node. + * The node here is a generic conception for a set of cpu. + * It Usually indecates one of sched domain for LLC level and above. + */ +static enum llc_mig can_migrate_node(int src_cpu, int dst_cpu, struct task_struct *p, bool to_pref) +{ + struct sched_domain *domain; + unsigned long dst_util, dst_cap, tsk_util = 0; + int k = 0; + + if (!get_llc_stats(dst_cpu, &dst_util, &dst_cap)) + return mig_unrestricted; + + if (p) + tsk_util = task_util(p); + + dst_util = dst_util + tsk_util; + + if (to_pref) { + if (fits_llc_capacity(dst_util, dst_cap)) + return mig_llc; + else + return mig_unrestricted; + } + + /* + * If the dest node decrase locality, decide if it should migrate by testing that + * if it is the closest place that is not overload. + */ + for_each_domain(src_cpu, domain) { + /* Skip sched domain lower than MC */ + if (domain->flags & SD_SHARE_LLC) + continue; + + /* Allow migration if we found dest cpu in this sched domain */ + if (cpumask_test_cpu(dst_cpu, sched_domain_span(domain))) + return mig_llc; + + /* + * For the special case: the workload is small and the dest cpu may far away + * from src cpu. + */ + if (p && (domain->span_weight > get_nr_threads(p) && k++)) + return mig_unrestricted; + + /* Don't migrate if there is a better place to live */ + if (!is_domain_overload(domain)) + return mig_forbid; + } + + return mig_unrestricted; +} + /* * Check if task p can migrate from source LLC to * destination LLC in terms of cache aware load balance. From 720c3b47162f89136e8af7a9500c55c621e27c82 Mon Sep 17 00:00:00 2001 From: Jianyong Wu Date: Mon, 30 Mar 2026 14:15:11 +0000 Subject: [PATCH 05/11] sched/fair: Add rq affinity boost calculation algorithm MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In the current implementation of the cache-aware scheduler, the policy for finding the best src group/rq only checks whether the preferred LLC of the task on the rq matches the dst LLC. However, migration may still improve affinity even if the preferred LLC does not match the dst LLC. For example, the src CPU resides in NODE0 and the dst CPU in NODE1. There is a task on the src CPU whose preferred LLC is in NODE2. Assume the node distance between NODE0 and NODE2 is 20, while the distance between NODE1 and NODE2 is 15. The task thus achieves better affinity after migrating to the dst CPU, but this case is not covered by the current implementation. To address this issue, introduce a new algorithm for calculating the affinity promotion given a dst LLC, src LLC, and rq. It can be roughly split into two steps: 1. Given the dst LLC and src LLC, iterate all LLCs in the system that can improve affinity when a task (with that LLC as its preferred LLC) migrates from the src CPU to the dst CPU. Compute the node distance difference for each corresponding LLC using the following formula: Di = llc_distance(src_llc, LLCi) - llc_distance(dst_llc, LLCi) (1) where i is the index of each system LLC. The minimum value of Di is clamped to 2 to avoid division by zero. 2. Given the LLCs from step 1 and a target rq, calculate the total affinity promotion quantity of the rq by accumulating the promotion value of each task on the rq. For each rq, the quantity is computed as: W_i = Rt_i * 1024 / Di (2) p = Σ_i W_i (3) where p represents the total affinity promotion quantity, and Rt_i is the number of tasks on the rq with LLCi as their preferred LLC. Rt_i can be obtained from rq->sd->pf. Signed-off-by: Jianyong Wu --- kernel/sched/fair.c | 77 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 73d94c7cd99df..bff49276eea00 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -11487,6 +11487,83 @@ static bool update_llc_busiest(struct lb_env *env, */ return sgs->nr_pref_dst_llc > busiest->nr_pref_dst_llc; } + +/* + * Get all LLCs that are closer to the destination LLC than to the + * source LLC. + * @affi_llcs: array to store LLCs satisfying the above condition + * @dist: array to store Di for each LLC in affi_llcs, computed as: + * + * Di = llc_distance(src_llc, LLCi) - llc_distance(dst_llc, LLCi) (1) + * where i is the index of affi_llcs. + */ +static int get_affi_llcs(int src_llc, int dst_llc, int *affi_llcs, int *dist) +{ + int j = 0, dis1, dis2; + + if (src_llc == dst_llc) + return 0; + + if (llc_to_node(src_llc) == llc_to_node(dst_llc)) { + affi_llcs[j] = dst_llc; + dist[j++] = 2; + return j; + } + + for (int i = 0; i < max_llcs; i++) { + dis1 = llc_distance(src_llc, i); + dis2 = llc_distance(dst_llc, i); + if (dis1 < 0 || dis2 < 0) + continue; + if (dis1 > dis2) { + dist[j] = clamp(dis1 - dis2, 4, 1024); + affi_llcs[j++] = i; + } + } + + return j; +} + +/* + * To find a src sched group/rq during load balancing, we need a method to + * calculate the benefit of each rq. For sched cache, we focus more on + * affinity improvement. + * + * This provides a way to quantify the affinity improvement for each rq + * by assigning an affinity score to each rq. + * + * Calculate the affinity score for a rq given src llc and dst llc. + * It is computed as: + * Di = llc_distance(src_llc, LLCi) - llc_distance(dst_llc, LLCi) (1) + * W_i = Rt_i * 1024 / Di (2) + * p = Σ_i W_i (3) + * + * where i is the index of an LLC, Di is obtained from get_affi_llcs, and + * Rt_i is the number of tasks on the rq with LLCi as their preferred LLC, + * obtainable from rq->sd->pf. + */ +static int cal_affinity_score(struct rq *rq, int src_cpu, int dst_llc) +{ + int *affi_llcs, *dist, num, wt = 0; + struct sched_domain *sd_tmp = rcu_dereference(rq->sd); + + affi_llcs = kmalloc_array(max_llcs, sizeof(*affi_llcs), GFP_NOWAIT); + if (!affi_llcs) + return wt; + + dist = kmalloc_array(max_llcs, sizeof(*dist), GFP_NOWAIT); + if (!dist) + goto fail; + + num = get_affi_llcs(llc_id(src_cpu), dst_llc, affi_llcs, dist); + for (int i = 0; i < num; i++) + wt += (sd_tmp->pf[affi_llcs[i]] << 10) / dist[i]; + + kfree(dist); +fail: + kfree(affi_llcs); + return wt; +} #else static inline void record_sg_llc_stats(struct lb_env *env, struct sg_lb_stats *sgs, struct sched_group *group) From 42a283105ed53601eb8b65f26cd2830ae344b19d Mon Sep 17 00:00:00 2001 From: Jianyong Wu Date: Wed, 1 Apr 2026 15:02:20 +0000 Subject: [PATCH 06/11] sched/fair: consider affinity improvement when find src group/rq In the current implementation, only the preferred LLC is consideredwhen finding the source group and source rq. The previous patch provides a way to calculate affinity improvementfor a rq given a source LLC and a destination LLC. We can utilize this method to find src group and rq. Signed-off-by: Jianyong Wu --- kernel/sched/fair.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index bff49276eea00..3b159fc6ec815 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -11471,8 +11471,8 @@ static inline bool llc_balance(struct lb_env *env, struct sg_lb_stats *sgs, return false; if (sgs->nr_pref_dst_llc && - can_migrate_llc(cpumask_first(sched_group_span(group)), - env->dst_cpu, 0, true) == mig_llc) + can_migrate_node(cpumask_first(sched_group_span(group)), + env->dst_cpu, NULL, true) == mig_llc) return true; return false; @@ -11630,7 +11630,7 @@ static inline void update_sg_lb_stats(struct lb_env *env, struct sched_domain *sd_tmp = rcu_dereference(rq->sd); if (valid_llc_buf(sd_tmp, dst_llc)) - sgs->nr_pref_dst_llc += sd_tmp->pf[dst_llc]; + sgs->nr_pref_dst_llc += cal_affinity_score(rq, i, dst_llc); } #endif @@ -12856,7 +12856,7 @@ static struct rq *sched_balance_find_src_rq(struct lb_env *env, sd_tmp = rcu_dereference(rq->sd); dst_llc = llc_id(env->dst_cpu); if (valid_llc_buf(sd_tmp, dst_llc)) { - unsigned int this_pref_llc = sd_tmp->pf[dst_llc]; + unsigned int this_pref_llc = cal_affinity_score(rq, i, dst_llc); if (busiest_pref_llc < this_pref_llc) { busiest_pref_llc = this_pref_llc; From 36388d92fde60a90d7b5fada2482874c47a97bab Mon Sep 17 00:00:00 2001 From: Jianyong Wu Date: Wed, 1 Apr 2026 15:21:58 +0000 Subject: [PATCH 07/11] sched/fair: remove the sibling bound for llc_balance As the group type for group_llc_balance focuses not only on the LLC level, but also on the NUMA level, the prefer_sibling constraint should be removed for it. Signed-off-by: Jianyong Wu --- kernel/sched/fair.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 3b159fc6ec815..a5f995e759eb2 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -12657,9 +12657,9 @@ static struct sched_group *sched_balance_find_src_group(struct lb_env *env) * Try to move all excess tasks to a sibling domain of the busiest * group's child domain. */ - if (sds.prefer_sibling && local->group_type == group_has_spare && - (busiest->group_type == group_llc_balance || - sibling_imbalance(env, &sds, busiest, local) > 1)) + if (local->group_type == group_has_spare && + ((busiest->group_type == group_llc_balance) || (sds.prefer_sibling && + sibling_imbalance(env, &sds, busiest, local) > 1))) goto force_balance; if (busiest->group_type != group_overloaded) { From af78982e9e4d1bd27ade9eb7c401156caca84562 Mon Sep 17 00:00:00 2001 From: Jianyong Wu Date: Tue, 24 Mar 2026 09:22:44 +0000 Subject: [PATCH 08/11] sched/fair: determine migration based on affinity improvement The current implementation only focuses on LLC level and ignores affinity in wider domains such as NUMA level. This change takes all affinity information, including cache locality and NUMA-wide affinity, to evaluate whether migration can improve locality. Node load check is added to make the final migration decision. Signed-off-by: Jianyong Wu --- kernel/sched/fair.c | 80 ++++++++++++++++++++++++++------------------- 1 file changed, 46 insertions(+), 34 deletions(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index a5f995e759eb2..8bc5b25ec5efb 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -10263,6 +10263,30 @@ static enum llc_mig can_migrate_node(int src_cpu, int dst_cpu, struct task_struc return mig_unrestricted; } +int llc_distance(int, int); + +/* Decide if the migration improve the affinity */ +static bool if_to_prefer(int src_cpu, int dst_cpu, int pref_llc) +{ + int src_dist, dst_dist, to_pref = false; + + src_dist = llc_distance(llc_id(src_cpu), pref_llc); + dst_dist = llc_distance(llc_id(dst_cpu), pref_llc); + + if (src_dist > dst_dist) { + to_pref = true; + } else if (src_dist == dst_dist) { + if (llc_id(dst_cpu) == pref_llc) + to_pref = true; + else + to_pref = false; + } else { + to_pref = false; + } + + return to_pref; +} + /* * Check if task p can migrate from source LLC to * destination LLC in terms of cache aware load balance. @@ -10293,15 +10317,10 @@ static enum llc_mig can_migrate_llc_task(int src_cpu, int dst_cpu, return mig_unrestricted; } - if (cpus_share_cache(dst_cpu, cpu)) - to_pref = true; - else if (cpus_share_cache(src_cpu, cpu)) - to_pref = false; - else - return mig_unrestricted; + to_pref = if_to_prefer(src_cpu, dst_cpu, llc_id(cpu)); - return can_migrate_llc(src_cpu, dst_cpu, - task_util(p), to_pref); + return can_migrate_node(src_cpu, dst_cpu, + p, to_pref); } /* @@ -10311,36 +10330,36 @@ static enum llc_mig can_migrate_llc_task(int src_cpu, int dst_cpu, static inline bool alb_break_llc(struct lb_env *env) { + int pref_llc = -1; + bool to_pref = false; + if (!sched_cache_enabled()) return false; if (cpus_share_cache(env->src_cpu, env->dst_cpu)) return false; /* - * All tasks prefer to stay on their current CPU. - * Do not pull a task from its preferred CPU if: - * 1. It is the only task running there; OR - * 2. Migrating it away from its preferred LLC would violate - * the cache-aware scheduling policy. + * We need the preferred LLC to decide whether we can perform migration. + * Therefore, we need to obtain task_struct, which is only available + * when there is a task running. + * For cases with more than one task on the rq, we need to check + * this in can_migrate_task(). */ - if (env->src_rq->nr_pref_llc_running && - env->src_rq->nr_pref_llc_running == env->src_rq->cfs.h_nr_runnable) { - unsigned long util = 0; + if (env->src_rq->nr_running == 1) { struct task_struct *cur; - if (env->src_rq->nr_running <= 1) - return true; - /* * Reach here in load balance with * rcu_read_lock() protected. */ cur = rcu_dereference(env->src_rq->curr); if (cur) - util = task_util(cur); + pref_llc = cur->preferred_llc; - if (can_migrate_llc(env->src_cpu, env->dst_cpu, - util, false) == mig_forbid) + to_pref = if_to_prefer(env->src_cpu, env->dst_cpu, pref_llc); + + if (can_migrate_node(env->src_cpu, env->dst_cpu, + cur, to_pref) == mig_forbid) return true; } @@ -10371,15 +10390,6 @@ static bool migrate_degrades_llc(struct task_struct *p, struct lb_env *env) if (env->sd->nr_balance_failed >= env->sd->cache_nice_tries + 1) return false; - /* - * We know the env->src_cpu has some tasks prefer to - * run on env->dst_cpu, skip the tasks do not prefer - * env->dst_cpu, and find the one that prefers. - */ - if (env->migration_type == migrate_llc_task && - task_llc(p) != llc_id(env->dst_cpu)) - return true; - if (can_migrate_llc_task(env->src_cpu, env->dst_cpu, p) != mig_forbid) return false; @@ -10498,15 +10508,14 @@ int can_migrate_task(struct task_struct *p, struct lb_env *env) * 3) task is cache cold, or * 4) too many balance attempts have failed. */ - if (env->flags & LBF_ACTIVE_LB) - return 1; - degrades = migrate_degrades_locality(p, env); if (!degrades) { /* * If the NUMA locality is not broken, * further check if migration would hurt * LLC locality. + * This should be done before check LBF_ACTIVE_LB + * as we has not check some cases in alb_break_llc */ if (migrate_degrades_llc(p, env)) return 0; @@ -10516,6 +10525,9 @@ int can_migrate_task(struct task_struct *p, struct lb_env *env) hot = degrades > 0; } + if (env->flags & LBF_ACTIVE_LB) + return 1; + if (!hot || env->sd->nr_balance_failed > env->sd->cache_nice_tries) { if (hot) p->sched_task_hot = 1; From 7a43f1899b2d2f25ed91dd19179097e05bef6052 Mon Sep 17 00:00:00 2001 From: Jianyong Wu Date: Tue, 24 Mar 2026 21:48:48 +0000 Subject: [PATCH 09/11] sched: make sched cache override numa balancing Cache-aware scheduling lacks memory awareness; remote memory access may still exist even with aggregated threads. Current designs let cache-aware scheduling follow NUMA balancing, but this breaks sched cache's preferred node selection logic. We propose an alternative: let sched cache override NUMA balancing. Sched cache handles task placement and migration, while NUMA balancing only manages memory migration. This combines the strengths of both mechanisms: sched cache is good at task aggregation and balancing, NUMA balancing is good at memory awareness. An interface is provided to disable this mode and revert to defaults. Signed-off-by: Jianyong Wu --- kernel/sched/debug.c | 2 ++ kernel/sched/fair.c | 17 ++++++++++++++--- kernel/sched/sched.h | 1 + 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c index dc4b7de6569f4..5999ac9531565 100644 --- a/kernel/sched/debug.c +++ b/kernel/sched/debug.c @@ -566,6 +566,8 @@ static __init int sched_init_debug(void) #ifdef CONFIG_SCHED_CACHE debugfs_create_file("llc_enabled", 0644, debugfs_sched, NULL, &sched_cache_enable_fops); + debugfs_create_u32("llc_override_numa_balance", 0644, debugfs_sched, + &llc_override_numa_balance); debugfs_create_u32("llc_aggr_tolerance", 0644, debugfs_sched, &llc_aggr_tolerance); debugfs_create_u32("llc_epoch_period", 0644, debugfs_sched, diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 8bc5b25ec5efb..460b0c958a987 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -1192,6 +1192,7 @@ static void set_next_buddy(struct sched_entity *se); #define EPOCH_LLC_AFFINITY_TIMEOUT 5 /* 50 ms */ __read_mostly unsigned int llc_aggr_tolerance = 1; +__read_mostly unsigned int llc_override_numa_balance = 1; __read_mostly unsigned int llc_epoch_period = EPOCH_PERIOD; __read_mostly unsigned int llc_epoch_affinity_timeout = EPOCH_LLC_AFFINITY_TIMEOUT; __read_mostly unsigned int llc_imb_pct = 20; @@ -1234,6 +1235,11 @@ static inline bool valid_llc_buf(struct sched_domain *sd, return valid_llc_id(id); } +static inline bool sched_cache_override_numa(void) +{ + return sched_cache_enabled() && llc_override_numa_balance; +} + static inline int get_sched_cache_scale(int mul) { if (!llc_aggr_tolerance) @@ -1481,9 +1487,10 @@ static int get_pref_llc(struct task_struct *p, struct mm_struct *mm) * than sched_setnuma() at least -- and thus the * conflict only exists for a short period of time. */ - if (static_branch_likely(&sched_numa_balancing) && - p->numa_preferred_nid >= 0 && - cpu_to_node(mm->sc_stat.cpu) != p->numa_preferred_nid) + if (!sched_cache_override_numa() && + static_branch_likely(&sched_numa_balancing) && + p->numa_preferred_nid >= 0 && + cpu_to_node(mm->sc_stat.cpu) != p->numa_preferred_nid) mm_sched_llc = -1; #endif } @@ -1763,6 +1770,7 @@ static void account_llc_enqueue(struct rq *rq, struct task_struct *p) {} static void account_llc_dequeue(struct rq *rq, struct task_struct *p) {} +static inline bool sched_cache_override_numa(void) {} #endif /* @@ -3729,6 +3737,9 @@ void task_numa_fault(int last_cpupid, int mem_node, int pages, int flags) if (!static_branch_likely(&sched_numa_balancing)) return; + if (sched_cache_override_numa()) + return; + /* for example, ksmd faulting in a user's mm */ if (!p->mm) return; diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index d847c1c12639c..375ed97a3aa78 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -3925,6 +3925,7 @@ extern unsigned int llc_epoch_period; extern unsigned int llc_epoch_affinity_timeout; extern unsigned int llc_imb_pct; extern unsigned int llc_overaggr_pct; +extern unsigned int llc_override_numa_balance; static inline bool sched_cache_enabled(void) { From a6948989a389ac473a44e3a392b2166d5ecf7d11 Mon Sep 17 00:00:00 2001 From: Jianyong Wu Date: Wed, 25 Mar 2026 14:07:18 +0000 Subject: [PATCH 10/11] sched/fair: dynamicly scale the period of cache work When a preferred LLC is selected and remains stable, task_cache_work does not need to run frequently. Because it scans all system CPUs for computation, high-frequency execution hurts performance. We thus reduce the scan rate in such cases. On the other hand, if the preferred node becomes suboptimal, we should increase the scan frequency to quickly find a better placement. The scan period is therefore dynamically adjusted. Signed-off-by: Jianyong Wu --- include/linux/sched.h | 4 +++ kernel/sched/debug.c | 6 ++++ kernel/sched/fair.c | 81 ++++++++++++++++++++++++++++++++++++------- kernel/sched/sched.h | 3 ++ 4 files changed, 81 insertions(+), 13 deletions(-) diff --git a/include/linux/sched.h b/include/linux/sched.h index 4236cacbb4099..505af39ed54fa 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -2347,7 +2347,11 @@ struct sched_cache_stat { struct sched_cache_time __percpu *pcpu_time; raw_spinlock_t lock; unsigned long epoch; + unsigned long last_reset_tick; + unsigned long next_scan; + unsigned long scan_period; u64 nr_running_avg; + int need_scan; int cpu; } ____cacheline_aligned_in_smp; diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c index 5999ac9531565..1cd532bfa9d98 100644 --- a/kernel/sched/debug.c +++ b/kernel/sched/debug.c @@ -568,6 +568,12 @@ static __init int sched_init_debug(void) &sched_cache_enable_fops); debugfs_create_u32("llc_override_numa_balance", 0644, debugfs_sched, &llc_override_numa_balance); + debugfs_create_u32("llc_scan_period_max", 0644, debugfs_sched, + &llc_scan_period_max); + debugfs_create_u32("llc_scan_period_min", 0644, debugfs_sched, + &llc_scan_period_min); + debugfs_create_u32("llc_scan_period_threshold", 0644, debugfs_sched, + &llc_scan_period_threshold); debugfs_create_u32("llc_aggr_tolerance", 0644, debugfs_sched, &llc_aggr_tolerance); debugfs_create_u32("llc_epoch_period", 0644, debugfs_sched, diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 460b0c958a987..5e49424eadeec 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -1197,6 +1197,9 @@ __read_mostly unsigned int llc_epoch_period = EPOCH_PERIOD; __read_mostly unsigned int llc_epoch_affinity_timeout = EPOCH_LLC_AFFINITY_TIMEOUT; __read_mostly unsigned int llc_imb_pct = 20; __read_mostly unsigned int llc_overaggr_pct = 50; +__read_mostly unsigned int llc_scan_period_min = 1; +__read_mostly unsigned int llc_scan_period_max = 64 * HZ; +__read_mostly unsigned int llc_scan_period_threshold = HZ; bool sched_cache_inuse(void) { @@ -1408,6 +1411,7 @@ void mm_init_sched(struct mm_struct *mm, raw_spin_lock_init(&mm->sc_stat.lock); mm->sc_stat.epoch = epoch; mm->sc_stat.cpu = -1; + mm->sc_stat.scan_period = llc_scan_period_min; /* * The update to mm->sc_stat should not be reordered @@ -1534,13 +1538,8 @@ void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec) epoch = rq->cpu_epoch; } - /* - * If this process hasn't hit task_cache_work() for a while, or it - * has only 1 thread, invalidate its preferred state. - */ - if (time_after(epoch, - READ_ONCE(mm->sc_stat.epoch) + llc_epoch_affinity_timeout) || - get_nr_threads(p) <= 1 || + /* If it has only 1 thread, invalidate its preferred state. */ + if (get_nr_threads(p) <= 1 || exceed_llc_nr(mm, cpu_of(rq), p) || exceed_llc_capacity(mm, cpu_of(rq), p)) { if (mm->sc_stat.cpu != -1) @@ -1575,6 +1574,10 @@ static void task_tick_cache(struct rq *rq, struct task_struct *p) if (time_after_eq(mm->sc_stat.epoch, epoch)) return; + if (time_before(jiffies, mm->sc_stat.next_scan) && + !mm->sc_stat.need_scan) + return; + guard(raw_spinlock)(&mm->sc_stat.lock); if (work->next == work) { @@ -1608,7 +1611,8 @@ static void task_cache_work(struct callback_head *work) unsigned long m_a_occ = 0; unsigned long m_a_n_occ = 0; unsigned long curr_m_a_n_occ = 0; - int cpu, m_a_cpu = -1, m_a_n_cpu = -1, nr_running = 0, curr_cpu; + unsigned long now; + int cpu, m_a_cpu = -1, m_a_n_cpu = -1, nr_running = 0, curr_cpu, need_scan = 0; cpumask_var_t cpus; WARN_ON_ONCE(work != &p->cache_work); @@ -1630,6 +1634,12 @@ static void task_cache_work(struct callback_head *work) if (!zalloc_cpumask_var(&cpus, GFP_KERNEL)) return; + now = jiffies; + if (time_before(now, READ_ONCE(mm->sc_stat.next_scan))) + return; + + WRITE_ONCE(mm->sc_stat.next_scan, (now + mm->sc_stat.scan_period)); + scoped_guard (cpus_read_lock) { cpumask_copy(cpus, cpu_online_mask); @@ -1698,7 +1708,8 @@ static void task_cache_work(struct callback_head *work) } } - if (m_a_n_occ > (2 * curr_m_a_n_occ)) { + need_scan = READ_ONCE(mm->sc_stat.need_scan); + if (m_a_n_occ > (2 * curr_m_a_n_occ) || need_scan) { /* * Avoid switching sc_stat.cpu too fast. * The reason to choose 2X is because: @@ -1709,9 +1720,35 @@ static void task_cache_work(struct callback_head *work) * 3. 2X is chosen based on test results, as it delivers * the optimal performance gain so far. */ - mm->sc_stat.cpu = m_a_n_cpu; + if (m_a_n_occ > (2 * curr_m_a_n_occ)) + mm->sc_stat.cpu = m_a_n_cpu; + + if (!mm->sc_stat.last_reset_tick) + mm->sc_stat.last_reset_tick = now; + + /* Change scan_period when preferred NUMA changed */ + if (((mm->sc_stat.cpu != -1) && (m_a_n_cpu != -1) + && (cpu_to_node(mm->sc_stat.cpu) != cpu_to_node(m_a_n_cpu))) + || need_scan) { + if (!need_scan) + need_scan = 1; + + WRITE_ONCE(mm->sc_stat.scan_period, + max(mm->sc_stat.scan_period >> 1, llc_scan_period_min)); + WRITE_ONCE(mm->sc_stat.last_reset_tick, now); + } + } + + if ((now - READ_ONCE(mm->sc_stat.last_reset_tick) > llc_scan_period_threshold) + && !need_scan) { + WRITE_ONCE(mm->sc_stat.scan_period, min(mm->sc_stat.scan_period << 1, + llc_scan_period_max)); + WRITE_ONCE(mm->sc_stat.last_reset_tick, now); } + if (READ_ONCE(mm->sc_stat.need_scan)) + WRITE_ONCE(mm->sc_stat.need_scan, 0); + update_avg_scale(&mm->sc_stat.nr_running_avg, nr_running); free_cpumask_var(cpus); } @@ -10046,6 +10083,13 @@ static inline int task_is_ineligible_on_dst_cpu(struct task_struct *p, int dest_ #define fits_llc_capacity(util, max) \ ((util) * 100 < (max) * llc_overaggr_pct) +/* + * Like fits_llc_capacity but consider bias. + * The bias here is the half of llc_imb_pct. + */ +#define fits_llc_cap_imb(util, max) \ + ((util) * 100 < (max) * (llc_overaggr_pct + llc_imb_pct / 2)) + /* * The margin used when comparing utilization. * is 'util1' noticeably greater than 'util2' @@ -10227,6 +10271,7 @@ static bool is_domain_overload(struct sched_domain *sd) */ static enum llc_mig can_migrate_node(int src_cpu, int dst_cpu, struct task_struct *p, bool to_pref) { + struct mm_struct *mm = NULL; struct sched_domain *domain; unsigned long dst_util, dst_cap, tsk_util = 0; int k = 0; @@ -10234,16 +10279,22 @@ static enum llc_mig can_migrate_node(int src_cpu, int dst_cpu, struct task_struc if (!get_llc_stats(dst_cpu, &dst_util, &dst_cap)) return mig_unrestricted; - if (p) + if (p) { + mm = p->mm; tsk_util = task_util(p); + } dst_util = dst_util + tsk_util; if (to_pref) { if (fits_llc_capacity(dst_util, dst_cap)) return mig_llc; - else + else { + if (mm && !fits_llc_cap_imb(dst_util, dst_cap)) + mm->sc_stat.need_scan = 1; + return mig_unrestricted; + } } /* @@ -10263,8 +10314,12 @@ static enum llc_mig can_migrate_node(int src_cpu, int dst_cpu, struct task_struc * For the special case: the workload is small and the dest cpu may far away * from src cpu. */ - if (p && (domain->span_weight > get_nr_threads(p) && k++)) + if (p && (domain->span_weight > get_nr_threads(p) && k++)) { + if (mm && !fits_llc_cap_imb(dst_util, dst_cap)) + mm->sc_stat.need_scan = 1; + return mig_unrestricted; + } /* Don't migrate if there is a better place to live */ if (!is_domain_overload(domain)) diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 375ed97a3aa78..9d324764315aa 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -3925,6 +3925,9 @@ extern unsigned int llc_epoch_period; extern unsigned int llc_epoch_affinity_timeout; extern unsigned int llc_imb_pct; extern unsigned int llc_overaggr_pct; +extern unsigned int llc_scan_period_min; +extern unsigned int llc_scan_period_max; +extern unsigned int llc_scan_period_threshold; extern unsigned int llc_override_numa_balance; static inline bool sched_cache_enabled(void) From e83bd8fc077d0ab1064368cb5072d79f4a4de1a3 Mon Sep 17 00:00:00 2001 From: Jianyong Wu Date: Wed, 25 Mar 2026 16:52:00 +0000 Subject: [PATCH 11/11] sched/fair: Introduce secondary best task mechanism Consider a scenario with two thread groups (one larger than the other) running on an unsuitable node. The load balancer currently migrates tasks randomly, which slows down thread group migration. A new preferred node is selected only after most tasks of a thread group have been migrated. A better policy is to prioritize tasks from the smaller thread group to accelerate group migration. This patch introduces a secondary best task mechanism. The best task comes from the thread group with the highest CPU utilization in the sched group; the secondary best task comes from the thread group with the next highest utilization. Signed-off-by: Jianyong Wu --- kernel/sched/fair.c | 98 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 95 insertions(+), 3 deletions(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 5e49424eadeec..e48853c87a5ce 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -9941,6 +9941,11 @@ struct lb_env { enum fbq_type fbq_type; enum migration_type migration_type; struct list_head tasks; + int local_idles; + int busiest_idles; +#ifdef CONFIG_SCHED_CACHE + int llc_imbalance; +#endif }; /* @@ -11732,8 +11737,10 @@ static inline void update_sg_lb_stats(struct lb_env *env, sgs->nr_preferred_running += rq->nr_preferred_running; } #endif - if (local_group) + if (local_group) { + env->local_idles = sgs->idle_cpus; continue; + } if (sd_flags & SD_ASYM_CPUCAPACITY) { /* Check for a misfit task on the cpu */ @@ -12398,6 +12405,14 @@ static inline void update_sd_lb_stats(struct lb_env *env, struct sd_lb_stats *sd update_sg_lb_stats(env, sds, sg, sgs, &sg_overloaded, &sg_overutilized); if (!local_group && update_sd_pick_busiest(env, sds, sg, sgs)) { +#ifdef CONFIG_SCHED_CACHE + if (!fits_llc_cap_imb(sgs->group_util, sgs->group_capacity) + && util_greater(sgs->group_util, local->group_util)) { + env->busiest_idles = sgs->idle_cpus; + env->llc_imbalance = 1; + } else + env->llc_imbalance = 0; +#endif sds->busiest = sg; sds->busiest_stat = *sgs; } @@ -12788,6 +12803,68 @@ static struct sched_group *sched_balance_find_src_group(struct lb_env *env) return NULL; } +#ifdef CONFIG_SCHED_CACHE + +/* + * Here, the best task refers to the thread group with the highest + * hit count in this sched group. + * The second best task is the one with the second highest hit count. + * + * This second best task mechanism is introduced to mitigate + * load imbalance caused by cache-aware scheduling. + */ +static struct task_struct * +find_second_best_task(struct lb_env *env, struct sched_group *group, int *second_best_score) +{ + struct task_struct *best_task = NULL, *second_best_task = NULL; + int best_task_score = 0, i; + int *pref_task = NULL, wt = 0; + + if (!sched_cache_enabled() || !group || !second_best_score) + return NULL; + + /* only allow NUMA domain to do this */ + if (!env->sd->child || env->sd->child->flags & SD_SHARE_LLC) + return NULL; + + wt = cpumask_weight(sched_group_span(group)); + pref_task = kmalloc_array(wt, sizeof(int), GFP_NOWAIT); + + if (!pref_task) + return NULL; + + memset(pref_task, 0, sizeof(int) * wt); + for_each_cpu_and(i, sched_group_span(group), env->cpus) { + struct rq *rq = cpu_rq(i); + struct task_struct *curr = rq->curr; + unsigned int tgid = 0, idx = 0; + + if (curr && curr->mm && curr->preferred_llc != -1) { + tgid = curr->tgid; + idx = tgid % wt; + if (cpumask_test_cpu(curr->mm->sc_stat.cpu, + sched_group_span(group))) { + pref_task[idx]++; + if (best_task_score < pref_task[idx]) { + if (!best_task) { + best_task = curr; + best_task_score = pref_task[idx]; + } else if (best_task->tgid != tgid) { + *second_best_score = best_task_score; + best_task_score = pref_task[idx]; + second_best_task = best_task; + best_task = curr; + } + } + } + } + } + + kfree(pref_task); + return second_best_task; +} +#endif + /* * sched_balance_find_src_rq - find the busiest runqueue among the CPUs in the group. */ @@ -12797,12 +12874,22 @@ static struct rq *sched_balance_find_src_rq(struct lb_env *env, struct rq *busiest = NULL, *rq; unsigned long busiest_util = 0, busiest_load = 0, busiest_capacity = 1; unsigned int busiest_nr = 0; + int i; #ifdef CONFIG_SCHED_CACHE unsigned int busiest_pref_llc = 0; + struct task_struct *second_best_task = NULL; struct sched_domain *sd_tmp; - int dst_llc; + int dst_llc, second_best_score = env->local_idles; + + if (sched_cache_enabled() && env->llc_imbalance) { + second_best_task = find_second_best_task(env, group, &second_best_score); + if (second_best_task && + env->migration_type == migrate_task && + (env->local_idles - env->busiest_idles) >> 1 > second_best_score && + env->local_idles - env->busiest_idles > get_nr_threads(second_best_task)) + env->migration_type = migrate_llc_task; + } #endif - int i; for_each_cpu_and(i, sched_group_span(group), env->cpus) { unsigned long capacity, load, util; @@ -12931,6 +13018,10 @@ static struct rq *sched_balance_find_src_rq(struct lb_env *env, case migrate_llc_task: #ifdef CONFIG_SCHED_CACHE + if (second_best_task && second_best_task->tgid == rq->curr->tgid) { + busiest = rq; + break; + } sd_tmp = rcu_dereference(rq->sd); dst_llc = llc_id(env->dst_cpu); if (valid_llc_buf(sd_tmp, dst_llc)) { @@ -13155,6 +13246,7 @@ static int sched_balance_rq(int this_cpu, struct rq *this_rq, .cpus = cpus, .fbq_type = all, .tasks = LIST_HEAD_INIT(env.tasks), + .llc_imbalance = 0, }; bool need_unlock = false;