diff --git a/include/linux/sched.h b/include/linux/sched.h index 4236cacbb409..505af39ed54f 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -2347,7 +2347,11 @@ struct sched_cache_stat { struct sched_cache_time __percpu *pcpu_time; raw_spinlock_t lock; unsigned long epoch; + unsigned long last_reset_tick; + unsigned long next_scan; + unsigned long scan_period; u64 nr_running_avg; + int need_scan; int cpu; } ____cacheline_aligned_in_smp; diff --git a/include/linux/topology.h b/include/linux/topology.h index 6575af39fd10..11232dd6a1a3 100644 --- a/include/linux/topology.h +++ b/include/linux/topology.h @@ -265,6 +265,10 @@ static inline const struct cpumask *cpu_node_mask(int cpu) return cpumask_of_node(cpu_to_node(cpu)); } +#define NR_LLCS NR_CPUS +int llc_to_node(int llc); +int llc_distance(int llc0, int llc1); + #ifdef CONFIG_NUMA int sched_numa_find_nth_cpu(const struct cpumask *cpus, int cpu, int node); extern const struct cpumask *sched_numa_hop_mask(unsigned int node, unsigned int hops); diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c index dc4b7de6569f..1cd532bfa9d9 100644 --- a/kernel/sched/debug.c +++ b/kernel/sched/debug.c @@ -566,6 +566,14 @@ static __init int sched_init_debug(void) #ifdef CONFIG_SCHED_CACHE debugfs_create_file("llc_enabled", 0644, debugfs_sched, NULL, &sched_cache_enable_fops); + debugfs_create_u32("llc_override_numa_balance", 0644, debugfs_sched, + &llc_override_numa_balance); + debugfs_create_u32("llc_scan_period_max", 0644, debugfs_sched, + &llc_scan_period_max); + debugfs_create_u32("llc_scan_period_min", 0644, debugfs_sched, + &llc_scan_period_min); + debugfs_create_u32("llc_scan_period_threshold", 0644, debugfs_sched, + &llc_scan_period_threshold); debugfs_create_u32("llc_aggr_tolerance", 0644, debugfs_sched, &llc_aggr_tolerance); debugfs_create_u32("llc_epoch_period", 0644, debugfs_sched, diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 977091fd0e49..e48853c87a5c 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -1192,10 +1192,14 @@ static void set_next_buddy(struct sched_entity *se); #define EPOCH_LLC_AFFINITY_TIMEOUT 5 /* 50 ms */ __read_mostly unsigned int llc_aggr_tolerance = 1; +__read_mostly unsigned int llc_override_numa_balance = 1; __read_mostly unsigned int llc_epoch_period = EPOCH_PERIOD; __read_mostly unsigned int llc_epoch_affinity_timeout = EPOCH_LLC_AFFINITY_TIMEOUT; __read_mostly unsigned int llc_imb_pct = 20; __read_mostly unsigned int llc_overaggr_pct = 50; +__read_mostly unsigned int llc_scan_period_min = 1; +__read_mostly unsigned int llc_scan_period_max = 64 * HZ; +__read_mostly unsigned int llc_scan_period_threshold = HZ; bool sched_cache_inuse(void) { @@ -1234,6 +1238,11 @@ static inline bool valid_llc_buf(struct sched_domain *sd, return valid_llc_id(id); } +static inline bool sched_cache_override_numa(void) +{ + return sched_cache_enabled() && llc_override_numa_balance; +} + static inline int get_sched_cache_scale(int mul) { if (!llc_aggr_tolerance) @@ -1402,6 +1411,7 @@ void mm_init_sched(struct mm_struct *mm, raw_spin_lock_init(&mm->sc_stat.lock); mm->sc_stat.epoch = epoch; mm->sc_stat.cpu = -1; + mm->sc_stat.scan_period = llc_scan_period_min; /* * The update to mm->sc_stat should not be reordered @@ -1481,9 +1491,10 @@ static int get_pref_llc(struct task_struct *p, struct mm_struct *mm) * than sched_setnuma() at least -- and thus the * conflict only exists for a short period of time. */ - if (static_branch_likely(&sched_numa_balancing) && - p->numa_preferred_nid >= 0 && - cpu_to_node(mm->sc_stat.cpu) != p->numa_preferred_nid) + if (!sched_cache_override_numa() && + static_branch_likely(&sched_numa_balancing) && + p->numa_preferred_nid >= 0 && + cpu_to_node(mm->sc_stat.cpu) != p->numa_preferred_nid) mm_sched_llc = -1; #endif } @@ -1527,13 +1538,8 @@ void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec) epoch = rq->cpu_epoch; } - /* - * If this process hasn't hit task_cache_work() for a while, or it - * has only 1 thread, invalidate its preferred state. - */ - if (time_after(epoch, - READ_ONCE(mm->sc_stat.epoch) + llc_epoch_affinity_timeout) || - get_nr_threads(p) <= 1 || + /* If it has only 1 thread, invalidate its preferred state. */ + if (get_nr_threads(p) <= 1 || exceed_llc_nr(mm, cpu_of(rq), p) || exceed_llc_capacity(mm, cpu_of(rq), p)) { if (mm->sc_stat.cpu != -1) @@ -1568,6 +1574,10 @@ static void task_tick_cache(struct rq *rq, struct task_struct *p) if (time_after_eq(mm->sc_stat.epoch, epoch)) return; + if (time_before(jiffies, mm->sc_stat.next_scan) && + !mm->sc_stat.need_scan) + return; + guard(raw_spinlock)(&mm->sc_stat.lock); if (work->next == work) { @@ -1599,8 +1609,10 @@ static void task_cache_work(struct callback_head *work) struct task_struct *p = current, *cur; struct mm_struct *mm = p->mm; unsigned long m_a_occ = 0; - unsigned long curr_m_a_occ = 0; - int cpu, m_a_cpu = -1, nr_running = 0, curr_cpu; + unsigned long m_a_n_occ = 0; + unsigned long curr_m_a_n_occ = 0; + unsigned long now; + int cpu, m_a_cpu = -1, m_a_n_cpu = -1, nr_running = 0, curr_cpu, need_scan = 0; cpumask_var_t cpus; WARN_ON_ONCE(work != &p->cache_work); @@ -1622,34 +1634,54 @@ static void task_cache_work(struct callback_head *work) if (!zalloc_cpumask_var(&cpus, GFP_KERNEL)) return; + now = jiffies; + if (time_before(now, READ_ONCE(mm->sc_stat.next_scan))) + return; + + WRITE_ONCE(mm->sc_stat.next_scan, (now + mm->sc_stat.scan_period)); + scoped_guard (cpus_read_lock) { cpumask_copy(cpus, cpu_online_mask); for_each_cpu(cpu, cpus) { - /* XXX sched_cluster_active */ - struct sched_domain *sd = per_cpu(sd_llc, cpu); - unsigned long occ, m_occ = 0, a_occ = 0; - int m_cpu = -1, i; + struct sched_domain *nsd = per_cpu(sd_node, cpu); + unsigned long occ, m_occ = 0, a_occ = 0, a_n_occ = 0; + int m_cpu = -1, i, k; - if (!sd) + if (!nsd) continue; - for_each_cpu(i, sched_domain_span(sd)) { - occ = fraction_mm_sched(cpu_rq(i), - per_cpu_ptr(mm->sc_stat.pcpu_sched, i)); - a_occ += occ; - if (occ > m_occ) { - m_occ = occ; - m_cpu = i; + for_each_cpu_and(k, sched_domain_span(nsd), cpus) { + a_occ = m_a_occ = m_occ = 0; + struct sched_domain *sd = per_cpu(sd_llc, k); + + if (!sd) + continue; + + for_each_cpu(i, sched_domain_span(sd)) { + occ = fraction_mm_sched(cpu_rq(i), + per_cpu_ptr(mm->sc_stat.pcpu_sched, i)); + a_occ += occ; + if (occ > m_occ) { + m_occ = occ; + m_cpu = i; + } + scoped_guard (rcu) { + cur = rcu_dereference(cpu_rq(i)->curr); + if (cur && !(cur->flags & (PF_EXITING | + PF_KTHREAD)) && cur->mm == mm) + nr_running++; + } } - scoped_guard (rcu) { - cur = rcu_dereference(cpu_rq(i)->curr); - if (cur && !(cur->flags & (PF_EXITING | PF_KTHREAD)) && - cur->mm == mm) - nr_running++; + + cpumask_andnot(cpus, cpus, sched_domain_span(sd)); + if (a_occ > m_a_occ) { + m_a_occ = a_occ; + m_a_cpu = m_cpu; } + /* record for numa node */ + a_n_occ += a_occ; } - /* * Compare the accumulated occupancy of each LLC. The * reason for using accumulated occupancy rather than average @@ -1665,19 +1697,19 @@ static void task_cache_work(struct callback_head *work) * the average number of faults per CPU. This strategy is also * followed here. */ - if (a_occ > m_a_occ) { - m_a_occ = a_occ; - m_a_cpu = m_cpu; + if (a_n_occ > m_a_n_occ) { + m_a_n_occ = a_n_occ; + m_a_n_cpu = m_a_cpu; } - if (llc_id(cpu) == llc_id(mm->sc_stat.cpu)) - curr_m_a_occ = a_occ; - - cpumask_andnot(cpus, cpus, sched_domain_span(sd)); + if (mm->sc_stat.cpu != -1 && cpu_to_node(cpu) == + cpu_to_node(mm->sc_stat.cpu)) + curr_m_a_n_occ = a_n_occ; } } - if (m_a_occ > (2 * curr_m_a_occ)) { + need_scan = READ_ONCE(mm->sc_stat.need_scan); + if (m_a_n_occ > (2 * curr_m_a_n_occ) || need_scan) { /* * Avoid switching sc_stat.cpu too fast. * The reason to choose 2X is because: @@ -1688,9 +1720,35 @@ static void task_cache_work(struct callback_head *work) * 3. 2X is chosen based on test results, as it delivers * the optimal performance gain so far. */ - mm->sc_stat.cpu = m_a_cpu; + if (m_a_n_occ > (2 * curr_m_a_n_occ)) + mm->sc_stat.cpu = m_a_n_cpu; + + if (!mm->sc_stat.last_reset_tick) + mm->sc_stat.last_reset_tick = now; + + /* Change scan_period when preferred NUMA changed */ + if (((mm->sc_stat.cpu != -1) && (m_a_n_cpu != -1) + && (cpu_to_node(mm->sc_stat.cpu) != cpu_to_node(m_a_n_cpu))) + || need_scan) { + if (!need_scan) + need_scan = 1; + + WRITE_ONCE(mm->sc_stat.scan_period, + max(mm->sc_stat.scan_period >> 1, llc_scan_period_min)); + WRITE_ONCE(mm->sc_stat.last_reset_tick, now); + } + } + + if ((now - READ_ONCE(mm->sc_stat.last_reset_tick) > llc_scan_period_threshold) + && !need_scan) { + WRITE_ONCE(mm->sc_stat.scan_period, min(mm->sc_stat.scan_period << 1, + llc_scan_period_max)); + WRITE_ONCE(mm->sc_stat.last_reset_tick, now); } + if (READ_ONCE(mm->sc_stat.need_scan)) + WRITE_ONCE(mm->sc_stat.need_scan, 0); + update_avg_scale(&mm->sc_stat.nr_running_avg, nr_running); free_cpumask_var(cpus); } @@ -1749,6 +1807,7 @@ static void account_llc_enqueue(struct rq *rq, struct task_struct *p) {} static void account_llc_dequeue(struct rq *rq, struct task_struct *p) {} +static inline bool sched_cache_override_numa(void) {} #endif /* @@ -3715,6 +3774,9 @@ void task_numa_fault(int last_cpupid, int mem_node, int pages, int flags) if (!static_branch_likely(&sched_numa_balancing)) return; + if (sched_cache_override_numa()) + return; + /* for example, ksmd faulting in a user's mm */ if (!p->mm) return; @@ -9879,6 +9941,11 @@ struct lb_env { enum fbq_type fbq_type; enum migration_type migration_type; struct list_head tasks; + int local_idles; + int busiest_idles; +#ifdef CONFIG_SCHED_CACHE + int llc_imbalance; +#endif }; /* @@ -10021,6 +10088,13 @@ static inline int task_is_ineligible_on_dst_cpu(struct task_struct *p, int dest_ #define fits_llc_capacity(util, max) \ ((util) * 100 < (max) * llc_overaggr_pct) +/* + * Like fits_llc_capacity but consider bias. + * The bias here is the half of llc_imb_pct. + */ +#define fits_llc_cap_imb(util, max) \ + ((util) * 100 < (max) * (llc_overaggr_pct + llc_imb_pct / 2)) + /* * The margin used when comparing utilization. * is 'util1' noticeably greater than 'util2' @@ -10047,6 +10121,7 @@ static __maybe_unused bool get_llc_stats(int cpu, unsigned long *util, return true; } + /* * Decision matrix according to the LLC utilization. To * decide whether we can do task aggregation across LLC. @@ -10148,6 +10223,141 @@ static enum llc_mig can_migrate_llc(int src_cpu, int dst_cpu, return mig_llc; } +/* + * Like get_llc_stats but for sched domain that above LLC level. + * Based on get_llc_stats, we can accumulate utility and cap for + * sched domain in the granularity of LLC. + */ +static bool get_sd_stats(struct sched_domain *sd, unsigned long *util_out, unsigned long *cap_out) +{ + struct cpumask mask; + int cpu; + unsigned long util_tmp, cap_tmp, util = 0, cap = 0; + struct sched_domain *sd_tmp; + + if (!sd || !util_out || !cap_out) + return false; + + cpumask_copy(&mask, sched_domain_span(sd)); + for_each_cpu(cpu, &mask) { + if (!get_llc_stats(cpu, &util_tmp, &cap_tmp)) + return false; + + sd_tmp = rcu_dereference(per_cpu(sd_llc, cpu)); + cpumask_andnot(&mask, &mask, sched_domain_span(sd_tmp)); + util += util_tmp; + cap += cap_tmp; + } + + *util_out = util; + *cap_out = cap; + + return true; +} + +/* Decide if a sched domain is overload. */ +static bool is_domain_overload(struct sched_domain *sd) +{ + int ret; + unsigned long util = 0, cap = 0; + + get_sd_stats(sd, &util, &cap); + + /* We are not a llc. Need change name? */ + ret = !fits_llc_capacity(util, cap); + + return ret; +} + +/* + * Decide if migration should happen on a specific node. + * The node here is a generic conception for a set of cpu. + * It Usually indecates one of sched domain for LLC level and above. + */ +static enum llc_mig can_migrate_node(int src_cpu, int dst_cpu, struct task_struct *p, bool to_pref) +{ + struct mm_struct *mm = NULL; + struct sched_domain *domain; + unsigned long dst_util, dst_cap, tsk_util = 0; + int k = 0; + + if (!get_llc_stats(dst_cpu, &dst_util, &dst_cap)) + return mig_unrestricted; + + if (p) { + mm = p->mm; + tsk_util = task_util(p); + } + + dst_util = dst_util + tsk_util; + + if (to_pref) { + if (fits_llc_capacity(dst_util, dst_cap)) + return mig_llc; + else { + if (mm && !fits_llc_cap_imb(dst_util, dst_cap)) + mm->sc_stat.need_scan = 1; + + return mig_unrestricted; + } + } + + /* + * If the dest node decrase locality, decide if it should migrate by testing that + * if it is the closest place that is not overload. + */ + for_each_domain(src_cpu, domain) { + /* Skip sched domain lower than MC */ + if (domain->flags & SD_SHARE_LLC) + continue; + + /* Allow migration if we found dest cpu in this sched domain */ + if (cpumask_test_cpu(dst_cpu, sched_domain_span(domain))) + return mig_llc; + + /* + * For the special case: the workload is small and the dest cpu may far away + * from src cpu. + */ + if (p && (domain->span_weight > get_nr_threads(p) && k++)) { + if (mm && !fits_llc_cap_imb(dst_util, dst_cap)) + mm->sc_stat.need_scan = 1; + + return mig_unrestricted; + } + + /* Don't migrate if there is a better place to live */ + if (!is_domain_overload(domain)) + return mig_forbid; + } + + return mig_unrestricted; +} + +int llc_distance(int, int); + +/* Decide if the migration improve the affinity */ +static bool if_to_prefer(int src_cpu, int dst_cpu, int pref_llc) +{ + int src_dist, dst_dist, to_pref = false; + + src_dist = llc_distance(llc_id(src_cpu), pref_llc); + dst_dist = llc_distance(llc_id(dst_cpu), pref_llc); + + if (src_dist > dst_dist) { + to_pref = true; + } else if (src_dist == dst_dist) { + if (llc_id(dst_cpu) == pref_llc) + to_pref = true; + else + to_pref = false; + } else { + to_pref = false; + } + + return to_pref; +} + /* * Check if task p can migrate from source LLC to * destination LLC in terms of cache aware load balance. @@ -10178,15 +10388,10 @@ static enum llc_mig can_migrate_llc_task(int src_cpu, int dst_cpu, return mig_unrestricted; } - if (cpus_share_cache(dst_cpu, cpu)) - to_pref = true; - else if (cpus_share_cache(src_cpu, cpu)) - to_pref = false; - else - return mig_unrestricted; + to_pref = if_to_prefer(src_cpu, dst_cpu, llc_id(cpu)); - return can_migrate_llc(src_cpu, dst_cpu, - task_util(p), to_pref); + return can_migrate_node(src_cpu, dst_cpu, + p, to_pref); } /* @@ -10196,36 +10401,36 @@ static enum llc_mig can_migrate_llc_task(int src_cpu, int dst_cpu, static inline bool alb_break_llc(struct lb_env *env) { + int pref_llc = -1; + bool to_pref = false; + if (!sched_cache_enabled()) return false; if (cpus_share_cache(env->src_cpu, env->dst_cpu)) return false; /* - * All tasks prefer to stay on their current CPU. - * Do not pull a task from its preferred CPU if: - * 1. It is the only task running there; OR - * 2. Migrating it away from its preferred LLC would violate - * the cache-aware scheduling policy. + * We need the preferred LLC to decide whether we can perform migration. + * Therefore, we need to obtain task_struct, which is only available + * when there is a task running. + * For cases with more than one task on the rq, we need to check + * this in can_migrate_task(). */ - if (env->src_rq->nr_pref_llc_running && - env->src_rq->nr_pref_llc_running == env->src_rq->cfs.h_nr_runnable) { - unsigned long util = 0; + if (env->src_rq->nr_running == 1) { struct task_struct *cur; - if (env->src_rq->nr_running <= 1) - return true; - /* * Reach here in load balance with * rcu_read_lock() protected. */ cur = rcu_dereference(env->src_rq->curr); if (cur) - util = task_util(cur); + pref_llc = cur->preferred_llc; - if (can_migrate_llc(env->src_cpu, env->dst_cpu, - util, false) == mig_forbid) + to_pref = if_to_prefer(env->src_cpu, env->dst_cpu, pref_llc); + + if (can_migrate_node(env->src_cpu, env->dst_cpu, + cur, to_pref) == mig_forbid) return true; } @@ -10256,15 +10461,6 @@ static bool migrate_degrades_llc(struct task_struct *p, struct lb_env *env) if (env->sd->nr_balance_failed >= env->sd->cache_nice_tries + 1) return false; - /* - * We know the env->src_cpu has some tasks prefer to - * run on env->dst_cpu, skip the tasks do not prefer - * env->dst_cpu, and find the one that prefers. - */ - if (env->migration_type == migrate_llc_task && - task_llc(p) != llc_id(env->dst_cpu)) - return true; - if (can_migrate_llc_task(env->src_cpu, env->dst_cpu, p) != mig_forbid) return false; @@ -10383,15 +10579,14 @@ int can_migrate_task(struct task_struct *p, struct lb_env *env) * 3) task is cache cold, or * 4) too many balance attempts have failed. */ - if (env->flags & LBF_ACTIVE_LB) - return 1; - degrades = migrate_degrades_locality(p, env); if (!degrades) { /* * If the NUMA locality is not broken, * further check if migration would hurt * LLC locality. + * This should be done before check LBF_ACTIVE_LB + * as we has not check some cases in alb_break_llc */ if (migrate_degrades_llc(p, env)) return 0; @@ -10401,6 +10596,9 @@ int can_migrate_task(struct task_struct *p, struct lb_env *env) hot = degrades > 0; } + if (env->flags & LBF_ACTIVE_LB) + return 1; + if (!hot || env->sd->nr_balance_failed > env->sd->cache_nice_tries) { if (hot) p->sched_task_hot = 1; @@ -11356,8 +11554,8 @@ static inline bool llc_balance(struct lb_env *env, struct sg_lb_stats *sgs, return false; if (sgs->nr_pref_dst_llc && - can_migrate_llc(cpumask_first(sched_group_span(group)), - env->dst_cpu, 0, true) == mig_llc) + can_migrate_node(cpumask_first(sched_group_span(group)), + env->dst_cpu, NULL, true) == mig_llc) return true; return false; @@ -11372,6 +11570,83 @@ static bool update_llc_busiest(struct lb_env *env, */ return sgs->nr_pref_dst_llc > busiest->nr_pref_dst_llc; } + +/* + * Get all LLCs that are closer to the destination LLC than to the + * source LLC. + * @affi_llcs: array to store LLCs satisfying the above condition + * @dist: array to store Di for each LLC in affi_llcs, computed as: + * + * Di = llc_distance(src_llc, LLCi) - llc_distance(dst_llc, LLCi) (1) + * where i is the index of affi_llcs. + */ +static int get_affi_llcs(int src_llc, int dst_llc, int *affi_llcs, int *dist) +{ + int j = 0, dis1, dis2; + + if (src_llc == dst_llc) + return 0; + + if (llc_to_node(src_llc) == llc_to_node(dst_llc)) { + affi_llcs[j] = dst_llc; + dist[j++] = 2; + return j; + } + + for (int i = 0; i < max_llcs; i++) { + dis1 = llc_distance(src_llc, i); + dis2 = llc_distance(dst_llc, i); + if (dis1 < 0 || dis2 < 0) + continue; + if (dis1 > dis2) { + dist[j] = clamp(dis1 - dis2, 4, 1024); + affi_llcs[j++] = i; + } + } + + return j; +} + +/* + * To find a src sched group/rq during load balancing, we need a method to + * calculate the benefit of each rq. For sched cache, we focus more on + * affinity improvement. + * + * This provides a way to quantify the affinity improvement for each rq + * by assigning an affinity score to each rq. + * + * Calculate the affinity score for a rq given src llc and dst llc. + * It is computed as: + * Di = llc_distance(src_llc, LLCi) - llc_distance(dst_llc, LLCi) (1) + * W_i = Rt_i * 1024 / Di (2) + * p = Σ_i W_i (3) + * + * where i is the index of an LLC, Di is obtained from get_affi_llcs, and + * Rt_i is the number of tasks on the rq with LLCi as their preferred LLC, + * obtainable from rq->sd->pf. + */ +static int cal_affinity_score(struct rq *rq, int src_cpu, int dst_llc) +{ + int *affi_llcs, *dist, num, wt = 0; + struct sched_domain *sd_tmp = rcu_dereference(rq->sd); + + affi_llcs = kmalloc_array(max_llcs, sizeof(*affi_llcs), GFP_NOWAIT); + if (!affi_llcs) + return wt; + + dist = kmalloc_array(max_llcs, sizeof(*dist), GFP_NOWAIT); + if (!dist) + goto fail; + + num = get_affi_llcs(llc_id(src_cpu), dst_llc, affi_llcs, dist); + for (int i = 0; i < num; i++) + wt += (sd_tmp->pf[affi_llcs[i]] << 10) / dist[i]; + + kfree(dist); +fail: + kfree(affi_llcs); + return wt; +} #else static inline void record_sg_llc_stats(struct lb_env *env, struct sg_lb_stats *sgs, struct sched_group *group) @@ -11438,7 +11713,7 @@ static inline void update_sg_lb_stats(struct lb_env *env, struct sched_domain *sd_tmp = rcu_dereference(rq->sd); if (valid_llc_buf(sd_tmp, dst_llc)) - sgs->nr_pref_dst_llc += sd_tmp->pf[dst_llc]; + sgs->nr_pref_dst_llc += cal_affinity_score(rq, i, dst_llc); } #endif @@ -11462,8 +11737,10 @@ static inline void update_sg_lb_stats(struct lb_env *env, sgs->nr_preferred_running += rq->nr_preferred_running; } #endif - if (local_group) + if (local_group) { + env->local_idles = sgs->idle_cpus; continue; + } if (sd_flags & SD_ASYM_CPUCAPACITY) { /* Check for a misfit task on the cpu */ @@ -12128,6 +12405,14 @@ static inline void update_sd_lb_stats(struct lb_env *env, struct sd_lb_stats *sd update_sg_lb_stats(env, sds, sg, sgs, &sg_overloaded, &sg_overutilized); if (!local_group && update_sd_pick_busiest(env, sds, sg, sgs)) { +#ifdef CONFIG_SCHED_CACHE + if (!fits_llc_cap_imb(sgs->group_util, sgs->group_capacity) + && util_greater(sgs->group_util, local->group_util)) { + env->busiest_idles = sgs->idle_cpus; + env->llc_imbalance = 1; + } else + env->llc_imbalance = 0; +#endif sds->busiest = sg; sds->busiest_stat = *sgs; } @@ -12465,9 +12750,9 @@ static struct sched_group *sched_balance_find_src_group(struct lb_env *env) * Try to move all excess tasks to a sibling domain of the busiest * group's child domain. */ - if (sds.prefer_sibling && local->group_type == group_has_spare && - (busiest->group_type == group_llc_balance || - sibling_imbalance(env, &sds, busiest, local) > 1)) + if (local->group_type == group_has_spare && + ((busiest->group_type == group_llc_balance) || (sds.prefer_sibling && + sibling_imbalance(env, &sds, busiest, local) > 1))) goto force_balance; if (busiest->group_type != group_overloaded) { @@ -12518,6 +12803,68 @@ static struct sched_group *sched_balance_find_src_group(struct lb_env *env) return NULL; } +#ifdef CONFIG_SCHED_CACHE + +/* + * Here, the best task refers to the thread group with the highest + * hit count in this sched group. + * The second best task is the one with the second highest hit count. + * + * This second best task mechanism is introduced to mitigate + * load imbalance caused by cache-aware scheduling. + */ +static struct task_struct * +find_second_best_task(struct lb_env *env, struct sched_group *group, int *second_best_score) +{ + struct task_struct *best_task = NULL, *second_best_task = NULL; + int best_task_score = 0, i; + int *pref_task = NULL, wt = 0; + + if (!sched_cache_enabled() || !group || !second_best_score) + return NULL; + + /* only allow NUMA domain to do this */ + if (!env->sd->child || env->sd->child->flags & SD_SHARE_LLC) + return NULL; + + wt = cpumask_weight(sched_group_span(group)); + pref_task = kmalloc_array(wt, sizeof(int), GFP_NOWAIT); + + if (!pref_task) + return NULL; + + memset(pref_task, 0, sizeof(int) * wt); + for_each_cpu_and(i, sched_group_span(group), env->cpus) { + struct rq *rq = cpu_rq(i); + struct task_struct *curr = rq->curr; + unsigned int tgid = 0, idx = 0; + + if (curr && curr->mm && curr->preferred_llc != -1) { + tgid = curr->tgid; + idx = tgid % wt; + if (cpumask_test_cpu(curr->mm->sc_stat.cpu, + sched_group_span(group))) { + pref_task[idx]++; + if (best_task_score < pref_task[idx]) { + if (!best_task) { + best_task = curr; + best_task_score = pref_task[idx]; + } else if (best_task->tgid != tgid) { + *second_best_score = best_task_score; + best_task_score = pref_task[idx]; + second_best_task = best_task; + best_task = curr; + } + } + } + } + } + + kfree(pref_task); + return second_best_task; +} +#endif + /* * sched_balance_find_src_rq - find the busiest runqueue among the CPUs in the group. */ @@ -12527,12 +12874,22 @@ static struct rq *sched_balance_find_src_rq(struct lb_env *env, struct rq *busiest = NULL, *rq; unsigned long busiest_util = 0, busiest_load = 0, busiest_capacity = 1; unsigned int busiest_nr = 0; + int i; #ifdef CONFIG_SCHED_CACHE unsigned int busiest_pref_llc = 0; + struct task_struct *second_best_task = NULL; struct sched_domain *sd_tmp; - int dst_llc; + int dst_llc, second_best_score = env->local_idles; + + if (sched_cache_enabled() && env->llc_imbalance) { + second_best_task = find_second_best_task(env, group, &second_best_score); + if (second_best_task && + env->migration_type == migrate_task && + (env->local_idles - env->busiest_idles) >> 1 > second_best_score && + env->local_idles - env->busiest_idles > get_nr_threads(second_best_task)) + env->migration_type = migrate_llc_task; + } #endif - int i; for_each_cpu_and(i, sched_group_span(group), env->cpus) { unsigned long capacity, load, util; @@ -12661,10 +13018,14 @@ static struct rq *sched_balance_find_src_rq(struct lb_env *env, case migrate_llc_task: #ifdef CONFIG_SCHED_CACHE + if (second_best_task && second_best_task->tgid == rq->curr->tgid) { + busiest = rq; + break; + } sd_tmp = rcu_dereference(rq->sd); dst_llc = llc_id(env->dst_cpu); if (valid_llc_buf(sd_tmp, dst_llc)) { - unsigned int this_pref_llc = sd_tmp->pf[dst_llc]; + unsigned int this_pref_llc = cal_affinity_score(rq, i, dst_llc); if (busiest_pref_llc < this_pref_llc) { busiest_pref_llc = this_pref_llc; @@ -12885,6 +13246,7 @@ static int sched_balance_rq(int this_cpu, struct rq *this_rq, .cpus = cpus, .fbq_type = all, .tasks = LIST_HEAD_INIT(env.tasks), + .llc_imbalance = 0, }; bool need_unlock = false; diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index f4785f84b1f1..9d324764315a 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2104,6 +2104,7 @@ DECLARE_PER_CPU(int, sd_llc_size); DECLARE_PER_CPU(int, sd_llc_id); DECLARE_PER_CPU(int, sd_share_id); DECLARE_PER_CPU(struct sched_domain_shared __rcu *, sd_llc_shared); +DECLARE_PER_CPU(struct sched_domain __rcu *, sd_node); DECLARE_PER_CPU(struct sched_domain __rcu *, sd_numa); DECLARE_PER_CPU(struct sched_domain __rcu *, sd_asym_packing); DECLARE_PER_CPU(struct sched_domain __rcu *, sd_asym_cpucapacity); @@ -3924,6 +3925,10 @@ extern unsigned int llc_epoch_period; extern unsigned int llc_epoch_affinity_timeout; extern unsigned int llc_imb_pct; extern unsigned int llc_overaggr_pct; +extern unsigned int llc_scan_period_min; +extern unsigned int llc_scan_period_max; +extern unsigned int llc_scan_period_threshold; +extern unsigned int llc_override_numa_balance; static inline bool sched_cache_enabled(void) { diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index e86dea1b9e86..99037ca8430e 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -668,6 +668,7 @@ DEFINE_PER_CPU(int, sd_llc_size); DEFINE_PER_CPU(int, sd_llc_id) = -1; DEFINE_PER_CPU(int, sd_share_id); DEFINE_PER_CPU(struct sched_domain_shared __rcu *, sd_llc_shared); +DEFINE_PER_CPU(struct sched_domain __rcu *, sd_node); DEFINE_PER_CPU(struct sched_domain __rcu *, sd_numa); DEFINE_PER_CPU(struct sched_domain __rcu *, sd_asym_packing); DEFINE_PER_CPU(struct sched_domain __rcu *, sd_asym_cpucapacity); @@ -675,6 +676,21 @@ DEFINE_PER_CPU(struct sched_domain __rcu *, sd_asym_cpucapacity); DEFINE_STATIC_KEY_FALSE(sched_asym_cpucapacity); DEFINE_STATIC_KEY_FALSE(sched_cluster_active); +static int llc_to_node_map[NR_LLCS] = { [0 ... NR_LLCS-1] = -1 }; + +/* Get sched domain by its name */ +static struct sched_domain *get_sched_domain_by_name(int cpu, char *name) +{ + struct sched_domain *sd = NULL; + + for_each_domain(cpu, sd) { + if (!strcmp(sd->name, name)) + break; + } + + return sd; +} + static void update_top_cache_domain(int cpu) { struct sched_domain_shared *sds = NULL; @@ -704,6 +720,9 @@ static void update_top_cache_domain(int cpu) */ per_cpu(sd_share_id, cpu) = id; + sd = get_sched_domain_by_name(cpu, "NODE"); + rcu_assign_pointer(per_cpu(sd_node, cpu), sd); + sd = lowest_flag_domain(cpu, SD_NUMA); rcu_assign_pointer(per_cpu(sd_numa, cpu), sd); @@ -811,6 +830,31 @@ DEFINE_STATIC_KEY_FALSE(sched_cache_active); /* user wants cache aware scheduling [0 or 1] */ int sysctl_sched_cache_user = 1; +/* Return the NUMA node containing the llc */ +int llc_to_node(int llc) +{ + if (llc < 0) + return -1; + + if (llc >= NR_LLCS) + return -1; + + return llc_to_node_map[llc]; +} + +/* Return the NUMA distance between the node containing LLC1 and the node containing LLC2 */ +int llc_distance(int llc1, int llc2) +{ + int numa1, numa2; + + numa1 = llc_to_node(llc1); + numa2 = llc_to_node(llc2); + if (numa1 < 0 || numa2 < 0) + return -1; + + return node_distance(numa1, numa2); +} + static bool alloc_sd_pref(const struct cpumask *cpu_map, struct s_data *d) { @@ -902,6 +946,9 @@ static bool alloc_sd_pref(const struct cpumask *cpu_map, { return false; } + +int llc_to_node(int llc) { return -1; } +int llc_distance(int llc1, int llc2) { return -1; } #endif /* @@ -2709,6 +2756,7 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att } lid = per_cpu(sd_llc_id, i); + llc_to_node_map[tl_max_llcs] = cpu_to_node(i); if (lid == -1) { int j;