diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 0507193b3ae3..158c1fba2393 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -323,6 +323,8 @@ enum node_stat_item { PGSCAN_PROACTIVE, PGSCAN_ANON, PGSCAN_FILE, + PGROTATE_ANON, + PGROTATE_FILE, PGREFILL, #ifdef CONFIG_HUGETLB_PAGE NR_HUGETLB, @@ -755,6 +757,12 @@ void lru_gen_reparent_memcg(struct mem_cgroup *memcg, struct mem_cgroup *parent, #endif /* CONFIG_LRU_GEN */ +struct lru_cost { + unsigned long count; + unsigned long last_rotated; + unsigned long last_io; +}; + struct lruvec { struct list_head lists[NR_LRU_LISTS]; /* per lruvec lru_lock for memcg */ @@ -763,9 +771,12 @@ struct lruvec { * These track the cost of reclaiming one LRU - file or anon - * over the other. As the observed cost of reclaiming one LRU * increases, the reclaim scan balance tips toward the other. + * Updated and decayed at prepare_scan_control() time; cost_lock + * serialises that update. */ - unsigned long anon_cost; - unsigned long file_cost; + struct lru_cost cost[ANON_AND_FILE]; + /* Protects cost[]. */ + spinlock_t cost_lock; /* Non-resident age, driven by LRU movement */ atomic_long_t nonresident_age; /* Refaults at the time of last reclaim cycle */ diff --git a/include/linux/vmstat.h b/include/linux/vmstat.h index fb8c76289e02..5b31d8e7ae40 100644 --- a/include/linux/vmstat.h +++ b/include/linux/vmstat.h @@ -20,7 +20,6 @@ struct reclaim_stat { unsigned nr_congested; unsigned nr_writeback; unsigned nr_immediate; - unsigned nr_pageout; unsigned nr_activate[ANON_AND_FILE]; unsigned nr_ref_keep; unsigned nr_unmap_fail; diff --git a/mm/folio.c b/mm/folio.c index d2937600cf72..a9e328c3f21b 100644 --- a/mm/folio.c +++ b/mm/folio.c @@ -265,73 +265,6 @@ void folio_rotate_reclaimable(struct folio *folio) folio_batch_add_and_move(folio, lru_move_tail); } -void lru_note_cost_unlock_irq(struct lruvec *lruvec, bool file, - unsigned int nr_io, unsigned int nr_rotated) - __releases(lruvec->lru_lock) - __releases(rcu) -{ - unsigned long cost; - - /* - * Reflect the relative cost of incurring IO and spending CPU - * time on rotations. This doesn't attempt to make a precise - * comparison, it just says: if reloads are about comparable - * between the LRU lists, or rotations are overwhelmingly - * different between them, adjust scan balance for CPU work. - */ - cost = nr_io * SWAP_CLUSTER_MAX + nr_rotated; - if (!cost) { - spin_unlock_irq(&lruvec->lru_lock); - rcu_read_unlock(); - return; - } - - for (;;) { - unsigned long lrusize; - - /* Record cost event */ - if (file) - lruvec->file_cost += cost; - else - lruvec->anon_cost += cost; - - /* - * Decay previous events - * - * Because workloads change over time (and to avoid - * overflow) we keep these statistics as a floating - * average, which ends up weighing recent refaults - * more than old ones. - */ - lrusize = lruvec_page_state(lruvec, NR_INACTIVE_ANON) + - lruvec_page_state(lruvec, NR_ACTIVE_ANON) + - lruvec_page_state(lruvec, NR_INACTIVE_FILE) + - lruvec_page_state(lruvec, NR_ACTIVE_FILE); - - if (lruvec->file_cost + lruvec->anon_cost > lrusize / 4) { - lruvec->file_cost /= 2; - lruvec->anon_cost /= 2; - } - - spin_unlock_irq(&lruvec->lru_lock); - lruvec = parent_lruvec(lruvec); - if (!lruvec) { - rcu_read_unlock(); - break; - } - spin_lock_irq(&lruvec->lru_lock); - } -} - -void lru_note_cost_refault(struct folio *folio) -{ - struct lruvec *lruvec; - - lruvec = folio_lruvec_lock_irq(folio); - lru_note_cost_unlock_irq(lruvec, folio_is_file_lru(folio), - folio_nr_pages(folio), 0); -} - static void lru_activate(struct lruvec *lruvec, struct folio *folio) { long nr_pages = folio_nr_pages(folio); @@ -1162,8 +1095,6 @@ void lru_reparent_memcg(struct mem_cgroup *memcg, struct mem_cgroup *parent, int child_lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid)); parent_lruvec = mem_cgroup_lruvec(parent, NODE_DATA(nid)); - parent_lruvec->anon_cost += child_lruvec->anon_cost; - parent_lruvec->file_cost += child_lruvec->file_cost; for_each_lru(lru) lruvec_reparent_lru(child_lruvec, parent_lruvec, lru, nid); diff --git a/mm/internal.h b/mm/internal.h index 5758dcaf4392..f47f06c55548 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -41,9 +41,6 @@ void workingset_refault(struct folio *folio, void *shadow); void workingset_activation(struct folio *folio); /* mm/folio.c */ -void lru_note_cost_unlock_irq(struct lruvec *lruvec, bool file, - unsigned int nr_io, unsigned int nr_rotated); -void lru_note_cost_refault(struct folio *folio); void folio_add_lru_vma(struct folio *folio, struct vm_area_struct *vma); static inline bool folio_may_be_lru_cached(struct folio *folio) diff --git a/mm/memcontrol-v1.c b/mm/memcontrol-v1.c index 2dc599484d00..835fc8e51184 100644 --- a/mm/memcontrol-v1.c +++ b/mm/memcontrol-v1.c @@ -2287,8 +2287,8 @@ void memcg1_stat_format(struct mem_cgroup *memcg, struct seq_buf *s) for_each_online_pgdat(pgdat) { mz = memcg->nodeinfo[pgdat->node_id]; - anon_cost += mz->lruvec.anon_cost; - file_cost += mz->lruvec.file_cost; + anon_cost += mz->lruvec.cost[WORKINGSET_ANON].count; + file_cost += mz->lruvec.cost[WORKINGSET_FILE].count; } seq_buf_printf(s, "anon_cost %lu\n", anon_cost); seq_buf_printf(s, "file_cost %lu\n", file_cost); diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 8472e37cbaf8..6b3d3e14a2f5 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -399,6 +399,7 @@ static const unsigned int memcg_node_stat_items[] = { NR_SHMEM_THPS, NR_FILE_THPS, NR_ANON_THPS, + NR_VMSCAN_WRITE, NR_VMALLOC, NR_KERNEL_STACK_KB, NR_PAGETABLE, @@ -425,6 +426,8 @@ static const unsigned int memcg_node_stat_items[] = { PGSCAN_PROACTIVE, PGSCAN_ANON, PGSCAN_FILE, + PGROTATE_ANON, + PGROTATE_FILE, PGREFILL, #ifdef CONFIG_HUGETLB_PAGE NR_HUGETLB, @@ -524,7 +527,7 @@ unsigned long lruvec_page_state(struct lruvec *lruvec, enum node_stat_item idx) * reading from per-CPU delta skew must present as zero. * * XXX: This helper (and its node/global peers) exists because we place - * monotonically-incremented event counters (PGROTATE_*, PGRECLAIM_PAGEOUT_*) + * monotonically-incremented event counters (NR_VMSCAN_WRITE and PGROTATE_*) * into enum node_stat_item. */ unsigned long lruvec_page_state_monotonic(struct lruvec *lruvec, diff --git a/mm/mmzone.c b/mm/mmzone.c index 59dc3f2076a6..9cc9ef588580 100644 --- a/mm/mmzone.c +++ b/mm/mmzone.c @@ -79,6 +79,7 @@ void lruvec_init(struct lruvec *lruvec) memset(lruvec, 0, sizeof(struct lruvec)); spin_lock_init(&lruvec->lru_lock); + spin_lock_init(&lruvec->cost_lock); zswap_lruvec_state_init(lruvec); for_each_lru(lru) diff --git a/mm/vmscan.c b/mm/vmscan.c index c3d9ed6df730..17d2b793cbfc 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -670,7 +670,7 @@ static pageout_t pageout(struct swap_io_ctx *ctx, struct address_space *mapping, folio_clear_reclaim(folio); trace_mm_vmscan_write_folio(folio); - node_stat_add_folio(folio, NR_VMSCAN_WRITE); + lruvec_stat_mod_folio(folio, NR_VMSCAN_WRITE, folio_nr_pages(folio)); return PAGE_SUCCESS; } @@ -1412,8 +1412,6 @@ retry: sc->nr_scanned -= (nr_pages - 1); nr_pages = 1; } - stat->nr_pageout += nr_pages; - if (folio_test_writeback(folio)) goto keep; if (folio_test_dirty(folio)) @@ -2036,10 +2034,10 @@ static unsigned long shrink_inactive_list(unsigned long nr_to_scan, item = PGSTEAL_KSWAPD + reclaimer_offset(sc); mod_lruvec_state(lruvec, item, nr_reclaimed); mod_lruvec_state(lruvec, PGSTEAL_ANON + file, nr_reclaimed); + if (nr_scanned > nr_reclaimed) + mod_lruvec_state(lruvec, PGROTATE_ANON + file, + nr_scanned - nr_reclaimed); - lruvec_lock_irq(lruvec); - lru_note_cost_unlock_irq(lruvec, file, stat.nr_pageout, - nr_scanned - nr_reclaimed); handle_reclaim_writeback(nr_taken, pgdat, sc, &stat); trace_mm_vmscan_lru_shrink_inactive(pgdat->node_id, nr_scanned, nr_reclaimed, &stat, sc->priority, file); @@ -2145,9 +2143,9 @@ static void shrink_active_list(unsigned long nr_to_scan, count_vm_events(PGDEACTIVATE, nr_deactivate); count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE, nr_deactivate); mod_node_page_state(pgdat, NR_ISOLATED_ANON + file, -nr_taken); + if (nr_rotated) + mod_lruvec_state(lruvec, PGROTATE_ANON + file, nr_rotated); - lruvec_lock_irq(lruvec); - lru_note_cost_unlock_irq(lruvec, file, 0, nr_rotated); trace_mm_vmscan_lru_shrink_active(pgdat->node_id, nr_taken, nr_activate, nr_deactivate, nr_rotated, sc->priority, file); } @@ -2280,8 +2278,10 @@ enum scan_balance { static void prepare_scan_control(pg_data_t *pgdat, struct scan_control *sc) { - unsigned long file; + struct lru_cost *anon_cost, *file_cost; struct lruvec *target_lruvec; + unsigned long lrusize; + unsigned long file; if (lru_gen_enabled() && !lru_gen_switching()) return; @@ -2297,11 +2297,69 @@ static void prepare_scan_control(pg_data_t *pgdat, struct scan_control *sc) /* * Determine the scan balance between anon and file LRUs. + * + * The cost model is based on rotations, refaults and + * reclaim-driven writes (anon only) on each side. + * + * These event counters are monotonic, so each reclaim cycle + * the delta since the last scan is extracted and incorporated + * into a decaying average. This ensures currency, as workloads + * change over time, and avoids overflow in the calculations. + * + * Use lruvec_page_state_monotonic() so unsigned subtraction + * yields the correct delta across a signed-long wraparound of + * the underlying counter (a real hazard on 32-bit that the + * clamp in lruvec_page_state() would otherwise turn into a huge + * spurious delta). */ - spin_lock_irq(&target_lruvec->lru_lock); - sc->anon_cost = target_lruvec->anon_cost; - sc->file_cost = target_lruvec->file_cost; - spin_unlock_irq(&target_lruvec->lru_lock); + spin_lock(&target_lruvec->cost_lock); + + for (int f = 0; f <= 1; f++) { + struct lru_cost *cost = &target_lruvec->cost[f]; + unsigned long rotated, io, nr_rotated, nr_io; + + rotated = lruvec_page_state_monotonic(target_lruvec, + PGROTATE_ANON + f); + io = lruvec_page_state_monotonic(target_lruvec, + WORKINGSET_RESTORE_BASE + f); + if (f == WORKINGSET_ANON) + io += lruvec_page_state_monotonic(target_lruvec, + NR_VMSCAN_WRITE); + + nr_rotated = rotated - cost->last_rotated; + nr_io = io - cost->last_io; + + /* + * Reflect the relative cost of incurring IO and spending + * CPU time on rotations. This doesn't attempt to make a + * precise comparison, it just says: if reloads are about + * comparable between the LRU lists, or rotations are + * overwhelmingly different between them, adjust scan + * balance for CPU work. + */ + cost->count += nr_io * SWAP_CLUSTER_MAX + nr_rotated; + + cost->last_rotated = rotated; + cost->last_io = io; + } + + anon_cost = &target_lruvec->cost[WORKINGSET_ANON]; + file_cost = &target_lruvec->cost[WORKINGSET_FILE]; + + lrusize = lruvec_page_state(target_lruvec, NR_INACTIVE_ANON) + + lruvec_page_state(target_lruvec, NR_ACTIVE_ANON) + + lruvec_page_state(target_lruvec, NR_INACTIVE_FILE) + + lruvec_page_state(target_lruvec, NR_ACTIVE_FILE); + + while (anon_cost->count + file_cost->count > lrusize / 4) { + anon_cost->count /= 2; + file_cost->count /= 2; + } + + sc->anon_cost = anon_cost->count; + sc->file_cost = file_cost->count; + + spin_unlock(&target_lruvec->cost_lock); /* * Target desirable inactive:active list ratios for the anon @@ -4826,7 +4884,8 @@ static int evict_folios(unsigned long nr_to_scan, struct lruvec *lruvec, struct reclaim_stat stat; struct lru_gen_mm_walk *walk; int scanned, reclaimed; - int isolated = 0, type, type_scanned; + int isolated = 0, nr_isolated = 0, type, type_scanned; + unsigned long total_reclaimed = 0; bool skip_retry = false; struct mem_cgroup *memcg = lruvec_memcg(lruvec); struct pglist_data *pgdat = lruvec_pgdat(lruvec); @@ -4838,6 +4897,7 @@ static int evict_folios(unsigned long nr_to_scan, struct lruvec *lruvec, scanned = isolate_folios(nr_to_scan, lruvec, sc, swappiness, &list, &isolated, &type, &type_scanned); + nr_isolated = isolated; /* Scanning may have emptied the oldest gen, flush it */ if (scanned) @@ -4850,6 +4910,7 @@ static int evict_folios(unsigned long nr_to_scan, struct lruvec *lruvec, retry: reclaimed = shrink_folio_list(&list, pgdat, sc, &stat, false, memcg); sc->nr_reclaimed += reclaimed; + total_reclaimed += reclaimed; /* Retry pass is only meant for clean folios without new isolation */ if (isolated) handle_reclaim_writeback(isolated, pgdat, sc, &stat); @@ -4901,6 +4962,10 @@ retry: goto retry; } + if (nr_isolated > total_reclaimed) + mod_lruvec_state(lruvec, PGROTATE_ANON + type, + nr_isolated - total_reclaimed); + return scanned; } diff --git a/mm/vmstat.c b/mm/vmstat.c index 9c50f0f59f57..cb57714539fb 100644 --- a/mm/vmstat.c +++ b/mm/vmstat.c @@ -1301,6 +1301,8 @@ const char * const vmstat_text[] = { [I(PGSCAN_PROACTIVE)] = "pgscan_proactive", [I(PGSCAN_ANON)] = "pgscan_anon", [I(PGSCAN_FILE)] = "pgscan_file", + [I(PGROTATE_ANON)] = "pgrotate_anon", + [I(PGROTATE_FILE)] = "pgrotate_file", [I(PGREFILL)] = "pgrefill", #ifdef CONFIG_HUGETLB_PAGE [I(NR_HUGETLB)] = "nr_hugetlb", diff --git a/mm/workingset.c b/mm/workingset.c index f351798e723a..7ac2b88c80ae 100644 --- a/mm/workingset.c +++ b/mm/workingset.c @@ -584,11 +584,6 @@ void workingset_refault(struct folio *folio, void *shadow) /* Folio was active prior to eviction */ if (workingset) { folio_set_workingset(folio); - /* - * XXX: Move to folio_add_lru() when it supports new vs - * putback - */ - lru_note_cost_refault(folio); mod_lruvec_state(lruvec, WORKINGSET_RESTORE_BASE + file, nr); } out: