Re: [PATCH v4 46/48] mm: shrinker: make memcg slab shrink lockless

[Dave Chinner <david@xxxxxxxxxxxxx>]

On Mon, Aug 07, 2023 at 07:09:34PM +0800, Qi Zheng wrote:
> Like global slab shrink, this commit also uses refcount+RCU method to make
> memcg slab shrink lockless.

This patch does random code cleanups amongst the actual RCU changes.
Can you please move the cleanups to a spearate patch to reduce the
noise in this one?

> diff --git a/mm/shrinker.c b/mm/shrinker.c
> index d318f5621862..fee6f62904fb 100644
> --- a/mm/shrinker.c
> +++ b/mm/shrinker.c
> @@ -107,6 +107,12 @@ static struct shrinker_info 
> *shrinker_info_protected(struct mem_cgroup *memcg,
>                                        lockdep_is_held(&shrinker_rwsem));
>  }
>  
> +static struct shrinker_info *shrinker_info_rcu(struct mem_cgroup *memcg,
> +                                            int nid)
> +{
> +     return rcu_dereference(memcg->nodeinfo[nid]->shrinker_info);
> +}

This helper doesn't add value. It doesn't tell me that
rcu_read_lock() needs to be held when it is called, for one....

>  static int expand_one_shrinker_info(struct mem_cgroup *memcg, int new_size,
>                                   int old_size, int new_nr_max)
>  {
> @@ -198,7 +204,7 @@ void set_shrinker_bit(struct mem_cgroup *memcg, int nid, 
> int shrinker_id)
>               struct shrinker_info_unit *unit;
>  
>               rcu_read_lock();
> -             info = rcu_dereference(memcg->nodeinfo[nid]->shrinker_info);
> +             info = shrinker_info_rcu(memcg, nid);

... whilst the original code here was obviously correct.

>               unit = info->unit[shriner_id_to_index(shrinker_id)];
>               if (!WARN_ON_ONCE(shrinker_id >= info->map_nr_max)) {
>                       /* Pairs with smp mb in shrink_slab() */
> @@ -211,7 +217,7 @@ void set_shrinker_bit(struct mem_cgroup *memcg, int nid, 
> int shrinker_id)
>  
>  static DEFINE_IDR(shrinker_idr);
>  
> -static int prealloc_memcg_shrinker(struct shrinker *shrinker)
> +static int shrinker_memcg_alloc(struct shrinker *shrinker)

Cleanups in a separate patch.

> @@ -253,10 +258,15 @@ static long xchg_nr_deferred_memcg(int nid, struct 
> shrinker *shrinker,
>  {
>       struct shrinker_info *info;
>       struct shrinker_info_unit *unit;
> +     long nr_deferred;
>  
> -     info = shrinker_info_protected(memcg, nid);
> +     rcu_read_lock();
> +     info = shrinker_info_rcu(memcg, nid);
>       unit = info->unit[shriner_id_to_index(shrinker->id)];
> -     return 
> atomic_long_xchg(&unit->nr_deferred[shriner_id_to_offset(shrinker->id)], 0);
> +     nr_deferred = 
> atomic_long_xchg(&unit->nr_deferred[shriner_id_to_offset(shrinker->id)], 0);
> +     rcu_read_unlock();
> +
> +     return nr_deferred;
>  }

This adds two rcu_read_lock() sections to every call to
do_shrink_slab(). It's not at all clear ifrom any of the other code
that do_shrink_slab() now has internal rcu_read_lock() sections....

> @@ -464,18 +480,23 @@ static unsigned long shrink_slab_memcg(gfp_t gfp_mask, 
> int nid,
>       if (!mem_cgroup_online(memcg))
>               return 0;
>  
> -     if (!down_read_trylock(&shrinker_rwsem))
> -             return 0;
> -
> -     info = shrinker_info_protected(memcg, nid);
> +again:
> +     rcu_read_lock();
> +     info = shrinker_info_rcu(memcg, nid);
>       if (unlikely(!info))
>               goto unlock;
>  
> -     for (; index < shriner_id_to_index(info->map_nr_max); index++) {
> +     if (index < shriner_id_to_index(info->map_nr_max)) {
>               struct shrinker_info_unit *unit;
>  
>               unit = info->unit[index];
>  
> +             /*
> +              * The shrinker_info_unit will not be freed, so we can
> +              * safely release the RCU lock here.
> +              */
> +             rcu_read_unlock();

Why - what guarantees that the shrinker_info_unit exists at this
point? We hold no reference to it, we hold no reference to any
shrinker, etc. What provides this existence guarantee?

> +
>               for_each_set_bit(offset, unit->map, SHRINKER_UNIT_BITS) {
>                       struct shrink_control sc = {
>                               .gfp_mask = gfp_mask,
> @@ -485,12 +506,14 @@ static unsigned long shrink_slab_memcg(gfp_t gfp_mask, 
> int nid,
>                       struct shrinker *shrinker;
>                       int shrinker_id = calc_shrinker_id(index, offset);
>  
> +                     rcu_read_lock();
>                       shrinker = idr_find(&shrinker_idr, shrinker_id);
> -                     if (unlikely(!shrinker || !(shrinker->flags & 
> SHRINKER_REGISTERED))) {
> -                             if (!shrinker)
> -                                     clear_bit(offset, unit->map);
> +                     if (unlikely(!shrinker || !shrinker_try_get(shrinker))) 
> {
> +                             clear_bit(offset, unit->map);
> +                             rcu_read_unlock();
>                               continue;
>                       }
> +                     rcu_read_unlock();
>  
>                       /* Call non-slab shrinkers even though kmem is disabled 
> */
>                       if (!memcg_kmem_online() &&
> @@ -523,15 +546,20 @@ static unsigned long shrink_slab_memcg(gfp_t gfp_mask, 
> int nid,
>                                       set_shrinker_bit(memcg, nid, 
> shrinker_id);
>                       }
>                       freed += ret;
> -
> -                     if (rwsem_is_contended(&shrinker_rwsem)) {
> -                             freed = freed ? : 1;
> -                             goto unlock;
> -                     }
> +                     shrinker_put(shrinker);

Ok, so why is this safe to call without holding the rcu read lock?
The global shrinker has to hold the rcu_read_lock() whilst calling
shrinker_put() to guarantee the validity of the list next pointer,
but we don't hold off RCU here so what guarantees a racing global
shrinker walk doesn't trip over this shrinker_put() call dropping
the refcount to zero and freeing occuring in a different context...


> +             /*
> +              * We have already exited the read-side of rcu critical section
> +              * before calling do_shrink_slab(), the shrinker_info may be
> +              * released in expand_one_shrinker_info(), so reacquire the
> +              * shrinker_info.
> +              */
> +             index++;
> +             goto again;

With that, what makes the use of shrinker_info in
xchg_nr_deferred_memcg() in do_shrink_slab() coherent and valid?

-Dave.
-- 
Dave Chinner
david@xxxxxxxxxxxxx