Skip to content

Remove task and cgroup local storage percpu counters #10507

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
kernel-patches-daemon-bpf wants to merge 16 commits into
base: bpf-next_base
Choose a base branch
from
+294 −480
Open

Remove task and cgroup local storage percpu counters #10507

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
ad18b7e
bpf: Convert bpf_selem_unlink_map to failable
ameryhung Dec 18, 2025
df272bc
bpf: Convert bpf_selem_link_map to failable
ameryhung Dec 18, 2025
251f572
bpf: Open code bpf_selem_unlink_storage in bpf_selem_unlink
ameryhung Dec 18, 2025
095c2df
bpf: Convert bpf_selem_unlink to failable
ameryhung Dec 18, 2025
d13df03
bpf: Change local_storage->lock and b->lock to rqspinlock
ameryhung Dec 18, 2025
28403a6
bpf: Remove task local storage percpu counter
ameryhung Dec 18, 2025
6fe1178
bpf: Remove cgroup local storage percpu counter
ameryhung Dec 18, 2025
f49de3f
bpf: Remove unused percpu counter from bpf_local_storage_map_free
ameryhung Dec 18, 2025
59ee9f5
bpf: Save memory allocation method and size in bpf_local_storage_elem
ameryhung Dec 18, 2025
9479a49
bpf: Support lockless unlink when freeing map or local storage
ameryhung Dec 18, 2025
ebaed77
bpf: Switch to bpf_selem_unlink_lockless in bpf_local_storage_{map_fr…
ameryhung Dec 18, 2025
153867a
selftests/bpf: Update sk_storage_omem_uncharge test
ameryhung Dec 18, 2025
d88984e
selftests/bpf: Update task_local_storage/recursion test
ameryhung Dec 18, 2025
7629531
selftests/bpf: Update task_local_storage/task_storage_nodeadlock test
ameryhung Dec 18, 2025
bd9d683
selftests/bpf: Remove test_task_storage_map_stress_lookup
ameryhung Dec 18, 2025
7871fae
selftests/bpf: Choose another percpu variable in bpf for btf_dump test
ameryhung Dec 18, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 11 additions & 9 deletions include/linux/bpf_local_storage.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,13 @@
#include <linux/types.h>
#include <linux/bpf_mem_alloc.h>
#include <uapi/linux/btf.h>
#include <asm/rqspinlock.h>

#define BPF_LOCAL_STORAGE_CACHE_SIZE 16

struct bpf_local_storage_map_bucket {
struct hlist_head list;
raw_spinlock_t lock;
rqspinlock_t lock;
};

/* Thp map is not the primary owner of a bpf_local_storage_elem.
Expand Down Expand Up @@ -79,7 +80,9 @@ struct bpf_local_storage_elem {
* after raw_spin_unlock
*/
};
/* 8 bytes hole */
atomic_t link_cnt;
u16 size;
bool use_kmalloc_nolock;
/* The data is stored in another cacheline to minimize
* the number of cachelines access during a cache hit.
*/
Expand All @@ -94,7 +97,7 @@ struct bpf_local_storage {
* bpf_local_storage_elem.
*/
struct rcu_head rcu;
raw_spinlock_t lock; /* Protect adding/removing from the "list" */
rqspinlock_t lock; /* Protect adding/removing from the "list" */
bool use_kmalloc_nolock;
};

Expand Down Expand Up @@ -162,11 +165,10 @@ bpf_local_storage_lookup(struct bpf_local_storage *local_storage,
return SDATA(selem);
}

void bpf_local_storage_destroy(struct bpf_local_storage *local_storage);
u32 bpf_local_storage_destroy(struct bpf_local_storage *local_storage);

void bpf_local_storage_map_free(struct bpf_map *map,
struct bpf_local_storage_cache *cache,
int __percpu *busy_counter);
struct bpf_local_storage_cache *cache);

int bpf_local_storage_map_check_btf(const struct bpf_map *map,
const struct btf *btf,
Expand All @@ -176,10 +178,10 @@ int bpf_local_storage_map_check_btf(const struct bpf_map *map,
void bpf_selem_link_storage_nolock(struct bpf_local_storage *local_storage,
struct bpf_local_storage_elem *selem);

void bpf_selem_unlink(struct bpf_local_storage_elem *selem, bool reuse_now);
int bpf_selem_unlink(struct bpf_local_storage_elem *selem, bool reuse_now);

void bpf_selem_link_map(struct bpf_local_storage_map *smap,
struct bpf_local_storage_elem *selem);
int bpf_selem_link_map(struct bpf_local_storage_map *smap,
struct bpf_local_storage_elem *selem);

struct bpf_local_storage_elem *
bpf_selem_alloc(struct bpf_local_storage_map *smap, void *owner, void *value,
Expand Down
63 changes: 10 additions & 53 deletions kernel/bpf/bpf_cgrp_storage.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,29 +11,6 @@

DEFINE_BPF_STORAGE_CACHE(cgroup_cache);

static DEFINE_PER_CPU(int, bpf_cgrp_storage_busy);

static void bpf_cgrp_storage_lock(void)
{
cant_migrate();
this_cpu_inc(bpf_cgrp_storage_busy);
}

static void bpf_cgrp_storage_unlock(void)
{
this_cpu_dec(bpf_cgrp_storage_busy);
}

static bool bpf_cgrp_storage_trylock(void)
{
cant_migrate();
if (unlikely(this_cpu_inc_return(bpf_cgrp_storage_busy) != 1)) {
this_cpu_dec(bpf_cgrp_storage_busy);
return false;
}
return true;
}

static struct bpf_local_storage __rcu **cgroup_storage_ptr(void *owner)
{
struct cgroup *cg = owner;
Expand All @@ -45,16 +22,15 @@ void bpf_cgrp_storage_free(struct cgroup *cgroup)
{
struct bpf_local_storage *local_storage;

rcu_read_lock_dont_migrate();
rcu_read_lock();
local_storage = rcu_dereference(cgroup->bpf_cgrp_storage);
if (!local_storage)
goto out;

bpf_cgrp_storage_lock();
bpf_local_storage_destroy(local_storage);
bpf_cgrp_storage_unlock();
RCU_INIT_POINTER(cgroup->bpf_cgrp_storage, NULL);
out:
rcu_read_unlock_migrate();
rcu_read_unlock();
}

static struct bpf_local_storage_data *
Expand Down Expand Up @@ -83,9 +59,7 @@ static void *bpf_cgrp_storage_lookup_elem(struct bpf_map *map, void *key)
if (IS_ERR(cgroup))
return ERR_CAST(cgroup);

bpf_cgrp_storage_lock();
sdata = cgroup_storage_lookup(cgroup, map, true);
bpf_cgrp_storage_unlock();
cgroup_put(cgroup);
return sdata ? sdata->data : NULL;
}
Expand All @@ -102,10 +76,8 @@ static long bpf_cgrp_storage_update_elem(struct bpf_map *map, void *key,
if (IS_ERR(cgroup))
return PTR_ERR(cgroup);

bpf_cgrp_storage_lock();
sdata = bpf_local_storage_update(cgroup, (struct bpf_local_storage_map *)map,
value, map_flags, false, GFP_ATOMIC);
bpf_cgrp_storage_unlock();
cgroup_put(cgroup);
return PTR_ERR_OR_ZERO(sdata);
}
Expand All @@ -118,8 +90,7 @@ static int cgroup_storage_delete(struct cgroup *cgroup, struct bpf_map *map)
if (!sdata)
return -ENOENT;

bpf_selem_unlink(SELEM(sdata), false);
return 0;
return bpf_selem_unlink(SELEM(sdata), false);
}

static long bpf_cgrp_storage_delete_elem(struct bpf_map *map, void *key)
Expand All @@ -132,9 +103,7 @@ static long bpf_cgrp_storage_delete_elem(struct bpf_map *map, void *key)
if (IS_ERR(cgroup))
return PTR_ERR(cgroup);

bpf_cgrp_storage_lock();
err = cgroup_storage_delete(cgroup, map);
bpf_cgrp_storage_unlock();
cgroup_put(cgroup);
return err;
}
Expand All @@ -151,15 +120,14 @@ static struct bpf_map *cgroup_storage_map_alloc(union bpf_attr *attr)

static void cgroup_storage_map_free(struct bpf_map *map)
{
bpf_local_storage_map_free(map, &cgroup_cache, &bpf_cgrp_storage_busy);
bpf_local_storage_map_free(map, &cgroup_cache);
}

/* *gfp_flags* is a hidden argument provided by the verifier */
BPF_CALL_5(bpf_cgrp_storage_get, struct bpf_map *, map, struct cgroup *, cgroup,
void *, value, u64, flags, gfp_t, gfp_flags)
{
struct bpf_local_storage_data *sdata;
bool nobusy;

WARN_ON_ONCE(!bpf_rcu_lock_held());
if (flags & ~(BPF_LOCAL_STORAGE_GET_F_CREATE))
Expand All @@ -168,38 +136,27 @@ BPF_CALL_5(bpf_cgrp_storage_get, struct bpf_map *, map, struct cgroup *, cgroup,
if (!cgroup)
return (unsigned long)NULL;

nobusy = bpf_cgrp_storage_trylock();

sdata = cgroup_storage_lookup(cgroup, map, nobusy);
sdata = cgroup_storage_lookup(cgroup, map, true);
if (sdata)
goto unlock;
goto out;

/* only allocate new storage, when the cgroup is refcounted */
if (!percpu_ref_is_dying(&cgroup->self.refcnt) &&
(flags & BPF_LOCAL_STORAGE_GET_F_CREATE) && nobusy)
(flags & BPF_LOCAL_STORAGE_GET_F_CREATE))
sdata = bpf_local_storage_update(cgroup, (struct bpf_local_storage_map *)map,
value, BPF_NOEXIST, false, gfp_flags);

unlock:
if (nobusy)
bpf_cgrp_storage_unlock();
out:
return IS_ERR_OR_NULL(sdata) ? (unsigned long)NULL : (unsigned long)sdata->data;
}

BPF_CALL_2(bpf_cgrp_storage_delete, struct bpf_map *, map, struct cgroup *, cgroup)
{
int ret;

WARN_ON_ONCE(!bpf_rcu_lock_held());
if (!cgroup)
return -EINVAL;

if (!bpf_cgrp_storage_trylock())
return -EBUSY;

ret = cgroup_storage_delete(cgroup, map);
bpf_cgrp_storage_unlock();
return ret;
return cgroup_storage_delete(cgroup, map);
}

const struct bpf_map_ops cgrp_storage_map_ops = {
Expand Down
7 changes: 3 additions & 4 deletions kernel/bpf/bpf_inode_storage.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ void bpf_inode_storage_free(struct inode *inode)
goto out;

bpf_local_storage_destroy(local_storage);
RCU_INIT_POINTER(bsb->storage, NULL);
out:
rcu_read_unlock_migrate();
}
Expand Down Expand Up @@ -110,9 +111,7 @@ static int inode_storage_delete(struct inode *inode, struct bpf_map *map)
if (!sdata)
return -ENOENT;

bpf_selem_unlink(SELEM(sdata), false);

return 0;
return bpf_selem_unlink(SELEM(sdata), false);
}

static long bpf_fd_inode_storage_delete_elem(struct bpf_map *map, void *key)
Expand Down Expand Up @@ -186,7 +185,7 @@ static struct bpf_map *inode_storage_map_alloc(union bpf_attr *attr)

static void inode_storage_map_free(struct bpf_map *map)
{
bpf_local_storage_map_free(map, &inode_cache, NULL);
bpf_local_storage_map_free(map, &inode_cache);
}

const struct bpf_map_ops inode_storage_map_ops = {
Expand Down
Loading
Loading