Commit d214e3a5 authored by Vikram Narayanan's avatar Vikram Narayanan

net: Support priv pool allocation for skb->data

skb->data needs to be shared across LCD and KLCD. To facilitate this, create a
shared area and an allocator on top and use this for allocating skb->data for
sk_buffs. Use NETIF flags to enable priv pool allocation.
Signed-off-by: Vikram Narayanan's avatarVikram Narayanan <vikram186@gmail.com>
parent 808bd7bb
......@@ -74,7 +74,7 @@ enum {
NETIF_F_BUSY_POLL_BIT, /* Busy poll */
NETIF_F_HW_TC_BIT, /* Offload TC infrastructure */
NETIF_F_PRIV_DATA_POOL_BIT, /* private pool for skb->data */
/*
* Add your fresh new feature above and remember to update
* netdev_features_strings[] in net/core/ethtool.c and maybe
......@@ -136,6 +136,7 @@ enum {
#define NETIF_F_HW_L2FW_DOFFLOAD __NETIF_F(HW_L2FW_DOFFLOAD)
#define NETIF_F_BUSY_POLL __NETIF_F(BUSY_POLL)
#define NETIF_F_HW_TC __NETIF_F(HW_TC)
#define NETIF_F_PRIV_DATA_POOL __NETIF_F(PRIV_DATA_POOL)
#define for_each_netdev_feature(mask_addr, bit) \
for_each_set_bit(bit, (unsigned long *)mask_addr, NETDEV_FEATURE_COUNT)
......
......@@ -222,6 +222,9 @@
/* Maximum value in skb->csum_level */
#define SKB_MAX_CSUM_LEVEL 3
/* for skb->data allocation */
#define SKB_DATA_PRIV_POOL (1 << 31)
#define SKB_DATA_ALIGN(X) ALIGN(X, SMP_CACHE_BYTES)
#define SKB_WITH_OVERHEAD(X) \
((X) - SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
......@@ -682,7 +685,8 @@ struct sk_buff {
fclone:2,
peeked:1,
head_frag:1,
xmit_more:1;
xmit_more:1,
private:1;
/* one bit hole */
kmemcheck_bitfield_end(flags1);
......
......@@ -104,6 +104,7 @@ static const char netdev_features_strings[NETDEV_FEATURE_COUNT][ETH_GSTRING_LEN]
[NETIF_F_HW_L2FW_DOFFLOAD_BIT] = "l2-fwd-offload",
[NETIF_F_BUSY_POLL_BIT] = "busy-poll",
[NETIF_F_HW_TC_BIT] = "hw-tc-offload",
[NETIF_F_PRIV_DATA_POOL_BIT] = "priv-data-pool",
};
static const char
......
......@@ -78,6 +78,8 @@
#include <linux/capability.h>
#include <linux/user_namespace.h>
#include <linux/priv_mempool.h>
struct kmem_cache *skbuff_head_cache __read_mostly;
static struct kmem_cache *skbuff_fclone_cache __read_mostly;
int sysctl_max_skb_frags __read_mostly = MAX_SKB_FRAGS;
......@@ -208,6 +210,7 @@ struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask,
struct sk_buff *skb;
u8 *data;
bool pfmemalloc;
bool privpool = false;
cache = (flags & SKB_ALLOC_FCLONE)
? skbuff_fclone_cache : skbuff_head_cache;
......@@ -221,22 +224,40 @@ struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask,
goto out;
prefetchw(skb);
if (size & SKB_DATA_PRIV_POOL) {
privpool = true;
size &= ~SKB_DATA_PRIV_POOL;
}
/* We do our best to align skb_shared_info on a separate cache
* line. It usually works because kmalloc(X > SMP_CACHE_BYTES) gives
* aligned memory blocks, unless SLUB/SLAB debug is enabled.
* Both skb->head and skb_shared_info are cache line aligned.
*/
size = SKB_DATA_ALIGN(size);
size += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
data = kmalloc_reserve(size, gfp_mask, node, &pfmemalloc);
if (unlikely(privpool)) {
/* private pool */
data = priv_alloc(SKB_DATA_POOL);
size = SKB_DATA_SIZE;
} else {
size = SKB_DATA_ALIGN(size);
size += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
data = kmalloc_reserve(size, gfp_mask, node, &pfmemalloc);
}
if (!data)
goto nodata;
/* kmalloc(size) might give us more room than requested.
* Put skb_shared_info exactly at the end of allocated zone,
* to allow max possible filling before reallocation.
*/
size = SKB_WITH_OVERHEAD(ksize(data));
prefetchw(data + size);
if (!privpool) {
size = SKB_WITH_OVERHEAD(ksize(data));
prefetchw(data + size);
} else {
size = SKB_WITH_OVERHEAD(SKB_DATA_SIZE);
prefetchw(data);
}
/*
* Only clear those fields we need to clear, not those that we will
......@@ -255,6 +276,9 @@ struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask,
skb->mac_header = (typeof(skb->mac_header))~0U;
skb->transport_header = (typeof(skb->transport_header))~0U;
if (privpool)
skb->private = true;
/* make sure we initialize shinfo sequentially */
shinfo = skb_shinfo(skb);
memset(shinfo, 0, offsetof(struct skb_shared_info, dataref));
......@@ -576,8 +600,12 @@ static void skb_free_head(struct sk_buff *skb)
if (skb->head_frag)
skb_free_frag(head);
else
kfree(head);
else {
if (skb->private)
priv_free(head, SKB_DATA_POOL);
else
kfree(head);
}
}
static void skb_release_data(struct sk_buff *skb)
......@@ -910,6 +938,7 @@ static struct sk_buff *__skb_clone(struct sk_buff *n, struct sk_buff *skb)
C(head_frag);
C(data);
C(truesize);
C(private);
atomic_set(&n->users, 1);
atomic_inc(&(skb_shinfo(skb)->dataref));
......
......@@ -984,15 +984,25 @@ alloc_new_skb:
alloclen += rt->dst.trailer_len;
if (transhdrlen) {
skb = sock_alloc_send_skb(sk,
alloclen + hh_len + 15,
unsigned long size = alloclen + hh_len + 15;
if (rt->dst.dev->features & NETIF_F_PRIV_DATA_POOL)
size |= SKB_DATA_PRIV_POOL;
skb = sock_alloc_send_skb(sk, size,
(flags & MSG_DONTWAIT), &err);
} else {
unsigned long size = alloclen + hh_len + 15;
if (rt->dst.dev->features & NETIF_F_PRIV_DATA_POOL)
size |= SKB_DATA_PRIV_POOL;
skb = NULL;
if (atomic_read(&sk->sk_wmem_alloc) <=
2 * sk->sk_sndbuf)
skb = sock_wmalloc(sk,
alloclen + hh_len + 15, 1,
size, 1,
sk->sk_allocation);
if (unlikely(!skb))
err = -ENOBUFS;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment