Commit d7e8883c authored by Eric Dumazet's avatar Eric Dumazet Committed by David S. Miller
Browse files

net: make GRO aware of skb->head_frag

GRO can check if skb to be merged has its skb->head mapped to a page
fragment, instead of a kmalloc() area.

We 'upgrade' skb->head as a fragment in itself

This avoids the frag_list fallback, and permits to build true GRO skb
(one sk_buff and up to 16 fragments), using less memory.

This reduces number of cache misses when user makes its copy, since a
single sk_buff is fetched.

This is a followup of patch "net: allow skb->head to be a page fragment"
Signed-off-by: default avatarEric Dumazet <>
Cc: Ilpo Järvinen <>
Cc: Herbert Xu <>
Cc: Maciej Żenczykowski <>
Cc: Neal Cardwell <>
Cc: Tom Herbert <>
Cc: Jeff Kirsher <>
Cc: Ben Hutchings <>
Cc: Matt Carlson <>
Cc: Michael Chan <>
Signed-off-by: default avatarDavid S. Miller <>
parent 8d4057a9
......@@ -1509,6 +1509,8 @@ struct napi_gro_cb {
/* Free the skb? */
int free;
#define NAPI_GRO_FREE 1
#define NAPI_GRO_CB(skb) ((struct napi_gro_cb *)(skb)->cb)
......@@ -561,6 +561,7 @@ static inline struct rtable *skb_rtable(const struct sk_buff *skb)
extern void kfree_skb(struct sk_buff *skb);
extern void consume_skb(struct sk_buff *skb);
extern void __kfree_skb(struct sk_buff *skb);
extern struct kmem_cache *skbuff_head_cache;
extern struct sk_buff *__alloc_skb(unsigned int size,
gfp_t priority, int fclone, int node);
extern struct sk_buff *build_skb(void *data, unsigned int frag_size);
......@@ -3546,7 +3546,10 @@ gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb)
kmem_cache_free(skbuff_head_cache, skb);
case GRO_HELD:
......@@ -69,7 +69,7 @@
#include <trace/events/skb.h>
#include <linux/highmem.h>
static struct kmem_cache *skbuff_head_cache __read_mostly;
struct kmem_cache *skbuff_head_cache __read_mostly;
static struct kmem_cache *skbuff_fclone_cache __read_mostly;
static void sock_pipe_buf_release(struct pipe_inode_info *pipe,
......@@ -2901,6 +2901,31 @@ int skb_gro_receive(struct sk_buff **head, struct sk_buff *skb)
NAPI_GRO_CB(skb)->free = 1;
goto done;
} else if (skb->head_frag) {
int nr_frags = pinfo->nr_frags;
skb_frag_t *frag = pinfo->frags + nr_frags;
struct page *page = virt_to_head_page(skb->head);
unsigned int first_size = headlen - offset;
unsigned int first_offset;
if (nr_frags + 1 + skbinfo->nr_frags > MAX_SKB_FRAGS)
return -E2BIG;
first_offset = skb->data -
(unsigned char *)page_address(page) +
pinfo->nr_frags = nr_frags + 1 + skbinfo->nr_frags;
frag->page.p = page;
frag->page_offset = first_offset;
skb_frag_size_set(frag, first_size);
memcpy(frag + 1, skbinfo->frags, sizeof(*frag) * skbinfo->nr_frags);
/* We dont need to clear skbinfo->nr_frags here */
goto done;
} else if (skb_gro_len(p) != pinfo->gso_size)
return -E2BIG;
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment