net: Support priv pool allocation for skb->data

skb->data needs to be shared across LCD and KLCD. To facilitate this, create a
shared area and an allocator on top and use this for allocating skb->data for
sk_buffs. Use NETIF flags to enable priv pool allocation.
Signed-off-by: Vikram Narayanan <vikram186@gmail.com>

include/linux/netdev_features.h

@@ -74,7 +74,7 @@ enum {
 	NETIF_F_BUSY_POLL_BIT,		/* Busy poll */
 
 	NETIF_F_HW_TC_BIT,		/* Offload TC infrastructure */
-
+	NETIF_F_PRIV_DATA_POOL_BIT,	/* private pool for skb->data */
 	/*
 	 * Add your fresh new feature above and remember to update
 	 * netdev_features_strings[] in net/core/ethtool.c and maybe
@@ -136,6 +136,7 @@ enum {
 #define NETIF_F_HW_L2FW_DOFFLOAD	__NETIF_F(HW_L2FW_DOFFLOAD)
 #define NETIF_F_BUSY_POLL	__NETIF_F(BUSY_POLL)
 #define NETIF_F_HW_TC		__NETIF_F(HW_TC)
+#define NETIF_F_PRIV_DATA_POOL	__NETIF_F(PRIV_DATA_POOL)
 
 #define for_each_netdev_feature(mask_addr, bit)	\
 	for_each_set_bit(bit, (unsigned long *)mask_addr, NETDEV_FEATURE_COUNT)

include/linux/skbuff.h

@@ -222,6 +222,9 @@
 /* Maximum value in skb->csum_level */
 #define SKB_MAX_CSUM_LEVEL	3
 
+/* for skb->data allocation */
+#define SKB_DATA_PRIV_POOL	(1 << 31)
+
 #define SKB_DATA_ALIGN(X)	ALIGN(X, SMP_CACHE_BYTES)
 #define SKB_WITH_OVERHEAD(X)	\
 	((X) - SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
@@ -682,7 +685,8 @@ struct sk_buff {
 				fclone:2,
 				peeked:1,
 				head_frag:1,
-				xmit_more:1;
+				xmit_more:1,
+				private:1;
 	/* one bit hole */
 	kmemcheck_bitfield_end(flags1);
 

net/core/ethtool.c

@@ -104,6 +104,7 @@ static const char netdev_features_strings[NETDEV_FEATURE_COUNT][ETH_GSTRING_LEN]
 	[NETIF_F_HW_L2FW_DOFFLOAD_BIT] = "l2-fwd-offload",
 	[NETIF_F_BUSY_POLL_BIT] =        "busy-poll",
 	[NETIF_F_HW_TC_BIT] =		 "hw-tc-offload",
+	[NETIF_F_PRIV_DATA_POOL_BIT] =	 "priv-data-pool",
 };
 
 static const char

net/core/skbuff.c

@@ -78,6 +78,8 @@
 #include <linux/capability.h>
 #include <linux/user_namespace.h>
 
+#include <linux/priv_mempool.h>
+
 struct kmem_cache *skbuff_head_cache __read_mostly;
 static struct kmem_cache *skbuff_fclone_cache __read_mostly;
 int sysctl_max_skb_frags __read_mostly = MAX_SKB_FRAGS;
@@ -208,6 +210,7 @@ struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask,
 	struct sk_buff *skb;
 	u8 *data;
 	bool pfmemalloc;
+	bool privpool = false;
 
 	cache = (flags & SKB_ALLOC_FCLONE)
 		? skbuff_fclone_cache : skbuff_head_cache;
@@ -221,22 +224,40 @@ struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask,
 		goto out;
 	prefetchw(skb);
 
+	if (size & SKB_DATA_PRIV_POOL) {
+		privpool = true;
+		size &= ~SKB_DATA_PRIV_POOL;
+	}
 	/* We do our best to align skb_shared_info on a separate cache
 	 * line. It usually works because kmalloc(X > SMP_CACHE_BYTES) gives
 	 * aligned memory blocks, unless SLUB/SLAB debug is enabled.
 	 * Both skb->head and skb_shared_info are cache line aligned.
 	 */
-	size = SKB_DATA_ALIGN(size);
-	size += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
-	data = kmalloc_reserve(size, gfp_mask, node, &pfmemalloc);
+	if (unlikely(privpool)) {
+		/* private pool */
+		data = priv_alloc(SKB_DATA_POOL);
+		size = SKB_DATA_SIZE;
+	} else {
+		size = SKB_DATA_ALIGN(size);
+		size += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
+
+		data = kmalloc_reserve(size, gfp_mask, node, &pfmemalloc);
+	}
+
 	if (!data)
 		goto nodata;
+
 	/* kmalloc(size) might give us more room than requested.
 	 * Put skb_shared_info exactly at the end of allocated zone,
 	 * to allow max possible filling before reallocation.
 	 */
-	size = SKB_WITH_OVERHEAD(ksize(data));
-	prefetchw(data + size);
+	if (!privpool) {
+		size = SKB_WITH_OVERHEAD(ksize(data));
+		prefetchw(data + size);
+	} else {
+		size = SKB_WITH_OVERHEAD(SKB_DATA_SIZE);
+		prefetchw(data);
+	}
 
 	/*
 	 * Only clear those fields we need to clear, not those that we will
@@ -255,6 +276,9 @@ struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask,
 	skb->mac_header = (typeof(skb->mac_header))~0U;
 	skb->transport_header = (typeof(skb->transport_header))~0U;
 
+	if (privpool)
+		skb->private = true;
+
 	/* make sure we initialize shinfo sequentially */
 	shinfo = skb_shinfo(skb);
 	memset(shinfo, 0, offsetof(struct skb_shared_info, dataref));
@@ -576,8 +600,12 @@ static void skb_free_head(struct sk_buff *skb)
 
 	if (skb->head_frag)
 		skb_free_frag(head);
-	else
-		kfree(head);
+	else {
+		if (skb->private)
+			priv_free(head, SKB_DATA_POOL);
+		else
+			kfree(head);
+	}
 }
 
 static void skb_release_data(struct sk_buff *skb)
@@ -910,6 +938,7 @@ static struct sk_buff *__skb_clone(struct sk_buff *n, struct sk_buff *skb)
 	C(head_frag);
 	C(data);
 	C(truesize);
+	C(private);
 	atomic_set(&n->users, 1);
 
 	atomic_inc(&(skb_shinfo(skb)->dataref));

net/ipv4/ip_output.c

@@ -984,15 +984,25 @@ alloc_new_skb:
 				alloclen += rt->dst.trailer_len;
 
 			if (transhdrlen) {
-				skb = sock_alloc_send_skb(sk,
-						alloclen + hh_len + 15,
+				unsigned long size = alloclen + hh_len + 15;
+
+				if (rt->dst.dev->features & NETIF_F_PRIV_DATA_POOL)
+					size |= SKB_DATA_PRIV_POOL;
+
+				skb = sock_alloc_send_skb(sk, size,
 						(flags & MSG_DONTWAIT), &err);
 			} else {
+				unsigned long size = alloclen + hh_len + 15;
+
+				if (rt->dst.dev->features & NETIF_F_PRIV_DATA_POOL)
+					size |= SKB_DATA_PRIV_POOL;
+
 				skb = NULL;
+
 				if (atomic_read(&sk->sk_wmem_alloc) <=
 				    2 * sk->sk_sndbuf)
 					skb = sock_wmalloc(sk,
-							   alloclen + hh_len + 15, 1,
+							   size, 1,
 							   sk->sk_allocation);
 				if (unlikely(!skb))
 					err = -ENOBUFS;