Commit bcdcb5cc authored by Vikram Narayanan's avatar Vikram Narayanan
Browse files

lcd/ixgbe: Refactor xmit and other optimizations



Refactor xmit for fastpath, softirq. Also replace kmalloc with
kmem_cache_alloc. Inc/dec page_ref_count is still a bit fragile and needs to be
fixed.
Signed-off-by: Vikram Narayanan's avatarVikram Narayanan <vikram186@gmail.com>
parent 2e30249f
......@@ -206,4 +206,19 @@ async_msg_blocking_send_start(struct thc_channel *chnl,
return -EIO;
}
}
static inline
int
async_msg_blocking_recv_start(struct thc_channel *chnl,
struct fipc_message** out)
{
int ret;
for (;;) {
/* Poll until we get a message or error */
ret = fipc_recv_msg_start(thc_channel_to_fipc(chnl),
out);
if ( !ret || ret != -EWOULDBLOCK )
return ret;
cpu_relax();
}
}
#endif /* __IXGBE_COMMON_H__ */
......@@ -11,6 +11,8 @@
#include <linux/hashtable.h>
#include <liblcd/netdev_helper.h>
#include "../../rdtsc_helper.h"
#include <lcd_config/post_hook.h>
struct cptr sync_ep;
......@@ -28,6 +30,10 @@ static struct net_device *g_net_device;
struct pcidev_info dev_assign = { 0x0000, 0x04, 0x00, 0x1 };
#endif
struct kmem_cache *skb_c_cache;
struct kmem_cache *skb_c_cache1;
struct kmem_cache *skbuff_cache;
/* XXX: How to determine this? */
#define CPTR_HASH_BITS 5
static DEFINE_HASHTABLE(cptr_table, CPTR_HASH_BITS);
......@@ -47,7 +53,32 @@ int glue_ixgbe_init(void)
}
ixgbe_cspace = c_cspace;
hash_init(cptr_table);
/* merge two datastructures into one for allocation */
skb_c_cache = kmem_cache_create("skb_c_cache",
sizeof(struct sk_buff_container)
+ sizeof(struct sk_buff),
0,
SLAB_HWCACHE_ALIGN|SLAB_PANIC,
NULL);
if (!skb_c_cache) {
LIBLCD_ERR("skb_container cache not created");
goto fail2;
}
/* container only slab for rx */
skb_c_cache1 = kmem_cache_create("skb_c_cache1",
sizeof(struct sk_buff_container),
0,
SLAB_HWCACHE_ALIGN|SLAB_PANIC,
NULL);
if (!skb_c_cache1) {
LIBLCD_ERR("skb_container cache not created");
goto fail2;
}
return 0;
fail2:
glue_cap_exit();
fail1:
......@@ -59,7 +90,11 @@ void glue_ixgbe_exit(void)
{
glue_cap_destroy(c_cspace);
glue_cap_exit();
if (skb_c_cache)
kmem_cache_destroy(skb_c_cache);
if (skb_c_cache1)
kmem_cache_destroy(skb_c_cache1);
}
int glue_insert_skbuff(struct hlist_head *htable, struct sk_buff_container *skb_c)
......@@ -1344,9 +1379,6 @@ void napi_consume_skb(struct sk_buff *skb, int budget)
lcd_cap_delete(skb_c->skb_cptr);
lcd_cap_delete(skb_c->skbh_cptr);
} else {
/* free skb memory that was allocate by us */
kfree(skb);
}
#ifdef NAPI_CONSUME_SEND_ONLY
......@@ -1359,7 +1391,7 @@ void napi_consume_skb(struct sk_buff *skb, int budget)
&_response);
#endif
glue_remove_skbuff(skb_c);
kfree(skb_c);
kmem_cache_free(skb_c_cache, skb_c);
if (ret) {
LIBLCD_ERR("thc_ipc_call");
......@@ -2395,6 +2427,10 @@ fail_lookup:
}
extern netdev_tx_t ixgbe_xmit_frame(struct sk_buff *skb,
struct net_device *dev);
extern struct net_device *g_netdev;
int ndo_start_xmit_callee(struct fipc_message *_request,
struct thc_channel *_channel,
struct glue_cspace *cspace,
......@@ -2402,7 +2438,6 @@ int ndo_start_xmit_callee(struct fipc_message *_request,
{
struct sk_buff *skb;
struct sk_buff_container *skb_c;
struct net_device_container *dev_container;
int ret;
struct fipc_message *_response;
unsigned int request_cookie;
......@@ -2417,19 +2452,10 @@ int ndo_start_xmit_callee(struct fipc_message *_request,
unsigned long skbh_offset, skb_end;
struct skbuff_members *skb_lcd;
__be16 proto;
void *mem;
request_cookie = thc_get_request_cookie(_request);
xmit_type = fipc_get_reg0(_request);
ret = glue_cap_lookup_net_device_type(cspace,
__cptr(fipc_get_reg1(_request)),
&dev_container);
if (ret) {
LIBLCD_ERR("lookup");
goto fail_lookup;
}
skb_ref = __cptr(fipc_get_reg2(_request));
skbh_offset = fipc_get_reg3(_request);
......@@ -2487,7 +2513,10 @@ int ndo_start_xmit_callee(struct fipc_message *_request,
break;
case SHARED_DATA_XMIT:
skb = kzalloc(sizeof(struct sk_buff), GFP_KERNEL);
skb_c = mem = kmem_cache_alloc(skb_c_cache,
GFP_KERNEL);
skb = (struct sk_buff*)((char*)mem +
sizeof(struct sk_buff_container));
if (!skb) {
LIBLCD_MSG("out of mmeory");
......@@ -2527,14 +2556,8 @@ int ndo_start_xmit_callee(struct fipc_message *_request,
break;
}
skb_c = kzalloc(sizeof(*skb_c), GFP_KERNEL);
if (!skb_c)
LIBLCD_MSG("no memory");
skb_c->skb = skb;
glue_insert_skbuff(cptr_table, skb_c);
skb_c->other_ref = skb_ref;
if (xmit_type == VOLUNTEER_XMIT) {
......@@ -2554,9 +2577,7 @@ int ndo_start_xmit_callee(struct fipc_message *_request,
#endif
} /* if */
func_ret = dev_container->net_device.
netdev_ops->ndo_start_xmit(
skb, &dev_container->net_device);
func_ret = ixgbe_xmit_frame(skb, g_netdev);
if (async_msg_blocking_send_start(_channel,
&_response)) {
......@@ -2569,7 +2590,6 @@ int ndo_start_xmit_callee(struct fipc_message *_request,
request_cookie,
_response);
fail_alloc:
fail_lookup:
fail_sync:
return ret;
......@@ -3381,6 +3401,7 @@ gro_result_t napi_gro_receive(struct napi_struct *napi,
unsigned long skb_sz, skb_off, skbh_sz, skbh_off;
cptr_t skb_cptr, skbh_cptr;
struct skb_shared_info *shinfo;
struct page *p = NULL;
ret = async_msg_blocking_send_start(ixgbe_async,
&_request);
......@@ -3402,11 +3423,11 @@ gro_result_t napi_gro_receive(struct napi_struct *napi,
if (shinfo->nr_frags) {
skb_frag_t *frag = &shinfo->frags[0];
printk("frag found, mapping");
fipc_set_reg1(_request, gpa_val(lcd_gva2gpa(
__gva(
(unsigned long)lcd_page_address(
skb_frag_page(frag))))));
p = skb_frag_page(frag);
}
ret = lcd_virt_to_cptr(__gva((unsigned long)skb),
......@@ -3437,7 +3458,7 @@ gro_result_t napi_gro_receive(struct napi_struct *napi,
glue_remove_skbuff(skb_c);
kfree(skb_c);
kmem_cache_free(skb_c_cache1, skb_c);
ret = thc_ipc_call(ixgbe_async,
_request,
......@@ -3449,6 +3470,14 @@ gro_result_t napi_gro_receive(struct napi_struct *napi,
func_ret = fipc_get_reg1(_response);
fipc_recv_msg_end(thc_channel_to_fipc(ixgbe_async),
_response);
/* simulate the effect of put_page called by kfree_skb
* a page_ref_inc is done by the driver to make sure that
* this page is not freed and reused again
*/
if (p)
page_ref_dec(p);
return func_ret;
fail_async:
fail_ipc:
......@@ -3538,7 +3567,7 @@ struct sk_buff *__napi_alloc_skb(struct napi_struct *napi,
skb = (void*)(gva_val(skb_gva) + skb_off);
skb->head = (void*)(gva_val(skbd_gva) + skbd_off);
skb->data = skb->head + data_off;
skb_c = kzalloc(sizeof(*skb_c), GFP_KERNEL);
skb_c = kmem_cache_alloc(skb_c_cache1, GFP_KERNEL);
if (!skb_c)
LIBLCD_MSG("no memory");
......@@ -3621,113 +3650,3 @@ fail_async:
fail_ipc:
return ret;
}
void skb_add_rx_frag(struct sk_buff *skb,
int i,
struct page *page,
int off,
int size,
unsigned int truesize)
{
int ret;
struct fipc_message *_request;
struct fipc_message *_response;
struct sk_buff_container *skb_c;
skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
glue_lookup_skbuff(cptr_table,
__cptr((unsigned long)skb), &skb_c);
ret = async_msg_blocking_send_start(ixgbe_async,
&_request);
if (ret) {
LIBLCD_ERR("failed to get a send slot");
goto fail_async;
}
async_msg_set_fn_type(_request,
SKB_ADD_RX_FRAG);
fipc_set_reg0(_request,
skb_c->other_ref.cptr);
fipc_set_reg1(_request,
i);
fipc_set_reg2(_request,
gpa_val(lcd_gva2gpa(
__gva(
(unsigned long)lcd_page_address(page)))));
fipc_set_reg3(_request,
off);
fipc_set_reg4(_request,
size);
fipc_set_reg5(_request,
truesize);
/* save pointers */
skb_c->head = skb->head;
skb_c->data = skb->data;
printk("calling rpc rx_flag");
ret = thc_ipc_call(ixgbe_async,
_request,
&_response);
if (ret) {
LIBLCD_ERR("thc_ipc_call");
goto fail_ipc;
}
fipc_recv_msg_end(thc_channel_to_fipc(ixgbe_async),
_response);
/* restore pointers */
skb->head = skb_c->head;
skb->data = skb_c->data;
/* set LCD page's address */
frag->page.p = page;
printk("call returns rx_frag");
return;
fail_async:
fail_ipc:
return;
}
unsigned int eth_get_headlen(void *data,
unsigned int len)
{
int ret;
struct fipc_message *_request;
struct fipc_message *_response;
unsigned int func_ret;
ret = async_msg_blocking_send_start(ixgbe_async,
&_request);
if (ret) {
LIBLCD_ERR("failed to get a send slot");
goto fail_async;
}
async_msg_set_fn_type(_request,
ETH_GET_HEADLEN);
fipc_set_reg1(_request,
gpa_val(lcd_gva2gpa(__gva((unsigned long)data))));
fipc_set_reg2(_request,
len);
ret = thc_ipc_call(ixgbe_async,
_request,
&_response);
if (ret) {
LIBLCD_ERR("thc_ipc_call");
goto fail_ipc;
}
func_ret = fipc_get_reg1(_response);
fipc_recv_msg_end(thc_channel_to_fipc(ixgbe_async),
_response);
return func_ret;
fail_async:
fail_ipc:
return ret;
}
......@@ -215,6 +215,7 @@ static void ixgbe_service_task(struct work_struct *work);
/* global instance of adapter struct */
struct ixgbe_adapter *g_adapter = NULL;
struct net_device *g_netdev = NULL;
extern int _request_irq(unsigned int irq, irq_handler_t handler, unsigned long flags,
const char *name, void *dev);
......@@ -1648,11 +1649,10 @@ static bool ixgbe_alloc_mapped_page(struct ixgbe_ring *rx_ring,
*/
flags = GFP_ATOMIC | __GFP_NOWARN | __GFP_COLD | __GFP_COMP | __GFP_MEMALLOC;
page = lcd_alloc_pages(flags, ixgbe_rx_pg_order(rx_ring));
if (page) {
unsigned ord = ixgbe_rx_pg_order(rx_ring);
pg_count += ord ? (1 << ord) : 1;
LIBLCD_MSG("total rx pages %u", pg_count);
}
/* increment ref count for an allocated page. This is not being
* done by the liblcd helper functions.
*/
page_ref_inc(page);
#endif
if (unlikely(!page)) {
......@@ -1682,6 +1682,12 @@ static bool ixgbe_alloc_mapped_page(struct ixgbe_ring *rx_ring,
bi->page = page;
bi->page_offset = 0;
if (page) {
unsigned ord = ixgbe_rx_pg_order(rx_ring);
pg_count += ord ? (1 << ord) : 1;
LIBLCD_MSG("total rx pages %u", pg_count);
}
return true;
}
......@@ -1902,8 +1908,6 @@ static void ixgbe_pull_tail(struct ixgbe_ring *rx_ring,
#ifndef LCD_ISOLATE
va = skb_frag_address(frag);
#else
printk("frag %p | frag_page_addr %p | offset %x", frag->page.p,
lcd_page_address(frag->page.p), frag->page_offset);
va = lcd_page_address(skb_frag_page(frag)) + frag->page_offset;
#endif /* LCD_ISOLATE */
......@@ -2107,7 +2111,7 @@ static bool ixgbe_add_rx_frag(struct ixgbe_ring *rx_ring,
#if (PAGE_SIZE < 8192)
/* if we are only owner of page we can reuse it */
if (unlikely(page_count(page) != 1))
if (unlikely(page_ref_count(page) != 1))
return false;
/* flip page offset to other buffer */
......@@ -2968,7 +2972,7 @@ int ixgbe_poll(struct napi_struct *napi, int budget)
struct ixgbe_ring *ring;
int per_ring_budget, work_done = 0;
bool clean_complete = true;
static int once = 0;
#ifdef CONFIG_IXGBE_DCA
if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
ixgbe_update_dca(q_vector);
......@@ -2990,6 +2994,11 @@ int ixgbe_poll(struct napi_struct *napi, int budget)
else
per_ring_budget = budget;
if (!once) {
LIBLCD_MSG("%s, budget %d | perring %d | qvec rx %d", __func__,
budget, per_ring_budget, q_vector->rx.count);
once++;
}
ixgbe_for_each_ring(ring, q_vector->rx) {
int cleaned = ixgbe_clean_rx_irq(q_vector, ring,
per_ring_budget);
......@@ -3872,6 +3881,8 @@ void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter,
IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(reg_idx), rxdctl);
ixgbe_rx_desc_queue_enable(adapter, ring);
printk("%s, calling rx_alloc %d | nextto use %u", __func__,
ixgbe_desc_unused(ring), ring->next_to_use);
ixgbe_alloc_rx_buffers(ring, ixgbe_desc_unused(ring));
}
......@@ -8167,7 +8178,7 @@ static netdev_tx_t __ixgbe_xmit_frame(struct sk_buff *skb,
return ixgbe_xmit_frame_ring(skb, adapter, tx_ring);
}
static netdev_tx_t ixgbe_xmit_frame(struct sk_buff *skb,
netdev_tx_t ixgbe_xmit_frame(struct sk_buff *skb,
struct net_device *netdev)
{
return __ixgbe_xmit_frame(skb, netdev, NULL);
......@@ -9957,6 +9968,7 @@ skip_sriov:
if (err)
goto err_register;
g_netdev = netdev;
pci_set_drvdata(pdev, adapter);
/* power down the optics for 82599 SFP+ fiber */
......
......@@ -9,6 +9,7 @@
#include "../../ixgbe_common.h"
#include "../ixgbe_callee.h"
#include "../../rdtsc_helper.h"
#include <linux/hashtable.h>
#include <asm/cacheflush.h>
......@@ -19,14 +20,25 @@
#include <lcd_config/post_hook.h>
//#define TIMESTAMP
//#define LCD_MEASUREMENT
struct glue_cspace *c_cspace = NULL;
struct thc_channel *ixgbe_async;
struct cptr sync_ep;
extern struct cspace *klcd_cspace;
extern struct thc_channel *xmit_chnl;
extern struct thc_channel *xmit_irq_chnl;
struct timer_list service_timer;
struct napi_struct *napi_q0;
#define NUM_PACKETS (170000)
#define VMALLOC_SZ (NUM_PACKETS * sizeof(uint64_t))
uint64_t *times_ndo_xmit = NULL;
uint64_t *times_lcd = NULL;
static struct rtnl_link_stats64 g_stats;
/* This is the only device we strive for */
#define IXGBE_DEV_ID_82599_SFP_SF2 0x154D
......@@ -45,18 +57,24 @@ static DEFINE_HASHTABLE(cptr_table, CPTR_HASH_BITS);
struct pci_dev *g_pdev = NULL;
struct net_device *g_ndev = NULL;
struct kmem_cache *skb_c_cache = NULL;
DEFINE_SPINLOCK(hspin_lock);
static unsigned long pool_pfn_start, pool_pfn_end;
priv_pool_t *pool;
priv_pool_t *skbc_pool;
void skb_data_pool_init(void)
{
pool = priv_pool_init(SKB_DATA_POOL, 10, SKB_DATA_SIZE);
skbc_pool = priv_pool_init(SKB_CONTAINER_POOL, 10,
SKB_CONTAINER_SIZE * 2);
}
void skb_data_pool_free(void)
{
priv_pool_destroy(pool);
priv_pool_destroy(skbc_pool);
}
xmit_type_t check_skb_range(struct sk_buff *skb)
......@@ -88,6 +106,20 @@ int glue_ixgbe_init(void)
/* initialize our private pool */
skb_data_pool_init();
times_ndo_xmit = vzalloc(NUM_PACKETS * sizeof(uint64_t));
times_lcd = vzalloc(NUM_PACKETS * sizeof(uint64_t));
skb_c_cache = kmem_cache_create("skb_c_cache",
sizeof(struct sk_buff_container),
0,
SLAB_HWCACHE_ALIGN|SLAB_PANIC,
NULL);
if (!skb_c_cache) {
LIBLCD_ERR("Could not create skb container cache");
goto fail2;
}
return 0;
fail2:
glue_cap_exit();
......@@ -100,6 +132,14 @@ void glue_ixgbe_exit(void)
{
glue_cap_destroy(c_cspace);
glue_cap_exit();
if (skb_c_cache)
kmem_cache_destroy(skb_c_cache);
if (times_ndo_xmit)
vfree(times_ndo_xmit);
if (times_lcd)
vfree(times_lcd);
}
int glue_insert_skbuff(struct hlist_head *htable, struct sk_buff_container *skb_c)
......@@ -1158,10 +1198,9 @@ ndo_stop_trampoline(struct net_device *dev)
}
#if 0
int ndo_start_xmit_user(struct sk_buff *skb,
int ndo_start_xmit_dofin(struct sk_buff *skb,
struct net_device *dev,
struct trampoline_hidden_args *hidden_args,
struct thc_channel *async_chnl,
xmit_type_t xmit_type)
{
struct net_device_container *dev_container;
......@@ -1181,10 +1220,13 @@ int ndo_start_xmit_user(struct sk_buff *skb,
struct net_device_container,
net_device);
skb_c = kzalloc(sizeof(*skb_c), GFP_KERNEL);
skb_c = kmem_cache_alloc(skb_c_cache, GFP_KERNEL);
if (!skb_c)
if (!skb_c) {
LIBLCD_MSG("no memory");
goto fail_alloc;
}
skb_c->skb = skb;
glue_insert_skbuff(cptr_table, skb_c);
......@@ -1196,11 +1238,8 @@ int ndo_start_xmit_user(struct sk_buff *skb,
/* pad to 17 bytes, don't care the ret val */
skb_put_padto(skb, 17);
/* enter LCD mode to have cspace tree */
lcd_enter();
ret = async_msg_blocking_send_start(async_chnl, &_request);
ret = async_msg_blocking_send_start(hidden_args->async_chnl,
&_request);
if (ret) {
LIBLCD_ERR("failed to get a send slot");
goto fail_async;
......@@ -1219,8 +1258,7 @@ int ndo_start_xmit_user(struct sk_buff *skb,
switch (xmit_type) {
case VOLUNTEER_XMIT:
ret = thc_ipc_send_request(hidden_args->async_chnl,
ret = thc_ipc_send_request(async_chnl,
_request, &request_cookie);
if (ret) {
......@@ -1228,7 +1266,7 @@ int ndo_start_xmit_user(struct sk_buff *skb,
goto fail_ipc;
}
ret = grant_sync_ep(&sync_end, hidden_args->sync_ep);
//ret = grant_sync_ep(&sync_end, hidden_args->sync_ep);
ret = sync_setup_memory(skb, sizeof(struct sk_buff),
&skb_ord, &skb_cptr, &skb_off);
......@@ -1287,44 +1325,218 @@ int ndo_start_xmit_user(struct sk_buff *skb,
skb_lcd->head_data_off = skb->data - skb->head;
ret = thc_ipc_send_request(hidden_args->async_chnl,
ret = thc_ipc_send_request(async_chnl,
_request, &request_cookie);
if (ret) {
LIBLCD_ERR("thc_ipc_call");
goto fail_ipc;
}
break;