...
 
Commits (11)
......@@ -23,7 +23,7 @@ int lcd_arch_ept_walk(struct lcd_arch *lcd, gpa_t a, int create,
/**
* Set the guest physical => host physical mapping in the ept entry.
*/
void lcd_arch_ept_set(lcd_arch_epte_t *epte, hpa_t a);
void lcd_arch_ept_set(lcd_arch_epte_t *epte, gpa_t gpa, hpa_t hpa);
/**
* Read the host physical address stored in epte.
*/
......
......@@ -10,6 +10,7 @@
#include <lcd_domains/types.h>
#include <asm/lcd_domains/types.h>
#include <asm/lcd_domains/microkernel.h>
#include <liblcd/address_spaces.h>
#ifdef CONFIG_LCD_SINGLE_EPT
u64 g_vmcs_ptr;
......@@ -204,7 +205,7 @@ enum vmx_epte_mts {
*
* See Intel SDM V3 Figure 28-1 and 28.2.2.
*/
static void vmx_epte_set(lcd_arch_epte_t *epte, hpa_t a, int level)
static void vmx_epte_set(lcd_arch_epte_t *epte, gpa_t ga, hpa_t a, int level)
{
/*
* zero out epte, and set
......@@ -223,6 +224,25 @@ static void vmx_epte_set(lcd_arch_epte_t *epte, hpa_t a, int level)
* & section 28.2.5.2 of the Intel Software Developer
* Manual Vol 3 for effective memory type.
*/
/*
* XXX: To support ioremap, set the effective memory type to be
* uncacheable. According to Intel SDM 28.2.6.2, If IPAT
* (ignore PAT) is set, the memory type set in EPT (bits 5:3)
* would take effect.
* TODO: create new iommu_map api which would propagate this
* setting.
*/
if ((gpa_val(ga) >= gpa_val(LCD_IOREMAP_GP_ADDR)) &&
(gpa_val(ga) <= gpa_val(gpa_add(LCD_IOREMAP_GP_ADDR,
LCD_IOREMAP_REGION_SIZE)))) {
*epte |= VMX_EPTE_MT_UC << VMX_EPT_MT_EPTE_SHIFT;
*epte |= VMX_EPT_IPAT_BIT;
//printk("%s, set (epte:%lx) UC to gpa:%lx hpa: %lx\n", __func__, *epte, gpa_val(ga));
} else {
*epte |= VMX_EPTE_MT_WB << VMX_EPT_MT_EPTE_SHIFT;
*epte &= ~VMX_EPT_IPAT_BIT;
}
*epte |= VMX_EPTE_MT_WB << VMX_EPT_MT_EPTE_SHIFT;
*epte &= ~VMX_EPT_IPAT_BIT;
}
......@@ -260,7 +280,7 @@ int lcd_arch_ept_walk(struct lcd_arch *lcd, gpa_t a, int create,
return -ENOMEM;
}
memset(hva2va(page), 0, PAGE_SIZE);
vmx_epte_set(&dir[idx], hva2hpa(page), i);
vmx_epte_set(&dir[idx], a, hva2hpa(page), i);
}
dir = (lcd_arch_epte_t *) hva2va(vmx_epte_hva(dir[idx]));
......@@ -273,9 +293,9 @@ int lcd_arch_ept_walk(struct lcd_arch *lcd, gpa_t a, int create,
return 0;
}
void lcd_arch_ept_set(lcd_arch_epte_t *epte, hpa_t a)
void lcd_arch_ept_set(lcd_arch_epte_t *epte, gpa_t gpa, hpa_t hpa)
{
vmx_epte_set(epte, a, 3);
vmx_epte_set(epte, gpa, hpa, 3);
}
int lcd_arch_ept_unset(lcd_arch_epte_t *epte)
......@@ -302,9 +322,12 @@ int lcd_arch_ept_map(struct lcd_arch *lcd, gpa_t ga, hpa_t ha,
mutex_lock(&g_ept_lock);
#endif
ret = lcd_arch_ept_walk(lcd, ga, create, &ept_entry);
if (ret)
if (ret) {
#ifdef CONFIG_LCD_SINGLE_EPT
mutex_unlock(&g_ept_lock);
#endif
return ret;
}
/*
* Check if guest physical address already mapped
*/
......@@ -321,7 +344,7 @@ int lcd_arch_ept_map(struct lcd_arch *lcd, gpa_t ga, hpa_t ha,
/*
* Map the guest physical addr to the host physical addr.
*/
lcd_arch_ept_set(ept_entry, ha);
lcd_arch_ept_set(ept_entry, ga, ha);
#ifdef CONFIG_LCD_SINGLE_EPT
mutex_unlock(&g_ept_lock);
......@@ -370,8 +393,12 @@ int lcd_arch_ept_unmap(struct lcd_arch *lcd, gpa_t a)
mutex_lock(&g_ept_lock);
#endif
ret = lcd_arch_ept_walk(lcd, a, 0, &ept_entry);
if (ret)
if (ret) {
#ifdef CONFIG_LCD_SINGLE_EPT
mutex_unlock(&g_ept_lock);
#endif
return ret;
}
/*
* Unset
......@@ -396,8 +423,13 @@ int lcd_arch_ept_unmap2(struct lcd_arch *lcd, gpa_t a, hpa_t *hpa_out)
mutex_lock(&g_ept_lock);
#endif
ret = lcd_arch_ept_walk(lcd, a, 0, &ept_entry);
if (ret)
if (ret) {
#ifdef CONFIG_LCD_SINGLE_EPT
mutex_unlock(&g_ept_lock);
#endif
return ret;
}
/*
* Extract hpa
*/
......@@ -446,8 +478,13 @@ int lcd_arch_ept_gpa_to_hpa(struct lcd_arch *lcd, gpa_t ga, hpa_t *ha_out, bool
#endif
ret = lcd_arch_ept_walk(lcd, ga, 0, &ept_entry);
if (ret)
if (ret) {
#ifdef CONFIG_LCD_SINGLE_EPT
mutex_unlock(&g_ept_lock);
#endif
return ret;
}
/*
* Confirm the entry is present
......
......@@ -697,7 +697,152 @@ int lcd_create_module_lcd(char *mdir, char *mname, cptr_t *lcd_out,
* Initialize empty LCD
*/
create_lcd:
ret = lcd_create(&lcd, lcd_id);
ret = lcd_create(&lcd, lcd_id, 1);
if (ret) {
LIBLCD_ERR("error creating empty LCD");
goto fail4;
}
/*
* Set up address spaces
*/
if (is_child) {
ret = setup_phys_addr_space_child_lcd(lcd, ctx, lcd_id);
} else {
ret = setup_addr_spaces(lcd, ctx, m_init_link_addr,
m_core_link_addr, m_init_size, m_core_size,
lcd_id);
}
if (ret) {
LIBLCD_ERR("error setting up address spaces");
goto fail5;
}
/*
* Configure initial control registers, etc. for LCD
*/
*((unsigned long *)ctx->gs_page) = gva_val(LCD_BOOTSTRAP_PAGES_GV_ADDR_CHILD(lcd_id));
ret = lcd_config_registers(lcd, m_init_func_addr,
/* implicity put a null return address and
* frame address */
gva_add(LCD_STACK_GV_ADDR_CHILD(lcd_id),
LCD_STACK_SIZE - sizeof(void *)),
LCD_BOOTSTRAP_PAGE_TABLES_GP_ADDR,
LCD_UTCB_GP_ADDR_CHILD(lcd_id),
LCD_ARCH_GS_GV_BASE_CHILD(lcd_id));
if (ret) {
LIBLCD_ERR("error configuring LCDs registers");
goto fail6;
}
#ifndef LCD_ISOLATE
/*
* For non-isolated code running this, we also remember where
* the struct module copy is located so we can do stack traces.
*/
ret = set_struct_module(lcd, ctx->m_core_bits,
m_struct_module_core_offset, is_child);
if (ret) {
LIBLCD_ERR("error setting struct module hva");
goto fail7;
}
#endif
/*
* Return context and lcd
*/
LIBLCD_MSG("%s, %s LCD, ctx %p, lcd %lu", __func__, is_child ? "child" : "parent",
ctx, lcd.cptr);
*ctx_out = ctx;
*lcd_out = lcd;
/*
* Save parent ctx for the future
*/
if (!is_child) {
parent_ctx = ctx;
}
#ifdef CONFIG_LCD_SINGLE_EPT
lcd_to_boot_info(ctx)->lcd_id = lcd_id;
#endif
return 0;
#ifndef LCD_ISOLATE
fail7:
#endif
fail6:
fail5:
lcd_cap_delete(lcd);
fail4:
lcd_release_module(ctx->m_init_bits, ctx->m_core_bits);
fail3:
fail2:
destroy_create_ctx(ctx);
fail1:
return ret;
}
int __lcd_create_module_lcd(char *mdir, char *mname, cptr_t *lcd_out,
struct lcd_create_ctx **ctx_out, int lcd_id, int num_lcds)
{
int ret;
bool is_child = (lcd_id > 0);
cptr_t m_init_cptr, m_core_cptr;
static gva_t m_init_link_addr, m_core_link_addr, m_init_func_addr;
static unsigned long m_init_size, m_core_size;
static unsigned long m_struct_module_core_offset;
struct lcd_create_ctx *ctx;
static struct lcd_create_ctx *parent_ctx;
cptr_t lcd;
/*
* Initialize create ctx
*/
ret = init_create_ctx(&ctx, mname);
if (ret) {
LIBLCD_ERR("error creating ctx");
goto fail1;
}
/*
* Alloc boot pages, stack pages, etc. for LCD
*/
ret = get_pages_for_lcd(ctx, is_child);
if (ret) {
LIBLCD_ERR("error alloc'ing boot, stack pages for LCD");
goto fail2;
}
/*
* if we are creating a child LCD, skip loading the module as it is
* already loaded in parent LCD. In child LCD, we just map the same set
* of pages to the EPT.
*/
if (is_child) {
ctx->m_init_bits = parent_ctx->m_init_bits;
ctx->m_core_bits = parent_ctx->m_core_bits;
goto create_lcd;
}
/*
* Load kernel module into caller's address space
*/
ret = lcd_load_module(mdir, mname,
&ctx->m_init_bits, &ctx->m_core_bits,
&m_init_cptr, &m_core_cptr,
&m_init_link_addr, &m_core_link_addr,
&m_init_size, &m_core_size,
&m_init_func_addr,
&m_struct_module_core_offset);
if (ret) {
LIBLCD_ERR("error loading kernel module");
goto fail3;
}
/*
* At this point, we have all of the data that will go in the LCD
* (the microkernel has the UTCB page)
*
* Initialize empty LCD
*/
create_lcd:
ret = lcd_create(&lcd, lcd_id, num_lcds);
if (ret) {
LIBLCD_ERR("error creating empty LCD");
goto fail4;
......@@ -796,8 +941,8 @@ int lcd_create_module_lcds(char *mdir, char *mname, cptr_t *lcd_out,
*/
for (i = 0; i < num_child; i++) {
LIBLCD_MSG("Creating LCD modules %d", i);
ret = lcd_create_module_lcd(mdir, mname, &lcd_out[i],
&ctx_out[i], i);
ret = __lcd_create_module_lcd(mdir, mname, &lcd_out[i],
&ctx_out[i], i, num_child);
if (ret) {
LIBLCD_ERR("error creating empty LCD");
goto fail2;
......
......@@ -288,14 +288,14 @@ int __lcd_create_no_vm_no_thread(struct lcd **out);
* which we need a new kthread to run the LCD's code, but we don't want
* the code to run inside an isolated VM.
*/
int __lcd_create_no_vm(struct lcd **out, const char *name, int lcd_id);
int __lcd_create_no_vm(struct lcd **out, const char *name, int lcd_id, int num_lcds);
/**
* __lcd_create -- Create an empty isolated LCD
* @caller: the LCD doing the creating
* @slot: the slot in the @caller's cspace where the LCD capability should
* be stored
*/
int __lcd_create(struct lcd *caller, cptr_t slot, int lcd_id);
int __lcd_create(struct lcd *caller, cptr_t slot, int lcd_id, int num_lcds);
/**
* __lcd_create_klcd -- Create an empty non-isolated LCD (kLCD)
* @caller: the LCD doing the creating
......
......@@ -38,7 +38,7 @@
* Creates a bare LCD with an empty cspace and address space (nothing is
* mapped it - except the dedicated page for the LCD's UTCB - see sync_ipc.h).
*/
int lcd_create(cptr_t *lcd, int lcd_id);
int lcd_create(cptr_t *lcd, int lcd_id, int num_lcds);
/**
* lcd_create_klcd -- Create an LCD that runs in the non-isolated host
......
......@@ -19,7 +19,7 @@
/* UTCB ---------------------------------------- */
#define LCD_NUM_REGS 8
#define LCD_NUM_REGS 42
struct lcd_utcb {
/**
* mr registers are general purpose 64-bit registers. You can
......@@ -63,6 +63,17 @@ static inline void lcd_set_cr##idx(cptr_t val) \
{ \
lcd_get_utcb()->cr[idx] = val; \
}
static inline void lcd_set_cr(int idx, cptr_t val)
{
lcd_get_utcb()->cr[idx] = val;
}
static inline cptr_t lcd_get_cr(int idx)
{
return lcd_get_utcb()->cr[idx];
}
LCD_MK_REG_ACCESS(0)
LCD_MK_REG_ACCESS(1)
LCD_MK_REG_ACCESS(2)
......@@ -72,6 +83,24 @@ LCD_MK_REG_ACCESS(5)
LCD_MK_REG_ACCESS(6)
LCD_MK_REG_ACCESS(7)
LCD_MK_REG_ACCESS(8)
LCD_MK_REG_ACCESS(9)
LCD_MK_REG_ACCESS(10)
LCD_MK_REG_ACCESS(11)
LCD_MK_REG_ACCESS(12)
LCD_MK_REG_ACCESS(13)
LCD_MK_REG_ACCESS(14)
LCD_MK_REG_ACCESS(15)
LCD_MK_REG_ACCESS(16)
LCD_MK_REG_ACCESS(17)
LCD_MK_REG_ACCESS(18)
LCD_MK_REG_ACCESS(19)
LCD_MK_REG_ACCESS(20)
LCD_MK_REG_ACCESS(21)
LCD_MK_REG_ACCESS(22)
LCD_MK_REG_ACCESS(23)
LCD_MK_REG_ACCESS(24)
/* SYNCHRONOUS IPC ---------------------------------------- */
/**
......
......@@ -8,7 +8,7 @@
#include <liblcd/liblcd.h>
#include <lcd_domains/microkernel.h>
int lcd_create(cptr_t *lcd, int lcd_id)
int lcd_create(cptr_t *lcd, int lcd_id, int num_lcds)
{
cptr_t slot;
int ret;
......@@ -23,7 +23,7 @@ int lcd_create(cptr_t *lcd, int lcd_id)
/*
* Make LCD
*/
ret = __lcd_create(current->lcd, slot, lcd_id);
ret = __lcd_create(current->lcd, slot, lcd_id, num_lcds);
if (ret) {
LIBLCD_ERR("lcd create failed");
goto fail2;
......
......@@ -13,7 +13,7 @@
#include <lcd_config/post_hook.h>
int lcd_create(cptr_t *lcd, int lcd_id)
int lcd_create(cptr_t *lcd, int lcd_id, int num_lcds)
{
cptr_t slot;
int ret;
......
......@@ -169,7 +169,7 @@ fail1:
return ret;
}
int __lcd_create_no_vm(struct lcd **out, const char *name, int lcd_id)
int __lcd_create_no_vm(struct lcd **out, const char *name, int lcd_id, int num_lcds)
{
struct lcd *lcd;
int ret;
......@@ -199,6 +199,59 @@ int __lcd_create_no_vm(struct lcd **out, const char *name, int lcd_id)
LCD_ERR("failed to create kthread");
goto fail2;
}
/* do bind only for LCDs */
if (!strncmp(name, "lcd/", strlen("lcd/"))) {
switch (num_lcds) {
case 1:
kthread_bind(lcd->kthread, 8);
break;
case 2:
/* lcds=2, each on one numa node */
if (lcd_id == 0)
kthread_bind(lcd->kthread, 8);
else
kthread_bind(lcd->kthread, 19);
break;
case 4:
switch (lcd_id) {
case 0:
kthread_bind(lcd->kthread, 7);
break;
case 1:
kthread_bind(lcd->kthread, 8);
break;
case 2:
kthread_bind(lcd->kthread, 18);
break;
case 3:
kthread_bind(lcd->kthread, 19);
break;
}
break;
case 6:
switch (lcd_id) {
case 0:
kthread_bind(lcd->kthread, 6);
break;
case 1:
kthread_bind(lcd->kthread, 7);
break;
case 2:
kthread_bind(lcd->kthread, 8);
break;
case 3:
kthread_bind(lcd->kthread, 17);
break;
case 4:
kthread_bind(lcd->kthread, 18);
break;
case 5:
kthread_bind(lcd->kthread, 19);
break;
}
break;
}
}
/*
* Bump reference count on kthread
*/
......@@ -219,7 +272,7 @@ fail1:
}
#define LCD_KTHREAD_NAME_SZ 32
int __lcd_create(struct lcd *caller, cptr_t slot, int lcd_id)
int __lcd_create(struct lcd *caller, cptr_t slot, int lcd_id, int num_lcds)
{
struct lcd *lcd;
int ret = 0;
......@@ -236,7 +289,7 @@ int __lcd_create(struct lcd *caller, cptr_t slot, int lcd_id)
snprintf(lcd_name, LCD_KTHREAD_NAME_SZ, "lcd/%d", lcd_id);
ret = __lcd_create_no_vm(&lcd, lcd_name, lcd_id);
ret = __lcd_create_no_vm(&lcd, lcd_name, lcd_id, num_lcds);
if (ret) {
LCD_ERR("lcd create");
goto fail1;
......@@ -280,7 +333,7 @@ int __lcd_create_klcd(struct lcd *caller, cptr_t slot)
/*
* Basic init of lcd
*/
ret = __lcd_create_no_vm(&lcd, "klcd", 0);
ret = __lcd_create_no_vm(&lcd, "klcd", 0, 0);
if (ret) {
LCD_ERR("lcd create");
goto fail1;
......
......@@ -217,6 +217,10 @@ void __lcd_mem_itree_exit(void)
* They should be empty if all LCDs (isolated and non-isolated)
* were torn down properly.
*/
BUG_ON(!RB_EMPTY_ROOT(&lcd_physical_mem_itree.root));
BUG_ON(!RB_EMPTY_ROOT(&lcd_vmalloc_mem_itree.root));
//BUG_ON(!RB_EMPTY_ROOT(&lcd_physical_mem_itree.root));
if (!RB_EMPTY_ROOT(&lcd_physical_mem_itree.root))
printk("%s, possible memory leak, check lcd_physical_mem_itree.root\n", __func__);
//BUG_ON(!RB_EMPTY_ROOT(&lcd_vmalloc_mem_itree.root));
if (!RB_EMPTY_ROOT(&lcd_vmalloc_mem_itree.root))
printk("%s, possible memory leak, check lcd_vmalloc_mem_itree.root\n", __func__);
}
......@@ -38,7 +38,7 @@ static int handle_syscall_create(struct lcd *lcd)
/*
* Do create
*/
return __lcd_create(lcd, lcd_slot, 0);
return __lcd_create(lcd, lcd_slot, 0, 0);
}
static int handle_syscall_cap_grant(struct lcd *lcd)
......
subdir-ccflags-y += -DNUM_LCDS="2"
subdir-ccflags-y += -DNUM_LCDS="1"
obj-$(LCD_CONFIG_BUILD_IXGBE_BOOT) += boot/
......
......@@ -142,6 +142,8 @@ static int boot_main(void)
goto fail8;
}
msleep_interruptible(3000);
LIBLCD_MSG("starting ixgbe ethernet...");
for (i = 0; i < num_lcds; i++) {
......@@ -151,6 +153,7 @@ static int boot_main(void)
LIBLCD_ERR("failed to start ixgbe lcd");
goto fail9;
}
msleep_interruptible(4000);
}
#ifdef BOOT_THREAD
// return
......
......@@ -24,10 +24,11 @@
#define NAPI_CONSUME_SEND_ONLY
#define LOCAL_SKB
#define NAPI_RX_SEND_ONLY
//#define SENDER_DISPATCH_LOOP
#define SENDER_DISPATCH_LOOP
#define CONFIG_PREALLOC_XMIT_CHANNELS
enum dispatch_t {
__PCI_REGISTER_DRIVER,
__PCI_REGISTER_DRIVER = 1,
PCI_UNREGISTER_DRIVER,
ALLOC_ETHERDEV_MQS,
REGISTER_NETDEV,
......@@ -159,6 +160,26 @@ struct skbuff_members {
#define C(x) skb_lcd->x = skb->x
#define P(x) skb->x = skb_lcd->x
#if NUM_LCDS == 1
/* total LCD cores = 2 (lcds=1,klcd=1), free cores = 18 */
#define MAX_CHANNELS_PER_LCD 18
#define NUM_THREADS_ON_NODE0 8
#elif NUM_LCDS == 2
/* total LCD cores = 3 (lcds=2,klcd=1), free cores = 17 */
#define MAX_CHANNELS_PER_LCD 9
#define NUM_THREADS_ON_NODE0 8
#elif NUM_LCDS == 4
/* total LCD cores = 5 (lcds=4,klcd=1), free cores = 15 */
#define MAX_CHANNELS_PER_LCD 4
#define NUM_THREADS_ON_NODE0 7
#elif NUM_LCDS == 6
/* total LCD cores = 7 (lcds=6,klcd=1), free cores = 13 */
#define MAX_CHANNELS_PER_LCD 3
#define NUM_THREADS_ON_NODE0 6
#endif
#define MAX_CHNL_PAIRS MAX_CHANNELS_PER_LCD
/* CSPACES ------------------------------------------------------------ */
int glue_cap_init(void);
......@@ -233,7 +254,6 @@ async_msg_blocking_recv_start(struct thc_channel *chnl,
struct fipc_message** out)
{
int ret;
static int count = 0;
for (;;) {
/* Poll until we get a message or error */
......@@ -242,8 +262,6 @@ async_msg_blocking_recv_start(struct thc_channel *chnl,
if ( !ret || ret != -EWOULDBLOCK )
return ret;
cpu_relax();
if (count++ % 512 == 0)
cond_resched();
}
}
#endif /* __IXGBE_COMMON_H__ */
......@@ -3006,8 +3006,11 @@ int ixgbe_poll(struct napi_struct *napi, int budget)
}
}
ixgbe_for_each_ring(ring, q_vector->tx) {
if (!ixgbe_clean_tx_irq(q_vector, ring, budget))
int ret;
if (!(ret = ixgbe_clean_tx_irq(q_vector, ring, budget))) {
clean_complete = false;
printk("%s, cleaning tx ring: %p, ret = %d", __func__, ring, ret);
}
}
/* Exit if we are called by netpoll or busy polling is active */
......
......@@ -26,7 +26,9 @@ extern int create_async_channel(int lcd_id);
unsigned long loops_per_jiffy;
bool poll_start = false;
extern int __ixgbe_poll(int lcd_id);
#define LCD_ROUNDROBIN
/* LOOP ---------------------------------------- */
static void main_and_loop(void)
......
......@@ -13,6 +13,19 @@
/* COMPILER: This is always included after all includes. */
#include <lcd_config/post_hook.h>
#define REGISTER_FREQ 50
extern struct trampoline_hidden_args *g_ndo_start_xmit_hidden_args;
extern struct timer_list service_timer;
extern struct glue_cspace *c_cspace;
atomic_t num_registered = ATOMIC_INIT(0);
static LIST_HEAD(net_infos);
struct thc_channel *xmit_chnl;
struct thc_channel *xmit_chnl2;
struct thc_channel *xmit_irq_chnl;
/* LOOP ------------------------------------------------------------ */
struct net_info {
struct thc_channel *chnl;
......@@ -20,22 +33,15 @@ struct net_info {
cptr_t sync_endpoint;
struct list_head list;
};
static LIST_HEAD(net_infos);
struct thc_channel *xmit_chnl;
struct thc_channel *xmit_chnl2;
struct thc_channel *xmit_irq_chnl;
struct thc_channel *klcd_chnl;
extern struct trampoline_hidden_args *g_ndo_start_xmit_hidden_args;
extern struct lcd_channels lcds[NUM_LCDS];
extern int setup_async_net_ring_channel(cptr_t tx, cptr_t rx,
int setup_async_net_ring_channel(cptr_t tx, cptr_t rx,
struct thc_channel **chnl_out);
extern void destroy_async_net_ring_channel(struct thc_channel *chnl);
extern int ixgbe_trigger_dump(struct thc_channel *_channel);
extern int ixgbe_service_event_sched(struct thc_channel *_channel);
extern int trigger_exit_to_lcd(struct thc_channel *_channel, enum dispatch_t);
extern struct timer_list service_timer;
extern struct glue_cspace *c_cspace;
void destroy_async_net_ring_channel(struct thc_channel *chnl);
int ixgbe_trigger_dump(struct thc_channel *_channel);
int ixgbe_service_event_sched(struct thc_channel *_channel);
int trigger_exit_to_lcd(struct thc_channel *_channel, enum dispatch_t);
int register_parent(int lcd_id);
int register_child(int lcd_id);
/* mechanism for unloading LCD gracefully */
static bool unload_lcd =0;
......@@ -159,10 +165,8 @@ static int do_one_register(cptr_t register_chnl)
int ret;
cptr_t sync_endpoint, tx, rx;
cptr_t tx_xmit, rx_xmit;
struct thc_channel *chnl;
struct net_info *net_info;
cptr_t _tx[MAX_CHNL_PAIRS], _rx[MAX_CHNL_PAIRS];
int i, j;
int lcd_id;
/*
......@@ -193,7 +197,20 @@ static int do_one_register(cptr_t register_chnl)
LIBLCD_ERR("cptr alloc failed");
goto fail3;
}
for (i = 0; i < MAX_CHNL_PAIRS; i++) {
ret = lcd_cptr_alloc(&_tx[i]);
if (ret) {
LIBLCD_ERR("cptr alloc failed");
goto fail3;
}
ret = lcd_cptr_alloc(&_rx[i]);
if (ret) {
LIBLCD_ERR("cptr alloc failed");
goto fail3;
}
}
/*
* Set up regs and poll
*/
......@@ -203,87 +220,41 @@ static int do_one_register(cptr_t register_chnl)
lcd_set_cr3(tx_xmit);
lcd_set_cr4(rx_xmit);
for (i = 0, j = 5; i < MAX_CHNL_PAIRS && j < LCD_NUM_REGS; i++) {
lcd_set_cr(j++, _tx[i]);
lcd_set_cr(j++, _rx[i]);
}
ret = lcd_sync_poll_recv(register_chnl);
if (ret) {
if (ret == -EWOULDBLOCK)
ret = 0;
goto free_cptrs;
}
#ifdef SYNC_TX
/*
* Dispatch to register handler
*/
ret = dispatch_sync_loop();
if (ret)
return ret; /* dispatch fn is responsible for cptr cleanup */
#endif
lcd_id = lcd_r1();
LIBLCD_MSG("settingup net_ring channel for LCD %d", lcd_id);
atomic_inc(&num_registered);
/*
* Set up async ring channel
*/
ret = setup_async_net_ring_channel(tx, rx, &chnl);
if (ret) {
LIBLCD_ERR("error setting up ring channel");
goto fail6;
}
klcd_chnl = chnl;
/* Populate LCD channel information */
lcds[lcd_id].lcd_async_chnl = chnl;
lcds[lcd_id].lcd_sync_end = sync_endpoint;
/* Only parent LCD creates extra channels */
if (lcd_id == 0) {
LIBLCD_MSG("settingup xmit channel for LCD %d", lcd_id);
/*
* Set up async ring channel
*/
ret = setup_async_net_ring_channel(tx_xmit, rx_xmit,
&xmit_chnl);
if (ret) {
LIBLCD_ERR("error setting up ring channel");
goto fail6;
}
register_parent(lcd_id);
} else {
register_child(lcd_id);
}
/*
* Add to dispatch loop
*/
LIBLCD_MSG("Adding to fsinfo cspace %lu | chnl %p | sync_ep %lu", c_cspace, chnl, sync_endpoint);
net_info = add_net(chnl, c_cspace, sync_endpoint);
if (!net_info) {
LIBLCD_ERR("error adding to dispatch loop");
goto fail7;
}
LIBLCD_MSG("Returning from %s", __func__);
lcd_set_cr0(CAP_CPTR_NULL);
lcd_set_cr1(CAP_CPTR_NULL);
lcd_set_cr2(CAP_CPTR_NULL);
lcd_set_cr3(CAP_CPTR_NULL);
lcd_set_cr4(CAP_CPTR_NULL);
if (lcd_sync_reply())
LIBLCD_ERR("Error reply");
return 0;
fail6:
destroy_async_net_ring_channel(chnl);
fail7:
free_cptrs:
lcd_set_cr0(CAP_CPTR_NULL);
lcd_set_cr1(CAP_CPTR_NULL);
lcd_set_cr2(CAP_CPTR_NULL);
lcd_set_cr3(CAP_CPTR_NULL);
lcd_set_cr4(CAP_CPTR_NULL);
for (i = 0; i < LCD_NUM_REGS; i++)
lcd_set_cr(i, CAP_CPTR_NULL);
lcd_cptr_free(sync_endpoint);
fail3:
for (i = 0; i < MAX_CHNL_PAIRS; i++) {
lcd_cptr_free(_tx[i]);
lcd_cptr_free(_rx[i]);
}
lcd_cptr_free(tx);
lcd_cptr_free(tx_xmit);
fail2:
......@@ -292,7 +263,6 @@ fail2:
fail1:
return ret;
}
#define REGISTER_FREQ 50
static void loop(cptr_t register_chnl)
{
......@@ -304,18 +274,19 @@ static void loop(cptr_t register_chnl)
DO_FINISH(
while (!stop) {
if (jiffies >= tics) {
/*
* Listen for a register call
*/
ret = do_one_register(register_chnl);
if (ret) {
LIBLCD_ERR("register error");
break;
if (atomic_read(&num_registered) != NUM_LCDS) {
if (jiffies >= tics) {
/*
* Listen for a register call
*/
ret = do_one_register(register_chnl);
if (ret) {
LIBLCD_ERR("register error");
break;
}
tics = jiffies + REGISTER_FREQ;
continue;
}
tics = jiffies + REGISTER_FREQ;
continue;
}
if (stop)
break;
......
subdir-ccflags-y += -DNUM_LCDS="2"
subdir-ccflags-y += -DNUM_LCDS="4"
obj-$(LCD_CONFIG_BUILD_NULLNET_BOOT) += boot/
......
......@@ -24,6 +24,10 @@ cptr_t net_chnl;
cptr_t net_chnl_domain_cptr, dummy_chnl_domain_cptr;
cptr_t *dummy_chnl_domain_cptrs;
static int bind_cpu = 2;
module_param(bind_cpu, int, 0);
MODULE_PARM_DESC(bind_cpu, "CPU to bind to");
static int num_lcds = NUM_LCDS;
module_param(num_lcds, int, 0);
MODULE_PARM_DESC(num_lcds, "Number of LCDs to launch");
......@@ -234,8 +238,10 @@ static int boot_init(void)
boot_task = kthread_create(boot_lcd_thread, NULL, "boot_lcd_thread");
if (!IS_ERR(boot_task))
if (!IS_ERR(boot_task)) {
kthread_bind(boot_task, bind_cpu);
wake_up_process(boot_task);
}
return 0;
}
......
......@@ -102,9 +102,14 @@ netdev_tx_t dummy_xmit(struct sk_buff *skb, struct net_device *dev)
dstats->tx_bytes += skb->len;
u64_stats_update_end(&dstats->syncp);
#else
/* XXX: Touching global variable brings down the bandwidth
* Do not do it for now
*/
#if 0
g_dstats.tx_packets++;
g_dstats.tx_bytes += skb->len;
#endif
#endif
#ifdef SENDER_DISPATCH_LOOP
dev_kfree_skb(skb);
......
......@@ -35,6 +35,11 @@ extern u64 tdiff_disp;
#define CPTR_HASH_BITS 5
#ifdef CONFIG_PREALLOC_XMIT_CHANNELS
int prep_xmit_channels_lcd(void);
void prep_xmit_channels_clean_lcd(void);
#endif
static DEFINE_HASHTABLE(cptr_table, CPTR_HASH_BITS);
struct lcd_sk_buff_container {
......@@ -232,6 +237,117 @@ fail1:
}
#endif
static int setup_async_channel_on_node(int node_id, cptr_t *buf1_cptr_out, cptr_t *buf2_cptr_out,
struct thc_channel **chnl_out)
{
int ret;
cptr_t buf1_cptr, buf2_cptr;
gva_t buf1_addr, buf2_addr;
struct fipc_ring_channel *fchnl;
struct thc_channel *chnl;
unsigned int pg_order = PMFS_ASYNC_RPC_BUFFER_ORDER - PAGE_SHIFT;
LIBLCD_MSG("%s\n",__func__);
/*
* Allocate buffers
*
* (We use the lower level alloc. If we used the heap, even though
* we may alloc only 1 - 2 pages, we would end up sharing around
* 4 MB chunks of memory, since the heap uses coarse microkernel
* allocations.)
*/
printk("%s, allocate pages for lcd:%d on node: %d\n", __func__, current_lcd_id,
node_id);
ret = _lcd_alloc_pages_exact_node(node_id, GFP_KERNEL, pg_order, &buf1_cptr);
if (ret) {
LIBLCD_ERR("buf1 alloc");
goto fail1;
}
ret = _lcd_alloc_pages_exact_node(node_id, GFP_KERNEL, pg_order, &buf2_cptr);
if (ret) {
LIBLCD_ERR("buf2 alloc");
goto fail2;
}
/*
* Map them somewhere
*/
ret = lcd_map_virt(buf1_cptr, pg_order, &buf1_addr);
if (ret) {
LIBLCD_ERR("error mapping buf1");
goto fail3;
}
ret = lcd_map_virt(buf2_cptr, pg_order, &buf2_addr);
if (ret) {
LIBLCD_ERR("error mapping buf2");
goto fail4;
}
/*
* Prep buffers for rpc
*/
ret = fipc_prep_buffers(PMFS_ASYNC_RPC_BUFFER_ORDER,
(void *)gva_val(buf1_addr),
(void *)gva_val(buf2_addr));
if (ret) {
LIBLCD_ERR("prep buffers");
goto fail5;
}
LIBLCD_MSG("==> Prep buffers");
/*
* Alloc and init channel header
*/
fchnl = kmalloc(sizeof(*fchnl), GFP_KERNEL);
if (!fchnl) {
ret = -ENOMEM;
LIBLCD_ERR("chnl alloc");
goto fail6;
}
ret = fipc_ring_channel_init(fchnl, PMFS_ASYNC_RPC_BUFFER_ORDER,
(void *)gva_val(buf1_addr),
(void *)gva_val(buf2_addr));
if (ret) {
LIBLCD_ERR("ring chnl init");
goto fail7;
}
/*
* Install async channel in async dispatch loop
*/
chnl = kzalloc(sizeof(*chnl), GFP_KERNEL);
if (!chnl) {
ret = -ENOMEM;
LIBLCD_ERR("alloc failed");
goto fail8;
}
ret = thc_channel_init(chnl, fchnl);
if (ret) {
LIBLCD_ERR("error init'ing async channel group item");
goto fail9;
}
*buf1_cptr_out = buf1_cptr;
*buf2_cptr_out = buf2_cptr;
*chnl_out = chnl;
return 0;
fail9:
kfree(chnl);
fail8:
fail7:
kfree(fchnl);
fail6:
fail5:
lcd_unmap_virt(buf1_addr, pg_order);
fail4:
lcd_unmap_virt(buf1_addr, pg_order);
fail3:
lcd_cap_delete(buf2_cptr);
fail2:
lcd_cap_delete(buf1_cptr);
fail1:
return ret;
}
static int setup_async_channel(cptr_t *buf1_cptr_out, cptr_t *buf2_cptr_out,
struct thc_channel **chnl_out)
{
......@@ -427,6 +543,11 @@ int create_async_channel(void)
// conveys the LCD id the call is coming from
lcd_set_r1(current_lcd_id);
#ifdef CONFIG_PREALLOC_XMIT_CHANNELS
prep_xmit_channels_lcd();
#endif
printk("%s, lcd_sync_call for lcd:%d\n", __func__, current_lcd_id);
ret = lcd_sync_call(nullnet_register_channels[current_lcd_id]);
/*
......@@ -438,6 +559,10 @@ int create_async_channel(void)
lcd_set_cr3(CAP_CPTR_NULL);
lcd_set_cr4(CAP_CPTR_NULL);
#ifdef CONFIG_PREALLOC_XMIT_CHANNELS
prep_xmit_channels_clean_lcd();
#endif
if (ret) {
LIBLCD_ERR("lcd_call");
goto fail3;
......@@ -467,6 +592,34 @@ fail1:
struct thc_channel_group_item *ptrs[NUM_LCDS][32];
static int idx[NUM_LCDS] = {0};
int create_one_async_channel_on_node(int node_id, struct thc_channel **chnl, cptr_t *tx, cptr_t *rx)
{
int ret;
struct thc_channel_group_item *xmit_ch_item;
ret = setup_async_channel_on_node(node_id, tx, rx, chnl);
if (ret) {
LIBLCD_ERR("async xmit chnl setup failed");
return -1;
}
xmit_ch_item = kzalloc(sizeof(*xmit_ch_item), GFP_KERNEL);
thc_channel_group_item_init(xmit_ch_item, *chnl, NULL);
xmit_ch_item->xmit_channel = true;
thc_channel_group_item_add(&ch_grp[current_lcd_id], xmit_ch_item);
printk("%s:%d adding chnl: %p to group: %p", __func__, current_lcd_id,
xmit_ch_item->channel, &ch_grp[current_lcd_id]);
ptrs[current_lcd_id][idx[current_lcd_id]++%32] = xmit_ch_item;
return 0;
}
int create_one_async_channel(struct thc_channel **chnl, cptr_t *tx, cptr_t *rx)
{
int ret;
......@@ -490,13 +643,72 @@ int create_one_async_channel(struct thc_channel **chnl, cptr_t *tx, cptr_t *rx)
thc_channel_group_item_add(&ch_grp[current_lcd_id], xmit_ch_item);
printk("%s:%d adding chnl: %p to group: %p", __func__, current_lcd_id,
xmit_ch_item, &ch_grp[current_lcd_id]);
xmit_ch_item->channel, &ch_grp[current_lcd_id]);
ptrs[current_lcd_id][idx[current_lcd_id]++%32] = xmit_ch_item;
return 0;
}
#ifdef CONFIG_PREALLOC_XMIT_CHANNELS
int prep_xmit_channels_lcd(void)
{
cptr_t tx[MAX_CHNL_PAIRS], rx[MAX_CHNL_PAIRS];
struct thc_channel *xmit;
int i, j;
int node_id;
for (i = 0; i < MAX_CHNL_PAIRS; i++) {
#if NUM_LCDS == 1
if (i >= 8) {
node_id = 1;
} else {
node_id = 0;
}
/* create half of the channel pairs on numa node 1 */
if (create_one_async_channel_on_node(node_id, &xmit, &tx[i], &rx[i]))
LIBLCD_ERR("async channel creation failed\n");
#elif NUM_LCDS == 2
if (current_lcd_id == 0)
node_id = 0;
else
node_id = 1;
if (create_one_async_channel_on_node(node_id, &xmit, &tx[i], &rx[i]))
LIBLCD_ERR("async channel creation failed\n");
#elif NUM_LCDS == 4
if (current_lcd_id < 2)
node_id = 0;
else
node_id = 1;
if (create_one_async_channel_on_node(node_id, &xmit, &tx[i], &rx[i]))
LIBLCD_ERR("async channel creation failed\n");
#elif NUM_LCDS == 6
if (current_lcd_id < 3)
node_id = 0;
else
node_id = 1;
if (create_one_async_channel_on_node(node_id, &xmit, &tx[i], &rx[i]))
LIBLCD_ERR("async channel creation failed\n");
#endif
}
for (i = 0, j = 5; i < MAX_CHNL_PAIRS && j < LCD_NUM_REGS; i++) {
lcd_set_cr(j++, rx[i]);
lcd_set_cr(j++, tx[i]);
}
return 0;
}
void prep_xmit_channels_clean_lcd(void)
{
int i;
for (i = 0; i < LCD_NUM_REGS; i++)
lcd_set_cr(i, CAP_CPTR_NULL);
}
#endif
//DONE
int __rtnl_link_register(struct rtnl_link_ops *ops)
{
......@@ -565,6 +777,10 @@ int __rtnl_link_register(struct rtnl_link_ops *ops)
lcd_set_cr3(rx_xmit);
lcd_set_cr4(tx_xmit);
#ifdef CONFIG_PREALLOC_XMIT_CHANNELS
prep_xmit_channels_lcd();
#endif
g_rtnl_link_ops = ops;
printk("%s, tx_xmit %lx | rx_xmit %lx", __func__,
......@@ -581,6 +797,10 @@ int __rtnl_link_register(struct rtnl_link_ops *ops)
lcd_set_cr3(CAP_CPTR_NULL);
lcd_set_cr4(CAP_CPTR_NULL);
#ifdef CONFIG_PREALLOC_XMIT_CHANNELS
prep_xmit_channels_clean_lcd();
#endif
if (ret) {
LIBLCD_ERR("lcd_call");
goto fail4;
......@@ -1253,9 +1473,11 @@ int ndo_start_xmit_noawe_callee(struct fipc_message *_request, struct thc_channe
fipc_set_reg1(response, ret);
thc_set_msg_type(response, msg_type_response);
fipc_send_msg_end(thc_channel_to_fipc(channel), response);
//printk("%s, response sent! chnl: %p", __func__, channel);
return ret;
}
#define MARSHAL
/* xmit_callee for async. This function receives the IPC and
* sends back a response
*/
......@@ -1263,19 +1485,23 @@ int ndo_start_xmit_async_bare_callee(struct fipc_message *_request, struct thc_c
{
struct fipc_message *response;
unsigned int request_cookie;
#ifdef MARSHAL
struct lcd_sk_buff_container static_skb_c;
struct lcd_sk_buff_container *skb_c = &static_skb_c;
struct sk_buff *skb = &skb_c->skbuff;
#endif
#ifdef COPY
struct skbuff_members *skb_lcd;
#endif
#ifdef MARSHAL
unsigned long skbh_offset, skb_end;
__be16 proto;
u32 len;
cptr_t skb_ref;
#endif
request_cookie = thc_get_request_cookie(_request);
#ifdef MARSHAL
skb_ref = __cptr(fipc_get_reg2(_request));
skbh_offset = fipc_get_reg3(_request);
......@@ -1283,13 +1509,16 @@ int ndo_start_xmit_async_bare_callee(struct fipc_message *_request, struct thc_c
skb_end = fipc_get_reg4(_request);
proto = fipc_get_reg5(_request);
len = fipc_get_reg6(_request);
#endif
fipc_recv_msg_end(thc_channel_to_fipc(channel),
_request);
#ifdef MARSHAL
skb->head = (char*)data_pool + skbh_offset;
skb->end = skb_end;
skb->len = len;
skb->private = true;
#endif
#ifdef COPY
skb_lcd = SKB_LCD_MEMBERS(skb);
......@@ -1309,14 +1538,13 @@ int ndo_start_xmit_async_bare_callee(struct fipc_message *_request, struct thc_c
skb->data = skb->head + skb_lcd->head_data_off;
#endif
fipc_recv_msg_end(thc_channel_to_fipc(channel),
_request);
#ifdef MARSHAL
skb_c->chnl = channel;
skb_c->cookie = request_cookie;
dummy_xmit(skb, NULL);
#endif
if (async_msg_blocking_send_start(channel, &response)) {
LIBLCD_ERR("error getting response msg");
......@@ -1654,9 +1882,11 @@ int validate_callee(struct fipc_message *request, struct thc_channel *channel, s
int cleanup_channel_group(struct fipc_message *request, struct thc_channel *channel)
{
int i;
fipc_recv_msg_end(thc_channel_to_fipc(channel), request);
#ifndef CONFIG_PREALLOC_XMIT_CHANNELS
{
int i;
for (i = 0; i < 32; i++) {
if (ptrs[current_lcd_id][i]) {
thc_channel_group_item_remove(&ch_grp[current_lcd_id], ptrs[current_lcd_id][i]);
......@@ -1665,6 +1895,7 @@ int cleanup_channel_group(struct fipc_message *request, struct thc_channel *chan
ptrs[current_lcd_id][i] = NULL;
} //if
} //for
}
#endif
return 0;
}
......@@ -27,7 +27,7 @@ bool tdiff_valid = false;
u64 tdiff_disp = 0ull;
TS_DECL(disp_loop);
/* LOOP ---------------------------------------- */
#define REPORT_LCD_LOAD
//#define REPORT_LCD_LOAD
void create_async_channel(void);
......@@ -35,6 +35,7 @@ static void main_and_loop(void)
{
int ret;
int stop = 0;
int once = 1;
#ifdef REPORT_LCD_LOAD
unsigned long long start_disp, end_disp;
unsigned long long start_g, end_g, diff_g;
......@@ -66,6 +67,15 @@ static void main_and_loop(void)
* channel). */
while (!stop && !dummy_done) {
struct thc_channel_group_item* curr_item;
if (once) {
printk("%s, LCD:%d looping", __func__, current_lcd_id);
once = 0;
list_for_each_entry(curr_item, &(ch_grp[current_lcd_id].head), list) {
printk("%s, curr_item->channel %p is_xmit: %d", __func__,
curr_item->channel, curr_item->xmit_channel);
}
}
#ifdef REPORT_LCD_LOAD
if (!start_g)
......@@ -115,15 +125,21 @@ static void main_and_loop(void)
if( !ret )
{
#ifdef REPORT_LCD_LOAD
start_disp = lcd_RDTSC_START();
#endif
/* message for us */
if (async_msg_get_fn_type(msg) == NDO_START_XMIT) {
//printk("%s, got msg on xmit ch: %p", __func__, curr_item->channel);
if (fipc_get_reg0(msg)) {
ret = ndo_start_xmit_async_bare_callee(msg,
curr_item->channel,
nullnet_cspace,
nullnet_sync_endpoints[current_lcd_id]);
} else {
//printk("%s, LCD:%d got XMIT msg on chnl: %p",
// __func__, current_lcd_id, curr_item->channel);
ret = ndo_start_xmit_noawe_callee(msg,
curr_item->channel,
nullnet_cspace,
......
......@@ -19,32 +19,32 @@
#include <liblcd/sync_ipc_poll.h>
enum dispatch_t {
REGISTER_NETDEVICE = 0,
UNREGISTER_NETDEVICE = 1,
ETHER_SETUP = 2,
ETH_MAC_ADDR =3,
ETH_VALIDATE_ADDR = 4,
FREE_NETDEV= 5,
NETIF_CARRIER_OFF=6,
REGISTER_NETDEVICE = 1,
UNREGISTER_NETDEVICE,
ETHER_SETUP,
ETH_MAC_ADDR,
ETH_VALIDATE_ADDR = 5,
FREE_NETDEV,
NETIF_CARRIER_OFF,
NETIF_CARRIER_ON,
__RTNL_LINK_REGISTER = 8,
__RTNL_LINK_UNREGISTER,
__RTNL_LINK_REGISTER,
__RTNL_LINK_UNREGISTER = 10,
RTNL_LINK_UNREGISTER,
ALLOC_NETDEV_MQS,
CONSUME_SKB = 12,
CONSUME_SKB,
NDO_INIT,
NDO_UNINIT,
NDO_UNINIT = 15,
NDO_START_XMIT,
NDO_VALIDATE_ADDR = 16,
NDO_VALIDATE_ADDR,
NDO_SET_RX_MODE,
NDO_SET_MAC_ADDRESS,
NDO_GET_STATS64,
NDO_CHANGE_CARRIER = 20,
VALIDATE = 21,
SETUP = 22,
TRIGGER_EXIT =23,
TRIGGER_CLEAN,
PREP_CHANNEL=25,
NDO_GET_STATS64 = 20,
NDO_CHANGE_CARRIER,
VALIDATE,
SETUP,
TRIGGER_EXIT,
TRIGGER_CLEAN = 25,
PREP_CHANNEL,
};
typedef enum {
......@@ -81,6 +81,27 @@ struct skbuff_members {
#define CONSUME_SKB_SEND_ONLY
#define CONSUME_SKB_NO_HASHING
#define SENDER_DISPATCH_LOOP
#define CONFIG_PREALLOC_XMIT_CHANNELS
#if NUM_LCDS == 1
/* total LCD cores = 2 (lcds=1,klcd=1), free cores = 18 */
#define MAX_CHANNELS_PER_LCD 18
#define NUM_THREADS_ON_NODE0 8
#elif NUM_LCDS == 2
/* total LCD cores = 3 (lcds=2,klcd=1), free cores = 17 */
#define MAX_CHANNELS_PER_LCD 9
#define NUM_THREADS_ON_NODE0 8
#elif NUM_LCDS == 4
/* total LCD cores = 5 (lcds=4,klcd=1), free cores = 15 */
#define MAX_CHANNELS_PER_LCD 4
#define NUM_THREADS_ON_NODE0 7
#elif NUM_LCDS == 6
/* total LCD cores = 7 (lcds=6,klcd=1), free cores = 13 */
#define MAX_CHANNELS_PER_LCD 3
#define NUM_THREADS_ON_NODE0 6
#endif
#define MAX_CHNL_PAIRS MAX_CHANNELS_PER_LCD
//#define NO_AWE
//#define NO_HASHING
......
......@@ -25,6 +25,7 @@ struct thc_channel *sirq_channels[4];
#define NUM_THREADS NUM_CORES
int prep_channel(struct trampoline_hidden_args *hidden_args, int queue);
int pick_channel(int lcd_id);
extern inline xmit_type_t check_skb_range(struct sk_buff *skb);
extern struct glue_cspace *c_cspace;
extern struct cptr sync_ep;
......@@ -34,20 +35,25 @@ extern u32 thread;
extern struct ptstate_t *ptrs[NUM_THREADS];
extern struct rtnl_link_stats64 g_stats;
extern struct thc_channel *xmit_chnl;
extern priv_pool_t *pool;
DEFINE_SPINLOCK(prep_lock);
extern priv_pool_t *skb_pool;
DEFINE_SPINLOCK(prep_lock);
/*
* setup a new channel for the first time when an application thread
* wishes to send a packet through this interface
*/
int setup_once(struct trampoline_hidden_args *hidden_args)
{
int queue;
int lcd_id;
#if NUM_LCDS == 1
#elif NUM_LCDS == 2
#else
static int count = 0;
printk("%s, %s:%d lcdenter \n", __func__,
current->comm, current->pid);
static int numa_count1 = 0;
static int numa_count0 = 0;
#endif
printk("%s, %s:%d lcdenter on cpu: %d\n", __func__,
current->comm, current->pid, smp_processor_id());
/* step 1. create lcd env */
lcd_enter();
......@@ -84,10 +90,51 @@ int setup_once(struct trampoline_hidden_args *hidden_args)
!strncmp(current->comm, "lt-iperf3",
strlen("lt-iperf3"))) {
#ifdef CONFIG_PREALLOC_XMIT_CHANNELS
#if NUM_LCDS == 1
lcd_id = 0;
#elif NUM_LCDS == 2
if (smp_processor_id() < NUM_THREADS_ON_NODE0) {
lcd_id = 0;
} else {
lcd_id = 1;
}
#elif NUM_LCDS == 4
spin_lock(&prep_lock);
lcd_id = count++ % NUM_LCDS;
if (smp_processor_id() < NUM_THREADS_ON_NODE0) {
numa_count0++;
if (numa_count0 % 2)
lcd_id = 0;
else
lcd_id = 1;
} else {
numa_count1++;
if (numa_count1 % 2)
lcd_id = 2;
else
lcd_id = 3;
}
spin_unlock(&prep_lock);
#elif NUM_LCDS == 6
spin_lock(&prep_lock);
queue = count++ % NUM_LCDS;
prep_channel(hidden_args, queue);
lcd_id = count++ % NUM_LCDS;
if (smp_processor_id() < NUM_THREADS_ON_NODE0) {
lcd_id = numa_count0++ % (NUM_LCDS >> 1);
} else {
lcd_id = numa_count1++ % (NUM_LCDS >> 1);
lcd_id += (NUM_LCDS >> 1);
}
spin_unlock(&prep_lock);
#endif
pick_channel(lcd_id);
#else
spin_lock(&prep_lock);
lcd_id = count++ % NUM_LCDS;
prep_channel(hidden_args, lcd_id);
spin_unlock(&prep_lock);
#endif
printk("===================================\n");
printk("===== Private Channel created (pid %d) =====\n", current->pid);
printk("===================================\n");
......@@ -174,7 +221,7 @@ int __ndo_start_xmit_inner_async(struct sk_buff *skb, struct net_device *dev, st
fipc_set_reg3(_request,
(unsigned long)
((void*)skb->head - pool->base));
((void*)skb->head - skb_pool->base));
fipc_set_reg4(_request, skb->end);
fipc_set_reg5(_request, skb->protocol);
......@@ -218,9 +265,16 @@ fail_ipc:
int __ndo_start_xmit_bare_async(struct sk_buff *skb, struct net_device *dev, struct trampoline_hidden_args *hidden_args)
{
struct fipc_message *_request;
struct fipc_message *_response;
struct thc_channel *async_chnl = NULL;
xmit_type_t xmit_type;
_TS_DECL(xmit);
unsigned int request_cookie;
int ret;
#ifdef TIMESTAMP
u32 i;
_TS_DECL(xmit);
#endif
xmit_type = check_skb_range(skb);
......@@ -236,10 +290,12 @@ int __ndo_start_xmit_bare_async(struct sk_buff *skb, struct net_device *dev, str
goto free;
}
async_chnl = (struct thc_channel*) PTS()->thc_chnl;
/*
* doesn't free the packet NUM_TRANSACTIONS times
* frees the packet only once
*/
#ifdef TIMESTAMP
_TS_START(xmit);
for (i = 0; i < NUM_TRANSACTIONS; i++) {
int j;
......@@ -255,6 +311,35 @@ int __ndo_start_xmit_bare_async(struct sk_buff *skb, struct net_device *dev, str
printk("%s, do_finish{async()}; %d transactions took %llu\n", __func__,
NUM_TRANSACTIONS, _TS_DIFF(xmit)/NUM_TRANSACTIONS);