...
 
Commits (5)
......@@ -2,26 +2,11 @@
#define PRIV_MEMPOOL_H
#define MTU 1470
#define HEADERS 42
#define HEADERS 42
#define SKB_LCD_MEMBERS_SZ 48
#define SKB_SHARED_INFO (sizeof(struct skb_shared_info))
#define DATA_ALIGNED_SZ (SKB_DATA_ALIGN(MTU + HEADERS + SKB_LCD_MEMBERS_SZ))
#define SKB_DATA_SIZE (DATA_ALIGNED_SZ + SKB_DATA_ALIGN(SKB_SHARED_INFO))
#define SKB_CONTAINER_SIZE 128
typedef enum {
/* for skb->data */
SKB_DATA_POOL = 0,
/* for skb->page_frag */
SKB_FRAG_POOL,
/* for skb_container */
SKB_CONTAINER_POOL,
POOL_MAX,
} pool_type_t;
#define DATA_ALIGNED_SZ (SKB_DATA_ALIGN(MTU + HEADERS + SKB_LCD_MEMBERS_SZ))
#define SKB_DATA_SIZE (DATA_ALIGNED_SZ + SKB_DATA_ALIGN(SKB_SHARED_INFO))
struct object {
struct object *next;
......@@ -38,34 +23,28 @@ struct atom {
} __attribute__((aligned(16)));
typedef struct {
struct object __percpu **head;
struct object __percpu **marker;
#ifdef PBUF
char __percpu **buf;
char __percpu **bufend;
#endif
int __percpu *cached;
unsigned int obj_size;
unsigned int total_pages;
unsigned int num_objs_percpu;
unsigned int obj_size;
unsigned int total_objs;
unsigned int num_cpus;
void *pool;
void *gpool;
unsigned int num_objs_percpu;
void *base;
struct atom stack;
unsigned int pool_order;
spinlock_t pool_spin_lock;
bool dump_once;
struct dentry *pstats;
struct object __percpu **head;
struct object __percpu **marker;
int __percpu *cached;
} priv_pool_t;
void *priv_alloc(pool_type_t type);
void priv_free(void *p, pool_type_t type);
//priv_pool_t *priv_pool_init(pool_type_t type, unsigned int num_objs, unsigned int obj_size);
void *priv_alloc(priv_pool_t *pool);
void priv_free(priv_pool_t *pool, void *obj);
priv_pool_t *priv_pool_init(void *pool_base, size_t pool_size,
unsigned int obj_size, const char* name);
priv_pool_t *priv_pool_init(pool_type_t type, void *pool_base,
size_t pool_size,
unsigned int obj_size);
void priv_pool_destroy(priv_pool_t *p);
#endif /* PRIV_MEMPOOL_H */
......@@ -298,6 +298,9 @@ int lcd_arch_ept_map(struct lcd_arch *lcd, gpa_t ga, hpa_t ha,
/*
* Walk ept
*/
#ifdef CONFIG_LCD_SINGLE_EPT
mutex_lock(&g_ept_lock);
#endif
ret = lcd_arch_ept_walk(lcd, ga, create, &ept_entry);
if (ret)
return ret;
......@@ -309,6 +312,9 @@ int lcd_arch_ept_map(struct lcd_arch *lcd, gpa_t ga, hpa_t ha,
LCD_ERR("would overwrite hpa %lx with hpa %lx\n",
hpa_val(lcd_arch_ept_hpa(ept_entry)),
hpa_val(ha));
#ifdef CONFIG_LCD_SINGLE_EPT
mutex_unlock(&g_ept_lock);
#endif
return -EINVAL;
}
......@@ -317,6 +323,10 @@ int lcd_arch_ept_map(struct lcd_arch *lcd, gpa_t ga, hpa_t ha,
*/
lcd_arch_ept_set(ept_entry, ha);
#ifdef CONFIG_LCD_SINGLE_EPT
mutex_unlock(&g_ept_lock);
#endif
return 0;
}
......@@ -327,6 +337,7 @@ int lcd_arch_ept_map_range(struct lcd_arch *lcd, gpa_t ga_start,
unsigned long len;
len = npages * PAGE_SIZE;
for (off = 0; off < len; off += PAGE_SIZE) {
if (lcd_arch_ept_map(lcd,
/* gpa */
......@@ -355,6 +366,9 @@ int lcd_arch_ept_unmap(struct lcd_arch *lcd, gpa_t a)
/*
* Walk ept
*/
#ifdef CONFIG_LCD_SINGLE_EPT
mutex_lock(&g_ept_lock);
#endif
ret = lcd_arch_ept_walk(lcd, a, 0, &ept_entry);
if (ret)
return ret;
......@@ -364,6 +378,9 @@ int lcd_arch_ept_unmap(struct lcd_arch *lcd, gpa_t a)
*/
lcd_arch_ept_unset(ept_entry);
#ifdef CONFIG_LCD_SINGLE_EPT
mutex_unlock(&g_ept_lock);
#endif
return 0;
}
......@@ -375,6 +392,9 @@ int lcd_arch_ept_unmap2(struct lcd_arch *lcd, gpa_t a, hpa_t *hpa_out)
/*
* Walk ept
*/
#ifdef CONFIG_LCD_SINGLE_EPT
mutex_lock(&g_ept_lock);
#endif
ret = lcd_arch_ept_walk(lcd, a, 0, &ept_entry);
if (ret)
return ret;
......@@ -388,6 +408,9 @@ int lcd_arch_ept_unmap2(struct lcd_arch *lcd, gpa_t a, hpa_t *hpa_out)
*/
lcd_arch_ept_unset(ept_entry);
#ifdef CONFIG_LCD_SINGLE_EPT
mutex_unlock(&g_ept_lock);
#endif
return 0;
}
......@@ -418,6 +441,10 @@ int lcd_arch_ept_gpa_to_hpa(struct lcd_arch *lcd, gpa_t ga, hpa_t *ha_out, bool
/*
* Walk ept
*/
#ifdef CONFIG_LCD_SINGLE_EPT
mutex_lock(&g_ept_lock);
#endif
ret = lcd_arch_ept_walk(lcd, ga, 0, &ept_entry);
if (ret)
return ret;
......@@ -428,6 +455,9 @@ int lcd_arch_ept_gpa_to_hpa(struct lcd_arch *lcd, gpa_t ga, hpa_t *ha_out, bool
if (!vmx_epte_present(*ept_entry) && verbose) {
LCD_ERR("gpa %lx is not mapped\n",
gpa_val(ga));
#ifdef CONFIG_LCD_SINGLE_EPT
mutex_unlock(&g_ept_lock);
#endif
return -EINVAL;
}
......@@ -438,6 +468,9 @@ int lcd_arch_ept_gpa_to_hpa(struct lcd_arch *lcd, gpa_t ga, hpa_t *ha_out, bool
hpa = hpa_add(hpa, vmx_ept_offset(ga));
*ha_out = hpa;
#ifdef CONFIG_LCD_SINGLE_EPT
mutex_unlock(&g_ept_lock);
#endif
return 0;
}
EXPORT_SYMBOL(lcd_arch_ept_gpa_to_hpa);
......
......@@ -697,7 +697,7 @@ int lcd_create_module_lcd(char *mdir, char *mname, cptr_t *lcd_out,
* Initialize empty LCD
*/
create_lcd:
ret = lcd_create(&lcd, is_child);
ret = lcd_create(&lcd, lcd_id);
if (ret) {
LIBLCD_ERR("error creating empty LCD");
goto fail4;
......
......@@ -288,14 +288,14 @@ int __lcd_create_no_vm_no_thread(struct lcd **out);
* which we need a new kthread to run the LCD's code, but we don't want
* the code to run inside an isolated VM.
*/
int __lcd_create_no_vm(struct lcd **out, const char *name, bool is_child);
int __lcd_create_no_vm(struct lcd **out, const char *name, int lcd_id);
/**
* __lcd_create -- Create an empty isolated LCD
* @caller: the LCD doing the creating
* @slot: the slot in the @caller's cspace where the LCD capability should
* be stored
*/
int __lcd_create(struct lcd *caller, cptr_t slot, bool is_child);
int __lcd_create(struct lcd *caller, cptr_t slot, int lcd_id);
/**
* __lcd_create_klcd -- Create an empty non-isolated LCD (kLCD)
* @caller: the LCD doing the creating
......
......@@ -38,7 +38,7 @@
* Creates a bare LCD with an empty cspace and address space (nothing is
* mapped it - except the dedicated page for the LCD's UTCB - see sync_ipc.h).
*/
int lcd_create(cptr_t *lcd, bool is_child);
int lcd_create(cptr_t *lcd, int lcd_id);
/**
* lcd_create_klcd -- Create an LCD that runs in the non-isolated host
......
......@@ -8,7 +8,7 @@
#include <liblcd/liblcd.h>
#include <lcd_domains/microkernel.h>
int lcd_create(cptr_t *lcd, bool is_child)
int lcd_create(cptr_t *lcd, int lcd_id)
{
cptr_t slot;
int ret;
......@@ -23,7 +23,7 @@ int lcd_create(cptr_t *lcd, bool is_child)
/*
* Make LCD
*/
ret = __lcd_create(current->lcd, slot, is_child);
ret = __lcd_create(current->lcd, slot, lcd_id);
if (ret) {
LIBLCD_ERR("lcd create failed");
goto fail2;
......
......@@ -13,7 +13,7 @@
#include <lcd_config/post_hook.h>
int lcd_create(cptr_t *lcd, bool is_child)
int lcd_create(cptr_t *lcd, int lcd_id)
{
cptr_t slot;
int ret;
......
......@@ -169,14 +169,14 @@ fail1:
return ret;
}
int __lcd_create_no_vm(struct lcd **out, const char *name, bool is_child)
int __lcd_create_no_vm(struct lcd **out, const char *name, int lcd_id)
{
struct lcd *lcd;
int ret;
/*
* More basic init
*/
if (is_child) {
if (lcd_id > 0) {
ret = __lcd_create_child_no_vm_no_thread(&lcd);
printk("%s, creating child LCD: %p\n", __func__, lcd);
if (ret) {
......@@ -218,14 +218,25 @@ fail1:
return ret;
}
int __lcd_create(struct lcd *caller, cptr_t slot, bool is_child)
#define LCD_KTHREAD_NAME_SZ 32
int __lcd_create(struct lcd *caller, cptr_t slot, int lcd_id)
{
struct lcd *lcd;
int ret;
int ret = 0;
char *lcd_name;
/*
* Basic init of lcd
*/
ret = __lcd_create_no_vm(&lcd, "lcd", is_child);
lcd_name = kzalloc(LCD_KTHREAD_NAME_SZ, GFP_KERNEL);
if (!lcd_name) {
LCD_ERR("cannot alloc memory");
goto fail1;
}
snprintf(lcd_name, LCD_KTHREAD_NAME_SZ, "lcd/%d", lcd_id);
ret = __lcd_create_no_vm(&lcd, lcd_name, lcd_id);
if (ret) {
LCD_ERR("lcd create");
goto fail1;
......@@ -233,7 +244,7 @@ int __lcd_create(struct lcd *caller, cptr_t slot, bool is_child)
/*
* Alloc vm / arch-dependent part
*/
ret = lcd_arch_create(&lcd->lcd_arch, is_child);
ret = lcd_arch_create(&lcd->lcd_arch, lcd_id > 0);
if(ret) {
LCD_ERR("error creating lcd_arch");
goto fail2;
......
......@@ -254,7 +254,22 @@ static int wait_for_transmit(struct lcd *lcd, struct cnode *ep_cnode,
/*
* Get ready to sleep
*/
set_current_state(TASK_INTERRUPTIBLE);
/*
* XXX: Network drivers, when isolated, does not have a way to
* create buffers by calling a function such as `open` before
* sending packets. They start sending packets right away. So,
* we need to setup private per-thread async buffers. However,
* `dev_queue_xmit` calls `rcu_read_lock_bh()` which increments
* the `preempt_count`. Sleeping here would cause the scheduler
* to emit 'BUG: scheduling while atomic'.
* There are a couple of fixes:
* 1) easy fix - pre-create a set of ring buffers (say 10 per LCD)
* 2) proper fix - Do a synchronous thread migration: Create a
* VMCS for the LCD, pick a stack and vmenter to launch the VM
* to setup ring buffers. Tear down the VMCS and return once we
* are done.
*/
/* set_current_state(TASK_INTERRUPTIBLE); */
/*
* Check if transmit activity
*/
......@@ -293,7 +308,10 @@ static int wait_for_transmit(struct lcd *lcd, struct cnode *ep_cnode,
* We weren't signaled, and message still not sent;
* go back to schleep
*/
schedule();
/*
* XXX: We can possibly be here in an atomic context
*/
/* schedule(); */
continue;
}
/*
......@@ -304,7 +322,10 @@ static int wait_for_transmit(struct lcd *lcd, struct cnode *ep_cnode,
break;
}
out:
set_current_state(TASK_RUNNING);
/*
* XXX: We can possibly be here in an atomic context
*/
/* set_current_state(TASK_RUNNING); */
/*
* Unset our flag
*/
......
......@@ -22,9 +22,12 @@ struct lcd_create_ctx **dummy_ctxs;
cptr_t *dummy_lcds;
cptr_t net_chnl;
cptr_t net_chnl_domain_cptr, dummy_chnl_domain_cptr;
int num_lcds = NUM_LCDS;
cptr_t *dummy_chnl_domain_cptrs;
static int num_lcds = NUM_LCDS;
module_param(num_lcds, int, 0);
MODULE_PARM_DESC(num_lcds, "Number of LCDs to launch");
static int boot_main(void)
{
int ret;
......@@ -142,6 +145,8 @@ static int boot_main(void)
goto fail8;
}
msleep_interruptible(3000);
LIBLCD_MSG("starting dummy ethernet...\n");
for (i = 0; i < num_lcds; i++) {
LIBLCD_MSG("Starting LCD %d ", i);
......@@ -150,6 +155,7 @@ static int boot_main(void)
LIBLCD_ERR("failed to start dummy lcd");
goto fail9;
}
msleep_interruptible(3000);
}
/*
* Wait for 4 seconds
......
......@@ -451,6 +451,8 @@ int create_async_channel(void)
xmit_ch_item->xmit_channel = true;
thc_channel_group_item_add(&ch_grp[current_lcd_id], xmit_ch_item);
printk("%s:%d adding chnl: %p to group: %p", __func__, current_lcd_id,
xmit_ch_item, &ch_grp[current_lcd_id]);
return ret;
fail3:
......@@ -462,12 +464,12 @@ fail1:
return ret;
}
struct thc_channel_group_item *ptrs[32];
struct thc_channel_group_item *ptrs[NUM_LCDS][32];
static int idx[NUM_LCDS] = {0};
int create_one_async_channel(struct thc_channel **chnl, cptr_t *tx, cptr_t *rx)
{
int ret;
static int idx = 0;
struct thc_channel_group_item *xmit_ch_item;
#ifdef ONE_SLOT
ret = setup_async_channel_0(tx, rx, chnl);
......@@ -487,7 +489,10 @@ int create_one_async_channel(struct thc_channel **chnl, cptr_t *tx, cptr_t *rx)
thc_channel_group_item_add(&ch_grp[current_lcd_id], xmit_ch_item);
ptrs[idx++%32] = xmit_ch_item;
printk("%s:%d adding chnl: %p to group: %p", __func__, current_lcd_id,
xmit_ch_item, &ch_grp[current_lcd_id]);
ptrs[current_lcd_id][idx[current_lcd_id]++%32] = xmit_ch_item;
return 0;
}
......@@ -1653,11 +1658,11 @@ int cleanup_channel_group(struct fipc_message *request, struct thc_channel *chan
fipc_recv_msg_end(thc_channel_to_fipc(channel), request);
for (i = 0; i < 32; i++) {
if (ptrs[i]) {
thc_channel_group_item_remove(&ch_grp[current_lcd_id], ptrs[i]);
destroy_async_channel(ptrs[i]->channel);
kfree(ptrs[i]);
ptrs[i] = NULL;
if (ptrs[current_lcd_id][i]) {
thc_channel_group_item_remove(&ch_grp[current_lcd_id], ptrs[current_lcd_id][i]);
destroy_async_channel(ptrs[current_lcd_id][i]->channel);
kfree(ptrs[current_lcd_id][i]);
ptrs[current_lcd_id][i] = NULL;
} //if
} //for
......
......@@ -41,7 +41,6 @@ static void main_and_loop(void)
unsigned long long disp_loop_cycles = 0, percent = 0;
#endif
struct fipc_message *msg;
int _local_lcd_id = current_lcd_id;
DO_FINISH(
......@@ -88,9 +87,12 @@ static void main_and_loop(void)
percent = 0;
}
#endif
if (disp_loop_cycles)
LIBLCD_MSG("total_time: %llu | disp_loop %llu | percent %llu",
diff_g, disp_loop_cycles, percent);
if (disp_loop_cycles) {
u64 div = diff_g / disp_loop_cycles;
percent = 100;
LIBLCD_MSG("total_time: %llu | disp_loop %llu | div: %llu percent: %llu",
diff_g, disp_loop_cycles, div, percent/div);
}
start_g = lcd_RDTSC_START();
// reset disp_loop_cycles
disp_loop_cycles = 0;
......@@ -100,7 +102,7 @@ static void main_and_loop(void)
/*
* Do RR async receive
*/
list_for_each_entry(curr_item, &(ch_grp[_local_lcd_id].head), list) {
list_for_each_entry(curr_item, &(ch_grp[current_lcd_id].head), list) {
if (curr_item->xmit_channel) {
ret = fipc_nonblocking_recv_start_if(
thc_channel_to_fipc(
......
......@@ -22,7 +22,15 @@
#include <lcd_config/post_hook.h>
#define NUM_PACKETS (1000000)
#define VMALLOC_SZ (NUM_PACKETS * sizeof(uint64_t))
#define CONFIG_VMALLOC_SHARED_POOL
#ifdef CONFIG_VMALLOC_SHARED_POOL
#define SKB_DATA_POOL_SIZE (256UL << 20)
#define SKB_DATA_POOL_PAGES (SKB_DATA_POOL_SIZE >> PAGE_SHIFT)
#define SKB_DATA_POOL_ORDER ilog2(roundup_pow_of_two(SKB_DATA_POOL_PAGES))
#endif
#define CORRECTION_VALUE 44
//#define NDO_XMIT_TS
......@@ -183,14 +191,12 @@ int pool_pick(void)
__func__, best_diff, best_idx, pools[best_idx].start_idx, pools[best_idx].end_idx);
return best_idx;
}
extern priv_pool_t *skb_pool;
void skb_data_pool_init(void)
{
//pool = priv_pool_init(SKB_DATA_POOL, 10, SKB_DATA_SIZE);
pool_base = base_pools[pools[pool_pick()].start_idx];
pool_size = best_diff * ((1 << pool_order) * PAGE_SIZE);
pool = priv_pool_init(SKB_DATA_POOL, (void*) pool_base, pool_size, 2048);
pool_base = vzalloc(SKB_DATA_POOL_SIZE);
pool_size = SKB_DATA_POOL_SIZE;
skb_pool = pool = priv_pool_init((void*) pool_base, pool_size, 2048, "skb_data_pool");
#ifdef SKBC_PRIVATE_POOL
skbc_pool = priv_pool_init(SKB_CONTAINER_POOL, 10,
......@@ -200,6 +206,8 @@ void skb_data_pool_init(void)
void skb_data_pool_free(void)
{
if (pool_base)
vfree(pool_base);
if (pool)
priv_pool_destroy(pool);
#ifdef SKBC_PRIVATE_POOL
......@@ -1534,8 +1542,8 @@ void setup_device_ops_trampolines(struct net_device_ops_container *netdev_ops_co
ndo_start_xmit_hidden_args->t_handle->hidden_args = ndo_start_xmit_hidden_args;
ndo_start_xmit_hidden_args->struct_container = netdev_ops_container;
ndo_start_xmit_hidden_args->cspace = c_cspace;
ndo_start_xmit_hidden_args->lcds[1].lcd_async_chnl = lcd_channels[1];
ndo_start_xmit_hidden_args->lcds[1].lcd_sync_ep = new_sync_eps[1];
//ndo_start_xmit_hidden_args->lcds[1].lcd_async_chnl = lcd_channels[1];
//ndo_start_xmit_hidden_args->lcds[1].lcd_sync_ep = new_sync_eps[1];
netdev_ops_container->net_device_ops.ndo_start_xmit = LCD_HANDLE_TO_TRAMPOLINE(ndo_start_xmit_hidden_args->t_handle);
ndo_validate_addr_hidden_args = kzalloc(sizeof( *ndo_validate_addr_hidden_args ), GFP_KERNEL);
......@@ -1723,7 +1731,9 @@ void setup(struct net_device *dev, struct trampoline_hidden_args *hidden_args)
struct setup_container *setup_container;
struct net_device_container *net_dev_container;
struct net_device_ops_container *netdev_ops_container;
#ifndef CONFIG_VMALLOC_SHARED_POOL
struct page *p;
#endif
unsigned int pool_ord;
cptr_t pool_cptr;
uint32_t request_cookie;
......@@ -1780,20 +1790,25 @@ void setup(struct net_device *dev, struct trampoline_hidden_args *hidden_args)
goto fail_ipc;
}
p = virt_to_head_page(pool->pool);
#ifdef CONFIG_VMALLOC_SHARED_POOL
ret = lcd_volunteer_vmalloc_mem(__gva((unsigned long)pool->base), SKB_DATA_POOL_SIZE >> PAGE_SHIFT, &pool_cptr);
pool_ord = SKB_DATA_POOL_ORDER;
#else
p = virt_to_head_page(pool->base);
pool_ord = ilog2(roundup_pow_of_two((
pool->total_pages * PAGE_SIZE)
>> PAGE_SHIFT));
ret = lcd_volunteer_pages(p, pool_ord, &pool_cptr);
#endif
if (ret) {
LIBLCD_ERR("volunteer shared data pool");
goto fail_vol;
}
pool_pfn_start = (unsigned long)pool->pool >> PAGE_SHIFT;
pool_pfn_start = (unsigned long)pool->base >> PAGE_SHIFT;
pool_pfn_end = pool_pfn_start + pool->total_pages;
printk("%s, pool pfn start %lu | end %lu\n", __func__,
......@@ -2155,11 +2170,12 @@ int register_child(void)
LIBLCD_ERR("error adding to dispatch loop");
goto fail2;
}
LIBLCD_MSG("%s, child %d registration complete\n",
__func__, lcd_r1());
lcd_channels[lcd_r1()] = chnl;
LIBLCD_MSG("%s, child %d registration complete!\n", __func__, lcd_r1());
printk("%s, chnl: %p lcd_channels[%lu]: %p\n", __func__, chnl, lcd_r1(), lcd_channels[lcd_r1()]);
g_ndo_start_xmit_hidden_args->lcds[lcd_r1()].lcd_async_chnl = lcd_channels[lcd_r1()];
g_ndo_start_xmit_hidden_args->lcds[lcd_r1()].lcd_sync_ep = new_sync_eps[lcd_r1()];
......@@ -2431,6 +2447,7 @@ int prep_channel(struct trampoline_hidden_args *hidden_args, int queue)
if (queue) {
from_sync_end = hidden_args->lcds[queue].lcd_sync_ep;
async_chnl = hidden_args->lcds[queue].lcd_async_chnl;
current->ptstate->queue_mapping = queue;
} else {
from_sync_end = hidden_args->sync_ep;
async_chnl = hidden_args->async_chnl;
......@@ -2735,33 +2752,57 @@ fail_async:
}
#endif /* CONSUME_SKB_NO_HASHING */
int trigger_exit_to_lcd(struct thc_channel *_channel, enum dispatch_t disp)
int trigger_exit_to_lcd(struct thc_channel *_channel)
{
struct fipc_message *_request;
int ret;
int i;
unsigned int request_cookie;
ret = async_msg_blocking_send_start(_channel,
&_request);
ret = async_msg_blocking_send_start(_channel, &_request);
if (ret) {
LIBLCD_ERR("failed to get a send slot");
goto fail_async;
}
async_msg_set_fn_type(_request,
disp);
async_msg_set_fn_type(_request, TRIGGER_EXIT);
/* No need to wait for a response here */
ret = thc_ipc_send_request(_channel,
_request,
&request_cookie);
printk("%s, sending EXIT to %p\n", __func__, _channel);
ret = thc_ipc_send_request(_channel, _request, &request_cookie);
awe_mapper_remove_id(request_cookie);
fail_async:
return ret;
}
int cleanup_channels(struct thc_channel *_channel)
{
struct fipc_message *_request;
int ret;
int i;
unsigned int request_cookie;
for (i = 0; i < NUM_LCDS; i++) {
ret = async_msg_blocking_send_start(lcd_channels[i], &_request);
if (ret) {
LIBLCD_ERR("failed to get a send slot");
goto fail_async;
}
async_msg_set_fn_type(_request, TRIGGER_CLEAN);
/* No need to wait for a response here */
printk("%s, sending CLEANUP to %p\n", __func__,
lcd_channels[i]);
ret = thc_ipc_send_request(lcd_channels[i], _request,
&request_cookie);
awe_mapper_remove_id(request_cookie);
}
// free the pointers
// there is a race if get_stats try to operate on this data at the same time
// but quite unlikely.
if (disp == TRIGGER_CLEAN) {
thread = 0;
for (i = 0; i < NUM_CORES; i++) {
if (ptrs[i]) {
if (ptrs[i]->exited) {
......@@ -2771,15 +2812,7 @@ int trigger_exit_to_lcd(struct thc_channel *_channel, enum dispatch_t disp)
}
}
}
}
if (ret) {
LIBLCD_ERR("thc_ipc send");
goto fail_ipc;
}
awe_mapper_remove_id(request_cookie);
fail_async:
fail_ipc:
return ret;
}
......@@ -35,15 +35,19 @@ extern struct ptstate_t *ptrs[NUM_THREADS];
extern struct rtnl_link_stats64 g_stats;
extern struct thc_channel *xmit_chnl;
extern priv_pool_t *pool;
DEFINE_SPINLOCK(prep_lock);
/*
* setup a new channel for the first time when an application thread
* wishes to send a packet through this interface
*/
int setup_once(struct trampoline_hidden_args *hidden_args, int queue)
int setup_once(struct trampoline_hidden_args *hidden_args)
{
printk("%s, %s:%d lcdenter | skbqueue %d\n", __func__,
current->comm, current->pid, queue);
int queue;
static int count = 0;
printk("%s, %s:%d lcdenter \n", __func__,
current->comm, current->pid);
/* step 1. create lcd env */
lcd_enter();
......@@ -80,7 +84,10 @@ int setup_once(struct trampoline_hidden_args *hidden_args, int queue)
!strncmp(current->comm, "lt-iperf3",
strlen("lt-iperf3"))) {
spin_lock(&prep_lock);
queue = count++ % NUM_LCDS;
prep_channel(hidden_args, queue);
spin_unlock(&prep_lock);
printk("===================================\n");
printk("===== Private Channel created (pid %d) =====\n", current->pid);
printk("===================================\n");
......@@ -167,7 +174,7 @@ int __ndo_start_xmit_inner_async(struct sk_buff *skb, struct net_device *dev, st
fipc_set_reg3(_request,
(unsigned long)
((void*)skb->head - pool->pool));
((void*)skb->head - pool->base));
fipc_set_reg4(_request, skb->end);
fipc_set_reg5(_request, skb->protocol);
......@@ -225,7 +232,7 @@ int __ndo_start_xmit_bare_async(struct sk_buff *skb, struct net_device *dev, str
}
if (unlikely(!current->ptstate)) {
if (setup_once(hidden_args, 0))
if (setup_once(hidden_args))
goto free;
}
......@@ -276,7 +283,7 @@ int __ndo_start_xmit_bare_fipc_nomarshal(struct sk_buff *skb, struct net_device
}
if (unlikely(!current->ptstate)) {
if (setup_once(hidden_args, 0))
if (setup_once(hidden_args))
goto free;
}
......@@ -368,7 +375,7 @@ int ndo_start_xmit_noasync(struct sk_buff *skb, struct net_device *dev, struct t
/* setup once for this thread */
if (unlikely(!current->ptstate)) {
if (setup_once(hidden_args, skb->queue_mapping))
if (setup_once(hidden_args))
goto free;
}
......@@ -393,7 +400,7 @@ int ndo_start_xmit_noasync(struct sk_buff *skb, struct net_device *dev, struct t
fipc_set_reg3(_request,
(unsigned long)
((void*)skb->head - pool->pool));
((void*)skb->head - pool->base));
fipc_set_reg4(_request, skb->end);
fipc_set_reg5(_request, skb->protocol);
......@@ -495,7 +502,7 @@ int ndo_start_xmit_async(struct sk_buff *skb, struct net_device *dev, struct tra
fipc_set_reg3(_request,
(unsigned long)
((void*)skb->head - pool->pool));
((void*)skb->head - pool->base));
fipc_set_reg4(_request, skb->end);
fipc_set_reg5(_request, skb->protocol);
......@@ -557,9 +564,9 @@ int ndo_start_xmit_async_landing(struct sk_buff *first, struct net_device *dev,
{
struct sk_buff *skb = first;
int rc = NETDEV_TX_OK;
static int count = 0;
skb->queue_mapping = (count++) & 0x1;
if (current->ptstate)
skb->queue_mapping = current->ptstate->queue_mapping;
if (!skb->next)
skb->chain_skb = false;
......@@ -569,7 +576,7 @@ int ndo_start_xmit_async_landing(struct sk_buff *first, struct net_device *dev,
/* chain skb */
if (unlikely(!current->ptstate)) {
if (setup_once(hidden_args, 0))
if (setup_once(hidden_args))
goto free;
}
......
......@@ -35,7 +35,8 @@ struct net_info {
};
static LIST_HEAD(net_infos);
extern int trigger_exit_to_lcd(struct thc_channel *_channel, enum dispatch_t);
extern int trigger_exit_to_lcd(struct thc_channel *_channel);
extern int cleanup_channels(struct thc_channel *_channel);
int register_child(void);
......@@ -250,12 +251,12 @@ static void loop(cptr_t register_chnl)
if (unload_lcd || clean_up) {
if (__get_net(&net)) {
if (unload_lcd) {
trigger_exit_to_lcd(net->chnl, TRIGGER_EXIT);
trigger_exit_to_lcd(net->chnl);
unload_lcd ^= unload_lcd;
}
if (clean_up) {
LIBLCD_MSG("cleanup triggered");
trigger_exit_to_lcd(net->chnl, TRIGGER_CLEAN);
cleanup_channels(net->chnl);
clean_up ^= clean_up;
}
}
......
......@@ -11,39 +11,15 @@
#undef pr_fmt
#define pr_fmt(fmt) "%s:%d : " fmt, __func__, smp_processor_id()
//#define CMA_ALLOC
static priv_pool_t pool_array[POOL_MAX];
static unsigned long pool_cma_size = 1024 * 1024 * 50;
static struct cma *pool_cma;
static int __init early_parse_pool_cma(char *p)
{
pool_cma_size = ALIGN(memparse(p, NULL), PAGE_SIZE);
return 0;
}
early_param("pool_cma", early_parse_pool_cma);
static priv_pool_t *get_pool(pool_type_t type)
{
if (type < 0 || type > POOL_MAX) {
printk("%s, unknown type requested %d\n", __func__, type);
return NULL;
}
return &pool_array[type];
}
#define CACHE_SIZE 0x80
void construct_global_pool(priv_pool_t *p)
{
unsigned int obj_size = p->obj_size;
unsigned int gpool_objs = p->total_objs;
char *gpool = p->gpool;
int i, b;
char *bpool = gpool;
char *bpool = p->base;
char *gpool = p->base;
unsigned int bundles;
struct object *objs;
struct bundle *prev_bund = NULL, *bund = NULL;
......@@ -53,18 +29,17 @@ void construct_global_pool(priv_pool_t *p)
*/
bundles = gpool_objs / CACHE_SIZE;
printk("%s, gpool_objs %d | bundles %u\n",
__func__, gpool_objs, bundles);
pr_info("gpool_objs %d | bundles %u\n", gpool_objs, bundles);
for (b = 0; b < bundles; b++) {
//printk("bundle ===> %d\n", b);
pr_debug("bundle ===> %d\n", b);
for (i = 0; i < CACHE_SIZE; i++) {
objs = (struct object*)((char*)bpool + (i * obj_size));
/* the last object's next is just null */
if (i == CACHE_SIZE - 1)
break;
objs->next = (struct object*)((char*)bpool + (i + 1) * obj_size);
// printk("\tobj %p | obj->next %p\n", objs, objs->next);
pr_debug("\tobj %p | obj->next %p\n", objs, objs->next);
}
/* break the last object's chain */
objs->next = NULL;
......@@ -76,8 +51,8 @@ void construct_global_pool(priv_pool_t *p)
/* assign the list we created to this bundle */
bund->list = (struct object *)objs->next;
printk("prev_bund %p | bund %p\n",
prev_bund, bund);
pr_info("prev_bund %p | bund %p\n", prev_bund, bund);
/* if prev_bundle is constructed, chain it */
if (prev_bund)
prev_bund->next = bund;
......@@ -93,7 +68,7 @@ void construct_global_pool(priv_pool_t *p)
/* our top bundle is at bpool */
p->stack.head = (struct bundle *)gpool;
printk("stak head %p | head->list %p | head->next %p\n",
pr_info("stack head %p | head->list %p | head->next %p\n",
p->stack.head, p->stack.head->list, p->stack.head->next);
}
......@@ -121,16 +96,6 @@ void priv_pool_destroy(priv_pool_t *p)
if (!p)
return;
if (p->pool)
free_pages((unsigned long)p->pool, p->pool_order);
#ifdef PBUF
if (p->buf)
free_percpu(p->buf);
if (p->bufend)
free_percpu(p->bufend);
#endif
if (p->head)
free_percpu(p->head);
......@@ -143,7 +108,7 @@ void priv_pool_destroy(priv_pool_t *p)
EXPORT_SYMBOL(priv_pool_destroy);
#if 0
priv_pool_t *priv_pool_init(pool_type_t type, unsigned int num_objs,
priv_pool_t *priv_pool_init(priv_pool_t type, unsigned int num_objs,
unsigned int obj_size)
{
priv_pool_t *p;
......@@ -261,58 +226,48 @@ priv_pool_t *priv_pool_init(pool_type_t type, unsigned int num_objs,
EXPORT_SYMBOL(priv_pool_init);
#endif
priv_pool_t *priv_pool_init(pool_type_t type, void *pool_base,
priv_pool_t *priv_pool_init(void *pool_base,
size_t pool_size,
unsigned int obj_size)
unsigned int obj_size,
const char *name)
{
priv_pool_t *p;
char *pool, *global_pool;
priv_pool_t *p = NULL;
int cpu;
if (!obj_size) {
printk("%s, invalid objsize (%u) requested\n",
__func__, obj_size);
return NULL;
pr_err("Invalid objsize %u\n", obj_size);
goto exit;
}
p = get_pool(type);
if (!pool_size) {
pr_err("Invalid pool_size %zu\n", pool_size);
goto exit;
}
if (!p)
return NULL;
if (!pool_base) {
pr_err("Invalid pool_base %p\n", pool_base);
goto exit;
}
memset(p, 0, sizeof(priv_pool_t));
p = kzalloc(sizeof(*p), GFP_KERNEL);
p->head = alloc_percpu(struct object *);
p->marker = alloc_percpu(struct object *);
p->cached = alloc_percpu(int);
if (!p) {
pr_err("unable to allocate pool struct\n");
return NULL;
}
/* align obj_size to 32 bit boundary */
p->obj_size = obj_size = ALIGN(obj_size, 32);
p->num_cpus = num_online_cpus() * 2;
p->pool_order = 10;
if (!pool_base) {
/* alloc total_size pages */
pool = p->pool = (char*) __get_free_pages(GFP_KERNEL | __GFP_ZERO,
p->pool_order);
p->total_objs = ((1 << p->pool_order) * PAGE_SIZE)
/ p->obj_size;
printk("Memory %p | size %lx\n", pool, (1 << p->pool_order) * PAGE_SIZE);
} else {
pool = p->pool = pool_base;
p->total_objs = pool_size / p->obj_size;
printk("Memory %p | size %lx\n", pool, pool_size);
p->total_pages = pool_size >> PAGE_SHIFT;
}
p->head = alloc_percpu(struct object *);
p->marker = alloc_percpu(struct object *);
p->cached = alloc_percpu(int);
if (!pool) {
printk("No memory %p\n", pool);
return NULL;
}
/* split the total pages between pcpu pool and the global pool */
p->gpool = global_pool = pool;
p->base = pool_base;
p->total_objs = pool_size / p->obj_size;
p->total_pages = pool_size >> PAGE_SHIFT;
pr_info("Initilaizing mempool, base: %p size: %lx objsize:%u\n",
pool_base, pool_size, obj_size);
/* update percpu vars */
for_each_online_cpu(cpu) {
......@@ -322,11 +277,13 @@ priv_pool_t *priv_pool_init(pool_type_t type, void *pool_base,
construct_global_pool(p);
spin_lock_init(&p->pool_spin_lock);
exit:
return p;
}
EXPORT_SYMBOL(priv_pool_init);
void *priv_alloc(pool_type_t type)
void *priv_alloc(priv_pool_t *p)
{
void *m = NULL;
#ifdef PBUF
......@@ -334,13 +291,13 @@ void *priv_alloc(pool_type_t type)
char *pbufend;
#endif
struct object *head;
priv_pool_t *p = get_pool(type);
unsigned long flags;
if (!p)
return NULL;
/* disable preempt until we manipulate all percpu pointers */
preempt_disable();
/* disable irqs until we manipulate all percpu pointers */
local_irq_save(flags);
head = (struct object*) *this_cpu_ptr(p->head);
......@@ -350,12 +307,19 @@ void *priv_alloc(pool_type_t type)
m = head;
*this_cpu_ptr(p->head) = head->next;
this_cpu_dec(*p->cached);
goto out;
/*
* An object is served from local per-cpu cache.
*/
local_irq_restore(flags);
return m;
} else {
pr_debug("reset cached\n");
this_cpu_write(*(p->cached), 0);
}
local_irq_restore(flags);
#ifdef PBUF
pbuf = (char*)*this_cpu_ptr(p->buf);
pbufend = (char*)*this_cpu_ptr(p->bufend);
......@@ -436,29 +400,29 @@ void *priv_alloc(pool_type_t type)
#endif
}
out:
/* enable preemption */
preempt_enable();
return m;
}
EXPORT_SYMBOL(priv_alloc);
void priv_free(void *addr, pool_type_t type)
void priv_free(priv_pool_t *pool, void *obj)
{
struct object *p, *head;
priv_pool_t *pool;
if (!addr)
return;
unsigned long flags;
pool = get_pool(type);
if (!obj) {
pr_err("Object pointer %p\n", obj);
return;
}
if (!pool)
if (!pool) {
pr_err("Pool pointer %p\n", pool);
return;
}
p = (struct object*) addr;
p = (struct object*) obj;
/* disable preempt until we manipulate all percpu pointers */
preempt_disable();
local_irq_save(flags);
head = (struct object*)*this_cpu_ptr(pool->head);
p->next = (struct object*)head;
......@@ -473,16 +437,19 @@ void priv_free(void *addr, pool_type_t type)
#endif
*this_cpu_ptr(pool->marker) = *this_cpu_ptr(pool->head);
pr_debug("set marker @ %p\n", *this_cpu_ptr(pool->marker));
if ((void*)(*this_cpu_ptr(pool->marker))->next < (void*)pool->pool ||
(void*)(*this_cpu_ptr(pool->marker))->next > (void*)(pool->pool + (pool->total_pages << PAGE_SHIFT))) {
if ((void*)(*this_cpu_ptr(pool->marker))->next < (void*)pool->base ||
(void*)(*this_cpu_ptr(pool->marker))->next > (void*)(pool->base + (pool->total_pages << PAGE_SHIFT))) {
printk("marker->next is corrupted!! marker %p | marker->next %p\n",
*this_cpu_ptr(pool->marker), (*this_cpu_ptr(pool->marker))->next);
dump_stack();
}
local_irq_restore(flags);
} else if (*this_cpu_ptr(pool->cached) == (CACHE_SIZE << 1)) {
struct bundle *donation = (struct bundle *)*this_cpu_ptr(pool->head);
struct atom snapshot, new;
local_irq_restore(flags);
#ifdef SPINLOCK
/* lock global pool */
spin_lock(&pool->pool_spin_lock);
......@@ -491,8 +458,8 @@ void priv_free(void *addr, pool_type_t type)
new.head = donation;
donation->list = ((struct object*)*this_cpu_ptr(pool->head))->next;
if ((void*)(*this_cpu_ptr(pool->marker))->next < (void*)pool->pool ||
(void*)(*this_cpu_ptr(pool->marker))->next > (void*)(pool->pool + (pool->total_pages << PAGE_SHIFT))) {
if ((void*)(*this_cpu_ptr(pool->marker))->next < (void*)pool->base ||
(void*)(*this_cpu_ptr(pool->marker))->next > (void*)(pool->base + (pool->total_pages << PAGE_SHIFT))) {
printk("update pool->head with corrupted marker %p | marker->next %p\n",
*this_cpu_ptr(pool->marker), (*this_cpu_ptr(pool->marker))->next);
dump_stack();
......@@ -528,10 +495,9 @@ void priv_free(void *addr, pool_type_t type)
donation, snapshot.head,
snapshot.head, snapshot.version,
new.head, new.version);
} else {
local_irq_restore(flags);
}
/* enable preemption */
preempt_enable();
}
EXPORT_SYMBOL(priv_free);
......
......@@ -84,6 +84,8 @@ struct kmem_cache *skbuff_head_cache __read_mostly;
static struct kmem_cache *skbuff_fclone_cache __read_mostly;
int sysctl_max_skb_frags __read_mostly = MAX_SKB_FRAGS;
EXPORT_SYMBOL(sysctl_max_skb_frags);
priv_pool_t *skb_pool;
EXPORT_SYMBOL(skb_pool);
/**
* skb_panic - private function for out-of-line support
......@@ -154,6 +156,8 @@ out:
return obj;
}
/* Allocate a new skbuff. We do this ourselves so we can fill in a few
* 'private' fields and also do memory statistics to find all the
* [BEEP] leaks.
......@@ -235,7 +239,7 @@ struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask,
*/
if (unlikely(privpool)) {
/* private pool */
data = priv_alloc(SKB_DATA_POOL);
data = priv_alloc(skb_pool);
size = SKB_DATA_SIZE;
} else {
size = SKB_DATA_ALIGN(size);
......@@ -602,7 +606,7 @@ static void skb_free_head(struct sk_buff *skb)
skb_free_frag(head);
else {
if (skb->private)
priv_free(head, SKB_DATA_POOL);
priv_free(skb_pool, head);
else
kfree(head);
}
......