...
 
Commits (5)
......@@ -302,8 +302,12 @@ int lcd_arch_ept_map(struct lcd_arch *lcd, gpa_t ga, hpa_t ha,
mutex_lock(&g_ept_lock);
#endif
ret = lcd_arch_ept_walk(lcd, ga, create, &ept_entry);
if (ret)
if (ret) {
#ifdef CONFIG_LCD_SINGLE_EPT
mutex_unlock(&g_ept_lock);
#endif
return ret;
}
/*
* Check if guest physical address already mapped
......@@ -370,8 +374,13 @@ int lcd_arch_ept_unmap(struct lcd_arch *lcd, gpa_t a)
mutex_lock(&g_ept_lock);
#endif
ret = lcd_arch_ept_walk(lcd, a, 0, &ept_entry);
if (ret)
if (ret) {
#ifdef CONFIG_LCD_SINGLE_EPT
mutex_unlock(&g_ept_lock);
#endif
return ret;
}
/*
* Unset
......@@ -396,8 +405,12 @@ int lcd_arch_ept_unmap2(struct lcd_arch *lcd, gpa_t a, hpa_t *hpa_out)
mutex_lock(&g_ept_lock);
#endif
ret = lcd_arch_ept_walk(lcd, a, 0, &ept_entry);
if (ret)
if (ret) {
#ifdef CONFIG_LCD_SINGLE_EPT
mutex_unlock(&g_ept_lock);
#endif
return ret;
}
/*
* Extract hpa
*/
......@@ -446,8 +459,12 @@ int lcd_arch_ept_gpa_to_hpa(struct lcd_arch *lcd, gpa_t ga, hpa_t *ha_out, bool
#endif
ret = lcd_arch_ept_walk(lcd, ga, 0, &ept_entry);
if (ret)
if (ret) {
#ifdef CONFIG_LCD_SINGLE_EPT
mutex_unlock(&g_ept_lock);
#endif
return ret;
}
/*
* Confirm the entry is present
......
......@@ -697,7 +697,152 @@ int lcd_create_module_lcd(char *mdir, char *mname, cptr_t *lcd_out,
* Initialize empty LCD
*/
create_lcd:
ret = lcd_create(&lcd, lcd_id);
ret = lcd_create(&lcd, lcd_id, 0);
if (ret) {
LIBLCD_ERR("error creating empty LCD");
goto fail4;
}
/*
* Set up address spaces
*/
if (is_child) {
ret = setup_phys_addr_space_child_lcd(lcd, ctx, lcd_id);
} else {
ret = setup_addr_spaces(lcd, ctx, m_init_link_addr,
m_core_link_addr, m_init_size, m_core_size,
lcd_id);
}
if (ret) {
LIBLCD_ERR("error setting up address spaces");
goto fail5;
}
/*
* Configure initial control registers, etc. for LCD
*/
*((unsigned long *)ctx->gs_page) = gva_val(LCD_BOOTSTRAP_PAGES_GV_ADDR_CHILD(lcd_id));
ret = lcd_config_registers(lcd, m_init_func_addr,
/* implicity put a null return address and
* frame address */
gva_add(LCD_STACK_GV_ADDR_CHILD(lcd_id),
LCD_STACK_SIZE - sizeof(void *)),
LCD_BOOTSTRAP_PAGE_TABLES_GP_ADDR,
LCD_UTCB_GP_ADDR_CHILD(lcd_id),
LCD_ARCH_GS_GV_BASE_CHILD(lcd_id));
if (ret) {
LIBLCD_ERR("error configuring LCDs registers");
goto fail6;
}
#ifndef LCD_ISOLATE
/*
* For non-isolated code running this, we also remember where
* the struct module copy is located so we can do stack traces.
*/
ret = set_struct_module(lcd, ctx->m_core_bits,
m_struct_module_core_offset, is_child);
if (ret) {
LIBLCD_ERR("error setting struct module hva");
goto fail7;
}
#endif
/*
* Return context and lcd
*/
LIBLCD_MSG("%s, %s LCD, ctx %p, lcd %lu", __func__, is_child ? "child" : "parent",
ctx, lcd.cptr);
*ctx_out = ctx;
*lcd_out = lcd;
/*
* Save parent ctx for the future
*/
if (!is_child) {
parent_ctx = ctx;
}
#ifdef CONFIG_LCD_SINGLE_EPT
lcd_to_boot_info(ctx)->lcd_id = lcd_id;
#endif
return 0;
#ifndef LCD_ISOLATE
fail7:
#endif
fail6:
fail5:
lcd_cap_delete(lcd);
fail4:
lcd_release_module(ctx->m_init_bits, ctx->m_core_bits);
fail3:
fail2:
destroy_create_ctx(ctx);
fail1:
return ret;
}
int __lcd_create_module_lcd(char *mdir, char *mname, cptr_t *lcd_out,
struct lcd_create_ctx **ctx_out, int lcd_id, int num_lcds)
{
int ret;
bool is_child = (lcd_id > 0);
cptr_t m_init_cptr, m_core_cptr;
static gva_t m_init_link_addr, m_core_link_addr, m_init_func_addr;
static unsigned long m_init_size, m_core_size;
static unsigned long m_struct_module_core_offset;
struct lcd_create_ctx *ctx;
static struct lcd_create_ctx *parent_ctx;
cptr_t lcd;
/*
* Initialize create ctx
*/
ret = init_create_ctx(&ctx, mname);
if (ret) {
LIBLCD_ERR("error creating ctx");
goto fail1;
}
/*
* Alloc boot pages, stack pages, etc. for LCD
*/
ret = get_pages_for_lcd(ctx, is_child);
if (ret) {
LIBLCD_ERR("error alloc'ing boot, stack pages for LCD");
goto fail2;
}
/*
* if we are creating a child LCD, skip loading the module as it is
* already loaded in parent LCD. In child LCD, we just map the same set
* of pages to the EPT.
*/
if (is_child) {
ctx->m_init_bits = parent_ctx->m_init_bits;
ctx->m_core_bits = parent_ctx->m_core_bits;
goto create_lcd;
}
/*
* Load kernel module into caller's address space
*/
ret = lcd_load_module(mdir, mname,
&ctx->m_init_bits, &ctx->m_core_bits,
&m_init_cptr, &m_core_cptr,
&m_init_link_addr, &m_core_link_addr,
&m_init_size, &m_core_size,
&m_init_func_addr,
&m_struct_module_core_offset);
if (ret) {
LIBLCD_ERR("error loading kernel module");
goto fail3;
}
/*
* At this point, we have all of the data that will go in the LCD
* (the microkernel has the UTCB page)
*
* Initialize empty LCD
*/
create_lcd:
ret = lcd_create(&lcd, lcd_id, num_lcds);
if (ret) {
LIBLCD_ERR("error creating empty LCD");
goto fail4;
......@@ -796,8 +941,8 @@ int lcd_create_module_lcds(char *mdir, char *mname, cptr_t *lcd_out,
*/
for (i = 0; i < num_child; i++) {
LIBLCD_MSG("Creating LCD modules %d", i);
ret = lcd_create_module_lcd(mdir, mname, &lcd_out[i],
&ctx_out[i], i);
ret = __lcd_create_module_lcd(mdir, mname, &lcd_out[i],
&ctx_out[i], i, num_child);
if (ret) {
LIBLCD_ERR("error creating empty LCD");
goto fail2;
......
......@@ -288,14 +288,14 @@ int __lcd_create_no_vm_no_thread(struct lcd **out);
* which we need a new kthread to run the LCD's code, but we don't want
* the code to run inside an isolated VM.
*/
int __lcd_create_no_vm(struct lcd **out, const char *name, int lcd_id);
int __lcd_create_no_vm(struct lcd **out, const char *name, int lcd_id, int num_lcds);
/**
* __lcd_create -- Create an empty isolated LCD
* @caller: the LCD doing the creating
* @slot: the slot in the @caller's cspace where the LCD capability should
* be stored
*/
int __lcd_create(struct lcd *caller, cptr_t slot, int lcd_id);
int __lcd_create(struct lcd *caller, cptr_t slot, int lcd_id, int num_lcds);
/**
* __lcd_create_klcd -- Create an empty non-isolated LCD (kLCD)
* @caller: the LCD doing the creating
......
......@@ -38,7 +38,7 @@
* Creates a bare LCD with an empty cspace and address space (nothing is
* mapped it - except the dedicated page for the LCD's UTCB - see sync_ipc.h).
*/
int lcd_create(cptr_t *lcd, int lcd_id);
int lcd_create(cptr_t *lcd, int lcd_id, int num_lcds);
/**
* lcd_create_klcd -- Create an LCD that runs in the non-isolated host
......
......@@ -19,7 +19,7 @@
/* UTCB ---------------------------------------- */
#define LCD_NUM_REGS 8
#define LCD_NUM_REGS 76
struct lcd_utcb {
/**
* mr registers are general purpose 64-bit registers. You can
......@@ -72,6 +72,17 @@ LCD_MK_REG_ACCESS(5)
LCD_MK_REG_ACCESS(6)
LCD_MK_REG_ACCESS(7)
static inline void lcd_set_cr(int idx, cptr_t val)
{
lcd_get_utcb()->cr[idx] = val;
}
static inline cptr_t lcd_get_cr(int idx)
{
return lcd_get_utcb()->cr[idx];
}
/* SYNCHRONOUS IPC ---------------------------------------- */
/**
......
......@@ -8,7 +8,7 @@
#include <liblcd/liblcd.h>
#include <lcd_domains/microkernel.h>
int lcd_create(cptr_t *lcd, int lcd_id)
int lcd_create(cptr_t *lcd, int lcd_id, int num_lcds)
{
cptr_t slot;
int ret;
......@@ -23,7 +23,7 @@ int lcd_create(cptr_t *lcd, int lcd_id)
/*
* Make LCD
*/
ret = __lcd_create(current->lcd, slot, lcd_id);
ret = __lcd_create(current->lcd, slot, lcd_id, num_lcds);
if (ret) {
LIBLCD_ERR("lcd create failed");
goto fail2;
......
......@@ -13,7 +13,7 @@
#include <lcd_config/post_hook.h>
int lcd_create(cptr_t *lcd, int lcd_id)
int lcd_create(cptr_t *lcd, int lcd_id, int num_lcds)
{
cptr_t slot;
int ret;
......
......@@ -169,7 +169,7 @@ fail1:
return ret;
}
int __lcd_create_no_vm(struct lcd **out, const char *name, int lcd_id)
int __lcd_create_no_vm(struct lcd **out, const char *name, int lcd_id, int num_lcds)
{
struct lcd *lcd;
int ret;
......@@ -199,6 +199,40 @@ int __lcd_create_no_vm(struct lcd **out, const char *name, int lcd_id)
LCD_ERR("failed to create kthread");
goto fail2;
}
#if 1
if (!strncmp(name, "lcd/", strlen("lcd/"))) {
switch (num_lcds) {
case 1:
kthread_bind(lcd->kthread, 24);
break;
case 2:
/* lcds=2, each on one numa node */
if (lcd_id == 0)
kthread_bind(lcd->kthread, 24);
else
kthread_bind(lcd->kthread, 25);
break;
case 4:
switch (lcd_id) {
case 0:
kthread_bind(lcd->kthread, 24);
break;
case 1:
kthread_bind(lcd->kthread, 29);
break;
case 2:
kthread_bind(lcd->kthread, 30);
break;
case 3:
kthread_bind(lcd->kthread, 31);
break;
}
break;
}
} else if (!strncmp(name, "klcd", 4)) {
kthread_bind(lcd->kthread, 28);
}
#endif
/*
* Bump reference count on kthread
*/
......@@ -219,7 +253,7 @@ fail1:
}
#define LCD_KTHREAD_NAME_SZ 32
int __lcd_create(struct lcd *caller, cptr_t slot, int lcd_id)
int __lcd_create(struct lcd *caller, cptr_t slot, int lcd_id, int num_lcds)
{
struct lcd *lcd;
int ret = 0;
......@@ -236,7 +270,7 @@ int __lcd_create(struct lcd *caller, cptr_t slot, int lcd_id)
snprintf(lcd_name, LCD_KTHREAD_NAME_SZ, "lcd/%d", lcd_id);
ret = __lcd_create_no_vm(&lcd, lcd_name, lcd_id);
ret = __lcd_create_no_vm(&lcd, lcd_name, lcd_id, num_lcds);
if (ret) {
LCD_ERR("lcd create");
......@@ -281,7 +315,7 @@ int __lcd_create_klcd(struct lcd *caller, cptr_t slot)
/*
* Basic init of lcd
*/
ret = __lcd_create_no_vm(&lcd, "klcd", 0);
ret = __lcd_create_no_vm(&lcd, "klcd", 0, 0);
if (ret) {
LCD_ERR("lcd create");
......
......@@ -38,7 +38,7 @@ static int handle_syscall_create(struct lcd *lcd)
/*
* Do create
*/
return __lcd_create(lcd, lcd_slot, 0);
return __lcd_create(lcd, lcd_slot, 0, 0);
}
static int handle_syscall_cap_grant(struct lcd *lcd)
......
subdir-ccflags-y += -DNUM_LCDS="2"
subdir-ccflags-y += -DNUM_LCDS="4"
obj-$(LCD_CONFIG_BUILD_NULLB_BOOT) += boot/
......
......@@ -31,6 +31,7 @@ static LIST_HEAD(drv_infos);
int pmfs_ready;
struct cspace *klcd_cspace;
int register_child(void);
atomic_t num_registered = ATOMIC_INIT(0);
/*drv_infos is a global list that has a list of drivers registered to it
* ch_grp is another list internal to a particular drv_info that has a list
......@@ -216,7 +217,9 @@ static int async_aux_loop(struct drv_info **drv_out, struct fipc_message **msg_o
static int do_one_register(cptr_t register_chnl)
{
int ret;
int i, j;
cptr_t sync_endpoint, tx, rx;
cptr_t _tx[MAX_CHNL_PAIRS], _rx[MAX_CHNL_PAIRS];
//cptr_t tx_aux, rx_aux;
/*
......@@ -238,6 +241,19 @@ static int do_one_register(cptr_t register_chnl)
goto fail3;
}
for (i = 0; i < MAX_CHNL_PAIRS; i++) {
ret = lcd_cptr_alloc(&_tx[i]);
if (ret) {
LIBLCD_ERR("cptr alloc failed");
goto fail3;
}
ret = lcd_cptr_alloc(&_rx[i]);
if (ret) {
LIBLCD_ERR("cptr alloc failed");
goto fail3;
}
}
//ret = lcd_cptr_alloc(&tx_aux);
//if (ret) {
// LIBLCD_ERR("cptr alloc failed");
......@@ -259,21 +275,28 @@ static int do_one_register(cptr_t register_chnl)
//lcd_set_cr3(tx_aux);
//lcd_set_cr4(rx_aux);
for (i = 0, j = 5; i < MAX_CHNL_PAIRS && j < LCD_NUM_REGS; i++) {
lcd_set_cr(j++, _tx[i]);
lcd_set_cr(j++, _rx[i]);
}
//printk("polling for sync recv...\n");
ret = lcd_sync_poll_recv(register_chnl);
if (ret) {
//if (ret == -EWOULDBLOCK) {
//printk("EWOULDBLOCK \n");
//ret = 0;
//}
if (ret == -EWOULDBLOCK) {
ret = 0;
}
goto free_cptrs;
}
atomic_inc(&num_registered);
/*
* Dispatch to register handler
*/
if (lcd_r0()) {
ret = dispatch_sync_loop();
} else {
printk("%s, calling register_child\n", __func__);
ret = register_child();
}
......@@ -283,9 +306,9 @@ static int do_one_register(cptr_t register_chnl)
return 0;
free_cptrs:
lcd_set_cr0(CAP_CPTR_NULL);
lcd_set_cr1(CAP_CPTR_NULL);
lcd_set_cr2(CAP_CPTR_NULL);
for (i = 0; i < LCD_NUM_REGS; i++)
lcd_set_cr(i, CAP_CPTR_NULL);
//lcd_set_cr3(CAP_CPTR_NULL);
//lcd_set_cr4(CAP_CPTR_NULL);
//lcd_cptr_free(rx_aux);
......@@ -294,6 +317,11 @@ free_cptrs:
//fail4:
lcd_cptr_free(rx);
fail3:
for (i = 0; i < MAX_CHNL_PAIRS; i++) {
lcd_cptr_free(_tx[i]);
lcd_cptr_free(_rx[i]);
}
lcd_cptr_free(tx);
fail2:
lcd_cptr_free(sync_endpoint);
......@@ -335,6 +363,14 @@ static void handle_loop(long id)
ret = async_loop(&drv, &curr_item, &msg);
if (!ret) {
#if 0
if (async_msg_get_fn_type(msg) == BLK_MQ_END_REQUEST) {
printk("KLCD: end_request seen \n");
}
if (async_msg_get_fn_type(msg) == BLK_MQ_START_REQUEST) {
printk("KLCD: start_request seen \n");
}
#endif
//(async_msg_get_fn_type(msg) == BLK_MQ_END_REQUEST) ? count ++ : -1;
//(async_msg_get_fn_type(msg) == BLK_MQ_START_REQUEST) ? bench_start() : -1;
//((id == 0) && (async_msg_get_fn_type(msg) == BLK_MQ_END_REQUEST)) ? marker_begin() : -1;
......@@ -473,31 +509,89 @@ int klcd_task(void *data) {
static void loop(cptr_t register_chnl)
{
unsigned long tics = jiffies + VFS_REGISTER_FREQ;
//struct fipc_message *msg;
//struct thc_channel_group_item *curr_item;
//struct drv_info *drv;
struct fipc_message *msg;
struct thc_channel_group_item *curr_item;
struct drv_info *drv;
int stop = 0;
int ret;
//struct task_struct *sub_task = NULL;
int id = 0;
while(!stop) {
if (jiffies >= tics) {
ret = do_one_register(register_chnl);
if (ret == 0) {
LIBLCD_MSG("registered driver successfully \n");
break;
DO_FINISH(
while(!stop) {
if (atomic_read(&num_registered) != NUM_LCDS) {
if (jiffies >= tics) {
ret = do_one_register(register_chnl);
if (ret) {
LIBLCD_MSG("register error\n");
break;
}
tics = jiffies + VFS_REGISTER_FREQ;
continue;
}
}
else if (ret == -EWOULDBLOCK) {
tics = jiffies + VFS_REGISTER_FREQ;
continue;
if (stop)
break;
ret = async_loop(&drv, &curr_item, &msg);
if (!ret) {
#if 0
if (async_msg_get_fn_type(msg) == BLK_MQ_END_REQUEST) {
printk("KLCD: end_request seen \n");
}
if (async_msg_get_fn_type(msg) == BLK_MQ_START_REQUEST) {
printk("KLCD: start_request seen \n");
}
#endif
//(async_msg_get_fn_type(msg) == BLK_MQ_END_REQUEST) ? count ++ : -1;
//(async_msg_get_fn_type(msg) == BLK_MQ_START_REQUEST) ? bench_start() : -1;
//((id == 0) && (async_msg_get_fn_type(msg) == BLK_MQ_END_REQUEST)) ? marker_begin() : -1;
//((id == 0xab) && (async_msg_get_fn_type(msg) == BLK_MQ_END_REQUEST)) ? bench_end() : -1;
ASYNC(
ret = dispatch_async_loop(
curr_item->channel,
msg,
drv->cspace,
drv->sync_endpoint);
if (ret) {
LIBLCD_ERR("drv dispatch err");
/* (break won't work here) */
stop = 1;
}
);
//((async_msg_get_fn_type(msg) == BLK_MQ_END_REQUEST)) ? bench_end() : -1;
} else if (ret != -EWOULDBLOCK) {
LIBLCD_ERR("async loop failed");
stop = 1;
break;
}
else {
LIBLCD_ERR("register error");
if (kthread_should_stop()) {
LIBLCD_ERR("kthread should stop");
stop = 1;
if (id == 0) {
blk_exit(curr_item->channel);
}
break;
}
#ifndef CONFIG_PREEMPT
/*
* Play nice with the rest of the system
*/
cond_resched();
#endif
}
}
if(id == 0) {
BENCH_COMPUTE_STAT(disp_loop);
//MARKER_DUMP(disp_loop);
//printk("main klcd count --> %d \n",count);
}
LIBLCD_MSG("blk exited loop, calling blk_exit");
);
//sub_task = kthread_create(klcd_task, NULL, "klcd_task");
//if (IS_ERR(sub_task)) {
// ret = PTR_ERR(sub_task);
......@@ -507,7 +601,7 @@ static void loop(cptr_t register_chnl)
//wake_up_process(sub_task);
handle_loop(0);
//handle_loop(0);
//kthread_stop(sub_task);
/*
DO_FINISH(
......
......@@ -63,6 +63,14 @@ static struct type_ops_id glue_libcap_type_ops[GLUE_NR_TYPES] = {
.revoke = dummy_func,
}
},
{
{
.name = "struct blk_dev_ops",
.delete = dummy_func,
.revoke = dummy_func,
}
},
{
{
.name = "struct gendisk",
......
......@@ -16,36 +16,40 @@
#define PMFS_ASYNC_RPC_BUFFER_ORDER 20
#define FIPC_MSG_STATUS_AVAILABLE 0xdeaddeadUL
#define FIPC_MSG_STATUS_SENT 0xfeedfeedUL
#define SENDER_DISPATCH_LOOP
#define CONFIG_PREALLOC_CHANNELS
enum dispatch_t {
BLK_MQ_ALLOC_TAG_SET,
BLK_MQ_ALLOC_TAG_SET = 1,
BLK_MQ_INIT_QUEUE,
BLK_CLEANUP_QUEUE,
BLK_MQ_END_REQUEST,
BLK_MQ_END_REQUEST = 4,
BLK_MQ_FREE_TAG_SET,
BLK_MQ_START_REQUEST,
BLK_MQ_MAP_QUEUE,
BLK_QUEUE_LOGICAL_BLOCK_SIZE,
BLK_QUEUE_LOGICAL_BLOCK_SIZE = 8,
BLK_QUEUE_PHYSICAL_BLOCK_SIZE,
ALLOC_DISK,
ADD_DISK,
PUT_DISK,
PUT_DISK = 12,
DEL_GENDISK,
DISK_NODE,
REGISTER_BLKDEV,
UNREGISTER_BLKDEV,
UNREGISTER_BLKDEV = 16,
REGISTER_CHARDEV,
QUEUE_RQ_FN,
MAP_QUEUE_FN,
INIT_HCTX_FN,
INIT_HCTX_FN = 20,
SOFTIRQ_DONE_FN,
OPEN,
RELEASE,
OPEN_CHARDEV,
OPEN_CHARDEV = 24,
RELEASE_CHARDEV,
MMAP_CHARDEV,
DESTROY_LCD
DESTROY_LCD = 27
};
/* CONTAINERS ------------------------------------------------------------ */
......@@ -104,6 +108,26 @@ struct request_queue_container {
cptr_t other_ref;
cptr_t my_ref;
};
#define NUM_CPUS 32
#if NUM_LCDS == 1
#define NUM_LCD_CPUS (NUM_LCDS + 1)
#define MAX_CHANNELS_PER_LCD (NUM_CPUS - NUM_LCD_CPUS)
#define NUM_THREADS_ON_NODE0 (NUM_CPUS_PER_NODE - NUM_LCD_CPUS)
#elif NUM_LCDS == 2
#define MAX_CHANNELS_PER_LCD 15
#define NUM_THREADS_ON_NODE0 5
#elif NUM_LCDS == 4
#define NUM_LCD_CPUS (NUM_LCDS + 1)
/* total LCD cores = 5 (lcds=4,klcd=1), free cores = 15 */
#define MAX_CHANNELS_PER_LCD 7
#define NUM_THREADS_ON_NODE0 6
#elif NUM_LCDS == 6
/* total LCD cores = 7 (lcds=6,klcd=1), free cores = 13 */
#define MAX_CHANNELS_PER_LCD 3
#define NUM_THREADS_ON_NODE0 6
#endif
#define MAX_CHNL_PAIRS MAX_CHANNELS_PER_LCD
/* CSPACES ------------------------------------------------------------ */
int glue_cap_init(void);
......@@ -224,6 +248,131 @@ async_msg_blocking_send_start(struct thc_channel *chnl,
return -EIO;
}
}
struct ring_stats {
unsigned long num_available;
unsigned long num_sent;
unsigned long num_other;
};
#endif /* _GLUE_HELPER_H_ */
static inline void collect_msg_statuses(struct ring_stats *stats, unsigned long num_slots,
unsigned long slot,
struct fipc_message *buffer)
{
int i;
for (i = 0; i < num_slots; i++) {
switch(buffer[(slot + i) % num_slots].msg_status) {
case FIPC_MSG_STATUS_AVAILABLE:
stats->num_available++;
break;
case FIPC_MSG_STATUS_SENT:
stats->num_sent++;
break;
default:
stats->num_other++;
break;
}
}
}
static inline void dump_ring_stats(struct thc_channel *chnl)
{
struct fipc_ring_channel *rc = thc_channel_to_fipc(chnl);
unsigned long tx_slot = rc->tx.slot;
unsigned long rx_slot = rc->rx.slot;
unsigned long num_tx_slots = rc->tx.order_two_mask;
unsigned long num_rx_slots = rc->rx.order_two_mask;
struct ring_stats tx_stats = {0}, rx_stats = {0};
collect_msg_statuses(&tx_stats, num_tx_slots, tx_slot, rc->tx.buffer);
collect_msg_statuses(&rx_stats, num_rx_slots, tx_slot, rc->rx.buffer);
printk("========== ring buf stats (Tx) ===========\n");
printk("[Tx] [%s:%d] Buffer: %p tx_slot: %lu num_slots: %lu\n",
current->comm, current->pid, rc->tx.buffer, tx_slot,
num_tx_slots);
printk("[Tx] current_slot:%lu status: %x\n", tx_slot, rc->tx.buffer[tx_slot].msg_status);
printk("[Tx] num_available: %lu num_sent: %lu num_other: %lu\n",
tx_stats.num_available, tx_stats.num_sent, tx_stats.num_other);
printk("========== ring buf stats (Rx) ===========\n");
printk("[Rx] [%s:%d] Buffer: %p tx_slot: %lu num_slots: %lu\n",
current->comm, current->pid, rc->rx.buffer, rx_slot,
num_rx_slots);
printk("[Rx] current_slot:%lu status: %x\n", rx_slot, rc->rx.buffer[rx_slot].msg_status);
printk("[Rx] num_available: %lu num_sent: %lu num_other: %lu\n",
rx_stats.num_available, rx_stats.num_sent, rx_stats.num_other);
printk("=====================================\n");
}
#define THRESHOLD (5 * 1000) /* 5 seconds */
#define fipc_test_pause() asm volatile ( "pause\n": : :"memory" );
static inline
int
fipc_msg_blocking_send_start(struct thc_channel *chnl,
struct fipc_message **out)
{
int ret;
int once = 1;
ktime_t start = ktime_get();
for (;;) {
/* Poll until we get a free slot or error */
ret = fipc_send_msg_start(thc_channel_to_fipc(chnl), out);
if (!ret || ret != -EWOULDBLOCK)
return ret;
fipc_test_pause();
if (ktime_to_ms(ktime_sub(ktime_get(), start)) >= THRESHOLD) {
if (once) {
once = 0;
dump_ring_stats(chnl);
printk("%s, could not get a slot for more than %d seconds!\n",
__func__, THRESHOLD / 1000);
}
}
}
}
static inline int
fipc_msg_blocking_recv_start(struct thc_channel *chnl,
struct fipc_message **response)
{
int ret;
int once = 1;
ktime_t start = ktime_get();
retry:
ret = fipc_recv_msg_start(thc_channel_to_fipc(chnl), response);
if (ret == 0) {
/*
* Message for us; remove request_cookie from awe mapper
*/
return 0;
} else if (ret == -EWOULDBLOCK) {
/*
* No messages in rx buffer; go to sleep.
*/
//cpu_relax();
fipc_test_pause();
if (ktime_to_ms(ktime_sub(ktime_get(), start)) >= THRESHOLD) {
if (once) {
once = 0;
dump_ring_stats(chnl);
printk("%s, could not get a slot for more than %d seconds!\n",
__func__, THRESHOLD / 1000);
}
}
goto retry;
} else {
/*
* Error
*/
printk(KERN_ERR "thc_ipc_recv_response: fipc returned %d\n",
ret);
return ret;
}
}
#endif /* _GLUE_HELPER_H_ */
......@@ -12,6 +12,7 @@
#include "../benchmark.h"
#include <lcd_config/post_hook.h>
#define CURRENT_LCD_ID current_lcd_id
cptr_t blk_register_chnls[NUM_LCDS];
cptr_t blk_sync_endpoints[NUM_LCDS];
......@@ -91,15 +92,14 @@ static void main_and_loop(void)
int ret;
int stop = 0;
struct fipc_message *msg;
struct thc_channel_group_item *curr_item;
/* initialize channel group list head */
init_chnl_group(&ch_grp[current_lcd_id]);
init_chnl_group(&ch_grp[CURRENT_LCD_ID]);
DO_FINISH(
/* parent LCD initializes the module */
if (current_lcd_id == 0) {
if (CURRENT_LCD_ID == 0) {
ASYNC(
ret = null_init();
if (ret) {
......@@ -120,13 +120,14 @@ static void main_and_loop(void)
* channel). */
LIBLCD_MSG("what is stop %d and nullb_done %d", stop, nullb_done);
while (!stop && !nullb_done) {
struct thc_channel_group_item *curr_item;
/*
* Do one async receive
*/
//ret = thc_ipc_poll_recv(blk_async_chl, &msg);
//TODO cleanup curr_item's memory!
//BENCH_BEGIN_LCD(disp_loop);
ret = thc_poll_recv_group_lcd(&ch_grp[current_lcd_id], &curr_item, &msg);
ret = thc_poll_recv_group_lcd(&ch_grp[CURRENT_LCD_ID], &curr_item, &msg);
if (ret) {
if (ret == -EWOULDBLOCK) {
continue;
......@@ -137,20 +138,32 @@ static void main_and_loop(void)
}
//(async_msg_get_fn_type(msg) == QUEUE_RQ_FN) ? bench_end() : -1;
if (async_msg_get_fn_type(msg) == QUEUE_RQ_FN) {
queue_rq_fn_callee(msg, curr_item->channel,
blk_cspace,
blk_sync_endpoints[CURRENT_LCD_ID]);
if (ret) {
LIBLCD_ERR("async dispatch failed");
stop = 1;
}
} else {
/*
* Got a message. Dispatch.
*/
ASYNC(
//printk("%s, dispatching msg: %d from channel: %p", __func__,
// async_msg_get_fn_type(msg), curr_item->channel);
ret = dispatch_async_loop(curr_item->channel, msg,
blk_cspace,
blk_sync_endpoints[current_lcd_id]);
blk_sync_endpoints[CURRENT_LCD_ID]);
if (ret) {
LIBLCD_ERR("async dispatch failed");
stop = 1;
}
);
}
}
LIBLCD_MSG("NULLB EXITED DISPATCH LOOP");
......@@ -195,12 +208,12 @@ static int __noreturn nullb_lcd_init(void)
* Remember - lcd_get_boot_info()->cptrs[0] holds the slot
* in the cspace where blk_chnl was granted by the boot module!
*/
blk_register_chnls[current_lcd_id] = lcd_get_boot_info()->cptrs[0];
printk("blk reg channel %lu\n", blk_register_chnls[current_lcd_id].cptr);
blk_register_chnls[CURRENT_LCD_ID] = lcd_get_boot_info()->cptrs[0];
printk("blk reg channel %lu\n", blk_register_chnls[CURRENT_LCD_ID].cptr);
/*
* Initialize nullb glue
*/
if (current_lcd_id == 0) {
if (CURRENT_LCD_ID == 0) {
r = glue_nullb_init();
if (r) {
LIBLCD_ERR("nullb init");
......@@ -214,7 +227,7 @@ static int __noreturn nullb_lcd_init(void)
/* DONE -------------------------------------------------- */
if (current_lcd_id == 0)
if (CURRENT_LCD_ID == 0)
glue_nullb_exit();
lcd_exit(0); /* doesn't return */
......
......@@ -15,6 +15,8 @@
#include <linux/lightnvm.h>
#include <liblcd/spinlock.h>
#include "../glue_helper.h"
#ifdef LCD_ISOLATE
#include <lcd_config/post_hook.h>
#endif
......@@ -45,12 +47,6 @@ struct nullb_queue {
struct nullb_cmd *cmds;
};
struct blk_mq_tag_set_container {
struct blk_mq_tag_set set;
u64 ref1;
u64 ref2;
};
struct nullb {
struct list_head list;
unsigned int index;
......@@ -95,7 +91,7 @@ enum {
#ifdef LCD_ISOLATE
/*TODO have to hardcode a value that nr_online_cpus return
* I doubt that nr_online_cpus will be accessible from here */
static int submit_queues;
static int submit_queues = NUM_CPUS - NUM_LCD_CPUS;
#else
static int submit_queues;
module_param(submit_queues, int, S_IRUGO);
......@@ -514,14 +510,8 @@ static int null_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
}
#ifdef LCD_ISOLATE
struct blk_mq_ops_container {
struct blk_mq_ops mq_ops;
u64 ref1;
u64 ref2;
};
static struct blk_mq_ops_container null_mq_ops_container = {
.mq_ops = {
.blk_mq_ops = {
.queue_rq = null_queue_rq,
.map_queue = blk_mq_map_queue,
.init_hctx = null_init_hctx,
......@@ -567,7 +557,7 @@ static void null_del_dev(struct nullb *nullb)
printk("calling blk_cleanup \n");
blk_cleanup_queue(nullb->q);
if (queue_mode == NULL_Q_MQ)
blk_mq_free_tag_set(&nullb->tag_set_container->set);
blk_mq_free_tag_set(&nullb->tag_set_container->blk_mq_tag_set);
if (!use_lightnvm)
put_disk(nullb->disk);
cleanup_queues(nullb);
......@@ -723,21 +713,11 @@ static void null_release(struct gendisk *disk, fmode_t mode)
}
#ifdef LCD_ISOLATE
struct block_device_operations_container {
struct block_device_operations null_fops;
u64 ref1;
u64 ref2;
};
struct module_container {
struct module module;
u64 ref1;
u64 ref2;
};
static struct module_container module_container;
static const struct block_device_operations_container null_ops_container = {
.null_fops = {
.block_device_operations = {
.owner = &module_container.module,
.open = null_open,
.release = null_release,
......@@ -842,8 +822,10 @@ static int null_add_dev(void)
#endif
//spin_lock_init(&nullb->lock);
if (queue_mode == NULL_Q_MQ && use_per_node_hctx)
if (queue_mode == NULL_Q_MQ && use_per_node_hctx) {
submit_queues = nr_online_nodes;
printk("submit_queues %d \n",submit_queues);
}
rv = setup_queues(nullb);
if (rv)
......@@ -851,32 +833,32 @@ static int null_add_dev(void)
if (queue_mode == NULL_Q_MQ) {
#ifdef LCD_ISOLATE
nullb->tag_set_container->set.ops = &null_mq_ops_container.mq_ops;
nullb->tag_set_container->set.nr_hw_queues = submit_queues;
nullb->tag_set_container->blk_mq_tag_set.ops = &null_mq_ops_container.blk_mq_ops;
nullb->tag_set_container->blk_mq_tag_set.nr_hw_queues = submit_queues;
printk("submit_queues %d \n",submit_queues);
nullb->tag_set_container->set.queue_depth = hw_queue_depth;
nullb->tag_set_container->set.numa_node = home_node;
nullb->tag_set_container->set.cmd_size = sizeof(struct nullb_cmd);
nullb->tag_set_container->set.flags = BLK_MQ_F_SHOULD_MERGE;
nullb->tag_set_container->blk_mq_tag_set.queue_depth = hw_queue_depth;
nullb->tag_set_container->blk_mq_tag_set.numa_node = home_node;
nullb->tag_set_container->blk_mq_tag_set.cmd_size = sizeof(struct nullb_cmd);
nullb->tag_set_container->blk_mq_tag_set.flags = BLK_MQ_F_SHOULD_MERGE;
/* TODO allocate memory for nullb in the klcd glue, exchange my_ref
and other_ref */
nullb->tag_set_container->set.driver_data = nullb;
nullb->tag_set_container->blk_mq_tag_set.driver_data = nullb;
rv = blk_mq_alloc_tag_set(&nullb->tag_set_container->set);
rv = blk_mq_alloc_tag_set(&nullb->tag_set_container->blk_mq_tag_set);
printk("blk_mq alloc tag set retruns %d \n",rv);
if (rv)
goto out_cleanup_queues;
/*TODO AB - allocate request_queue container in the glue. IDL doesn't
create container for ptr returned! */
nullb->q = blk_mq_init_queue(&nullb->tag_set_container->set);
nullb->q = blk_mq_init_queue(&nullb->tag_set_container->blk_mq_tag_set);
if (IS_ERR(nullb->q)) {
rv = -ENOMEM;
printk("blk_mq_init_queue returns NULL! \n");
goto out_cleanup_tags;
}
#else
nullb->tag_set.ops = &null_mq_ops_container.mq_ops;
nullb->tag_set.ops = &null_mq_ops_container.blk_mq_ops;
nullb->tag_set.nr_hw_queues = submit_queues;
nullb->tag_set.queue_depth = hw_queue_depth;
nullb->tag_set.numa_node = home_node;
......@@ -955,7 +937,7 @@ static int null_add_dev(void)
disk->flags |= GENHD_FL_EXT_DEVT | GENHD_FL_SUPPRESS_PARTITION_INFO;
disk->major = null_major;
disk->first_minor = nullb->index;
disk->fops = &null_ops_container.null_fops;
disk->fops = &null_ops_container.block_device_operations;
/* TODO Nullb's memory will be allocated in the klcd glue, so my_ref for nullb
should be marshalled here */
disk->private_data = nullb;
......@@ -989,7 +971,7 @@ out_cleanup_blk_queue:
out_cleanup_tags:
if (queue_mode == NULL_Q_MQ)
blk_mq_free_tag_set(&nullb->tag_set_container->set);
blk_mq_free_tag_set(&nullb->tag_set_container->blk_mq_tag_set);
out_cleanup_queues:
cleanup_queues(nullb);
......@@ -1038,11 +1020,13 @@ int null_init(void)
nr_online_nodes);
submit_queues = nr_online_nodes;
}
} else if (submit_queues > nr_cpu_ids)
}
#if 0
else if (submit_queues > nr_cpu_ids)
submit_queues = nr_cpu_ids;
else if (!submit_queues)
submit_queues = 1;
#endif
mutex_init(&lock);
null_major = register_blkdev(0, "nullb");
......