...
 
Commits (6)
......@@ -10,6 +10,9 @@
#define SKB_CONTAINER_SIZE 128
#define BLK_USER_BUF_SIZE (1 << 20) /* 1 MB blocks */
typedef enum {
/* for skb->data */
SKB_DATA_POOL = 0,
......@@ -20,6 +23,9 @@ typedef enum {
/* for skb_container */
SKB_CONTAINER_POOL,
/* nullblk iocb pool */
BLK_USER_BUF_POOL,
POOL_MAX,
} pool_type_t;
......
......@@ -164,9 +164,10 @@ static void do_mem_itree_delete(struct lcd_mem_itree *tree,
__lcd_memory_object_start(mo),
&node);
if (ret) {
LCD_ERR("couldn't find memory object in tree, skipping delete");
LCD_ERR("couldn't find memory object %p (nr_pages %d) in tree, skipping delete!",
mo->object, mo->nr_pages);
mutex_unlock(&tree->lock);
BUG();
WARN(ret, "%s:\n", __func__);
return;
}
/*
......
......@@ -10,6 +10,9 @@
#include "../nullb_callee.h"
#include <asm/cacheflush.h>
#include <linux/atomic.h>
#include <linux/priv_mempool.h>
#include <linux/sort.h>
#include <lcd_domains/microkernel.h>
#include <lcd_config/post_hook.h>
......@@ -35,9 +38,33 @@ struct trampoline_hidden_args {
};
#endif
priv_pool_t *iocb_pool;
void *pool_base = NULL;
size_t pool_size = 0;
static unsigned long pool_pfn_start, pool_pfn_end;
static struct glue_cspace *c_cspace;
extern struct cspace *klcd_cspace;
#define IOCB_POOL_SIZE (256UL << 20)
#define IOCB_POOL_PAGES (IOCB_POOL_SIZE >> PAGE_SHIFT)
#define IOCB_POOL_ORDER ilog2(IOCB_POOL_PAGES)
void iocb_data_pool_init(void)
{
pool_base = vzalloc(IOCB_POOL_SIZE);
pool_size = IOCB_POOL_SIZE;
iocb_pool = priv_pool_init(BLK_USER_BUF_POOL, (void*) pool_base, pool_size, BLK_USER_BUF_SIZE);
}
void iocb_data_pool_free(void)
{
if (iocb_pool) {
vfree(iocb_pool->pool);
priv_pool_destroy(iocb_pool);
}
}
int glue_blk_init(void)
{
int ret;
......@@ -51,6 +78,8 @@ int glue_blk_init(void)
LIBLCD_ERR("cap create");
goto fail2;
}
iocb_data_pool_init();
return 0;
fail2:
glue_cap_exit();
......@@ -63,6 +92,7 @@ void glue_blk_exit(void)
{
glue_cap_destroy(c_cspace);
glue_cap_exit();
iocb_data_pool_free();
}
int blk_mq_init_queue_callee(struct fipc_message *request)
......@@ -92,7 +122,8 @@ int blk_mq_init_queue_callee(struct fipc_message *request)
rq_container = container_of(rq, struct request_queue_container,
request_queue);
ret = glue_cap_insert_request_queue_type(c_cspace, rq_container, &rq_container->my_ref);
ret = glue_cap_insert_request_queue_type(c_cspace, rq_container,
&rq_container->my_ref);
if (ret) {
LIBLCD_ERR("lcd insert");
goto fail_insert;
......@@ -232,8 +263,9 @@ int blk_queue_physical_block_size_callee(struct fipc_message *request)
int ret;
size = fipc_get_reg0(request);
ret = glue_cap_lookup_request_queue_type(c_cspace, __cptr(fipc_get_reg1(request)),
&rq_container);
ret = glue_cap_lookup_request_queue_type(c_cspace,
__cptr(fipc_get_reg1(request)),
&rq_container);
if (ret) {
LIBLCD_ERR("lookup");
goto fail_lookup;
......@@ -269,7 +301,8 @@ int alloc_disk_node_callee(struct fipc_message *request)
/* Hack for remove */
disk_g = disk;
ret = glue_cap_insert_gendisk_type(c_cspace, disk_container, &disk_container->my_ref);
ret = glue_cap_insert_gendisk_type(c_cspace, disk_container,
&disk_container->my_ref);
if (ret) {
LIBLCD_ERR("lcd insert");
goto fail_insert;
......@@ -344,13 +377,8 @@ int del_gendisk_callee(struct fipc_message *request)
disk = &disk_container->gendisk;
printk("Calling del_gendisk on cpu: %d, disk %p \n", raw_smp_processor_id(), disk);
{
struct kobject *kobj = &disk_to_dev(disk)->kobj;
printk("%s, disk_to_dev(disk) %p, kobj %p, kobj->sd %p\n", __func__,
disk_to_dev(disk), kobj, kobj->sd);
}
printk("Calling del_gendisk on cpu: %d, disk %p \n",
raw_smp_processor_id(), disk);
del_gendisk(disk);
......@@ -358,20 +386,6 @@ fail_lookup:
return ret;
}
#ifndef CONFIG_LVD
LCD_TRAMPOLINE_DATA(mmap_fops_trampoline);
int LCD_TRAMPOLINE_LINKAGE(mmap_fops_trampoline) mmap_fops_trampoline(struct file *filp, struct vm_area_struct *vma)
{
int ( *volatile mmap_fp )(struct file *filp, struct vm_area_struct *vma, struct trampoline_hidden_args *);
struct trampoline_hidden_args *hidden_args;
LCD_TRAMPOLINE_PROLOGUE(hidden_args, mmap_fops_trampoline);
mmap_fp = nullbu_mmap;
return mmap_fp(filp, vma, hidden_args);
}
#endif
int register_blkdev_callee(struct fipc_message *request)
{
int ret;
......@@ -382,7 +396,7 @@ int register_blkdev_callee(struct fipc_message *request)
ret = register_blkdev(devno, "nullb");
LIBLCD_MSG("register_blkdev returns %d", ret);
printk("register_blkdev returns %d\n", ret);
if (ret < 0) {
LIBLCD_ERR("register_blkdev failed! ret = %d", ret);
......@@ -410,7 +424,8 @@ int blk_cleanup_queue_callee(struct fipc_message *request)
struct request_queue_container *rq_container;
int ret = 0;
ret = glue_cap_lookup_request_queue_type(c_cspace, __cptr(fipc_get_reg0(request)),
ret = glue_cap_lookup_request_queue_type(c_cspace,
__cptr(fipc_get_reg0(request)),
&rq_container);
if (ret) {
LIBLCD_ERR("lookup");
......@@ -439,19 +454,16 @@ int _queue_rq_fn(struct blk_mq_hw_ctx *ctx, const struct blk_mq_queue_data *bd,
#endif
struct blk_mq_hw_ctx_container *ctx_container;
//printk("^^^^^^^^^^ [Klcd-queue-rq] enter ^^^^^^^^^^^ \n");
/*XXX Beware!! hwctx can be unique per hw context of the driver, if multiple
* exists, then we need one cspace insert function per hwctx. Should be handled
* in the init_hctx routine */
ctx_container = container_of(ctx, struct blk_mq_hw_ctx_container, blk_mq_hw_ctx);
ctx_container = container_of(ctx, struct blk_mq_hw_ctx_container,
blk_mq_hw_ctx);
#ifndef CONFIG_LVD
ops_container = (struct blk_mq_ops_container *)hidden_args->struct_container;
#endif
//printk("hctx %p | cptr %lu\n",ctx, ctx_container->other_ref.cptr);
#if 0
async_msg_set_fn_type(request, QUEUE_RQ_FN);
fipc_set_reg0(request, ctx->queue_num);
......@@ -460,26 +472,18 @@ int _queue_rq_fn(struct blk_mq_hw_ctx *ctx, const struct blk_mq_queue_data *bd,
#ifndef CONFIG_LVD
fipc_set_reg2(request, ops_container->other_ref.cptr);
#endif
fipc_set_reg3(request, bd->rq->tag);
vmfunc_klcd_wrapper(request, 1);
#endif
blk_mq_start_request(bd->rq);
blk_mq_end_request(bd->rq, 0);
vmfunc_klcd_wrapper(request, 1);
/* function ret - makes no sense now but keeping it this way! */
ret = fipc_get_reg0(request);
/* printk("^^^^^ [Klcd-queue-rq] done ^^^^^^ \n"); */
return BLK_MQ_RQ_QUEUE_OK;
//return ret;
printk("[Klcd-queue-rq] done with err \n");
return ret;
}
int init_request(void *data, struct request *req, unsigned int hctx_idx, unsigned int rq_idx,
unsigned int numa_node)
int init_request(void *data, struct request *req, unsigned int hctx_idx,
unsigned int rq_idx,
unsigned int numa_node)
{
static int init_done = 0;
......@@ -523,6 +527,7 @@ struct blk_mq_hw_ctx *_map_queue_fn(struct request_queue *rq, int m, struct tram
* the map. All the kernel does is to call blk_mq_map_queue.
* So instead of going to the LCD, I am going to call
* blk_mq_map_queue here! */
/* TODO: Call to LCD and call this function from there */
return blk_mq_map_queue(rq, m);
}
......@@ -540,6 +545,44 @@ struct blk_mq_hw_ctx *LCD_TRAMPOLINE_LINKAGE(map_queue_fn_trampoline) map_queue_
}
#endif
int init_hctx_sync_callee(struct fipc_message *_request)
{
int ret;
unsigned int pool_ord;
cptr_t pool_cptr;
cptr_t lcd_pool_cptr;
/* get LCD's pool cptr */
lcd_pool_cptr = __cptr(fipc_get_reg0(_request));
pool_ord = ilog2(roundup_pow_of_two((
iocb_pool->total_pages * PAGE_SIZE)
>> PAGE_SHIFT));
ret = lcd_volunteer_vmalloc_mem(__gva((unsigned long) iocb_pool->pool),
IOCB_POOL_PAGES, &pool_cptr);
if (ret) {
LIBLCD_ERR("volunteer shared data pool");
goto fail_vol;
}
pool_pfn_start = (unsigned long)iocb_pool->pool >> PAGE_SHIFT;
pool_pfn_end = pool_pfn_start + iocb_pool->total_pages;
printk("%s, pool pfn start %lu | end %lu\n", __func__,
pool_pfn_start, pool_pfn_end);
copy_msg_cap_vmfunc(current->lcd, current->vmfunc_lcd, pool_cptr, lcd_pool_cptr);
fipc_set_reg0(_request, pool_ord);
fail_vol:
return 0;
}
#ifdef CONFIG_LVD
int _init_hctx_fn(struct blk_mq_hw_ctx *ctx, void *data, unsigned int index)
#else
......@@ -581,9 +624,8 @@ int _init_hctx_fn(struct blk_mq_hw_ctx *ctx, void *data, unsigned int index, str
ctx_container->other_ref.cptr = fipc_get_reg0(request);
func_ret = fipc_get_reg1(request);
printk("init_hctx hctx %p | cptr %lu\n",ctx, ctx_container->other_ref.cptr);
printk("init_hctx hctx %p | cptr %lu\n", ctx, ctx_container->other_ref.cptr);
printk("end of init_hctx procedure \n");
fail_insert:
return func_ret;
......@@ -646,7 +688,8 @@ int open(struct block_device *device, fmode_t mode, struct trampoline_hidden_arg
async_msg_set_fn_type(request, OPEN);
fipc_set_reg0(request, mode);
printk("%s, %s:%d on cpu:%d\n", __func__, current->comm, current->pid, smp_processor_id());
printk("%s, %s:%d on cpu:%d\n", __func__, current->comm, current->pid,
smp_processor_id());
vmfunc_klcd_wrapper(request, 1);
ret = fipc_get_reg0(request);
......@@ -666,7 +709,8 @@ void release(struct gendisk *disk, fmode_t mode, struct trampoline_hidden_args *
disk_container = container_of(disk, struct gendisk_container, gendisk);
printk("%s, %s:%d on cpu:%d\n", __func__, current->comm, current->pid, smp_processor_id());
printk("%s, %s:%d on cpu:%d\n", __func__, current->comm, current->pid,
smp_processor_id());
async_msg_set_fn_type(request, RELEASE);
fipc_set_reg0(request, disk_container->other_ref.cptr);
......@@ -891,7 +935,7 @@ int blk_mq_alloc_tag_set_callee(struct fipc_message *request)
set_container->tag_set.cmd_size = fipc_get_reg4(request);
set_container->tag_set.flags = fipc_get_reg5(request);
/* init request */
/* XXX: Populate function pointers including init_request */
ops_container->blk_mq_ops.init_request = init_request;
ops_container->blk_mq_ops.queue_rq = _queue_rq_fn;
ops_container->blk_mq_ops.map_queue = _map_queue_fn;
......@@ -1107,6 +1151,7 @@ int device_add_disk_callee(struct fipc_message *request)
strncpy(disk->disk_name, disk_name, DISK_NAME_LEN);
printk("Calling add_disk on cpu: %d, disk %p \n", raw_smp_processor_id(), disk);
add_disk(disk);
{
......
......@@ -11,6 +11,9 @@ int blk_klcd_syncipc_dispatch(struct fipc_message *message)
fn_type = async_msg_get_fn_type(message);
switch (fn_type) {
case INIT_HCTX_SYNC:
trace(INIT_HCTX_SYNC);
return init_hctx_sync_callee(message);
default:
LIBLCD_ERR("unexpected function label %d", fn_type);
return -EINVAL;
......@@ -34,7 +37,7 @@ int blk_klcd_dispatch_loop(struct fipc_message *message)
return blk_mq_init_queue_callee(message);
case BLK_MQ_END_REQUEST:
trace(BLK_MQ_END_REQUEST);
/* trace(BLK_MQ_END_REQUEST); */
return blk_mq_end_request_callee(message);
case BLK_MQ_FREE_TAG_SET:
......@@ -42,7 +45,7 @@ int blk_klcd_dispatch_loop(struct fipc_message *message)
return blk_mq_free_tag_set_callee(message);
case BLK_MQ_START_REQUEST:
trace(BLK_MQ_START_REQUEST);
/* trace(BLK_MQ_START_REQUEST); */
return blk_mq_start_request_callee(message);
case BLK_MQ_MAP_QUEUE:
......
......@@ -24,4 +24,6 @@ int unregister_blkdev_callee(struct fipc_message *request);
int blk_cleanup_queue_callee(struct fipc_message *request);
int lcd_register_chardev_callee(struct fipc_message *request);
int init_hctx_sync_callee(struct fipc_message *request);
#endif /* __NULLB_CALLEE_H__ */
......@@ -38,6 +38,7 @@ enum dispatch_t {
QUEUE_RQ_FN,
MAP_QUEUE_FN,
INIT_HCTX_FN,
INIT_HCTX_SYNC,
SOFTIRQ_DONE_FN,
OPEN,
RELEASE,
......
#include <lcd_config/pre_hook.h>
//#include <linux/fs.h>
#include <libcap.h>
#include <liblcd/liblcd.h>
#include <liblcd/sync_ipc_poll.h>
......@@ -12,6 +10,7 @@
#include "../../benchmark.h"
static struct glue_cspace *c_cspace;
void *iocb_data_pool;
struct blk_mq_hw_ctx_container *ctx_container_g;
struct blk_mq_ops_container *ops_container_g;
......@@ -170,8 +169,7 @@ void blk_mq_end_request(struct request *rq, int error)
struct fipc_message *request = &r;
struct lcd_request_container *rq_c;
rq_c = container_of(rq,
struct lcd_request_container, rq);
rq_c = container_of(rq, struct lcd_request_container, rq);
async_msg_set_fn_type(request, BLK_MQ_END_REQUEST);
......@@ -193,7 +191,7 @@ void blk_mq_free_tag_set(struct blk_mq_tag_set *set)
set_container = container_of(set, struct blk_mq_tag_set_container, tag_set);
ops_container = container_of(set->ops, struct blk_mq_ops_container,
blk_mq_ops);
blk_mq_ops);
async_msg_set_fn_type(request, BLK_MQ_FREE_TAG_SET);
......@@ -327,7 +325,8 @@ void device_add_disk(struct device *parent, struct gendisk *disk)
disk_container = container_of(disk, struct gendisk_container, gendisk);
blo_container = container_of(disk->fops,
struct block_device_operations_container, block_device_operations);
struct block_device_operations_container,
block_device_operations);
module_container = container_of(disk->fops->owner, struct module_container,
module);
......@@ -362,14 +361,14 @@ void device_add_disk(struct device *parent, struct gendisk *disk)
fipc_set_reg5(request, disk->major);
fipc_set_reg6(request, disk->first_minor);
/* Ran out of registers to marshall the string, so hardcoding it
* in the klcd */
/*
* Ran out of registers to marshall the string, so hardcoding it
* in the klcd
*/
vmfunc_wrapper(request);
blo_container->other_ref.cptr = fipc_get_reg0(request);
module_container->other_ref.cptr = fipc_get_reg1(request);
printk("add_disk ends here in lcd glue \n");
return;
......@@ -534,7 +533,9 @@ int init_hctx_fn_callee(struct fipc_message *request)
struct blk_mq_hw_ctx_container *ctx_container;
struct blk_mq_ops_container *ops_container;
unsigned int index;
cptr_t pool_cptr;
gva_t pool_addr;
unsigned int pool_ord;
int ret = 0;
ctx_container = kzalloc(sizeof(*ctx_container), GFP_KERNEL);
......@@ -560,7 +561,29 @@ int init_hctx_fn_callee(struct fipc_message *request)
LIBLCD_ERR("lookup");
goto fail_lookup;
}
/* receive shared data pool */
ret = lcd_cptr_alloc(&pool_cptr);
if (ret) {
LIBLCD_ERR("failed to get cptr");
goto fail_cptr;
}
fipc_set_reg0(request, cptr_val(pool_cptr));
vmfunc_sync_call(request, INIT_HCTX_SYNC);
pool_ord = fipc_get_reg0(request);
ret = lcd_map_virt(pool_cptr, pool_ord, &pool_addr);
if (ret) {
LIBLCD_ERR("failed to map pool");
goto fail_pool;
}
LIBLCD_MSG("%s, mapping private pool %p | ord %d", __func__,
gva_val(pool_addr), pool_ord);
iocb_data_pool = (void*)gva_val(pool_addr);
/* Passing NULL to data arg, hack to get nullb's address within the driver */
ret = ops_container->blk_mq_ops.init_hctx(&ctx_container->blk_mq_hw_ctx, NULL, index);
if(ret) {
......@@ -568,11 +591,12 @@ int init_hctx_fn_callee(struct fipc_message *request)
goto fail_hctx;
}
LIBLCD_MSG("lcd ctx.cptr %lu", ctx_container->my_ref.cptr);
fipc_set_reg0(request, ctx_container->my_ref.cptr);
fipc_set_reg1(request, ret);
return ret;
fail_pool:
fail_cptr:
fail_hctx:
fail_lookup:
glue_cap_remove(c_cspace, ctx_container->my_ref);
......@@ -600,6 +624,7 @@ int open_callee(struct fipc_message *_request)
int ret;
fmode_t mode = fipc_get_reg0(_request);
ret = null_open(NULL, mode);
fipc_set_reg0(_request, ret);
......
......@@ -192,7 +192,10 @@ static void dummy_setup(struct net_device *dev)
dev->features |= NETIF_F_SG | NETIF_F_FRAGLIST;
dev->features |= NETIF_F_ALL_TSO | NETIF_F_UFO;
dev->features |= NETIF_F_HW_CSUM | NETIF_F_HIGHDMA | NETIF_F_LLTX | NETIF_F_PRIV_DATA_POOL;
dev->features |= NETIF_F_GSO_ENCAP_ALL | NETIF_F_CHAIN_SKB;
dev->features |= NETIF_F_GSO_ENCAP_ALL;
#ifndef CONFIG_LVD
dev->features |= NETIF_F_CHAIN_SKB;
#endif
dev->hw_features |= dev->features;
dev->hw_enc_features |= dev->features;
......
......@@ -955,7 +955,7 @@ fail8:
}
#endif
void setup_sync_callee(struct fipc_message *msg)
void setup_sync_callee(struct fipc_message *_request)
{
int ret;
struct page *p;
......@@ -964,7 +964,7 @@ void setup_sync_callee(struct fipc_message *msg)
cptr_t lcd_pool_cptr;
/* get LCD's pool cptr */
lcd_pool_cptr = __cptr(msg->regs[0]);
lcd_pool_cptr = __cptr(fipc_get_reg0(_request));
p = virt_to_head_page(pool->pool);
......@@ -987,7 +987,7 @@ void setup_sync_callee(struct fipc_message *msg)
copy_msg_cap_vmfunc(current->lcd, current->vmfunc_lcd, pool_cptr, lcd_pool_cptr);
fipc_set_reg0(msg, pool_ord);
fipc_set_reg0(_request, pool_ord);
fail_vol:
return;
......