...
 
Commits (29)
......@@ -33,12 +33,58 @@
#include <linux/ratelimit.h>
#include <linux/pm_runtime.h>
#include <linux/blk-cgroup.h>
#include <linux/blk-bench.h>
#include <linux/bdump.h>
#define CREATE_TRACE_POINTS
#include <trace/events/block.h>
#include "blk.h"
#include "blk-mq.h"
#include <linux/blk-lcd.h>
//static bool __always_inline check_fio(void) {
// if(strcmp(current->comm, "fio") == 0) {
// return true;
// }
// return false;
//}
//INIT_BENCHMARK_DATA(plug);
//INIT_BENCHMARK_DATA(plug_ot);
//INIT_BENCHMARK_DATA(plug_ots);
//static void __always_inline bench_start_plos(void) {
// BENCH_BEGIN(plug_ots);
//}
//static void __always_inline bench_end_plos(void) {
// BENCH_END(plug_ots);
//}
//static void __always_inline bench_start_plo(void) {
// BENCH_BEGIN(plug_ot);
//}
//static void __always_inline bench_end_plo(void) {
// BENCH_END(plug_ot);
//}
//static void __always_inline bench_start_pl(void) {
// BENCH_BEGIN(plug);
//}
//static void __always_inline bench_end_pl(void) {
// BENCH_END(plug);
//}
//void bdump_data(void) {
// BENCH_COMPUTE_STAT(plug);
/// BENCH_COMPUTE_STAT(plug_ot);
// BENCH_COMPUTE_STAT(plug_ots);
//}
struct request_queue *queue_nullb = NULL;
EXPORT_SYMBOL(queue_nullb);
EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_remap);
EXPORT_TRACEPOINT_SYMBOL_GPL(block_rq_remap);
......@@ -148,11 +194,17 @@ static void req_bio_endio(struct request *rq, struct bio *bio,
if (unlikely(rq->cmd_flags & REQ_QUIET))
bio_set_flag(bio, BIO_QUIET);
//queue_nullb ? printk("bio_advance: %d \n",nbytes) : queue_nullb = NULL;
bio_advance(bio, nbytes);
/* don't actually finish bio if it's part of flush sequence */
if (bio->bi_iter.bi_size == 0 && !(rq->cmd_flags & REQ_FLUSH_SEQ))
if (bio->bi_iter.bi_size == 0 && !(rq->cmd_flags & REQ_FLUSH_SEQ)) {
//queue_nullb ? printk("calling bio_endio \n") : queue_nullb = NULL;
//BENCH_BEGIN(io_complete);
bio_endio(bio);
//BENCH_END(io_complete);
}
}
void blk_dump_rq_flags(struct request *rq, char *msg)
......@@ -545,6 +597,12 @@ void blk_cleanup_queue(struct request_queue *q)
{
spinlock_t *lock = q->queue_lock;
//if(queue_nullb) {
// BENCH_COMPUTE_STAT(io_complete);
// bdump_data();
//}
/* mark @q DYING, no new request or merges will be allowed afterwards */
mutex_lock(&q->sysfs_lock);
blk_set_queue_dying(q);
......@@ -691,14 +749,18 @@ static void blk_rq_timed_out_timer(unsigned long data)
struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id)
{
struct request_queue_container *rq_cnt;
struct request_queue *q;
int err;
q = kmem_cache_alloc_node(blk_requestq_cachep,
rq_cnt = kmem_cache_alloc_node(blk_requestq_cachep,
gfp_mask | __GFP_ZERO, node_id);
if (!q)
if (!rq_cnt)
return NULL;
q = &rq_cnt->rq;
q->id = ida_simple_get(&blk_queue_ida, 0, 0, gfp_mask);
if (q->id < 0)
goto fail_q;
......@@ -773,7 +835,8 @@ fail_split:
fail_id:
ida_simple_remove(&blk_queue_ida, q->id);
fail_q:
kmem_cache_free(blk_requestq_cachep, q);
//kmem_cache_free(blk_requestq_cachep, q);
kmem_cache_free(blk_requestq_cachep, rq_cnt);
return NULL;
}
EXPORT_SYMBOL(blk_alloc_queue_node);
......@@ -1997,6 +2060,8 @@ blk_qc_t generic_make_request(struct bio *bio)
struct bio_list bio_list_on_stack;
blk_qc_t ret = BLK_QC_T_NONE;
//(check_fio() == true) ? printk("[%s:%s] --> in \n",__FILE__,__func__) : -1;
if (!generic_make_request_checks(bio))
goto out;
......@@ -2011,6 +2076,7 @@ blk_qc_t generic_make_request(struct bio *bio)
* should be added at the tail
*/
if (current->bio_list) {
//(check_fio() == true) ? printk("adding bio to the current->bio_list \n") : -1;
bio_list_add(current->bio_list, bio);
goto out;
}
......@@ -2036,20 +2102,25 @@ blk_qc_t generic_make_request(struct bio *bio)
struct request_queue *q = bdev_get_queue(bio->bi_bdev);
if (likely(blk_queue_enter(q, false) == 0)) {
//(check_fio() == true) ? printk("calling make_request_fn \n") : -1;
//(check_fio() == true) ? bench_start_pl() : -1;
ret = q->make_request_fn(q, bio);
//(check_fio() == true) ? bench_end_pl() : -1;
blk_queue_exit(q);
bio = bio_list_pop(current->bio_list);
} else {
struct bio *bio_next = bio_list_pop(current->bio_list);
//(check_fio() == true) ? printk("pop bio list \n") : -1;
bio_io_error(bio);
bio = bio_next;
}
} while (bio);
current->bio_list = NULL; /* deactivate */
//(check_fio() == true) ? printk("[%s:%s] <-- out \n",__FILE__,__func__) : -1;
out:
return ret;
}
......@@ -2066,13 +2137,16 @@ EXPORT_SYMBOL(generic_make_request);
*/
blk_qc_t submit_bio(struct bio *bio)
{
//blk_qc_t ret;
/*
* If it's a regular read/write or a barrier with data attached,
* go through the normal accounting stuff before submission.
*/
//(check_fio() == true) ? printk("[%s:%s] --> in \n",__FILE__,__func__) : -1;
if (bio_has_data(bio)) {
unsigned int count;
//(check_fio() == true) ? printk("bio has data \n") : -1;
if (unlikely(bio_op(bio) == REQ_OP_WRITE_SAME))
count = bdev_logical_block_size(bio->bi_bdev) >> 9;
else
......@@ -2081,6 +2155,7 @@ blk_qc_t submit_bio(struct bio *bio)
if (op_is_write(bio_op(bio))) {
count_vm_events(PGPGOUT, count);
} else {
//(check_fio() == true) ? printk("task_io_account \n") : -1;
task_io_account_read(bio->bi_iter.bi_size);
count_vm_events(PGPGIN, count);
}
......@@ -2096,7 +2171,10 @@ blk_qc_t submit_bio(struct bio *bio)
}
}
//(check_fio() == true) ? bench_start_pl() : -1;
return generic_make_request(bio);
//(check_fio() == true) ? bench_end_pl() : -1;
//return ret;
}
EXPORT_SYMBOL(submit_bio);
......@@ -2593,22 +2671,34 @@ bool blk_update_request(struct request *req, int error, unsigned int nr_bytes)
blk_account_io_completion(req, nr_bytes);
//if(queue_nullb) {
//BENCH_BEGIN(io_complete);
//printk("----> into while \n");
//}
total_bytes = 0;
while (req->bio) {
struct bio *bio = req->bio;
unsigned bio_bytes = min(bio->bi_iter.bi_size, nr_bytes);
if (bio_bytes == bio->bi_iter.bi_size)
if (bio_bytes == bio->bi_iter.bi_size) {
// queue_nullb ? printk("hitting here \n") : queue_nullb = NULL;
req->bio = bio->bi_next;
}
//queue_nullb ? printk("calling req_bio_endio: %d \n", bio_bytes) : queue_nullb= NULL;
req_bio_endio(req, bio, bio_bytes, error);
total_bytes += bio_bytes;
nr_bytes -= bio_bytes;
//queue_nullb ? printk("nr_bytes: %d, total_bytes: %d \n", nr_bytes, total_bytes) : queue_nullb = NULL;
if (!nr_bytes)
break;
}
//if(queue_nullb) {
//printk("<--- out of while \n");
//BENCH_END(io_complete);
//}
/*
* completely done
......@@ -3224,17 +3314,24 @@ void blk_flush_plug_list(struct blk_plug *plug, bool from_schedule)
LIST_HEAD(list);
unsigned int depth;
//(check_fio() == true) ? printk("[%s:%s] --> in \n",__FILE__,__func__) : -1;
flush_plug_callbacks(plug, from_schedule);
if (!list_empty(&plug->mq_list))
if (!list_empty(&plug->mq_list)) {
//(check_fio() == true) ? printk("calling flush plug list \n") : -1;
//(check_fio() == true) ? bench_start_pl() : -1;
blk_mq_flush_plug_list(plug, from_schedule);
//(check_fio() == true) ? bench_end_pl() : -1;
}
if (list_empty(&plug->list))
return;
list_splice_init(&plug->list, &list);
//(check_fio() == true) ? bench_start_plos() : -1;
list_sort(NULL, &list, plug_rq_cmp);
//(check_fio() == true) ? bench_end_plos() : -1;
q = NULL;
depth = 0;
......@@ -3243,6 +3340,7 @@ void blk_flush_plug_list(struct blk_plug *plug, bool from_schedule)
* Save and disable interrupts here, to avoid doing it for every
* queue lock we have to take.
*/
//(check_fio() == true) ? bench_start_plo() : -1;
local_irq_save(flags);
while (!list_empty(&list)) {
rq = list_entry_rq(list.next);
......@@ -3285,6 +3383,8 @@ void blk_flush_plug_list(struct blk_plug *plug, bool from_schedule)
queue_unplugged(q, depth, from_schedule);
local_irq_restore(flags);
//(check_fio() == true) ? bench_end_plo() : -1;
//(check_fio() == true) ? printk("[%s:%s] <-- in \n",__FILE__,__func__) : -1;
}
void blk_finish_plug(struct blk_plug *plug)
......@@ -3532,8 +3632,12 @@ int __init blk_dev_init(void)
request_cachep = kmem_cache_create("blkdev_requests",
sizeof(struct request), 0, SLAB_PANIC, NULL);
//blk_requestq_cachep = kmem_cache_create("request_queue",
// sizeof(struct request_queue), 0, SLAB_PANIC, NULL);
/* AB - This is for LCD */
blk_requestq_cachep = kmem_cache_create("request_queue",
sizeof(struct request_queue), 0, SLAB_PANIC, NULL);
sizeof(struct request_queue_container), 0, SLAB_PANIC, NULL);
return 0;
}
......@@ -11,6 +11,13 @@
#include "blk.h"
static bool __always_inline check_fio(void) {
if(strcmp(current->comm, "fio") == 0) {
return true;
}
return false;
}
static struct bio *blk_bio_discard_split(struct request_queue *q,
struct bio *bio,
struct bio_set *bs,
......@@ -194,15 +201,19 @@ void blk_queue_split(struct request_queue *q, struct bio **bio,
struct bio *split, *res;
unsigned nsegs;
//(check_fio() == true) ? printk("[%s:%s] in --> \n",__FILE__,__func__) : -1;
switch (bio_op(*bio)) {
case REQ_OP_DISCARD:
case REQ_OP_SECURE_ERASE:
//(check_fio() == true) ? printk("REQ_OP_SECURE_ERASE: \n") : -1;
split = blk_bio_discard_split(q, *bio, bs, &nsegs);
break;
case REQ_OP_WRITE_SAME:
//(check_fio() == true) ? printk("REQ_OP_WRITE_SAME: \n") : -1;
split = blk_bio_write_same_split(q, *bio, bs, &nsegs);
break;
default:
//(check_fio() == true) ? printk("default: \n") : -1;
split = blk_bio_segment_split(q, *bio, q->bio_split, &nsegs);
break;
}
......@@ -218,9 +229,11 @@ void blk_queue_split(struct request_queue *q, struct bio **bio,
bio_chain(split, *bio);
trace_block_split(q, split, (*bio)->bi_iter.bi_sector);
//(check_fio() == true) ? printk("calling make req again \n") : -1;
generic_make_request(*bio);
*bio = split;
}
//(check_fio() == true) ? printk("[%s:%s] <-- out \n",__FILE__,__func__) : -1;
}
EXPORT_SYMBOL(blk_queue_split);
......
......@@ -19,6 +19,29 @@
#include "blk-mq.h"
#include "blk-mq-tag.h"
#include <linux/blk-bench.h>
#include <linux/bdump.h>
//INIT_BENCHMARK_DATA(gettag);
//static bool __always_inline check_fio(void) {
// if(strcmp(current->comm, "fio") == 0) {
// return true;
// }
// return false;
//}
//static void __always_inline bench_start(void) {
// BENCH_BEGIN(gettag);
//}
//static void __always_inline bench_end(void) {
// BENCH_END(gettag);
//}
//void bdump_data(void) {
// BENCH_COMPUTE_STAT(gettag);
//}
static bool bt_has_free_tags(struct blk_mq_bitmap_tags *bt)
{
int i;
......@@ -264,7 +287,9 @@ static int bt_get(struct blk_mq_alloc_data *data,
DEFINE_WAIT(wait);
int tag;
//(check_fio() == true) ? bench_start() : -1;
tag = __bt_get(hctx, bt, last_tag, tags);
//(check_fio() == true) ? bench_end() : -1;
if (tag != -1)
return tag;
......@@ -278,7 +303,6 @@ static int bt_get(struct blk_mq_alloc_data *data,
tag = __bt_get(hctx, bt, last_tag, tags);
if (tag != -1)
break;
/*
* We're out of tags on this hardware queue, kick any
* pending IO submits before going to sleep waiting for
......@@ -310,6 +334,7 @@ static int bt_get(struct blk_mq_alloc_data *data,
hctx = data->hctx;
bt = &hctx->tags->bitmap_tags;
}
finish_wait(&bs->wait, &wait);
bs = bt_wait_ptr(bt, hctx);
} while (1);
......@@ -322,8 +347,10 @@ static unsigned int __blk_mq_get_tag(struct blk_mq_alloc_data *data)
{
int tag;
//(check_fio() == true) ? bench_start() : -1;
tag = bt_get(data, &data->hctx->tags->bitmap_tags, data->hctx,
&data->ctx->last_tag, data->hctx->tags);
//(check_fio() == true) ? bench_end() : -1;
if (tag >= 0)
return tag + data->hctx->tags->nr_reserved_tags;
......@@ -349,8 +376,9 @@ static unsigned int __blk_mq_get_reserved_tag(struct blk_mq_alloc_data *data)
unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data)
{
if (data->flags & BLK_MQ_REQ_RESERVED)
if (data->flags & BLK_MQ_REQ_RESERVED) {
return __blk_mq_get_reserved_tag(data);
}
return __blk_mq_get_tag(data);
}
......
This diff is collapsed.
......@@ -13,6 +13,7 @@
#include "blk.h"
#include "blk-mq.h"
#include <linux/blk-lcd.h>
struct queue_sysfs_entry {
struct attribute attr;
......@@ -604,7 +605,11 @@ static void blk_free_queue_rcu(struct rcu_head *rcu_head)
{
struct request_queue *q = container_of(rcu_head, struct request_queue,
rcu_head);
kmem_cache_free(blk_requestq_cachep, q);
struct request_queue_container *rq_cnt = container_of(q,
struct request_queue_container,
rq);
kmem_cache_free(blk_requestq_cachep, rq_cnt);
}
/**
......
......@@ -183,6 +183,14 @@ unsigned long blk_rq_timeout(unsigned long timeout)
return timeout;
}
static bool __always_inline check_fio_klcd(void) {
if(strcmp(current->comm, "fio") == 0 || strcmp(current->comm, "klcd") == 0) {
return true;
}
return false;
}
/**
* blk_add_timer - Start timeout timer for a single request
* @req: request that is about to start running.
......@@ -207,8 +215,10 @@ void blk_add_timer(struct request *req)
* Some LLDs, like scsi, peek at the timeout to prevent a
* command from being retried forever.
*/
if (!req->timeout)
if (!req->timeout) {
//check_fio_klcd() ? printk("[timeout_null] name: %s req: %p, timeout: %d,\n",current->comm, req, req->timeout) : -1;
req->timeout = q->rq_timeout;
}
req->deadline = jiffies + req->timeout;
......@@ -225,6 +235,7 @@ void blk_add_timer(struct request *req)
* second.
*/
expiry = blk_rq_timeout(round_jiffies_up(req->deadline));
//check_fio_klcd() ? printk("[expiry_check] name: %s req: %p, timeout: %d, expiry: %d\n",current->comm, req, req->timeout, expiry) : -1;
if (!timer_pending(&q->timeout) ||
time_before(expiry, q->timeout.expires)) {
......@@ -238,6 +249,7 @@ void blk_add_timer(struct request *req)
* will be X + something.
*/
if (!timer_pending(&q->timeout) || (diff >= HZ / 2))
//check_fio_klcd() ? printk("[calling mod-timer] name: %s req: %p, timeout: %d, expiry: %d\n",current->comm, req, q->timeout, expiry) : -1;
mod_timer(&q->timeout, expiry);
}
......
......@@ -23,6 +23,7 @@
#include <linux/badblocks.h>
#include "blk.h"
#include <linux/blk-lcd.h>
static DEFINE_MUTEX(block_class_lock);
struct kobject *block_depr;
......@@ -1144,7 +1145,10 @@ static void disk_release(struct device *dev)
hd_free_part(&disk->part0);
if (disk->queue)
blk_put_queue(disk->queue);
kfree(disk);
//kfree(disk);
kfree((container_of(disk,
struct gendisk_container,
gendisk)));
}
struct class block_class = {
.name = "block",
......@@ -1289,18 +1293,25 @@ EXPORT_SYMBOL(alloc_disk);
struct gendisk *alloc_disk_node(int minors, int node_id)
{
struct gendisk_container *disk_container;
struct gendisk *disk;
disk = kzalloc_node(sizeof(struct gendisk), GFP_KERNEL, node_id);
if (disk) {
//disk = kzalloc_node(sizeof(struct gendisk), GFP_KERNEL, node_id);
disk_container = kzalloc_node(sizeof(struct gendisk_container), GFP_KERNEL, node_id);
if (disk_container) {
disk = &disk_container->gendisk;
if (!init_part_stats(&disk->part0)) {
kfree(disk);
//kfree(disk);
kfree(disk_container);
return NULL;
}
disk->node_id = node_id;
if (disk_expand_part_tbl(disk, 0)) {
free_part_stats(&disk->part0);
kfree(disk);
//kfree(disk);
kfree(disk_container);
return NULL;
}
disk->part_tbl->part[0] = &disk->part0;
......@@ -1317,7 +1328,8 @@ struct gendisk *alloc_disk_node(int minors, int node_id)
seqcount_init(&disk->part0.nr_sects_seq);
if (hd_ref_init(&disk->part0)) {
hd_free_part(&disk->part0);
kfree(disk);
//kfree(disk);
kfree(disk_container);
return NULL;
}
......
This diff is collapsed.
......@@ -46,9 +46,62 @@
#include "internal.h"
#include <linux/blk-bench.h>
#include <linux/bdump.h>
#define AIO_RING_MAGIC 0xa10a10a1
#define AIO_RING_COMPAT_FEATURES 1
#define AIO_RING_INCOMPAT_FEATURES 0
extern struct request_queue *queue_nullb;
//INIT_BENCHMARK_DATA(io_sub);
//INIT_BENCHMARK_DATA(io_gu);
//INIT_BENCHMARK_DATA(io_cu);
//static void __always_inline bench_start(void) {
// BENCH_BEGIN(io_sub);
//}
//static void __always_inline bench_end(void) {
// BENCH_END(io_sub);
//}
//static void __always_inline bench_start_gu(void) {
// BENCH_BEGIN(io_gu);
//}
//static void __always_inline bench_end_gu(void) {
// BENCH_END(io_gu);
//}
//static void __always_inline bench_start_cu(void) {
// BENCH_BEGIN(io_cu);
//}
//static void __always_inline bench_end_cu(void) {
// BENCH_END(io_cu);
//}
//void bdump_data(void) {
// BENCH_COMPUTE_STAT(io_sub);
//BENCH_COMPUTE_STAT(io_gu);
//BENCH_COMPUTE_STAT(io_cu);
//}
static bool __always_inline check_fio(void) {
if(strcmp(current->comm, "fio") == 0) {
return true;
}
return false;
}
//static inline void bbegin(void) {
// BENCH_BEGIN(aio_complete);
// return;
//}
//static inline void bend(void) {
// BENCH_END(aio_complete);
// return;
//}
struct aio_ring {
unsigned id; /* kernel internal index number */
unsigned nr; /* number of io_events */
......@@ -1074,7 +1127,8 @@ static void aio_complete(struct kiocb *kiocb, long res, long res2)
struct io_event *ev_page, *event;
unsigned tail, pos, head;
unsigned long flags;
//(queue_nullb != NULL) ? bbegin() : -1;
/*
* Special case handling for sync iocbs:
* - events go directly into the iocb for fast handling
......@@ -1145,8 +1199,11 @@ static void aio_complete(struct kiocb *kiocb, long res, long res2)
* eventfd. The eventfd_signal() function is safe to be called
* from IRQ context.
*/
if (iocb->ki_eventfd != NULL)
if (iocb->ki_eventfd != NULL) {
//(queue_nullb != NULL) ? printk("eventfd?\n") : -1;
eventfd_signal(iocb->ki_eventfd, 1);
}
/* everything turned out well, dispose of the aiocb. */
kiocb_free(iocb);
......@@ -1159,12 +1216,19 @@ static void aio_complete(struct kiocb *kiocb, long res, long res2)
*/
smp_mb();
if (waitqueue_active(&ctx->wait))
if (waitqueue_active(&ctx->wait)) {
//(queue_nullb != NULL) ? printk("wakeup wait?\n") : -1;
wake_up(&ctx->wait);
}
percpu_ref_put(&ctx->reqs);
//(queue_nullb != NULL) ? bend() : -1;
}
//void bdump_data(void) {
// BENCH_COMPUTE_STAT(aio_complete);
//}
/* aio_read_events_ring
* Pull an event off of the ioctx's event ring. Returns the number of
* events fetched
......@@ -1280,7 +1344,9 @@ static long read_events(struct kioctx *ctx, long min_nr, long nr,
if (unlikely(copy_from_user(&ts, timeout, sizeof(ts))))
return -EFAULT;
//(queue_nullb != NULL) ? printk("us ts.sec: %lld \n", ts.tv_sec) : -1;
until = timespec_to_ktime(ts);
//(queue_nullb != NULL) ? printk("reset timer: until.tv64: %lld \n", until.tv64) : -1;
}
/*
......@@ -1297,13 +1363,18 @@ static long read_events(struct kioctx *ctx, long min_nr, long nr,
* the ringbuffer empty. So in practice we should be ok, but it's
* something to be aware of when touching this code.
*/
if (until.tv64 == 0)
if (until.tv64 == 0) {
//(queue_nullb != NULL) ? printk("regular read_ev \n") : -1;
//(check_fio() == true) ? printk("regular read min: %d nr: %d \n",min_nr, nr) : -1;
aio_read_events(ctx, min_nr, nr, event, &ret);
else
}
else {
//(queue_nullb != NULL) ? printk("waited read_ev \n") : -1;
//(check_fio() == true) ? printk("waited read min: %d nr: %d \n",min_nr, nr) : -1;
wait_event_interruptible_hrtimeout(ctx->wait,
aio_read_events(ctx, min_nr, nr, event, &ret),
until);
}
if (!ret && signal_pending(current))
ret = -EINTR;
......@@ -1446,6 +1517,7 @@ rw_common:
ret = aio_setup_vectored_rw(rw, buf, len,
&iovec, compat, &iter);
else {
//(check_fio() == true) ? printk("calling ISR -> len:%d \n", len) : -1;
ret = import_single_range(rw, buf, len, iovec, &iter);
iovec = NULL;
}
......@@ -1460,7 +1532,10 @@ rw_common:
if (rw == WRITE)
file_start_write(file);
//(check_fio() == true) ? printk("calling read/write op \n") : -1;
//(check_fio() == true) ? bench_start() : -1;
ret = iter_op(req, &iter);
//(check_fio() == true) ? bench_end() : -1;
if (rw == WRITE)
file_end_write(file);
......@@ -1495,6 +1570,7 @@ rw_common:
ret == -ERESTARTNOHAND ||
ret == -ERESTART_RESTARTBLOCK))
ret = -EINTR;
//(check_fio() == true) ? printk("***** Failing IO with EINTR \n") : -1;
aio_complete(req, ret, 0);
}
......@@ -1562,10 +1638,14 @@ static int io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb,
req->ki_user_iocb = user_iocb;
req->ki_user_data = iocb->aio_data;
//(check_fio() == true) ? printk("-> aio_run_iocb \n") : -1;
//(check_fio() == true) ? bench_start() : -1;
ret = aio_run_iocb(&req->common, iocb->aio_lio_opcode,
(char __user *)(unsigned long)iocb->aio_buf,
iocb->aio_nbytes,
compat);
//(check_fio() == true) ? bench_end() : -1;
if (ret)
goto out_put_req;
......@@ -1594,12 +1674,13 @@ long do_io_submit(aio_context_t ctx_id, long nr,
if (unlikely(!access_ok(VERIFY_READ, iocbpp, (nr*sizeof(*iocbpp)))))
return -EFAULT;
//(check_fio() == true) ? bench_start() : -1;
ctx = lookup_ioctx(ctx_id);
if (unlikely(!ctx)) {
pr_debug("EINVAL: invalid context id\n");
return -EINVAL;
}
//(check_fio() == true) ? bench_end() : -1;
blk_start_plug(&plug);
/*
......@@ -1610,23 +1691,36 @@ long do_io_submit(aio_context_t ctx_id, long nr,
struct iocb __user *user_iocb;
struct iocb tmp;
//(check_fio() == true) ? bench_start_gu() : -1;
if (unlikely(__get_user(user_iocb, iocbpp + i))) {
ret = -EFAULT;
break;
}
//(check_fio() == true) ? bench_end_gu() : -1;
//(check_fio() == true) ? bench_start_cu() : -1;
if (unlikely(copy_from_user(&tmp, user_iocb, sizeof(tmp)))) {
ret = -EFAULT;
break;
}
//(check_fio() == true) ? bench_end_cu() : -1;
//(check_fio() == true) ? printk("io_submit_one: --> %d \n", i) : -1;
//(check_fio() == true) ? bench_start_gu() : -1;
ret = io_submit_one(ctx, user_iocb, &tmp, compat);
//(check_fio() == true) ? bench_end_gu() : -1;
if (ret)
break;
}
//(check_fio() == true) ? printk(" -------> finish plug %d \n", i) : -1;
//(check_fio() == true) ? bench_start() : -1;
blk_finish_plug(&plug);
//(check_fio() == true) ? bench_end() : -1;
//(check_fio() == true) ? bench_start_gu() : -1;
percpu_ref_put(&ctx->users);
//(check_fio() == true) ? bench_end_gu() : -1;
return i ? i : ret;
}
......@@ -1645,7 +1739,13 @@ long do_io_submit(aio_context_t ctx_id, long nr,
SYSCALL_DEFINE3(io_submit, aio_context_t, ctx_id, long, nr,
struct iocb __user * __user *, iocbpp)
{
return do_io_submit(ctx_id, nr, iocbpp, 0);
long ret = 0;
//(check_fio() == true) ? printk("--------------> syscall enter \n") : -1;
//(check_fio() == true) ? bench_start() : -1;
ret = do_io_submit(ctx_id, nr, iocbpp, 0);
//(check_fio() == true) ? printk("<-------------- syscall enter \n") : -1;
//(check_fio() == true) ? bench_end() : -1;
return ret;
}
/* lookup_kiocb
......
This diff is collapsed.
This diff is collapsed.
#ifndef __BLK_LCD_H_
#define __BLK_LCD_H_
typedef struct cptr {
unsigned long cptr;
} cptr_t;
struct request_queue_container {
struct request_queue rq;
cptr_t other_ref;
cptr_t my_ref;
};
struct blk_mq_hw_ctx_container {
struct blk_mq_hw_ctx blk_mq_hw_ctx;
cptr_t other_ref;
cptr_t my_ref;
};
struct gendisk_container {
struct gendisk gendisk;
cptr_t other_ref;
cptr_t my_ref;
};
#endif
......@@ -87,7 +87,15 @@ struct blk_mq_queue_data {
bool last;
};
struct blk_mq_queue_data_async {
struct list_head *rq_list;
struct list_head *list;
struct list_head *drv_list;
int *queued;
};
typedef int (queue_rq_fn)(struct blk_mq_hw_ctx *, const struct blk_mq_queue_data *);
typedef void (queue_rq_fn_async)(struct blk_mq_hw_ctx *, struct blk_mq_queue_data_async *);
typedef struct blk_mq_hw_ctx *(map_queue_fn)(struct request_queue *, const int);
typedef enum blk_eh_timer_return (timeout_fn)(struct request *, bool);
typedef int (init_hctx_fn)(struct blk_mq_hw_ctx *, void *, unsigned int);
......@@ -105,6 +113,10 @@ typedef int (poll_fn)(struct blk_mq_hw_ctx *, unsigned int);
struct blk_mq_ops {
/*
* Queue request asynchronously
*/
queue_rq_fn_async *queue_rq_async;
/*
* Queue request
*/
......@@ -229,6 +241,7 @@ void blk_mq_end_request(struct request *rq, int error);
void __blk_mq_end_request(struct request *rq, int error);
void blk_mq_requeue_request(struct request *rq);
void __blk_mq_requeue_request(struct request *rq);
void blk_mq_add_to_requeue_list(struct request *rq, bool at_head);
void blk_mq_cancel_requeue_work(struct request_queue *q);
void blk_mq_kick_requeue_list(struct request_queue *q);
......
......@@ -344,6 +344,9 @@ static int vmx_handle_other_exits(struct lcd_arch *lcd_arch)
exit_reason = lcd_arch->exit_reason;
switch (exit_reason) {
case EXIT_REASON_EXCEPTION_NMI:
ret = vmx_handle_nmi(lcd_arch);
break;
case EXIT_REASON_VMCALL:
ret = vmx_handle_vmcall(lcd_arch);
break;
......
......@@ -269,7 +269,7 @@ int __lcd_create_no_vm_no_thread(struct lcd **out);
* which we need a new kthread to run the LCD's code, but we don't want
* the code to run inside an isolated VM.
*/
int __lcd_create_no_vm(struct lcd **out);
int __lcd_create_no_vm(struct lcd **out, const char *name);
/**
* __lcd_create -- Create an empty isolated LCD
* @caller: the LCD doing the creating
......
......@@ -61,5 +61,5 @@ int lcd_cptr_alloc(cptr_t *slot_out);
* @slot: the slot to free
*/
void lcd_cptr_free(cptr_t slot);
struct cspace *get_current_cspace(struct task_struct *c);
#endif /* LCD_DOMAINS_CAP_H */
......@@ -110,6 +110,9 @@ int lcd_enter(void);
* regular user-level exit).
*/
void LCD_MAYBE_NORETURN lcd_exit(int retval);
struct task_struct;
void LCD_MAYBE_NORETURN lcd_exit_info(struct task_struct *info);
#define LCD_EXIT() ({ \
*((volatile void**)__builtin_frame_address(0) + 1) = NULL; \
lcd_enter(); \
......
......@@ -28,10 +28,14 @@ void lcd_cptr_free(cptr_t slot)
{
cptr_free(current->cptr_cache, slot);
}
struct cspace *get_current_cspace(struct task_struct *c)
{
return current->lcd->cspace;
}
/* EXPORTS -------------------------------------------------- */
EXPORT_SYMBOL(lcd_cap_delete);
EXPORT_SYMBOL(lcd_cap_revoke);
EXPORT_SYMBOL(lcd_cptr_alloc);
EXPORT_SYMBOL(lcd_cptr_free);
EXPORT_SYMBOL(get_current_cspace);
......@@ -130,6 +130,34 @@ static void do_destroy_cptr_cache(struct cptr_cache *c)
cptr_cache_free(c);
}
void lcd_exit_info(struct task_struct *info)
{
if(!info) {
LCD_ERR("invalid task struct");
return;
}
if (info->lcd) {
do_destroy_lcd(info->lcd);
}
if (info->cptr_cache) {
do_destroy_cptr_cache(info->cptr_cache);
}
if (info->lcd_resource_trees[0]) {
lcd_destroy_free_resource_tree(info->lcd_resource_trees[0]);
}
if (info->lcd_resource_trees[1]) {
lcd_destroy_free_resource_tree(info->lcd_resource_trees[1]);
}
if (info->ptstate) {
thc_done_with_pts(info->ptstate);
}
info->lcd = NULL;
info->cptr_cache = NULL;
info->lcd_resource_trees[0] = NULL;
info->lcd_resource_trees[1] = NULL;
}
void lcd_exit(int retval)
{
/*
......@@ -165,4 +193,5 @@ void lcd_abort(void)
EXPORT_SYMBOL(lcd_enter);
EXPORT_SYMBOL(lcd_exit);
EXPORT_SYMBOL(lcd_exit_info);
EXPORT_SYMBOL(lcd_abort);
/*
* This file is meant to support interface with the user-process
* which attempts to share a piece of memory with the LCD.
*
* (C) Abhiram Balasubramanian (abhiram@cs.utah.edu)
*/
/* Includes */
#include <lcd_domains/user_interface.h>
/* Defines */
#define DRV_MAX_DEVS 1
/* Globals */
static struct class *drv_class;
static struct cdev *cdev_local = NULL;
static struct device *dev = NULL;
dev_t dev_no = 0;
/*
* Creates a device node in the name of dev_name,
* registers a set of file operations that is left
* to the caller's control!
*/
int lcd_setup_chardev(const char* dev_name, const struct file_operations* fops)
{
int ret = 0;
printk("mod init \n");
drv_class = class_create(THIS_MODULE, dev_name);
if (IS_ERR(drv_class)) {
printk(KERN_ERR "class_create failed \n");
ret = PTR_ERR(drv_class);
goto exit_no_class;
}
/* Dynamic registration of major number */
printk("alloc chardev \n");
ret = alloc_chrdev_region(&dev_no, 0, DRV_MAX_DEVS, dev_name);
if (ret < 0){
printk(KERN_ERR "Couldn't alloc chardev region \n");
goto exit_chrdev_reg;
}
printk("cdev alloc \n");
cdev_local = cdev_alloc();
if(!cdev_local) {
ret = -ENOMEM;
printk("cdev_alloc- not enough memory \n");
goto exit_cdev_alloc;
}
cdev_local->owner = THIS_MODULE;
cdev_local->ops = fops;
printk("cdev add \n");
ret = cdev_add(cdev_local, dev_no, 1);
if(ret) {
printk("Cannot add cdev device \n");
goto exit_dev_add;
}
printk("dev create \n");
dev = device_create(drv_class, NULL, dev_no, NULL, "%s", dev_name);
if(IS_ERR(dev)) {
ret = PTR_ERR(dev);
printk("Cannot create device node entry \n");
goto exit_dev_create;
}
printk("init done \n");
return 0;
exit_dev_create:
exit_dev_add:
cdev_del(cdev_local);
exit_cdev_alloc:
unregister_chrdev_region(dev_no, DRV_MAX_DEVS);
exit_chrdev_reg:
class_destroy(drv_class);
exit_no_class:
return ret;
}
void lcd_teardown_chardev(void)
{
cdev_del(cdev_local);
device_del(dev);
unregister_chrdev_region(dev_no, DRV_MAX_DEVS);
class_destroy(drv_class);
}
EXPORT_SYMBOL(lcd_setup_chardev);
EXPORT_SYMBOL(lcd_teardown_chardev);
Subproject commit 58432a5225b4c53bc670f92895a1926d93110083
Subproject commit e96f2d4f8bf56e56540e803e7dc089640eb4109b
......@@ -70,13 +70,14 @@ long strnlen_user(const char __user *str, long count)
return 0L;
}
#if 0
void __lockfunc _raw_spin_unlock_irqrestore(raw_spinlock_t *lock,
unsigned long flags)
{
lcd_printk("resolve.c: trying to call dummy _raw_spin_unlock_irqrestore!");
return;
}
#endif
/* MUTEXES ------------------------------------------------------------ */
void __mutex_init(struct mutex *lock, const char *name,
......
......@@ -85,7 +85,7 @@ fail1:
return ret;
}
int __lcd_create_no_vm(struct lcd **out)
int __lcd_create_no_vm(struct lcd **out, const char *name)
{
struct lcd *lcd;
int ret;
......@@ -100,7 +100,7 @@ int __lcd_create_no_vm(struct lcd **out)
/*
* Create a kernel thread (won't run till we wake it up)
*/
lcd->kthread = kthread_create(__lcd_kthread_main, NULL, "lcd");
lcd->kthread = kthread_create(__lcd_kthread_main, NULL, name);
if (!lcd->kthread) {
LCD_ERR("failed to create kthread");
goto fail2;
......@@ -131,7 +131,7 @@ int __lcd_create(struct lcd *caller, cptr_t slot)
/*
* Basic init of lcd
*/
ret = __lcd_create_no_vm(&lcd);
ret = __lcd_create_no_vm(&lcd, "lcd");
if (ret) {
LCD_ERR("lcd create");
goto fail1;
......@@ -174,7 +174,7 @@ int __lcd_create_klcd(struct lcd *caller, cptr_t slot)
/*
* Basic init of lcd
*/
ret = __lcd_create_no_vm(&lcd);
ret = __lcd_create_no_vm(&lcd, "klcd");
if (ret) {
LCD_ERR("lcd create");
goto fail1;
......
......@@ -49,6 +49,7 @@ lcd_domains-y += $(addprefix kliblcd/, \
sync_ipc.o \
trace.o \
irq.o \
user_interface.o \
)
# code shared by kliblcd and isolated liblcd
......
......@@ -25,3 +25,4 @@ obj-m += bug/
obj-m += async_rpc/
obj-m += llvm_example/
obj-m += ioremap/
obj-m += nullb/
......@@ -49,3 +49,8 @@ llvm_example/lcd isolated
ioremap/boot nonisolated
ioremap/lcd isolated
nullb/boot nonisolated
nullb/nullb_lcd isolated
nullb/blk_klcd nonisolated
# Main targets
obj-m += lcd_test_mod_bug_boot.o
lcd_test_mod_bug_boot-y += boot.o
obj-m += lcd_test_mod_bug_lcd.o
lcd_test_mod_bug_lcd-y += lcd.o
lcd_test_mod_bug_lcd-y += $(LIBLCD)
# We don't want the loop to be optimized away
CFLAGS_lcd.o = -O0
/**
* boot.c - non-isolated kernel module, does setup
*
*/
#include <lcd-domains/kliblcd.h>
#include <linux/delay.h>
#include <linux/module.h>
#include <linux/kernel.h>
static int boot_main(void)
{
int ret;
struct lcd_info *mi;
cptr_t lcd;
/*
* Enter lcd mode
*/
ret = lcd_enter();
if (ret) {
LIBLCD_ERR("enter");
goto fail1;
}
/*
* Create a new lcd
*/
ret = lcd_create_module_lcd(LCD_DIR("bug/lcd"),
"lcd_test_mod_bug_lcd",
LCD_CPTR_NULL, &mi);
if (ret) {
LIBLCD_ERR("create module lcd");
goto fail2;
}
/*
* Set up boot info
*/
ret = lcd_dump_boot_info(mi);
if (ret) {
LIBLCD_ERR("dump boot info");
goto fail3;
}
/*
* Run it
*/
ret = lcd_run(lcd);
if (ret) {
LIBLCD_ERR("run lcd");
goto fail4;
}
/*
* Wait for 4 seconds
*/
msleep(4000);
/*
* Tear everything down
*/
ret = 0;
goto out;
out:
fail4:
fail3:
lcd_destroy_module_lcd(lcd, mi, LCD_CPTR_NULL);
fail2:
lcd_exit(0);
fail1:
return ret;
}
static int boot_init(void)
{
int ret;
LCD_MAIN({
ret = boot_main();
});
return ret;
}
static void boot_exit(void)
{
/* nothing to do */
}
module_init(boot_init);
module_exit(boot_exit);
/*
* LCD BUG() and oops test. This LCD crashes to generate a
* kernel oops.
*
* This code *must* be compiled with optimizations turned off, or
* else it won't do what we want.
*/
#include <linux/jiffies.h>
#include <lcd-domains/liblcd-config.h>
#include <linux/module.h>
#include <linux/kernel.h>
#include <lcd-domains/liblcd.h>
#include <lcd-domains/liblcd-hacks.h>
void foo5(void) {
unsigned char stack_array [] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 1, 2, 3, 4};
register long rax asm ("rax");
register long rbx asm ("rbx");
register long rcx asm ("rcx");
register long rdx asm ("rdx");
register long rdi asm ("rdi");
register long rsi asm ("rsi");
register long r8 asm ("r8");
register long r9 asm ("r9");
register long r10 asm ("r10");
register long r11 asm ("r11");
register long r12 asm ("r12");
register long r13 asm ("r13");
register long r14 asm ("r14");
register long r15 asm ("r15");
rax = 1;
rbx = 2;
rcx = 3;
rdx = 4;
rsi = 5;
rdi = 6;
r8 = 7;
r9 = 8;
r10 = 9;
r11 = 0xa;
r12 = 0xb;
r13 = 0xc;
r14 = 0xd;
r15 = 0xe;
jiffies_to_clock_t(100000);
rax = stack_array[0];
return;
};
void foo4 (void) {
foo5();
return;
}
void foo3 (void) {
foo4();
return;
}
void foo2 (void) {
foo3();
return;
}
void foo1 (void) {
foo2();
return;
}
static int __noreturn test_init(void)
{
int r;
r = lcd_enter();
if (r)
goto fail1;
foo1();
fail1:
lcd_exit(r);
}
static int __test_init(void)
{
int ret;
LCD_MAIN({
ret = test_init();
});
return ret;
}
/*
* make module loader happy (so we can unload). we don't actually call
* this before unloading the lcd (yet)
*/
static void __exit test_exit(void)
{
return;
}
module_init(__test_init);
module_exit(test_exit);
obj-$(LCD_CONFIG_BUILD_NULLB_BOOT) += boot/
obj-$(LCD_CONFIG_BUILD_NULLB_NULLB_LCD) += nullb_lcd/
obj-$(LCD_CONFIG_BUILD_NULLB_BLK_KLCD) += blk_klcd/
This diff is collapsed.
This diff is collapsed.
[global]
direct=1
filename=/dev/nullb0
numjobs=1
iodepth=4
ioengine=libaio
size=2G
rw=randrw
; Synchronous reads
[sync-read-4k]
rw=read
bs=4k
[sync-write-4k]
rw=write
bs=4k
; Varying block sizes
[block-512]
bs=512
[block-4k]
bs=4k
[block-1M]
bs=1M
; Only applicable for async rw
[vary-iodepth1]
bs=4k
iodepth=8
[vary-iodepth2]
bs=4k
iodepth=16
[vary-iodepth3]
bs=4k
iodepth=32
[seq-rw-block-512]
bs=512
rw=readwrite
[seq-rw-block-4k]
bs=4k
rw=readwrite
[seq-rw-block-1M]
bs=1M
rw=readwrite
This diff is collapsed.
This diff is collapsed.
obj-m += lcd_test_mod_nullb_blk_klcd.o
lcd_test_mod_nullb_blk_klcd-y += main.o
# glue code
lcd_test_mod_nullb_blk_klcd-y += $(addprefix glue/, nullb_callee.o \
nullb_callee_dispatch.o )
lcd_test_mod_nullb_blk_klcd-y += $(addprefix ../, cap.o)
cppflags-y += $(NONISOLATED_CFLAGS)
extra-y += glue/nullb_callee.lds
ldflags-y += -T $(LCD_TEST_MODULES_BUILD_DIR)/nullb/blk_klcd/glue/nullb_callee.lds
ccflags-y += $(NONISOLATED_CFLAGS)
This diff is collapsed.
#include <liblcd/trampoline_link.h>
SECTIONS
{
LCD_TRAMPOLINE_LINKER_SECTION(queue_rq_fn_trampoline)
LCD_TRAMPOLINE_LINKER_SECTION(queue_rq_async_fn_trampoline)
LCD_TRAMPOLINE_LINKER_SECTION(map_queue_fn_trampoline)
LCD_TRAMPOLINE_LINKER_SECTION(init_hctx_fn_trampoline)
LCD_TRAMPOLINE_LINKER_SECTION(softirq_done_fn_trampoline)
LCD_TRAMPOLINE_LINKER_SECTION(open_trampoline)
LCD_TRAMPOLINE_LINKER_SECTION(release_trampoline)
LCD_TRAMPOLINE_LINKER_SECTION(mmap_fops_trampoline)
}
INSERT AFTER .text;
#include <lcd_config/pre_hook.h>
#include <liblcd/liblcd.h>
#include "../nullb_callee.h"
#include <lcd_config/post_hook.h>
#define trace(x) LIBLCD_MSG("blk got " #x " msg")
int dispatch_sync_loop(void)
{
int fn_type = lcd_r0();
switch (fn_type) {
case REGISTER_BLKDEV:
trace(REGISTER_BLKDEV:);