dm-io.c 11.9 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1 2
/*
 * Copyright (C) 2003 Sistina Software
3
 * Copyright (C) 2006 Red Hat GmbH
Linus Torvalds's avatar
Linus Torvalds committed
4 5 6 7
 *
 * This file is released under the GPL.
 */

8 9
#include "dm.h"

10
#include <linux/device-mapper.h>
Linus Torvalds's avatar
Linus Torvalds committed
11 12 13 14 15 16

#include <linux/bio.h>
#include <linux/mempool.h>
#include <linux/module.h>
#include <linux/sched.h>
#include <linux/slab.h>
Alasdair G Kergon's avatar
Alasdair G Kergon committed
17
#include <linux/dm-io.h>
Linus Torvalds's avatar
Linus Torvalds committed
18

19 20 21 22
#define DM_MSG_PREFIX "io"

#define DM_IO_MAX_REGIONS	BITS_PER_LONG

23 24 25 26 27
struct dm_io_client {
	mempool_t *pool;
	struct bio_set *bios;
};

28 29 30 31
/*
 * Aligning 'struct io' reduces the number of bits required to store
 * its address.  Refer to store_io_and_region_in_bio() below.
 */
Linus Torvalds's avatar
Linus Torvalds committed
32
struct io {
33
	unsigned long error_bits;
Linus Torvalds's avatar
Linus Torvalds committed
34 35
	atomic_t count;
	struct task_struct *sleeper;
36
	struct dm_io_client *client;
Linus Torvalds's avatar
Linus Torvalds committed
37 38
	io_notify_fn callback;
	void *context;
39
} __attribute__((aligned(DM_IO_MAX_REGIONS)));
Linus Torvalds's avatar
Linus Torvalds committed
40

41 42
static struct kmem_cache *_dm_io_cache;

Linus Torvalds's avatar
Linus Torvalds committed
43 44 45
/*
 * io contexts are only dynamically allocated for asynchronous
 * io.  Since async io is likely to be the majority of io we'll
46
 * have the same number of io contexts as bios! (FIXME: must reduce this).
Linus Torvalds's avatar
Linus Torvalds committed
47
 */
48

Linus Torvalds's avatar
Linus Torvalds committed
49 50 51 52 53
static unsigned int pages_to_ios(unsigned int pages)
{
	return 4 * pages;	/* too many ? */
}

Heinz Mauelshagen's avatar
Heinz Mauelshagen committed
54 55 56 57 58 59 60 61 62 63 64 65
/*
 * Create a client with mempool and bioset.
 */
struct dm_io_client *dm_io_client_create(unsigned num_pages)
{
	unsigned ios = pages_to_ios(num_pages);
	struct dm_io_client *client;

	client = kmalloc(sizeof(*client), GFP_KERNEL);
	if (!client)
		return ERR_PTR(-ENOMEM);

66
	client->pool = mempool_create_slab_pool(ios, _dm_io_cache);
Heinz Mauelshagen's avatar
Heinz Mauelshagen committed
67 68 69
	if (!client->pool)
		goto bad;

70
	client->bios = bioset_create(16, 0);
Heinz Mauelshagen's avatar
Heinz Mauelshagen committed
71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98
	if (!client->bios)
		goto bad;

	return client;

   bad:
	if (client->pool)
		mempool_destroy(client->pool);
	kfree(client);
	return ERR_PTR(-ENOMEM);
}
EXPORT_SYMBOL(dm_io_client_create);

int dm_io_client_resize(unsigned num_pages, struct dm_io_client *client)
{
	return mempool_resize(client->pool, pages_to_ios(num_pages),
			      GFP_KERNEL);
}
EXPORT_SYMBOL(dm_io_client_resize);

void dm_io_client_destroy(struct dm_io_client *client)
{
	mempool_destroy(client->pool);
	bioset_free(client->bios);
	kfree(client);
}
EXPORT_SYMBOL(dm_io_client_destroy);

Linus Torvalds's avatar
Linus Torvalds committed
99 100
/*-----------------------------------------------------------------
 * We need to keep track of which region a bio is doing io for.
101 102 103 104
 * To avoid a memory allocation to store just 5 or 6 bits, we
 * ensure the 'struct io' pointer is aligned so enough low bits are
 * always zero and then combine it with the region number directly in
 * bi_private.
Linus Torvalds's avatar
Linus Torvalds committed
105
 *---------------------------------------------------------------*/
106 107
static void store_io_and_region_in_bio(struct bio *bio, struct io *io,
				       unsigned region)
Linus Torvalds's avatar
Linus Torvalds committed
108
{
109 110 111 112 113 114
	if (unlikely(!IS_ALIGNED((unsigned long)io, DM_IO_MAX_REGIONS))) {
		DMCRIT("Unaligned struct io pointer %p", io);
		BUG();
	}

	bio->bi_private = (void *)((unsigned long)io | region);
Linus Torvalds's avatar
Linus Torvalds committed
115 116
}

117 118
static void retrieve_io_and_region_from_bio(struct bio *bio, struct io **io,
				       unsigned *region)
Linus Torvalds's avatar
Linus Torvalds committed
119
{
120 121 122 123
	unsigned long val = (unsigned long)bio->bi_private;

	*io = (void *)(val & -(unsigned long)DM_IO_MAX_REGIONS);
	*region = val & (DM_IO_MAX_REGIONS - 1);
Linus Torvalds's avatar
Linus Torvalds committed
124 125 126 127 128 129 130 131
}

/*-----------------------------------------------------------------
 * We need an io object to keep track of the number of bios that
 * have been dispatched for a particular io.
 *---------------------------------------------------------------*/
static void dec_count(struct io *io, unsigned int region, int error)
{
132
	if (error)
133
		set_bit(region, &io->error_bits);
Linus Torvalds's avatar
Linus Torvalds committed
134 135 136 137 138 139

	if (atomic_dec_and_test(&io->count)) {
		if (io->sleeper)
			wake_up_process(io->sleeper);

		else {
140
			unsigned long r = io->error_bits;
Linus Torvalds's avatar
Linus Torvalds committed
141 142 143
			io_notify_fn fn = io->callback;
			void *context = io->context;

Milan Broz's avatar
Milan Broz committed
144
			mempool_free(io, io->client->pool);
Linus Torvalds's avatar
Linus Torvalds committed
145 146 147 148 149
			fn(r, context);
		}
	}
}

150
static void endio(struct bio *bio, int error)
Linus Torvalds's avatar
Linus Torvalds committed
151
{
Heinz Mauelshagen's avatar
Heinz Mauelshagen committed
152 153
	struct io *io;
	unsigned region;
Linus Torvalds's avatar
Linus Torvalds committed
154 155 156 157

	if (error && bio_data_dir(bio) == READ)
		zero_fill_bio(bio);

Heinz Mauelshagen's avatar
Heinz Mauelshagen committed
158 159 160
	/*
	 * The bio destructor in bio_put() may use the io object.
	 */
161
	retrieve_io_and_region_from_bio(bio, &io, &region);
Heinz Mauelshagen's avatar
Heinz Mauelshagen committed
162

Linus Torvalds's avatar
Linus Torvalds committed
163 164
	bio_put(bio);

Heinz Mauelshagen's avatar
Heinz Mauelshagen committed
165
	dec_count(io, region, error);
Linus Torvalds's avatar
Linus Torvalds committed
166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234
}

/*-----------------------------------------------------------------
 * These little objects provide an abstraction for getting a new
 * destination page for io.
 *---------------------------------------------------------------*/
struct dpages {
	void (*get_page)(struct dpages *dp,
			 struct page **p, unsigned long *len, unsigned *offset);
	void (*next_page)(struct dpages *dp);

	unsigned context_u;
	void *context_ptr;
};

/*
 * Functions for getting the pages from a list.
 */
static void list_get_page(struct dpages *dp,
		  struct page **p, unsigned long *len, unsigned *offset)
{
	unsigned o = dp->context_u;
	struct page_list *pl = (struct page_list *) dp->context_ptr;

	*p = pl->page;
	*len = PAGE_SIZE - o;
	*offset = o;
}

static void list_next_page(struct dpages *dp)
{
	struct page_list *pl = (struct page_list *) dp->context_ptr;
	dp->context_ptr = pl->next;
	dp->context_u = 0;
}

static void list_dp_init(struct dpages *dp, struct page_list *pl, unsigned offset)
{
	dp->get_page = list_get_page;
	dp->next_page = list_next_page;
	dp->context_u = offset;
	dp->context_ptr = pl;
}

/*
 * Functions for getting the pages from a bvec.
 */
static void bvec_get_page(struct dpages *dp,
		  struct page **p, unsigned long *len, unsigned *offset)
{
	struct bio_vec *bvec = (struct bio_vec *) dp->context_ptr;
	*p = bvec->bv_page;
	*len = bvec->bv_len;
	*offset = bvec->bv_offset;
}

static void bvec_next_page(struct dpages *dp)
{
	struct bio_vec *bvec = (struct bio_vec *) dp->context_ptr;
	dp->context_ptr = bvec + 1;
}

static void bvec_dp_init(struct dpages *dp, struct bio_vec *bvec)
{
	dp->get_page = bvec_get_page;
	dp->next_page = bvec_next_page;
	dp->context_ptr = bvec;
}

Heinz Mauelshagen's avatar
Heinz Mauelshagen committed
235 236 237
/*
 * Functions for getting the pages from a VMA.
 */
Linus Torvalds's avatar
Linus Torvalds committed
238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259
static void vm_get_page(struct dpages *dp,
		 struct page **p, unsigned long *len, unsigned *offset)
{
	*p = vmalloc_to_page(dp->context_ptr);
	*offset = dp->context_u;
	*len = PAGE_SIZE - dp->context_u;
}

static void vm_next_page(struct dpages *dp)
{
	dp->context_ptr += PAGE_SIZE - dp->context_u;
	dp->context_u = 0;
}

static void vm_dp_init(struct dpages *dp, void *data)
{
	dp->get_page = vm_get_page;
	dp->next_page = vm_next_page;
	dp->context_u = ((unsigned long) data) & (PAGE_SIZE - 1);
	dp->context_ptr = data;
}

Peter Osterlund's avatar
Peter Osterlund committed
260 261
static void dm_bio_destructor(struct bio *bio)
{
262 263 264 265
	unsigned region;
	struct io *io;

	retrieve_io_and_region_from_bio(bio, &io, &region);
266

Milan Broz's avatar
Milan Broz committed
267
	bio_free(bio, io->client->bios);
Peter Osterlund's avatar
Peter Osterlund committed
268 269
}

Heinz Mauelshagen's avatar
Heinz Mauelshagen committed
270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294
/*
 * Functions for getting the pages from kernel memory.
 */
static void km_get_page(struct dpages *dp, struct page **p, unsigned long *len,
			unsigned *offset)
{
	*p = virt_to_page(dp->context_ptr);
	*offset = dp->context_u;
	*len = PAGE_SIZE - dp->context_u;
}

static void km_next_page(struct dpages *dp)
{
	dp->context_ptr += PAGE_SIZE - dp->context_u;
	dp->context_u = 0;
}

static void km_dp_init(struct dpages *dp, void *data)
{
	dp->get_page = km_get_page;
	dp->next_page = km_next_page;
	dp->context_u = ((unsigned long) data) & (PAGE_SIZE - 1);
	dp->context_ptr = data;
}

Linus Torvalds's avatar
Linus Torvalds committed
295 296 297
/*-----------------------------------------------------------------
 * IO routines that accept a list of pages.
 *---------------------------------------------------------------*/
Heinz Mauelshagen's avatar
Heinz Mauelshagen committed
298
static void do_region(int rw, unsigned region, struct dm_io_region *where,
Linus Torvalds's avatar
Linus Torvalds committed
299 300 301 302 303 304 305 306 307
		      struct dpages *dp, struct io *io)
{
	struct bio *bio;
	struct page *page;
	unsigned long len;
	unsigned offset;
	unsigned num_bvecs;
	sector_t remaining = where->count;

Mikulas Patocka's avatar
Mikulas Patocka committed
308
	/*
309 310
	 * where->count may be zero if rw holds a flush and we need to
	 * send a zero-sized flush.
Mikulas Patocka's avatar
Mikulas Patocka committed
311 312
	 */
	do {
Linus Torvalds's avatar
Linus Torvalds committed
313
		/*
314
		 * Allocate a suitably sized-bio.
Linus Torvalds's avatar
Linus Torvalds committed
315
		 */
316 317
		num_bvecs = dm_sector_div_up(remaining,
					     (PAGE_SIZE >> SECTOR_SHIFT));
318
		num_bvecs = min_t(int, bio_get_nr_vecs(where->bdev), num_bvecs);
Milan Broz's avatar
Milan Broz committed
319
		bio = bio_alloc_bioset(GFP_NOIO, num_bvecs, io->client->bios);
Linus Torvalds's avatar
Linus Torvalds committed
320 321 322
		bio->bi_sector = where->sector + (where->count - remaining);
		bio->bi_bdev = where->bdev;
		bio->bi_end_io = endio;
Peter Osterlund's avatar
Peter Osterlund committed
323
		bio->bi_destructor = dm_bio_destructor;
324
		store_io_and_region_in_bio(bio, io, region);
Linus Torvalds's avatar
Linus Torvalds committed
325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341

		/*
		 * Try and add as many pages as possible.
		 */
		while (remaining) {
			dp->get_page(dp, &page, &len, &offset);
			len = min(len, to_bytes(remaining));
			if (!bio_add_page(bio, page, len, offset))
				break;

			offset = 0;
			remaining -= to_sector(len);
			dp->next_page(dp);
		}

		atomic_inc(&io->count);
		submit_bio(rw, bio);
Mikulas Patocka's avatar
Mikulas Patocka committed
342
	} while (remaining);
Linus Torvalds's avatar
Linus Torvalds committed
343 344 345
}

static void dispatch_io(int rw, unsigned int num_regions,
Heinz Mauelshagen's avatar
Heinz Mauelshagen committed
346
			struct dm_io_region *where, struct dpages *dp,
Linus Torvalds's avatar
Linus Torvalds committed
347 348 349 350 351
			struct io *io, int sync)
{
	int i;
	struct dpages old_pages = *dp;

352 353
	BUG_ON(num_regions > DM_IO_MAX_REGIONS);

Linus Torvalds's avatar
Linus Torvalds committed
354
	if (sync)
Jens Axboe's avatar
Jens Axboe committed
355
		rw |= REQ_SYNC;
Linus Torvalds's avatar
Linus Torvalds committed
356 357 358 359 360 361 362

	/*
	 * For multiple regions we need to be careful to rewind
	 * the dp object for each call to do_region.
	 */
	for (i = 0; i < num_regions; i++) {
		*dp = old_pages;
363
		if (where[i].count || (rw & REQ_FLUSH))
Linus Torvalds's avatar
Linus Torvalds committed
364 365 366 367
			do_region(rw, i, where + i, dp, io);
	}

	/*
368
	 * Drop the extra reference that we were holding to avoid
Linus Torvalds's avatar
Linus Torvalds committed
369 370 371 372 373
	 * the io being completed too early.
	 */
	dec_count(io, 0, 0);
}

374
static int sync_io(struct dm_io_client *client, unsigned int num_regions,
Heinz Mauelshagen's avatar
Heinz Mauelshagen committed
375
		   struct dm_io_region *where, int rw, struct dpages *dp,
376
		   unsigned long *error_bits)
Linus Torvalds's avatar
Linus Torvalds committed
377
{
378 379 380 381 382 383 384 385
	/*
	 * gcc <= 4.3 can't do the alignment for stack variables, so we must
	 * align it on our own.
	 * volatile prevents the optimizer from removing or reusing
	 * "io_" field from the stack frame (allowed in ANSI C).
	 */
	volatile char io_[sizeof(struct io) + __alignof__(struct io) - 1];
	struct io *io = (struct io *)PTR_ALIGN(&io_, __alignof__(struct io));
Linus Torvalds's avatar
Linus Torvalds committed
386

Mikulas Patocka's avatar
Mikulas Patocka committed
387
	if (num_regions > 1 && (rw & RW_MASK) != WRITE) {
Linus Torvalds's avatar
Linus Torvalds committed
388 389 390 391
		WARN_ON(1);
		return -EIO;
	}

392 393 394 395
	io->error_bits = 0;
	atomic_set(&io->count, 1); /* see dispatch_io() */
	io->sleeper = current;
	io->client = client;
Linus Torvalds's avatar
Linus Torvalds committed
396

397
	dispatch_io(rw, num_regions, where, dp, io, 1);
Linus Torvalds's avatar
Linus Torvalds committed
398 399 400 401

	while (1) {
		set_current_state(TASK_UNINTERRUPTIBLE);

402
		if (!atomic_read(&io->count))
Linus Torvalds's avatar
Linus Torvalds committed
403 404 405 406 407 408
			break;

		io_schedule();
	}
	set_current_state(TASK_RUNNING);

409
	if (error_bits)
410
		*error_bits = io->error_bits;
411

412
	return io->error_bits ? -EIO : 0;
Linus Torvalds's avatar
Linus Torvalds committed
413 414
}

415
static int async_io(struct dm_io_client *client, unsigned int num_regions,
Heinz Mauelshagen's avatar
Heinz Mauelshagen committed
416
		    struct dm_io_region *where, int rw, struct dpages *dp,
417
		    io_notify_fn fn, void *context)
Linus Torvalds's avatar
Linus Torvalds committed
418 419 420
{
	struct io *io;

Mikulas Patocka's avatar
Mikulas Patocka committed
421
	if (num_regions > 1 && (rw & RW_MASK) != WRITE) {
Linus Torvalds's avatar
Linus Torvalds committed
422 423 424 425 426
		WARN_ON(1);
		fn(1, context);
		return -EIO;
	}

Milan Broz's avatar
Milan Broz committed
427
	io = mempool_alloc(client->pool, GFP_NOIO);
428
	io->error_bits = 0;
Linus Torvalds's avatar
Linus Torvalds committed
429 430
	atomic_set(&io->count, 1); /* see dispatch_io() */
	io->sleeper = NULL;
431
	io->client = client;
Linus Torvalds's avatar
Linus Torvalds committed
432 433 434 435 436 437 438
	io->callback = fn;
	io->context = context;

	dispatch_io(rw, num_regions, where, dp, io, 0);
	return 0;
}

Heinz Mauelshagen's avatar
Heinz Mauelshagen committed
439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466
static int dp_init(struct dm_io_request *io_req, struct dpages *dp)
{
	/* Set up dpages based on memory type */
	switch (io_req->mem.type) {
	case DM_IO_PAGE_LIST:
		list_dp_init(dp, io_req->mem.ptr.pl, io_req->mem.offset);
		break;

	case DM_IO_BVEC:
		bvec_dp_init(dp, io_req->mem.ptr.bvec);
		break;

	case DM_IO_VMA:
		vm_dp_init(dp, io_req->mem.ptr.vma);
		break;

	case DM_IO_KMEM:
		km_dp_init(dp, io_req->mem.ptr.addr);
		break;

	default:
		return -EINVAL;
	}

	return 0;
}

/*
Mikulas Patocka's avatar
Mikulas Patocka committed
467 468 469
 * New collapsed (a)synchronous interface.
 *
 * If the IO is asynchronous (i.e. it has notify.fn), you must either unplug
470 471
 * the queue with blk_unplug() some time later or set REQ_SYNC in
io_req->bi_rw. If you fail to do one of these, the IO will be submitted to
Mikulas Patocka's avatar
Mikulas Patocka committed
472
 * the disk after q->unplug_delay, which defaults to 3ms in blk-settings.c.
Heinz Mauelshagen's avatar
Heinz Mauelshagen committed
473 474
 */
int dm_io(struct dm_io_request *io_req, unsigned num_regions,
Heinz Mauelshagen's avatar
Heinz Mauelshagen committed
475
	  struct dm_io_region *where, unsigned long *sync_error_bits)
Heinz Mauelshagen's avatar
Heinz Mauelshagen committed
476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491
{
	int r;
	struct dpages dp;

	r = dp_init(io_req, &dp);
	if (r)
		return r;

	if (!io_req->notify.fn)
		return sync_io(io_req->client, num_regions, where,
			       io_req->bi_rw, &dp, sync_error_bits);

	return async_io(io_req->client, num_regions, where, io_req->bi_rw,
			&dp, io_req->notify.fn, io_req->notify.context);
}
EXPORT_SYMBOL(dm_io);
492 493 494 495 496 497 498 499 500 501 502 503 504 505 506

int __init dm_io_init(void)
{
	_dm_io_cache = KMEM_CACHE(io, 0);
	if (!_dm_io_cache)
		return -ENOMEM;

	return 0;
}

void dm_io_exit(void)
{
	kmem_cache_destroy(_dm_io_cache);
	_dm_io_cache = NULL;
}