dmatest.c 26.6 KB
Newer Older
1 2 3 4
/*
 * DMA Engine test module
 *
 * Copyright (C) 2007 Atmel Corporation
5
 * Copyright (C) 2013 Intel Corporation
6 7 8 9 10
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */
11 12
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

13
#include <linux/delay.h>
14
#include <linux/dma-mapping.h>
15
#include <linux/dmaengine.h>
16
#include <linux/freezer.h>
17 18 19 20 21
#include <linux/init.h>
#include <linux/kthread.h>
#include <linux/module.h>
#include <linux/moduleparam.h>
#include <linux/random.h>
22
#include <linux/slab.h>
23 24 25
#include <linux/wait.h>

static unsigned int test_buf_size = 16384;
26
module_param(test_buf_size, uint, S_IRUGO | S_IWUSR);
27 28
MODULE_PARM_DESC(test_buf_size, "Size of the memcpy test buffer");

29
static char test_channel[20];
30 31
module_param_string(channel, test_channel, sizeof(test_channel),
		S_IRUGO | S_IWUSR);
32 33
MODULE_PARM_DESC(channel, "Bus ID of the channel to test (default: any)");

34
static char test_device[32];
35 36
module_param_string(device, test_device, sizeof(test_device),
		S_IRUGO | S_IWUSR);
37 38 39
MODULE_PARM_DESC(device, "Bus ID of the DMA Engine to test (default: any)");

static unsigned int threads_per_chan = 1;
40
module_param(threads_per_chan, uint, S_IRUGO | S_IWUSR);
41 42 43 44
MODULE_PARM_DESC(threads_per_chan,
		"Number of threads to start per channel (default: 1)");

static unsigned int max_channels;
45
module_param(max_channels, uint, S_IRUGO | S_IWUSR);
46
MODULE_PARM_DESC(max_channels,
47 48
		"Maximum number of channels to use (default: all)");

49
static unsigned int iterations;
50
module_param(iterations, uint, S_IRUGO | S_IWUSR);
51 52 53
MODULE_PARM_DESC(iterations,
		"Iterations before stopping test (default: infinite)");

54 55 56 57 58 59 60 61 62 63
static unsigned int sg_buffers = 1;
module_param(sg_buffers, uint, S_IRUGO | S_IWUSR);
MODULE_PARM_DESC(sg_buffers,
		"Number of scatter gather buffers (default: 1)");

static unsigned int dmatest = 1;
module_param(dmatest, uint, S_IRUGO | S_IWUSR);
MODULE_PARM_DESC(dmatest,
		"dmatest 0-memcpy 1-slave_sg (default: 1)");

Dan Williams's avatar
Dan Williams committed
64
static unsigned int xor_sources = 3;
65
module_param(xor_sources, uint, S_IRUGO | S_IWUSR);
Dan Williams's avatar
Dan Williams committed
66 67 68
MODULE_PARM_DESC(xor_sources,
		"Number of xor source buffers (default: 3)");

Dan Williams's avatar
Dan Williams committed
69
static unsigned int pq_sources = 3;
70
module_param(pq_sources, uint, S_IRUGO | S_IWUSR);
Dan Williams's avatar
Dan Williams committed
71 72 73
MODULE_PARM_DESC(pq_sources,
		"Number of p+q source buffers (default: 3)");

74
static int timeout = 3000;
75
module_param(timeout, uint, S_IRUGO | S_IWUSR);
76 77
MODULE_PARM_DESC(timeout, "Transfer Timeout in msec (default: 3000), "
		 "Pass -1 for infinite timeout");
78

79 80 81
static bool noverify;
module_param(noverify, bool, S_IRUGO | S_IWUSR);
MODULE_PARM_DESC(noverify, "Disable random data setup and verification");
82

Dan Williams's avatar
Dan Williams committed
83 84 85
static bool verbose;
module_param(verbose, bool, S_IRUGO | S_IWUSR);
MODULE_PARM_DESC(verbose, "Enable \"success\" result messages (default: off)");
86

87
/**
88
 * struct dmatest_params - test parameters.
89 90 91 92 93 94 95 96 97 98
 * @buf_size:		size of the memcpy test buffer
 * @channel:		bus ID of the channel to test
 * @device:		bus ID of the DMA Engine to test
 * @threads_per_chan:	number of threads to start per channel
 * @max_channels:	maximum number of channels to use
 * @iterations:		iterations before stopping test
 * @xor_sources:	number of xor source buffers
 * @pq_sources:		number of p+q source buffers
 * @timeout:		transfer timeout in msec, -1 for infinite timeout
 */
99
struct dmatest_params {
100 101
	unsigned int	buf_size;
	char		channel[20];
102
	char		device[32];
103 104 105 106 107 108
	unsigned int	threads_per_chan;
	unsigned int	max_channels;
	unsigned int	iterations;
	unsigned int	xor_sources;
	unsigned int	pq_sources;
	int		timeout;
109
	bool		noverify;
110 111 112 113 114
};

/**
 * struct dmatest_info - test information.
 * @params:		test parameters
115
 * @lock:		access protection to the fields of this structure
116
 */
117
static struct dmatest_info {
118 119
	/* Test parameters */
	struct dmatest_params	params;
120 121 122 123

	/* Internal state */
	struct list_head	channels;
	unsigned int		nr_channels;
124
	struct mutex		lock;
125 126 127 128 129
	bool			did_init;
} test_info = {
	.channels = LIST_HEAD_INIT(test_info.channels),
	.lock = __MUTEX_INITIALIZER(test_info.lock),
};
130

131 132
static int dmatest_run_set(const char *val, const struct kernel_param *kp);
static int dmatest_run_get(char *val, const struct kernel_param *kp);
133
static const struct kernel_param_ops run_ops = {
134 135
	.set = dmatest_run_set,
	.get = dmatest_run_get,
136
};
137 138 139
static bool dmatest_run;
module_param_cb(run, &run_ops, &dmatest_run, S_IRUGO | S_IWUSR);
MODULE_PARM_DESC(run, "Run the test (default: false)");
140

141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159
/* Maximum amount of mismatched bytes in buffer to print */
#define MAX_ERROR_COUNT		32

/*
 * Initialization patterns. All bytes in the source buffer has bit 7
 * set, all bytes in the destination buffer has bit 7 cleared.
 *
 * Bit 6 is set for all bytes which are to be copied by the DMA
 * engine. Bit 5 is set for all bytes which are to be overwritten by
 * the DMA engine.
 *
 * The remaining bits are the inverse of a counter which increments by
 * one for each byte address.
 */
#define PATTERN_SRC		0x80
#define PATTERN_DST		0x00
#define PATTERN_COPY		0x40
#define PATTERN_OVERWRITE	0x20
#define PATTERN_COUNT_MASK	0x1f
160

161 162 163 164 165 166 167 168 169 170
struct dmatest_thread {
	struct list_head	node;
	struct dmatest_info	*info;
	struct task_struct	*task;
	struct dma_chan		*chan;
	u8			**srcs;
	u8			**dsts;
	enum dma_transaction_type type;
	bool			done;
};
171

172 173 174 175
struct dmatest_chan {
	struct list_head	node;
	struct dma_chan		*chan;
	struct list_head	threads;
176 177
};

178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207
static DECLARE_WAIT_QUEUE_HEAD(thread_wait);
static bool wait;

static bool is_threaded_test_run(struct dmatest_info *info)
{
	struct dmatest_chan *dtc;

	list_for_each_entry(dtc, &info->channels, node) {
		struct dmatest_thread *thread;

		list_for_each_entry(thread, &dtc->threads, node) {
			if (!thread->done)
				return true;
		}
	}

	return false;
}

static int dmatest_wait_get(char *val, const struct kernel_param *kp)
{
	struct dmatest_info *info = &test_info;
	struct dmatest_params *params = &info->params;

	if (params->iterations)
		wait_event(thread_wait, !is_threaded_test_run(info));
	wait = true;
	return param_get_bool(val, kp);
}

208
static const struct kernel_param_ops wait_ops = {
209 210 211 212 213
	.get = dmatest_wait_get,
	.set = param_set_bool,
};
module_param_cb(wait, &wait_ops, &wait, S_IRUGO);
MODULE_PARM_DESC(wait, "Wait for tests to complete (default: false)");
214

215
static bool dmatest_match_channel(struct dmatest_params *params,
216
		struct dma_chan *chan)
217
{
218
	if (params->channel[0] == '\0')
219
		return true;
220
	return strcmp(dma_chan_name(chan), params->channel) == 0;
221 222
}

223
static bool dmatest_match_device(struct dmatest_params *params,
224
		struct dma_device *device)
225
{
226
	if (params->device[0] == '\0')
227
		return true;
228
	return strcmp(dev_name(device->dev), params->device) == 0;
229 230 231 232 233 234
}

static unsigned long dmatest_random(void)
{
	unsigned long buf;

235
	prandom_bytes(&buf, sizeof(buf));
236 237 238
	return buf;
}

239 240
static void dmatest_init_srcs(u8 **bufs, unsigned int start, unsigned int len,
		unsigned int buf_size)
241 242
{
	unsigned int i;
Dan Williams's avatar
Dan Williams committed
243 244 245 246 247 248 249
	u8 *buf;

	for (; (buf = *bufs); bufs++) {
		for (i = 0; i < start; i++)
			buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK);
		for ( ; i < start + len; i++)
			buf[i] = PATTERN_SRC | PATTERN_COPY
250
				| (~i & PATTERN_COUNT_MASK);
251
		for ( ; i < buf_size; i++)
Dan Williams's avatar
Dan Williams committed
252 253 254
			buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK);
		buf++;
	}
255 256
}

257 258
static void dmatest_init_dsts(u8 **bufs, unsigned int start, unsigned int len,
		unsigned int buf_size)
259 260
{
	unsigned int i;
Dan Williams's avatar
Dan Williams committed
261 262 263 264 265 266 267 268
	u8 *buf;

	for (; (buf = *bufs); bufs++) {
		for (i = 0; i < start; i++)
			buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK);
		for ( ; i < start + len; i++)
			buf[i] = PATTERN_DST | PATTERN_OVERWRITE
				| (~i & PATTERN_COUNT_MASK);
269
		for ( ; i < buf_size; i++)
Dan Williams's avatar
Dan Williams committed
270 271
			buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK);
	}
272 273
}

274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298
static void dmatest_mismatch(u8 actual, u8 pattern, unsigned int index,
		unsigned int counter, bool is_srcbuf)
{
	u8		diff = actual ^ pattern;
	u8		expected = pattern | (~counter & PATTERN_COUNT_MASK);
	const char	*thread_name = current->comm;

	if (is_srcbuf)
		pr_warn("%s: srcbuf[0x%x] overwritten! Expected %02x, got %02x\n",
			thread_name, index, expected, actual);
	else if ((pattern & PATTERN_COPY)
			&& (diff & (PATTERN_COPY | PATTERN_OVERWRITE)))
		pr_warn("%s: dstbuf[0x%x] not copied! Expected %02x, got %02x\n",
			thread_name, index, expected, actual);
	else if (diff & PATTERN_SRC)
		pr_warn("%s: dstbuf[0x%x] was copied! Expected %02x, got %02x\n",
			thread_name, index, expected, actual);
	else
		pr_warn("%s: dstbuf[0x%x] mismatch! Expected %02x, got %02x\n",
			thread_name, index, expected, actual);
}

static unsigned int dmatest_verify(u8 **bufs, unsigned int start,
		unsigned int end, unsigned int counter, u8 pattern,
		bool is_srcbuf)
299 300 301 302
{
	unsigned int i;
	unsigned int error_count = 0;
	u8 actual;
Dan Williams's avatar
Dan Williams committed
303 304 305 306 307 308 309 310 311 312
	u8 expected;
	u8 *buf;
	unsigned int counter_orig = counter;

	for (; (buf = *bufs); bufs++) {
		counter = counter_orig;
		for (i = start; i < end; i++) {
			actual = buf[i];
			expected = pattern | (~counter & PATTERN_COUNT_MASK);
			if (actual != expected) {
313 314 315
				if (error_count < MAX_ERROR_COUNT)
					dmatest_mismatch(actual, pattern, i,
							 counter, is_srcbuf);
Dan Williams's avatar
Dan Williams committed
316 317 318
				error_count++;
			}
			counter++;
319 320 321
		}
	}

322
	if (error_count > MAX_ERROR_COUNT)
323
		pr_warn("%s: %u errors suppressed\n",
324
			current->comm, error_count - MAX_ERROR_COUNT);
325 326 327 328

	return error_count;
}

329 330 331 332 333 334 335
/* poor man's completion - we want to use wait_event_freezable() on it */
struct dmatest_done {
	bool			done;
	wait_queue_head_t	*wait;
};

static void dmatest_callback(void *arg)
336
{
337 338 339 340
	struct dmatest_done *done = arg;

	done->done = true;
	wake_up_all(done->wait);
341 342
}

343 344 345 346 347 348 349
static unsigned int min_odd(unsigned int x, unsigned int y)
{
	unsigned int val = min(x, y);

	return val % 2 ? val : val - 1;
}

350 351
static void result(const char *err, unsigned int n, unsigned int src_off,
		   unsigned int dst_off, unsigned int len, unsigned long data)
352
{
353
	pr_info("%s: result #%u: '%s' with src_off=0x%x dst_off=0x%x len=0x%x (%lu)\n",
354
		current->comm, n, err, src_off, dst_off, len, data);
355 356
}

357 358 359
static void dbg_result(const char *err, unsigned int n, unsigned int src_off,
		       unsigned int dst_off, unsigned int len,
		       unsigned long data)
360
{
361
	pr_debug("%s: result #%u: '%s' with src_off=0x%x dst_off=0x%x len=0x%x (%lu)\n",
Andy Shevchenko's avatar
Andy Shevchenko committed
362
		 current->comm, n, err, src_off, dst_off, len, data);
363 364
}

Andy Shevchenko's avatar
Andy Shevchenko committed
365 366 367 368 369
#define verbose_result(err, n, src_off, dst_off, len, data) ({	\
	if (verbose)						\
		result(err, n, src_off, dst_off, len, data);	\
	else							\
		dbg_result(err, n, src_off, dst_off, len, data);\
Dan Williams's avatar
Dan Williams committed
370
})
371

372
static unsigned long long dmatest_persec(s64 runtime, unsigned int val)
373
{
374
	unsigned long long per_sec = 1000000;
375

376 377
	if (runtime <= 0)
		return 0;
378

379 380 381 382
	/* drop precision until runtime is 32-bits */
	while (runtime > UINT_MAX) {
		runtime >>= 1;
		per_sec <<= 1;
383 384
	}

385 386 387
	per_sec *= val;
	do_div(per_sec, runtime);
	return per_sec;
388 389
}

390
static unsigned long long dmatest_KBs(s64 runtime, unsigned long long len)
391
{
392
	return dmatest_persec(runtime, len >> 10);
393 394
}

395 396
/*
 * This function repeatedly tests DMA transfers of various lengths and
Dan Williams's avatar
Dan Williams committed
397 398 399 400
 * offsets for a given operation type until it is told to exit by
 * kthread_stop(). There may be multiple threads running this function
 * in parallel for a single channel, and there may be multiple channels
 * being tested in parallel.
401 402 403 404 405 406 407 408 409 410
 *
 * Before each test, the source and destination buffer is initialized
 * with a known pattern. This pattern is different depending on
 * whether it's in an area which is supposed to be copied or
 * overwritten, and different in the source and destination buffers.
 * So if the DMA engine doesn't copy exactly what we tell it to copy,
 * we'll notice.
 */
static int dmatest_func(void *data)
{
411
	DECLARE_WAIT_QUEUE_HEAD_ONSTACK(done_wait);
412
	struct dmatest_thread	*thread = data;
413
	struct dmatest_done	done = { .wait = &done_wait };
414
	struct dmatest_info	*info;
415
	struct dmatest_params	*params;
416
	struct dma_chan		*chan;
417
	struct dma_device	*dev;
418 419 420 421 422
	unsigned int		error_count;
	unsigned int		failed_tests = 0;
	unsigned int		total_tests = 0;
	dma_cookie_t		cookie;
	enum dma_status		status;
Dan Williams's avatar
Dan Williams committed
423
	enum dma_ctrl_flags 	flags;
424
	u8			*pq_coefs = NULL;
425
	int			ret;
Dan Williams's avatar
Dan Williams committed
426 427 428
	int			src_cnt;
	int			dst_cnt;
	int			i;
429 430 431
	ktime_t			ktime;
	s64			runtime = 0;
	unsigned long long	total_len = 0;
432

433
	set_freezable();
434 435 436 437

	ret = -ENOMEM;

	smp_rmb();
438
	info = thread->info;
439
	params = &info->params;
440
	chan = thread->chan;
441
	dev = chan->device;
Dan Williams's avatar
Dan Williams committed
442 443
	if (thread->type == DMA_MEMCPY)
		src_cnt = dst_cnt = 1;
444 445
	else if (thread->type == DMA_SG)
		src_cnt = dst_cnt = sg_buffers;
Dan Williams's avatar
Dan Williams committed
446
	else if (thread->type == DMA_XOR) {
447
		/* force odd to ensure dst = src */
448
		src_cnt = min_odd(params->xor_sources | 1, dev->max_xor);
Dan Williams's avatar
Dan Williams committed
449
		dst_cnt = 1;
Dan Williams's avatar
Dan Williams committed
450
	} else if (thread->type == DMA_PQ) {
451
		/* force odd to ensure dst = src */
452
		src_cnt = min_odd(params->pq_sources | 1, dma_maxpq(dev, 0));
Dan Williams's avatar
Dan Williams committed
453
		dst_cnt = 2;
454

455
		pq_coefs = kmalloc(params->pq_sources+1, GFP_KERNEL);
456 457 458
		if (!pq_coefs)
			goto err_thread_type;

459
		for (i = 0; i < src_cnt; i++)
Dan Williams's avatar
Dan Williams committed
460
			pq_coefs[i] = 1;
Dan Williams's avatar
Dan Williams committed
461
	} else
462
		goto err_thread_type;
Dan Williams's avatar
Dan Williams committed
463 464 465 466 467

	thread->srcs = kcalloc(src_cnt+1, sizeof(u8 *), GFP_KERNEL);
	if (!thread->srcs)
		goto err_srcs;
	for (i = 0; i < src_cnt; i++) {
468
		thread->srcs[i] = kmalloc(params->buf_size, GFP_KERNEL);
Dan Williams's avatar
Dan Williams committed
469 470 471 472 473 474 475 476 477
		if (!thread->srcs[i])
			goto err_srcbuf;
	}
	thread->srcs[i] = NULL;

	thread->dsts = kcalloc(dst_cnt+1, sizeof(u8 *), GFP_KERNEL);
	if (!thread->dsts)
		goto err_dsts;
	for (i = 0; i < dst_cnt; i++) {
478
		thread->dsts[i] = kmalloc(params->buf_size, GFP_KERNEL);
Dan Williams's avatar
Dan Williams committed
479 480 481 482 483
		if (!thread->dsts[i])
			goto err_dstbuf;
	}
	thread->dsts[i] = NULL;

484 485
	set_user_nice(current, 10);

486
	/*
487
	 * src and dst buffers are freed by ourselves below
488
	 */
489
	flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT;
490

491
	ktime = ktime_get();
492
	while (!kthread_should_stop()
493
	       && !(params->iterations && total_tests >= params->iterations)) {
Dan Williams's avatar
Dan Williams committed
494
		struct dma_async_tx_descriptor *tx = NULL;
495 496 497
		struct dmaengine_unmap_data *um;
		dma_addr_t srcs[src_cnt];
		dma_addr_t *dsts;
498
		unsigned int src_off, dst_off, len;
499
		u8 align = 0;
500 501
		struct scatterlist tx_sg[src_cnt];
		struct scatterlist rx_sg[src_cnt];
502

503 504
		total_tests++;

505 506 507 508 509 510 511 512
		/* honor alignment restrictions */
		if (thread->type == DMA_MEMCPY)
			align = dev->copy_align;
		else if (thread->type == DMA_XOR)
			align = dev->xor_align;
		else if (thread->type == DMA_PQ)
			align = dev->pq_align;

513
		if (1 << align > params->buf_size) {
514
			pr_err("%u-byte buffer too small for %d-byte alignment\n",
515
			       params->buf_size, 1 << align);
516 517 518
			break;
		}

519
		if (params->noverify)
520
			len = params->buf_size;
521 522 523 524 525 526 527 528 529 530
		else
			len = dmatest_random() % params->buf_size + 1;

		len = (len >> align) << align;
		if (!len)
			len = 1 << align;

		total_len += len;

		if (params->noverify) {
531 532 533 534 535 536 537 538 539 540 541 542 543 544 545
			src_off = 0;
			dst_off = 0;
		} else {
			src_off = dmatest_random() % (params->buf_size - len + 1);
			dst_off = dmatest_random() % (params->buf_size - len + 1);

			src_off = (src_off >> align) << align;
			dst_off = (dst_off >> align) << align;

			dmatest_init_srcs(thread->srcs, src_off, len,
					  params->buf_size);
			dmatest_init_dsts(thread->dsts, dst_off, len,
					  params->buf_size);
		}

546 547 548 549 550 551 552 553
		um = dmaengine_get_unmap_data(dev->dev, src_cnt+dst_cnt,
					      GFP_KERNEL);
		if (!um) {
			failed_tests++;
			result("unmap data NULL", total_tests,
			       src_off, dst_off, len, ret);
			continue;
		}
554

555
		um->len = params->buf_size;
Dan Williams's avatar
Dan Williams committed
556
		for (i = 0; i < src_cnt; i++) {
557
			void *buf = thread->srcs[i];
558
			struct page *pg = virt_to_page(buf);
559
			unsigned pg_off = (unsigned long) buf & ~PAGE_MASK;
560 561 562 563 564

			um->addr[i] = dma_map_page(dev->dev, pg, pg_off,
						   um->len, DMA_TO_DEVICE);
			srcs[i] = um->addr[i] + src_off;
			ret = dma_mapping_error(dev->dev, um->addr[i]);
565
			if (ret) {
566
				dmaengine_unmap_put(um);
567 568
				result("src mapping error", total_tests,
				       src_off, dst_off, len, ret);
569 570 571
				failed_tests++;
				continue;
			}
572
			um->to_cnt++;
Dan Williams's avatar
Dan Williams committed
573
		}
574
		/* map with DMA_BIDIRECTIONAL to force writeback/invalidate */
575
		dsts = &um->addr[src_cnt];
Dan Williams's avatar
Dan Williams committed
576
		for (i = 0; i < dst_cnt; i++) {
577
			void *buf = thread->dsts[i];
578
			struct page *pg = virt_to_page(buf);
579
			unsigned pg_off = (unsigned long) buf & ~PAGE_MASK;
580 581 582 583

			dsts[i] = dma_map_page(dev->dev, pg, pg_off, um->len,
					       DMA_BIDIRECTIONAL);
			ret = dma_mapping_error(dev->dev, dsts[i]);
584
			if (ret) {
585
				dmaengine_unmap_put(um);
586 587
				result("dst mapping error", total_tests,
				       src_off, dst_off, len, ret);
588 589 590
				failed_tests++;
				continue;
			}
591
			um->bidi_cnt++;
Dan Williams's avatar
Dan Williams committed
592 593
		}

594 595 596 597 598 599 600 601 602
		sg_init_table(tx_sg, src_cnt);
		sg_init_table(rx_sg, src_cnt);
		for (i = 0; i < src_cnt; i++) {
			sg_dma_address(&rx_sg[i]) = srcs[i];
			sg_dma_address(&tx_sg[i]) = dsts[i] + dst_off;
			sg_dma_len(&tx_sg[i]) = len;
			sg_dma_len(&rx_sg[i]) = len;
		}

Dan Williams's avatar
Dan Williams committed
603 604
		if (thread->type == DMA_MEMCPY)
			tx = dev->device_prep_dma_memcpy(chan,
605 606
							 dsts[0] + dst_off,
							 srcs[0], len, flags);
607 608 609
		else if (thread->type == DMA_SG)
			tx = dev->device_prep_dma_sg(chan, tx_sg, src_cnt,
						     rx_sg, src_cnt, flags);
Dan Williams's avatar
Dan Williams committed
610 611
		else if (thread->type == DMA_XOR)
			tx = dev->device_prep_dma_xor(chan,
612 613
						      dsts[0] + dst_off,
						      srcs, src_cnt,
Dan Williams's avatar
Dan Williams committed
614
						      len, flags);
Dan Williams's avatar
Dan Williams committed
615 616 617 618
		else if (thread->type == DMA_PQ) {
			dma_addr_t dma_pq[dst_cnt];

			for (i = 0; i < dst_cnt; i++)
619 620
				dma_pq[i] = dsts[i] + dst_off;
			tx = dev->device_prep_dma_pq(chan, dma_pq, srcs,
621
						     src_cnt, pq_coefs,
Dan Williams's avatar
Dan Williams committed
622 623
						     len, flags);
		}
624 625

		if (!tx) {
626
			dmaengine_unmap_put(um);
627 628
			result("prep error", total_tests, src_off,
			       dst_off, len, ret);
629 630 631 632
			msleep(100);
			failed_tests++;
			continue;
		}
633

634
		done.done = false;
635
		tx->callback = dmatest_callback;
636
		tx->callback_param = &done;
637 638
		cookie = tx->tx_submit(tx);

639
		if (dma_submit_error(cookie)) {
640
			dmaengine_unmap_put(um);
641 642
			result("submit error", total_tests, src_off,
			       dst_off, len, ret);
643 644 645 646
			msleep(100);
			failed_tests++;
			continue;
		}
Dan Williams's avatar
Dan Williams committed
647
		dma_async_issue_pending(chan);
648

649
		wait_event_freezable_timeout(done_wait, done.done,
650
					     msecs_to_jiffies(params->timeout));
651

652
		status = dma_async_is_tx_complete(chan, cookie, NULL, NULL);
653

654 655 656 657 658 659 660 661 662
		if (!done.done) {
			/*
			 * We're leaving the timed out dma operation with
			 * dangling pointer to done_wait.  To make this
			 * correct, we'll need to allocate wait_done for
			 * each test iteration and perform "who's gonna
			 * free it this time?" dancing.  For now, just
			 * leave it dangling.
			 */
663
			dmaengine_unmap_put(um);
664 665
			result("test timed out", total_tests, src_off, dst_off,
			       len, 0);
666 667
			failed_tests++;
			continue;
668
		} else if (status != DMA_COMPLETE) {
669
			dmaengine_unmap_put(um);
670 671 672 673
			result(status == DMA_ERROR ?
			       "completion error status" :
			       "completion busy status", total_tests, src_off,
			       dst_off, len, ret);
674 675 676
			failed_tests++;
			continue;
		}
677

678
		dmaengine_unmap_put(um);
679

680
		if (params->noverify) {
Dan Williams's avatar
Dan Williams committed
681 682
			verbose_result("test passed", total_tests, src_off,
				       dst_off, len, 0);
683 684
			continue;
		}
685

686
		pr_debug("%s: verifying source buffer...\n", current->comm);
687
		error_count = dmatest_verify(thread->srcs, 0, src_off,
688
				0, PATTERN_SRC, true);
689 690 691 692 693 694 695
		error_count += dmatest_verify(thread->srcs, src_off,
				src_off + len, src_off,
				PATTERN_SRC | PATTERN_COPY, true);
		error_count += dmatest_verify(thread->srcs, src_off + len,
				params->buf_size, src_off + len,
				PATTERN_SRC, true);

696
		pr_debug("%s: verifying dest buffer...\n", current->comm);
697
		error_count += dmatest_verify(thread->dsts, 0, dst_off,
698
				0, PATTERN_DST, false);
699 700 701 702 703 704
		error_count += dmatest_verify(thread->dsts, dst_off,
				dst_off + len, src_off,
				PATTERN_SRC | PATTERN_COPY, false);
		error_count += dmatest_verify(thread->dsts, dst_off + len,
				params->buf_size, dst_off + len,
				PATTERN_DST, false);
705 706

		if (error_count) {
707 708
			result("data error", total_tests, src_off, dst_off,
			       len, error_count);
709 710
			failed_tests++;
		} else {
Dan Williams's avatar
Dan Williams committed
711 712
			verbose_result("test passed", total_tests, src_off,
				       dst_off, len, 0);
713 714
		}
	}
715
	runtime = ktime_us_delta(ktime_get(), ktime);
716 717

	ret = 0;
718
err_dstbuf:
Dan Williams's avatar
Dan Williams committed
719 720 721 722
	for (i = 0; thread->dsts[i]; i++)
		kfree(thread->dsts[i]);
	kfree(thread->dsts);
err_dsts:
723
err_srcbuf:
Dan Williams's avatar
Dan Williams committed
724 725 726 727
	for (i = 0; thread->srcs[i]; i++)
		kfree(thread->srcs[i]);
	kfree(thread->srcs);
err_srcs:
728 729
	kfree(pq_coefs);
err_thread_type:
730 731 732 733
	pr_info("%s: summary %u tests, %u failures %llu iops %llu KB/s (%d)\n",
		current->comm, total_tests, failed_tests,
		dmatest_persec(runtime, total_tests),
		dmatest_KBs(runtime, total_len), ret);
734

735
	/* terminate all transfers on specified channels */
736 737 738
	if (ret)
		dmaengine_terminate_all(chan);

739
	thread->done = true;
740
	wake_up(&thread_wait);
741

742 743 744 745 746 747 748 749 750 751 752
	return ret;
}

static void dmatest_cleanup_channel(struct dmatest_chan *dtc)
{
	struct dmatest_thread	*thread;
	struct dmatest_thread	*_thread;
	int			ret;

	list_for_each_entry_safe(thread, _thread, &dtc->threads, node) {
		ret = kthread_stop(thread->task);
753 754
		pr_debug("thread %s exited with status %d\n",
			 thread->task->comm, ret);
755
		list_del(&thread->node);
756
		put_task_struct(thread->task);
757 758
		kfree(thread);
	}
759 760

	/* terminate all transfers on specified channels */
761
	dmaengine_terminate_all(dtc->chan);
762

763 764 765
	kfree(dtc);
}

766 767
static int dmatest_add_threads(struct dmatest_info *info,
		struct dmatest_chan *dtc, enum dma_transaction_type type)
768
{
769
	struct dmatest_params *params = &info->params;
Dan Williams's avatar
Dan Williams committed
770 771 772 773
	struct dmatest_thread *thread;
	struct dma_chan *chan = dtc->chan;
	char *op;
	unsigned int i;
774

Dan Williams's avatar
Dan Williams committed
775 776
	if (type == DMA_MEMCPY)
		op = "copy";
777 778
	else if (type == DMA_SG)
		op = "sg";
Dan Williams's avatar
Dan Williams committed
779 780
	else if (type == DMA_XOR)
		op = "xor";
Dan Williams's avatar
Dan Williams committed
781 782
	else if (type == DMA_PQ)
		op = "pq";
Dan Williams's avatar
Dan Williams committed
783 784
	else
		return -EINVAL;
785

786
	for (i = 0; i < params->threads_per_chan; i++) {
787 788
		thread = kzalloc(sizeof(struct dmatest_thread), GFP_KERNEL);
		if (!thread) {
789 790
			pr_warn("No memory for %s-%s%u\n",
				dma_chan_name(chan), op, i);
791 792
			break;
		}
793
		thread->info = info;
794
		thread->chan = dtc->chan;
Dan Williams's avatar
Dan Williams committed
795
		thread->type = type;
796
		smp_wmb();
797
		thread->task = kthread_create(dmatest_func, thread, "%s-%s%u",
Dan Williams's avatar
Dan Williams committed
798
				dma_chan_name(chan), op, i);
799
		if (IS_ERR(thread->task)) {
800
			pr_warn("Failed to create thread %s-%s%u\n",
801
				dma_chan_name(chan), op, i);
802 803 804 805 806
			kfree(thread);
			break;
		}

		/* srcbuf and dstbuf are allocated by the thread itself */
807
		get_task_struct(thread->task);
808
		list_add_tail(&thread->node, &dtc->threads);
809
		wake_up_process(thread->task);
810 811
	}

Dan Williams's avatar
Dan Williams committed
812 813 814
	return i;
}

815 816
static int dmatest_add_channel(struct dmatest_info *info,
		struct dma_chan *chan)
Dan Williams's avatar
Dan Williams committed
817 818 819 820
{
	struct dmatest_chan	*dtc;
	struct dma_device	*dma_dev = chan->device;
	unsigned int		thread_count = 0;
821
	int cnt;
Dan Williams's avatar
Dan Williams committed
822 823 824

	dtc = kmalloc(sizeof(struct dmatest_chan), GFP_KERNEL);
	if (!dtc) {
825
		pr_warn("No memory for %s\n", dma_chan_name(chan));
Dan Williams's avatar
Dan Williams committed
826 827 828 829 830 831 832
		return -ENOMEM;
	}

	dtc->chan = chan;
	INIT_LIST_HEAD(&dtc->threads);

	if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) {
833 834 835 836
		if (dmatest == 0) {
			cnt = dmatest_add_threads(info, dtc, DMA_MEMCPY);
			thread_count += cnt > 0 ? cnt : 0;
		}
Dan Williams's avatar
Dan Williams committed
837
	}
838 839 840 841 842 843 844 845

	if (dma_has_cap(DMA_SG, dma_dev->cap_mask)) {
		if (dmatest == 1) {
			cnt = dmatest_add_threads(info, dtc, DMA_SG);
			thread_count += cnt > 0 ? cnt : 0;
		}
	}

Dan Williams's avatar
Dan Williams committed
846
	if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
847
		cnt = dmatest_add_threads(info, dtc, DMA_XOR);
848
		thread_count += cnt > 0 ? cnt : 0;
Dan Williams's avatar
Dan Williams committed
849
	}
Dan Williams's avatar
Dan Williams committed
850
	if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) {
851
		cnt = dmatest_add_threads(info, dtc, DMA_PQ);
852
		thread_count += cnt > 0 ? cnt : 0;
Dan Williams's avatar
Dan Williams committed
853
	}
Dan Williams's avatar
Dan Williams committed
854

855
	pr_info("Started %u threads using %s\n",
Dan Williams's avatar
Dan Williams committed
856
		thread_count, dma_chan_name(chan));
857

858 859
	list_add_tail(&dtc->node, &info->channels);
	info->nr_channels++;
860

861
	return 0;
862 863
}

864
static bool filter(struct dma_chan *chan, void *param)
865
{
866
	struct dmatest_params *params = param;
867

868 869
	if (!dmatest_match_channel(params, chan) ||
	    !dmatest_match_device(params, chan->device))
870
		return false;
871
	else
872
		return true;
873 874
}

875 876
static void request_channels(struct dmatest_info *info,
			     enum dma_transaction_type type)
877
{
878 879 880
	dma_cap_mask_t mask;

	dma_cap_zero(mask);
881
	dma_cap_set(type, mask);
882
	for (;;) {
883 884 885
		struct dmatest_params *params = &info->params;
		struct dma_chan *chan;

886
		chan = dma_request_channel(mask, filter, params);
887
		if (chan) {
888
			if (dmatest_add_channel(info, chan)) {
889 890 891 892 893
				dma_release_channel(chan);
				break; /* add_channel failed, punt */
			}
		} else
			break; /* no more channels available */
894 895
		if (params->max_channels &&
		    info->nr_channels >= params->max_channels)
896 897
			break; /* we have all we need */
	}
898 899
}

900
static void run_threaded_test(struct dmatest_info *info)
901
{
902
	struct dmatest_params *params = &info->params;
903

904 905 906 907 908 909 910 911 912 913
	/* Copy test parameters */
	params->buf_size = test_buf_size;
	strlcpy(params->channel, strim(test_channel), sizeof(params->channel));
	strlcpy(params->device, strim(test_device), sizeof(params->device));
	params->threads_per_chan = threads_per_chan;
	params->max_channels = max_channels;
	params->iterations = iterations;
	params->xor_sources = xor_sources;
	params->pq_sources = pq_sources;
	params->timeout = timeout;
914
	params->noverify = noverify;
915 916 917

	request_channels(info, DMA_MEMCPY);
	request_channels(info, DMA_XOR);
918
	request_channels(info, DMA_SG);
919
	request_channels(info, DMA_PQ);
920 921
}

922
static void stop_threaded_test(struct dmatest_info *info)
923
{
924
	struct dmatest_chan *dtc, *_dtc;
925
	struct dma_chan *chan;
926

927
	list_for_each_entry_safe(dtc, _dtc, &info->channels, node) {
928
		list_del(&dtc->node);
929
		chan = dtc->chan;
930
		dmatest_cleanup_channel(dtc);
931
		pr_debug("dropped channel %s\n", dma_chan_name(chan));
932
		dma_release_channel(chan);
933
	}
934 935

	info->nr_channels = 0;
936
}
937

938
static void restart_threaded_test(struct dmatest_info *info, bool run)
939
{
940 941 942 943
	/* we might be called early to set run=, defer running until all
	 * parameters have been evaluated
	 */
	if (!info->did_init)
944
		return;
945 946

	/* Stop any running test first */
947
	stop_threaded_test(info);
948 949

	/* Run test with new parameters */
950
	run_threaded_test(info);
951 952
}

953
static int dmatest_run_get(char *val, const struct kernel_param *kp)
954
{
955
	struct dmatest_info *info = &test_info;
956 957

	mutex_lock(&info->lock);
958 959
	if (is_threaded_test_run(info)) {
		dmatest_run = true;
960
	} else {
961 962
		stop_threaded_test(info);
		dmatest_run = false;
963
	}
964 965
	mutex_unlock(&info->lock);

966
	return param_get_bool(val, kp);
967 968
}

969
static int dmatest_run_set(const char *val, const struct kernel_param *kp)
970
{
971 972
	struct dmatest_info *info = &test_info;
	int ret;
973

974 975 976
	mutex_lock(&info->lock);
	ret = param_set_bool(val, kp);
	if (ret) {
977
		mutex_unlock(&info->lock);
978
		return ret;
979 980
	}

981 982 983
	if (is_threaded_test_run(info))
		ret = -EBUSY;
	else if (dmatest_run)
984
		restart_threaded_test(info, dmatest_run);
985

986
	mutex_unlock(&info->lock);
987

988
	return ret;
989 990
}