ntb_transport.c 49 KB
Newer Older
1
2
3
4
5
6
7
/*
 * This file is provided under a dual BSD/GPLv2 license.  When using or
 *   redistributing this file, you may do so under either license.
 *
 *   GPL LICENSE SUMMARY
 *
 *   Copyright(c) 2012 Intel Corporation. All rights reserved.
8
 *   Copyright (C) 2015 EMC Corporation. All Rights Reserved.
9
10
11
12
13
14
15
16
 *
 *   This program is free software; you can redistribute it and/or modify
 *   it under the terms of version 2 of the GNU General Public License as
 *   published by the Free Software Foundation.
 *
 *   BSD LICENSE
 *
 *   Copyright(c) 2012 Intel Corporation. All rights reserved.
17
 *   Copyright (C) 2015 EMC Corporation. All Rights Reserved.
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
 *
 *   Redistribution and use in source and binary forms, with or without
 *   modification, are permitted provided that the following conditions
 *   are met:
 *
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copy
 *       notice, this list of conditions and the following disclaimer in
 *       the documentation and/or other materials provided with the
 *       distribution.
 *     * Neither the name of Intel Corporation nor the names of its
 *       contributors may be used to endorse or promote products derived
 *       from this software without specific prior written permission.
 *
 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
45
 * PCIe NTB Transport Linux driver
46
47
48
49
50
51
 *
 * Contact Information:
 * Jon Mason <jon.mason@intel.com>
 */
#include <linux/debugfs.h>
#include <linux/delay.h>
52
#include <linux/dmaengine.h>
53
54
55
56
57
58
59
60
#include <linux/dma-mapping.h>
#include <linux/errno.h>
#include <linux/export.h>
#include <linux/interrupt.h>
#include <linux/module.h>
#include <linux/pci.h>
#include <linux/slab.h>
#include <linux/types.h>
61
62
#include "linux/ntb.h"
#include "linux/ntb_transport.h"
63

64
65
66
67
68
69
70
71
72
73
74
75
76
#define NTB_TRANSPORT_VERSION	4
#define NTB_TRANSPORT_VER	"4"
#define NTB_TRANSPORT_NAME	"ntb_transport"
#define NTB_TRANSPORT_DESC	"Software Queue-Pair Transport over NTB"

MODULE_DESCRIPTION(NTB_TRANSPORT_DESC);
MODULE_VERSION(NTB_TRANSPORT_VER);
MODULE_LICENSE("Dual BSD/GPL");
MODULE_AUTHOR("Intel Corporation");

static unsigned long max_mw_size;
module_param(max_mw_size, ulong, 0644);
MODULE_PARM_DESC(max_mw_size, "Limit size of large memory windows");
77

78
static unsigned int transport_mtu = 0x401E;
79
80
81
module_param(transport_mtu, uint, 0644);
MODULE_PARM_DESC(transport_mtu, "Maximum size of NTB transport packets");

Jon Mason's avatar
Jon Mason committed
82
static unsigned char max_num_clients;
83
84
85
module_param(max_num_clients, byte, 0644);
MODULE_PARM_DESC(max_num_clients, "Maximum number of NTB transport clients");

86
87
88
89
static unsigned int copy_bytes = 1024;
module_param(copy_bytes, uint, 0644);
MODULE_PARM_DESC(copy_bytes, "Threshold under which NTB will use the CPU to copy instead of DMA");

90
91
static struct dentry *nt_debugfs_dir;

92
93
94
struct ntb_queue_entry {
	/* ntb_queue list reference */
	struct list_head entry;
95
	/* pointers to data to be transferred */
96
97
98
99
	void *cb_data;
	void *buf;
	unsigned int len;
	unsigned int flags;
100
101
102
103
104
105
106

	struct ntb_transport_qp *qp;
	union {
		struct ntb_payload_header __iomem *tx_hdr;
		struct ntb_payload_header *rx_hdr;
	};
	unsigned int index;
107
108
};

Jon Mason's avatar
Jon Mason committed
109
110
111
112
struct ntb_rx_info {
	unsigned int entry;
};

113
struct ntb_transport_qp {
114
115
	struct ntb_transport_ctx *transport;
	struct ntb_dev *ndev;
116
	void *cb_data;
117
	struct dma_chan *dma_chan;
118
119

	bool client_ready;
120
121
	bool link_is_up;

122
	u8 qp_num;	/* Only 64 QP's are allowed.  0-63 */
123
	u64 qp_bit;
124

Jon Mason's avatar
Jon Mason committed
125
	struct ntb_rx_info __iomem *rx_info;
Jon Mason's avatar
Jon Mason committed
126
127
	struct ntb_rx_info *remote_rx_info;

Jon Mason's avatar
Jon Mason committed
128
129
	void (*tx_handler)(struct ntb_transport_qp *qp, void *qp_data,
			   void *data, int len);
130
131
	struct list_head tx_free_q;
	spinlock_t ntb_tx_free_q_lock;
Jon Mason's avatar
Jon Mason committed
132
	void __iomem *tx_mw;
133
	dma_addr_t tx_mw_phys;
Jon Mason's avatar
Jon Mason committed
134
135
	unsigned int tx_index;
	unsigned int tx_max_entry;
136
	unsigned int tx_max_frame;
137

Jon Mason's avatar
Jon Mason committed
138
139
	void (*rx_handler)(struct ntb_transport_qp *qp, void *qp_data,
			   void *data, int len);
140
141
142
143
	struct list_head rx_pend_q;
	struct list_head rx_free_q;
	spinlock_t ntb_rx_pend_q_lock;
	spinlock_t ntb_rx_free_q_lock;
Jon Mason's avatar
Jon Mason committed
144
145
146
	void *rx_buff;
	unsigned int rx_index;
	unsigned int rx_max_entry;
147
	unsigned int rx_max_frame;
148
	dma_cookie_t last_cookie;
149
	struct tasklet_struct rxc_db_work;
150

Jon Mason's avatar
Jon Mason committed
151
	void (*event_handler)(void *data, int status);
152
	struct delayed_work link_work;
153
	struct work_struct link_cleanup;
154
155
156
157
158
159
160
161
162
163
164

	struct dentry *debugfs_dir;
	struct dentry *debugfs_stats;

	/* Stats */
	u64 rx_bytes;
	u64 rx_pkts;
	u64 rx_ring_empty;
	u64 rx_err_no_buf;
	u64 rx_err_oflow;
	u64 rx_err_ver;
165
166
	u64 rx_memcpy;
	u64 rx_async;
167
168
169
	u64 tx_bytes;
	u64 tx_pkts;
	u64 tx_ring_full;
170
171
172
	u64 tx_err_no_buf;
	u64 tx_memcpy;
	u64 tx_async;
173
174
175
};

struct ntb_transport_mw {
176
177
178
179
180
181
182
	phys_addr_t phys_addr;
	resource_size_t phys_size;
	resource_size_t xlat_align;
	resource_size_t xlat_align_size;
	void __iomem *vbase;
	size_t xlat_size;
	size_t buff_size;
183
184
185
186
187
188
	void *virt_addr;
	dma_addr_t dma_addr;
};

struct ntb_transport_client_dev {
	struct list_head entry;
189
	struct ntb_transport_ctx *nt;
190
191
192
	struct device dev;
};

193
struct ntb_transport_ctx {
194
195
196
	struct list_head entry;
	struct list_head client_devs;

197
198
199
200
201
202
203
204
205
206
	struct ntb_dev *ndev;

	struct ntb_transport_mw *mw_vec;
	struct ntb_transport_qp *qp_vec;
	unsigned int mw_count;
	unsigned int qp_count;
	u64 qp_bitmap;
	u64 qp_bitmap_free;

	bool link_is_up;
207
	struct delayed_work link_work;
208
	struct work_struct link_cleanup;
209
210
211
};

enum {
212
213
	DESC_DONE_FLAG = BIT(0),
	LINK_DOWN_FLAG = BIT(1),
214
215
216
};

struct ntb_payload_header {
Jon Mason's avatar
Jon Mason committed
217
	unsigned int ver;
218
219
220
221
222
223
224
	unsigned int len;
	unsigned int flags;
};

enum {
	VERSION = 0,
	QP_LINKS,
Jon Mason's avatar
Jon Mason committed
225
226
227
228
229
230
	NUM_QPS,
	NUM_MWS,
	MW0_SZ_HIGH,
	MW0_SZ_LOW,
	MW1_SZ_HIGH,
	MW1_SZ_LOW,
231
232
233
	MAX_SPAD,
};

234
235
236
237
238
239
240
#define dev_client_dev(__dev) \
	container_of((__dev), struct ntb_transport_client_dev, dev)

#define drv_client(__drv) \
	container_of((__drv), struct ntb_transport_client, driver)

#define QP_TO_MW(nt, qp)	((qp) % nt->mw_count)
241
242
243
#define NTB_QP_DEF_NUM_ENTRIES	100
#define NTB_LINK_DOWN_TIMEOUT	10

244
245
246
247
248
249
static void ntb_transport_rxc_db(unsigned long data);
static const struct ntb_ctx_ops ntb_transport_ops;
static struct ntb_client ntb_transport_client;

static int ntb_transport_bus_match(struct device *dev,
				   struct device_driver *drv)
250
251
252
253
{
	return !strncmp(dev_name(dev), drv->name, strlen(drv->name));
}

254
static int ntb_transport_bus_probe(struct device *dev)
255
{
256
	const struct ntb_transport_client *client;
257
258
259
	int rc = -EINVAL;

	get_device(dev);
260
261
262

	client = drv_client(dev->driver);
	rc = client->probe(dev);
263
264
265
266
267
268
	if (rc)
		put_device(dev);

	return rc;
}

269
static int ntb_transport_bus_remove(struct device *dev)
270
{
271
	const struct ntb_transport_client *client;
272

273
274
	client = drv_client(dev->driver);
	client->remove(dev);
275
276
277
278
279
280

	put_device(dev);

	return 0;
}

281
282
283
284
285
static struct bus_type ntb_transport_bus = {
	.name = "ntb_transport",
	.match = ntb_transport_bus_match,
	.probe = ntb_transport_bus_probe,
	.remove = ntb_transport_bus_remove,
286
287
288
289
};

static LIST_HEAD(ntb_transport_list);

290
static int ntb_bus_init(struct ntb_transport_ctx *nt)
291
292
293
294
295
{
	list_add(&nt->entry, &ntb_transport_list);
	return 0;
}

296
static void ntb_bus_remove(struct ntb_transport_ctx *nt)
297
298
299
300
301
302
303
304
305
306
307
308
309
{
	struct ntb_transport_client_dev *client_dev, *cd;

	list_for_each_entry_safe(client_dev, cd, &nt->client_devs, entry) {
		dev_err(client_dev->dev.parent, "%s still attached to bus, removing\n",
			dev_name(&client_dev->dev));
		list_del(&client_dev->entry);
		device_unregister(&client_dev->dev);
	}

	list_del(&nt->entry);
}

310
static void ntb_transport_client_release(struct device *dev)
311
312
313
{
	struct ntb_transport_client_dev *client_dev;

314
	client_dev = dev_client_dev(dev);
315
316
317
318
	kfree(client_dev);
}

/**
319
 * ntb_transport_unregister_client_dev - Unregister NTB client device
320
321
322
323
 * @device_name: Name of NTB client device
 *
 * Unregister an NTB client device with the NTB transport layer
 */
324
void ntb_transport_unregister_client_dev(char *device_name)
325
326
{
	struct ntb_transport_client_dev *client, *cd;
327
	struct ntb_transport_ctx *nt;
328
329
330
331
332
333
334
335
336

	list_for_each_entry(nt, &ntb_transport_list, entry)
		list_for_each_entry_safe(client, cd, &nt->client_devs, entry)
			if (!strncmp(dev_name(&client->dev), device_name,
				     strlen(device_name))) {
				list_del(&client->entry);
				device_unregister(&client->dev);
			}
}
337
EXPORT_SYMBOL_GPL(ntb_transport_unregister_client_dev);
338
339

/**
340
 * ntb_transport_register_client_dev - Register NTB client device
341
342
343
344
 * @device_name: Name of NTB client device
 *
 * Register an NTB client device with the NTB transport layer
 */
345
int ntb_transport_register_client_dev(char *device_name)
346
347
{
	struct ntb_transport_client_dev *client_dev;
348
	struct ntb_transport_ctx *nt;
349
	int node;
Jon Mason's avatar
Jon Mason committed
350
	int rc, i = 0;
351

352
353
354
	if (list_empty(&ntb_transport_list))
		return -ENODEV;

355
356
357
	list_for_each_entry(nt, &ntb_transport_list, entry) {
		struct device *dev;

358
359
360
361
		node = dev_to_node(&nt->ndev->dev);

		client_dev = kzalloc_node(sizeof(*client_dev),
					  GFP_KERNEL, node);
362
363
364
365
366
367
368
369
		if (!client_dev) {
			rc = -ENOMEM;
			goto err;
		}

		dev = &client_dev->dev;

		/* setup and register client devices */
Jon Mason's avatar
Jon Mason committed
370
		dev_set_name(dev, "%s%d", device_name, i);
371
372
373
		dev->bus = &ntb_transport_bus;
		dev->release = ntb_transport_client_release;
		dev->parent = &nt->ndev->dev;
374
375
376
377
378
379
380
381

		rc = device_register(dev);
		if (rc) {
			kfree(client_dev);
			goto err;
		}

		list_add_tail(&client_dev->entry, &nt->client_devs);
Jon Mason's avatar
Jon Mason committed
382
		i++;
383
384
385
386
387
	}

	return 0;

err:
388
	ntb_transport_unregister_client_dev(device_name);
389
390
391

	return rc;
}
392
EXPORT_SYMBOL_GPL(ntb_transport_register_client_dev);
393
394

/**
395
 * ntb_transport_register_client - Register NTB client driver
396
397
398
399
400
401
 * @drv: NTB client driver to be registered
 *
 * Register an NTB client driver with the NTB transport layer
 *
 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
 */
402
int ntb_transport_register_client(struct ntb_transport_client *drv)
403
{
404
	drv->driver.bus = &ntb_transport_bus;
405

406
407
408
	if (list_empty(&ntb_transport_list))
		return -ENODEV;

409
410
	return driver_register(&drv->driver);
}
411
EXPORT_SYMBOL_GPL(ntb_transport_register_client);
412
413

/**
414
 * ntb_transport_unregister_client - Unregister NTB client driver
415
416
417
418
419
420
 * @drv: NTB client driver to be unregistered
 *
 * Unregister an NTB client driver with the NTB transport layer
 *
 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
 */
421
void ntb_transport_unregister_client(struct ntb_transport_client *drv)
422
423
424
{
	driver_unregister(&drv->driver);
}
425
EXPORT_SYMBOL_GPL(ntb_transport_unregister_client);
426
427
428
429
430

static ssize_t debugfs_read(struct file *filp, char __user *ubuf, size_t count,
			    loff_t *offp)
{
	struct ntb_transport_qp *qp;
431
	char *buf;
432
433
	ssize_t ret, out_offset, out_count;

434
	out_count = 1000;
435
436
437
438

	buf = kmalloc(out_count, GFP_KERNEL);
	if (!buf)
		return -ENOMEM;
439
440
441
442
443
444
445
446
447

	qp = filp->private_data;
	out_offset = 0;
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "NTB QP stats\n");
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_bytes - \t%llu\n", qp->rx_bytes);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_pkts - \t%llu\n", qp->rx_pkts);
448
449
450
451
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_memcpy - \t%llu\n", qp->rx_memcpy);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_async - \t%llu\n", qp->rx_async);
452
453
454
455
456
457
458
459
460
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_ring_empty - %llu\n", qp->rx_ring_empty);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_err_no_buf - %llu\n", qp->rx_err_no_buf);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_err_oflow - \t%llu\n", qp->rx_err_oflow);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_err_ver - \t%llu\n", qp->rx_err_ver);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
Jon Mason's avatar
Jon Mason committed
461
			       "rx_buff - \t%p\n", qp->rx_buff);
462
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
Jon Mason's avatar
Jon Mason committed
463
			       "rx_index - \t%u\n", qp->rx_index);
464
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
Jon Mason's avatar
Jon Mason committed
465
			       "rx_max_entry - \t%u\n", qp->rx_max_entry);
466
467
468
469
470

	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "tx_bytes - \t%llu\n", qp->tx_bytes);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "tx_pkts - \t%llu\n", qp->tx_pkts);
471
472
473
474
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "tx_memcpy - \t%llu\n", qp->tx_memcpy);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "tx_async - \t%llu\n", qp->tx_async);
475
476
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "tx_ring_full - \t%llu\n", qp->tx_ring_full);
477
478
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "tx_err_no_buf - %llu\n", qp->tx_err_no_buf);
479
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
Jon Mason's avatar
Jon Mason committed
480
			       "tx_mw - \t%p\n", qp->tx_mw);
481
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
Jon Mason's avatar
Jon Mason committed
482
			       "tx_index - \t%u\n", qp->tx_index);
483
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
Jon Mason's avatar
Jon Mason committed
484
			       "tx_max_entry - \t%u\n", qp->tx_max_entry);
485
486

	out_offset += snprintf(buf + out_offset, out_count - out_offset,
487
488
			       "\nQP Link %s\n",
			       qp->link_is_up ? "Up" : "Down");
489
490
	if (out_offset > out_count)
		out_offset = out_count;
491
492

	ret = simple_read_from_buffer(ubuf, count, offp, buf, out_offset);
493
	kfree(buf);
494
495
496
497
498
	return ret;
}

static const struct file_operations ntb_qp_debugfs_stats = {
	.owner = THIS_MODULE,
Jon Mason's avatar
Jon Mason committed
499
	.open = simple_open,
500
501
502
503
504
505
506
507
508
509
510
511
512
513
	.read = debugfs_read,
};

static void ntb_list_add(spinlock_t *lock, struct list_head *entry,
			 struct list_head *list)
{
	unsigned long flags;

	spin_lock_irqsave(lock, flags);
	list_add_tail(entry, list);
	spin_unlock_irqrestore(lock, flags);
}

static struct ntb_queue_entry *ntb_list_rm(spinlock_t *lock,
Jon Mason's avatar
Jon Mason committed
514
					   struct list_head *list)
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
{
	struct ntb_queue_entry *entry;
	unsigned long flags;

	spin_lock_irqsave(lock, flags);
	if (list_empty(list)) {
		entry = NULL;
		goto out;
	}
	entry = list_first_entry(list, struct ntb_queue_entry, entry);
	list_del(&entry->entry);
out:
	spin_unlock_irqrestore(lock, flags);

	return entry;
}

532
533
static int ntb_transport_setup_qp_mw(struct ntb_transport_ctx *nt,
				     unsigned int qp_num)
534
{
535
536
	struct ntb_transport_qp *qp = &nt->qp_vec[qp_num];
	struct ntb_transport_mw *mw;
537
	unsigned int rx_size, num_qps_mw;
538
	unsigned int mw_num, mw_count, qp_count;
Jon Mason's avatar
Jon Mason committed
539
	unsigned int i;
540

541
542
	mw_count = nt->mw_count;
	qp_count = nt->qp_count;
Jon Mason's avatar
Jon Mason committed
543

544
545
546
547
548
	mw_num = QP_TO_MW(nt, qp_num);
	mw = &nt->mw_vec[mw_num];

	if (!mw->virt_addr)
		return -ENOMEM;
549

550
551
	if (qp_count % mw_count && mw_num + 1 < qp_count / mw_count)
		num_qps_mw = qp_count / mw_count + 1;
552
	else
553
		num_qps_mw = qp_count / mw_count;
554

555
556
	rx_size = (unsigned int)mw->xlat_size / num_qps_mw;
	qp->rx_buff = mw->virt_addr + rx_size * qp_num / mw_count;
Jon Mason's avatar
Jon Mason committed
557
558
	rx_size -= sizeof(struct ntb_rx_info);

559
560
	qp->remote_rx_info = qp->rx_buff + rx_size;

561
562
	/* Due to housekeeping, there must be atleast 2 buffs */
	qp->rx_max_frame = min(transport_mtu, rx_size / 2);
Jon Mason's avatar
Jon Mason committed
563
564
565
	qp->rx_max_entry = rx_size / qp->rx_max_frame;
	qp->rx_index = 0;

566
	qp->remote_rx_info->entry = qp->rx_max_entry - 1;
567

568
	/* setup the hdr offsets with 0's */
Jon Mason's avatar
Jon Mason committed
569
	for (i = 0; i < qp->rx_max_entry; i++) {
570
571
		void *offset = (qp->rx_buff + qp->rx_max_frame * (i + 1) -
				sizeof(struct ntb_payload_header));
572
		memset(offset, 0, sizeof(struct ntb_payload_header));
Jon Mason's avatar
Jon Mason committed
573
	}
574
575
576

	qp->rx_pkts = 0;
	qp->tx_pkts = 0;
Jon Mason's avatar
Jon Mason committed
577
	qp->tx_index = 0;
578
579

	return 0;
580
581
}

582
static void ntb_free_mw(struct ntb_transport_ctx *nt, int num_mw)
Jon Mason's avatar
Jon Mason committed
583
{
584
585
	struct ntb_transport_mw *mw = &nt->mw_vec[num_mw];
	struct pci_dev *pdev = nt->ndev->pdev;
Jon Mason's avatar
Jon Mason committed
586
587
588
589

	if (!mw->virt_addr)
		return;

590
591
592
593
594
	ntb_mw_clear_trans(nt->ndev, num_mw);
	dma_free_coherent(&pdev->dev, mw->buff_size,
			  mw->virt_addr, mw->dma_addr);
	mw->xlat_size = 0;
	mw->buff_size = 0;
Jon Mason's avatar
Jon Mason committed
595
596
597
	mw->virt_addr = NULL;
}

598
599
static int ntb_set_mw(struct ntb_transport_ctx *nt, int num_mw,
		      unsigned int size)
600
{
601
602
603
604
605
606
607
	struct ntb_transport_mw *mw = &nt->mw_vec[num_mw];
	struct pci_dev *pdev = nt->ndev->pdev;
	unsigned int xlat_size, buff_size;
	int rc;

	xlat_size = round_up(size, mw->xlat_align_size);
	buff_size = round_up(size, mw->xlat_align);
608

Jon Mason's avatar
Jon Mason committed
609
	/* No need to re-setup */
610
	if (mw->xlat_size == xlat_size)
Jon Mason's avatar
Jon Mason committed
611
612
		return 0;

613
	if (mw->buff_size)
Jon Mason's avatar
Jon Mason committed
614
615
		ntb_free_mw(nt, num_mw);

616
617
618
	/* Alloc memory for receiving data.  Must be aligned */
	mw->xlat_size = xlat_size;
	mw->buff_size = buff_size;
619

620
621
	mw->virt_addr = dma_alloc_coherent(&pdev->dev, buff_size,
					   &mw->dma_addr, GFP_KERNEL);
622
	if (!mw->virt_addr) {
623
624
625
626
		mw->xlat_size = 0;
		mw->buff_size = 0;
		dev_err(&pdev->dev, "Unable to alloc MW buff of size %d\n",
			buff_size);
627
628
629
		return -ENOMEM;
	}

630
631
632
633
634
635
	/*
	 * we must ensure that the memory address allocated is BAR size
	 * aligned in order for the XLAT register to take the value. This
	 * is a requirement of the hardware. It is recommended to setup CMA
	 * for BAR sizes equal or greater than 4MB.
	 */
636
637
	if (!IS_ALIGNED(mw->dma_addr, mw->xlat_align)) {
		dev_err(&pdev->dev, "DMA memory %pad is not aligned\n",
638
639
640
641
642
			&mw->dma_addr);
		ntb_free_mw(nt, num_mw);
		return -ENOMEM;
	}

643
	/* Notify HW the memory location of the receive buffer */
644
645
646
647
648
649
	rc = ntb_mw_set_trans(nt->ndev, num_mw, mw->dma_addr, mw->xlat_size);
	if (rc) {
		dev_err(&pdev->dev, "Unable to set mw%d translation", num_mw);
		ntb_free_mw(nt, num_mw);
		return -EIO;
	}
650
651
652
653

	return 0;
}

654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
static void ntb_qp_link_down_reset(struct ntb_transport_qp *qp)
{
	qp->link_is_up = false;

	qp->tx_index = 0;
	qp->rx_index = 0;
	qp->rx_bytes = 0;
	qp->rx_pkts = 0;
	qp->rx_ring_empty = 0;
	qp->rx_err_no_buf = 0;
	qp->rx_err_oflow = 0;
	qp->rx_err_ver = 0;
	qp->rx_memcpy = 0;
	qp->rx_async = 0;
	qp->tx_bytes = 0;
	qp->tx_pkts = 0;
	qp->tx_ring_full = 0;
	qp->tx_err_no_buf = 0;
	qp->tx_memcpy = 0;
	qp->tx_async = 0;
}

676
static void ntb_qp_link_cleanup(struct ntb_transport_qp *qp)
677
{
678
679
	struct ntb_transport_ctx *nt = qp->transport;
	struct pci_dev *pdev = nt->ndev->pdev;
680

681
	dev_info(&pdev->dev, "qp %d: Link Cleanup\n", qp->qp_num);
682
683
684

	cancel_delayed_work_sync(&qp->link_work);
	ntb_qp_link_down_reset(qp);
685
686
687

	if (qp->event_handler)
		qp->event_handler(qp->cb_data, qp->link_is_up);
688
689
690
691
692
693
694
}

static void ntb_qp_link_cleanup_work(struct work_struct *work)
{
	struct ntb_transport_qp *qp = container_of(work,
						   struct ntb_transport_qp,
						   link_cleanup);
695
	struct ntb_transport_ctx *nt = qp->transport;
696
697

	ntb_qp_link_cleanup(qp);
698

699
	if (nt->link_is_up)
700
701
702
703
		schedule_delayed_work(&qp->link_work,
				      msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT));
}

704
705
706
707
708
static void ntb_qp_link_down(struct ntb_transport_qp *qp)
{
	schedule_work(&qp->link_cleanup);
}

709
static void ntb_transport_link_cleanup(struct ntb_transport_ctx *nt)
710
{
711
712
	struct ntb_transport_qp *qp;
	u64 qp_bitmap_alloc;
713
714
	int i;

715
716
	qp_bitmap_alloc = nt->qp_bitmap & ~nt->qp_bitmap_free;

717
	/* Pass along the info to any clients */
718
719
720
721
722
723
724
	for (i = 0; i < nt->qp_count; i++)
		if (qp_bitmap_alloc & BIT_ULL(i)) {
			qp = &nt->qp_vec[i];
			ntb_qp_link_cleanup(qp);
			cancel_work_sync(&qp->link_cleanup);
			cancel_delayed_work_sync(&qp->link_work);
		}
725

726
	if (!nt->link_is_up)
727
728
729
730
731
732
733
		cancel_delayed_work_sync(&nt->link_work);

	/* The scratchpad registers keep the values if the remote side
	 * goes down, blast them now to give them a sane value the next
	 * time they are accessed
	 */
	for (i = 0; i < MAX_SPAD; i++)
734
		ntb_spad_write(nt->ndev, i, 0);
735
736
}

737
738
static void ntb_transport_link_cleanup_work(struct work_struct *work)
{
739
740
	struct ntb_transport_ctx *nt =
		container_of(work, struct ntb_transport_ctx, link_cleanup);
741
742
743
744

	ntb_transport_link_cleanup(nt);
}

745
static void ntb_transport_event_callback(void *data)
746
{
747
	struct ntb_transport_ctx *nt = data;
748

749
	if (ntb_link_is_up(nt->ndev, NULL, NULL) == 1)
750
		schedule_delayed_work(&nt->link_work, 0);
751
	else
752
		schedule_work(&nt->link_cleanup);
753
754
755
756
}

static void ntb_transport_link_work(struct work_struct *work)
{
757
758
759
760
761
	struct ntb_transport_ctx *nt =
		container_of(work, struct ntb_transport_ctx, link_work.work);
	struct ntb_dev *ndev = nt->ndev;
	struct pci_dev *pdev = ndev->pdev;
	resource_size_t size;
762
	u32 val;
763
	int rc, i, spad;
764

Jon Mason's avatar
Jon Mason committed
765
	/* send the local info, in the opposite order of the way we read it */
766
767
	for (i = 0; i < nt->mw_count; i++) {
		size = nt->mw_vec[i].phys_size;
768

769
770
		if (max_mw_size && size > max_mw_size)
			size = max_mw_size;
771

772
773
		spad = MW0_SZ_HIGH + (i * 2);
		ntb_peer_spad_write(ndev, spad, (u32)(size >> 32));
774

775
776
		spad = MW0_SZ_LOW + (i * 2);
		ntb_peer_spad_write(ndev, spad, (u32)size);
777
778
	}

779
	ntb_peer_spad_write(ndev, NUM_MWS, nt->mw_count);
780

781
	ntb_peer_spad_write(ndev, NUM_QPS, nt->qp_count);
782

783
	ntb_peer_spad_write(ndev, VERSION, NTB_TRANSPORT_VERSION);
784

785
	/* Query the remote side for its info */
786
	val = ntb_spad_read(ndev, VERSION);
787
788
	dev_dbg(&pdev->dev, "Remote version = %d\n", val);
	if (val != NTB_TRANSPORT_VERSION)
789
790
		goto out;

791
	val = ntb_spad_read(ndev, NUM_QPS);
792
	dev_dbg(&pdev->dev, "Remote max number of qps = %d\n", val);
793
	if (val != nt->qp_count)
794
795
		goto out;

796
	val = ntb_spad_read(ndev, NUM_MWS);
Jon Mason's avatar
Jon Mason committed
797
	dev_dbg(&pdev->dev, "Remote number of mws = %d\n", val);
798
799
	if (val != nt->mw_count)
		goto out;
800

801
	for (i = 0; i < nt->mw_count; i++) {
Jon Mason's avatar
Jon Mason committed
802
		u64 val64;
803

804
		val = ntb_spad_read(ndev, MW0_SZ_HIGH + (i * 2));
805
		val64 = (u64)val << 32;
Jon Mason's avatar
Jon Mason committed
806

807
		val = ntb_spad_read(ndev, MW0_SZ_LOW + (i * 2));
Jon Mason's avatar
Jon Mason committed
808
809
		val64 |= val;

810
		dev_dbg(&pdev->dev, "Remote MW%d size = %#llx\n", i, val64);
Jon Mason's avatar
Jon Mason committed
811
812
813
814
815

		rc = ntb_set_mw(nt, i, val64);
		if (rc)
			goto out1;
	}
816

817
	nt->link_is_up = true;
818

819
820
	for (i = 0; i < nt->qp_count; i++) {
		struct ntb_transport_qp *qp = &nt->qp_vec[i];
821
822
823

		ntb_transport_setup_qp_mw(nt, i);

824
		if (qp->client_ready)
825
826
827
828
829
			schedule_delayed_work(&qp->link_work, 0);
	}

	return;

Jon Mason's avatar
Jon Mason committed
830
out1:
831
	for (i = 0; i < nt->mw_count; i++)
Jon Mason's avatar
Jon Mason committed
832
		ntb_free_mw(nt, i);
833
out:
834
	if (ntb_link_is_up(ndev, NULL, NULL) == 1)
835
836
837
838
839
840
841
842
843
		schedule_delayed_work(&nt->link_work,
				      msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT));
}

static void ntb_qp_link_work(struct work_struct *work)
{
	struct ntb_transport_qp *qp = container_of(work,
						   struct ntb_transport_qp,
						   link_work.work);
844
845
846
	struct pci_dev *pdev = qp->ndev->pdev;
	struct ntb_transport_ctx *nt = qp->transport;
	int val;
847

848
	WARN_ON(!nt->link_is_up);
849

850
	val = ntb_spad_read(nt->ndev, QP_LINKS);
851

852
	ntb_peer_spad_write(nt->ndev, QP_LINKS, val | BIT(qp->qp_num));
853
854

	/* query remote spad for qp ready bits */
855
	ntb_peer_spad_read(nt->ndev, QP_LINKS);
Allen Hubbe's avatar
Allen Hubbe committed
856
	dev_dbg_ratelimited(&pdev->dev, "Remote QP link status = %x\n", val);
857
858

	/* See if the remote side is up */
859
	if (val & BIT(qp->qp_num)) {
860
		dev_info(&pdev->dev, "qp %d: Link Up\n", qp->qp_num);
861
862
		qp->link_is_up = true;

863
		if (qp->event_handler)
864
865
			qp->event_handler(qp->cb_data, qp->link_is_up);
	} else if (nt->link_is_up)
866
867
868
869
		schedule_delayed_work(&qp->link_work,
				      msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT));
}

870
static int ntb_transport_init_queue(struct ntb_transport_ctx *nt,
Jon Mason's avatar
Jon Mason committed
871
				    unsigned int qp_num)
872
873
{
	struct ntb_transport_qp *qp;
874
875
876
	struct ntb_transport_mw *mw;
	phys_addr_t mw_base;
	resource_size_t mw_size;
877
	unsigned int num_qps_mw, tx_size;
878
	unsigned int mw_num, mw_count, qp_count;
879
	u64 qp_offset;
Jon Mason's avatar
Jon Mason committed
880

881
882
	mw_count = nt->mw_count;
	qp_count = nt->qp_count;
883

884
885
886
887
	mw_num = QP_TO_MW(nt, qp_num);
	mw = &nt->mw_vec[mw_num];

	qp = &nt->qp_vec[qp_num];
888
889
890
	qp->qp_num = qp_num;
	qp->transport = nt;
	qp->ndev = nt->ndev;
891
	qp->client_ready = false;
892
	qp->event_handler = NULL;
893
	ntb_qp_link_down_reset(qp);
894

895
896
	if (qp_count % mw_count && mw_num + 1 < qp_count / mw_count)
		num_qps_mw = qp_count / mw_count + 1;
897
	else
898
899
900
901
		num_qps_mw = qp_count / mw_count;

	mw_base = nt->mw_vec[mw_num].phys_addr;
	mw_size = nt->mw_vec[mw_num].phys_size;
902

903
904
905
906
	tx_size = (unsigned int)mw_size / num_qps_mw;
	qp_offset = tx_size * qp_num / mw_count;

	qp->tx_mw = nt->mw_vec[mw_num].vbase + qp_offset;
907
908
909
	if (!qp->tx_mw)
		return -EINVAL;

910
	qp->tx_mw_phys = mw_base + qp_offset;
911
912
913
	if (!qp->tx_mw_phys)
		return -EINVAL;

Jon Mason's avatar
Jon Mason committed
914
	tx_size -= sizeof(struct ntb_rx_info);
915
	qp->rx_info = qp->tx_mw + tx_size;
Jon Mason's avatar
Jon Mason committed
916

917
918
	/* Due to housekeeping, there must be atleast 2 buffs */
	qp->tx_max_frame = min(transport_mtu, tx_size / 2);
Jon Mason's avatar
Jon Mason committed
919
	qp->tx_max_entry = tx_size / qp->tx_max_frame;
920

921
	if (nt_debugfs_dir) {
922
923
924
925
		char debugfs_name[4];

		snprintf(debugfs_name, 4, "qp%d", qp_num);
		qp->debugfs_dir = debugfs_create_dir(debugfs_name,
926
						     nt_debugfs_dir);
927
928
929
930

		qp->debugfs_stats = debugfs_create_file("stats", S_IRUSR,
							qp->debugfs_dir, qp,
							&ntb_qp_debugfs_stats);
931
932
933
	} else {
		qp->debugfs_dir = NULL;
		qp->debugfs_stats = NULL;
934
935
936
	}

	INIT_DELAYED_WORK(&qp->link_work, ntb_qp_link_work);
937
	INIT_WORK(&qp->link_cleanup, ntb_qp_link_cleanup_work);
938
939
940
941
942
943
944
945

	spin_lock_init(&qp->ntb_rx_pend_q_lock);
	spin_lock_init(&qp->ntb_rx_free_q_lock);
	spin_lock_init(&qp->ntb_tx_free_q_lock);

	INIT_LIST_HEAD(&qp->rx_pend_q);
	INIT_LIST_HEAD(&qp->rx_free_q);
	INIT_LIST_HEAD(&qp->tx_free_q);
946

947
948
949
	tasklet_init(&qp->rxc_db_work, ntb_transport_rxc_db,
		     (unsigned long)qp);

950
	return 0;
951
952
}

953
static int ntb_transport_probe(struct ntb_client *self, struct ntb_dev *ndev)
954
{
955
956
957
958
	struct ntb_transport_ctx *nt;
	struct ntb_transport_mw *mw;
	unsigned int mw_count, qp_count;
	u64 qp_bitmap;
959
	int node;
960
961
	int rc, i;

962
963
964
965
966
967
968
	if (ntb_db_is_unsafe(ndev))
		dev_dbg(&ndev->dev,
			"doorbell is unsafe, proceed anyway...\n");
	if (ntb_spad_is_unsafe(ndev))
		dev_dbg(&ndev->dev,
			"scratchpad is unsafe, proceed anyway...\n");

969
970
971
	node = dev_to_node(&ndev->dev);

	nt = kzalloc_node(sizeof(*nt), GFP_KERNEL, node);
972
973
974
	if (!nt)
		return -ENOMEM;

975
976
977
978
979
980
	nt->ndev = ndev;

	mw_count = ntb_mw_count(ndev);

	nt->mw_count = mw_count;

981
982
	nt->mw_vec = kzalloc_node(mw_count * sizeof(*nt->mw_vec),
				  GFP_KERNEL, node);
983
984
	if (!nt->mw_vec) {
		rc = -ENOMEM;
985
986
987
		goto err;
	}

988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
	for (i = 0; i < mw_count; i++) {
		mw = &nt->mw_vec[i];

		rc = ntb_mw_get_range(ndev, i, &mw->phys_addr, &mw->phys_size,
				      &mw->xlat_align, &mw->xlat_align_size);
		if (rc)
			goto err1;

		mw->vbase = ioremap(mw->phys_addr, mw->phys_size);
		if (!mw->vbase) {
			rc = -ENOMEM;
			goto err1;
		}

		mw->buff_size = 0;
		mw->xlat_size = 0