ntb_transport.c 52.6 KB
Newer Older
1 2 3 4 5 6 7
/*
 * This file is provided under a dual BSD/GPLv2 license.  When using or
 *   redistributing this file, you may do so under either license.
 *
 *   GPL LICENSE SUMMARY
 *
 *   Copyright(c) 2012 Intel Corporation. All rights reserved.
8
 *   Copyright (C) 2015 EMC Corporation. All Rights Reserved.
9 10 11 12 13 14 15 16
 *
 *   This program is free software; you can redistribute it and/or modify
 *   it under the terms of version 2 of the GNU General Public License as
 *   published by the Free Software Foundation.
 *
 *   BSD LICENSE
 *
 *   Copyright(c) 2012 Intel Corporation. All rights reserved.
17
 *   Copyright (C) 2015 EMC Corporation. All Rights Reserved.
18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44
 *
 *   Redistribution and use in source and binary forms, with or without
 *   modification, are permitted provided that the following conditions
 *   are met:
 *
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copy
 *       notice, this list of conditions and the following disclaimer in
 *       the documentation and/or other materials provided with the
 *       distribution.
 *     * Neither the name of Intel Corporation nor the names of its
 *       contributors may be used to endorse or promote products derived
 *       from this software without specific prior written permission.
 *
 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
45
 * PCIe NTB Transport Linux driver
46 47 48 49 50 51
 *
 * Contact Information:
 * Jon Mason <jon.mason@intel.com>
 */
#include <linux/debugfs.h>
#include <linux/delay.h>
52
#include <linux/dmaengine.h>
53 54 55 56 57 58 59 60
#include <linux/dma-mapping.h>
#include <linux/errno.h>
#include <linux/export.h>
#include <linux/interrupt.h>
#include <linux/module.h>
#include <linux/pci.h>
#include <linux/slab.h>
#include <linux/types.h>
61
#include <linux/uaccess.h>
62 63
#include "linux/ntb.h"
#include "linux/ntb_transport.h"
64

65 66 67 68 69 70 71 72 73 74 75 76 77
#define NTB_TRANSPORT_VERSION	4
#define NTB_TRANSPORT_VER	"4"
#define NTB_TRANSPORT_NAME	"ntb_transport"
#define NTB_TRANSPORT_DESC	"Software Queue-Pair Transport over NTB"

MODULE_DESCRIPTION(NTB_TRANSPORT_DESC);
MODULE_VERSION(NTB_TRANSPORT_VER);
MODULE_LICENSE("Dual BSD/GPL");
MODULE_AUTHOR("Intel Corporation");

static unsigned long max_mw_size;
module_param(max_mw_size, ulong, 0644);
MODULE_PARM_DESC(max_mw_size, "Limit size of large memory windows");
78

79
static unsigned int transport_mtu = 0x10000;
80 81 82
module_param(transport_mtu, uint, 0644);
MODULE_PARM_DESC(transport_mtu, "Maximum size of NTB transport packets");

Jon Mason's avatar
Jon Mason committed
83
static unsigned char max_num_clients;
84 85 86
module_param(max_num_clients, byte, 0644);
MODULE_PARM_DESC(max_num_clients, "Maximum number of NTB transport clients");

87 88 89 90
static unsigned int copy_bytes = 1024;
module_param(copy_bytes, uint, 0644);
MODULE_PARM_DESC(copy_bytes, "Threshold under which NTB will use the CPU to copy instead of DMA");

91 92 93 94
static bool use_dma;
module_param(use_dma, bool, 0644);
MODULE_PARM_DESC(use_dma, "Use DMA engine to perform large data copy");

95 96
static struct dentry *nt_debugfs_dir;

97 98 99
struct ntb_queue_entry {
	/* ntb_queue list reference */
	struct list_head entry;
100
	/* pointers to data to be transferred */
101 102 103 104
	void *cb_data;
	void *buf;
	unsigned int len;
	unsigned int flags;
105 106 107 108 109 110 111

	struct ntb_transport_qp *qp;
	union {
		struct ntb_payload_header __iomem *tx_hdr;
		struct ntb_payload_header *rx_hdr;
	};
	unsigned int index;
112 113
};

Jon Mason's avatar
Jon Mason committed
114 115 116 117
struct ntb_rx_info {
	unsigned int entry;
};

118
struct ntb_transport_qp {
119 120
	struct ntb_transport_ctx *transport;
	struct ntb_dev *ndev;
121
	void *cb_data;
122 123
	struct dma_chan *tx_dma_chan;
	struct dma_chan *rx_dma_chan;
124 125

	bool client_ready;
126 127
	bool link_is_up;

128
	u8 qp_num;	/* Only 64 QP's are allowed.  0-63 */
129
	u64 qp_bit;
130

Jon Mason's avatar
Jon Mason committed
131
	struct ntb_rx_info __iomem *rx_info;
Jon Mason's avatar
Jon Mason committed
132 133
	struct ntb_rx_info *remote_rx_info;

Jon Mason's avatar
Jon Mason committed
134 135
	void (*tx_handler)(struct ntb_transport_qp *qp, void *qp_data,
			   void *data, int len);
136 137
	struct list_head tx_free_q;
	spinlock_t ntb_tx_free_q_lock;
Jon Mason's avatar
Jon Mason committed
138
	void __iomem *tx_mw;
139
	dma_addr_t tx_mw_phys;
Jon Mason's avatar
Jon Mason committed
140 141
	unsigned int tx_index;
	unsigned int tx_max_entry;
142
	unsigned int tx_max_frame;
143

Jon Mason's avatar
Jon Mason committed
144 145
	void (*rx_handler)(struct ntb_transport_qp *qp, void *qp_data,
			   void *data, int len);
146
	struct list_head rx_post_q;
147 148
	struct list_head rx_pend_q;
	struct list_head rx_free_q;
149 150
	/* ntb_rx_q_lock: synchronize access to rx_XXXX_q */
	spinlock_t ntb_rx_q_lock;
Jon Mason's avatar
Jon Mason committed
151 152 153
	void *rx_buff;
	unsigned int rx_index;
	unsigned int rx_max_entry;
154
	unsigned int rx_max_frame;
155
	dma_cookie_t last_cookie;
156
	struct tasklet_struct rxc_db_work;
157

Jon Mason's avatar
Jon Mason committed
158
	void (*event_handler)(void *data, int status);
159
	struct delayed_work link_work;
160
	struct work_struct link_cleanup;
161 162 163 164 165 166 167 168 169 170 171

	struct dentry *debugfs_dir;
	struct dentry *debugfs_stats;

	/* Stats */
	u64 rx_bytes;
	u64 rx_pkts;
	u64 rx_ring_empty;
	u64 rx_err_no_buf;
	u64 rx_err_oflow;
	u64 rx_err_ver;
172 173
	u64 rx_memcpy;
	u64 rx_async;
174
	u64 dma_rx_prep_err;
175 176 177
	u64 tx_bytes;
	u64 tx_pkts;
	u64 tx_ring_full;
178 179 180
	u64 tx_err_no_buf;
	u64 tx_memcpy;
	u64 tx_async;
181
	u64 dma_tx_prep_err;
182 183 184
};

struct ntb_transport_mw {
185 186 187 188 189 190 191
	phys_addr_t phys_addr;
	resource_size_t phys_size;
	resource_size_t xlat_align;
	resource_size_t xlat_align_size;
	void __iomem *vbase;
	size_t xlat_size;
	size_t buff_size;
192 193 194 195 196 197
	void *virt_addr;
	dma_addr_t dma_addr;
};

struct ntb_transport_client_dev {
	struct list_head entry;
198
	struct ntb_transport_ctx *nt;
199 200 201
	struct device dev;
};

202
struct ntb_transport_ctx {
203 204 205
	struct list_head entry;
	struct list_head client_devs;

206 207 208 209 210 211 212 213 214 215
	struct ntb_dev *ndev;

	struct ntb_transport_mw *mw_vec;
	struct ntb_transport_qp *qp_vec;
	unsigned int mw_count;
	unsigned int qp_count;
	u64 qp_bitmap;
	u64 qp_bitmap_free;

	bool link_is_up;
216
	struct delayed_work link_work;
217
	struct work_struct link_cleanup;
218 219

	struct dentry *debugfs_node_dir;
220 221 222
};

enum {
223 224
	DESC_DONE_FLAG = BIT(0),
	LINK_DOWN_FLAG = BIT(1),
225 226 227
};

struct ntb_payload_header {
Jon Mason's avatar
Jon Mason committed
228
	unsigned int ver;
229 230 231 232 233 234 235
	unsigned int len;
	unsigned int flags;
};

enum {
	VERSION = 0,
	QP_LINKS,
Jon Mason's avatar
Jon Mason committed
236 237 238 239 240 241
	NUM_QPS,
	NUM_MWS,
	MW0_SZ_HIGH,
	MW0_SZ_LOW,
	MW1_SZ_HIGH,
	MW1_SZ_LOW,
242 243 244
	MAX_SPAD,
};

245 246 247 248 249 250 251
#define dev_client_dev(__dev) \
	container_of((__dev), struct ntb_transport_client_dev, dev)

#define drv_client(__drv) \
	container_of((__drv), struct ntb_transport_client, driver)

#define QP_TO_MW(nt, qp)	((qp) % nt->mw_count)
252 253
#define NTB_QP_DEF_NUM_ENTRIES	100
#define NTB_LINK_DOWN_TIMEOUT	10
254 255
#define DMA_RETRIES		20
#define DMA_OUT_RESOURCE_TO	50
256

257 258 259 260 261 262
static void ntb_transport_rxc_db(unsigned long data);
static const struct ntb_ctx_ops ntb_transport_ops;
static struct ntb_client ntb_transport_client;

static int ntb_transport_bus_match(struct device *dev,
				   struct device_driver *drv)
263 264 265 266
{
	return !strncmp(dev_name(dev), drv->name, strlen(drv->name));
}

267
static int ntb_transport_bus_probe(struct device *dev)
268
{
269
	const struct ntb_transport_client *client;
270 271 272
	int rc = -EINVAL;

	get_device(dev);
273 274 275

	client = drv_client(dev->driver);
	rc = client->probe(dev);
276 277 278 279 280 281
	if (rc)
		put_device(dev);

	return rc;
}

282
static int ntb_transport_bus_remove(struct device *dev)
283
{
284
	const struct ntb_transport_client *client;
285

286 287
	client = drv_client(dev->driver);
	client->remove(dev);
288 289 290 291 292 293

	put_device(dev);

	return 0;
}

294 295 296 297 298
static struct bus_type ntb_transport_bus = {
	.name = "ntb_transport",
	.match = ntb_transport_bus_match,
	.probe = ntb_transport_bus_probe,
	.remove = ntb_transport_bus_remove,
299 300 301 302
};

static LIST_HEAD(ntb_transport_list);

303
static int ntb_bus_init(struct ntb_transport_ctx *nt)
304
{
305
	list_add_tail(&nt->entry, &ntb_transport_list);
306 307 308
	return 0;
}

309
static void ntb_bus_remove(struct ntb_transport_ctx *nt)
310 311 312 313 314 315 316 317 318 319 320 321 322
{
	struct ntb_transport_client_dev *client_dev, *cd;

	list_for_each_entry_safe(client_dev, cd, &nt->client_devs, entry) {
		dev_err(client_dev->dev.parent, "%s still attached to bus, removing\n",
			dev_name(&client_dev->dev));
		list_del(&client_dev->entry);
		device_unregister(&client_dev->dev);
	}

	list_del(&nt->entry);
}

323
static void ntb_transport_client_release(struct device *dev)
324 325 326
{
	struct ntb_transport_client_dev *client_dev;

327
	client_dev = dev_client_dev(dev);
328 329 330 331
	kfree(client_dev);
}

/**
332
 * ntb_transport_unregister_client_dev - Unregister NTB client device
333 334 335 336
 * @device_name: Name of NTB client device
 *
 * Unregister an NTB client device with the NTB transport layer
 */
337
void ntb_transport_unregister_client_dev(char *device_name)
338 339
{
	struct ntb_transport_client_dev *client, *cd;
340
	struct ntb_transport_ctx *nt;
341 342 343 344 345 346 347 348 349

	list_for_each_entry(nt, &ntb_transport_list, entry)
		list_for_each_entry_safe(client, cd, &nt->client_devs, entry)
			if (!strncmp(dev_name(&client->dev), device_name,
				     strlen(device_name))) {
				list_del(&client->entry);
				device_unregister(&client->dev);
			}
}
350
EXPORT_SYMBOL_GPL(ntb_transport_unregister_client_dev);
351 352

/**
353
 * ntb_transport_register_client_dev - Register NTB client device
354 355 356 357
 * @device_name: Name of NTB client device
 *
 * Register an NTB client device with the NTB transport layer
 */
358
int ntb_transport_register_client_dev(char *device_name)
359 360
{
	struct ntb_transport_client_dev *client_dev;
361
	struct ntb_transport_ctx *nt;
362
	int node;
Jon Mason's avatar
Jon Mason committed
363
	int rc, i = 0;
364

365 366 367
	if (list_empty(&ntb_transport_list))
		return -ENODEV;

368 369 370
	list_for_each_entry(nt, &ntb_transport_list, entry) {
		struct device *dev;

371 372 373 374
		node = dev_to_node(&nt->ndev->dev);

		client_dev = kzalloc_node(sizeof(*client_dev),
					  GFP_KERNEL, node);
375 376 377 378 379 380 381 382
		if (!client_dev) {
			rc = -ENOMEM;
			goto err;
		}

		dev = &client_dev->dev;

		/* setup and register client devices */
Jon Mason's avatar
Jon Mason committed
383
		dev_set_name(dev, "%s%d", device_name, i);
384 385 386
		dev->bus = &ntb_transport_bus;
		dev->release = ntb_transport_client_release;
		dev->parent = &nt->ndev->dev;
387 388 389 390 391 392 393 394

		rc = device_register(dev);
		if (rc) {
			kfree(client_dev);
			goto err;
		}

		list_add_tail(&client_dev->entry, &nt->client_devs);
Jon Mason's avatar
Jon Mason committed
395
		i++;
396 397 398 399 400
	}

	return 0;

err:
401
	ntb_transport_unregister_client_dev(device_name);
402 403 404

	return rc;
}
405
EXPORT_SYMBOL_GPL(ntb_transport_register_client_dev);
406 407

/**
408
 * ntb_transport_register_client - Register NTB client driver
409 410 411 412 413 414
 * @drv: NTB client driver to be registered
 *
 * Register an NTB client driver with the NTB transport layer
 *
 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
 */
415
int ntb_transport_register_client(struct ntb_transport_client *drv)
416
{
417
	drv->driver.bus = &ntb_transport_bus;
418

419 420 421
	if (list_empty(&ntb_transport_list))
		return -ENODEV;

422 423
	return driver_register(&drv->driver);
}
424
EXPORT_SYMBOL_GPL(ntb_transport_register_client);
425 426

/**
427
 * ntb_transport_unregister_client - Unregister NTB client driver
428 429 430 431 432 433
 * @drv: NTB client driver to be unregistered
 *
 * Unregister an NTB client driver with the NTB transport layer
 *
 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
 */
434
void ntb_transport_unregister_client(struct ntb_transport_client *drv)
435 436 437
{
	driver_unregister(&drv->driver);
}
438
EXPORT_SYMBOL_GPL(ntb_transport_unregister_client);
439 440 441 442 443

static ssize_t debugfs_read(struct file *filp, char __user *ubuf, size_t count,
			    loff_t *offp)
{
	struct ntb_transport_qp *qp;
444
	char *buf;
445 446
	ssize_t ret, out_offset, out_count;

447 448 449 450 451
	qp = filp->private_data;

	if (!qp || !qp->link_is_up)
		return 0;

452
	out_count = 1000;
453 454 455 456

	buf = kmalloc(out_count, GFP_KERNEL);
	if (!buf)
		return -ENOMEM;
457 458 459

	out_offset = 0;
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
Dave Jiang's avatar
Dave Jiang committed
460
			       "\nNTB QP stats:\n\n");
461 462 463 464
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_bytes - \t%llu\n", qp->rx_bytes);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_pkts - \t%llu\n", qp->rx_pkts);
465 466 467 468
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_memcpy - \t%llu\n", qp->rx_memcpy);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_async - \t%llu\n", qp->rx_async);
469 470 471 472 473 474 475 476 477
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_ring_empty - %llu\n", qp->rx_ring_empty);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_err_no_buf - %llu\n", qp->rx_err_no_buf);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_err_oflow - \t%llu\n", qp->rx_err_oflow);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_err_ver - \t%llu\n", qp->rx_err_ver);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
Dave Jiang's avatar
Dave Jiang committed
478
			       "rx_buff - \t0x%p\n", qp->rx_buff);
479
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
Jon Mason's avatar
Jon Mason committed
480
			       "rx_index - \t%u\n", qp->rx_index);
481
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
Dave Jiang's avatar
Dave Jiang committed
482
			       "rx_max_entry - \t%u\n\n", qp->rx_max_entry);
483 484 485 486 487

	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "tx_bytes - \t%llu\n", qp->tx_bytes);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "tx_pkts - \t%llu\n", qp->tx_pkts);
488 489 490 491
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "tx_memcpy - \t%llu\n", qp->tx_memcpy);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "tx_async - \t%llu\n", qp->tx_async);
492 493
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "tx_ring_full - \t%llu\n", qp->tx_ring_full);
494 495
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "tx_err_no_buf - %llu\n", qp->tx_err_no_buf);
496
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
Dave Jiang's avatar
Dave Jiang committed
497
			       "tx_mw - \t0x%p\n", qp->tx_mw);
498
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
Dave Jiang's avatar
Dave Jiang committed
499
			       "tx_index (H) - \t%u\n", qp->tx_index);
500
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
Dave Jiang's avatar
Dave Jiang committed
501
			       "RRI (T) - \t%u\n",
502
			       qp->remote_rx_info->entry);
Dave Jiang's avatar
Dave Jiang committed
503 504
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "tx_max_entry - \t%u\n", qp->tx_max_entry);
505 506 507
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "free tx - \t%u\n",
			       ntb_transport_tx_free_entry(qp));
508 509 510 511 512 513
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "DMA tx prep err - \t%llu\n",
			       qp->dma_tx_prep_err);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "DMA rx prep err - \t%llu\n",
			       qp->dma_rx_prep_err);
514 515

	out_offset += snprintf(buf + out_offset, out_count - out_offset,
Dave Jiang's avatar
Dave Jiang committed
516 517
			       "\n");
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
518 519 520 521 522
			       "Using TX DMA - \t%s\n",
			       qp->tx_dma_chan ? "Yes" : "No");
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "Using RX DMA - \t%s\n",
			       qp->rx_dma_chan ? "Yes" : "No");
Dave Jiang's avatar
Dave Jiang committed
523 524
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "QP Link - \t%s\n",
525
			       qp->link_is_up ? "Up" : "Down");
Dave Jiang's avatar
Dave Jiang committed
526 527 528
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "\n");

529 530
	if (out_offset > out_count)
		out_offset = out_count;
531 532

	ret = simple_read_from_buffer(ubuf, count, offp, buf, out_offset);
533
	kfree(buf);
534 535 536 537 538
	return ret;
}

static const struct file_operations ntb_qp_debugfs_stats = {
	.owner = THIS_MODULE,
Jon Mason's avatar
Jon Mason committed
539
	.open = simple_open,
540 541 542 543 544 545 546 547 548 549 550 551 552 553
	.read = debugfs_read,
};

static void ntb_list_add(spinlock_t *lock, struct list_head *entry,
			 struct list_head *list)
{
	unsigned long flags;

	spin_lock_irqsave(lock, flags);
	list_add_tail(entry, list);
	spin_unlock_irqrestore(lock, flags);
}

static struct ntb_queue_entry *ntb_list_rm(spinlock_t *lock,
Jon Mason's avatar
Jon Mason committed
554
					   struct list_head *list)
555 556 557 558 559 560 561 562 563 564 565
{
	struct ntb_queue_entry *entry;
	unsigned long flags;

	spin_lock_irqsave(lock, flags);
	if (list_empty(list)) {
		entry = NULL;
		goto out;
	}
	entry = list_first_entry(list, struct ntb_queue_entry, entry);
	list_del(&entry->entry);
566

567 568 569 570 571 572
out:
	spin_unlock_irqrestore(lock, flags);

	return entry;
}

573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593
static struct ntb_queue_entry *ntb_list_mv(spinlock_t *lock,
					   struct list_head *list,
					   struct list_head *to_list)
{
	struct ntb_queue_entry *entry;
	unsigned long flags;

	spin_lock_irqsave(lock, flags);

	if (list_empty(list)) {
		entry = NULL;
	} else {
		entry = list_first_entry(list, struct ntb_queue_entry, entry);
		list_move_tail(&entry->entry, to_list);
	}

	spin_unlock_irqrestore(lock, flags);

	return entry;
}

594 595
static int ntb_transport_setup_qp_mw(struct ntb_transport_ctx *nt,
				     unsigned int qp_num)
596
{
597 598
	struct ntb_transport_qp *qp = &nt->qp_vec[qp_num];
	struct ntb_transport_mw *mw;
599
	unsigned int rx_size, num_qps_mw;
600
	unsigned int mw_num, mw_count, qp_count;
Jon Mason's avatar
Jon Mason committed
601
	unsigned int i;
602

603 604
	mw_count = nt->mw_count;
	qp_count = nt->qp_count;
Jon Mason's avatar
Jon Mason committed
605

606 607 608 609 610
	mw_num = QP_TO_MW(nt, qp_num);
	mw = &nt->mw_vec[mw_num];

	if (!mw->virt_addr)
		return -ENOMEM;
611

612 613
	if (qp_count % mw_count && mw_num + 1 < qp_count / mw_count)
		num_qps_mw = qp_count / mw_count + 1;
614
	else
615
		num_qps_mw = qp_count / mw_count;
616

617
	rx_size = (unsigned int)mw->xlat_size / num_qps_mw;
618
	qp->rx_buff = mw->virt_addr + rx_size * (qp_num / mw_count);
Jon Mason's avatar
Jon Mason committed
619 620
	rx_size -= sizeof(struct ntb_rx_info);

621 622
	qp->remote_rx_info = qp->rx_buff + rx_size;

623 624
	/* Due to housekeeping, there must be atleast 2 buffs */
	qp->rx_max_frame = min(transport_mtu, rx_size / 2);
Jon Mason's avatar
Jon Mason committed
625 626 627
	qp->rx_max_entry = rx_size / qp->rx_max_frame;
	qp->rx_index = 0;

628
	qp->remote_rx_info->entry = qp->rx_max_entry - 1;
629

630
	/* setup the hdr offsets with 0's */
Jon Mason's avatar
Jon Mason committed
631
	for (i = 0; i < qp->rx_max_entry; i++) {
632 633
		void *offset = (qp->rx_buff + qp->rx_max_frame * (i + 1) -
				sizeof(struct ntb_payload_header));
634
		memset(offset, 0, sizeof(struct ntb_payload_header));
Jon Mason's avatar
Jon Mason committed
635
	}
636 637 638

	qp->rx_pkts = 0;
	qp->tx_pkts = 0;
Jon Mason's avatar
Jon Mason committed
639
	qp->tx_index = 0;
640 641

	return 0;
642 643
}

644
static void ntb_free_mw(struct ntb_transport_ctx *nt, int num_mw)
Jon Mason's avatar
Jon Mason committed
645
{
646 647
	struct ntb_transport_mw *mw = &nt->mw_vec[num_mw];
	struct pci_dev *pdev = nt->ndev->pdev;
Jon Mason's avatar
Jon Mason committed
648 649 650 651

	if (!mw->virt_addr)
		return;

652 653 654 655 656
	ntb_mw_clear_trans(nt->ndev, num_mw);
	dma_free_coherent(&pdev->dev, mw->buff_size,
			  mw->virt_addr, mw->dma_addr);
	mw->xlat_size = 0;
	mw->buff_size = 0;
Jon Mason's avatar
Jon Mason committed
657 658 659
	mw->virt_addr = NULL;
}

660
static int ntb_set_mw(struct ntb_transport_ctx *nt, int num_mw,
661
		      resource_size_t size)
662
{
663 664
	struct ntb_transport_mw *mw = &nt->mw_vec[num_mw];
	struct pci_dev *pdev = nt->ndev->pdev;
665
	size_t xlat_size, buff_size;
666 667
	int rc;

668 669 670
	if (!size)
		return -EINVAL;

671 672
	xlat_size = round_up(size, mw->xlat_align_size);
	buff_size = round_up(size, mw->xlat_align);
673

Jon Mason's avatar
Jon Mason committed
674
	/* No need to re-setup */
675
	if (mw->xlat_size == xlat_size)
Jon Mason's avatar
Jon Mason committed
676 677
		return 0;

678
	if (mw->buff_size)
Jon Mason's avatar
Jon Mason committed
679 680
		ntb_free_mw(nt, num_mw);

681 682 683
	/* Alloc memory for receiving data.  Must be aligned */
	mw->xlat_size = xlat_size;
	mw->buff_size = buff_size;
684

685 686
	mw->virt_addr = dma_alloc_coherent(&pdev->dev, buff_size,
					   &mw->dma_addr, GFP_KERNEL);
687
	if (!mw->virt_addr) {
688 689
		mw->xlat_size = 0;
		mw->buff_size = 0;
690
		dev_err(&pdev->dev, "Unable to alloc MW buff of size %zu\n",
691
			buff_size);
692 693 694
		return -ENOMEM;
	}

695 696 697 698 699 700
	/*
	 * we must ensure that the memory address allocated is BAR size
	 * aligned in order for the XLAT register to take the value. This
	 * is a requirement of the hardware. It is recommended to setup CMA
	 * for BAR sizes equal or greater than 4MB.
	 */
701 702
	if (!IS_ALIGNED(mw->dma_addr, mw->xlat_align)) {
		dev_err(&pdev->dev, "DMA memory %pad is not aligned\n",
703 704 705 706 707
			&mw->dma_addr);
		ntb_free_mw(nt, num_mw);
		return -ENOMEM;
	}

708
	/* Notify HW the memory location of the receive buffer */
709 710 711 712 713 714
	rc = ntb_mw_set_trans(nt->ndev, num_mw, mw->dma_addr, mw->xlat_size);
	if (rc) {
		dev_err(&pdev->dev, "Unable to set mw%d translation", num_mw);
		ntb_free_mw(nt, num_mw);
		return -EIO;
	}
715 716 717 718

	return 0;
}

719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738
static void ntb_qp_link_down_reset(struct ntb_transport_qp *qp)
{
	qp->link_is_up = false;

	qp->tx_index = 0;
	qp->rx_index = 0;
	qp->rx_bytes = 0;
	qp->rx_pkts = 0;
	qp->rx_ring_empty = 0;
	qp->rx_err_no_buf = 0;
	qp->rx_err_oflow = 0;
	qp->rx_err_ver = 0;
	qp->rx_memcpy = 0;
	qp->rx_async = 0;
	qp->tx_bytes = 0;
	qp->tx_pkts = 0;
	qp->tx_ring_full = 0;
	qp->tx_err_no_buf = 0;
	qp->tx_memcpy = 0;
	qp->tx_async = 0;
739 740
	qp->dma_tx_prep_err = 0;
	qp->dma_rx_prep_err = 0;
741 742
}

743
static void ntb_qp_link_cleanup(struct ntb_transport_qp *qp)
744
{
745 746
	struct ntb_transport_ctx *nt = qp->transport;
	struct pci_dev *pdev = nt->ndev->pdev;
747

748
	dev_info(&pdev->dev, "qp %d: Link Cleanup\n", qp->qp_num);
749 750 751

	cancel_delayed_work_sync(&qp->link_work);
	ntb_qp_link_down_reset(qp);
752 753 754

	if (qp->event_handler)
		qp->event_handler(qp->cb_data, qp->link_is_up);
755 756 757 758 759 760 761
}

static void ntb_qp_link_cleanup_work(struct work_struct *work)
{
	struct ntb_transport_qp *qp = container_of(work,
						   struct ntb_transport_qp,
						   link_cleanup);
762
	struct ntb_transport_ctx *nt = qp->transport;
763 764

	ntb_qp_link_cleanup(qp);
765

766
	if (nt->link_is_up)
767 768 769 770
		schedule_delayed_work(&qp->link_work,
				      msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT));
}

771 772 773 774 775
static void ntb_qp_link_down(struct ntb_transport_qp *qp)
{
	schedule_work(&qp->link_cleanup);
}

776
static void ntb_transport_link_cleanup(struct ntb_transport_ctx *nt)
777
{
778 779
	struct ntb_transport_qp *qp;
	u64 qp_bitmap_alloc;
780 781
	int i;

782 783
	qp_bitmap_alloc = nt->qp_bitmap & ~nt->qp_bitmap_free;

784
	/* Pass along the info to any clients */
785 786 787 788 789 790 791
	for (i = 0; i < nt->qp_count; i++)
		if (qp_bitmap_alloc & BIT_ULL(i)) {
			qp = &nt->qp_vec[i];
			ntb_qp_link_cleanup(qp);
			cancel_work_sync(&qp->link_cleanup);
			cancel_delayed_work_sync(&qp->link_work);
		}
792

793
	if (!nt->link_is_up)
794 795 796 797 798 799 800
		cancel_delayed_work_sync(&nt->link_work);

	/* The scratchpad registers keep the values if the remote side
	 * goes down, blast them now to give them a sane value the next
	 * time they are accessed
	 */
	for (i = 0; i < MAX_SPAD; i++)
801
		ntb_spad_write(nt->ndev, i, 0);
802 803
}

804 805
static void ntb_transport_link_cleanup_work(struct work_struct *work)
{
806 807
	struct ntb_transport_ctx *nt =
		container_of(work, struct ntb_transport_ctx, link_cleanup);
808 809 810 811

	ntb_transport_link_cleanup(nt);
}

812
static void ntb_transport_event_callback(void *data)
813
{
814
	struct ntb_transport_ctx *nt = data;
815

816
	if (ntb_link_is_up(nt->ndev, NULL, NULL) == 1)
817
		schedule_delayed_work(&nt->link_work, 0);
818
	else
819
		schedule_work(&nt->link_cleanup);
820 821 822 823
}

static void ntb_transport_link_work(struct work_struct *work)
{
824 825 826 827 828
	struct ntb_transport_ctx *nt =
		container_of(work, struct ntb_transport_ctx, link_work.work);
	struct ntb_dev *ndev = nt->ndev;
	struct pci_dev *pdev = ndev->pdev;
	resource_size_t size;
829
	u32 val;
830
	int rc, i, spad;
831

Jon Mason's avatar
Jon Mason committed
832
	/* send the local info, in the opposite order of the way we read it */
833 834
	for (i = 0; i < nt->mw_count; i++) {
		size = nt->mw_vec[i].phys_size;
835

836 837
		if (max_mw_size && size > max_mw_size)
			size = max_mw_size;
838

839
		spad = MW0_SZ_HIGH + (i * 2);
840
		ntb_peer_spad_write(ndev, spad, upper_32_bits(size));
841

842
		spad = MW0_SZ_LOW + (i * 2);
843
		ntb_peer_spad_write(ndev, spad, lower_32_bits(size));
844 845
	}

846
	ntb_peer_spad_write(ndev, NUM_MWS, nt->mw_count);
847

848
	ntb_peer_spad_write(ndev, NUM_QPS, nt->qp_count);
849

850
	ntb_peer_spad_write(ndev, VERSION, NTB_TRANSPORT_VERSION);
851

852
	/* Query the remote side for its info */
853
	val = ntb_spad_read(ndev, VERSION);
854 855
	dev_dbg(&pdev->dev, "Remote version = %d\n", val);
	if (val != NTB_TRANSPORT_VERSION)
856 857
		goto out;

858
	val = ntb_spad_read(ndev, NUM_QPS);
859
	dev_dbg(&pdev->dev, "Remote max number of qps = %d\n", val);
860
	if (val != nt->qp_count)
861 862
		goto out;

863
	val = ntb_spad_read(ndev, NUM_MWS);
Jon Mason's avatar
Jon Mason committed
864
	dev_dbg(&pdev->dev, "Remote number of mws = %d\n", val);
865 866
	if (val != nt->mw_count)
		goto out;
867

868
	for (i = 0; i < nt->mw_count; i++) {
Jon Mason's avatar
Jon Mason committed
869
		u64 val64;
870

871
		val = ntb_spad_read(ndev, MW0_SZ_HIGH + (i * 2));
872
		val64 = (u64)val << 32;
Jon Mason's avatar
Jon Mason committed
873

874
		val = ntb_spad_read(ndev, MW0_SZ_LOW + (i * 2));
Jon Mason's avatar
Jon Mason committed
875 876
		val64 |= val;

877
		dev_dbg(&pdev->dev, "Remote MW%d size = %#llx\n", i, val64);
Jon Mason's avatar
Jon Mason committed
878 879 880 881 882

		rc = ntb_set_mw(nt, i, val64);
		if (rc)
			goto out1;
	}
883

884
	nt->link_is_up = true;
885

886 887
	for (i = 0; i < nt->qp_count; i++) {
		struct ntb_transport_qp *qp = &nt->qp_vec[i];
888 889 890

		ntb_transport_setup_qp_mw(nt, i);

891
		if (qp->client_ready)
892 893 894 895 896
			schedule_delayed_work(&qp->link_work, 0);
	}

	return;

Jon Mason's avatar
Jon Mason committed
897
out1:
898
	for (i = 0; i < nt->mw_count; i++)
Jon Mason's avatar
Jon Mason committed
899
		ntb_free_mw(nt, i);
900
out:
901
	if (ntb_link_is_up(ndev, NULL, NULL) == 1)
902 903 904 905 906 907 908 909 910
		schedule_delayed_work(&nt->link_work,
				      msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT));
}

static void ntb_qp_link_work(struct work_struct *work)
{
	struct ntb_transport_qp *qp = container_of(work,
						   struct ntb_transport_qp,
						   link_work.work);
911 912 913
	struct pci_dev *pdev = qp->ndev->pdev;
	struct ntb_transport_ctx *nt = qp->transport;
	int val;
914

915
	WARN_ON(!nt->link_is_up);
916

917
	val = ntb_spad_read(nt->ndev, QP_LINKS);
918

919
	ntb_peer_spad_write(nt->ndev, QP_LINKS, val | BIT(qp->qp_num));