ntb_transport.c 52.9 KB
Newer Older
1 2 3 4 5 6 7
/*
 * This file is provided under a dual BSD/GPLv2 license.  When using or
 *   redistributing this file, you may do so under either license.
 *
 *   GPL LICENSE SUMMARY
 *
 *   Copyright(c) 2012 Intel Corporation. All rights reserved.
8
 *   Copyright (C) 2015 EMC Corporation. All Rights Reserved.
9 10 11 12 13 14 15 16
 *
 *   This program is free software; you can redistribute it and/or modify
 *   it under the terms of version 2 of the GNU General Public License as
 *   published by the Free Software Foundation.
 *
 *   BSD LICENSE
 *
 *   Copyright(c) 2012 Intel Corporation. All rights reserved.
17
 *   Copyright (C) 2015 EMC Corporation. All Rights Reserved.
18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44
 *
 *   Redistribution and use in source and binary forms, with or without
 *   modification, are permitted provided that the following conditions
 *   are met:
 *
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copy
 *       notice, this list of conditions and the following disclaimer in
 *       the documentation and/or other materials provided with the
 *       distribution.
 *     * Neither the name of Intel Corporation nor the names of its
 *       contributors may be used to endorse or promote products derived
 *       from this software without specific prior written permission.
 *
 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
45
 * PCIe NTB Transport Linux driver
46 47 48 49 50 51
 *
 * Contact Information:
 * Jon Mason <jon.mason@intel.com>
 */
#include <linux/debugfs.h>
#include <linux/delay.h>
52
#include <linux/dmaengine.h>
53 54 55 56 57 58 59 60
#include <linux/dma-mapping.h>
#include <linux/errno.h>
#include <linux/export.h>
#include <linux/interrupt.h>
#include <linux/module.h>
#include <linux/pci.h>
#include <linux/slab.h>
#include <linux/types.h>
61
#include <linux/uaccess.h>
62 63
#include "linux/ntb.h"
#include "linux/ntb_transport.h"
64

65 66 67 68 69 70 71 72 73 74 75 76 77
#define NTB_TRANSPORT_VERSION	4
#define NTB_TRANSPORT_VER	"4"
#define NTB_TRANSPORT_NAME	"ntb_transport"
#define NTB_TRANSPORT_DESC	"Software Queue-Pair Transport over NTB"

MODULE_DESCRIPTION(NTB_TRANSPORT_DESC);
MODULE_VERSION(NTB_TRANSPORT_VER);
MODULE_LICENSE("Dual BSD/GPL");
MODULE_AUTHOR("Intel Corporation");

static unsigned long max_mw_size;
module_param(max_mw_size, ulong, 0644);
MODULE_PARM_DESC(max_mw_size, "Limit size of large memory windows");
78

79
static unsigned int transport_mtu = 0x10000;
80 81 82
module_param(transport_mtu, uint, 0644);
MODULE_PARM_DESC(transport_mtu, "Maximum size of NTB transport packets");

Jon Mason's avatar
Jon Mason committed
83
static unsigned char max_num_clients;
84 85 86
module_param(max_num_clients, byte, 0644);
MODULE_PARM_DESC(max_num_clients, "Maximum number of NTB transport clients");

87 88 89 90
static unsigned int copy_bytes = 1024;
module_param(copy_bytes, uint, 0644);
MODULE_PARM_DESC(copy_bytes, "Threshold under which NTB will use the CPU to copy instead of DMA");

91 92 93 94
static bool use_dma;
module_param(use_dma, bool, 0644);
MODULE_PARM_DESC(use_dma, "Use DMA engine to perform large data copy");

95 96
static struct dentry *nt_debugfs_dir;

97 98 99
struct ntb_queue_entry {
	/* ntb_queue list reference */
	struct list_head entry;
100
	/* pointers to data to be transferred */
101 102 103 104
	void *cb_data;
	void *buf;
	unsigned int len;
	unsigned int flags;
105 106 107 108 109 110 111

	struct ntb_transport_qp *qp;
	union {
		struct ntb_payload_header __iomem *tx_hdr;
		struct ntb_payload_header *rx_hdr;
	};
	unsigned int index;
112 113
};

Jon Mason's avatar
Jon Mason committed
114 115 116 117
struct ntb_rx_info {
	unsigned int entry;
};

118
struct ntb_transport_qp {
119 120
	struct ntb_transport_ctx *transport;
	struct ntb_dev *ndev;
121
	void *cb_data;
122 123
	struct dma_chan *tx_dma_chan;
	struct dma_chan *rx_dma_chan;
124 125

	bool client_ready;
126
	bool link_is_up;
127
	bool active;
128

129
	u8 qp_num;	/* Only 64 QP's are allowed.  0-63 */
130
	u64 qp_bit;
131

Jon Mason's avatar
Jon Mason committed
132
	struct ntb_rx_info __iomem *rx_info;
Jon Mason's avatar
Jon Mason committed
133 134
	struct ntb_rx_info *remote_rx_info;

Jon Mason's avatar
Jon Mason committed
135 136
	void (*tx_handler)(struct ntb_transport_qp *qp, void *qp_data,
			   void *data, int len);
137 138
	struct list_head tx_free_q;
	spinlock_t ntb_tx_free_q_lock;
Jon Mason's avatar
Jon Mason committed
139
	void __iomem *tx_mw;
140
	dma_addr_t tx_mw_phys;
Jon Mason's avatar
Jon Mason committed
141 142
	unsigned int tx_index;
	unsigned int tx_max_entry;
143
	unsigned int tx_max_frame;
144

Jon Mason's avatar
Jon Mason committed
145 146
	void (*rx_handler)(struct ntb_transport_qp *qp, void *qp_data,
			   void *data, int len);
147
	struct list_head rx_post_q;
148 149
	struct list_head rx_pend_q;
	struct list_head rx_free_q;
150 151
	/* ntb_rx_q_lock: synchronize access to rx_XXXX_q */
	spinlock_t ntb_rx_q_lock;
Jon Mason's avatar
Jon Mason committed
152 153 154
	void *rx_buff;
	unsigned int rx_index;
	unsigned int rx_max_entry;
155
	unsigned int rx_max_frame;
156
	dma_cookie_t last_cookie;
157
	struct tasklet_struct rxc_db_work;
158

Jon Mason's avatar
Jon Mason committed
159
	void (*event_handler)(void *data, int status);
160
	struct delayed_work link_work;
161
	struct work_struct link_cleanup;
162 163 164 165 166 167 168 169 170 171 172

	struct dentry *debugfs_dir;
	struct dentry *debugfs_stats;

	/* Stats */
	u64 rx_bytes;
	u64 rx_pkts;
	u64 rx_ring_empty;
	u64 rx_err_no_buf;
	u64 rx_err_oflow;
	u64 rx_err_ver;
173 174
	u64 rx_memcpy;
	u64 rx_async;
175
	u64 dma_rx_prep_err;
176 177 178
	u64 tx_bytes;
	u64 tx_pkts;
	u64 tx_ring_full;
179 180 181
	u64 tx_err_no_buf;
	u64 tx_memcpy;
	u64 tx_async;
182
	u64 dma_tx_prep_err;
183 184 185
};

struct ntb_transport_mw {
186 187 188 189 190 191 192
	phys_addr_t phys_addr;
	resource_size_t phys_size;
	resource_size_t xlat_align;
	resource_size_t xlat_align_size;
	void __iomem *vbase;
	size_t xlat_size;
	size_t buff_size;
193 194 195 196 197 198
	void *virt_addr;
	dma_addr_t dma_addr;
};

struct ntb_transport_client_dev {
	struct list_head entry;
199
	struct ntb_transport_ctx *nt;
200 201 202
	struct device dev;
};

203
struct ntb_transport_ctx {
204 205 206
	struct list_head entry;
	struct list_head client_devs;

207 208 209 210 211 212 213 214 215 216
	struct ntb_dev *ndev;

	struct ntb_transport_mw *mw_vec;
	struct ntb_transport_qp *qp_vec;
	unsigned int mw_count;
	unsigned int qp_count;
	u64 qp_bitmap;
	u64 qp_bitmap_free;

	bool link_is_up;
217
	struct delayed_work link_work;
218
	struct work_struct link_cleanup;
219 220

	struct dentry *debugfs_node_dir;
221 222 223
};

enum {
224 225
	DESC_DONE_FLAG = BIT(0),
	LINK_DOWN_FLAG = BIT(1),
226 227 228
};

struct ntb_payload_header {
Jon Mason's avatar
Jon Mason committed
229
	unsigned int ver;
230 231 232 233 234 235 236
	unsigned int len;
	unsigned int flags;
};

enum {
	VERSION = 0,
	QP_LINKS,
Jon Mason's avatar
Jon Mason committed
237 238 239 240 241 242
	NUM_QPS,
	NUM_MWS,
	MW0_SZ_HIGH,
	MW0_SZ_LOW,
	MW1_SZ_HIGH,
	MW1_SZ_LOW,
243 244 245
	MAX_SPAD,
};

246 247 248 249 250 251 252
#define dev_client_dev(__dev) \
	container_of((__dev), struct ntb_transport_client_dev, dev)

#define drv_client(__drv) \
	container_of((__drv), struct ntb_transport_client, driver)

#define QP_TO_MW(nt, qp)	((qp) % nt->mw_count)
253 254
#define NTB_QP_DEF_NUM_ENTRIES	100
#define NTB_LINK_DOWN_TIMEOUT	10
255 256
#define DMA_RETRIES		20
#define DMA_OUT_RESOURCE_TO	50
257

258 259 260 261 262 263
static void ntb_transport_rxc_db(unsigned long data);
static const struct ntb_ctx_ops ntb_transport_ops;
static struct ntb_client ntb_transport_client;

static int ntb_transport_bus_match(struct device *dev,
				   struct device_driver *drv)
264 265 266 267
{
	return !strncmp(dev_name(dev), drv->name, strlen(drv->name));
}

268
static int ntb_transport_bus_probe(struct device *dev)
269
{
270
	const struct ntb_transport_client *client;
271 272 273
	int rc = -EINVAL;

	get_device(dev);
274 275 276

	client = drv_client(dev->driver);
	rc = client->probe(dev);
277 278 279 280 281 282
	if (rc)
		put_device(dev);

	return rc;
}

283
static int ntb_transport_bus_remove(struct device *dev)
284
{
285
	const struct ntb_transport_client *client;
286

287 288
	client = drv_client(dev->driver);
	client->remove(dev);
289 290 291 292 293 294

	put_device(dev);

	return 0;
}

295 296 297 298 299
static struct bus_type ntb_transport_bus = {
	.name = "ntb_transport",
	.match = ntb_transport_bus_match,
	.probe = ntb_transport_bus_probe,
	.remove = ntb_transport_bus_remove,
300 301 302 303
};

static LIST_HEAD(ntb_transport_list);

304
static int ntb_bus_init(struct ntb_transport_ctx *nt)
305
{
306
	list_add_tail(&nt->entry, &ntb_transport_list);
307 308 309
	return 0;
}

310
static void ntb_bus_remove(struct ntb_transport_ctx *nt)
311 312 313 314 315 316 317 318 319 320 321 322 323
{
	struct ntb_transport_client_dev *client_dev, *cd;

	list_for_each_entry_safe(client_dev, cd, &nt->client_devs, entry) {
		dev_err(client_dev->dev.parent, "%s still attached to bus, removing\n",
			dev_name(&client_dev->dev));
		list_del(&client_dev->entry);
		device_unregister(&client_dev->dev);
	}

	list_del(&nt->entry);
}

324
static void ntb_transport_client_release(struct device *dev)
325 326 327
{
	struct ntb_transport_client_dev *client_dev;

328
	client_dev = dev_client_dev(dev);
329 330 331 332
	kfree(client_dev);
}

/**
333
 * ntb_transport_unregister_client_dev - Unregister NTB client device
334 335 336 337
 * @device_name: Name of NTB client device
 *
 * Unregister an NTB client device with the NTB transport layer
 */
338
void ntb_transport_unregister_client_dev(char *device_name)
339 340
{
	struct ntb_transport_client_dev *client, *cd;
341
	struct ntb_transport_ctx *nt;
342 343 344 345 346 347 348 349 350

	list_for_each_entry(nt, &ntb_transport_list, entry)
		list_for_each_entry_safe(client, cd, &nt->client_devs, entry)
			if (!strncmp(dev_name(&client->dev), device_name,
				     strlen(device_name))) {
				list_del(&client->entry);
				device_unregister(&client->dev);
			}
}
351
EXPORT_SYMBOL_GPL(ntb_transport_unregister_client_dev);
352 353

/**
354
 * ntb_transport_register_client_dev - Register NTB client device
355 356 357 358
 * @device_name: Name of NTB client device
 *
 * Register an NTB client device with the NTB transport layer
 */
359
int ntb_transport_register_client_dev(char *device_name)
360 361
{
	struct ntb_transport_client_dev *client_dev;
362
	struct ntb_transport_ctx *nt;
363
	int node;
Jon Mason's avatar
Jon Mason committed
364
	int rc, i = 0;
365

366 367 368
	if (list_empty(&ntb_transport_list))
		return -ENODEV;

369 370 371
	list_for_each_entry(nt, &ntb_transport_list, entry) {
		struct device *dev;

372 373 374 375
		node = dev_to_node(&nt->ndev->dev);

		client_dev = kzalloc_node(sizeof(*client_dev),
					  GFP_KERNEL, node);
376 377 378 379 380 381 382 383
		if (!client_dev) {
			rc = -ENOMEM;
			goto err;
		}

		dev = &client_dev->dev;

		/* setup and register client devices */
Jon Mason's avatar
Jon Mason committed
384
		dev_set_name(dev, "%s%d", device_name, i);
385 386 387
		dev->bus = &ntb_transport_bus;
		dev->release = ntb_transport_client_release;
		dev->parent = &nt->ndev->dev;
388 389 390 391 392 393 394 395

		rc = device_register(dev);
		if (rc) {
			kfree(client_dev);
			goto err;
		}

		list_add_tail(&client_dev->entry, &nt->client_devs);
Jon Mason's avatar
Jon Mason committed
396
		i++;
397 398 399 400 401
	}

	return 0;

err:
402
	ntb_transport_unregister_client_dev(device_name);
403 404 405

	return rc;
}
406
EXPORT_SYMBOL_GPL(ntb_transport_register_client_dev);
407 408

/**
409
 * ntb_transport_register_client - Register NTB client driver
410 411 412 413 414 415
 * @drv: NTB client driver to be registered
 *
 * Register an NTB client driver with the NTB transport layer
 *
 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
 */
416
int ntb_transport_register_client(struct ntb_transport_client *drv)
417
{
418
	drv->driver.bus = &ntb_transport_bus;
419

420 421 422
	if (list_empty(&ntb_transport_list))
		return -ENODEV;

423 424
	return driver_register(&drv->driver);
}
425
EXPORT_SYMBOL_GPL(ntb_transport_register_client);
426 427

/**
428
 * ntb_transport_unregister_client - Unregister NTB client driver
429 430 431 432 433 434
 * @drv: NTB client driver to be unregistered
 *
 * Unregister an NTB client driver with the NTB transport layer
 *
 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
 */
435
void ntb_transport_unregister_client(struct ntb_transport_client *drv)
436 437 438
{
	driver_unregister(&drv->driver);
}
439
EXPORT_SYMBOL_GPL(ntb_transport_unregister_client);
440 441 442 443 444

static ssize_t debugfs_read(struct file *filp, char __user *ubuf, size_t count,
			    loff_t *offp)
{
	struct ntb_transport_qp *qp;
445
	char *buf;
446 447
	ssize_t ret, out_offset, out_count;

448 449 450 451 452
	qp = filp->private_data;

	if (!qp || !qp->link_is_up)
		return 0;

453
	out_count = 1000;
454 455 456 457

	buf = kmalloc(out_count, GFP_KERNEL);
	if (!buf)
		return -ENOMEM;
458 459 460

	out_offset = 0;
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
Dave Jiang's avatar
Dave Jiang committed
461
			       "\nNTB QP stats:\n\n");
462 463 464 465
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_bytes - \t%llu\n", qp->rx_bytes);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_pkts - \t%llu\n", qp->rx_pkts);
466 467 468 469
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_memcpy - \t%llu\n", qp->rx_memcpy);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_async - \t%llu\n", qp->rx_async);
470 471 472 473 474 475 476 477 478
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_ring_empty - %llu\n", qp->rx_ring_empty);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_err_no_buf - %llu\n", qp->rx_err_no_buf);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_err_oflow - \t%llu\n", qp->rx_err_oflow);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_err_ver - \t%llu\n", qp->rx_err_ver);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
Dave Jiang's avatar
Dave Jiang committed
479
			       "rx_buff - \t0x%p\n", qp->rx_buff);
480
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
Jon Mason's avatar
Jon Mason committed
481
			       "rx_index - \t%u\n", qp->rx_index);
482
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
Dave Jiang's avatar
Dave Jiang committed
483
			       "rx_max_entry - \t%u\n\n", qp->rx_max_entry);
484 485 486 487 488

	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "tx_bytes - \t%llu\n", qp->tx_bytes);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "tx_pkts - \t%llu\n", qp->tx_pkts);
489 490 491 492
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "tx_memcpy - \t%llu\n", qp->tx_memcpy);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "tx_async - \t%llu\n", qp->tx_async);
493 494
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "tx_ring_full - \t%llu\n", qp->tx_ring_full);
495 496
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "tx_err_no_buf - %llu\n", qp->tx_err_no_buf);
497
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
Dave Jiang's avatar
Dave Jiang committed
498
			       "tx_mw - \t0x%p\n", qp->tx_mw);
499
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
Dave Jiang's avatar
Dave Jiang committed
500
			       "tx_index (H) - \t%u\n", qp->tx_index);
501
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
Dave Jiang's avatar
Dave Jiang committed
502
			       "RRI (T) - \t%u\n",
503
			       qp->remote_rx_info->entry);
Dave Jiang's avatar
Dave Jiang committed
504 505
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "tx_max_entry - \t%u\n", qp->tx_max_entry);
506 507 508
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "free tx - \t%u\n",
			       ntb_transport_tx_free_entry(qp));
509 510 511 512 513 514
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "DMA tx prep err - \t%llu\n",
			       qp->dma_tx_prep_err);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "DMA rx prep err - \t%llu\n",
			       qp->dma_rx_prep_err);
515 516

	out_offset += snprintf(buf + out_offset, out_count - out_offset,
Dave Jiang's avatar
Dave Jiang committed
517 518
			       "\n");
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
519 520 521 522 523
			       "Using TX DMA - \t%s\n",
			       qp->tx_dma_chan ? "Yes" : "No");
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "Using RX DMA - \t%s\n",
			       qp->rx_dma_chan ? "Yes" : "No");
Dave Jiang's avatar
Dave Jiang committed
524 525
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "QP Link - \t%s\n",
526
			       qp->link_is_up ? "Up" : "Down");
Dave Jiang's avatar
Dave Jiang committed
527 528 529
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "\n");

530 531
	if (out_offset > out_count)
		out_offset = out_count;
532 533

	ret = simple_read_from_buffer(ubuf, count, offp, buf, out_offset);
534
	kfree(buf);
535 536 537 538 539
	return ret;
}

static const struct file_operations ntb_qp_debugfs_stats = {
	.owner = THIS_MODULE,
Jon Mason's avatar
Jon Mason committed
540
	.open = simple_open,
541 542 543 544 545 546 547 548 549 550 551 552 553 554
	.read = debugfs_read,
};

static void ntb_list_add(spinlock_t *lock, struct list_head *entry,
			 struct list_head *list)
{
	unsigned long flags;

	spin_lock_irqsave(lock, flags);
	list_add_tail(entry, list);
	spin_unlock_irqrestore(lock, flags);
}

static struct ntb_queue_entry *ntb_list_rm(spinlock_t *lock,
Jon Mason's avatar
Jon Mason committed
555
					   struct list_head *list)
556 557 558 559 560 561 562 563 564 565 566
{
	struct ntb_queue_entry *entry;
	unsigned long flags;

	spin_lock_irqsave(lock, flags);
	if (list_empty(list)) {
		entry = NULL;
		goto out;
	}
	entry = list_first_entry(list, struct ntb_queue_entry, entry);
	list_del(&entry->entry);
567

568 569 570 571 572 573
out:
	spin_unlock_irqrestore(lock, flags);

	return entry;
}

574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594
static struct ntb_queue_entry *ntb_list_mv(spinlock_t *lock,
					   struct list_head *list,
					   struct list_head *to_list)
{
	struct ntb_queue_entry *entry;
	unsigned long flags;

	spin_lock_irqsave(lock, flags);

	if (list_empty(list)) {
		entry = NULL;
	} else {
		entry = list_first_entry(list, struct ntb_queue_entry, entry);
		list_move_tail(&entry->entry, to_list);
	}

	spin_unlock_irqrestore(lock, flags);

	return entry;
}

595 596
static int ntb_transport_setup_qp_mw(struct ntb_transport_ctx *nt,
				     unsigned int qp_num)
597
{
598 599
	struct ntb_transport_qp *qp = &nt->qp_vec[qp_num];
	struct ntb_transport_mw *mw;
600
	unsigned int rx_size, num_qps_mw;
601
	unsigned int mw_num, mw_count, qp_count;
Jon Mason's avatar
Jon Mason committed
602
	unsigned int i;
603

604 605
	mw_count = nt->mw_count;
	qp_count = nt->qp_count;
Jon Mason's avatar
Jon Mason committed
606

607 608 609 610 611
	mw_num = QP_TO_MW(nt, qp_num);
	mw = &nt->mw_vec[mw_num];

	if (!mw->virt_addr)
		return -ENOMEM;
612

613 614
	if (qp_count % mw_count && mw_num + 1 < qp_count / mw_count)
		num_qps_mw = qp_count / mw_count + 1;
615
	else
616
		num_qps_mw = qp_count / mw_count;
617

618
	rx_size = (unsigned int)mw->xlat_size / num_qps_mw;
619
	qp->rx_buff = mw->virt_addr + rx_size * (qp_num / mw_count);
Jon Mason's avatar
Jon Mason committed
620 621
	rx_size -= sizeof(struct ntb_rx_info);

622 623
	qp->remote_rx_info = qp->rx_buff + rx_size;

624 625
	/* Due to housekeeping, there must be atleast 2 buffs */
	qp->rx_max_frame = min(transport_mtu, rx_size / 2);
Jon Mason's avatar
Jon Mason committed
626 627 628
	qp->rx_max_entry = rx_size / qp->rx_max_frame;
	qp->rx_index = 0;

629
	qp->remote_rx_info->entry = qp->rx_max_entry - 1;
630

631
	/* setup the hdr offsets with 0's */
Jon Mason's avatar
Jon Mason committed
632
	for (i = 0; i < qp->rx_max_entry; i++) {
633 634
		void *offset = (qp->rx_buff + qp->rx_max_frame * (i + 1) -
				sizeof(struct ntb_payload_header));
635
		memset(offset, 0, sizeof(struct ntb_payload_header));
Jon Mason's avatar
Jon Mason committed
636
	}
637 638 639

	qp->rx_pkts = 0;
	qp->tx_pkts = 0;
Jon Mason's avatar
Jon Mason committed
640
	qp->tx_index = 0;
641 642

	return 0;
643 644
}

645
static void ntb_free_mw(struct ntb_transport_ctx *nt, int num_mw)
Jon Mason's avatar
Jon Mason committed
646
{
647 648
	struct ntb_transport_mw *mw = &nt->mw_vec[num_mw];
	struct pci_dev *pdev = nt->ndev->pdev;
Jon Mason's avatar
Jon Mason committed
649 650 651 652

	if (!mw->virt_addr)
		return;

653 654 655 656 657
	ntb_mw_clear_trans(nt->ndev, num_mw);
	dma_free_coherent(&pdev->dev, mw->buff_size,
			  mw->virt_addr, mw->dma_addr);
	mw->xlat_size = 0;
	mw->buff_size = 0;
Jon Mason's avatar
Jon Mason committed
658 659 660
	mw->virt_addr = NULL;
}

661
static int ntb_set_mw(struct ntb_transport_ctx *nt, int num_mw,
662
		      resource_size_t size)
663
{
664 665
	struct ntb_transport_mw *mw = &nt->mw_vec[num_mw];
	struct pci_dev *pdev = nt->ndev->pdev;
666
	size_t xlat_size, buff_size;
667 668
	int rc;

669 670 671
	if (!size)
		return -EINVAL;

672 673
	xlat_size = round_up(size, mw->xlat_align_size);
	buff_size = round_up(size, mw->xlat_align);
674

Jon Mason's avatar
Jon Mason committed
675
	/* No need to re-setup */
676
	if (mw->xlat_size == xlat_size)
Jon Mason's avatar
Jon Mason committed
677 678
		return 0;

679
	if (mw->buff_size)
Jon Mason's avatar
Jon Mason committed
680 681
		ntb_free_mw(nt, num_mw);

682 683 684
	/* Alloc memory for receiving data.  Must be aligned */
	mw->xlat_size = xlat_size;
	mw->buff_size = buff_size;
685

686 687
	mw->virt_addr = dma_alloc_coherent(&pdev->dev, buff_size,
					   &mw->dma_addr, GFP_KERNEL);
688
	if (!mw->virt_addr) {
689 690
		mw->xlat_size = 0;
		mw->buff_size = 0;
691
		dev_err(&pdev->dev, "Unable to alloc MW buff of size %zu\n",
692
			buff_size);
693 694 695
		return -ENOMEM;
	}

696 697 698 699 700 701
	/*
	 * we must ensure that the memory address allocated is BAR size
	 * aligned in order for the XLAT register to take the value. This
	 * is a requirement of the hardware. It is recommended to setup CMA
	 * for BAR sizes equal or greater than 4MB.
	 */
702 703
	if (!IS_ALIGNED(mw->dma_addr, mw->xlat_align)) {
		dev_err(&pdev->dev, "DMA memory %pad is not aligned\n",
704 705 706 707 708
			&mw->dma_addr);
		ntb_free_mw(nt, num_mw);
		return -ENOMEM;
	}

709
	/* Notify HW the memory location of the receive buffer */
710 711 712 713 714 715
	rc = ntb_mw_set_trans(nt->ndev, num_mw, mw->dma_addr, mw->xlat_size);
	if (rc) {
		dev_err(&pdev->dev, "Unable to set mw%d translation", num_mw);
		ntb_free_mw(nt, num_mw);
		return -EIO;
	}
716 717 718 719

	return 0;
}

720 721 722
static void ntb_qp_link_down_reset(struct ntb_transport_qp *qp)
{
	qp->link_is_up = false;
723
	qp->active = false;
724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740

	qp->tx_index = 0;
	qp->rx_index = 0;
	qp->rx_bytes = 0;
	qp->rx_pkts = 0;
	qp->rx_ring_empty = 0;
	qp->rx_err_no_buf = 0;
	qp->rx_err_oflow = 0;
	qp->rx_err_ver = 0;
	qp->rx_memcpy = 0;
	qp->rx_async = 0;
	qp->tx_bytes = 0;
	qp->tx_pkts = 0;
	qp->tx_ring_full = 0;
	qp->tx_err_no_buf = 0;
	qp->tx_memcpy = 0;
	qp->tx_async = 0;
741 742
	qp->dma_tx_prep_err = 0;
	qp->dma_rx_prep_err = 0;
743 744
}

745
static void ntb_qp_link_cleanup(struct ntb_transport_qp *qp)
746
{
747 748
	struct ntb_transport_ctx *nt = qp->transport;
	struct pci_dev *pdev = nt->ndev->pdev;
749

750
	dev_info(&pdev->dev, "qp %d: Link Cleanup\n", qp->qp_num);
751 752 753

	cancel_delayed_work_sync(&qp->link_work);
	ntb_qp_link_down_reset(qp);
754 755 756

	if (qp->event_handler)
		qp->event_handler(qp->cb_data, qp->link_is_up);
757 758 759 760 761 762 763
}

static void ntb_qp_link_cleanup_work(struct work_struct *work)
{
	struct ntb_transport_qp *qp = container_of(work,
						   struct ntb_transport_qp,
						   link_cleanup);
764
	struct ntb_transport_ctx *nt = qp->transport;
765 766

	ntb_qp_link_cleanup(qp);
767

768
	if (nt->link_is_up)
769 770 771 772
		schedule_delayed_work(&qp->link_work,
				      msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT));
}

773 774 775 776 777
static void ntb_qp_link_down(struct ntb_transport_qp *qp)
{
	schedule_work(&qp->link_cleanup);
}

778
static void ntb_transport_link_cleanup(struct ntb_transport_ctx *nt)
779
{
780 781
	struct ntb_transport_qp *qp;
	u64 qp_bitmap_alloc;
782 783
	int i;

784 785
	qp_bitmap_alloc = nt->qp_bitmap & ~nt->qp_bitmap_free;

786
	/* Pass along the info to any clients */
787 788 789 790 791 792 793
	for (i = 0; i < nt->qp_count; i++)
		if (qp_bitmap_alloc & BIT_ULL(i)) {
			qp = &nt->qp_vec[i];
			ntb_qp_link_cleanup(qp);
			cancel_work_sync(&qp->link_cleanup);
			cancel_delayed_work_sync(&qp->link_work);
		}
794

795
	if (!nt->link_is_up)
796 797 798 799 800 801 802
		cancel_delayed_work_sync(&nt->link_work);

	/* The scratchpad registers keep the values if the remote side
	 * goes down, blast them now to give them a sane value the next
	 * time they are accessed
	 */
	for (i = 0; i < MAX_SPAD; i++)
803
		ntb_spad_write(nt->ndev, i, 0);
804 805
}

806 807
static void ntb_transport_link_cleanup_work(struct work_struct *work)
{
808 809
	struct ntb_transport_ctx *nt =
		container_of(work, struct ntb_transport_ctx, link_cleanup);
810 811 812 813

	ntb_transport_link_cleanup(nt);
}

814
static void ntb_transport_event_callback(void *data)
815
{
816
	struct ntb_transport_ctx *nt = data;
817

818
	if (ntb_link_is_up(nt->ndev, NULL, NULL) == 1)
819
		schedule_delayed_work(&nt->link_work, 0);
820
	else
821
		schedule_work(&nt->link_cleanup);
822 823 824 825
}

static void ntb_transport_link_work(struct work_struct *work)
{
826 827 828 829 830
	struct ntb_transport_ctx *nt =
		container_of(work, struct ntb_transport_ctx, link_work.work);
	struct ntb_dev *ndev = nt->ndev;
	struct pci_dev *pdev = ndev->pdev;
	resource_size_t size;
831
	u32 val;
832
	int rc = 0, i, spad;
833

Jon Mason's avatar
Jon Mason committed
834
	/* send the local info, in the opposite order of the way we read it */
835 836
	for (i = 0; i < nt->mw_count; i++) {
		size = nt->mw_vec[i].phys_size;
837

838 839
		if (max_mw_size && size > max_mw_size)
			size = max_mw_size;
840

841
		spad = MW0_SZ_HIGH + (i * 2);
842
		ntb_peer_spad_write(ndev, spad, upper_32_bits(size));
843

844
		spad = MW0_SZ_LOW + (i * 2);
845
		ntb_peer_spad_write(ndev, spad, lower_32_bits(size));
846 847
	}

848
	ntb_peer_spad_write(ndev, NUM_MWS, nt->mw_count);
849

850
	ntb_peer_spad_write(ndev, NUM_QPS, nt->qp_count);
851

852
	ntb_peer_spad_write(ndev, VERSION, NTB_TRANSPORT_VERSION);
853

854
	/* Query the remote side for its info */
855
	val = ntb_spad_read(ndev, VERSION);
856 857
	dev_dbg(&pdev->dev, "Remote version = %d\n", val);
	if (val != NTB_TRANSPORT_VERSION)
858 859
		goto out;

860
	val = ntb_spad_read(ndev, NUM_QPS);
861
	dev_dbg(&pdev->dev, "Remote max number of qps = %d\n", val);
862
	if (val != nt->qp_count)
863 864
		goto out;

865
	val = ntb_spad_read(ndev, NUM_MWS);
Jon Mason's avatar
Jon Mason committed
866
	dev_dbg(&pdev->dev, "Remote number of mws = %d\n", val);
867 868
	if (val != nt->mw_count)
		goto out;
869

870
	for (i = 0; i < nt->mw_count; i++) {
Jon Mason's avatar
Jon Mason committed
871
		u64 val64;
872

873
		val = ntb_spad_read(ndev, MW0_SZ_HIGH + (i * 2));
874
		val64 = (u64)val << 32;
Jon Mason's avatar
Jon Mason committed
875

876
		val = ntb_spad_read(ndev, MW0_SZ_LOW + (i * 2));
Jon Mason's avatar
Jon Mason committed
877 878
		val64 |= val;

879
		dev_dbg(&pdev->dev, "Remote MW%d size = %#llx\n", i, val64);
Jon Mason's avatar
Jon Mason committed
880 881 882 883 884

		rc = ntb_set_mw(nt, i, val64);
		if (rc)
			goto out1;
	}
885

886
	nt->link_is_up = true;
887

888 889
	for (i = 0; i < nt->qp_count; i++) {
		struct ntb_transport_qp *qp = &nt->qp_vec[i];
890 891 892

		ntb_transport_setup_qp_mw(nt, i);

893
		if (qp->client_ready)
894 895 896 897 898
			schedule_delayed_work(&qp->link_work, 0);
	}

	return;

Jon Mason's avatar
Jon Mason committed
899
out1:
900
	for (i = 0; i < nt->mw_count; i++)
Jon Mason's avatar
Jon Mason committed
901
		ntb_free_mw(nt, i);
902 903 904 905 906 907 908

	/* if there's an actual failure, we should just bail */
	if (rc < 0) {
		ntb_link_disable(ndev);
		return;
	}

909
out:
910
	if (ntb_link_is_up(ndev, NULL, NULL) == 1)
911 912 913 914 915 916 917 918 919
		schedule_delayed_work(&nt->link_work,
				      msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT));
}

static void ntb_qp_link_work(struct work_struct *work)
{
	struct ntb_transport_qp *qp = container_of(work,
						   struct ntb_transport_qp,
						   link_work.work);
920 921 922
	struct pci_dev *pdev = qp->ndev->pdev;
	struct ntb_transport_ctx *nt = qp->transport;
	int val;