ntb_transport.c 49.5 KB
Newer Older
1 2 3 4 5 6 7
/*
 * This file is provided under a dual BSD/GPLv2 license.  When using or
 *   redistributing this file, you may do so under either license.
 *
 *   GPL LICENSE SUMMARY
 *
 *   Copyright(c) 2012 Intel Corporation. All rights reserved.
8
 *   Copyright (C) 2015 EMC Corporation. All Rights Reserved.
9 10 11 12 13 14 15 16
 *
 *   This program is free software; you can redistribute it and/or modify
 *   it under the terms of version 2 of the GNU General Public License as
 *   published by the Free Software Foundation.
 *
 *   BSD LICENSE
 *
 *   Copyright(c) 2012 Intel Corporation. All rights reserved.
17
 *   Copyright (C) 2015 EMC Corporation. All Rights Reserved.
18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44
 *
 *   Redistribution and use in source and binary forms, with or without
 *   modification, are permitted provided that the following conditions
 *   are met:
 *
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copy
 *       notice, this list of conditions and the following disclaimer in
 *       the documentation and/or other materials provided with the
 *       distribution.
 *     * Neither the name of Intel Corporation nor the names of its
 *       contributors may be used to endorse or promote products derived
 *       from this software without specific prior written permission.
 *
 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
45
 * PCIe NTB Transport Linux driver
46 47 48 49 50 51
 *
 * Contact Information:
 * Jon Mason <jon.mason@intel.com>
 */
#include <linux/debugfs.h>
#include <linux/delay.h>
52
#include <linux/dmaengine.h>
53 54 55 56 57 58 59 60
#include <linux/dma-mapping.h>
#include <linux/errno.h>
#include <linux/export.h>
#include <linux/interrupt.h>
#include <linux/module.h>
#include <linux/pci.h>
#include <linux/slab.h>
#include <linux/types.h>
61
#include <linux/uaccess.h>
62 63
#include "linux/ntb.h"
#include "linux/ntb_transport.h"
64

65 66 67 68 69 70 71 72 73 74 75 76 77
#define NTB_TRANSPORT_VERSION	4
#define NTB_TRANSPORT_VER	"4"
#define NTB_TRANSPORT_NAME	"ntb_transport"
#define NTB_TRANSPORT_DESC	"Software Queue-Pair Transport over NTB"

MODULE_DESCRIPTION(NTB_TRANSPORT_DESC);
MODULE_VERSION(NTB_TRANSPORT_VER);
MODULE_LICENSE("Dual BSD/GPL");
MODULE_AUTHOR("Intel Corporation");

static unsigned long max_mw_size;
module_param(max_mw_size, ulong, 0644);
MODULE_PARM_DESC(max_mw_size, "Limit size of large memory windows");
78

79
static unsigned int transport_mtu = 0x10000;
80 81 82
module_param(transport_mtu, uint, 0644);
MODULE_PARM_DESC(transport_mtu, "Maximum size of NTB transport packets");

Jon Mason's avatar
Jon Mason committed
83
static unsigned char max_num_clients;
84 85 86
module_param(max_num_clients, byte, 0644);
MODULE_PARM_DESC(max_num_clients, "Maximum number of NTB transport clients");

87 88 89 90
static unsigned int copy_bytes = 1024;
module_param(copy_bytes, uint, 0644);
MODULE_PARM_DESC(copy_bytes, "Threshold under which NTB will use the CPU to copy instead of DMA");

91 92 93 94
static bool use_dma;
module_param(use_dma, bool, 0644);
MODULE_PARM_DESC(use_dma, "Use DMA engine to perform large data copy");

95 96
static struct dentry *nt_debugfs_dir;

97 98 99
struct ntb_queue_entry {
	/* ntb_queue list reference */
	struct list_head entry;
100
	/* pointers to data to be transferred */
101 102 103 104
	void *cb_data;
	void *buf;
	unsigned int len;
	unsigned int flags;
105 106 107 108 109 110 111

	struct ntb_transport_qp *qp;
	union {
		struct ntb_payload_header __iomem *tx_hdr;
		struct ntb_payload_header *rx_hdr;
	};
	unsigned int index;
112 113
};

Jon Mason's avatar
Jon Mason committed
114 115 116 117
struct ntb_rx_info {
	unsigned int entry;
};

118
struct ntb_transport_qp {
119 120
	struct ntb_transport_ctx *transport;
	struct ntb_dev *ndev;
121
	void *cb_data;
122
	struct dma_chan *dma_chan;
123 124

	bool client_ready;
125 126
	bool link_is_up;

127
	u8 qp_num;	/* Only 64 QP's are allowed.  0-63 */
128
	u64 qp_bit;
129

Jon Mason's avatar
Jon Mason committed
130
	struct ntb_rx_info __iomem *rx_info;
Jon Mason's avatar
Jon Mason committed
131 132
	struct ntb_rx_info *remote_rx_info;

Jon Mason's avatar
Jon Mason committed
133 134
	void (*tx_handler)(struct ntb_transport_qp *qp, void *qp_data,
			   void *data, int len);
135 136
	struct list_head tx_free_q;
	spinlock_t ntb_tx_free_q_lock;
Jon Mason's avatar
Jon Mason committed
137
	void __iomem *tx_mw;
138
	dma_addr_t tx_mw_phys;
Jon Mason's avatar
Jon Mason committed
139 140
	unsigned int tx_index;
	unsigned int tx_max_entry;
141
	unsigned int tx_max_frame;
142

Jon Mason's avatar
Jon Mason committed
143 144
	void (*rx_handler)(struct ntb_transport_qp *qp, void *qp_data,
			   void *data, int len);
145 146 147 148
	struct list_head rx_pend_q;
	struct list_head rx_free_q;
	spinlock_t ntb_rx_pend_q_lock;
	spinlock_t ntb_rx_free_q_lock;
Jon Mason's avatar
Jon Mason committed
149 150 151
	void *rx_buff;
	unsigned int rx_index;
	unsigned int rx_max_entry;
152
	unsigned int rx_max_frame;
153
	dma_cookie_t last_cookie;
154
	struct tasklet_struct rxc_db_work;
155

Jon Mason's avatar
Jon Mason committed
156
	void (*event_handler)(void *data, int status);
157
	struct delayed_work link_work;
158
	struct work_struct link_cleanup;
159 160 161 162 163 164 165 166 167 168 169

	struct dentry *debugfs_dir;
	struct dentry *debugfs_stats;

	/* Stats */
	u64 rx_bytes;
	u64 rx_pkts;
	u64 rx_ring_empty;
	u64 rx_err_no_buf;
	u64 rx_err_oflow;
	u64 rx_err_ver;
170 171
	u64 rx_memcpy;
	u64 rx_async;
172 173 174
	u64 tx_bytes;
	u64 tx_pkts;
	u64 tx_ring_full;
175 176 177
	u64 tx_err_no_buf;
	u64 tx_memcpy;
	u64 tx_async;
178 179 180
};

struct ntb_transport_mw {
181 182 183 184 185 186 187
	phys_addr_t phys_addr;
	resource_size_t phys_size;
	resource_size_t xlat_align;
	resource_size_t xlat_align_size;
	void __iomem *vbase;
	size_t xlat_size;
	size_t buff_size;
188 189 190 191 192 193
	void *virt_addr;
	dma_addr_t dma_addr;
};

struct ntb_transport_client_dev {
	struct list_head entry;
194
	struct ntb_transport_ctx *nt;
195 196 197
	struct device dev;
};

198
struct ntb_transport_ctx {
199 200 201
	struct list_head entry;
	struct list_head client_devs;

202 203 204 205 206 207 208 209 210 211
	struct ntb_dev *ndev;

	struct ntb_transport_mw *mw_vec;
	struct ntb_transport_qp *qp_vec;
	unsigned int mw_count;
	unsigned int qp_count;
	u64 qp_bitmap;
	u64 qp_bitmap_free;

	bool link_is_up;
212
	struct delayed_work link_work;
213
	struct work_struct link_cleanup;
214 215 216
};

enum {
217 218
	DESC_DONE_FLAG = BIT(0),
	LINK_DOWN_FLAG = BIT(1),
219 220 221
};

struct ntb_payload_header {
Jon Mason's avatar
Jon Mason committed
222
	unsigned int ver;
223 224 225 226 227 228 229
	unsigned int len;
	unsigned int flags;
};

enum {
	VERSION = 0,
	QP_LINKS,
Jon Mason's avatar
Jon Mason committed
230 231 232 233 234 235
	NUM_QPS,
	NUM_MWS,
	MW0_SZ_HIGH,
	MW0_SZ_LOW,
	MW1_SZ_HIGH,
	MW1_SZ_LOW,
236 237 238
	MAX_SPAD,
};

239 240 241 242 243 244 245
#define dev_client_dev(__dev) \
	container_of((__dev), struct ntb_transport_client_dev, dev)

#define drv_client(__drv) \
	container_of((__drv), struct ntb_transport_client, driver)

#define QP_TO_MW(nt, qp)	((qp) % nt->mw_count)
246 247 248
#define NTB_QP_DEF_NUM_ENTRIES	100
#define NTB_LINK_DOWN_TIMEOUT	10

249 250 251 252 253 254
static void ntb_transport_rxc_db(unsigned long data);
static const struct ntb_ctx_ops ntb_transport_ops;
static struct ntb_client ntb_transport_client;

static int ntb_transport_bus_match(struct device *dev,
				   struct device_driver *drv)
255 256 257 258
{
	return !strncmp(dev_name(dev), drv->name, strlen(drv->name));
}

259
static int ntb_transport_bus_probe(struct device *dev)
260
{
261
	const struct ntb_transport_client *client;
262 263 264
	int rc = -EINVAL;

	get_device(dev);
265 266 267

	client = drv_client(dev->driver);
	rc = client->probe(dev);
268 269 270 271 272 273
	if (rc)
		put_device(dev);

	return rc;
}

274
static int ntb_transport_bus_remove(struct device *dev)
275
{
276
	const struct ntb_transport_client *client;
277

278 279
	client = drv_client(dev->driver);
	client->remove(dev);
280 281 282 283 284 285

	put_device(dev);

	return 0;
}

286 287 288 289 290
static struct bus_type ntb_transport_bus = {
	.name = "ntb_transport",
	.match = ntb_transport_bus_match,
	.probe = ntb_transport_bus_probe,
	.remove = ntb_transport_bus_remove,
291 292 293 294
};

static LIST_HEAD(ntb_transport_list);

295
static int ntb_bus_init(struct ntb_transport_ctx *nt)
296 297 298 299 300
{
	list_add(&nt->entry, &ntb_transport_list);
	return 0;
}

301
static void ntb_bus_remove(struct ntb_transport_ctx *nt)
302 303 304 305 306 307 308 309 310 311 312 313 314
{
	struct ntb_transport_client_dev *client_dev, *cd;

	list_for_each_entry_safe(client_dev, cd, &nt->client_devs, entry) {
		dev_err(client_dev->dev.parent, "%s still attached to bus, removing\n",
			dev_name(&client_dev->dev));
		list_del(&client_dev->entry);
		device_unregister(&client_dev->dev);
	}

	list_del(&nt->entry);
}

315
static void ntb_transport_client_release(struct device *dev)
316 317 318
{
	struct ntb_transport_client_dev *client_dev;

319
	client_dev = dev_client_dev(dev);
320 321 322 323
	kfree(client_dev);
}

/**
324
 * ntb_transport_unregister_client_dev - Unregister NTB client device
325 326 327 328
 * @device_name: Name of NTB client device
 *
 * Unregister an NTB client device with the NTB transport layer
 */
329
void ntb_transport_unregister_client_dev(char *device_name)
330 331
{
	struct ntb_transport_client_dev *client, *cd;
332
	struct ntb_transport_ctx *nt;
333 334 335 336 337 338 339 340 341

	list_for_each_entry(nt, &ntb_transport_list, entry)
		list_for_each_entry_safe(client, cd, &nt->client_devs, entry)
			if (!strncmp(dev_name(&client->dev), device_name,
				     strlen(device_name))) {
				list_del(&client->entry);
				device_unregister(&client->dev);
			}
}
342
EXPORT_SYMBOL_GPL(ntb_transport_unregister_client_dev);
343 344

/**
345
 * ntb_transport_register_client_dev - Register NTB client device
346 347 348 349
 * @device_name: Name of NTB client device
 *
 * Register an NTB client device with the NTB transport layer
 */
350
int ntb_transport_register_client_dev(char *device_name)
351 352
{
	struct ntb_transport_client_dev *client_dev;
353
	struct ntb_transport_ctx *nt;
354
	int node;
Jon Mason's avatar
Jon Mason committed
355
	int rc, i = 0;
356

357 358 359
	if (list_empty(&ntb_transport_list))
		return -ENODEV;

360 361 362
	list_for_each_entry(nt, &ntb_transport_list, entry) {
		struct device *dev;

363 364 365 366
		node = dev_to_node(&nt->ndev->dev);

		client_dev = kzalloc_node(sizeof(*client_dev),
					  GFP_KERNEL, node);
367 368 369 370 371 372 373 374
		if (!client_dev) {
			rc = -ENOMEM;
			goto err;
		}

		dev = &client_dev->dev;

		/* setup and register client devices */
Jon Mason's avatar
Jon Mason committed
375
		dev_set_name(dev, "%s%d", device_name, i);
376 377 378
		dev->bus = &ntb_transport_bus;
		dev->release = ntb_transport_client_release;
		dev->parent = &nt->ndev->dev;
379 380 381 382 383 384 385 386

		rc = device_register(dev);
		if (rc) {
			kfree(client_dev);
			goto err;
		}

		list_add_tail(&client_dev->entry, &nt->client_devs);
Jon Mason's avatar
Jon Mason committed
387
		i++;
388 389 390 391 392
	}

	return 0;

err:
393
	ntb_transport_unregister_client_dev(device_name);
394 395 396

	return rc;
}
397
EXPORT_SYMBOL_GPL(ntb_transport_register_client_dev);
398 399

/**
400
 * ntb_transport_register_client - Register NTB client driver
401 402 403 404 405 406
 * @drv: NTB client driver to be registered
 *
 * Register an NTB client driver with the NTB transport layer
 *
 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
 */
407
int ntb_transport_register_client(struct ntb_transport_client *drv)
408
{
409
	drv->driver.bus = &ntb_transport_bus;
410

411 412 413
	if (list_empty(&ntb_transport_list))
		return -ENODEV;

414 415
	return driver_register(&drv->driver);
}
416
EXPORT_SYMBOL_GPL(ntb_transport_register_client);
417 418

/**
419
 * ntb_transport_unregister_client - Unregister NTB client driver
420 421 422 423 424 425
 * @drv: NTB client driver to be unregistered
 *
 * Unregister an NTB client driver with the NTB transport layer
 *
 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
 */
426
void ntb_transport_unregister_client(struct ntb_transport_client *drv)
427 428 429
{
	driver_unregister(&drv->driver);
}
430
EXPORT_SYMBOL_GPL(ntb_transport_unregister_client);
431 432 433 434 435

static ssize_t debugfs_read(struct file *filp, char __user *ubuf, size_t count,
			    loff_t *offp)
{
	struct ntb_transport_qp *qp;
436
	char *buf;
437 438
	ssize_t ret, out_offset, out_count;

439
	out_count = 1000;
440 441 442 443

	buf = kmalloc(out_count, GFP_KERNEL);
	if (!buf)
		return -ENOMEM;
444 445 446 447 448 449 450 451 452

	qp = filp->private_data;
	out_offset = 0;
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "NTB QP stats\n");
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_bytes - \t%llu\n", qp->rx_bytes);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_pkts - \t%llu\n", qp->rx_pkts);
453 454 455 456
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_memcpy - \t%llu\n", qp->rx_memcpy);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_async - \t%llu\n", qp->rx_async);
457 458 459 460 461 462 463 464 465
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_ring_empty - %llu\n", qp->rx_ring_empty);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_err_no_buf - %llu\n", qp->rx_err_no_buf);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_err_oflow - \t%llu\n", qp->rx_err_oflow);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_err_ver - \t%llu\n", qp->rx_err_ver);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
Jon Mason's avatar
Jon Mason committed
466
			       "rx_buff - \t%p\n", qp->rx_buff);
467
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
Jon Mason's avatar
Jon Mason committed
468
			       "rx_index - \t%u\n", qp->rx_index);
469
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
Jon Mason's avatar
Jon Mason committed
470
			       "rx_max_entry - \t%u\n", qp->rx_max_entry);
471 472 473 474 475

	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "tx_bytes - \t%llu\n", qp->tx_bytes);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "tx_pkts - \t%llu\n", qp->tx_pkts);
476 477 478 479
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "tx_memcpy - \t%llu\n", qp->tx_memcpy);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "tx_async - \t%llu\n", qp->tx_async);
480 481
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "tx_ring_full - \t%llu\n", qp->tx_ring_full);
482 483
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "tx_err_no_buf - %llu\n", qp->tx_err_no_buf);
484
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
Jon Mason's avatar
Jon Mason committed
485
			       "tx_mw - \t%p\n", qp->tx_mw);
486
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
Jon Mason's avatar
Jon Mason committed
487
			       "tx_index - \t%u\n", qp->tx_index);
488
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
Jon Mason's avatar
Jon Mason committed
489
			       "tx_max_entry - \t%u\n", qp->tx_max_entry);
490 491

	out_offset += snprintf(buf + out_offset, out_count - out_offset,
492 493
			       "\nQP Link %s\n",
			       qp->link_is_up ? "Up" : "Down");
494 495
	if (out_offset > out_count)
		out_offset = out_count;
496 497

	ret = simple_read_from_buffer(ubuf, count, offp, buf, out_offset);
498
	kfree(buf);
499 500 501 502 503
	return ret;
}

static const struct file_operations ntb_qp_debugfs_stats = {
	.owner = THIS_MODULE,
Jon Mason's avatar
Jon Mason committed
504
	.open = simple_open,
505 506 507 508 509 510 511 512 513 514 515 516 517 518
	.read = debugfs_read,
};

static void ntb_list_add(spinlock_t *lock, struct list_head *entry,
			 struct list_head *list)
{
	unsigned long flags;

	spin_lock_irqsave(lock, flags);
	list_add_tail(entry, list);
	spin_unlock_irqrestore(lock, flags);
}

static struct ntb_queue_entry *ntb_list_rm(spinlock_t *lock,
Jon Mason's avatar
Jon Mason committed
519
					   struct list_head *list)
520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536
{
	struct ntb_queue_entry *entry;
	unsigned long flags;

	spin_lock_irqsave(lock, flags);
	if (list_empty(list)) {
		entry = NULL;
		goto out;
	}
	entry = list_first_entry(list, struct ntb_queue_entry, entry);
	list_del(&entry->entry);
out:
	spin_unlock_irqrestore(lock, flags);

	return entry;
}

537 538
static int ntb_transport_setup_qp_mw(struct ntb_transport_ctx *nt,
				     unsigned int qp_num)
539
{
540 541
	struct ntb_transport_qp *qp = &nt->qp_vec[qp_num];
	struct ntb_transport_mw *mw;
542
	unsigned int rx_size, num_qps_mw;
543
	unsigned int mw_num, mw_count, qp_count;
Jon Mason's avatar
Jon Mason committed
544
	unsigned int i;
545

546 547
	mw_count = nt->mw_count;
	qp_count = nt->qp_count;
Jon Mason's avatar
Jon Mason committed
548

549 550 551 552 553
	mw_num = QP_TO_MW(nt, qp_num);
	mw = &nt->mw_vec[mw_num];

	if (!mw->virt_addr)
		return -ENOMEM;
554

555 556
	if (qp_count % mw_count && mw_num + 1 < qp_count / mw_count)
		num_qps_mw = qp_count / mw_count + 1;
557
	else
558
		num_qps_mw = qp_count / mw_count;
559

560 561
	rx_size = (unsigned int)mw->xlat_size / num_qps_mw;
	qp->rx_buff = mw->virt_addr + rx_size * qp_num / mw_count;
Jon Mason's avatar
Jon Mason committed
562 563
	rx_size -= sizeof(struct ntb_rx_info);

564 565
	qp->remote_rx_info = qp->rx_buff + rx_size;

566 567
	/* Due to housekeeping, there must be atleast 2 buffs */
	qp->rx_max_frame = min(transport_mtu, rx_size / 2);
Jon Mason's avatar
Jon Mason committed
568 569 570
	qp->rx_max_entry = rx_size / qp->rx_max_frame;
	qp->rx_index = 0;

571
	qp->remote_rx_info->entry = qp->rx_max_entry - 1;
572

573
	/* setup the hdr offsets with 0's */
Jon Mason's avatar
Jon Mason committed
574
	for (i = 0; i < qp->rx_max_entry; i++) {
575 576
		void *offset = (qp->rx_buff + qp->rx_max_frame * (i + 1) -
				sizeof(struct ntb_payload_header));
577
		memset(offset, 0, sizeof(struct ntb_payload_header));
Jon Mason's avatar
Jon Mason committed
578
	}
579 580 581

	qp->rx_pkts = 0;
	qp->tx_pkts = 0;
Jon Mason's avatar
Jon Mason committed
582
	qp->tx_index = 0;
583 584

	return 0;
585 586
}

587
static void ntb_free_mw(struct ntb_transport_ctx *nt, int num_mw)
Jon Mason's avatar
Jon Mason committed
588
{
589 590
	struct ntb_transport_mw *mw = &nt->mw_vec[num_mw];
	struct pci_dev *pdev = nt->ndev->pdev;
Jon Mason's avatar
Jon Mason committed
591 592 593 594

	if (!mw->virt_addr)
		return;

595 596 597 598 599
	ntb_mw_clear_trans(nt->ndev, num_mw);
	dma_free_coherent(&pdev->dev, mw->buff_size,
			  mw->virt_addr, mw->dma_addr);
	mw->xlat_size = 0;
	mw->buff_size = 0;
Jon Mason's avatar
Jon Mason committed
600 601 602
	mw->virt_addr = NULL;
}

603 604
static int ntb_set_mw(struct ntb_transport_ctx *nt, int num_mw,
		      unsigned int size)
605
{
606 607 608 609 610 611 612
	struct ntb_transport_mw *mw = &nt->mw_vec[num_mw];
	struct pci_dev *pdev = nt->ndev->pdev;
	unsigned int xlat_size, buff_size;
	int rc;

	xlat_size = round_up(size, mw->xlat_align_size);
	buff_size = round_up(size, mw->xlat_align);
613

Jon Mason's avatar
Jon Mason committed
614
	/* No need to re-setup */
615
	if (mw->xlat_size == xlat_size)
Jon Mason's avatar
Jon Mason committed
616 617
		return 0;

618
	if (mw->buff_size)
Jon Mason's avatar
Jon Mason committed
619 620
		ntb_free_mw(nt, num_mw);

621 622 623
	/* Alloc memory for receiving data.  Must be aligned */
	mw->xlat_size = xlat_size;
	mw->buff_size = buff_size;
624

625 626
	mw->virt_addr = dma_alloc_coherent(&pdev->dev, buff_size,
					   &mw->dma_addr, GFP_KERNEL);
627
	if (!mw->virt_addr) {
628 629 630 631
		mw->xlat_size = 0;
		mw->buff_size = 0;
		dev_err(&pdev->dev, "Unable to alloc MW buff of size %d\n",
			buff_size);
632 633 634
		return -ENOMEM;
	}

635 636 637 638 639 640
	/*
	 * we must ensure that the memory address allocated is BAR size
	 * aligned in order for the XLAT register to take the value. This
	 * is a requirement of the hardware. It is recommended to setup CMA
	 * for BAR sizes equal or greater than 4MB.
	 */
641 642
	if (!IS_ALIGNED(mw->dma_addr, mw->xlat_align)) {
		dev_err(&pdev->dev, "DMA memory %pad is not aligned\n",
643 644 645 646 647
			&mw->dma_addr);
		ntb_free_mw(nt, num_mw);
		return -ENOMEM;
	}

648
	/* Notify HW the memory location of the receive buffer */
649 650 651 652 653 654
	rc = ntb_mw_set_trans(nt->ndev, num_mw, mw->dma_addr, mw->xlat_size);
	if (rc) {
		dev_err(&pdev->dev, "Unable to set mw%d translation", num_mw);
		ntb_free_mw(nt, num_mw);
		return -EIO;
	}
655 656 657 658

	return 0;
}

659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680
static void ntb_qp_link_down_reset(struct ntb_transport_qp *qp)
{
	qp->link_is_up = false;

	qp->tx_index = 0;
	qp->rx_index = 0;
	qp->rx_bytes = 0;
	qp->rx_pkts = 0;
	qp->rx_ring_empty = 0;
	qp->rx_err_no_buf = 0;
	qp->rx_err_oflow = 0;
	qp->rx_err_ver = 0;
	qp->rx_memcpy = 0;
	qp->rx_async = 0;
	qp->tx_bytes = 0;
	qp->tx_pkts = 0;
	qp->tx_ring_full = 0;
	qp->tx_err_no_buf = 0;
	qp->tx_memcpy = 0;
	qp->tx_async = 0;
}

681
static void ntb_qp_link_cleanup(struct ntb_transport_qp *qp)
682
{
683 684
	struct ntb_transport_ctx *nt = qp->transport;
	struct pci_dev *pdev = nt->ndev->pdev;
685

686
	dev_info(&pdev->dev, "qp %d: Link Cleanup\n", qp->qp_num);
687 688 689

	cancel_delayed_work_sync(&qp->link_work);
	ntb_qp_link_down_reset(qp);
690 691 692

	if (qp->event_handler)
		qp->event_handler(qp->cb_data, qp->link_is_up);
693 694 695 696 697 698 699
}

static void ntb_qp_link_cleanup_work(struct work_struct *work)
{
	struct ntb_transport_qp *qp = container_of(work,
						   struct ntb_transport_qp,
						   link_cleanup);
700
	struct ntb_transport_ctx *nt = qp->transport;
701 702

	ntb_qp_link_cleanup(qp);
703

704
	if (nt->link_is_up)
705 706 707 708
		schedule_delayed_work(&qp->link_work,
				      msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT));
}

709 710 711 712 713
static void ntb_qp_link_down(struct ntb_transport_qp *qp)
{
	schedule_work(&qp->link_cleanup);
}

714
static void ntb_transport_link_cleanup(struct ntb_transport_ctx *nt)
715
{
716 717
	struct ntb_transport_qp *qp;
	u64 qp_bitmap_alloc;
718 719
	int i;

720 721
	qp_bitmap_alloc = nt->qp_bitmap & ~nt->qp_bitmap_free;

722
	/* Pass along the info to any clients */
723 724 725 726 727 728 729
	for (i = 0; i < nt->qp_count; i++)
		if (qp_bitmap_alloc & BIT_ULL(i)) {
			qp = &nt->qp_vec[i];
			ntb_qp_link_cleanup(qp);
			cancel_work_sync(&qp->link_cleanup);
			cancel_delayed_work_sync(&qp->link_work);
		}
730

731
	if (!nt->link_is_up)
732 733 734 735 736 737 738
		cancel_delayed_work_sync(&nt->link_work);

	/* The scratchpad registers keep the values if the remote side
	 * goes down, blast them now to give them a sane value the next
	 * time they are accessed
	 */
	for (i = 0; i < MAX_SPAD; i++)
739
		ntb_spad_write(nt->ndev, i, 0);
740 741
}

742 743
static void ntb_transport_link_cleanup_work(struct work_struct *work)
{
744 745
	struct ntb_transport_ctx *nt =
		container_of(work, struct ntb_transport_ctx, link_cleanup);
746 747 748 749

	ntb_transport_link_cleanup(nt);
}

750
static void ntb_transport_event_callback(void *data)
751
{
752
	struct ntb_transport_ctx *nt = data;
753

754
	if (ntb_link_is_up(nt->ndev, NULL, NULL) == 1)
755
		schedule_delayed_work(&nt->link_work, 0);
756
	else
757
		schedule_work(&nt->link_cleanup);
758 759 760 761
}

static void ntb_transport_link_work(struct work_struct *work)
{
762 763 764 765 766
	struct ntb_transport_ctx *nt =
		container_of(work, struct ntb_transport_ctx, link_work.work);
	struct ntb_dev *ndev = nt->ndev;
	struct pci_dev *pdev = ndev->pdev;
	resource_size_t size;
767
	u32 val;
768
	int rc, i, spad;
769

Jon Mason's avatar
Jon Mason committed
770
	/* send the local info, in the opposite order of the way we read it */
771 772
	for (i = 0; i < nt->mw_count; i++) {
		size = nt->mw_vec[i].phys_size;
773

774 775
		if (max_mw_size && size > max_mw_size)
			size = max_mw_size;
776

777 778
		spad = MW0_SZ_HIGH + (i * 2);
		ntb_peer_spad_write(ndev, spad, (u32)(size >> 32));
779

780 781
		spad = MW0_SZ_LOW + (i * 2);
		ntb_peer_spad_write(ndev, spad, (u32)size);
782 783
	}

784
	ntb_peer_spad_write(ndev, NUM_MWS, nt->mw_count);
785

786
	ntb_peer_spad_write(ndev, NUM_QPS, nt->qp_count);
787

788
	ntb_peer_spad_write(ndev, VERSION, NTB_TRANSPORT_VERSION);
789

790
	/* Query the remote side for its info */
791
	val = ntb_spad_read(ndev, VERSION);
792 793
	dev_dbg(&pdev->dev, "Remote version = %d\n", val);
	if (val != NTB_TRANSPORT_VERSION)
794 795
		goto out;

796
	val = ntb_spad_read(ndev, NUM_QPS);
797
	dev_dbg(&pdev->dev, "Remote max number of qps = %d\n", val);
798
	if (val != nt->qp_count)
799 800
		goto out;

801
	val = ntb_spad_read(ndev, NUM_MWS);
Jon Mason's avatar
Jon Mason committed
802
	dev_dbg(&pdev->dev, "Remote number of mws = %d\n", val);
803 804
	if (val != nt->mw_count)
		goto out;
805

806
	for (i = 0; i < nt->mw_count; i++) {
Jon Mason's avatar
Jon Mason committed
807
		u64 val64;
808

809
		val = ntb_spad_read(ndev, MW0_SZ_HIGH + (i * 2));
810
		val64 = (u64)val << 32;
Jon Mason's avatar
Jon Mason committed
811

812
		val = ntb_spad_read(ndev, MW0_SZ_LOW + (i * 2));
Jon Mason's avatar
Jon Mason committed
813 814
		val64 |= val;

815
		dev_dbg(&pdev->dev, "Remote MW%d size = %#llx\n", i, val64);
Jon Mason's avatar
Jon Mason committed
816 817 818 819 820

		rc = ntb_set_mw(nt, i, val64);
		if (rc)
			goto out1;
	}
821

822
	nt->link_is_up = true;
823

824 825
	for (i = 0; i < nt->qp_count; i++) {
		struct ntb_transport_qp *qp = &nt->qp_vec[i];
826 827 828

		ntb_transport_setup_qp_mw(nt, i);

829
		if (qp->client_ready)
830 831 832 833 834
			schedule_delayed_work(&qp->link_work, 0);
	}

	return;

Jon Mason's avatar
Jon Mason committed
835
out1:
836
	for (i = 0; i < nt->mw_count; i++)
Jon Mason's avatar
Jon Mason committed
837
		ntb_free_mw(nt, i);
838
out:
839
	if (ntb_link_is_up(ndev, NULL, NULL) == 1)
840 841 842 843 844 845 846 847 848
		schedule_delayed_work(&nt->link_work,
				      msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT));
}

static void ntb_qp_link_work(struct work_struct *work)
{
	struct ntb_transport_qp *qp = container_of(work,
						   struct ntb_transport_qp,
						   link_work.work);
849 850 851
	struct pci_dev *pdev = qp->ndev->pdev;
	struct ntb_transport_ctx *nt = qp->transport;
	int val;
852

853
	WARN_ON(!nt->link_is_up);
854

855
	val = ntb_spad_read(nt->ndev, QP_LINKS);
856

857
	ntb_peer_spad_write(nt->ndev, QP_LINKS, val | BIT(qp->qp_num));
858 859

	/* query remote spad for qp ready bits */
860
	ntb_peer_spad_read(nt->ndev, QP_LINKS);
Allen Hubbe's avatar
Allen Hubbe committed
861
	dev_dbg_ratelimited(&pdev->dev, "Remote QP link status = %x\n", val);
862 863

	/* See if the remote side is up */
864
	if (val & BIT(qp->qp_num)) {
865
		dev_info(&pdev->dev, "qp %d: Link Up\n", qp->qp_num);
866 867
		qp->link_is_up = true;

868
		if (qp->event_handler)
869 870
			qp->event_handler(qp->cb_data, qp->link_is_up);
	} else if (nt->link_is_up)
871 872 873 874
		schedule_delayed_work(&qp->link_work,
				      msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT));
}

875
static int ntb_transport_init_queue(struct ntb_transport_ctx *nt,
Jon Mason's avatar
Jon Mason committed
876
				    unsigned int qp_num)
877 878
{
	struct ntb_transport_qp *qp;
879 880 881
	struct ntb_transport_mw *mw;
	phys_addr_t mw_base;
	resource_size_t mw_size;
882
	unsigned int num_qps_mw, tx_size;
883
	unsigned int mw_num, mw_count, qp_count;
884
	u64 qp_offset;
Jon Mason's avatar
Jon Mason committed
885

886 887
	mw_count = nt->mw_count;
	qp_count = nt->qp_count;
888

889 890 891 892
	mw_num = QP_TO_MW(nt, qp_num);
	mw = &nt->mw_vec[mw_num];

	qp = &nt->qp_vec[qp_num];
893 894 895
	qp->qp_num = qp_num;
	qp->transport = nt;
	qp->ndev = nt->ndev;
896
	qp->client_ready = false;
897
	qp->event_handler = NULL;
898
	ntb_qp_link_down_reset(qp);
899

900 901
	if (qp_count % mw_count && mw_num + 1 < qp_count / mw_count)
		num_qps_mw = qp_count / mw_count + 1;
902
	else
903 904 905 906
		num_qps_mw = qp_count / mw_count;

	mw_base = nt->mw_vec[mw_num].phys_addr;
	mw_size = nt->mw_vec[mw_num].phys_size;
907

908 909 910 911
	tx_size = (unsigned int)mw_size / num_qps_mw;
	qp_offset = tx_size * qp_num / mw_count;

	qp->tx_mw = nt->mw_vec[mw_num].vbase + qp_offset;
912 913 914
	if (!qp->tx_mw)
		return -EINVAL;

915
	qp->tx_mw_phys = mw_base + qp_offset;
916 917 918
	if (!qp->tx_mw_phys)
		return -EINVAL;

Jon Mason's avatar
Jon Mason committed
919
	tx_size -= sizeof(struct ntb_rx_info);