ntb_transport.c 49.8 KB
Newer Older
1 2 3 4 5 6 7
/*
 * This file is provided under a dual BSD/GPLv2 license.  When using or
 *   redistributing this file, you may do so under either license.
 *
 *   GPL LICENSE SUMMARY
 *
 *   Copyright(c) 2012 Intel Corporation. All rights reserved.
8
 *   Copyright (C) 2015 EMC Corporation. All Rights Reserved.
9 10 11 12 13 14 15 16
 *
 *   This program is free software; you can redistribute it and/or modify
 *   it under the terms of version 2 of the GNU General Public License as
 *   published by the Free Software Foundation.
 *
 *   BSD LICENSE
 *
 *   Copyright(c) 2012 Intel Corporation. All rights reserved.
17
 *   Copyright (C) 2015 EMC Corporation. All Rights Reserved.
18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44
 *
 *   Redistribution and use in source and binary forms, with or without
 *   modification, are permitted provided that the following conditions
 *   are met:
 *
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copy
 *       notice, this list of conditions and the following disclaimer in
 *       the documentation and/or other materials provided with the
 *       distribution.
 *     * Neither the name of Intel Corporation nor the names of its
 *       contributors may be used to endorse or promote products derived
 *       from this software without specific prior written permission.
 *
 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
45
 * PCIe NTB Transport Linux driver
46 47 48 49 50 51
 *
 * Contact Information:
 * Jon Mason <jon.mason@intel.com>
 */
#include <linux/debugfs.h>
#include <linux/delay.h>
52
#include <linux/dmaengine.h>
53 54 55 56 57 58 59 60
#include <linux/dma-mapping.h>
#include <linux/errno.h>
#include <linux/export.h>
#include <linux/interrupt.h>
#include <linux/module.h>
#include <linux/pci.h>
#include <linux/slab.h>
#include <linux/types.h>
61
#include <linux/uaccess.h>
62 63
#include "linux/ntb.h"
#include "linux/ntb_transport.h"
64

65 66 67 68 69 70 71 72 73 74 75 76 77
#define NTB_TRANSPORT_VERSION	4
#define NTB_TRANSPORT_VER	"4"
#define NTB_TRANSPORT_NAME	"ntb_transport"
#define NTB_TRANSPORT_DESC	"Software Queue-Pair Transport over NTB"

MODULE_DESCRIPTION(NTB_TRANSPORT_DESC);
MODULE_VERSION(NTB_TRANSPORT_VER);
MODULE_LICENSE("Dual BSD/GPL");
MODULE_AUTHOR("Intel Corporation");

static unsigned long max_mw_size;
module_param(max_mw_size, ulong, 0644);
MODULE_PARM_DESC(max_mw_size, "Limit size of large memory windows");
78

79
static unsigned int transport_mtu = 0x10000;
80 81 82
module_param(transport_mtu, uint, 0644);
MODULE_PARM_DESC(transport_mtu, "Maximum size of NTB transport packets");

Jon Mason's avatar
Jon Mason committed
83
static unsigned char max_num_clients;
84 85 86
module_param(max_num_clients, byte, 0644);
MODULE_PARM_DESC(max_num_clients, "Maximum number of NTB transport clients");

87 88 89 90
static unsigned int copy_bytes = 1024;
module_param(copy_bytes, uint, 0644);
MODULE_PARM_DESC(copy_bytes, "Threshold under which NTB will use the CPU to copy instead of DMA");

91 92 93 94
static bool use_dma;
module_param(use_dma, bool, 0644);
MODULE_PARM_DESC(use_dma, "Use DMA engine to perform large data copy");

95 96
static struct dentry *nt_debugfs_dir;

97 98 99
struct ntb_queue_entry {
	/* ntb_queue list reference */
	struct list_head entry;
100
	/* pointers to data to be transferred */
101 102 103 104
	void *cb_data;
	void *buf;
	unsigned int len;
	unsigned int flags;
105 106 107 108 109 110 111

	struct ntb_transport_qp *qp;
	union {
		struct ntb_payload_header __iomem *tx_hdr;
		struct ntb_payload_header *rx_hdr;
	};
	unsigned int index;
112 113
};

Jon Mason's avatar
Jon Mason committed
114 115 116 117
struct ntb_rx_info {
	unsigned int entry;
};

118
struct ntb_transport_qp {
119 120
	struct ntb_transport_ctx *transport;
	struct ntb_dev *ndev;
121
	void *cb_data;
122
	struct dma_chan *dma_chan;
123 124

	bool client_ready;
125 126
	bool link_is_up;

127
	u8 qp_num;	/* Only 64 QP's are allowed.  0-63 */
128
	u64 qp_bit;
129

Jon Mason's avatar
Jon Mason committed
130
	struct ntb_rx_info __iomem *rx_info;
Jon Mason's avatar
Jon Mason committed
131 132
	struct ntb_rx_info *remote_rx_info;

Jon Mason's avatar
Jon Mason committed
133 134
	void (*tx_handler)(struct ntb_transport_qp *qp, void *qp_data,
			   void *data, int len);
135 136
	struct list_head tx_free_q;
	spinlock_t ntb_tx_free_q_lock;
Jon Mason's avatar
Jon Mason committed
137
	void __iomem *tx_mw;
138
	dma_addr_t tx_mw_phys;
Jon Mason's avatar
Jon Mason committed
139 140
	unsigned int tx_index;
	unsigned int tx_max_entry;
141
	unsigned int tx_max_frame;
142

Jon Mason's avatar
Jon Mason committed
143 144
	void (*rx_handler)(struct ntb_transport_qp *qp, void *qp_data,
			   void *data, int len);
145
	struct list_head rx_post_q;
146 147
	struct list_head rx_pend_q;
	struct list_head rx_free_q;
148 149
	/* ntb_rx_q_lock: synchronize access to rx_XXXX_q */
	spinlock_t ntb_rx_q_lock;
Jon Mason's avatar
Jon Mason committed
150 151 152
	void *rx_buff;
	unsigned int rx_index;
	unsigned int rx_max_entry;
153
	unsigned int rx_max_frame;
154
	dma_cookie_t last_cookie;
155
	struct tasklet_struct rxc_db_work;
156

Jon Mason's avatar
Jon Mason committed
157
	void (*event_handler)(void *data, int status);
158
	struct delayed_work link_work;
159
	struct work_struct link_cleanup;
160 161 162 163 164 165 166 167 168 169 170

	struct dentry *debugfs_dir;
	struct dentry *debugfs_stats;

	/* Stats */
	u64 rx_bytes;
	u64 rx_pkts;
	u64 rx_ring_empty;
	u64 rx_err_no_buf;
	u64 rx_err_oflow;
	u64 rx_err_ver;
171 172
	u64 rx_memcpy;
	u64 rx_async;
173 174 175
	u64 tx_bytes;
	u64 tx_pkts;
	u64 tx_ring_full;
176 177 178
	u64 tx_err_no_buf;
	u64 tx_memcpy;
	u64 tx_async;
179 180 181
};

struct ntb_transport_mw {
182 183 184 185 186 187 188
	phys_addr_t phys_addr;
	resource_size_t phys_size;
	resource_size_t xlat_align;
	resource_size_t xlat_align_size;
	void __iomem *vbase;
	size_t xlat_size;
	size_t buff_size;
189 190 191 192 193 194
	void *virt_addr;
	dma_addr_t dma_addr;
};

struct ntb_transport_client_dev {
	struct list_head entry;
195
	struct ntb_transport_ctx *nt;
196 197 198
	struct device dev;
};

199
struct ntb_transport_ctx {
200 201 202
	struct list_head entry;
	struct list_head client_devs;

203 204 205 206 207 208 209 210 211 212
	struct ntb_dev *ndev;

	struct ntb_transport_mw *mw_vec;
	struct ntb_transport_qp *qp_vec;
	unsigned int mw_count;
	unsigned int qp_count;
	u64 qp_bitmap;
	u64 qp_bitmap_free;

	bool link_is_up;
213
	struct delayed_work link_work;
214
	struct work_struct link_cleanup;
215 216 217
};

enum {
218 219
	DESC_DONE_FLAG = BIT(0),
	LINK_DOWN_FLAG = BIT(1),
220 221 222
};

struct ntb_payload_header {
Jon Mason's avatar
Jon Mason committed
223
	unsigned int ver;
224 225 226 227 228 229 230
	unsigned int len;
	unsigned int flags;
};

enum {
	VERSION = 0,
	QP_LINKS,
Jon Mason's avatar
Jon Mason committed
231 232 233 234 235 236
	NUM_QPS,
	NUM_MWS,
	MW0_SZ_HIGH,
	MW0_SZ_LOW,
	MW1_SZ_HIGH,
	MW1_SZ_LOW,
237 238 239
	MAX_SPAD,
};

240 241 242 243 244 245 246
#define dev_client_dev(__dev) \
	container_of((__dev), struct ntb_transport_client_dev, dev)

#define drv_client(__drv) \
	container_of((__drv), struct ntb_transport_client, driver)

#define QP_TO_MW(nt, qp)	((qp) % nt->mw_count)
247 248 249
#define NTB_QP_DEF_NUM_ENTRIES	100
#define NTB_LINK_DOWN_TIMEOUT	10

250 251 252 253 254 255
static void ntb_transport_rxc_db(unsigned long data);
static const struct ntb_ctx_ops ntb_transport_ops;
static struct ntb_client ntb_transport_client;

static int ntb_transport_bus_match(struct device *dev,
				   struct device_driver *drv)
256 257 258 259
{
	return !strncmp(dev_name(dev), drv->name, strlen(drv->name));
}

260
static int ntb_transport_bus_probe(struct device *dev)
261
{
262
	const struct ntb_transport_client *client;
263 264 265
	int rc = -EINVAL;

	get_device(dev);
266 267 268

	client = drv_client(dev->driver);
	rc = client->probe(dev);
269 270 271 272 273 274
	if (rc)
		put_device(dev);

	return rc;
}

275
static int ntb_transport_bus_remove(struct device *dev)
276
{
277
	const struct ntb_transport_client *client;
278

279 280
	client = drv_client(dev->driver);
	client->remove(dev);
281 282 283 284 285 286

	put_device(dev);

	return 0;
}

287 288 289 290 291
static struct bus_type ntb_transport_bus = {
	.name = "ntb_transport",
	.match = ntb_transport_bus_match,
	.probe = ntb_transport_bus_probe,
	.remove = ntb_transport_bus_remove,
292 293 294 295
};

static LIST_HEAD(ntb_transport_list);

296
static int ntb_bus_init(struct ntb_transport_ctx *nt)
297 298 299 300 301
{
	list_add(&nt->entry, &ntb_transport_list);
	return 0;
}

302
static void ntb_bus_remove(struct ntb_transport_ctx *nt)
303 304 305 306 307 308 309 310 311 312 313 314 315
{
	struct ntb_transport_client_dev *client_dev, *cd;

	list_for_each_entry_safe(client_dev, cd, &nt->client_devs, entry) {
		dev_err(client_dev->dev.parent, "%s still attached to bus, removing\n",
			dev_name(&client_dev->dev));
		list_del(&client_dev->entry);
		device_unregister(&client_dev->dev);
	}

	list_del(&nt->entry);
}

316
static void ntb_transport_client_release(struct device *dev)
317 318 319
{
	struct ntb_transport_client_dev *client_dev;

320
	client_dev = dev_client_dev(dev);
321 322 323 324
	kfree(client_dev);
}

/**
325
 * ntb_transport_unregister_client_dev - Unregister NTB client device
326 327 328 329
 * @device_name: Name of NTB client device
 *
 * Unregister an NTB client device with the NTB transport layer
 */
330
void ntb_transport_unregister_client_dev(char *device_name)
331 332
{
	struct ntb_transport_client_dev *client, *cd;
333
	struct ntb_transport_ctx *nt;
334 335 336 337 338 339 340 341 342

	list_for_each_entry(nt, &ntb_transport_list, entry)
		list_for_each_entry_safe(client, cd, &nt->client_devs, entry)
			if (!strncmp(dev_name(&client->dev), device_name,
				     strlen(device_name))) {
				list_del(&client->entry);
				device_unregister(&client->dev);
			}
}
343
EXPORT_SYMBOL_GPL(ntb_transport_unregister_client_dev);
344 345

/**
346
 * ntb_transport_register_client_dev - Register NTB client device
347 348 349 350
 * @device_name: Name of NTB client device
 *
 * Register an NTB client device with the NTB transport layer
 */
351
int ntb_transport_register_client_dev(char *device_name)
352 353
{
	struct ntb_transport_client_dev *client_dev;
354
	struct ntb_transport_ctx *nt;
355
	int node;
Jon Mason's avatar
Jon Mason committed
356
	int rc, i = 0;
357

358 359 360
	if (list_empty(&ntb_transport_list))
		return -ENODEV;

361 362 363
	list_for_each_entry(nt, &ntb_transport_list, entry) {
		struct device *dev;

364 365 366 367
		node = dev_to_node(&nt->ndev->dev);

		client_dev = kzalloc_node(sizeof(*client_dev),
					  GFP_KERNEL, node);
368 369 370 371 372 373 374 375
		if (!client_dev) {
			rc = -ENOMEM;
			goto err;
		}

		dev = &client_dev->dev;

		/* setup and register client devices */
Jon Mason's avatar
Jon Mason committed
376
		dev_set_name(dev, "%s%d", device_name, i);
377 378 379
		dev->bus = &ntb_transport_bus;
		dev->release = ntb_transport_client_release;
		dev->parent = &nt->ndev->dev;
380 381 382 383 384 385 386 387

		rc = device_register(dev);
		if (rc) {
			kfree(client_dev);
			goto err;
		}

		list_add_tail(&client_dev->entry, &nt->client_devs);
Jon Mason's avatar
Jon Mason committed
388
		i++;
389 390 391 392 393
	}

	return 0;

err:
394
	ntb_transport_unregister_client_dev(device_name);
395 396 397

	return rc;
}
398
EXPORT_SYMBOL_GPL(ntb_transport_register_client_dev);
399 400

/**
401
 * ntb_transport_register_client - Register NTB client driver
402 403 404 405 406 407
 * @drv: NTB client driver to be registered
 *
 * Register an NTB client driver with the NTB transport layer
 *
 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
 */
408
int ntb_transport_register_client(struct ntb_transport_client *drv)
409
{
410
	drv->driver.bus = &ntb_transport_bus;
411

412 413 414
	if (list_empty(&ntb_transport_list))
		return -ENODEV;

415 416
	return driver_register(&drv->driver);
}
417
EXPORT_SYMBOL_GPL(ntb_transport_register_client);
418 419

/**
420
 * ntb_transport_unregister_client - Unregister NTB client driver
421 422 423 424 425 426
 * @drv: NTB client driver to be unregistered
 *
 * Unregister an NTB client driver with the NTB transport layer
 *
 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
 */
427
void ntb_transport_unregister_client(struct ntb_transport_client *drv)
428 429 430
{
	driver_unregister(&drv->driver);
}
431
EXPORT_SYMBOL_GPL(ntb_transport_unregister_client);
432 433 434 435 436

static ssize_t debugfs_read(struct file *filp, char __user *ubuf, size_t count,
			    loff_t *offp)
{
	struct ntb_transport_qp *qp;
437
	char *buf;
438 439
	ssize_t ret, out_offset, out_count;

440
	out_count = 1000;
441 442 443 444

	buf = kmalloc(out_count, GFP_KERNEL);
	if (!buf)
		return -ENOMEM;
445 446 447 448 449 450 451 452 453

	qp = filp->private_data;
	out_offset = 0;
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "NTB QP stats\n");
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_bytes - \t%llu\n", qp->rx_bytes);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_pkts - \t%llu\n", qp->rx_pkts);
454 455 456 457
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_memcpy - \t%llu\n", qp->rx_memcpy);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_async - \t%llu\n", qp->rx_async);
458 459 460 461 462 463 464 465 466
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_ring_empty - %llu\n", qp->rx_ring_empty);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_err_no_buf - %llu\n", qp->rx_err_no_buf);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_err_oflow - \t%llu\n", qp->rx_err_oflow);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_err_ver - \t%llu\n", qp->rx_err_ver);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
Jon Mason's avatar
Jon Mason committed
467
			       "rx_buff - \t%p\n", qp->rx_buff);
468
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
Jon Mason's avatar
Jon Mason committed
469
			       "rx_index - \t%u\n", qp->rx_index);
470
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
Jon Mason's avatar
Jon Mason committed
471
			       "rx_max_entry - \t%u\n", qp->rx_max_entry);
472 473 474 475 476

	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "tx_bytes - \t%llu\n", qp->tx_bytes);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "tx_pkts - \t%llu\n", qp->tx_pkts);
477 478 479 480
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "tx_memcpy - \t%llu\n", qp->tx_memcpy);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "tx_async - \t%llu\n", qp->tx_async);
481 482
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "tx_ring_full - \t%llu\n", qp->tx_ring_full);
483 484
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "tx_err_no_buf - %llu\n", qp->tx_err_no_buf);
485
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
Jon Mason's avatar
Jon Mason committed
486
			       "tx_mw - \t%p\n", qp->tx_mw);
487
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
Jon Mason's avatar
Jon Mason committed
488
			       "tx_index - \t%u\n", qp->tx_index);
489
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
Jon Mason's avatar
Jon Mason committed
490
			       "tx_max_entry - \t%u\n", qp->tx_max_entry);
491 492

	out_offset += snprintf(buf + out_offset, out_count - out_offset,
493 494
			       "\nQP Link %s\n",
			       qp->link_is_up ? "Up" : "Down");
495 496
	if (out_offset > out_count)
		out_offset = out_count;
497 498

	ret = simple_read_from_buffer(ubuf, count, offp, buf, out_offset);
499
	kfree(buf);
500 501 502 503 504
	return ret;
}

static const struct file_operations ntb_qp_debugfs_stats = {
	.owner = THIS_MODULE,
Jon Mason's avatar
Jon Mason committed
505
	.open = simple_open,
506 507 508 509 510 511 512 513 514 515 516 517 518 519
	.read = debugfs_read,
};

static void ntb_list_add(spinlock_t *lock, struct list_head *entry,
			 struct list_head *list)
{
	unsigned long flags;

	spin_lock_irqsave(lock, flags);
	list_add_tail(entry, list);
	spin_unlock_irqrestore(lock, flags);
}

static struct ntb_queue_entry *ntb_list_rm(spinlock_t *lock,
Jon Mason's avatar
Jon Mason committed
520
					   struct list_head *list)
521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537
{
	struct ntb_queue_entry *entry;
	unsigned long flags;

	spin_lock_irqsave(lock, flags);
	if (list_empty(list)) {
		entry = NULL;
		goto out;
	}
	entry = list_first_entry(list, struct ntb_queue_entry, entry);
	list_del(&entry->entry);
out:
	spin_unlock_irqrestore(lock, flags);

	return entry;
}

538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558
static struct ntb_queue_entry *ntb_list_mv(spinlock_t *lock,
					   struct list_head *list,
					   struct list_head *to_list)
{
	struct ntb_queue_entry *entry;
	unsigned long flags;

	spin_lock_irqsave(lock, flags);

	if (list_empty(list)) {
		entry = NULL;
	} else {
		entry = list_first_entry(list, struct ntb_queue_entry, entry);
		list_move_tail(&entry->entry, to_list);
	}

	spin_unlock_irqrestore(lock, flags);

	return entry;
}

559 560
static int ntb_transport_setup_qp_mw(struct ntb_transport_ctx *nt,
				     unsigned int qp_num)
561
{
562 563
	struct ntb_transport_qp *qp = &nt->qp_vec[qp_num];
	struct ntb_transport_mw *mw;
564
	unsigned int rx_size, num_qps_mw;
565
	unsigned int mw_num, mw_count, qp_count;
Jon Mason's avatar
Jon Mason committed
566
	unsigned int i;
567

568 569
	mw_count = nt->mw_count;
	qp_count = nt->qp_count;
Jon Mason's avatar
Jon Mason committed
570

571 572 573 574 575
	mw_num = QP_TO_MW(nt, qp_num);
	mw = &nt->mw_vec[mw_num];

	if (!mw->virt_addr)
		return -ENOMEM;
576

577 578
	if (qp_count % mw_count && mw_num + 1 < qp_count / mw_count)
		num_qps_mw = qp_count / mw_count + 1;
579
	else
580
		num_qps_mw = qp_count / mw_count;
581

582 583
	rx_size = (unsigned int)mw->xlat_size / num_qps_mw;
	qp->rx_buff = mw->virt_addr + rx_size * qp_num / mw_count;
Jon Mason's avatar
Jon Mason committed
584 585
	rx_size -= sizeof(struct ntb_rx_info);

586 587
	qp->remote_rx_info = qp->rx_buff + rx_size;

588 589
	/* Due to housekeeping, there must be atleast 2 buffs */
	qp->rx_max_frame = min(transport_mtu, rx_size / 2);
Jon Mason's avatar
Jon Mason committed
590 591 592
	qp->rx_max_entry = rx_size / qp->rx_max_frame;
	qp->rx_index = 0;

593
	qp->remote_rx_info->entry = qp->rx_max_entry - 1;
594

595
	/* setup the hdr offsets with 0's */
Jon Mason's avatar
Jon Mason committed
596
	for (i = 0; i < qp->rx_max_entry; i++) {
597 598
		void *offset = (qp->rx_buff + qp->rx_max_frame * (i + 1) -
				sizeof(struct ntb_payload_header));
599
		memset(offset, 0, sizeof(struct ntb_payload_header));
Jon Mason's avatar
Jon Mason committed
600
	}
601 602 603

	qp->rx_pkts = 0;
	qp->tx_pkts = 0;
Jon Mason's avatar
Jon Mason committed
604
	qp->tx_index = 0;
605 606

	return 0;
607 608
}

609
static void ntb_free_mw(struct ntb_transport_ctx *nt, int num_mw)
Jon Mason's avatar
Jon Mason committed
610
{
611 612
	struct ntb_transport_mw *mw = &nt->mw_vec[num_mw];
	struct pci_dev *pdev = nt->ndev->pdev;
Jon Mason's avatar
Jon Mason committed
613 614 615 616

	if (!mw->virt_addr)
		return;

617 618 619 620 621
	ntb_mw_clear_trans(nt->ndev, num_mw);
	dma_free_coherent(&pdev->dev, mw->buff_size,
			  mw->virt_addr, mw->dma_addr);
	mw->xlat_size = 0;
	mw->buff_size = 0;
Jon Mason's avatar
Jon Mason committed
622 623 624
	mw->virt_addr = NULL;
}

625 626
static int ntb_set_mw(struct ntb_transport_ctx *nt, int num_mw,
		      unsigned int size)
627
{
628 629 630 631 632 633 634
	struct ntb_transport_mw *mw = &nt->mw_vec[num_mw];
	struct pci_dev *pdev = nt->ndev->pdev;
	unsigned int xlat_size, buff_size;
	int rc;

	xlat_size = round_up(size, mw->xlat_align_size);
	buff_size = round_up(size, mw->xlat_align);
635

Jon Mason's avatar
Jon Mason committed
636
	/* No need to re-setup */
637
	if (mw->xlat_size == xlat_size)
Jon Mason's avatar
Jon Mason committed
638 639
		return 0;

640
	if (mw->buff_size)
Jon Mason's avatar
Jon Mason committed
641 642
		ntb_free_mw(nt, num_mw);

643 644 645
	/* Alloc memory for receiving data.  Must be aligned */
	mw->xlat_size = xlat_size;
	mw->buff_size = buff_size;
646

647 648
	mw->virt_addr = dma_alloc_coherent(&pdev->dev, buff_size,
					   &mw->dma_addr, GFP_KERNEL);
649
	if (!mw->virt_addr) {
650 651 652 653
		mw->xlat_size = 0;
		mw->buff_size = 0;
		dev_err(&pdev->dev, "Unable to alloc MW buff of size %d\n",
			buff_size);
654 655 656
		return -ENOMEM;
	}

657 658 659 660 661 662
	/*
	 * we must ensure that the memory address allocated is BAR size
	 * aligned in order for the XLAT register to take the value. This
	 * is a requirement of the hardware. It is recommended to setup CMA
	 * for BAR sizes equal or greater than 4MB.
	 */
663 664
	if (!IS_ALIGNED(mw->dma_addr, mw->xlat_align)) {
		dev_err(&pdev->dev, "DMA memory %pad is not aligned\n",
665 666 667 668 669
			&mw->dma_addr);
		ntb_free_mw(nt, num_mw);
		return -ENOMEM;
	}

670
	/* Notify HW the memory location of the receive buffer */
671 672 673 674 675 676
	rc = ntb_mw_set_trans(nt->ndev, num_mw, mw->dma_addr, mw->xlat_size);
	if (rc) {
		dev_err(&pdev->dev, "Unable to set mw%d translation", num_mw);
		ntb_free_mw(nt, num_mw);
		return -EIO;
	}
677 678 679 680

	return 0;
}

681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702
static void ntb_qp_link_down_reset(struct ntb_transport_qp *qp)
{
	qp->link_is_up = false;

	qp->tx_index = 0;
	qp->rx_index = 0;
	qp->rx_bytes = 0;
	qp->rx_pkts = 0;
	qp->rx_ring_empty = 0;
	qp->rx_err_no_buf = 0;
	qp->rx_err_oflow = 0;
	qp->rx_err_ver = 0;
	qp->rx_memcpy = 0;
	qp->rx_async = 0;
	qp->tx_bytes = 0;
	qp->tx_pkts = 0;
	qp->tx_ring_full = 0;
	qp->tx_err_no_buf = 0;
	qp->tx_memcpy = 0;
	qp->tx_async = 0;
}

703
static void ntb_qp_link_cleanup(struct ntb_transport_qp *qp)
704
{
705 706
	struct ntb_transport_ctx *nt = qp->transport;
	struct pci_dev *pdev = nt->ndev->pdev;
707

708
	dev_info(&pdev->dev, "qp %d: Link Cleanup\n", qp->qp_num);
709 710 711

	cancel_delayed_work_sync(&qp->link_work);
	ntb_qp_link_down_reset(qp);
712 713 714

	if (qp->event_handler)
		qp->event_handler(qp->cb_data, qp->link_is_up);
715 716 717 718 719 720 721
}

static void ntb_qp_link_cleanup_work(struct work_struct *work)
{
	struct ntb_transport_qp *qp = container_of(work,
						   struct ntb_transport_qp,
						   link_cleanup);
722
	struct ntb_transport_ctx *nt = qp->transport;
723 724

	ntb_qp_link_cleanup(qp);
725

726
	if (nt->link_is_up)
727 728 729 730
		schedule_delayed_work(&qp->link_work,
				      msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT));
}

731 732 733 734 735
static void ntb_qp_link_down(struct ntb_transport_qp *qp)
{
	schedule_work(&qp->link_cleanup);
}

736
static void ntb_transport_link_cleanup(struct ntb_transport_ctx *nt)
737
{
738 739
	struct ntb_transport_qp *qp;
	u64 qp_bitmap_alloc;
740 741
	int i;

742 743
	qp_bitmap_alloc = nt->qp_bitmap & ~nt->qp_bitmap_free;

744
	/* Pass along the info to any clients */
745 746 747 748 749 750 751
	for (i = 0; i < nt->qp_count; i++)
		if (qp_bitmap_alloc & BIT_ULL(i)) {
			qp = &nt->qp_vec[i];
			ntb_qp_link_cleanup(qp);
			cancel_work_sync(&qp->link_cleanup);
			cancel_delayed_work_sync(&qp->link_work);
		}
752

753
	if (!nt->link_is_up)
754 755 756 757 758 759 760
		cancel_delayed_work_sync(&nt->link_work);

	/* The scratchpad registers keep the values if the remote side
	 * goes down, blast them now to give them a sane value the next
	 * time they are accessed
	 */
	for (i = 0; i < MAX_SPAD; i++)
761
		ntb_spad_write(nt->ndev, i, 0);
762 763
}

764 765
static void ntb_transport_link_cleanup_work(struct work_struct *work)
{
766 767
	struct ntb_transport_ctx *nt =
		container_of(work, struct ntb_transport_ctx, link_cleanup);
768 769 770 771

	ntb_transport_link_cleanup(nt);
}

772
static void ntb_transport_event_callback(void *data)
773
{
774
	struct ntb_transport_ctx *nt = data;
775

776
	if (ntb_link_is_up(nt->ndev, NULL, NULL) == 1)
777
		schedule_delayed_work(&nt->link_work, 0);
778
	else
779
		schedule_work(&nt->link_cleanup);
780 781 782 783
}

static void ntb_transport_link_work(struct work_struct *work)
{
784 785 786 787 788
	struct ntb_transport_ctx *nt =
		container_of(work, struct ntb_transport_ctx, link_work.work);
	struct ntb_dev *ndev = nt->ndev;
	struct pci_dev *pdev = ndev->pdev;
	resource_size_t size;
789
	u32 val;
790
	int rc, i, spad;
791

Jon Mason's avatar
Jon Mason committed
792
	/* send the local info, in the opposite order of the way we read it */
793 794
	for (i = 0; i < nt->mw_count; i++) {
		size = nt->mw_vec[i].phys_size;
795

796 797
		if (max_mw_size && size > max_mw_size)
			size = max_mw_size;
798

799 800
		spad = MW0_SZ_HIGH + (i * 2);
		ntb_peer_spad_write(ndev, spad, (u32)(size >> 32));
801

802 803
		spad = MW0_SZ_LOW + (i * 2);
		ntb_peer_spad_write(ndev, spad, (u32)size);
804 805
	}

806
	ntb_peer_spad_write(ndev, NUM_MWS, nt->mw_count);
807

808
	ntb_peer_spad_write(ndev, NUM_QPS, nt->qp_count);
809

810
	ntb_peer_spad_write(ndev, VERSION, NTB_TRANSPORT_VERSION);
811

812
	/* Query the remote side for its info */
813
	val = ntb_spad_read(ndev, VERSION);
814 815
	dev_dbg(&pdev->dev, "Remote version = %d\n", val);
	if (val != NTB_TRANSPORT_VERSION)
816 817
		goto out;

818
	val = ntb_spad_read(ndev, NUM_QPS);
819
	dev_dbg(&pdev->dev, "Remote max number of qps = %d\n", val);
820
	if (val != nt->qp_count)
821 822
		goto out;

823
	val = ntb_spad_read(ndev, NUM_MWS);
Jon Mason's avatar
Jon Mason committed
824
	dev_dbg(&pdev->dev, "Remote number of mws = %d\n", val);
825 826
	if (val != nt->mw_count)
		goto out;
827

828
	for (i = 0; i < nt->mw_count; i++) {
Jon Mason's avatar
Jon Mason committed
829
		u64 val64;
830

831
		val = ntb_spad_read(ndev, MW0_SZ_HIGH + (i * 2));
832
		val64 = (u64)val << 32;
Jon Mason's avatar
Jon Mason committed
833

834
		val = ntb_spad_read(ndev, MW0_SZ_LOW + (i * 2));
Jon Mason's avatar
Jon Mason committed
835 836
		val64 |= val;

837
		dev_dbg(&pdev->dev, "Remote MW%d size = %#llx\n", i, val64);
Jon Mason's avatar
Jon Mason committed
838 839 840 841 842

		rc = ntb_set_mw(nt, i, val64);
		if (rc)
			goto out1;
	}
843

844
	nt->link_is_up = true;
845

846 847
	for (i = 0; i < nt->qp_count; i++) {
		struct ntb_transport_qp *qp = &nt->qp_vec[i];
848 849 850

		ntb_transport_setup_qp_mw(nt, i);

851
		if (qp->client_ready)
852 853 854 855 856
			schedule_delayed_work(&qp->link_work, 0);
	}

	return;

Jon Mason's avatar
Jon Mason committed
857
out1:
858
	for (i = 0; i < nt->mw_count; i++)
Jon Mason's avatar
Jon Mason committed
859
		ntb_free_mw(nt, i);
860
out:
861
	if (ntb_link_is_up(ndev, NULL, NULL) == 1)
862 863 864 865 866 867 868 869 870
		schedule_delayed_work(&nt->link_work,
				      msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT));
}

static void ntb_qp_link_work(struct work_struct *work)
{
	struct ntb_transport_qp *qp = container_of(work,
						   struct ntb_transport_qp,
						   link_work.work);
871 872 873
	struct pci_dev *pdev = qp->ndev->pdev;
	struct ntb_transport_ctx *nt = qp->transport;
	int val;
874

875
	WARN_ON(!nt->link_is_up);
876

877
	val = ntb_spad_read(nt->ndev, QP_LINKS);
878

879
	ntb_peer_spad_write(nt->ndev, QP_LINKS, val | BIT(qp->qp_num));
880 881

	/* query remote spad for qp ready bits */
882
	ntb_peer_spad_read(nt->ndev, QP_LINKS);
Allen Hubbe's avatar
Allen Hubbe committed
883
	dev_dbg_ratelimited(&pdev->dev, "Remote QP link status = %x\n", val);
884 885

	/* See if the remote side is up */
886
	if (val & BIT(qp->qp_num)) {
887
		dev_info(&pdev->dev, "qp %d: Link Up\n", qp->qp_num);
888 889
		qp->link_is_up = true;

890
		if (qp->event_handler)
891 892
			qp->event_handler(qp->cb_data, qp->link_is_up);
	} else if (nt->link_is_up)
893 894 895 896
		schedule_delayed_work(&qp->link_work,
				      msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT));
}

897
static int ntb_transport_init_queue(struct ntb_transport_ctx *nt,
Jon Mason's avatar
Jon Mason committed
898
				    unsigned int qp_num)
899 900
{
	struct ntb_transport_qp *qp;
901 902 903
	struct ntb_transport_mw *mw;
	phys_addr_t mw_base;
	resource_size_t mw_size;
904
	unsigned int num_qps_mw, tx_size;
905
	unsigned int mw_num, mw_count, qp_count;
906
	u64 qp_offset;
Jon Mason's avatar
Jon Mason committed
907

908 909
	mw_count = nt->mw_count;
	qp_count = nt->qp_count;
910

911 912 913 914
	mw_num = QP_TO_MW(nt, qp_num);
	mw = &nt->mw_vec[mw_num];

	qp = &nt->qp_vec[qp_num];
915 916 917
	qp->qp_num = qp_num;
	qp->transport = nt;
	qp->ndev = nt->ndev;
918
	qp->client_ready = false;
919
	qp->event_handler = NULL;
920
	ntb_qp_link_down_reset(qp);
921

922 923
	if (qp_count % mw_count && mw_num + 1 < qp_count / mw_count)
		num_qps_mw = qp_count / mw_count + 1;
924
	else
925 926 927 928
		num_qps_mw = qp_count / mw_count;

	mw_base = nt->mw_vec[mw_num].phys_addr;
	mw_size = nt->mw_vec[mw_num].phys_size;