ntb_transport.c 48.5 KB
Newer Older
1 2 3 4 5 6 7
/*
 * This file is provided under a dual BSD/GPLv2 license.  When using or
 *   redistributing this file, you may do so under either license.
 *
 *   GPL LICENSE SUMMARY
 *
 *   Copyright(c) 2012 Intel Corporation. All rights reserved.
8
 *   Copyright (C) 2015 EMC Corporation. All Rights Reserved.
9 10 11 12 13 14 15 16
 *
 *   This program is free software; you can redistribute it and/or modify
 *   it under the terms of version 2 of the GNU General Public License as
 *   published by the Free Software Foundation.
 *
 *   BSD LICENSE
 *
 *   Copyright(c) 2012 Intel Corporation. All rights reserved.
17
 *   Copyright (C) 2015 EMC Corporation. All Rights Reserved.
18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44
 *
 *   Redistribution and use in source and binary forms, with or without
 *   modification, are permitted provided that the following conditions
 *   are met:
 *
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copy
 *       notice, this list of conditions and the following disclaimer in
 *       the documentation and/or other materials provided with the
 *       distribution.
 *     * Neither the name of Intel Corporation nor the names of its
 *       contributors may be used to endorse or promote products derived
 *       from this software without specific prior written permission.
 *
 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
45
 * PCIe NTB Transport Linux driver
46 47 48 49 50 51
 *
 * Contact Information:
 * Jon Mason <jon.mason@intel.com>
 */
#include <linux/debugfs.h>
#include <linux/delay.h>
52
#include <linux/dmaengine.h>
53 54 55 56 57 58 59 60
#include <linux/dma-mapping.h>
#include <linux/errno.h>
#include <linux/export.h>
#include <linux/interrupt.h>
#include <linux/module.h>
#include <linux/pci.h>
#include <linux/slab.h>
#include <linux/types.h>
61 62
#include "linux/ntb.h"
#include "linux/ntb_transport.h"
63

64 65 66 67 68 69 70 71 72 73 74 75 76
#define NTB_TRANSPORT_VERSION	4
#define NTB_TRANSPORT_VER	"4"
#define NTB_TRANSPORT_NAME	"ntb_transport"
#define NTB_TRANSPORT_DESC	"Software Queue-Pair Transport over NTB"

MODULE_DESCRIPTION(NTB_TRANSPORT_DESC);
MODULE_VERSION(NTB_TRANSPORT_VER);
MODULE_LICENSE("Dual BSD/GPL");
MODULE_AUTHOR("Intel Corporation");

static unsigned long max_mw_size;
module_param(max_mw_size, ulong, 0644);
MODULE_PARM_DESC(max_mw_size, "Limit size of large memory windows");
77

78
static unsigned int transport_mtu = 0x401E;
79 80 81
module_param(transport_mtu, uint, 0644);
MODULE_PARM_DESC(transport_mtu, "Maximum size of NTB transport packets");

Jon Mason's avatar
Jon Mason committed
82
static unsigned char max_num_clients;
83 84 85
module_param(max_num_clients, byte, 0644);
MODULE_PARM_DESC(max_num_clients, "Maximum number of NTB transport clients");

86 87 88 89
static unsigned int copy_bytes = 1024;
module_param(copy_bytes, uint, 0644);
MODULE_PARM_DESC(copy_bytes, "Threshold under which NTB will use the CPU to copy instead of DMA");

90 91
static struct dentry *nt_debugfs_dir;

92 93 94
struct ntb_queue_entry {
	/* ntb_queue list reference */
	struct list_head entry;
95
	/* pointers to data to be transferred */
96 97 98 99
	void *cb_data;
	void *buf;
	unsigned int len;
	unsigned int flags;
100 101 102 103 104 105 106

	struct ntb_transport_qp *qp;
	union {
		struct ntb_payload_header __iomem *tx_hdr;
		struct ntb_payload_header *rx_hdr;
	};
	unsigned int index;
107 108
};

Jon Mason's avatar
Jon Mason committed
109 110 111 112
struct ntb_rx_info {
	unsigned int entry;
};

113
struct ntb_transport_qp {
114 115
	struct ntb_transport_ctx *transport;
	struct ntb_dev *ndev;
116
	void *cb_data;
117
	struct dma_chan *dma_chan;
118 119

	bool client_ready;
120 121
	bool link_is_up;

122
	u8 qp_num;	/* Only 64 QP's are allowed.  0-63 */
123
	u64 qp_bit;
124

Jon Mason's avatar
Jon Mason committed
125
	struct ntb_rx_info __iomem *rx_info;
Jon Mason's avatar
Jon Mason committed
126 127
	struct ntb_rx_info *remote_rx_info;

Jon Mason's avatar
Jon Mason committed
128 129
	void (*tx_handler)(struct ntb_transport_qp *qp, void *qp_data,
			   void *data, int len);
130 131
	struct list_head tx_free_q;
	spinlock_t ntb_tx_free_q_lock;
Jon Mason's avatar
Jon Mason committed
132
	void __iomem *tx_mw;
133
	dma_addr_t tx_mw_phys;
Jon Mason's avatar
Jon Mason committed
134 135
	unsigned int tx_index;
	unsigned int tx_max_entry;
136
	unsigned int tx_max_frame;
137

Jon Mason's avatar
Jon Mason committed
138 139
	void (*rx_handler)(struct ntb_transport_qp *qp, void *qp_data,
			   void *data, int len);
140 141 142 143
	struct list_head rx_pend_q;
	struct list_head rx_free_q;
	spinlock_t ntb_rx_pend_q_lock;
	spinlock_t ntb_rx_free_q_lock;
Jon Mason's avatar
Jon Mason committed
144 145 146
	void *rx_buff;
	unsigned int rx_index;
	unsigned int rx_max_entry;
147
	unsigned int rx_max_frame;
148
	dma_cookie_t last_cookie;
149
	struct tasklet_struct rxc_db_work;
150

Jon Mason's avatar
Jon Mason committed
151
	void (*event_handler)(void *data, int status);
152
	struct delayed_work link_work;
153
	struct work_struct link_cleanup;
154 155 156 157 158 159 160 161 162 163 164

	struct dentry *debugfs_dir;
	struct dentry *debugfs_stats;

	/* Stats */
	u64 rx_bytes;
	u64 rx_pkts;
	u64 rx_ring_empty;
	u64 rx_err_no_buf;
	u64 rx_err_oflow;
	u64 rx_err_ver;
165 166
	u64 rx_memcpy;
	u64 rx_async;
167 168 169
	u64 tx_bytes;
	u64 tx_pkts;
	u64 tx_ring_full;
170 171 172
	u64 tx_err_no_buf;
	u64 tx_memcpy;
	u64 tx_async;
173 174 175
};

struct ntb_transport_mw {
176 177 178 179 180 181 182
	phys_addr_t phys_addr;
	resource_size_t phys_size;
	resource_size_t xlat_align;
	resource_size_t xlat_align_size;
	void __iomem *vbase;
	size_t xlat_size;
	size_t buff_size;
183 184 185 186 187 188
	void *virt_addr;
	dma_addr_t dma_addr;
};

struct ntb_transport_client_dev {
	struct list_head entry;
189
	struct ntb_transport_ctx *nt;
190 191 192
	struct device dev;
};

193
struct ntb_transport_ctx {
194 195 196
	struct list_head entry;
	struct list_head client_devs;

197 198 199 200 201 202 203 204 205 206
	struct ntb_dev *ndev;

	struct ntb_transport_mw *mw_vec;
	struct ntb_transport_qp *qp_vec;
	unsigned int mw_count;
	unsigned int qp_count;
	u64 qp_bitmap;
	u64 qp_bitmap_free;

	bool link_is_up;
207
	struct delayed_work link_work;
208
	struct work_struct link_cleanup;
209 210 211
};

enum {
212 213
	DESC_DONE_FLAG = BIT(0),
	LINK_DOWN_FLAG = BIT(1),
214 215 216
};

struct ntb_payload_header {
Jon Mason's avatar
Jon Mason committed
217
	unsigned int ver;
218 219 220 221 222 223 224
	unsigned int len;
	unsigned int flags;
};

enum {
	VERSION = 0,
	QP_LINKS,
Jon Mason's avatar
Jon Mason committed
225 226 227 228 229 230
	NUM_QPS,
	NUM_MWS,
	MW0_SZ_HIGH,
	MW0_SZ_LOW,
	MW1_SZ_HIGH,
	MW1_SZ_LOW,
231 232 233
	MAX_SPAD,
};

234 235 236 237 238 239 240
#define dev_client_dev(__dev) \
	container_of((__dev), struct ntb_transport_client_dev, dev)

#define drv_client(__drv) \
	container_of((__drv), struct ntb_transport_client, driver)

#define QP_TO_MW(nt, qp)	((qp) % nt->mw_count)
241 242 243
#define NTB_QP_DEF_NUM_ENTRIES	100
#define NTB_LINK_DOWN_TIMEOUT	10

244 245 246 247 248 249
static void ntb_transport_rxc_db(unsigned long data);
static const struct ntb_ctx_ops ntb_transport_ops;
static struct ntb_client ntb_transport_client;

static int ntb_transport_bus_match(struct device *dev,
				   struct device_driver *drv)
250 251 252 253
{
	return !strncmp(dev_name(dev), drv->name, strlen(drv->name));
}

254
static int ntb_transport_bus_probe(struct device *dev)
255
{
256
	const struct ntb_transport_client *client;
257 258 259
	int rc = -EINVAL;

	get_device(dev);
260 261 262

	client = drv_client(dev->driver);
	rc = client->probe(dev);
263 264 265 266 267 268
	if (rc)
		put_device(dev);

	return rc;
}

269
static int ntb_transport_bus_remove(struct device *dev)
270
{
271
	const struct ntb_transport_client *client;
272

273 274
	client = drv_client(dev->driver);
	client->remove(dev);
275 276 277 278 279 280

	put_device(dev);

	return 0;
}

281 282 283 284 285
static struct bus_type ntb_transport_bus = {
	.name = "ntb_transport",
	.match = ntb_transport_bus_match,
	.probe = ntb_transport_bus_probe,
	.remove = ntb_transport_bus_remove,
286 287 288 289
};

static LIST_HEAD(ntb_transport_list);

290
static int ntb_bus_init(struct ntb_transport_ctx *nt)
291 292 293 294 295
{
	list_add(&nt->entry, &ntb_transport_list);
	return 0;
}

296
static void ntb_bus_remove(struct ntb_transport_ctx *nt)
297 298 299 300 301 302 303 304 305 306 307 308 309
{
	struct ntb_transport_client_dev *client_dev, *cd;

	list_for_each_entry_safe(client_dev, cd, &nt->client_devs, entry) {
		dev_err(client_dev->dev.parent, "%s still attached to bus, removing\n",
			dev_name(&client_dev->dev));
		list_del(&client_dev->entry);
		device_unregister(&client_dev->dev);
	}

	list_del(&nt->entry);
}

310
static void ntb_transport_client_release(struct device *dev)
311 312 313
{
	struct ntb_transport_client_dev *client_dev;

314
	client_dev = dev_client_dev(dev);
315 316 317 318
	kfree(client_dev);
}

/**
319
 * ntb_transport_unregister_client_dev - Unregister NTB client device
320 321 322 323
 * @device_name: Name of NTB client device
 *
 * Unregister an NTB client device with the NTB transport layer
 */
324
void ntb_transport_unregister_client_dev(char *device_name)
325 326
{
	struct ntb_transport_client_dev *client, *cd;
327
	struct ntb_transport_ctx *nt;
328 329 330 331 332 333 334 335 336

	list_for_each_entry(nt, &ntb_transport_list, entry)
		list_for_each_entry_safe(client, cd, &nt->client_devs, entry)
			if (!strncmp(dev_name(&client->dev), device_name,
				     strlen(device_name))) {
				list_del(&client->entry);
				device_unregister(&client->dev);
			}
}
337
EXPORT_SYMBOL_GPL(ntb_transport_unregister_client_dev);
338 339

/**
340
 * ntb_transport_register_client_dev - Register NTB client device
341 342 343 344
 * @device_name: Name of NTB client device
 *
 * Register an NTB client device with the NTB transport layer
 */
345
int ntb_transport_register_client_dev(char *device_name)
346 347
{
	struct ntb_transport_client_dev *client_dev;
348
	struct ntb_transport_ctx *nt;
Jon Mason's avatar
Jon Mason committed
349
	int rc, i = 0;
350

351 352 353
	if (list_empty(&ntb_transport_list))
		return -ENODEV;

354 355 356
	list_for_each_entry(nt, &ntb_transport_list, entry) {
		struct device *dev;

357
		client_dev = kzalloc(sizeof(*client_dev),
358 359 360 361 362 363 364 365 366
				     GFP_KERNEL);
		if (!client_dev) {
			rc = -ENOMEM;
			goto err;
		}

		dev = &client_dev->dev;

		/* setup and register client devices */
Jon Mason's avatar
Jon Mason committed
367
		dev_set_name(dev, "%s%d", device_name, i);
368 369 370
		dev->bus = &ntb_transport_bus;
		dev->release = ntb_transport_client_release;
		dev->parent = &nt->ndev->dev;
371 372 373 374 375 376 377 378

		rc = device_register(dev);
		if (rc) {
			kfree(client_dev);
			goto err;
		}

		list_add_tail(&client_dev->entry, &nt->client_devs);
Jon Mason's avatar
Jon Mason committed
379
		i++;
380 381 382 383 384
	}

	return 0;

err:
385
	ntb_transport_unregister_client_dev(device_name);
386 387 388

	return rc;
}
389
EXPORT_SYMBOL_GPL(ntb_transport_register_client_dev);
390 391

/**
392
 * ntb_transport_register_client - Register NTB client driver
393 394 395 396 397 398
 * @drv: NTB client driver to be registered
 *
 * Register an NTB client driver with the NTB transport layer
 *
 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
 */
399
int ntb_transport_register_client(struct ntb_transport_client *drv)
400
{
401
	drv->driver.bus = &ntb_transport_bus;
402

403 404 405
	if (list_empty(&ntb_transport_list))
		return -ENODEV;

406 407
	return driver_register(&drv->driver);
}
408
EXPORT_SYMBOL_GPL(ntb_transport_register_client);
409 410

/**
411
 * ntb_transport_unregister_client - Unregister NTB client driver
412 413 414 415 416 417
 * @drv: NTB client driver to be unregistered
 *
 * Unregister an NTB client driver with the NTB transport layer
 *
 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
 */
418
void ntb_transport_unregister_client(struct ntb_transport_client *drv)
419 420 421
{
	driver_unregister(&drv->driver);
}
422
EXPORT_SYMBOL_GPL(ntb_transport_unregister_client);
423 424 425 426 427

static ssize_t debugfs_read(struct file *filp, char __user *ubuf, size_t count,
			    loff_t *offp)
{
	struct ntb_transport_qp *qp;
428
	char *buf;
429 430
	ssize_t ret, out_offset, out_count;

431
	out_count = 1000;
432 433 434 435

	buf = kmalloc(out_count, GFP_KERNEL);
	if (!buf)
		return -ENOMEM;
436 437 438 439 440 441 442 443 444

	qp = filp->private_data;
	out_offset = 0;
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "NTB QP stats\n");
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_bytes - \t%llu\n", qp->rx_bytes);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_pkts - \t%llu\n", qp->rx_pkts);
445 446 447 448
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_memcpy - \t%llu\n", qp->rx_memcpy);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_async - \t%llu\n", qp->rx_async);
449 450 451 452 453 454 455 456 457
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_ring_empty - %llu\n", qp->rx_ring_empty);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_err_no_buf - %llu\n", qp->rx_err_no_buf);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_err_oflow - \t%llu\n", qp->rx_err_oflow);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_err_ver - \t%llu\n", qp->rx_err_ver);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
Jon Mason's avatar
Jon Mason committed
458
			       "rx_buff - \t%p\n", qp->rx_buff);
459
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
Jon Mason's avatar
Jon Mason committed
460
			       "rx_index - \t%u\n", qp->rx_index);
461
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
Jon Mason's avatar
Jon Mason committed
462
			       "rx_max_entry - \t%u\n", qp->rx_max_entry);
463 464 465 466 467

	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "tx_bytes - \t%llu\n", qp->tx_bytes);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "tx_pkts - \t%llu\n", qp->tx_pkts);
468 469 470 471
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "tx_memcpy - \t%llu\n", qp->tx_memcpy);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "tx_async - \t%llu\n", qp->tx_async);
472 473
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "tx_ring_full - \t%llu\n", qp->tx_ring_full);
474 475
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "tx_err_no_buf - %llu\n", qp->tx_err_no_buf);
476
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
Jon Mason's avatar
Jon Mason committed
477
			       "tx_mw - \t%p\n", qp->tx_mw);
478
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
Jon Mason's avatar
Jon Mason committed
479
			       "tx_index - \t%u\n", qp->tx_index);
480
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
Jon Mason's avatar
Jon Mason committed
481
			       "tx_max_entry - \t%u\n", qp->tx_max_entry);
482 483

	out_offset += snprintf(buf + out_offset, out_count - out_offset,
484 485
			       "\nQP Link %s\n",
			       qp->link_is_up ? "Up" : "Down");
486 487
	if (out_offset > out_count)
		out_offset = out_count;
488 489

	ret = simple_read_from_buffer(ubuf, count, offp, buf, out_offset);
490
	kfree(buf);
491 492 493 494 495
	return ret;
}

static const struct file_operations ntb_qp_debugfs_stats = {
	.owner = THIS_MODULE,
Jon Mason's avatar
Jon Mason committed
496
	.open = simple_open,
497 498 499 500 501 502 503 504 505 506 507 508 509 510
	.read = debugfs_read,
};

static void ntb_list_add(spinlock_t *lock, struct list_head *entry,
			 struct list_head *list)
{
	unsigned long flags;

	spin_lock_irqsave(lock, flags);
	list_add_tail(entry, list);
	spin_unlock_irqrestore(lock, flags);
}

static struct ntb_queue_entry *ntb_list_rm(spinlock_t *lock,
Jon Mason's avatar
Jon Mason committed
511
					   struct list_head *list)
512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528
{
	struct ntb_queue_entry *entry;
	unsigned long flags;

	spin_lock_irqsave(lock, flags);
	if (list_empty(list)) {
		entry = NULL;
		goto out;
	}
	entry = list_first_entry(list, struct ntb_queue_entry, entry);
	list_del(&entry->entry);
out:
	spin_unlock_irqrestore(lock, flags);

	return entry;
}

529 530
static int ntb_transport_setup_qp_mw(struct ntb_transport_ctx *nt,
				     unsigned int qp_num)
531
{
532 533
	struct ntb_transport_qp *qp = &nt->qp_vec[qp_num];
	struct ntb_transport_mw *mw;
534
	unsigned int rx_size, num_qps_mw;
535
	unsigned int mw_num, mw_count, qp_count;
Jon Mason's avatar
Jon Mason committed
536
	unsigned int i;
537

538 539
	mw_count = nt->mw_count;
	qp_count = nt->qp_count;
Jon Mason's avatar
Jon Mason committed
540

541 542 543 544 545
	mw_num = QP_TO_MW(nt, qp_num);
	mw = &nt->mw_vec[mw_num];

	if (!mw->virt_addr)
		return -ENOMEM;
546

547 548
	if (qp_count % mw_count && mw_num + 1 < qp_count / mw_count)
		num_qps_mw = qp_count / mw_count + 1;
549
	else
550
		num_qps_mw = qp_count / mw_count;
551

552 553
	rx_size = (unsigned int)mw->xlat_size / num_qps_mw;
	qp->rx_buff = mw->virt_addr + rx_size * qp_num / mw_count;
Jon Mason's avatar
Jon Mason committed
554 555
	rx_size -= sizeof(struct ntb_rx_info);

556 557
	qp->remote_rx_info = qp->rx_buff + rx_size;

558 559
	/* Due to housekeeping, there must be atleast 2 buffs */
	qp->rx_max_frame = min(transport_mtu, rx_size / 2);
Jon Mason's avatar
Jon Mason committed
560 561 562
	qp->rx_max_entry = rx_size / qp->rx_max_frame;
	qp->rx_index = 0;

563
	qp->remote_rx_info->entry = qp->rx_max_entry - 1;
564

565
	/* setup the hdr offsets with 0's */
Jon Mason's avatar
Jon Mason committed
566
	for (i = 0; i < qp->rx_max_entry; i++) {
567 568
		void *offset = (qp->rx_buff + qp->rx_max_frame * (i + 1) -
				sizeof(struct ntb_payload_header));
569
		memset(offset, 0, sizeof(struct ntb_payload_header));
Jon Mason's avatar
Jon Mason committed
570
	}
571 572 573

	qp->rx_pkts = 0;
	qp->tx_pkts = 0;
Jon Mason's avatar
Jon Mason committed
574
	qp->tx_index = 0;
575 576

	return 0;
577 578
}

579
static void ntb_free_mw(struct ntb_transport_ctx *nt, int num_mw)
Jon Mason's avatar
Jon Mason committed
580
{
581 582
	struct ntb_transport_mw *mw = &nt->mw_vec[num_mw];
	struct pci_dev *pdev = nt->ndev->pdev;
Jon Mason's avatar
Jon Mason committed
583 584 585 586

	if (!mw->virt_addr)
		return;

587 588 589 590 591
	ntb_mw_clear_trans(nt->ndev, num_mw);
	dma_free_coherent(&pdev->dev, mw->buff_size,
			  mw->virt_addr, mw->dma_addr);
	mw->xlat_size = 0;
	mw->buff_size = 0;
Jon Mason's avatar
Jon Mason committed
592 593 594
	mw->virt_addr = NULL;
}

595 596
static int ntb_set_mw(struct ntb_transport_ctx *nt, int num_mw,
		      unsigned int size)
597
{
598 599 600 601 602 603 604
	struct ntb_transport_mw *mw = &nt->mw_vec[num_mw];
	struct pci_dev *pdev = nt->ndev->pdev;
	unsigned int xlat_size, buff_size;
	int rc;

	xlat_size = round_up(size, mw->xlat_align_size);
	buff_size = round_up(size, mw->xlat_align);
605

Jon Mason's avatar
Jon Mason committed
606
	/* No need to re-setup */
607
	if (mw->xlat_size == xlat_size)
Jon Mason's avatar
Jon Mason committed
608 609
		return 0;

610
	if (mw->buff_size)
Jon Mason's avatar
Jon Mason committed
611 612
		ntb_free_mw(nt, num_mw);

613 614 615
	/* Alloc memory for receiving data.  Must be aligned */
	mw->xlat_size = xlat_size;
	mw->buff_size = buff_size;
616

617 618
	mw->virt_addr = dma_alloc_coherent(&pdev->dev, buff_size,
					   &mw->dma_addr, GFP_KERNEL);
619
	if (!mw->virt_addr) {
620 621 622 623
		mw->xlat_size = 0;
		mw->buff_size = 0;
		dev_err(&pdev->dev, "Unable to alloc MW buff of size %d\n",
			buff_size);
624 625 626
		return -ENOMEM;
	}

627 628 629 630 631 632
	/*
	 * we must ensure that the memory address allocated is BAR size
	 * aligned in order for the XLAT register to take the value. This
	 * is a requirement of the hardware. It is recommended to setup CMA
	 * for BAR sizes equal or greater than 4MB.
	 */
633 634
	if (!IS_ALIGNED(mw->dma_addr, mw->xlat_align)) {
		dev_err(&pdev->dev, "DMA memory %pad is not aligned\n",
635 636 637 638 639
			&mw->dma_addr);
		ntb_free_mw(nt, num_mw);
		return -ENOMEM;
	}

640
	/* Notify HW the memory location of the receive buffer */
641 642 643 644 645 646
	rc = ntb_mw_set_trans(nt->ndev, num_mw, mw->dma_addr, mw->xlat_size);
	if (rc) {
		dev_err(&pdev->dev, "Unable to set mw%d translation", num_mw);
		ntb_free_mw(nt, num_mw);
		return -EIO;
	}
647 648 649 650

	return 0;
}

651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672
static void ntb_qp_link_down_reset(struct ntb_transport_qp *qp)
{
	qp->link_is_up = false;

	qp->tx_index = 0;
	qp->rx_index = 0;
	qp->rx_bytes = 0;
	qp->rx_pkts = 0;
	qp->rx_ring_empty = 0;
	qp->rx_err_no_buf = 0;
	qp->rx_err_oflow = 0;
	qp->rx_err_ver = 0;
	qp->rx_memcpy = 0;
	qp->rx_async = 0;
	qp->tx_bytes = 0;
	qp->tx_pkts = 0;
	qp->tx_ring_full = 0;
	qp->tx_err_no_buf = 0;
	qp->tx_memcpy = 0;
	qp->tx_async = 0;
}

673
static void ntb_qp_link_cleanup(struct ntb_transport_qp *qp)
674
{
675 676
	struct ntb_transport_ctx *nt = qp->transport;
	struct pci_dev *pdev = nt->ndev->pdev;
677

678
	dev_info(&pdev->dev, "qp %d: Link Cleanup\n", qp->qp_num);
679 680 681

	cancel_delayed_work_sync(&qp->link_work);
	ntb_qp_link_down_reset(qp);
682 683 684

	if (qp->event_handler)
		qp->event_handler(qp->cb_data, qp->link_is_up);
685 686 687 688 689 690 691
}

static void ntb_qp_link_cleanup_work(struct work_struct *work)
{
	struct ntb_transport_qp *qp = container_of(work,
						   struct ntb_transport_qp,
						   link_cleanup);
692
	struct ntb_transport_ctx *nt = qp->transport;
693 694

	ntb_qp_link_cleanup(qp);
695

696
	if (nt->link_is_up)
697 698 699 700
		schedule_delayed_work(&qp->link_work,
				      msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT));
}

701 702 703 704 705
static void ntb_qp_link_down(struct ntb_transport_qp *qp)
{
	schedule_work(&qp->link_cleanup);
}

706
static void ntb_transport_link_cleanup(struct ntb_transport_ctx *nt)
707
{
708 709
	struct ntb_transport_qp *qp;
	u64 qp_bitmap_alloc;
710 711
	int i;

712 713
	qp_bitmap_alloc = nt->qp_bitmap & ~nt->qp_bitmap_free;

714
	/* Pass along the info to any clients */
715 716 717 718 719 720 721
	for (i = 0; i < nt->qp_count; i++)
		if (qp_bitmap_alloc & BIT_ULL(i)) {
			qp = &nt->qp_vec[i];
			ntb_qp_link_cleanup(qp);
			cancel_work_sync(&qp->link_cleanup);
			cancel_delayed_work_sync(&qp->link_work);
		}
722

723
	if (!nt->link_is_up)
724 725 726 727 728 729 730
		cancel_delayed_work_sync(&nt->link_work);

	/* The scratchpad registers keep the values if the remote side
	 * goes down, blast them now to give them a sane value the next
	 * time they are accessed
	 */
	for (i = 0; i < MAX_SPAD; i++)
731
		ntb_spad_write(nt->ndev, i, 0);
732 733
}

734 735
static void ntb_transport_link_cleanup_work(struct work_struct *work)
{
736 737
	struct ntb_transport_ctx *nt =
		container_of(work, struct ntb_transport_ctx, link_cleanup);
738 739 740 741

	ntb_transport_link_cleanup(nt);
}

742
static void ntb_transport_event_callback(void *data)
743
{
744
	struct ntb_transport_ctx *nt = data;
745

746
	if (ntb_link_is_up(nt->ndev, NULL, NULL) == 1)
747
		schedule_delayed_work(&nt->link_work, 0);
748
	else
749
		schedule_work(&nt->link_cleanup);
750 751 752 753
}

static void ntb_transport_link_work(struct work_struct *work)
{
754 755 756 757 758
	struct ntb_transport_ctx *nt =
		container_of(work, struct ntb_transport_ctx, link_work.work);
	struct ntb_dev *ndev = nt->ndev;
	struct pci_dev *pdev = ndev->pdev;
	resource_size_t size;
759
	u32 val;
760
	int rc, i, spad;
761

Jon Mason's avatar
Jon Mason committed
762
	/* send the local info, in the opposite order of the way we read it */
763 764
	for (i = 0; i < nt->mw_count; i++) {
		size = nt->mw_vec[i].phys_size;
765

766 767
		if (max_mw_size && size > max_mw_size)
			size = max_mw_size;
768

769 770
		spad = MW0_SZ_HIGH + (i * 2);
		ntb_peer_spad_write(ndev, spad, (u32)(size >> 32));
771

772 773
		spad = MW0_SZ_LOW + (i * 2);
		ntb_peer_spad_write(ndev, spad, (u32)size);
774 775
	}

776
	ntb_peer_spad_write(ndev, NUM_MWS, nt->mw_count);
777

778
	ntb_peer_spad_write(ndev, NUM_QPS, nt->qp_count);
779

780
	ntb_peer_spad_write(ndev, VERSION, NTB_TRANSPORT_VERSION);
781

782
	/* Query the remote side for its info */
783
	val = ntb_spad_read(ndev, VERSION);
784 785
	dev_dbg(&pdev->dev, "Remote version = %d\n", val);
	if (val != NTB_TRANSPORT_VERSION)
786 787
		goto out;

788
	val = ntb_spad_read(ndev, NUM_QPS);
789
	dev_dbg(&pdev->dev, "Remote max number of qps = %d\n", val);
790
	if (val != nt->qp_count)
791 792
		goto out;

793
	val = ntb_spad_read(ndev, NUM_MWS);
Jon Mason's avatar
Jon Mason committed
794
	dev_dbg(&pdev->dev, "Remote number of mws = %d\n", val);
795 796
	if (val != nt->mw_count)
		goto out;
797

798
	for (i = 0; i < nt->mw_count; i++) {
Jon Mason's avatar
Jon Mason committed
799
		u64 val64;
800

801
		val = ntb_spad_read(ndev, MW0_SZ_HIGH + (i * 2));
802
		val64 = (u64)val << 32;
Jon Mason's avatar
Jon Mason committed
803

804
		val = ntb_spad_read(ndev, MW0_SZ_LOW + (i * 2));
Jon Mason's avatar
Jon Mason committed
805 806
		val64 |= val;

807
		dev_dbg(&pdev->dev, "Remote MW%d size = %#llx\n", i, val64);
Jon Mason's avatar
Jon Mason committed
808 809 810 811 812

		rc = ntb_set_mw(nt, i, val64);
		if (rc)
			goto out1;
	}
813

814
	nt->link_is_up = true;
815

816 817
	for (i = 0; i < nt->qp_count; i++) {
		struct ntb_transport_qp *qp = &nt->qp_vec[i];
818 819 820

		ntb_transport_setup_qp_mw(nt, i);

821
		if (qp->client_ready)
822 823 824 825 826
			schedule_delayed_work(&qp->link_work, 0);
	}

	return;

Jon Mason's avatar
Jon Mason committed
827
out1:
828
	for (i = 0; i < nt->mw_count; i++)
Jon Mason's avatar
Jon Mason committed
829
		ntb_free_mw(nt, i);
830
out:
831
	if (ntb_link_is_up(ndev, NULL, NULL) == 1)
832 833 834 835 836 837 838 839 840
		schedule_delayed_work(&nt->link_work,
				      msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT));
}

static void ntb_qp_link_work(struct work_struct *work)
{
	struct ntb_transport_qp *qp = container_of(work,
						   struct ntb_transport_qp,
						   link_work.work);
841 842 843
	struct pci_dev *pdev = qp->ndev->pdev;
	struct ntb_transport_ctx *nt = qp->transport;
	int val;
844

845
	WARN_ON(!nt->link_is_up);
846

847
	val = ntb_spad_read(nt->ndev, QP_LINKS);
848

849
	ntb_peer_spad_write(nt->ndev, QP_LINKS, val | BIT(qp->qp_num));
850 851

	/* query remote spad for qp ready bits */
852
	ntb_peer_spad_read(nt->ndev, QP_LINKS);
853 854 855
	dev_dbg(&pdev->dev, "Remote QP link status = %x\n", val);

	/* See if the remote side is up */
856
	if (val & BIT(qp->qp_num)) {
857
		dev_info(&pdev->dev, "qp %d: Link Up\n", qp->qp_num);
858 859
		qp->link_is_up = true;

860
		if (qp->event_handler)
861 862
			qp->event_handler(qp->cb_data, qp->link_is_up);
	} else if (nt->link_is_up)
863 864 865 866
		schedule_delayed_work(&qp->link_work,
				      msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT));
}

867
static int ntb_transport_init_queue(struct ntb_transport_ctx *nt,
Jon Mason's avatar
Jon Mason committed
868
				    unsigned int qp_num)
869 870
{
	struct ntb_transport_qp *qp;
871 872 873
	struct ntb_transport_mw *mw;
	phys_addr_t mw_base;
	resource_size_t mw_size;
874
	unsigned int num_qps_mw, tx_size;
875
	unsigned int mw_num, mw_count, qp_count;
876
	u64 qp_offset;
Jon Mason's avatar
Jon Mason committed
877

878 879
	mw_count = nt->mw_count;
	qp_count = nt->qp_count;
880

881 882 883 884
	mw_num = QP_TO_MW(nt, qp_num);
	mw = &nt->mw_vec[mw_num];

	qp = &nt->qp_vec[qp_num];
885 886 887
	qp->qp_num = qp_num;
	qp->transport = nt;
	qp->ndev = nt->ndev;
888
	qp->client_ready = false;
889
	qp->event_handler = NULL;
890
	ntb_qp_link_down_reset(qp);
891

892 893
	if (qp_count % mw_count && mw_num + 1 < qp_count / mw_count)
		num_qps_mw = qp_count / mw_count + 1;
894
	else
895 896 897 898
		num_qps_mw = qp_count / mw_count;

	mw_base = nt->mw_vec[mw_num].phys_addr;
	mw_size = nt->mw_vec[mw_num].phys_size;
899

900 901 902 903
	tx_size = (unsigned int)mw_size / num_qps_mw;
	qp_offset = tx_size * qp_num / mw_count;

	qp->tx_mw = nt->mw_vec[mw_num].vbase + qp_offset;
904 905 906
	if (!qp->tx_mw)
		return -EINVAL;

907
	qp->tx_mw_phys = mw_base + qp_offset;
908 909 910
	if (!qp->tx_mw_phys)
		return -EINVAL;

Jon Mason's avatar
Jon Mason committed
911
	tx_size -= sizeof(struct ntb_rx_info);
912
	qp->rx_info = qp->tx_mw + tx_size;
Jon Mason's avatar
Jon Mason committed
913

914 915
	/* Due to housekeeping, there must be atleast 2 buffs */
	qp->tx_max_frame = min(transport_mtu, tx_size / 2);
Jon Mason's avatar
Jon Mason committed
916
	qp->tx_max_entry = tx_size / qp->tx_max_frame;