ntb_transport.c 51.7 KB
Newer Older
1 2 3 4 5 6 7
/*
 * This file is provided under a dual BSD/GPLv2 license.  When using or
 *   redistributing this file, you may do so under either license.
 *
 *   GPL LICENSE SUMMARY
 *
 *   Copyright(c) 2012 Intel Corporation. All rights reserved.
8
 *   Copyright (C) 2015 EMC Corporation. All Rights Reserved.
9 10 11 12 13 14 15 16
 *
 *   This program is free software; you can redistribute it and/or modify
 *   it under the terms of version 2 of the GNU General Public License as
 *   published by the Free Software Foundation.
 *
 *   BSD LICENSE
 *
 *   Copyright(c) 2012 Intel Corporation. All rights reserved.
17
 *   Copyright (C) 2015 EMC Corporation. All Rights Reserved.
18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44
 *
 *   Redistribution and use in source and binary forms, with or without
 *   modification, are permitted provided that the following conditions
 *   are met:
 *
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copy
 *       notice, this list of conditions and the following disclaimer in
 *       the documentation and/or other materials provided with the
 *       distribution.
 *     * Neither the name of Intel Corporation nor the names of its
 *       contributors may be used to endorse or promote products derived
 *       from this software without specific prior written permission.
 *
 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
45
 * PCIe NTB Transport Linux driver
46 47 48 49 50 51
 *
 * Contact Information:
 * Jon Mason <jon.mason@intel.com>
 */
#include <linux/debugfs.h>
#include <linux/delay.h>
52
#include <linux/dmaengine.h>
53 54 55 56 57 58 59 60
#include <linux/dma-mapping.h>
#include <linux/errno.h>
#include <linux/export.h>
#include <linux/interrupt.h>
#include <linux/module.h>
#include <linux/pci.h>
#include <linux/slab.h>
#include <linux/types.h>
61
#include <linux/uaccess.h>
62 63
#include "linux/ntb.h"
#include "linux/ntb_transport.h"
64

65 66 67 68 69 70 71 72 73 74 75 76 77
#define NTB_TRANSPORT_VERSION	4
#define NTB_TRANSPORT_VER	"4"
#define NTB_TRANSPORT_NAME	"ntb_transport"
#define NTB_TRANSPORT_DESC	"Software Queue-Pair Transport over NTB"

MODULE_DESCRIPTION(NTB_TRANSPORT_DESC);
MODULE_VERSION(NTB_TRANSPORT_VER);
MODULE_LICENSE("Dual BSD/GPL");
MODULE_AUTHOR("Intel Corporation");

static unsigned long max_mw_size;
module_param(max_mw_size, ulong, 0644);
MODULE_PARM_DESC(max_mw_size, "Limit size of large memory windows");
78

79
static unsigned int transport_mtu = 0x10000;
80 81 82
module_param(transport_mtu, uint, 0644);
MODULE_PARM_DESC(transport_mtu, "Maximum size of NTB transport packets");

Jon Mason's avatar
Jon Mason committed
83
static unsigned char max_num_clients;
84 85 86
module_param(max_num_clients, byte, 0644);
MODULE_PARM_DESC(max_num_clients, "Maximum number of NTB transport clients");

87 88 89 90
static unsigned int copy_bytes = 1024;
module_param(copy_bytes, uint, 0644);
MODULE_PARM_DESC(copy_bytes, "Threshold under which NTB will use the CPU to copy instead of DMA");

91 92 93 94
static bool use_dma;
module_param(use_dma, bool, 0644);
MODULE_PARM_DESC(use_dma, "Use DMA engine to perform large data copy");

95 96
static struct dentry *nt_debugfs_dir;

97 98 99
struct ntb_queue_entry {
	/* ntb_queue list reference */
	struct list_head entry;
100
	/* pointers to data to be transferred */
101 102 103 104
	void *cb_data;
	void *buf;
	unsigned int len;
	unsigned int flags;
105 106 107 108 109 110 111

	struct ntb_transport_qp *qp;
	union {
		struct ntb_payload_header __iomem *tx_hdr;
		struct ntb_payload_header *rx_hdr;
	};
	unsigned int index;
112 113
};

Jon Mason's avatar
Jon Mason committed
114 115 116 117
struct ntb_rx_info {
	unsigned int entry;
};

118
struct ntb_transport_qp {
119 120
	struct ntb_transport_ctx *transport;
	struct ntb_dev *ndev;
121
	void *cb_data;
122 123
	struct dma_chan *tx_dma_chan;
	struct dma_chan *rx_dma_chan;
124 125

	bool client_ready;
126 127
	bool link_is_up;

128
	u8 qp_num;	/* Only 64 QP's are allowed.  0-63 */
129
	u64 qp_bit;
130

Jon Mason's avatar
Jon Mason committed
131
	struct ntb_rx_info __iomem *rx_info;
Jon Mason's avatar
Jon Mason committed
132 133
	struct ntb_rx_info *remote_rx_info;

Jon Mason's avatar
Jon Mason committed
134 135
	void (*tx_handler)(struct ntb_transport_qp *qp, void *qp_data,
			   void *data, int len);
136 137
	struct list_head tx_free_q;
	spinlock_t ntb_tx_free_q_lock;
Jon Mason's avatar
Jon Mason committed
138
	void __iomem *tx_mw;
139
	dma_addr_t tx_mw_phys;
Jon Mason's avatar
Jon Mason committed
140 141
	unsigned int tx_index;
	unsigned int tx_max_entry;
142
	unsigned int tx_max_frame;
143

Jon Mason's avatar
Jon Mason committed
144 145
	void (*rx_handler)(struct ntb_transport_qp *qp, void *qp_data,
			   void *data, int len);
146
	struct list_head rx_post_q;
147 148
	struct list_head rx_pend_q;
	struct list_head rx_free_q;
149 150
	/* ntb_rx_q_lock: synchronize access to rx_XXXX_q */
	spinlock_t ntb_rx_q_lock;
Jon Mason's avatar
Jon Mason committed
151 152 153
	void *rx_buff;
	unsigned int rx_index;
	unsigned int rx_max_entry;
154
	unsigned int rx_max_frame;
155
	dma_cookie_t last_cookie;
156
	struct tasklet_struct rxc_db_work;
157

Jon Mason's avatar
Jon Mason committed
158
	void (*event_handler)(void *data, int status);
159
	struct delayed_work link_work;
160
	struct work_struct link_cleanup;
161 162 163 164 165 166 167 168 169 170 171

	struct dentry *debugfs_dir;
	struct dentry *debugfs_stats;

	/* Stats */
	u64 rx_bytes;
	u64 rx_pkts;
	u64 rx_ring_empty;
	u64 rx_err_no_buf;
	u64 rx_err_oflow;
	u64 rx_err_ver;
172 173
	u64 rx_memcpy;
	u64 rx_async;
174 175 176
	u64 tx_bytes;
	u64 tx_pkts;
	u64 tx_ring_full;
177 178 179
	u64 tx_err_no_buf;
	u64 tx_memcpy;
	u64 tx_async;
180 181 182
};

struct ntb_transport_mw {
183 184 185 186 187 188 189
	phys_addr_t phys_addr;
	resource_size_t phys_size;
	resource_size_t xlat_align;
	resource_size_t xlat_align_size;
	void __iomem *vbase;
	size_t xlat_size;
	size_t buff_size;
190 191 192 193 194 195
	void *virt_addr;
	dma_addr_t dma_addr;
};

struct ntb_transport_client_dev {
	struct list_head entry;
196
	struct ntb_transport_ctx *nt;
197 198 199
	struct device dev;
};

200
struct ntb_transport_ctx {
201 202 203
	struct list_head entry;
	struct list_head client_devs;

204 205 206 207 208 209 210 211 212 213
	struct ntb_dev *ndev;

	struct ntb_transport_mw *mw_vec;
	struct ntb_transport_qp *qp_vec;
	unsigned int mw_count;
	unsigned int qp_count;
	u64 qp_bitmap;
	u64 qp_bitmap_free;

	bool link_is_up;
214
	struct delayed_work link_work;
215
	struct work_struct link_cleanup;
216 217

	struct dentry *debugfs_node_dir;
218 219 220
};

enum {
221 222
	DESC_DONE_FLAG = BIT(0),
	LINK_DOWN_FLAG = BIT(1),
223 224 225
};

struct ntb_payload_header {
Jon Mason's avatar
Jon Mason committed
226
	unsigned int ver;
227 228 229 230 231 232 233
	unsigned int len;
	unsigned int flags;
};

enum {
	VERSION = 0,
	QP_LINKS,
Jon Mason's avatar
Jon Mason committed
234 235 236 237 238 239
	NUM_QPS,
	NUM_MWS,
	MW0_SZ_HIGH,
	MW0_SZ_LOW,
	MW1_SZ_HIGH,
	MW1_SZ_LOW,
240 241 242
	MAX_SPAD,
};

243 244 245 246 247 248 249
#define dev_client_dev(__dev) \
	container_of((__dev), struct ntb_transport_client_dev, dev)

#define drv_client(__drv) \
	container_of((__drv), struct ntb_transport_client, driver)

#define QP_TO_MW(nt, qp)	((qp) % nt->mw_count)
250 251 252
#define NTB_QP_DEF_NUM_ENTRIES	100
#define NTB_LINK_DOWN_TIMEOUT	10

253 254 255 256 257 258
static void ntb_transport_rxc_db(unsigned long data);
static const struct ntb_ctx_ops ntb_transport_ops;
static struct ntb_client ntb_transport_client;

static int ntb_transport_bus_match(struct device *dev,
				   struct device_driver *drv)
259 260 261 262
{
	return !strncmp(dev_name(dev), drv->name, strlen(drv->name));
}

263
static int ntb_transport_bus_probe(struct device *dev)
264
{
265
	const struct ntb_transport_client *client;
266 267 268
	int rc = -EINVAL;

	get_device(dev);
269 270 271

	client = drv_client(dev->driver);
	rc = client->probe(dev);
272 273 274 275 276 277
	if (rc)
		put_device(dev);

	return rc;
}

278
static int ntb_transport_bus_remove(struct device *dev)
279
{
280
	const struct ntb_transport_client *client;
281

282 283
	client = drv_client(dev->driver);
	client->remove(dev);
284 285 286 287 288 289

	put_device(dev);

	return 0;
}

290 291 292 293 294
static struct bus_type ntb_transport_bus = {
	.name = "ntb_transport",
	.match = ntb_transport_bus_match,
	.probe = ntb_transport_bus_probe,
	.remove = ntb_transport_bus_remove,
295 296 297 298
};

static LIST_HEAD(ntb_transport_list);

299
static int ntb_bus_init(struct ntb_transport_ctx *nt)
300
{
301
	list_add_tail(&nt->entry, &ntb_transport_list);
302 303 304
	return 0;
}

305
static void ntb_bus_remove(struct ntb_transport_ctx *nt)
306 307 308 309 310 311 312 313 314 315 316 317 318
{
	struct ntb_transport_client_dev *client_dev, *cd;

	list_for_each_entry_safe(client_dev, cd, &nt->client_devs, entry) {
		dev_err(client_dev->dev.parent, "%s still attached to bus, removing\n",
			dev_name(&client_dev->dev));
		list_del(&client_dev->entry);
		device_unregister(&client_dev->dev);
	}

	list_del(&nt->entry);
}

319
static void ntb_transport_client_release(struct device *dev)
320 321 322
{
	struct ntb_transport_client_dev *client_dev;

323
	client_dev = dev_client_dev(dev);
324 325 326 327
	kfree(client_dev);
}

/**
328
 * ntb_transport_unregister_client_dev - Unregister NTB client device
329 330 331 332
 * @device_name: Name of NTB client device
 *
 * Unregister an NTB client device with the NTB transport layer
 */
333
void ntb_transport_unregister_client_dev(char *device_name)
334 335
{
	struct ntb_transport_client_dev *client, *cd;
336
	struct ntb_transport_ctx *nt;
337 338 339 340 341 342 343 344 345

	list_for_each_entry(nt, &ntb_transport_list, entry)
		list_for_each_entry_safe(client, cd, &nt->client_devs, entry)
			if (!strncmp(dev_name(&client->dev), device_name,
				     strlen(device_name))) {
				list_del(&client->entry);
				device_unregister(&client->dev);
			}
}
346
EXPORT_SYMBOL_GPL(ntb_transport_unregister_client_dev);
347 348

/**
349
 * ntb_transport_register_client_dev - Register NTB client device
350 351 352 353
 * @device_name: Name of NTB client device
 *
 * Register an NTB client device with the NTB transport layer
 */
354
int ntb_transport_register_client_dev(char *device_name)
355 356
{
	struct ntb_transport_client_dev *client_dev;
357
	struct ntb_transport_ctx *nt;
358
	int node;
Jon Mason's avatar
Jon Mason committed
359
	int rc, i = 0;
360

361 362 363
	if (list_empty(&ntb_transport_list))
		return -ENODEV;

364 365 366
	list_for_each_entry(nt, &ntb_transport_list, entry) {
		struct device *dev;

367 368 369 370
		node = dev_to_node(&nt->ndev->dev);

		client_dev = kzalloc_node(sizeof(*client_dev),
					  GFP_KERNEL, node);
371 372 373 374 375 376 377 378
		if (!client_dev) {
			rc = -ENOMEM;
			goto err;
		}

		dev = &client_dev->dev;

		/* setup and register client devices */
Jon Mason's avatar
Jon Mason committed
379
		dev_set_name(dev, "%s%d", device_name, i);
380 381 382
		dev->bus = &ntb_transport_bus;
		dev->release = ntb_transport_client_release;
		dev->parent = &nt->ndev->dev;
383 384 385 386 387 388 389 390

		rc = device_register(dev);
		if (rc) {
			kfree(client_dev);
			goto err;
		}

		list_add_tail(&client_dev->entry, &nt->client_devs);
Jon Mason's avatar
Jon Mason committed
391
		i++;
392 393 394 395 396
	}

	return 0;

err:
397
	ntb_transport_unregister_client_dev(device_name);
398 399 400

	return rc;
}
401
EXPORT_SYMBOL_GPL(ntb_transport_register_client_dev);
402 403

/**
404
 * ntb_transport_register_client - Register NTB client driver
405 406 407 408 409 410
 * @drv: NTB client driver to be registered
 *
 * Register an NTB client driver with the NTB transport layer
 *
 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
 */
411
int ntb_transport_register_client(struct ntb_transport_client *drv)
412
{
413
	drv->driver.bus = &ntb_transport_bus;
414

415 416 417
	if (list_empty(&ntb_transport_list))
		return -ENODEV;

418 419
	return driver_register(&drv->driver);
}
420
EXPORT_SYMBOL_GPL(ntb_transport_register_client);
421 422

/**
423
 * ntb_transport_unregister_client - Unregister NTB client driver
424 425 426 427 428 429
 * @drv: NTB client driver to be unregistered
 *
 * Unregister an NTB client driver with the NTB transport layer
 *
 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
 */
430
void ntb_transport_unregister_client(struct ntb_transport_client *drv)
431 432 433
{
	driver_unregister(&drv->driver);
}
434
EXPORT_SYMBOL_GPL(ntb_transport_unregister_client);
435 436 437 438 439

static ssize_t debugfs_read(struct file *filp, char __user *ubuf, size_t count,
			    loff_t *offp)
{
	struct ntb_transport_qp *qp;
440
	char *buf;
441 442
	ssize_t ret, out_offset, out_count;

443 444 445 446 447
	qp = filp->private_data;

	if (!qp || !qp->link_is_up)
		return 0;

448
	out_count = 1000;
449 450 451 452

	buf = kmalloc(out_count, GFP_KERNEL);
	if (!buf)
		return -ENOMEM;
453 454 455

	out_offset = 0;
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
Dave Jiang's avatar
Dave Jiang committed
456
			       "\nNTB QP stats:\n\n");
457 458 459 460
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_bytes - \t%llu\n", qp->rx_bytes);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_pkts - \t%llu\n", qp->rx_pkts);
461 462 463 464
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_memcpy - \t%llu\n", qp->rx_memcpy);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_async - \t%llu\n", qp->rx_async);
465 466 467 468 469 470 471 472 473
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_ring_empty - %llu\n", qp->rx_ring_empty);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_err_no_buf - %llu\n", qp->rx_err_no_buf);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_err_oflow - \t%llu\n", qp->rx_err_oflow);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_err_ver - \t%llu\n", qp->rx_err_ver);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
Dave Jiang's avatar
Dave Jiang committed
474
			       "rx_buff - \t0x%p\n", qp->rx_buff);
475
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
Jon Mason's avatar
Jon Mason committed
476
			       "rx_index - \t%u\n", qp->rx_index);
477
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
Dave Jiang's avatar
Dave Jiang committed
478
			       "rx_max_entry - \t%u\n\n", qp->rx_max_entry);
479 480 481 482 483

	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "tx_bytes - \t%llu\n", qp->tx_bytes);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "tx_pkts - \t%llu\n", qp->tx_pkts);
484 485 486 487
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "tx_memcpy - \t%llu\n", qp->tx_memcpy);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "tx_async - \t%llu\n", qp->tx_async);
488 489
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "tx_ring_full - \t%llu\n", qp->tx_ring_full);
490 491
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "tx_err_no_buf - %llu\n", qp->tx_err_no_buf);
492
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
Dave Jiang's avatar
Dave Jiang committed
493
			       "tx_mw - \t0x%p\n", qp->tx_mw);
494
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
Dave Jiang's avatar
Dave Jiang committed
495
			       "tx_index (H) - \t%u\n", qp->tx_index);
496
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
Dave Jiang's avatar
Dave Jiang committed
497
			       "RRI (T) - \t%u\n",
498
			       qp->remote_rx_info->entry);
Dave Jiang's avatar
Dave Jiang committed
499 500
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "tx_max_entry - \t%u\n", qp->tx_max_entry);
501 502 503
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "free tx - \t%u\n",
			       ntb_transport_tx_free_entry(qp));
504 505

	out_offset += snprintf(buf + out_offset, out_count - out_offset,
Dave Jiang's avatar
Dave Jiang committed
506 507
			       "\n");
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
508 509 510 511 512
			       "Using TX DMA - \t%s\n",
			       qp->tx_dma_chan ? "Yes" : "No");
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "Using RX DMA - \t%s\n",
			       qp->rx_dma_chan ? "Yes" : "No");
Dave Jiang's avatar
Dave Jiang committed
513 514
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "QP Link - \t%s\n",
515
			       qp->link_is_up ? "Up" : "Down");
Dave Jiang's avatar
Dave Jiang committed
516 517 518
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "\n");

519 520
	if (out_offset > out_count)
		out_offset = out_count;
521 522

	ret = simple_read_from_buffer(ubuf, count, offp, buf, out_offset);
523
	kfree(buf);
524 525 526 527 528
	return ret;
}

static const struct file_operations ntb_qp_debugfs_stats = {
	.owner = THIS_MODULE,
Jon Mason's avatar
Jon Mason committed
529
	.open = simple_open,
530 531 532 533 534 535 536 537 538 539 540 541 542 543
	.read = debugfs_read,
};

static void ntb_list_add(spinlock_t *lock, struct list_head *entry,
			 struct list_head *list)
{
	unsigned long flags;

	spin_lock_irqsave(lock, flags);
	list_add_tail(entry, list);
	spin_unlock_irqrestore(lock, flags);
}

static struct ntb_queue_entry *ntb_list_rm(spinlock_t *lock,
Jon Mason's avatar
Jon Mason committed
544
					   struct list_head *list)
545 546 547 548 549 550 551 552 553 554 555
{
	struct ntb_queue_entry *entry;
	unsigned long flags;

	spin_lock_irqsave(lock, flags);
	if (list_empty(list)) {
		entry = NULL;
		goto out;
	}
	entry = list_first_entry(list, struct ntb_queue_entry, entry);
	list_del(&entry->entry);
556

557 558 559 560 561 562
out:
	spin_unlock_irqrestore(lock, flags);

	return entry;
}

563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583
static struct ntb_queue_entry *ntb_list_mv(spinlock_t *lock,
					   struct list_head *list,
					   struct list_head *to_list)
{
	struct ntb_queue_entry *entry;
	unsigned long flags;

	spin_lock_irqsave(lock, flags);

	if (list_empty(list)) {
		entry = NULL;
	} else {
		entry = list_first_entry(list, struct ntb_queue_entry, entry);
		list_move_tail(&entry->entry, to_list);
	}

	spin_unlock_irqrestore(lock, flags);

	return entry;
}

584 585
static int ntb_transport_setup_qp_mw(struct ntb_transport_ctx *nt,
				     unsigned int qp_num)
586
{
587 588
	struct ntb_transport_qp *qp = &nt->qp_vec[qp_num];
	struct ntb_transport_mw *mw;
589
	unsigned int rx_size, num_qps_mw;
590
	unsigned int mw_num, mw_count, qp_count;
Jon Mason's avatar
Jon Mason committed
591
	unsigned int i;
592

593 594
	mw_count = nt->mw_count;
	qp_count = nt->qp_count;
Jon Mason's avatar
Jon Mason committed
595

596 597 598 599 600
	mw_num = QP_TO_MW(nt, qp_num);
	mw = &nt->mw_vec[mw_num];

	if (!mw->virt_addr)
		return -ENOMEM;
601

602 603
	if (qp_count % mw_count && mw_num + 1 < qp_count / mw_count)
		num_qps_mw = qp_count / mw_count + 1;
604
	else
605
		num_qps_mw = qp_count / mw_count;
606

607
	rx_size = (unsigned int)mw->xlat_size / num_qps_mw;
608
	qp->rx_buff = mw->virt_addr + rx_size * (qp_num / mw_count);
Jon Mason's avatar
Jon Mason committed
609 610
	rx_size -= sizeof(struct ntb_rx_info);

611 612
	qp->remote_rx_info = qp->rx_buff + rx_size;

613 614
	/* Due to housekeeping, there must be atleast 2 buffs */
	qp->rx_max_frame = min(transport_mtu, rx_size / 2);
Jon Mason's avatar
Jon Mason committed
615 616 617
	qp->rx_max_entry = rx_size / qp->rx_max_frame;
	qp->rx_index = 0;

618
	qp->remote_rx_info->entry = qp->rx_max_entry - 1;
619

620
	/* setup the hdr offsets with 0's */
Jon Mason's avatar
Jon Mason committed
621
	for (i = 0; i < qp->rx_max_entry; i++) {
622 623
		void *offset = (qp->rx_buff + qp->rx_max_frame * (i + 1) -
				sizeof(struct ntb_payload_header));
624
		memset(offset, 0, sizeof(struct ntb_payload_header));
Jon Mason's avatar
Jon Mason committed
625
	}
626 627 628

	qp->rx_pkts = 0;
	qp->tx_pkts = 0;
Jon Mason's avatar
Jon Mason committed
629
	qp->tx_index = 0;
630 631

	return 0;
632 633
}

634
static void ntb_free_mw(struct ntb_transport_ctx *nt, int num_mw)
Jon Mason's avatar
Jon Mason committed
635
{
636 637
	struct ntb_transport_mw *mw = &nt->mw_vec[num_mw];
	struct pci_dev *pdev = nt->ndev->pdev;
Jon Mason's avatar
Jon Mason committed
638 639 640 641

	if (!mw->virt_addr)
		return;

642 643 644 645 646
	ntb_mw_clear_trans(nt->ndev, num_mw);
	dma_free_coherent(&pdev->dev, mw->buff_size,
			  mw->virt_addr, mw->dma_addr);
	mw->xlat_size = 0;
	mw->buff_size = 0;
Jon Mason's avatar
Jon Mason committed
647 648 649
	mw->virt_addr = NULL;
}

650
static int ntb_set_mw(struct ntb_transport_ctx *nt, int num_mw,
651
		      resource_size_t size)
652
{
653 654
	struct ntb_transport_mw *mw = &nt->mw_vec[num_mw];
	struct pci_dev *pdev = nt->ndev->pdev;
655
	size_t xlat_size, buff_size;
656 657
	int rc;

658 659 660
	if (!size)
		return -EINVAL;

661 662
	xlat_size = round_up(size, mw->xlat_align_size);
	buff_size = round_up(size, mw->xlat_align);
663

Jon Mason's avatar
Jon Mason committed
664
	/* No need to re-setup */
665
	if (mw->xlat_size == xlat_size)
Jon Mason's avatar
Jon Mason committed
666 667
		return 0;

668
	if (mw->buff_size)
Jon Mason's avatar
Jon Mason committed
669 670
		ntb_free_mw(nt, num_mw);

671 672 673
	/* Alloc memory for receiving data.  Must be aligned */
	mw->xlat_size = xlat_size;
	mw->buff_size = buff_size;
674

675 676
	mw->virt_addr = dma_alloc_coherent(&pdev->dev, buff_size,
					   &mw->dma_addr, GFP_KERNEL);
677
	if (!mw->virt_addr) {
678 679
		mw->xlat_size = 0;
		mw->buff_size = 0;
680
		dev_err(&pdev->dev, "Unable to alloc MW buff of size %zu\n",
681
			buff_size);
682 683 684
		return -ENOMEM;
	}

685 686 687 688 689 690
	/*
	 * we must ensure that the memory address allocated is BAR size
	 * aligned in order for the XLAT register to take the value. This
	 * is a requirement of the hardware. It is recommended to setup CMA
	 * for BAR sizes equal or greater than 4MB.
	 */
691 692
	if (!IS_ALIGNED(mw->dma_addr, mw->xlat_align)) {
		dev_err(&pdev->dev, "DMA memory %pad is not aligned\n",
693 694 695 696 697
			&mw->dma_addr);
		ntb_free_mw(nt, num_mw);
		return -ENOMEM;
	}

698
	/* Notify HW the memory location of the receive buffer */
699 700 701 702 703 704
	rc = ntb_mw_set_trans(nt->ndev, num_mw, mw->dma_addr, mw->xlat_size);
	if (rc) {
		dev_err(&pdev->dev, "Unable to set mw%d translation", num_mw);
		ntb_free_mw(nt, num_mw);
		return -EIO;
	}
705 706 707 708

	return 0;
}

709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730
static void ntb_qp_link_down_reset(struct ntb_transport_qp *qp)
{
	qp->link_is_up = false;

	qp->tx_index = 0;
	qp->rx_index = 0;
	qp->rx_bytes = 0;
	qp->rx_pkts = 0;
	qp->rx_ring_empty = 0;
	qp->rx_err_no_buf = 0;
	qp->rx_err_oflow = 0;
	qp->rx_err_ver = 0;
	qp->rx_memcpy = 0;
	qp->rx_async = 0;
	qp->tx_bytes = 0;
	qp->tx_pkts = 0;
	qp->tx_ring_full = 0;
	qp->tx_err_no_buf = 0;
	qp->tx_memcpy = 0;
	qp->tx_async = 0;
}

731
static void ntb_qp_link_cleanup(struct ntb_transport_qp *qp)
732
{
733 734
	struct ntb_transport_ctx *nt = qp->transport;
	struct pci_dev *pdev = nt->ndev->pdev;
735

736
	dev_info(&pdev->dev, "qp %d: Link Cleanup\n", qp->qp_num);
737 738 739

	cancel_delayed_work_sync(&qp->link_work);
	ntb_qp_link_down_reset(qp);
740 741 742

	if (qp->event_handler)
		qp->event_handler(qp->cb_data, qp->link_is_up);
743 744 745 746 747 748 749
}

static void ntb_qp_link_cleanup_work(struct work_struct *work)
{
	struct ntb_transport_qp *qp = container_of(work,
						   struct ntb_transport_qp,
						   link_cleanup);
750
	struct ntb_transport_ctx *nt = qp->transport;
751 752

	ntb_qp_link_cleanup(qp);
753

754
	if (nt->link_is_up)
755 756 757 758
		schedule_delayed_work(&qp->link_work,
				      msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT));
}

759 760 761 762 763
static void ntb_qp_link_down(struct ntb_transport_qp *qp)
{
	schedule_work(&qp->link_cleanup);
}

764
static void ntb_transport_link_cleanup(struct ntb_transport_ctx *nt)
765
{
766 767
	struct ntb_transport_qp *qp;
	u64 qp_bitmap_alloc;
768 769
	int i;

770 771
	qp_bitmap_alloc = nt->qp_bitmap & ~nt->qp_bitmap_free;

772
	/* Pass along the info to any clients */
773 774 775 776 777 778 779
	for (i = 0; i < nt->qp_count; i++)
		if (qp_bitmap_alloc & BIT_ULL(i)) {
			qp = &nt->qp_vec[i];
			ntb_qp_link_cleanup(qp);
			cancel_work_sync(&qp->link_cleanup);
			cancel_delayed_work_sync(&qp->link_work);
		}
780

781
	if (!nt->link_is_up)
782 783 784 785 786 787 788
		cancel_delayed_work_sync(&nt->link_work);

	/* The scratchpad registers keep the values if the remote side
	 * goes down, blast them now to give them a sane value the next
	 * time they are accessed
	 */
	for (i = 0; i < MAX_SPAD; i++)
789
		ntb_spad_write(nt->ndev, i, 0);
790 791
}

792 793
static void ntb_transport_link_cleanup_work(struct work_struct *work)
{
794 795
	struct ntb_transport_ctx *nt =
		container_of(work, struct ntb_transport_ctx, link_cleanup);
796 797 798 799

	ntb_transport_link_cleanup(nt);
}

800
static void ntb_transport_event_callback(void *data)
801
{
802
	struct ntb_transport_ctx *nt = data;
803

804
	if (ntb_link_is_up(nt->ndev, NULL, NULL) == 1)
805
		schedule_delayed_work(&nt->link_work, 0);
806
	else
807
		schedule_work(&nt->link_cleanup);
808 809 810 811
}

static void ntb_transport_link_work(struct work_struct *work)
{
812 813 814 815 816
	struct ntb_transport_ctx *nt =
		container_of(work, struct ntb_transport_ctx, link_work.work);
	struct ntb_dev *ndev = nt->ndev;
	struct pci_dev *pdev = ndev->pdev;
	resource_size_t size;
817
	u32 val;
818
	int rc, i, spad;
819

Jon Mason's avatar
Jon Mason committed
820
	/* send the local info, in the opposite order of the way we read it */
821 822
	for (i = 0; i < nt->mw_count; i++) {
		size = nt->mw_vec[i].phys_size;
823

824 825
		if (max_mw_size && size > max_mw_size)
			size = max_mw_size;
826

827 828
		spad = MW0_SZ_HIGH + (i * 2);
		ntb_peer_spad_write(ndev, spad, (u32)(size >> 32));
829

830 831
		spad = MW0_SZ_LOW + (i * 2);
		ntb_peer_spad_write(ndev, spad, (u32)size);
832 833
	}

834
	ntb_peer_spad_write(ndev, NUM_MWS, nt->mw_count);
835

836
	ntb_peer_spad_write(ndev, NUM_QPS, nt->qp_count);
837

838
	ntb_peer_spad_write(ndev, VERSION, NTB_TRANSPORT_VERSION);
839

840
	/* Query the remote side for its info */
841
	val = ntb_spad_read(ndev, VERSION);
842 843
	dev_dbg(&pdev->dev, "Remote version = %d\n", val);
	if (val != NTB_TRANSPORT_VERSION)
844 845
		goto out;

846
	val = ntb_spad_read(ndev, NUM_QPS);
847
	dev_dbg(&pdev->dev, "Remote max number of qps = %d\n", val);
848
	if (val != nt->qp_count)
849 850
		goto out;

851
	val = ntb_spad_read(ndev, NUM_MWS);
Jon Mason's avatar
Jon Mason committed
852
	dev_dbg(&pdev->dev, "Remote number of mws = %d\n", val);
853 854
	if (val != nt->mw_count)
		goto out;
855

856
	for (i = 0; i < nt->mw_count; i++) {
Jon Mason's avatar
Jon Mason committed
857
		u64 val64;
858

859
		val = ntb_spad_read(ndev, MW0_SZ_HIGH + (i * 2));
860
		val64 = (u64)val << 32;
Jon Mason's avatar
Jon Mason committed
861

862
		val = ntb_spad_read(ndev, MW0_SZ_LOW + (i * 2));
Jon Mason's avatar
Jon Mason committed
863 864
		val64 |= val;

865
		dev_dbg(&pdev->dev, "Remote MW%d size = %#llx\n", i, val64);
Jon Mason's avatar
Jon Mason committed
866 867 868 869 870

		rc = ntb_set_mw(nt, i, val64);
		if (rc)
			goto out1;
	}
871

872
	nt->link_is_up = true;
873

874 875
	for (i = 0; i < nt->qp_count; i++) {
		struct ntb_transport_qp *qp = &nt->qp_vec[i];
876 877 878

		ntb_transport_setup_qp_mw(nt, i);

879
		if (qp->client_ready)
880 881 882 883 884
			schedule_delayed_work(&qp->link_work, 0);
	}

	return;

Jon Mason's avatar
Jon Mason committed
885
out1:
886
	for (i = 0; i < nt->mw_count; i++)
Jon Mason's avatar
Jon Mason committed
887
		ntb_free_mw(nt, i);
888
out:
889
	if (ntb_link_is_up(ndev, NULL, NULL) == 1)
890 891 892 893 894 895 896 897 898
		schedule_delayed_work(&nt->link_work,
				      msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT));
}

static void ntb_qp_link_work(struct work_struct *work)
{
	struct ntb_transport_qp *qp = container_of(work,
						   struct ntb_transport_qp,
						   link_work.work);
899 900 901
	struct pci_dev *pdev = qp->ndev->pdev;
	struct ntb_transport_ctx *nt = qp->transport;
	int val;
902

903
	WARN_ON(!nt->link_is_up);
904

905
	val = ntb_spad_read(nt->ndev, QP_LINKS);
906

907
	ntb_peer_spad_write(nt->ndev, QP_LINKS, val | BIT(qp->qp_num));
908 909

	/* query remote spad for qp ready bits */
910
	ntb_peer_spad_read(nt->ndev, QP_LINKS);
Allen Hubbe's avatar
Allen Hubbe committed
911
	dev_dbg_ratelimited(&pdev->dev, "Remote QP link status = %x\n", val);
912 913

	/* See if the remote side is up */
914
	if (val & BIT(qp->qp_num)) {
915
		dev_info(&pdev->dev, "qp %d: Link Up\n", qp->qp_num);
916 917
		qp->link_is_up = true;

918
		if (qp->event_handler)
919
			qp->event_handler(qp->cb_data, qp->link_is_up);
Allen Hubbe's avatar