ntb_transport.c 50.6 KB
Newer Older
1 2 3 4 5 6 7
/*
 * This file is provided under a dual BSD/GPLv2 license.  When using or
 *   redistributing this file, you may do so under either license.
 *
 *   GPL LICENSE SUMMARY
 *
 *   Copyright(c) 2012 Intel Corporation. All rights reserved.
8
 *   Copyright (C) 2015 EMC Corporation. All Rights Reserved.
9 10 11 12 13 14 15 16
 *
 *   This program is free software; you can redistribute it and/or modify
 *   it under the terms of version 2 of the GNU General Public License as
 *   published by the Free Software Foundation.
 *
 *   BSD LICENSE
 *
 *   Copyright(c) 2012 Intel Corporation. All rights reserved.
17
 *   Copyright (C) 2015 EMC Corporation. All Rights Reserved.
18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44
 *
 *   Redistribution and use in source and binary forms, with or without
 *   modification, are permitted provided that the following conditions
 *   are met:
 *
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copy
 *       notice, this list of conditions and the following disclaimer in
 *       the documentation and/or other materials provided with the
 *       distribution.
 *     * Neither the name of Intel Corporation nor the names of its
 *       contributors may be used to endorse or promote products derived
 *       from this software without specific prior written permission.
 *
 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
45
 * PCIe NTB Transport Linux driver
46 47 48 49 50 51
 *
 * Contact Information:
 * Jon Mason <jon.mason@intel.com>
 */
#include <linux/debugfs.h>
#include <linux/delay.h>
52
#include <linux/dmaengine.h>
53 54 55 56 57 58 59 60
#include <linux/dma-mapping.h>
#include <linux/errno.h>
#include <linux/export.h>
#include <linux/interrupt.h>
#include <linux/module.h>
#include <linux/pci.h>
#include <linux/slab.h>
#include <linux/types.h>
61
#include <linux/uaccess.h>
62 63
#include "linux/ntb.h"
#include "linux/ntb_transport.h"
64

65 66 67 68 69 70 71 72 73 74 75 76 77
#define NTB_TRANSPORT_VERSION	4
#define NTB_TRANSPORT_VER	"4"
#define NTB_TRANSPORT_NAME	"ntb_transport"
#define NTB_TRANSPORT_DESC	"Software Queue-Pair Transport over NTB"

MODULE_DESCRIPTION(NTB_TRANSPORT_DESC);
MODULE_VERSION(NTB_TRANSPORT_VER);
MODULE_LICENSE("Dual BSD/GPL");
MODULE_AUTHOR("Intel Corporation");

static unsigned long max_mw_size;
module_param(max_mw_size, ulong, 0644);
MODULE_PARM_DESC(max_mw_size, "Limit size of large memory windows");
78

79
static unsigned int transport_mtu = 0x10000;
80 81 82
module_param(transport_mtu, uint, 0644);
MODULE_PARM_DESC(transport_mtu, "Maximum size of NTB transport packets");

Jon Mason's avatar
Jon Mason committed
83
static unsigned char max_num_clients;
84 85 86
module_param(max_num_clients, byte, 0644);
MODULE_PARM_DESC(max_num_clients, "Maximum number of NTB transport clients");

87 88 89 90
static unsigned int copy_bytes = 1024;
module_param(copy_bytes, uint, 0644);
MODULE_PARM_DESC(copy_bytes, "Threshold under which NTB will use the CPU to copy instead of DMA");

91 92 93 94
static bool use_dma;
module_param(use_dma, bool, 0644);
MODULE_PARM_DESC(use_dma, "Use DMA engine to perform large data copy");

95 96
static struct dentry *nt_debugfs_dir;

97 98 99
struct ntb_queue_entry {
	/* ntb_queue list reference */
	struct list_head entry;
100
	/* pointers to data to be transferred */
101 102 103 104
	void *cb_data;
	void *buf;
	unsigned int len;
	unsigned int flags;
105 106 107 108 109 110 111

	struct ntb_transport_qp *qp;
	union {
		struct ntb_payload_header __iomem *tx_hdr;
		struct ntb_payload_header *rx_hdr;
	};
	unsigned int index;
112 113
};

Jon Mason's avatar
Jon Mason committed
114 115 116 117
struct ntb_rx_info {
	unsigned int entry;
};

118
struct ntb_transport_qp {
119 120
	struct ntb_transport_ctx *transport;
	struct ntb_dev *ndev;
121
	void *cb_data;
122
	struct dma_chan *dma_chan;
123 124

	bool client_ready;
125 126
	bool link_is_up;

127
	u8 qp_num;	/* Only 64 QP's are allowed.  0-63 */
128
	u64 qp_bit;
129

Jon Mason's avatar
Jon Mason committed
130
	struct ntb_rx_info __iomem *rx_info;
Jon Mason's avatar
Jon Mason committed
131 132
	struct ntb_rx_info *remote_rx_info;

Jon Mason's avatar
Jon Mason committed
133 134
	void (*tx_handler)(struct ntb_transport_qp *qp, void *qp_data,
			   void *data, int len);
135 136
	struct list_head tx_free_q;
	spinlock_t ntb_tx_free_q_lock;
Jon Mason's avatar
Jon Mason committed
137
	void __iomem *tx_mw;
138
	dma_addr_t tx_mw_phys;
Jon Mason's avatar
Jon Mason committed
139 140
	unsigned int tx_index;
	unsigned int tx_max_entry;
141
	unsigned int tx_max_frame;
142

Jon Mason's avatar
Jon Mason committed
143 144
	void (*rx_handler)(struct ntb_transport_qp *qp, void *qp_data,
			   void *data, int len);
145
	struct list_head rx_post_q;
146 147
	struct list_head rx_pend_q;
	struct list_head rx_free_q;
148 149
	/* ntb_rx_q_lock: synchronize access to rx_XXXX_q */
	spinlock_t ntb_rx_q_lock;
Jon Mason's avatar
Jon Mason committed
150 151 152
	void *rx_buff;
	unsigned int rx_index;
	unsigned int rx_max_entry;
153
	unsigned int rx_max_frame;
154
	dma_cookie_t last_cookie;
155
	struct tasklet_struct rxc_db_work;
156

Jon Mason's avatar
Jon Mason committed
157
	void (*event_handler)(void *data, int status);
158
	struct delayed_work link_work;
159
	struct work_struct link_cleanup;
160 161 162 163 164 165 166 167 168 169 170

	struct dentry *debugfs_dir;
	struct dentry *debugfs_stats;

	/* Stats */
	u64 rx_bytes;
	u64 rx_pkts;
	u64 rx_ring_empty;
	u64 rx_err_no_buf;
	u64 rx_err_oflow;
	u64 rx_err_ver;
171 172
	u64 rx_memcpy;
	u64 rx_async;
173 174 175
	u64 tx_bytes;
	u64 tx_pkts;
	u64 tx_ring_full;
176 177 178
	u64 tx_err_no_buf;
	u64 tx_memcpy;
	u64 tx_async;
179 180 181
};

struct ntb_transport_mw {
182 183 184 185 186 187 188
	phys_addr_t phys_addr;
	resource_size_t phys_size;
	resource_size_t xlat_align;
	resource_size_t xlat_align_size;
	void __iomem *vbase;
	size_t xlat_size;
	size_t buff_size;
189 190 191 192 193 194
	void *virt_addr;
	dma_addr_t dma_addr;
};

struct ntb_transport_client_dev {
	struct list_head entry;
195
	struct ntb_transport_ctx *nt;
196 197 198
	struct device dev;
};

199
struct ntb_transport_ctx {
200 201 202
	struct list_head entry;
	struct list_head client_devs;

203 204 205 206 207 208 209 210 211 212
	struct ntb_dev *ndev;

	struct ntb_transport_mw *mw_vec;
	struct ntb_transport_qp *qp_vec;
	unsigned int mw_count;
	unsigned int qp_count;
	u64 qp_bitmap;
	u64 qp_bitmap_free;

	bool link_is_up;
213
	struct delayed_work link_work;
214
	struct work_struct link_cleanup;
215 216

	struct dentry *debugfs_node_dir;
217 218 219
};

enum {
220 221
	DESC_DONE_FLAG = BIT(0),
	LINK_DOWN_FLAG = BIT(1),
222 223 224
};

struct ntb_payload_header {
Jon Mason's avatar
Jon Mason committed
225
	unsigned int ver;
226 227 228 229 230 231 232
	unsigned int len;
	unsigned int flags;
};

enum {
	VERSION = 0,
	QP_LINKS,
Jon Mason's avatar
Jon Mason committed
233 234 235 236 237 238
	NUM_QPS,
	NUM_MWS,
	MW0_SZ_HIGH,
	MW0_SZ_LOW,
	MW1_SZ_HIGH,
	MW1_SZ_LOW,
239 240 241
	MAX_SPAD,
};

242 243 244 245 246 247 248
#define dev_client_dev(__dev) \
	container_of((__dev), struct ntb_transport_client_dev, dev)

#define drv_client(__drv) \
	container_of((__drv), struct ntb_transport_client, driver)

#define QP_TO_MW(nt, qp)	((qp) % nt->mw_count)
249 250 251
#define NTB_QP_DEF_NUM_ENTRIES	100
#define NTB_LINK_DOWN_TIMEOUT	10

252 253 254 255 256 257
static void ntb_transport_rxc_db(unsigned long data);
static const struct ntb_ctx_ops ntb_transport_ops;
static struct ntb_client ntb_transport_client;

static int ntb_transport_bus_match(struct device *dev,
				   struct device_driver *drv)
258 259 260 261
{
	return !strncmp(dev_name(dev), drv->name, strlen(drv->name));
}

262
static int ntb_transport_bus_probe(struct device *dev)
263
{
264
	const struct ntb_transport_client *client;
265 266 267
	int rc = -EINVAL;

	get_device(dev);
268 269 270

	client = drv_client(dev->driver);
	rc = client->probe(dev);
271 272 273 274 275 276
	if (rc)
		put_device(dev);

	return rc;
}

277
static int ntb_transport_bus_remove(struct device *dev)
278
{
279
	const struct ntb_transport_client *client;
280

281 282
	client = drv_client(dev->driver);
	client->remove(dev);
283 284 285 286 287 288

	put_device(dev);

	return 0;
}

289 290 291 292 293
static struct bus_type ntb_transport_bus = {
	.name = "ntb_transport",
	.match = ntb_transport_bus_match,
	.probe = ntb_transport_bus_probe,
	.remove = ntb_transport_bus_remove,
294 295 296 297
};

static LIST_HEAD(ntb_transport_list);

298
static int ntb_bus_init(struct ntb_transport_ctx *nt)
299 300 301 302 303
{
	list_add(&nt->entry, &ntb_transport_list);
	return 0;
}

304
static void ntb_bus_remove(struct ntb_transport_ctx *nt)
305 306 307 308 309 310 311 312 313 314 315 316 317
{
	struct ntb_transport_client_dev *client_dev, *cd;

	list_for_each_entry_safe(client_dev, cd, &nt->client_devs, entry) {
		dev_err(client_dev->dev.parent, "%s still attached to bus, removing\n",
			dev_name(&client_dev->dev));
		list_del(&client_dev->entry);
		device_unregister(&client_dev->dev);
	}

	list_del(&nt->entry);
}

318
static void ntb_transport_client_release(struct device *dev)
319 320 321
{
	struct ntb_transport_client_dev *client_dev;

322
	client_dev = dev_client_dev(dev);
323 324 325 326
	kfree(client_dev);
}

/**
327
 * ntb_transport_unregister_client_dev - Unregister NTB client device
328 329 330 331
 * @device_name: Name of NTB client device
 *
 * Unregister an NTB client device with the NTB transport layer
 */
332
void ntb_transport_unregister_client_dev(char *device_name)
333 334
{
	struct ntb_transport_client_dev *client, *cd;
335
	struct ntb_transport_ctx *nt;
336 337 338 339 340 341 342 343 344

	list_for_each_entry(nt, &ntb_transport_list, entry)
		list_for_each_entry_safe(client, cd, &nt->client_devs, entry)
			if (!strncmp(dev_name(&client->dev), device_name,
				     strlen(device_name))) {
				list_del(&client->entry);
				device_unregister(&client->dev);
			}
}
345
EXPORT_SYMBOL_GPL(ntb_transport_unregister_client_dev);
346 347

/**
348
 * ntb_transport_register_client_dev - Register NTB client device
349 350 351 352
 * @device_name: Name of NTB client device
 *
 * Register an NTB client device with the NTB transport layer
 */
353
int ntb_transport_register_client_dev(char *device_name)
354 355
{
	struct ntb_transport_client_dev *client_dev;
356
	struct ntb_transport_ctx *nt;
357
	int node;
Jon Mason's avatar
Jon Mason committed
358
	int rc, i = 0;
359

360 361 362
	if (list_empty(&ntb_transport_list))
		return -ENODEV;

363 364 365
	list_for_each_entry(nt, &ntb_transport_list, entry) {
		struct device *dev;

366 367 368 369
		node = dev_to_node(&nt->ndev->dev);

		client_dev = kzalloc_node(sizeof(*client_dev),
					  GFP_KERNEL, node);
370 371 372 373 374 375 376 377
		if (!client_dev) {
			rc = -ENOMEM;
			goto err;
		}

		dev = &client_dev->dev;

		/* setup and register client devices */
Jon Mason's avatar
Jon Mason committed
378
		dev_set_name(dev, "%s%d", device_name, i);
379 380 381
		dev->bus = &ntb_transport_bus;
		dev->release = ntb_transport_client_release;
		dev->parent = &nt->ndev->dev;
382 383 384 385 386 387 388 389

		rc = device_register(dev);
		if (rc) {
			kfree(client_dev);
			goto err;
		}

		list_add_tail(&client_dev->entry, &nt->client_devs);
Jon Mason's avatar
Jon Mason committed
390
		i++;
391 392 393 394 395
	}

	return 0;

err:
396
	ntb_transport_unregister_client_dev(device_name);
397 398 399

	return rc;
}
400
EXPORT_SYMBOL_GPL(ntb_transport_register_client_dev);
401 402

/**
403
 * ntb_transport_register_client - Register NTB client driver
404 405 406 407 408 409
 * @drv: NTB client driver to be registered
 *
 * Register an NTB client driver with the NTB transport layer
 *
 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
 */
410
int ntb_transport_register_client(struct ntb_transport_client *drv)
411
{
412
	drv->driver.bus = &ntb_transport_bus;
413

414 415 416
	if (list_empty(&ntb_transport_list))
		return -ENODEV;

417 418
	return driver_register(&drv->driver);
}
419
EXPORT_SYMBOL_GPL(ntb_transport_register_client);
420 421

/**
422
 * ntb_transport_unregister_client - Unregister NTB client driver
423 424 425 426 427 428
 * @drv: NTB client driver to be unregistered
 *
 * Unregister an NTB client driver with the NTB transport layer
 *
 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
 */
429
void ntb_transport_unregister_client(struct ntb_transport_client *drv)
430 431 432
{
	driver_unregister(&drv->driver);
}
433
EXPORT_SYMBOL_GPL(ntb_transport_unregister_client);
434 435 436 437 438

static ssize_t debugfs_read(struct file *filp, char __user *ubuf, size_t count,
			    loff_t *offp)
{
	struct ntb_transport_qp *qp;
439
	char *buf;
440 441
	ssize_t ret, out_offset, out_count;

442 443 444 445 446
	qp = filp->private_data;

	if (!qp || !qp->link_is_up)
		return 0;

447
	out_count = 1000;
448 449 450 451

	buf = kmalloc(out_count, GFP_KERNEL);
	if (!buf)
		return -ENOMEM;
452 453 454 455 456 457 458 459

	out_offset = 0;
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "NTB QP stats\n");
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_bytes - \t%llu\n", qp->rx_bytes);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_pkts - \t%llu\n", qp->rx_pkts);
460 461 462 463
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_memcpy - \t%llu\n", qp->rx_memcpy);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_async - \t%llu\n", qp->rx_async);
464 465 466 467 468 469 470 471 472
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_ring_empty - %llu\n", qp->rx_ring_empty);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_err_no_buf - %llu\n", qp->rx_err_no_buf);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_err_oflow - \t%llu\n", qp->rx_err_oflow);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "rx_err_ver - \t%llu\n", qp->rx_err_ver);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
Jon Mason's avatar
Jon Mason committed
473
			       "rx_buff - \t%p\n", qp->rx_buff);
474
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
Jon Mason's avatar
Jon Mason committed
475
			       "rx_index - \t%u\n", qp->rx_index);
476
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
Jon Mason's avatar
Jon Mason committed
477
			       "rx_max_entry - \t%u\n", qp->rx_max_entry);
478 479 480 481 482

	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "tx_bytes - \t%llu\n", qp->tx_bytes);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "tx_pkts - \t%llu\n", qp->tx_pkts);
483 484 485 486
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "tx_memcpy - \t%llu\n", qp->tx_memcpy);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "tx_async - \t%llu\n", qp->tx_async);
487 488
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "tx_ring_full - \t%llu\n", qp->tx_ring_full);
489 490
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "tx_err_no_buf - %llu\n", qp->tx_err_no_buf);
491
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
Jon Mason's avatar
Jon Mason committed
492
			       "tx_mw - \t%p\n", qp->tx_mw);
493
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
Jon Mason's avatar
Jon Mason committed
494
			       "tx_index - \t%u\n", qp->tx_index);
495
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
Jon Mason's avatar
Jon Mason committed
496
			       "tx_max_entry - \t%u\n", qp->tx_max_entry);
497 498 499 500 501 502
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "qp->remote_rx_info->entry - \t%u\n",
			       qp->remote_rx_info->entry);
	out_offset += snprintf(buf + out_offset, out_count - out_offset,
			       "free tx - \t%u\n",
			       ntb_transport_tx_free_entry(qp));
503 504

	out_offset += snprintf(buf + out_offset, out_count - out_offset,
505 506
			       "\nQP Link %s\n",
			       qp->link_is_up ? "Up" : "Down");
507 508
	if (out_offset > out_count)
		out_offset = out_count;
509 510

	ret = simple_read_from_buffer(ubuf, count, offp, buf, out_offset);
511
	kfree(buf);
512 513 514 515 516
	return ret;
}

static const struct file_operations ntb_qp_debugfs_stats = {
	.owner = THIS_MODULE,
Jon Mason's avatar
Jon Mason committed
517
	.open = simple_open,
518 519 520 521 522 523 524 525 526 527 528 529 530 531
	.read = debugfs_read,
};

static void ntb_list_add(spinlock_t *lock, struct list_head *entry,
			 struct list_head *list)
{
	unsigned long flags;

	spin_lock_irqsave(lock, flags);
	list_add_tail(entry, list);
	spin_unlock_irqrestore(lock, flags);
}

static struct ntb_queue_entry *ntb_list_rm(spinlock_t *lock,
Jon Mason's avatar
Jon Mason committed
532
					   struct list_head *list)
533 534 535 536 537 538 539 540 541 542 543
{
	struct ntb_queue_entry *entry;
	unsigned long flags;

	spin_lock_irqsave(lock, flags);
	if (list_empty(list)) {
		entry = NULL;
		goto out;
	}
	entry = list_first_entry(list, struct ntb_queue_entry, entry);
	list_del(&entry->entry);
544

545 546 547 548 549 550
out:
	spin_unlock_irqrestore(lock, flags);

	return entry;
}

551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571
static struct ntb_queue_entry *ntb_list_mv(spinlock_t *lock,
					   struct list_head *list,
					   struct list_head *to_list)
{
	struct ntb_queue_entry *entry;
	unsigned long flags;

	spin_lock_irqsave(lock, flags);

	if (list_empty(list)) {
		entry = NULL;
	} else {
		entry = list_first_entry(list, struct ntb_queue_entry, entry);
		list_move_tail(&entry->entry, to_list);
	}

	spin_unlock_irqrestore(lock, flags);

	return entry;
}

572 573
static int ntb_transport_setup_qp_mw(struct ntb_transport_ctx *nt,
				     unsigned int qp_num)
574
{
575 576
	struct ntb_transport_qp *qp = &nt->qp_vec[qp_num];
	struct ntb_transport_mw *mw;
577
	unsigned int rx_size, num_qps_mw;
578
	unsigned int mw_num, mw_count, qp_count;
Jon Mason's avatar
Jon Mason committed
579
	unsigned int i;
580

581 582
	mw_count = nt->mw_count;
	qp_count = nt->qp_count;
Jon Mason's avatar
Jon Mason committed
583

584 585 586 587 588
	mw_num = QP_TO_MW(nt, qp_num);
	mw = &nt->mw_vec[mw_num];

	if (!mw->virt_addr)
		return -ENOMEM;
589

590 591
	if (qp_count % mw_count && mw_num + 1 < qp_count / mw_count)
		num_qps_mw = qp_count / mw_count + 1;
592
	else
593
		num_qps_mw = qp_count / mw_count;
594

595 596
	rx_size = (unsigned int)mw->xlat_size / num_qps_mw;
	qp->rx_buff = mw->virt_addr + rx_size * qp_num / mw_count;
Jon Mason's avatar
Jon Mason committed
597 598
	rx_size -= sizeof(struct ntb_rx_info);

599 600
	qp->remote_rx_info = qp->rx_buff + rx_size;

601 602
	/* Due to housekeeping, there must be atleast 2 buffs */
	qp->rx_max_frame = min(transport_mtu, rx_size / 2);
Jon Mason's avatar
Jon Mason committed
603 604 605
	qp->rx_max_entry = rx_size / qp->rx_max_frame;
	qp->rx_index = 0;

606
	qp->remote_rx_info->entry = qp->rx_max_entry - 1;
607

608
	/* setup the hdr offsets with 0's */
Jon Mason's avatar
Jon Mason committed
609
	for (i = 0; i < qp->rx_max_entry; i++) {
610 611
		void *offset = (qp->rx_buff + qp->rx_max_frame * (i + 1) -
				sizeof(struct ntb_payload_header));
612
		memset(offset, 0, sizeof(struct ntb_payload_header));
Jon Mason's avatar
Jon Mason committed
613
	}
614 615 616

	qp->rx_pkts = 0;
	qp->tx_pkts = 0;
Jon Mason's avatar
Jon Mason committed
617
	qp->tx_index = 0;
618 619

	return 0;
620 621
}

622
static void ntb_free_mw(struct ntb_transport_ctx *nt, int num_mw)
Jon Mason's avatar
Jon Mason committed
623
{
624 625
	struct ntb_transport_mw *mw = &nt->mw_vec[num_mw];
	struct pci_dev *pdev = nt->ndev->pdev;
Jon Mason's avatar
Jon Mason committed
626 627 628 629

	if (!mw->virt_addr)
		return;

630 631 632 633 634
	ntb_mw_clear_trans(nt->ndev, num_mw);
	dma_free_coherent(&pdev->dev, mw->buff_size,
			  mw->virt_addr, mw->dma_addr);
	mw->xlat_size = 0;
	mw->buff_size = 0;
Jon Mason's avatar
Jon Mason committed
635 636 637
	mw->virt_addr = NULL;
}

638
static int ntb_set_mw(struct ntb_transport_ctx *nt, int num_mw,
639
		      resource_size_t size)
640
{
641 642
	struct ntb_transport_mw *mw = &nt->mw_vec[num_mw];
	struct pci_dev *pdev = nt->ndev->pdev;
643
	size_t xlat_size, buff_size;
644 645
	int rc;

646 647 648
	if (!size)
		return -EINVAL;

649 650
	xlat_size = round_up(size, mw->xlat_align_size);
	buff_size = round_up(size, mw->xlat_align);
651

Jon Mason's avatar
Jon Mason committed
652
	/* No need to re-setup */
653
	if (mw->xlat_size == xlat_size)
Jon Mason's avatar
Jon Mason committed
654 655
		return 0;

656
	if (mw->buff_size)
Jon Mason's avatar
Jon Mason committed
657 658
		ntb_free_mw(nt, num_mw);

659 660 661
	/* Alloc memory for receiving data.  Must be aligned */
	mw->xlat_size = xlat_size;
	mw->buff_size = buff_size;
662

663 664
	mw->virt_addr = dma_alloc_coherent(&pdev->dev, buff_size,
					   &mw->dma_addr, GFP_KERNEL);
665
	if (!mw->virt_addr) {
666 667
		mw->xlat_size = 0;
		mw->buff_size = 0;
668
		dev_err(&pdev->dev, "Unable to alloc MW buff of size %zu\n",
669
			buff_size);
670 671 672
		return -ENOMEM;
	}

673 674 675 676 677 678
	/*
	 * we must ensure that the memory address allocated is BAR size
	 * aligned in order for the XLAT register to take the value. This
	 * is a requirement of the hardware. It is recommended to setup CMA
	 * for BAR sizes equal or greater than 4MB.
	 */
679 680
	if (!IS_ALIGNED(mw->dma_addr, mw->xlat_align)) {
		dev_err(&pdev->dev, "DMA memory %pad is not aligned\n",
681 682 683 684 685
			&mw->dma_addr);
		ntb_free_mw(nt, num_mw);
		return -ENOMEM;
	}

686
	/* Notify HW the memory location of the receive buffer */
687 688 689 690 691 692
	rc = ntb_mw_set_trans(nt->ndev, num_mw, mw->dma_addr, mw->xlat_size);
	if (rc) {
		dev_err(&pdev->dev, "Unable to set mw%d translation", num_mw);
		ntb_free_mw(nt, num_mw);
		return -EIO;
	}
693 694 695 696

	return 0;
}

697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718
static void ntb_qp_link_down_reset(struct ntb_transport_qp *qp)
{
	qp->link_is_up = false;

	qp->tx_index = 0;
	qp->rx_index = 0;
	qp->rx_bytes = 0;
	qp->rx_pkts = 0;
	qp->rx_ring_empty = 0;
	qp->rx_err_no_buf = 0;
	qp->rx_err_oflow = 0;
	qp->rx_err_ver = 0;
	qp->rx_memcpy = 0;
	qp->rx_async = 0;
	qp->tx_bytes = 0;
	qp->tx_pkts = 0;
	qp->tx_ring_full = 0;
	qp->tx_err_no_buf = 0;
	qp->tx_memcpy = 0;
	qp->tx_async = 0;
}

719
static void ntb_qp_link_cleanup(struct ntb_transport_qp *qp)
720
{
721 722
	struct ntb_transport_ctx *nt = qp->transport;
	struct pci_dev *pdev = nt->ndev->pdev;
723

724
	dev_info(&pdev->dev, "qp %d: Link Cleanup\n", qp->qp_num);
725 726 727

	cancel_delayed_work_sync(&qp->link_work);
	ntb_qp_link_down_reset(qp);
728 729 730

	if (qp->event_handler)
		qp->event_handler(qp->cb_data, qp->link_is_up);
731 732 733 734 735 736 737
}

static void ntb_qp_link_cleanup_work(struct work_struct *work)
{
	struct ntb_transport_qp *qp = container_of(work,
						   struct ntb_transport_qp,
						   link_cleanup);
738
	struct ntb_transport_ctx *nt = qp->transport;
739 740

	ntb_qp_link_cleanup(qp);
741

742
	if (nt->link_is_up)
743 744 745 746
		schedule_delayed_work(&qp->link_work,
				      msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT));
}

747 748 749 750 751
static void ntb_qp_link_down(struct ntb_transport_qp *qp)
{
	schedule_work(&qp->link_cleanup);
}

752
static void ntb_transport_link_cleanup(struct ntb_transport_ctx *nt)
753
{
754 755
	struct ntb_transport_qp *qp;
	u64 qp_bitmap_alloc;
756 757
	int i;

758 759
	qp_bitmap_alloc = nt->qp_bitmap & ~nt->qp_bitmap_free;

760
	/* Pass along the info to any clients */
761 762 763 764 765 766 767
	for (i = 0; i < nt->qp_count; i++)
		if (qp_bitmap_alloc & BIT_ULL(i)) {
			qp = &nt->qp_vec[i];
			ntb_qp_link_cleanup(qp);
			cancel_work_sync(&qp->link_cleanup);
			cancel_delayed_work_sync(&qp->link_work);
		}
768

769
	if (!nt->link_is_up)
770 771 772 773 774 775 776
		cancel_delayed_work_sync(&nt->link_work);

	/* The scratchpad registers keep the values if the remote side
	 * goes down, blast them now to give them a sane value the next
	 * time they are accessed
	 */
	for (i = 0; i < MAX_SPAD; i++)
777
		ntb_spad_write(nt->ndev, i, 0);
778 779
}

780 781
static void ntb_transport_link_cleanup_work(struct work_struct *work)
{
782 783
	struct ntb_transport_ctx *nt =
		container_of(work, struct ntb_transport_ctx, link_cleanup);
784 785 786 787

	ntb_transport_link_cleanup(nt);
}

788
static void ntb_transport_event_callback(void *data)
789
{
790
	struct ntb_transport_ctx *nt = data;
791

792
	if (ntb_link_is_up(nt->ndev, NULL, NULL) == 1)
793
		schedule_delayed_work(&nt->link_work, 0);
794
	else
795
		schedule_work(&nt->link_cleanup);
796 797 798 799
}

static void ntb_transport_link_work(struct work_struct *work)
{
800 801 802 803 804
	struct ntb_transport_ctx *nt =
		container_of(work, struct ntb_transport_ctx, link_work.work);
	struct ntb_dev *ndev = nt->ndev;
	struct pci_dev *pdev = ndev->pdev;
	resource_size_t size;
805
	u32 val;
806
	int rc, i, spad;
807

Jon Mason's avatar
Jon Mason committed
808
	/* send the local info, in the opposite order of the way we read it */
809 810
	for (i = 0; i < nt->mw_count; i++) {
		size = nt->mw_vec[i].phys_size;
811

812 813
		if (max_mw_size && size > max_mw_size)
			size = max_mw_size;
814

815 816
		spad = MW0_SZ_HIGH + (i * 2);
		ntb_peer_spad_write(ndev, spad, (u32)(size >> 32));
817

818 819
		spad = MW0_SZ_LOW + (i * 2);
		ntb_peer_spad_write(ndev, spad, (u32)size);
820 821
	}

822
	ntb_peer_spad_write(ndev, NUM_MWS, nt->mw_count);
823

824
	ntb_peer_spad_write(ndev, NUM_QPS, nt->qp_count);
825

826
	ntb_peer_spad_write(ndev, VERSION, NTB_TRANSPORT_VERSION);
827

828
	/* Query the remote side for its info */
829
	val = ntb_spad_read(ndev, VERSION);
830 831
	dev_dbg(&pdev->dev, "Remote version = %d\n", val);
	if (val != NTB_TRANSPORT_VERSION)
832 833
		goto out;

834
	val = ntb_spad_read(ndev, NUM_QPS);
835
	dev_dbg(&pdev->dev, "Remote max number of qps = %d\n", val);
836
	if (val != nt->qp_count)
837 838
		goto out;

839
	val = ntb_spad_read(ndev, NUM_MWS);
Jon Mason's avatar
Jon Mason committed
840
	dev_dbg(&pdev->dev, "Remote number of mws = %d\n", val);
841 842
	if (val != nt->mw_count)
		goto out;
843

844
	for (i = 0; i < nt->mw_count; i++) {
Jon Mason's avatar
Jon Mason committed
845
		u64 val64;
846

847
		val = ntb_spad_read(ndev, MW0_SZ_HIGH + (i * 2));
848
		val64 = (u64)val << 32;
Jon Mason's avatar
Jon Mason committed
849

850
		val = ntb_spad_read(ndev, MW0_SZ_LOW + (i * 2));
Jon Mason's avatar
Jon Mason committed
851 852
		val64 |= val;

853
		dev_dbg(&pdev->dev, "Remote MW%d size = %#llx\n", i, val64);
Jon Mason's avatar
Jon Mason committed
854 855 856 857 858

		rc = ntb_set_mw(nt, i, val64);
		if (rc)
			goto out1;
	}
859

860
	nt->link_is_up = true;
861

862 863
	for (i = 0; i < nt->qp_count; i++) {
		struct ntb_transport_qp *qp = &nt->qp_vec[i];
864 865 866

		ntb_transport_setup_qp_mw(nt, i);

867
		if (qp->client_ready)
868 869 870 871 872
			schedule_delayed_work(&qp->link_work, 0);
	}

	return;

Jon Mason's avatar
Jon Mason committed
873
out1:
874
	for (i = 0; i < nt->mw_count; i++)
Jon Mason's avatar
Jon Mason committed
875
		ntb_free_mw(nt, i);
876
out:
877
	if (ntb_link_is_up(ndev, NULL, NULL) == 1)
878 879 880 881 882 883 884 885 886
		schedule_delayed_work(&nt->link_work,
				      msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT));
}

static void ntb_qp_link_work(struct work_struct *work)
{
	struct ntb_transport_qp *qp = container_of(work,
						   struct ntb_transport_qp,
						   link_work.work);
887 888 889
	struct pci_dev *pdev = qp->ndev->pdev;
	struct ntb_transport_ctx *nt = qp->transport;
	int val;
890

891
	WARN_ON(!nt->link_is_up);
892

893
	val = ntb_spad_read(nt->ndev, QP_LINKS);
894

895
	ntb_peer_spad_write(nt->ndev, QP_LINKS, val | BIT(qp->qp_num));
896 897

	/* query remote spad for qp ready bits */
898
	ntb_peer_spad_read(nt->ndev, QP_LINKS);
Allen Hubbe's avatar
Allen Hubbe committed
899
	dev_dbg_ratelimited(&pdev->dev, "Remote QP link status = %x\n", val);
900 901

	/* See if the remote side is up */
902
	if (val & BIT(qp->qp_num)) {
903
		dev_info(&pdev->dev, "qp %d: Link Up\n", qp->qp_num);
904 905
		qp->link_is_up = true;

906
		if (qp->event_handler)
907
			qp->event_handler(qp->cb_data, qp->link_is_up);
908 909

		tasklet_schedule(&qp->rxc_db_work);
910
	} else if (nt->link_is_up)
911 912 913 914
		schedule_delayed_work(&qp->link_work,
				      msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT));
}

915
static int ntb_transport_init_queue(struct ntb_transport_ctx *nt,
Jon Mason's avatar
Jon Mason committed
916
				    unsigned int qp_num)
917 918
{
	struct ntb_transport_qp *qp;
919 920 921
	struct ntb_transport_mw *mw;
	phys_addr_t mw_base;
	resource_size_t mw_size;
922
	unsigned int num_qps_mw, tx_size;
923
	unsigned int mw_num, mw_count, qp_count;
Jon Mason's avatar
<