Commit b3e27d5d authored by Linus Torvalds's avatar Linus Torvalds

Merge tag 'ntb-4.5' of git://github.com/jonmason/ntb

Pull NTB updates from Jon Mason:
 "A new driver to support AMD NTB, a NTB performance test driver, NTB
  bugs fixes, and the ability to recover from running out of DMA
  descriptors"

* tag 'ntb-4.5' of git://github.com/jonmason/ntb:
  NTB: Fix macro parameter conflict with field name
  NTB: Add support for AMD PCI-Express Non-Transparent Bridge
  ntb: ntb perf tool
  NTB: Address out of DMA descriptor issue with NTB
  NTB: Clear property bits in BAR value
  NTB: ntb_process_tx error path bug
parents cc673757 03beaec8
......@@ -7702,6 +7702,12 @@ W: https://github.com/jonmason/ntb/wiki
T: git git://github.com/jonmason/ntb.git
F: drivers/ntb/hw/intel/
NTB AMD DRIVER
M: Xiangliang Yu <Xiangliang.Yu@amd.com>
L: linux-ntb@googlegroups.com
S: Supported
F: drivers/ntb/hw/amd/
NTFS FILESYSTEM
M: Anton Altaparmakov <anton@tuxera.com>
L: linux-ntfs-dev@lists.sourceforge.net
......
source "drivers/ntb/hw/amd/Kconfig"
source "drivers/ntb/hw/intel/Kconfig"
obj-$(CONFIG_NTB_AMD) += amd/
obj-$(CONFIG_NTB_INTEL) += intel/
config NTB_AMD
tristate "AMD Non-Transparent Bridge support"
depends on X86_64
help
This driver supports AMD NTB on capable Zeppelin hardware.
If unsure, say N.
obj-$(CONFIG_NTB_AMD) += ntb_hw_amd.o
This diff is collapsed.
/*
* This file is provided under a dual BSD/GPLv2 license. When using or
* redistributing this file, you may do so under either license.
*
* GPL LICENSE SUMMARY
*
* Copyright (C) 2016 Advanced Micro Devices, Inc. All Rights Reserved.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of version 2 of the GNU General Public License as
* published by the Free Software Foundation.
*
* BSD LICENSE
*
* Copyright (C) 2016 Advanced Micro Devices, Inc. All Rights Reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copy
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of AMD Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* AMD PCIe NTB Linux driver
*
* Contact Information:
* Xiangliang Yu <Xiangliang.Yu@amd.com>
*/
#ifndef NTB_HW_AMD_H
#define NTB_HW_AMD_H
#include <linux/ntb.h>
#include <linux/pci.h>
#define PCI_DEVICE_ID_AMD_NTB 0x145B
#define AMD_LINK_HB_TIMEOUT msecs_to_jiffies(1000)
#define AMD_LINK_STATUS_OFFSET 0x68
#define NTB_LIN_STA_ACTIVE_BIT 0x00000002
#define NTB_LNK_STA_SPEED_MASK 0x000F0000
#define NTB_LNK_STA_WIDTH_MASK 0x03F00000
#define NTB_LNK_STA_ACTIVE(x) (!!((x) & NTB_LIN_STA_ACTIVE_BIT))
#define NTB_LNK_STA_SPEED(x) (((x) & NTB_LNK_STA_SPEED_MASK) >> 16)
#define NTB_LNK_STA_WIDTH(x) (((x) & NTB_LNK_STA_WIDTH_MASK) >> 20)
#ifndef read64
#ifdef readq
#define read64 readq
#else
#define read64 _read64
static inline u64 _read64(void __iomem *mmio)
{
u64 low, high;
low = readl(mmio);
high = readl(mmio + sizeof(u32));
return low | (high << 32);
}
#endif
#endif
#ifndef write64
#ifdef writeq
#define write64 writeq
#else
#define write64 _write64
static inline void _write64(u64 val, void __iomem *mmio)
{
writel(val, mmio);
writel(val >> 32, mmio + sizeof(u32));
}
#endif
#endif
enum {
/* AMD NTB Capability */
AMD_MW_CNT = 3,
AMD_DB_CNT = 16,
AMD_MSIX_VECTOR_CNT = 24,
AMD_SPADS_CNT = 16,
/* AMD NTB register offset */
AMD_CNTL_OFFSET = 0x200,
/* NTB control register bits */
PMM_REG_CTL = BIT(21),
SMM_REG_CTL = BIT(20),
SMM_REG_ACC_PATH = BIT(18),
PMM_REG_ACC_PATH = BIT(17),
NTB_CLK_EN = BIT(16),
AMD_STA_OFFSET = 0x204,
AMD_PGSLV_OFFSET = 0x208,
AMD_SPAD_MUX_OFFSET = 0x20C,
AMD_SPAD_OFFSET = 0x210,
AMD_RSMU_HCID = 0x250,
AMD_RSMU_SIID = 0x254,
AMD_PSION_OFFSET = 0x300,
AMD_SSION_OFFSET = 0x330,
AMD_MMINDEX_OFFSET = 0x400,
AMD_MMDATA_OFFSET = 0x404,
AMD_SIDEINFO_OFFSET = 0x408,
AMD_SIDE_MASK = BIT(0),
AMD_SIDE_READY = BIT(1),
/* limit register */
AMD_ROMBARLMT_OFFSET = 0x410,
AMD_BAR1LMT_OFFSET = 0x414,
AMD_BAR23LMT_OFFSET = 0x418,
AMD_BAR45LMT_OFFSET = 0x420,
/* xlat address */
AMD_POMBARXLAT_OFFSET = 0x428,
AMD_BAR1XLAT_OFFSET = 0x430,
AMD_BAR23XLAT_OFFSET = 0x438,
AMD_BAR45XLAT_OFFSET = 0x440,
/* doorbell and interrupt */
AMD_DBFM_OFFSET = 0x450,
AMD_DBREQ_OFFSET = 0x454,
AMD_MIRRDBSTAT_OFFSET = 0x458,
AMD_DBMASK_OFFSET = 0x45C,
AMD_DBSTAT_OFFSET = 0x460,
AMD_INTMASK_OFFSET = 0x470,
AMD_INTSTAT_OFFSET = 0x474,
/* event type */
AMD_PEER_FLUSH_EVENT = BIT(0),
AMD_PEER_RESET_EVENT = BIT(1),
AMD_PEER_D3_EVENT = BIT(2),
AMD_PEER_PMETO_EVENT = BIT(3),
AMD_PEER_D0_EVENT = BIT(4),
AMD_EVENT_INTMASK = (AMD_PEER_FLUSH_EVENT |
AMD_PEER_RESET_EVENT | AMD_PEER_D3_EVENT |
AMD_PEER_PMETO_EVENT | AMD_PEER_D0_EVENT),
AMD_PMESTAT_OFFSET = 0x480,
AMD_PMSGTRIG_OFFSET = 0x490,
AMD_LTRLATENCY_OFFSET = 0x494,
AMD_FLUSHTRIG_OFFSET = 0x498,
/* SMU register*/
AMD_SMUACK_OFFSET = 0x4A0,
AMD_SINRST_OFFSET = 0x4A4,
AMD_RSPNUM_OFFSET = 0x4A8,
AMD_SMU_SPADMUTEX = 0x4B0,
AMD_SMU_SPADOFFSET = 0x4B4,
AMD_PEER_OFFSET = 0x400,
};
struct amd_ntb_dev;
struct amd_ntb_vec {
struct amd_ntb_dev *ndev;
int num;
};
struct amd_ntb_dev {
struct ntb_dev ntb;
u32 ntb_side;
u32 lnk_sta;
u32 cntl_sta;
u32 peer_sta;
unsigned char mw_count;
unsigned char spad_count;
unsigned char db_count;
unsigned char msix_vec_count;
u64 db_valid_mask;
u64 db_mask;
u32 int_mask;
struct msix_entry *msix;
struct amd_ntb_vec *vec;
/* synchronize rmw access of db_mask and hw reg */
spinlock_t db_mask_lock;
void __iomem *self_mmio;
void __iomem *peer_mmio;
unsigned int self_spad;
unsigned int peer_spad;
struct delayed_work hb_timer;
struct dentry *debugfs_dir;
struct dentry *debugfs_info;
};
#define ndev_pdev(ndev) ((ndev)->ntb.pdev)
#define ndev_name(ndev) pci_name(ndev_pdev(ndev))
#define ndev_dev(ndev) (&ndev_pdev(ndev)->dev)
#define ntb_ndev(__ntb) container_of(__ntb, struct amd_ntb_dev, ntb)
#define hb_ndev(__work) container_of(__work, struct amd_ntb_dev, hb_timer.work)
#endif
......@@ -875,7 +875,7 @@ static int intel_ntb_mw_set_trans(struct ntb_dev *ntb, int idx,
limit_reg = bar2_off(ndev->xlat_reg->bar2_limit, bar);
if (bar < 4 || !ndev->bar4_split) {
base = ioread64(mmio + base_reg);
base = ioread64(mmio + base_reg) & NTB_BAR_MASK_64;
/* Set the limit if supported, if size is not mw_size */
if (limit_reg && size != mw_size)
......@@ -906,7 +906,7 @@ static int intel_ntb_mw_set_trans(struct ntb_dev *ntb, int idx,
if ((addr + size) & (~0ull << 32))
return -EINVAL;
base = ioread32(mmio + base_reg);
base = ioread32(mmio + base_reg) & NTB_BAR_MASK_32;
/* Set the limit if supported, if size is not mw_size */
if (limit_reg && size != mw_size)
......
......@@ -245,6 +245,9 @@
#define NTB_UNSAFE_DB BIT_ULL(0)
#define NTB_UNSAFE_SPAD BIT_ULL(1)
#define NTB_BAR_MASK_64 ~(0xfull)
#define NTB_BAR_MASK_32 ~(0xfu)
struct intel_ntb_dev;
struct intel_ntb_reg {
......@@ -334,7 +337,8 @@ struct intel_ntb_dev {
#define ndev_pdev(ndev) ((ndev)->ntb.pdev)
#define ndev_name(ndev) pci_name(ndev_pdev(ndev))
#define ndev_dev(ndev) (&ndev_pdev(ndev)->dev)
#define ntb_ndev(ntb) container_of(ntb, struct intel_ntb_dev, ntb)
#define hb_ndev(work) container_of(work, struct intel_ntb_dev, hb_timer.work)
#define ntb_ndev(__ntb) container_of(__ntb, struct intel_ntb_dev, ntb)
#define hb_ndev(__work) container_of(__work, struct intel_ntb_dev, \
hb_timer.work)
#endif
......@@ -171,12 +171,14 @@ struct ntb_transport_qp {
u64 rx_err_ver;
u64 rx_memcpy;
u64 rx_async;
u64 dma_rx_prep_err;
u64 tx_bytes;
u64 tx_pkts;
u64 tx_ring_full;
u64 tx_err_no_buf;
u64 tx_memcpy;
u64 tx_async;
u64 dma_tx_prep_err;
};
struct ntb_transport_mw {
......@@ -249,6 +251,8 @@ enum {
#define QP_TO_MW(nt, qp) ((qp) % nt->mw_count)
#define NTB_QP_DEF_NUM_ENTRIES 100
#define NTB_LINK_DOWN_TIMEOUT 10
#define DMA_RETRIES 20
#define DMA_OUT_RESOURCE_TO 50
static void ntb_transport_rxc_db(unsigned long data);
static const struct ntb_ctx_ops ntb_transport_ops;
......@@ -501,6 +505,12 @@ static ssize_t debugfs_read(struct file *filp, char __user *ubuf, size_t count,
out_offset += snprintf(buf + out_offset, out_count - out_offset,
"free tx - \t%u\n",
ntb_transport_tx_free_entry(qp));
out_offset += snprintf(buf + out_offset, out_count - out_offset,
"DMA tx prep err - \t%llu\n",
qp->dma_tx_prep_err);
out_offset += snprintf(buf + out_offset, out_count - out_offset,
"DMA rx prep err - \t%llu\n",
qp->dma_rx_prep_err);
out_offset += snprintf(buf + out_offset, out_count - out_offset,
"\n");
......@@ -726,6 +736,8 @@ static void ntb_qp_link_down_reset(struct ntb_transport_qp *qp)
qp->tx_err_no_buf = 0;
qp->tx_memcpy = 0;
qp->tx_async = 0;
qp->dma_tx_prep_err = 0;
qp->dma_rx_prep_err = 0;
}
static void ntb_qp_link_cleanup(struct ntb_transport_qp *qp)
......@@ -1228,6 +1240,7 @@ static void ntb_async_rx(struct ntb_queue_entry *entry, void *offset)
struct dmaengine_unmap_data *unmap;
dma_cookie_t cookie;
void *buf = entry->buf;
int retries = 0;
len = entry->len;
......@@ -1263,11 +1276,21 @@ static void ntb_async_rx(struct ntb_queue_entry *entry, void *offset)
unmap->from_cnt = 1;
txd = device->device_prep_dma_memcpy(chan, unmap->addr[1],
unmap->addr[0], len,
DMA_PREP_INTERRUPT);
if (!txd)
for (retries = 0; retries < DMA_RETRIES; retries++) {
txd = device->device_prep_dma_memcpy(chan, unmap->addr[1],
unmap->addr[0], len,
DMA_PREP_INTERRUPT);
if (txd)
break;
set_current_state(TASK_INTERRUPTIBLE);
schedule_timeout(DMA_OUT_RESOURCE_TO);
}
if (!txd) {
qp->dma_rx_prep_err++;
goto err_get_unmap;
}
txd->callback = ntb_rx_copy_callback;
txd->callback_param = entry;
......@@ -1460,6 +1483,7 @@ static void ntb_async_tx(struct ntb_transport_qp *qp,
void __iomem *offset;
size_t len = entry->len;
void *buf = entry->buf;
int retries = 0;
offset = qp->tx_mw + qp->tx_max_frame * qp->tx_index;
hdr = offset + qp->tx_max_frame - sizeof(struct ntb_payload_header);
......@@ -1494,10 +1518,20 @@ static void ntb_async_tx(struct ntb_transport_qp *qp,
unmap->to_cnt = 1;
txd = device->device_prep_dma_memcpy(chan, dest, unmap->addr[0], len,
DMA_PREP_INTERRUPT);
if (!txd)
for (retries = 0; retries < DMA_RETRIES; retries++) {
txd = device->device_prep_dma_memcpy(chan, dest, unmap->addr[0],
len, DMA_PREP_INTERRUPT);
if (txd)
break;
set_current_state(TASK_INTERRUPTIBLE);
schedule_timeout(DMA_OUT_RESOURCE_TO);
}
if (!txd) {
qp->dma_tx_prep_err++;
goto err_get_unmap;
}
txd->callback = ntb_tx_copy_callback;
txd->callback_param = entry;
......@@ -1532,7 +1566,7 @@ static int ntb_process_tx(struct ntb_transport_qp *qp,
if (entry->len > qp->tx_max_frame - sizeof(struct ntb_payload_header)) {
if (qp->tx_handler)
qp->tx_handler(qp->cb_data, qp, NULL, -EIO);
qp->tx_handler(qp, qp->cb_data, NULL, -EIO);
ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry,
&qp->tx_free_q);
......
......@@ -17,3 +17,11 @@ config NTB_TOOL
functioning at a basic level.
If unsure, say N.
config NTB_PERF
tristate "NTB RAW Perf Measuring Tool"
help
This is a tool to measure raw NTB performance by transferring data
to and from the window without additional software interaction.
If unsure, say N.
obj-$(CONFIG_NTB_PINGPONG) += ntb_pingpong.o
obj-$(CONFIG_NTB_TOOL) += ntb_tool.o
obj-$(CONFIG_NTB_PERF) += ntb_perf.o
/*
* This file is provided under a dual BSD/GPLv2 license. When using or
* redistributing this file, you may do so under either license.
*
* GPL LICENSE SUMMARY
*
* Copyright(c) 2015 Intel Corporation. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of version 2 of the GNU General Public License as
* published by the Free Software Foundation.
*
* BSD LICENSE
*
* Copyright(c) 2015 Intel Corporation. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copy
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* PCIe NTB Perf Linux driver
*/
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/kthread.h>
#include <linux/time.h>
#include <linux/timer.h>
#include <linux/dma-mapping.h>
#include <linux/pci.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/debugfs.h>
#include <linux/dmaengine.h>
#include <linux/delay.h>
#include <linux/sizes.h>
#include <linux/ntb.h>
#define DRIVER_NAME "ntb_perf"
#define DRIVER_DESCRIPTION "PCIe NTB Performance Measurement Tool"
#define DRIVER_LICENSE "Dual BSD/GPL"
#define DRIVER_VERSION "1.0"
#define DRIVER_AUTHOR "Dave Jiang <dave.jiang@intel.com>"
#define PERF_LINK_DOWN_TIMEOUT 10
#define PERF_VERSION 0xffff0001
#define MAX_THREADS 32
#define MAX_TEST_SIZE SZ_1M
#define MAX_SRCS 32
#define DMA_OUT_RESOURCE_TO 50
#define DMA_RETRIES 20
#define SZ_4G (1ULL << 32)
#define MAX_SEG_ORDER 20 /* no larger than 1M for kmalloc buffer */
MODULE_LICENSE(DRIVER_LICENSE);
MODULE_VERSION(DRIVER_VERSION);
MODULE_AUTHOR(DRIVER_AUTHOR);
MODULE_DESCRIPTION(DRIVER_DESCRIPTION);
static struct dentry *perf_debugfs_dir;
static unsigned int seg_order = 19; /* 512K */
module_param(seg_order, uint, 0644);
MODULE_PARM_DESC(seg_order, "size order [n^2] of buffer segment for testing");
static unsigned int run_order = 32; /* 4G */
module_param(run_order, uint, 0644);
MODULE_PARM_DESC(run_order, "size order [n^2] of total data to transfer");
static bool use_dma; /* default to 0 */
module_param(use_dma, bool, 0644);
MODULE_PARM_DESC(use_dma, "Using DMA engine to measure performance");
struct perf_mw {
phys_addr_t phys_addr;
resource_size_t phys_size;
resource_size_t xlat_align;
resource_size_t xlat_align_size;
void __iomem *vbase;
size_t xlat_size;
size_t buf_size;
void *virt_addr;
dma_addr_t dma_addr;
};
struct perf_ctx;
struct pthr_ctx {
struct task_struct *thread;
struct perf_ctx *perf;
atomic_t dma_sync;
struct dma_chan *dma_chan;
int dma_prep_err;
int src_idx;
void *srcs[MAX_SRCS];
};
struct perf_ctx {
struct ntb_dev *ntb;
spinlock_t db_lock;
struct perf_mw mw;
bool link_is_up;
struct work_struct link_cleanup;
struct delayed_work link_work;
struct dentry *debugfs_node_dir;
struct dentry *debugfs_run;
struct dentry *debugfs_threads;
u8 perf_threads;
bool run;
struct pthr_ctx pthr_ctx[MAX_THREADS];
atomic_t tsync;
};
enum {
VERSION = 0,
MW_SZ_HIGH,
MW_SZ_LOW,
SPAD_MSG,
SPAD_ACK,
MAX_SPAD
};
static void perf_link_event(void *ctx)
{
struct perf_ctx *perf = ctx;
if (ntb_link_is_up(perf->ntb, NULL, NULL) == 1)
schedule_delayed_work(&perf->link_work, 2*HZ);
else
schedule_work(&perf->link_cleanup);
}
static void perf_db_event(void *ctx, int vec)
{
struct perf_ctx *perf = ctx;
u64 db_bits, db_mask;
db_mask = ntb_db_vector_mask(perf->ntb, vec);
db_bits = ntb_db_read(perf->ntb);
dev_dbg(&perf->ntb->dev, "doorbell vec %d mask %#llx bits %#llx\n",
vec, db_mask, db_bits);
}
static const struct ntb_ctx_ops perf_ops = {
.link_event = perf_link_event,
.db_event = perf_db_event,
};
static void perf_copy_callback(void *data)
{
struct pthr_ctx *pctx = data;
atomic_dec(&pctx->dma_sync);
}
static ssize_t perf_copy(struct pthr_ctx *pctx, char *dst,
char *src, size_t size)
{
struct perf_ctx *perf = pctx->perf;
struct dma_async_tx_descriptor *txd;
struct dma_chan *chan = pctx->dma_chan;
struct dma_device *device;
struct dmaengine_unmap_data *unmap;
dma_cookie_t cookie;
size_t src_off, dst_off;
struct perf_mw *mw = &perf->mw;
u64 vbase, dst_vaddr;
dma_addr_t dst_phys;
int retries = 0;
if (!use_dma) {
memcpy_toio(dst, src, size);
return size;
}
if (!chan) {