Commit 7bb67c14 authored by Shannon Nelson's avatar Shannon Nelson Committed by Linus Torvalds

I/OAT: Add support for version 2 of ioatdma device

Add support for version 2 of the ioatdma device.  This device handles
the descriptor chain and DCA services slightly differently:
 - Instead of moving the dma descriptors between a busy and an idle chain,
   this new version uses a single circular chain so that we don't have
   rewrite the next_descriptor pointers as we add new requests, and the
   device doesn't need to re-read the last descriptor.
 - The new device has the DCA tags defined internally instead of needing
   them defined statically.
Signed-off-by: default avatarShannon Nelson <shannon.nelson@intel.com>
Cc: "Williams, Dan J" <dan.j.williams@intel.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent cc9f2f8f
......@@ -39,10 +39,14 @@ MODULE_LICENSE("GPL");
MODULE_AUTHOR("Intel Corporation");
static struct pci_device_id ioat_pci_tbl[] = {
/* I/OAT v1 platforms */
{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT) },
{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_CNB) },
{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SCNB) },
{ PCI_DEVICE(PCI_VENDOR_ID_UNISYS, PCI_DEVICE_ID_UNISYS_DMA_DIRECTOR) },
/* I/OAT v2 platforms */
{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB) },
{ 0, }
};
......@@ -74,10 +78,17 @@ static int ioat_setup_functionality(struct pci_dev *pdev, void __iomem *iobase)
if (device->dma && ioat_dca_enabled)
device->dca = ioat_dca_init(pdev, iobase);
break;
case IOAT_VER_2_0:
device->dma = ioat_dma_probe(pdev, iobase);
if (device->dma && ioat_dca_enabled)
device->dca = ioat2_dca_init(pdev, iobase);
break;
default:
err = -ENODEV;
break;
}
if (!device->dma)
err = -ENODEV;
return err;
}
......
......@@ -261,3 +261,167 @@ struct dca_provider *ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase)
return dca;
}
static int ioat2_dca_add_requester(struct dca_provider *dca, struct device *dev)
{
struct ioat_dca_priv *ioatdca = dca_priv(dca);
struct pci_dev *pdev;
int i;
u16 id;
u16 global_req_table;
/* This implementation only supports PCI-Express */
if (dev->bus != &pci_bus_type)
return -ENODEV;
pdev = to_pci_dev(dev);
id = dcaid_from_pcidev(pdev);
if (ioatdca->requester_count == ioatdca->max_requesters)
return -ENODEV;
for (i = 0; i < ioatdca->max_requesters; i++) {
if (ioatdca->req_slots[i].pdev == NULL) {
/* found an empty slot */
ioatdca->requester_count++;
ioatdca->req_slots[i].pdev = pdev;
ioatdca->req_slots[i].rid = id;
global_req_table =
readw(ioatdca->dca_base + IOAT_DCA_GREQID_OFFSET);
writel(id | IOAT_DCA_GREQID_VALID,
ioatdca->iobase + global_req_table + (i * 4));
return i;
}
}
/* Error, ioatdma->requester_count is out of whack */
return -EFAULT;
}
static int ioat2_dca_remove_requester(struct dca_provider *dca,
struct device *dev)
{
struct ioat_dca_priv *ioatdca = dca_priv(dca);
struct pci_dev *pdev;
int i;
u16 global_req_table;
/* This implementation only supports PCI-Express */
if (dev->bus != &pci_bus_type)
return -ENODEV;
pdev = to_pci_dev(dev);
for (i = 0; i < ioatdca->max_requesters; i++) {
if (ioatdca->req_slots[i].pdev == pdev) {
global_req_table =
readw(ioatdca->dca_base + IOAT_DCA_GREQID_OFFSET);
writel(0, ioatdca->iobase + global_req_table + (i * 4));
ioatdca->req_slots[i].pdev = NULL;
ioatdca->req_slots[i].rid = 0;
ioatdca->requester_count--;
return i;
}
}
return -ENODEV;
}
static u8 ioat2_dca_get_tag(struct dca_provider *dca, int cpu)
{
u8 tag;
tag = ioat_dca_get_tag(dca, cpu);
tag = (~tag) & 0x1F;
return tag;
}
static struct dca_ops ioat2_dca_ops = {
.add_requester = ioat2_dca_add_requester,
.remove_requester = ioat2_dca_remove_requester,
.get_tag = ioat2_dca_get_tag,
};
static int ioat2_dca_count_dca_slots(void *iobase, u16 dca_offset)
{
int slots = 0;
u32 req;
u16 global_req_table;
global_req_table = readw(iobase + dca_offset + IOAT_DCA_GREQID_OFFSET);
if (global_req_table == 0)
return 0;
do {
req = readl(iobase + global_req_table + (slots * sizeof(u32)));
slots++;
} while ((req & IOAT_DCA_GREQID_LASTID) == 0);
return slots;
}
struct dca_provider *ioat2_dca_init(struct pci_dev *pdev, void __iomem *iobase)
{
struct dca_provider *dca;
struct ioat_dca_priv *ioatdca;
int slots;
int i;
int err;
u32 tag_map;
u16 dca_offset;
u16 csi_fsb_control;
u16 pcie_control;
u8 bit;
if (!system_has_dca_enabled(pdev))
return NULL;
dca_offset = readw(iobase + IOAT_DCAOFFSET_OFFSET);
if (dca_offset == 0)
return NULL;
slots = ioat2_dca_count_dca_slots(iobase, dca_offset);
if (slots == 0)
return NULL;
dca = alloc_dca_provider(&ioat2_dca_ops,
sizeof(*ioatdca)
+ (sizeof(struct ioat_dca_slot) * slots));
if (!dca)
return NULL;
ioatdca = dca_priv(dca);
ioatdca->iobase = iobase;
ioatdca->dca_base = iobase + dca_offset;
ioatdca->max_requesters = slots;
/* some bios might not know to turn these on */
csi_fsb_control = readw(ioatdca->dca_base + IOAT_FSB_CAP_ENABLE_OFFSET);
if ((csi_fsb_control & IOAT_FSB_CAP_ENABLE_PREFETCH) == 0) {
csi_fsb_control |= IOAT_FSB_CAP_ENABLE_PREFETCH;
writew(csi_fsb_control,
ioatdca->dca_base + IOAT_FSB_CAP_ENABLE_OFFSET);
}
pcie_control = readw(ioatdca->dca_base + IOAT_PCI_CAP_ENABLE_OFFSET);
if ((pcie_control & IOAT_PCI_CAP_ENABLE_MEMWR) == 0) {
pcie_control |= IOAT_PCI_CAP_ENABLE_MEMWR;
writew(pcie_control,
ioatdca->dca_base + IOAT_PCI_CAP_ENABLE_OFFSET);
}
/* TODO version, compatibility and configuration checks */
/* copy out the APIC to DCA tag map */
tag_map = readl(ioatdca->dca_base + IOAT_APICID_TAG_MAP_OFFSET);
for (i = 0; i < 5; i++) {
bit = (tag_map >> (4 * i)) & 0x0f;
if (bit < 8)
ioatdca->tag_map[i] = bit | DCA_TAG_MAP_VALID;
else
ioatdca->tag_map[i] = 0;
}
err = register_dca_provider(dca, &pdev->dev);
if (err) {
free_dca_provider(dca);
return NULL;
}
return dca;
}
This diff is collapsed.
/*
* Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved.
* Copyright(c) 2004 - 2007 Intel Corporation. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
......@@ -28,7 +28,7 @@
#include <linux/cache.h>
#include <linux/pci_ids.h>
#define IOAT_DMA_VERSION "1.26"
#define IOAT_DMA_VERSION "2.04"
enum ioat_interrupt {
none = 0,
......@@ -39,6 +39,8 @@ enum ioat_interrupt {
};
#define IOAT_LOW_COMPLETION_MASK 0xffffffc0
#define IOAT_DMA_DCA_ANY_CPU ~0
/**
* struct ioatdma_device - internal representation of a IOAT device
......@@ -47,6 +49,9 @@ enum ioat_interrupt {
* @dma_pool: for allocating DMA descriptors
* @common: embedded struct dma_device
* @version: version of ioatdma device
* @irq_mode: which style irq to use
* @msix_entries: irq handlers
* @idx: per channel data
*/
struct ioatdma_device {
......@@ -63,23 +68,7 @@ struct ioatdma_device {
/**
* struct ioat_dma_chan - internal representation of a DMA channel
* @device:
* @reg_base:
* @sw_in_use:
* @completion:
* @completion_low:
* @completion_high:
* @completed_cookie: last cookie seen completed on cleanup
* @cookie: value of last cookie given to client
* @last_completion:
* @xfercap:
* @desc_lock:
* @free_desc:
* @used_desc:
* @resource:
* @device_node:
*/
struct ioat_dma_chan {
void __iomem *reg_base;
......@@ -95,6 +84,8 @@ struct ioat_dma_chan {
struct list_head used_desc;
int pending;
int dmacount;
int desccount;
struct ioatdma_device *device;
struct dma_chan common;
......@@ -134,12 +125,13 @@ struct ioat_desc_sw {
struct ioatdma_device *ioat_dma_probe(struct pci_dev *pdev,
void __iomem *iobase);
void ioat_dma_remove(struct ioatdma_device *device);
struct dca_provider *ioat_dca_init(struct pci_dev *pdev,
void __iomem *iobase);
struct dca_provider *ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase);
struct dca_provider *ioat2_dca_init(struct pci_dev *pdev, void __iomem *iobase);
#else
#define ioat_dma_probe(pdev, iobase) NULL
#define ioat_dma_remove(device) do { } while (0)
#define ioat_dca_init(pdev, iobase) NULL
#define ioat2_dca_init(pdev, iobase) NULL
#endif
#endif /* IOATDMA_H */
/*
* Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved.
* Copyright(c) 2004 - 2007 Intel Corporation. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
......@@ -22,12 +22,19 @@
#define _IOAT_HW_H_
/* PCI Configuration Space Values */
#define IOAT_PCI_VID 0x8086
#define IOAT_PCI_DID 0x1A38
#define IOAT_PCI_RID 0x00
#define IOAT_PCI_SVID 0x8086
#define IOAT_PCI_SID 0x8086
#define IOAT_VER_1_2 0x12 /* Version 1.2 */
#define IOAT_PCI_VID 0x8086
/* CB device ID's */
#define IOAT_PCI_DID_5000 0x1A38
#define IOAT_PCI_DID_CNB 0x360B
#define IOAT_PCI_DID_SCNB 0x65FF
#define IOAT_PCI_DID_SNB 0x402F
#define IOAT_PCI_RID 0x00
#define IOAT_PCI_SVID 0x8086
#define IOAT_PCI_SID 0x8086
#define IOAT_VER_1_2 0x12 /* Version 1.2 */
#define IOAT_VER_2_0 0x20 /* Version 2.0 */
struct ioat_dma_descriptor {
uint32_t size;
......@@ -47,6 +54,16 @@ struct ioat_dma_descriptor {
#define IOAT_DMA_DESCRIPTOR_CTL_CP_STS 0x00000008
#define IOAT_DMA_DESCRIPTOR_CTL_FRAME 0x00000010
#define IOAT_DMA_DESCRIPTOR_NUL 0x00000020
#define IOAT_DMA_DESCRIPTOR_OPCODE 0xFF000000
#define IOAT_DMA_DESCRIPTOR_CTL_SP_BRK 0x00000040
#define IOAT_DMA_DESCRIPTOR_CTL_DP_BRK 0x00000080
#define IOAT_DMA_DESCRIPTOR_CTL_BNDL 0x00000100
#define IOAT_DMA_DESCRIPTOR_CTL_DCA 0x00000200
#define IOAT_DMA_DESCRIPTOR_CTL_BUFHINT 0x00000400
#define IOAT_DMA_DESCRIPTOR_CTL_OPCODE_CONTEXT 0xFF000000
#define IOAT_DMA_DESCRIPTOR_CTL_OPCODE_DMA 0x00000000
#define IOAT_DMA_DESCRIPTOR_CTL_CONTEXT_DCA 0x00000001
#define IOAT_DMA_DESCRIPTOR_CTL_OPCODE_MASK 0xFF000000
#endif
......@@ -42,26 +42,25 @@
#define IOAT_INTRCTRL_MASTER_INT_EN 0x01 /* Master Interrupt Enable */
#define IOAT_INTRCTRL_INT_STATUS 0x02 /* ATTNSTATUS -or- Channel Int */
#define IOAT_INTRCTRL_INT 0x04 /* INT_STATUS -and- MASTER_INT_EN */
#define IOAT_INTRCTRL_MSIX_VECTOR_CONTROL 0x08 /* Enable all MSI-X vectors */
#define IOAT_INTRCTRL_MSIX_VECTOR_CONTROL 0x08 /* Enable all MSI-X vectors */
#define IOAT_ATTNSTATUS_OFFSET 0x04 /* Each bit is a channel */
#define IOAT_VER_OFFSET 0x08 /* 8-bit */
#define IOAT_VER_MAJOR_MASK 0xF0
#define IOAT_VER_MINOR_MASK 0x0F
#define GET_IOAT_VER_MAJOR(x) ((x) & IOAT_VER_MAJOR_MASK)
#define GET_IOAT_VER_MAJOR(x) (((x) & IOAT_VER_MAJOR_MASK) >> 4)
#define GET_IOAT_VER_MINOR(x) ((x) & IOAT_VER_MINOR_MASK)
#define IOAT_PERPORTOFFSET_OFFSET 0x0A /* 16-bit */
#define IOAT_INTRDELAY_OFFSET 0x0C /* 16-bit */
#define IOAT_INTRDELAY_INT_DELAY_MASK 0x3FFF /* Interrupt Delay Time */
#define IOAT_INTRDELAY_COALESE_SUPPORT 0x8000 /* Interrupt Coalesing Supported */
#define IOAT_INTRDELAY_COALESE_SUPPORT 0x8000 /* Interrupt Coalescing Supported */
#define IOAT_DEVICE_STATUS_OFFSET 0x0E /* 16-bit */
#define IOAT_DEVICE_STATUS_DEGRADED_MODE 0x0001
#define IOAT_CHANNEL_MMIO_SIZE 0x80 /* Each Channel MMIO space is this size */
/* DMA Channel Registers */
......@@ -74,25 +73,101 @@
#define IOAT_CHANCTRL_ERR_COMPLETION_EN 0x0004
#define IOAT_CHANCTRL_INT_DISABLE 0x0001
#define IOAT_DMA_COMP_OFFSET 0x02 /* 16-bit DMA channel compatability */
#define IOAT_DMA_COMP_V1 0x0001 /* Compatability with DMA version 1 */
#define IOAT_CHANSTS_OFFSET 0x04 /* 64-bit Channel Status Register */
#define IOAT_CHANSTS_OFFSET_LOW 0x04
#define IOAT_CHANSTS_OFFSET_HIGH 0x08
#define IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR 0xFFFFFFFFFFFFFFC0UL
#define IOAT_DMA_COMP_OFFSET 0x02 /* 16-bit DMA channel compatibility */
#define IOAT_DMA_COMP_V1 0x0001 /* Compatibility with DMA version 1 */
#define IOAT_DMA_COMP_V2 0x0002 /* Compatibility with DMA version 2 */
#define IOAT1_CHANSTS_OFFSET 0x04 /* 64-bit Channel Status Register */
#define IOAT2_CHANSTS_OFFSET 0x08 /* 64-bit Channel Status Register */
#define IOAT_CHANSTS_OFFSET(ver) ((ver) < IOAT_VER_2_0 \
? IOAT1_CHANSTS_OFFSET : IOAT2_CHANSTS_OFFSET)
#define IOAT1_CHANSTS_OFFSET_LOW 0x04
#define IOAT2_CHANSTS_OFFSET_LOW 0x08
#define IOAT_CHANSTS_OFFSET_LOW(ver) ((ver) < IOAT_VER_2_0 \
? IOAT1_CHANSTS_OFFSET_LOW : IOAT2_CHANSTS_OFFSET_LOW)
#define IOAT1_CHANSTS_OFFSET_HIGH 0x08
#define IOAT2_CHANSTS_OFFSET_HIGH 0x0C
#define IOAT_CHANSTS_OFFSET_HIGH(ver) ((ver) < IOAT_VER_2_0 \
? IOAT1_CHANSTS_OFFSET_HIGH : IOAT2_CHANSTS_OFFSET_HIGH)
#define IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR ~0x3F
#define IOAT_CHANSTS_SOFT_ERR 0x0000000000000010
#define IOAT_CHANSTS_UNAFFILIATED_ERR 0x0000000000000008
#define IOAT_CHANSTS_DMA_TRANSFER_STATUS 0x0000000000000007
#define IOAT_CHANSTS_DMA_TRANSFER_STATUS_ACTIVE 0x0
#define IOAT_CHANSTS_DMA_TRANSFER_STATUS_DONE 0x1
#define IOAT_CHANSTS_DMA_TRANSFER_STATUS_SUSPENDED 0x2
#define IOAT_CHANSTS_DMA_TRANSFER_STATUS_HALTED 0x3
#define IOAT_CHAINADDR_OFFSET 0x0C /* 64-bit Descriptor Chain Address Register */
#define IOAT_CHAINADDR_OFFSET_LOW 0x0C
#define IOAT_CHAINADDR_OFFSET_HIGH 0x10
#define IOAT_CHANCMD_OFFSET 0x14 /* 8-bit DMA Channel Command Register */
#define IOAT_CHAN_DMACOUNT_OFFSET 0x06 /* 16-bit DMA Count register */
#define IOAT_DCACTRL_OFFSET 0x30 /* 32 bit Direct Cache Access Control Register */
#define IOAT_DCACTRL_CMPL_WRITE_ENABLE 0x10000
#define IOAT_DCACTRL_TARGET_CPU_MASK 0xFFFF /* APIC ID */
/* CB DCA Memory Space Registers */
#define IOAT_DCAOFFSET_OFFSET 0x14
/* CB_BAR + IOAT_DCAOFFSET value */
#define IOAT_DCA_VER_OFFSET 0x00
#define IOAT_DCA_VER_MAJOR_MASK 0xF0
#define IOAT_DCA_VER_MINOR_MASK 0x0F
#define IOAT_DCA_COMP_OFFSET 0x02
#define IOAT_DCA_COMP_V1 0x1
#define IOAT_FSB_CAPABILITY_OFFSET 0x04
#define IOAT_FSB_CAPABILITY_PREFETCH 0x1
#define IOAT_PCI_CAPABILITY_OFFSET 0x06
#define IOAT_PCI_CAPABILITY_MEMWR 0x1
#define IOAT_FSB_CAP_ENABLE_OFFSET 0x08
#define IOAT_FSB_CAP_ENABLE_PREFETCH 0x1
#define IOAT_PCI_CAP_ENABLE_OFFSET 0x0A
#define IOAT_PCI_CAP_ENABLE_MEMWR 0x1
#define IOAT_APICID_TAG_MAP_OFFSET 0x0C
#define IOAT_APICID_TAG_MAP_TAG0 0x0000000F
#define IOAT_APICID_TAG_MAP_TAG0_SHIFT 0
#define IOAT_APICID_TAG_MAP_TAG1 0x000000F0
#define IOAT_APICID_TAG_MAP_TAG1_SHIFT 4
#define IOAT_APICID_TAG_MAP_TAG2 0x00000F00
#define IOAT_APICID_TAG_MAP_TAG2_SHIFT 8
#define IOAT_APICID_TAG_MAP_TAG3 0x0000F000
#define IOAT_APICID_TAG_MAP_TAG3_SHIFT 12
#define IOAT_APICID_TAG_MAP_TAG4 0x000F0000
#define IOAT_APICID_TAG_MAP_TAG4_SHIFT 16
#define IOAT_APICID_TAG_CB2_VALID 0x8080808080
#define IOAT_DCA_GREQID_OFFSET 0x10
#define IOAT_DCA_GREQID_SIZE 0x04
#define IOAT_DCA_GREQID_MASK 0xFFFF
#define IOAT_DCA_GREQID_IGNOREFUN 0x10000000
#define IOAT_DCA_GREQID_VALID 0x20000000
#define IOAT_DCA_GREQID_LASTID 0x80000000
#define IOAT1_CHAINADDR_OFFSET 0x0C /* 64-bit Descriptor Chain Address Register */
#define IOAT2_CHAINADDR_OFFSET 0x10 /* 64-bit Descriptor Chain Address Register */
#define IOAT_CHAINADDR_OFFSET(ver) ((ver) < IOAT_VER_2_0 \
? IOAT1_CHAINADDR_OFFSET : IOAT2_CHAINADDR_OFFSET)
#define IOAT1_CHAINADDR_OFFSET_LOW 0x0C
#define IOAT2_CHAINADDR_OFFSET_LOW 0x10
#define IOAT_CHAINADDR_OFFSET_LOW(ver) ((ver) < IOAT_VER_2_0 \
? IOAT1_CHAINADDR_OFFSET_LOW : IOAT2_CHAINADDR_OFFSET_LOW)
#define IOAT1_CHAINADDR_OFFSET_HIGH 0x10
#define IOAT2_CHAINADDR_OFFSET_HIGH 0x14
#define IOAT_CHAINADDR_OFFSET_HIGH(ver) ((ver) < IOAT_VER_2_0 \
? IOAT1_CHAINADDR_OFFSET_HIGH : IOAT2_CHAINADDR_OFFSET_HIGH)
#define IOAT1_CHANCMD_OFFSET 0x14 /* 8-bit DMA Channel Command Register */
#define IOAT2_CHANCMD_OFFSET 0x04 /* 8-bit DMA Channel Command Register */
#define IOAT_CHANCMD_OFFSET(ver) ((ver) < IOAT_VER_2_0 \
? IOAT1_CHANCMD_OFFSET : IOAT2_CHANCMD_OFFSET)
#define IOAT_CHANCMD_RESET 0x20
#define IOAT_CHANCMD_RESUME 0x10
#define IOAT_CHANCMD_ABORT 0x08
......@@ -124,6 +199,7 @@
#define IOAT_CHANERR_COMPLETION_ADDR_ERR 0x1000
#define IOAT_CHANERR_INT_CONFIGURATION_ERR 0x2000
#define IOAT_CHANERR_SOFT_ERR 0x4000
#define IOAT_CHANERR_UNAFFILIATED_ERR 0x8000
#define IOAT_CHANERR_MASK_OFFSET 0x2C /* 32-bit Channel Error Register */
......
......@@ -2332,6 +2332,7 @@
#define PCI_DEVICE_ID_INTEL_MCH_PC1 0x359a
#define PCI_DEVICE_ID_INTEL_E7525_MCH 0x359e
#define PCI_DEVICE_ID_INTEL_IOAT_CNB 0x360b
#define PCI_DEVICE_ID_INTEL_IOAT_SNB 0x402f
#define PCI_DEVICE_ID_INTEL_IOAT_SCNB 0x65ff
#define PCI_DEVICE_ID_INTEL_TOLAPAI_0 0x5031
#define PCI_DEVICE_ID_INTEL_TOLAPAI_1 0x5032
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment