pci-calgary_64.c 40.4 KB
Newer Older
1 2 3
/*
 * Derived from arch/powerpc/kernel/iommu.c
 *
4
 * Copyright IBM Corporation, 2006-2007
5
 * Copyright (C) 2006  Jon Mason <jdmason@kudzu.us>
6
 *
7
 * Author: Jon Mason <jdmason@kudzu.us>
8 9
 * Author: Muli Ben-Yehuda <muli@il.ibm.com>

10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
 */

#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/types.h>
#include <linux/slab.h>
#include <linux/mm.h>
#include <linux/spinlock.h>
#include <linux/string.h>
32
#include <linux/crash_dump.h>
33
#include <linux/dma-mapping.h>
34
#include <linux/bitmap.h>
35 36 37
#include <linux/pci_ids.h>
#include <linux/pci.h>
#include <linux/delay.h>
38
#include <linux/scatterlist.h>
39
#include <linux/iommu-helper.h>
40

41
#include <asm/iommu.h>
42 43 44 45 46
#include <asm/calgary.h>
#include <asm/tce.h>
#include <asm/pci-direct.h>
#include <asm/system.h>
#include <asm/dma.h>
47
#include <asm/rio.h>
48
#include <asm/bios_ebda.h>
49
#include <asm/x86_init.h>
50
#include <asm/iommu_table.h>
51

52 53 54 55 56 57
#ifdef CONFIG_CALGARY_IOMMU_ENABLED_BY_DEFAULT
int use_calgary __read_mostly = 1;
#else
int use_calgary __read_mostly = 0;
#endif /* CONFIG_CALGARY_DEFAULT_ENABLED */

58
#define PCI_DEVICE_ID_IBM_CALGARY 0x02a1
59
#define PCI_DEVICE_ID_IBM_CALIOC2 0x0308
60 61

/* register offsets inside the host bridge space */
62 63
#define CALGARY_CONFIG_REG	0x0108
#define PHB_CSR_OFFSET		0x0110 /* Channel Status */
64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82
#define PHB_PLSSR_OFFSET	0x0120
#define PHB_CONFIG_RW_OFFSET	0x0160
#define PHB_IOBASE_BAR_LOW	0x0170
#define PHB_IOBASE_BAR_HIGH	0x0180
#define PHB_MEM_1_LOW		0x0190
#define PHB_MEM_1_HIGH		0x01A0
#define PHB_IO_ADDR_SIZE	0x01B0
#define PHB_MEM_1_SIZE		0x01C0
#define PHB_MEM_ST_OFFSET	0x01D0
#define PHB_AER_OFFSET		0x0200
#define PHB_CONFIG_0_HIGH	0x0220
#define PHB_CONFIG_0_LOW	0x0230
#define PHB_CONFIG_0_END	0x0240
#define PHB_MEM_2_LOW		0x02B0
#define PHB_MEM_2_HIGH		0x02C0
#define PHB_MEM_2_SIZE_HIGH	0x02D0
#define PHB_MEM_2_SIZE_LOW	0x02E0
#define PHB_DOSHOLE_OFFSET	0x08E0

83
/* CalIOC2 specific */
84 85 86
#define PHB_SAVIOR_L2		0x0DB0
#define PHB_PAGE_MIG_CTRL	0x0DA8
#define PHB_PAGE_MIG_DEBUG	0x0DA0
87
#define PHB_ROOT_COMPLEX_STATUS 0x0CB0
88

89 90 91 92 93 94 95 96 97 98 99
/* PHB_CONFIG_RW */
#define PHB_TCE_ENABLE		0x20000000
#define PHB_SLOT_DISABLE	0x1C000000
#define PHB_DAC_DISABLE		0x01000000
#define PHB_MEM2_ENABLE		0x00400000
#define PHB_MCSR_ENABLE		0x00100000
/* TAR (Table Address Register) */
#define TAR_SW_BITS		0x0000ffffffff800fUL
#define TAR_VALID		0x0000000000000008UL
/* CSR (Channel/DMA Status Register) */
#define CSR_AGENT_MASK		0xffe0ffff
100
/* CCR (Calgary Configuration Register) */
101
#define CCR_2SEC_TIMEOUT	0x000000000000000EUL
102
/* PMCR/PMDR (Page Migration Control/Debug Registers */
103 104 105
#define PMR_SOFTSTOP		0x80000000
#define PMR_SOFTSTOPFAULT	0x40000000
#define PMR_HARDSTOP		0x20000000
106

107 108 109 110 111 112 113
/*
 * The maximum PHB bus number.
 * x3950M2 (rare): 8 chassis, 48 PHBs per chassis = 384
 * x3950M2: 4 chassis, 48 PHBs per chassis        = 192
 * x3950 (PCIE): 8 chassis, 32 PHBs per chassis   = 256
 * x3950 (PCIX): 8 chassis, 16 PHBs per chassis   = 128
 */
114
#define MAX_PHB_BUS_NUM		256
115 116

#define PHBS_PER_CALGARY	  4
117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139

/* register offsets in Calgary's internal register space */
static const unsigned long tar_offsets[] = {
	0x0580 /* TAR0 */,
	0x0588 /* TAR1 */,
	0x0590 /* TAR2 */,
	0x0598 /* TAR3 */
};

static const unsigned long split_queue_offsets[] = {
	0x4870 /* SPLIT QUEUE 0 */,
	0x5870 /* SPLIT QUEUE 1 */,
	0x6870 /* SPLIT QUEUE 2 */,
	0x7870 /* SPLIT QUEUE 3 */
};

static const unsigned long phb_offsets[] = {
	0x8000 /* PHB0 */,
	0x9000 /* PHB1 */,
	0xA000 /* PHB2 */,
	0xB000 /* PHB3 */
};

140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155
/* PHB debug registers */

static const unsigned long phb_debug_offsets[] = {
	0x4000	/* PHB 0 DEBUG */,
	0x5000	/* PHB 1 DEBUG */,
	0x6000	/* PHB 2 DEBUG */,
	0x7000	/* PHB 3 DEBUG */
};

/*
 * STUFF register for each debug PHB,
 * byte 1 = start bus number, byte 2 = end bus number
 */

#define PHB_DEBUG_STUFF_OFFSET	0x0020

156 157
#define EMERGENCY_PAGES 32 /* = 128KB */

158 159 160 161
unsigned int specified_table_size = TCE_TABLE_SIZE_UNSPECIFIED;
static int translate_empty_slots __read_mostly = 0;
static int calgary_detected __read_mostly = 0;

162 163
static struct rio_table_hdr	*rio_table_hdr __initdata;
static struct scal_detail	*scal_devs[MAX_NUMNODES] __initdata;
164
static struct rio_detail	*rio_devs[MAX_NUMNODES * 4] __initdata;
165

166 167
struct calgary_bus_info {
	void *tce_space;
168
	unsigned char translation_disabled;
169
	signed char phbid;
170
	void __iomem *bbar;
171 172
};

173 174
static void calgary_handle_quirks(struct iommu_table *tbl, struct pci_dev *dev);
static void calgary_tce_cache_blast(struct iommu_table *tbl);
175
static void calgary_dump_error_regs(struct iommu_table *tbl);
176
static void calioc2_handle_quirks(struct iommu_table *tbl, struct pci_dev *dev);
177
static void calioc2_tce_cache_blast(struct iommu_table *tbl);
178
static void calioc2_dump_error_regs(struct iommu_table *tbl);
179 180
static void calgary_init_bitmap_from_tce_table(struct iommu_table *tbl);
static void get_tce_space_from_tar(void);
181 182 183

static struct cal_chipset_ops calgary_chip_ops = {
	.handle_quirks = calgary_handle_quirks,
184 185
	.tce_cache_blast = calgary_tce_cache_blast,
	.dump_error_regs = calgary_dump_error_regs
186
};
187

188 189
static struct cal_chipset_ops calioc2_chip_ops = {
	.handle_quirks = calioc2_handle_quirks,
190 191
	.tce_cache_blast = calioc2_tce_cache_blast,
	.dump_error_regs = calioc2_dump_error_regs
192 193
};

194
static struct calgary_bus_info bus_info[MAX_PHB_BUS_NUM] = { { NULL, 0, 0 }, };
195

196 197 198 199 200 201
static inline int translation_enabled(struct iommu_table *tbl)
{
	/* only PHBs with translation enabled have an IOMMU table */
	return (tbl != NULL);
}

202
static void iommu_range_reserve(struct iommu_table *tbl,
203
	unsigned long start_addr, unsigned int npages)
204 205 206
{
	unsigned long index;
	unsigned long end;
207
	unsigned long flags;
208 209 210 211 212 213 214 215 216 217 218

	index = start_addr >> PAGE_SHIFT;

	/* bail out if we're asked to reserve a region we don't cover */
	if (index >= tbl->it_size)
		return;

	end = index + npages;
	if (end > tbl->it_size) /* don't go off the table */
		end = tbl->it_size;

219 220
	spin_lock_irqsave(&tbl->it_lock, flags);

221
	bitmap_set(tbl->it_map, index, npages);
222 223

	spin_unlock_irqrestore(&tbl->it_lock, flags);
224 225
}

226 227 228
static unsigned long iommu_range_alloc(struct device *dev,
				       struct iommu_table *tbl,
				       unsigned int npages)
229
{
230
	unsigned long flags;
231
	unsigned long offset;
232 233 234 235
	unsigned long boundary_size;

	boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1,
			      PAGE_SIZE) >> PAGE_SHIFT;
236 237 238

	BUG_ON(npages == 0);

239 240
	spin_lock_irqsave(&tbl->it_lock, flags);

241 242
	offset = iommu_area_alloc(tbl->it_map, tbl->it_size, tbl->it_hint,
				  npages, 0, boundary_size, 0);
243
	if (offset == ~0UL) {
244
		tbl->chip_ops->tce_cache_blast(tbl);
245 246 247

		offset = iommu_area_alloc(tbl->it_map, tbl->it_size, 0,
					  npages, 0, boundary_size, 0);
248 249
		if (offset == ~0UL) {
			printk(KERN_WARNING "Calgary: IOMMU full.\n");
250
			spin_unlock_irqrestore(&tbl->it_lock, flags);
251 252 253
			if (panic_on_overflow)
				panic("Calgary: fix the allocator.\n");
			else
254
				return DMA_ERROR_CODE;
255 256 257 258 259 260
		}
	}

	tbl->it_hint = offset + npages;
	BUG_ON(tbl->it_hint > tbl->it_size);

261 262
	spin_unlock_irqrestore(&tbl->it_lock, flags);

263 264 265
	return offset;
}

266 267
static dma_addr_t iommu_alloc(struct device *dev, struct iommu_table *tbl,
			      void *vaddr, unsigned int npages, int direction)
268
{
269
	unsigned long entry;
270
	dma_addr_t ret;
271

272
	entry = iommu_range_alloc(dev, tbl, npages);
273

274 275 276 277 278
	if (unlikely(entry == DMA_ERROR_CODE)) {
		printk(KERN_WARNING "Calgary: failed to allocate %u pages in "
		       "iommu %p\n", npages, tbl);
		return DMA_ERROR_CODE;
	}
279 280 281 282 283 284 285 286 287 288

	/* set the return dma address */
	ret = (entry << PAGE_SHIFT) | ((unsigned long)vaddr & ~PAGE_MASK);

	/* put the TCEs in the HW table */
	tce_build(tbl, entry, npages, (unsigned long)vaddr & PAGE_MASK,
		  direction);
	return ret;
}

289
static void iommu_free(struct iommu_table *tbl, dma_addr_t dma_addr,
290 291 292
	unsigned int npages)
{
	unsigned long entry;
293
	unsigned long badend;
294
	unsigned long flags;
295 296

	/* were we called with bad_dma_address? */
297 298
	badend = DMA_ERROR_CODE + (EMERGENCY_PAGES * PAGE_SIZE);
	if (unlikely((dma_addr >= DMA_ERROR_CODE) && (dma_addr < badend))) {
299
		WARN(1, KERN_ERR "Calgary: driver tried unmapping bad DMA "
300 301 302
		       "address 0x%Lx\n", dma_addr);
		return;
	}
303 304 305 306 307 308 309

	entry = dma_addr >> PAGE_SHIFT;

	BUG_ON(entry + npages > tbl->it_size);

	tce_free(tbl, entry, npages);

310 311
	spin_lock_irqsave(&tbl->it_lock, flags);

312
	bitmap_clear(tbl->it_map, entry, npages);
313 314

	spin_unlock_irqrestore(&tbl->it_lock, flags);
315 316
}

317 318
static inline struct iommu_table *find_iommu_table(struct device *dev)
{
319 320
	struct pci_dev *pdev;
	struct pci_bus *pbus;
321 322
	struct iommu_table *tbl;

323 324
	pdev = to_pci_dev(dev);

325
	/* search up the device tree for an iommu */
326
	pbus = pdev->bus;
327 328 329 330 331
	do {
		tbl = pci_iommu(pbus);
		if (tbl && tbl->it_busno == pbus->number)
			break;
		tbl = NULL;
332
		pbus = pbus->parent;
333
	} while (pbus);
334

335
	BUG_ON(tbl && (tbl->it_busno != pbus->number));
336 337 338 339

	return tbl;
}

340 341 342
static void calgary_unmap_sg(struct device *dev, struct scatterlist *sglist,
			     int nelems,enum dma_data_direction dir,
			     struct dma_attrs *attrs)
343
{
344
	struct iommu_table *tbl = find_iommu_table(dev);
345 346
	struct scatterlist *s;
	int i;
347

348
	if (!translation_enabled(tbl))
349 350
		return;

351
	for_each_sg(sglist, s, nelems, i) {
352
		unsigned int npages;
353 354
		dma_addr_t dma = s->dma_address;
		unsigned int dmalen = s->dma_length;
355 356 357 358

		if (dmalen == 0)
			break;

359
		npages = iommu_num_pages(dma, dmalen, PAGE_SIZE);
360
		iommu_free(tbl, dma, npages);
361 362 363
	}
}

364
static int calgary_map_sg(struct device *dev, struct scatterlist *sg,
365 366
			  int nelems, enum dma_data_direction dir,
			  struct dma_attrs *attrs)
367
{
368
	struct iommu_table *tbl = find_iommu_table(dev);
369
	struct scatterlist *s;
370 371 372 373 374
	unsigned long vaddr;
	unsigned int npages;
	unsigned long entry;
	int i;

375
	for_each_sg(sg, s, nelems, i) {
Jens Axboe's avatar
Jens Axboe committed
376
		BUG_ON(!sg_page(s));
377

Jens Axboe's avatar
Jens Axboe committed
378
		vaddr = (unsigned long) sg_virt(s);
379
		npages = iommu_num_pages(vaddr, s->length, PAGE_SIZE);
380

381
		entry = iommu_range_alloc(dev, tbl, npages);
382
		if (entry == DMA_ERROR_CODE) {
383 384 385 386 387 388 389 390
			/* makes sure unmap knows to stop */
			s->dma_length = 0;
			goto error;
		}

		s->dma_address = (entry << PAGE_SHIFT) | s->offset;

		/* insert into HW table */
391
		tce_build(tbl, entry, npages, vaddr & PAGE_MASK, dir);
392 393 394 395 396 397

		s->dma_length = s->length;
	}

	return nelems;
error:
398
	calgary_unmap_sg(dev, sg, nelems, dir, NULL);
399
	for_each_sg(sg, s, nelems, i) {
400
		sg->dma_address = DMA_ERROR_CODE;
401
		sg->dma_length = 0;
402 403 404 405
	}
	return 0;
}

406 407 408 409
static dma_addr_t calgary_map_page(struct device *dev, struct page *page,
				   unsigned long offset, size_t size,
				   enum dma_data_direction dir,
				   struct dma_attrs *attrs)
410
{
411
	void *vaddr = page_address(page) + offset;
412 413
	unsigned long uaddr;
	unsigned int npages;
414
	struct iommu_table *tbl = find_iommu_table(dev);
415 416

	uaddr = (unsigned long)vaddr;
417
	npages = iommu_num_pages(uaddr, size, PAGE_SIZE);
418

419
	return iommu_alloc(dev, tbl, vaddr, npages, dir);
420 421
}

422 423 424
static void calgary_unmap_page(struct device *dev, dma_addr_t dma_addr,
			       size_t size, enum dma_data_direction dir,
			       struct dma_attrs *attrs)
425
{
426
	struct iommu_table *tbl = find_iommu_table(dev);
427 428
	unsigned int npages;

429 430 431 432
	npages = iommu_num_pages(dma_addr, size, PAGE_SIZE);
	iommu_free(tbl, dma_addr, npages);
}

433
static void* calgary_alloc_coherent(struct device *dev, size_t size,
434
	dma_addr_t *dma_handle, gfp_t flag, struct dma_attrs *attrs)
435 436 437 438
{
	void *ret = NULL;
	dma_addr_t mapping;
	unsigned int npages, order;
439
	struct iommu_table *tbl = find_iommu_table(dev);
440 441 442 443 444

	size = PAGE_ALIGN(size); /* size rounded up to full pages */
	npages = size >> PAGE_SHIFT;
	order = get_order(size);

445 446
	flag &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32);

447 448 449 450 451 452
	/* alloc enough pages (and possibly more) */
	ret = (void *)__get_free_pages(flag, order);
	if (!ret)
		goto error;
	memset(ret, 0, size);

453 454
	/* set up tces to cover the allocated range */
	mapping = iommu_alloc(dev, tbl, ret, npages, DMA_BIDIRECTIONAL);
455
	if (mapping == DMA_ERROR_CODE)
456 457
		goto free;
	*dma_handle = mapping;
458 459 460 461 462 463 464 465
	return ret;
free:
	free_pages((unsigned long)ret, get_order(size));
	ret = NULL;
error:
	return ret;
}

466
static void calgary_free_coherent(struct device *dev, size_t size,
467 468
				  void *vaddr, dma_addr_t dma_handle,
				  struct dma_attrs *attrs)
469 470 471 472 473 474 475 476 477 478 479
{
	unsigned int npages;
	struct iommu_table *tbl = find_iommu_table(dev);

	size = PAGE_ALIGN(size);
	npages = size >> PAGE_SHIFT;

	iommu_free(tbl, dma_handle, npages);
	free_pages((unsigned long)vaddr, get_order(size));
}

480
static struct dma_map_ops calgary_dma_ops = {
481 482
	.alloc = calgary_alloc_coherent,
	.free = calgary_free_coherent,
483 484
	.map_sg = calgary_map_sg,
	.unmap_sg = calgary_unmap_sg,
485 486
	.map_page = calgary_map_page,
	.unmap_page = calgary_unmap_page,
487 488
};

489 490 491 492 493
static inline void __iomem * busno_to_bbar(unsigned char num)
{
	return bus_info[num].bbar;
}

494 495
static inline int busno_to_phbid(unsigned char num)
{
496
	return bus_info[num].phbid;
497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525
}

static inline unsigned long split_queue_offset(unsigned char num)
{
	size_t idx = busno_to_phbid(num);

	return split_queue_offsets[idx];
}

static inline unsigned long tar_offset(unsigned char num)
{
	size_t idx = busno_to_phbid(num);

	return tar_offsets[idx];
}

static inline unsigned long phb_offset(unsigned char num)
{
	size_t idx = busno_to_phbid(num);

	return phb_offsets[idx];
}

static inline void __iomem* calgary_reg(void __iomem *bar, unsigned long offset)
{
	unsigned long target = ((unsigned long)bar) | offset;
	return (void __iomem*)target;
}

526 527 528 529 530 531 532 533 534 535 536 537 538 539 540
static inline int is_calioc2(unsigned short device)
{
	return (device == PCI_DEVICE_ID_IBM_CALIOC2);
}

static inline int is_calgary(unsigned short device)
{
	return (device == PCI_DEVICE_ID_IBM_CALGARY);
}

static inline int is_cal_pci_dev(unsigned short device)
{
	return (is_calgary(device) || is_calioc2(device));
}

541
static void calgary_tce_cache_blast(struct iommu_table *tbl)
542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577
{
	u64 val;
	u32 aer;
	int i = 0;
	void __iomem *bbar = tbl->bbar;
	void __iomem *target;

	/* disable arbitration on the bus */
	target = calgary_reg(bbar, phb_offset(tbl->it_busno) | PHB_AER_OFFSET);
	aer = readl(target);
	writel(0, target);

	/* read plssr to ensure it got there */
	target = calgary_reg(bbar, phb_offset(tbl->it_busno) | PHB_PLSSR_OFFSET);
	val = readl(target);

	/* poll split queues until all DMA activity is done */
	target = calgary_reg(bbar, split_queue_offset(tbl->it_busno));
	do {
		val = readq(target);
		i++;
	} while ((val & 0xff) != 0xff && i < 100);
	if (i == 100)
		printk(KERN_WARNING "Calgary: PCI bus not quiesced, "
		       "continuing anyway\n");

	/* invalidate TCE cache */
	target = calgary_reg(bbar, tar_offset(tbl->it_busno));
	writeq(tbl->tar_val, target);

	/* enable arbitration */
	target = calgary_reg(bbar, phb_offset(tbl->it_busno) | PHB_AER_OFFSET);
	writel(aer, target);
	(void)readl(target); /* flush */
}

578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656
static void calioc2_tce_cache_blast(struct iommu_table *tbl)
{
	void __iomem *bbar = tbl->bbar;
	void __iomem *target;
	u64 val64;
	u32 val;
	int i = 0;
	int count = 1;
	unsigned char bus = tbl->it_busno;

begin:
	printk(KERN_DEBUG "Calgary: CalIOC2 bus 0x%x entering tce cache blast "
	       "sequence - count %d\n", bus, count);

	/* 1. using the Page Migration Control reg set SoftStop */
	target = calgary_reg(bbar, phb_offset(bus) | PHB_PAGE_MIG_CTRL);
	val = be32_to_cpu(readl(target));
	printk(KERN_DEBUG "1a. read 0x%x [LE] from %p\n", val, target);
	val |= PMR_SOFTSTOP;
	printk(KERN_DEBUG "1b. writing 0x%x [LE] to %p\n", val, target);
	writel(cpu_to_be32(val), target);

	/* 2. poll split queues until all DMA activity is done */
	printk(KERN_DEBUG "2a. starting to poll split queues\n");
	target = calgary_reg(bbar, split_queue_offset(bus));
	do {
		val64 = readq(target);
		i++;
	} while ((val64 & 0xff) != 0xff && i < 100);
	if (i == 100)
		printk(KERN_WARNING "CalIOC2: PCI bus not quiesced, "
		       "continuing anyway\n");

	/* 3. poll Page Migration DEBUG for SoftStopFault */
	target = calgary_reg(bbar, phb_offset(bus) | PHB_PAGE_MIG_DEBUG);
	val = be32_to_cpu(readl(target));
	printk(KERN_DEBUG "3. read 0x%x [LE] from %p\n", val, target);

	/* 4. if SoftStopFault - goto (1) */
	if (val & PMR_SOFTSTOPFAULT) {
		if (++count < 100)
			goto begin;
		else {
			printk(KERN_WARNING "CalIOC2: too many SoftStopFaults, "
			       "aborting TCE cache flush sequence!\n");
			return; /* pray for the best */
		}
	}

	/* 5. Slam into HardStop by reading PHB_PAGE_MIG_CTRL */
	target = calgary_reg(bbar, phb_offset(bus) | PHB_PAGE_MIG_CTRL);
	printk(KERN_DEBUG "5a. slamming into HardStop by reading %p\n", target);
	val = be32_to_cpu(readl(target));
	printk(KERN_DEBUG "5b. read 0x%x [LE] from %p\n", val, target);
	target = calgary_reg(bbar, phb_offset(bus) | PHB_PAGE_MIG_DEBUG);
	val = be32_to_cpu(readl(target));
	printk(KERN_DEBUG "5c. read 0x%x [LE] from %p (debug)\n", val, target);

	/* 6. invalidate TCE cache */
	printk(KERN_DEBUG "6. invalidating TCE cache\n");
	target = calgary_reg(bbar, tar_offset(bus));
	writeq(tbl->tar_val, target);

	/* 7. Re-read PMCR */
	printk(KERN_DEBUG "7a. Re-reading PMCR\n");
	target = calgary_reg(bbar, phb_offset(bus) | PHB_PAGE_MIG_CTRL);
	val = be32_to_cpu(readl(target));
	printk(KERN_DEBUG "7b. read 0x%x [LE] from %p\n", val, target);

	/* 8. Remove HardStop */
	printk(KERN_DEBUG "8a. removing HardStop from PMCR\n");
	target = calgary_reg(bbar, phb_offset(bus) | PHB_PAGE_MIG_CTRL);
	val = 0;
	printk(KERN_DEBUG "8b. writing 0x%x [LE] to %p\n", val, target);
	writel(cpu_to_be32(val), target);
	val = be32_to_cpu(readl(target));
	printk(KERN_DEBUG "8c. read 0x%x [LE] from %p\n", val, target);
}

657 658 659 660 661 662 663 664 665
static void __init calgary_reserve_mem_region(struct pci_dev *dev, u64 start,
	u64 limit)
{
	unsigned int numpages;

	limit = limit | 0xfffff;
	limit++;

	numpages = ((limit - start) >> PAGE_SHIFT);
666
	iommu_range_reserve(pci_iommu(dev->bus), start, numpages);
667 668 669 670 671 672 673
}

static void __init calgary_reserve_peripheral_mem_1(struct pci_dev *dev)
{
	void __iomem *target;
	u64 low, high, sizelow;
	u64 start, limit;
674
	struct iommu_table *tbl = pci_iommu(dev->bus);
675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697
	unsigned char busnum = dev->bus->number;
	void __iomem *bbar = tbl->bbar;

	/* peripheral MEM_1 region */
	target = calgary_reg(bbar, phb_offset(busnum) | PHB_MEM_1_LOW);
	low = be32_to_cpu(readl(target));
	target = calgary_reg(bbar, phb_offset(busnum) | PHB_MEM_1_HIGH);
	high = be32_to_cpu(readl(target));
	target = calgary_reg(bbar, phb_offset(busnum) | PHB_MEM_1_SIZE);
	sizelow = be32_to_cpu(readl(target));

	start = (high << 32) | low;
	limit = sizelow;

	calgary_reserve_mem_region(dev, start, limit);
}

static void __init calgary_reserve_peripheral_mem_2(struct pci_dev *dev)
{
	void __iomem *target;
	u32 val32;
	u64 low, high, sizelow, sizehigh;
	u64 start, limit;
698
	struct iommu_table *tbl = pci_iommu(dev->bus);
699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733
	unsigned char busnum = dev->bus->number;
	void __iomem *bbar = tbl->bbar;

	/* is it enabled? */
	target = calgary_reg(bbar, phb_offset(busnum) | PHB_CONFIG_RW_OFFSET);
	val32 = be32_to_cpu(readl(target));
	if (!(val32 & PHB_MEM2_ENABLE))
		return;

	target = calgary_reg(bbar, phb_offset(busnum) | PHB_MEM_2_LOW);
	low = be32_to_cpu(readl(target));
	target = calgary_reg(bbar, phb_offset(busnum) | PHB_MEM_2_HIGH);
	high = be32_to_cpu(readl(target));
	target = calgary_reg(bbar, phb_offset(busnum) | PHB_MEM_2_SIZE_LOW);
	sizelow = be32_to_cpu(readl(target));
	target = calgary_reg(bbar, phb_offset(busnum) | PHB_MEM_2_SIZE_HIGH);
	sizehigh = be32_to_cpu(readl(target));

	start = (high << 32) | low;
	limit = (sizehigh << 32) | sizelow;

	calgary_reserve_mem_region(dev, start, limit);
}

/*
 * some regions of the IO address space do not get translated, so we
 * must not give devices IO addresses in those regions. The regions
 * are the 640KB-1MB region and the two PCI peripheral memory holes.
 * Reserve all of them in the IOMMU bitmap to avoid giving them out
 * later.
 */
static void __init calgary_reserve_regions(struct pci_dev *dev)
{
	unsigned int npages;
	u64 start;
734
	struct iommu_table *tbl = pci_iommu(dev->bus);
735

736
	/* reserve EMERGENCY_PAGES from bad_dma_address and up */
737
	iommu_range_reserve(tbl, DMA_ERROR_CODE, EMERGENCY_PAGES);
738 739

	/* avoid the BIOS/VGA first 640KB-1MB region */
740
	/* for CalIOC2 - avoid the entire first MB */
741 742 743 744 745
	if (is_calgary(dev->device)) {
		start = (640 * 1024);
		npages = ((1024 - 640) * 1024) >> PAGE_SHIFT;
	} else { /* calioc2 */
		start = 0;
746
		npages = (1 * 1024 * 1024) >> PAGE_SHIFT;
747
	}
748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767
	iommu_range_reserve(tbl, start, npages);

	/* reserve the two PCI peripheral memory regions in IO space */
	calgary_reserve_peripheral_mem_1(dev);
	calgary_reserve_peripheral_mem_2(dev);
}

static int __init calgary_setup_tar(struct pci_dev *dev, void __iomem *bbar)
{
	u64 val64;
	u64 table_phys;
	void __iomem *target;
	int ret;
	struct iommu_table *tbl;

	/* build TCE tables for each PHB */
	ret = build_tce_table(dev, bbar);
	if (ret)
		return ret;

768
	tbl = pci_iommu(dev->bus);
769
	tbl->it_base = (unsigned long)bus_info[dev->bus->number].tce_space;
770 771 772 773 774

	if (is_kdump_kernel())
		calgary_init_bitmap_from_tce_table(tbl);
	else
		tce_free(tbl, 0, tbl->it_size);
775

776 777
	if (is_calgary(dev->device))
		tbl->chip_ops = &calgary_chip_ops;
778 779
	else if (is_calioc2(dev->device))
		tbl->chip_ops = &calioc2_chip_ops;
780 781
	else
		BUG();
782

783 784 785 786 787 788 789 790 791
	calgary_reserve_regions(dev);

	/* set TARs for each PHB */
	target = calgary_reg(bbar, tar_offset(dev->bus->number));
	val64 = be64_to_cpu(readq(target));

	/* zero out all TAR bits under sw control */
	val64 &= ~TAR_SW_BITS;
	table_phys = (u64)__pa(tbl->it_base);
792

793 794 795 796 797 798
	val64 |= table_phys;

	BUG_ON(specified_table_size > TCE_TABLE_SIZE_8M);
	val64 |= (u64) specified_table_size;

	tbl->tar_val = cpu_to_be64(val64);
799

800 801 802 803 804 805
	writeq(tbl->tar_val, target);
	readq(target); /* flush */

	return 0;
}

806
static void __init calgary_free_bus(struct pci_dev *dev)
807 808
{
	u64 val64;
809
	struct iommu_table *tbl = pci_iommu(dev->bus);
810
	void __iomem *target;
811
	unsigned int bitmapsz;
812 813 814 815 816 817 818

	target = calgary_reg(tbl->bbar, tar_offset(dev->bus->number));
	val64 = be64_to_cpu(readq(target));
	val64 &= ~TAR_SW_BITS;
	writeq(cpu_to_be64(val64), target);
	readq(target); /* flush */

819 820 821 822
	bitmapsz = tbl->it_size / BITS_PER_BYTE;
	free_pages((unsigned long)tbl->it_map, get_order(bitmapsz));
	tbl->it_map = NULL;

823
	kfree(tbl);
824 825
	
	set_pci_iommu(dev->bus, NULL);
826 827 828

	/* Can't free bootmem allocated memory after system is up :-( */
	bus_info[dev->bus->number].tce_space = NULL;
829 830
}

831 832 833
static void calgary_dump_error_regs(struct iommu_table *tbl)
{
	void __iomem *bbar = tbl->bbar;
834
	void __iomem *target;
835
	u32 csr, plssr;
836 837

	target = calgary_reg(bbar, phb_offset(tbl->it_busno) | PHB_CSR_OFFSET);
838 839 840 841
	csr = be32_to_cpu(readl(target));

	target = calgary_reg(bbar, phb_offset(tbl->it_busno) | PHB_PLSSR_OFFSET);
	plssr = be32_to_cpu(readl(target));
842 843 844

	/* If no error, the agent ID in the CSR is not valid */
	printk(KERN_EMERG "Calgary: DMA error on Calgary PHB 0x%x, "
845
	       "0x%08x@CSR 0x%08x@PLSSR\n", tbl->it_busno, csr, plssr);
846 847 848 849 850 851
}

static void calioc2_dump_error_regs(struct iommu_table *tbl)
{
	void __iomem *bbar = tbl->bbar;
	u32 csr, csmr, plssr, mck, rcstat;
852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870
	void __iomem *target;
	unsigned long phboff = phb_offset(tbl->it_busno);
	unsigned long erroff;
	u32 errregs[7];
	int i;

	/* dump CSR */
	target = calgary_reg(bbar, phboff | PHB_CSR_OFFSET);
	csr = be32_to_cpu(readl(target));
	/* dump PLSSR */
	target = calgary_reg(bbar, phboff | PHB_PLSSR_OFFSET);
	plssr = be32_to_cpu(readl(target));
	/* dump CSMR */
	target = calgary_reg(bbar, phboff | 0x290);
	csmr = be32_to_cpu(readl(target));
	/* dump mck */
	target = calgary_reg(bbar, phboff | 0x800);
	mck = be32_to_cpu(readl(target));

871 872 873 874 875
	printk(KERN_EMERG "Calgary: DMA error on CalIOC2 PHB 0x%x\n",
	       tbl->it_busno);

	printk(KERN_EMERG "Calgary: 0x%08x@CSR 0x%08x@PLSSR 0x%08x@CSMR 0x%08x@MCK\n",
	       csr, plssr, csmr, mck);
876 877 878 879

	/* dump rest of error regs */
	printk(KERN_EMERG "Calgary: ");
	for (i = 0; i < ARRAY_SIZE(errregs); i++) {
880 881
		/* err regs are at 0x810 - 0x870 */
		erroff = (0x810 + (i * 0x10));
882 883 884 885 886
		target = calgary_reg(bbar, phboff | erroff);
		errregs[i] = be32_to_cpu(readl(target));
		printk("0x%08x@0x%lx ", errregs[i], erroff);
	}
	printk("\n");
887 888 889 890 891 892

	/* root complex status */
	target = calgary_reg(bbar, phboff | PHB_ROOT_COMPLEX_STATUS);
	rcstat = be32_to_cpu(readl(target));
	printk(KERN_EMERG "Calgary: 0x%08x@0x%x\n", rcstat,
	       PHB_ROOT_COMPLEX_STATUS);
893 894
}

895 896 897
static void calgary_watchdog(unsigned long data)
{
	struct pci_dev *dev = (struct pci_dev *)data;
898
	struct iommu_table *tbl = pci_iommu(dev->bus);
899 900 901 902 903 904 905 906 907
	void __iomem *bbar = tbl->bbar;
	u32 val32;
	void __iomem *target;

	target = calgary_reg(bbar, phb_offset(tbl->it_busno) | PHB_CSR_OFFSET);
	val32 = be32_to_cpu(readl(target));

	/* If no error, the agent ID in the CSR is not valid */
	if (val32 & CSR_AGENT_MASK) {
908
		tbl->chip_ops->dump_error_regs(tbl);
909 910

		/* reset error */
911 912 913 914
		writel(0, target);

		/* Disable bus that caused the error */
		target = calgary_reg(bbar, phb_offset(tbl->it_busno) |
915
				     PHB_CONFIG_RW_OFFSET);
916 917 918 919 920 921 922 923 924 925
		val32 = be32_to_cpu(readl(target));
		val32 |= PHB_SLOT_DISABLE;
		writel(cpu_to_be32(val32), target);
		readl(target); /* flush */
	} else {
		/* Reset the timer */
		mod_timer(&tbl->watchdog_timer, jiffies + 2 * HZ);
	}
}

926 927
static void __init calgary_set_split_completion_timeout(void __iomem *bbar,
	unsigned char busnum, unsigned long timeout)
928 929 930
{
	u64 val64;
	void __iomem *target;
931
	unsigned int phb_shift = ~0; /* silence gcc */
932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952
	u64 mask;

	switch (busno_to_phbid(busnum)) {
	case 0: phb_shift = (63 - 19);
		break;
	case 1: phb_shift = (63 - 23);
		break;
	case 2: phb_shift = (63 - 27);
		break;
	case 3: phb_shift = (63 - 35);
		break;
	default:
		BUG_ON(busno_to_phbid(busnum));
	}

	target = calgary_reg(bbar, CALGARY_CONFIG_REG);
	val64 = be64_to_cpu(readq(target));

	/* zero out this PHB's timer bits */
	mask = ~(0xFUL << phb_shift);
	val64 &= mask;
953
	val64 |= (timeout << phb_shift);
954 955 956 957
	writeq(cpu_to_be64(val64), target);
	readq(target); /* flush */
}

958
static void __init calioc2_handle_quirks(struct iommu_table *tbl, struct pci_dev *dev)
959 960 961 962 963 964
{
	unsigned char busnum = dev->bus->number;
	void __iomem *bbar = tbl->bbar;
	void __iomem *target;
	u32 val;

965 966 967 968 969 970 971
	/*
	 * CalIOC2 designers recommend setting bit 8 in 0xnDB0 to 1
	 */
	target = calgary_reg(bbar, phb_offset(busnum) | PHB_SAVIOR_L2);
	val = cpu_to_be32(readl(target));
	val |= 0x00800000;
	writel(cpu_to_be32(val), target);
972 973
}

974
static void __init calgary_handle_quirks(struct iommu_table *tbl, struct pci_dev *dev)
975 976 977 978 979 980 981
{
	unsigned char busnum = dev->bus->number;

	/*
	 * Give split completion a longer timeout on bus 1 for aic94xx
	 * http://bugzilla.kernel.org/show_bug.cgi?id=7180
	 */
982
	if (is_calgary(dev->device) && (busnum == 1))
983 984 985 986
		calgary_set_split_completion_timeout(tbl->bbar, busnum,
						     CCR_2SEC_TIMEOUT);
}

987 988 989 990 991 992 993 994 995
static void __init calgary_enable_translation(struct pci_dev *dev)
{
	u32 val32;
	unsigned char busnum;
	void __iomem *target;
	void __iomem *bbar;
	struct iommu_table *tbl;

	busnum = dev->bus->number;
996
	tbl = pci_iommu(dev->bus);
997 998 999 1000 1001 1002 1003
	bbar = tbl->bbar;

	/* enable TCE in PHB Config Register */
	target = calgary_reg(bbar, phb_offset(busnum) | PHB_CONFIG_RW_OFFSET);
	val32 = be32_to_cpu(readl(target));
	val32 |= PHB_TCE_ENABLE | PHB_DAC_DISABLE | PHB_MCSR_ENABLE;

1004 1005 1006
	printk(KERN_INFO "Calgary: enabling translation on %s PHB %#x\n",
	       (dev->device == PCI_DEVICE_ID_IBM_CALGARY) ?
	       "Calgary" : "CalIOC2", busnum);
1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027
	printk(KERN_INFO "Calgary: errant DMAs will now be prevented on this "
	       "bus.\n");

	writel(cpu_to_be32(val32), target);
	readl(target); /* flush */

	init_timer(&tbl->watchdog_timer);
	tbl->watchdog_timer.function = &calgary_watchdog;
	tbl->watchdog_timer.data = (unsigned long)dev;
	mod_timer(&tbl->watchdog_timer, jiffies);
}

static void __init calgary_disable_translation(struct pci_dev *dev)
{
	u32 val32;
	unsigned char busnum;
	void __iomem *target;
	void __iomem *bbar;
	struct iommu_table *tbl;

	busnum = dev->bus->number;
1028
	tbl = pci_iommu(dev->bus);
1029 1030 1031 1032 1033 1034 1035
	bbar = tbl->bbar;

	/* disable TCE in PHB Config Register */
	target = calgary_reg(bbar, phb_offset(busnum) | PHB_CONFIG_RW_OFFSET);
	val32 = be32_to_cpu(readl(target));
	val32 &= ~(PHB_TCE_ENABLE | PHB_DAC_DISABLE | PHB_MCSR_ENABLE);

1036
	printk(KERN_INFO "Calgary: disabling translation on PHB %#x!\n", busnum);
1037 1038 1039 1040 1041 1042
	writel(cpu_to_be32(val32), target);
	readl(target); /* flush */

	del_timer_sync(&tbl->watchdog_timer);
}

1043
static void __init calgary_init_one_nontraslated(struct pci_dev *dev)
1044
{
1045
	pci_dev_get(dev);
1046
	set_pci_iommu(dev->bus, NULL);
1047 1048 1049 1050 1051 1052

	/* is the device behind a bridge? */
	if (dev->bus->parent)
		dev->bus->parent->self = dev;
	else
		dev->bus->self = dev;
1053 1054 1055 1056 1057
}

static int __init calgary_init_one(struct pci_dev *dev)
{
	void __iomem *bbar;
1058
	struct iommu_table *tbl;
1059 1060
	int ret;

1061
	bbar = busno_to_bbar(dev->bus->number);
1062 1063
	ret = calgary_setup_tar(dev, bbar);
	if (ret)
1064
		goto done;
1065

1066
	pci_dev_get(dev);
1067 1068 1069 1070 1071 1072 1073 1074

	if (dev->bus->parent) {
		if (dev->bus->parent->self)
			printk(KERN_WARNING "Calgary: IEEEE, dev %p has "
			       "bus->parent->self!\n", dev);
		dev->bus->parent->self = dev;
	} else
		dev->bus->self = dev;
1075

1076
	tbl = pci_iommu(dev->bus);
1077
	tbl->chip_ops->handle_quirks(tbl, dev);
1078

1079 1080 1081 1082 1083 1084 1085 1086
	calgary_enable_translation(dev);

	return 0;