iommu.c 34.7 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1 2 3
/*
 * Copyright (C) 2001 Mike Corrigan & Dave Engebretsen, IBM Corporation
 *
4
 * Rewrite, cleanup:
Linus Torvalds's avatar
Linus Torvalds committed
5
 *
6
 * Copyright (C) 2004 Olof Johansson <olof@lixom.net>, IBM Corporation
7
 * Copyright (C) 2006 Olof Johansson <olof@lixom.net>
Linus Torvalds's avatar
Linus Torvalds committed
8 9 10
 *
 * Dynamic DMA mapping support, pSeries-specific parts, both SMP and LPAR.
 *
11
 *
Linus Torvalds's avatar
Linus Torvalds committed
12 13 14 15
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
16
 *
Linus Torvalds's avatar
Linus Torvalds committed
17 18 19 20
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
21
 *
Linus Torvalds's avatar
Linus Torvalds committed
22 23 24 25 26 27 28 29 30 31
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
 */

#include <linux/init.h>
#include <linux/types.h>
#include <linux/slab.h>
#include <linux/mm.h>
#include <linux/spinlock.h>
32
#include <linux/sched.h>	/* for show_stack */
Linus Torvalds's avatar
Linus Torvalds committed
33 34 35
#include <linux/string.h>
#include <linux/pci.h>
#include <linux/dma-mapping.h>
36
#include <linux/crash_dump.h>
37
#include <linux/memory.h>
Linus Torvalds's avatar
Linus Torvalds committed
38 39 40 41 42 43 44 45
#include <asm/io.h>
#include <asm/prom.h>
#include <asm/rtas.h>
#include <asm/iommu.h>
#include <asm/pci-bridge.h>
#include <asm/machdep.h>
#include <asm/abs_addr.h>
#include <asm/pSeries_reconfig.h>
46
#include <asm/firmware.h>
47
#include <asm/tce.h>
48
#include <asm/ppc-pci.h>
49
#include <asm/udbg.h>
50
#include <asm/mmzone.h>
Linus Torvalds's avatar
Linus Torvalds committed
51

52 53
#include "plpar_wrappers.h"

Linus Torvalds's avatar
Linus Torvalds committed
54

55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83
static void tce_invalidate_pSeries_sw(struct iommu_table *tbl,
				      u64 *startp, u64 *endp)
{
	u64 __iomem *invalidate = (u64 __iomem *)tbl->it_index;
	unsigned long start, end, inc;

	start = __pa(startp);
	end = __pa(endp);
	inc = L1_CACHE_BYTES; /* invalidate a cacheline of TCEs at a time */

	/* If this is non-zero, change the format.  We shift the
	 * address and or in the magic from the device tree. */
	if (tbl->it_busno) {
		start <<= 12;
		end <<= 12;
		inc <<= 12;
		start |= tbl->it_busno;
		end |= tbl->it_busno;
	}

	end |= inc - 1; /* round up end to be different than start */

	mb(); /* Make sure TCEs in memory are written */
	while (start <= end) {
		out_be64(invalidate, start);
		start += inc;
	}
}

84
static int tce_build_pSeries(struct iommu_table *tbl, long index,
85
			      long npages, unsigned long uaddr,
86 87
			      enum dma_data_direction direction,
			      struct dma_attrs *attrs)
Linus Torvalds's avatar
Linus Torvalds committed
88
{
89
	u64 proto_tce;
90
	u64 *tcep, *tces;
91
	u64 rpn;
Linus Torvalds's avatar
Linus Torvalds committed
92

93
	proto_tce = TCE_PCI_READ; // Read allowed
Linus Torvalds's avatar
Linus Torvalds committed
94 95

	if (direction != DMA_TO_DEVICE)
96
		proto_tce |= TCE_PCI_WRITE;
Linus Torvalds's avatar
Linus Torvalds committed
97

98
	tces = tcep = ((u64 *)tbl->it_base) + index;
Linus Torvalds's avatar
Linus Torvalds committed
99 100

	while (npages--) {
Yinghai Lu's avatar
Yinghai Lu committed
101
		/* can't move this out since we might cross MEMBLOCK boundary */
102 103
		rpn = (virt_to_abs(uaddr)) >> TCE_SHIFT;
		*tcep = proto_tce | (rpn & TCE_RPN_MASK) << TCE_RPN_SHIFT;
Linus Torvalds's avatar
Linus Torvalds committed
104

105
		uaddr += TCE_PAGE_SIZE;
106
		tcep++;
Linus Torvalds's avatar
Linus Torvalds committed
107
	}
108

109
	if (tbl->it_type & TCE_PCI_SWINV_CREATE)
110
		tce_invalidate_pSeries_sw(tbl, tces, tcep - 1);
111
	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
112 113 114 115 116
}


static void tce_free_pSeries(struct iommu_table *tbl, long index, long npages)
{
117
	u64 *tcep, *tces;
Linus Torvalds's avatar
Linus Torvalds committed
118

119
	tces = tcep = ((u64 *)tbl->it_base) + index;
120 121 122

	while (npages--)
		*(tcep++) = 0;
123

124
	if (tbl->it_type & TCE_PCI_SWINV_FREE)
125
		tce_invalidate_pSeries_sw(tbl, tces, tcep - 1);
Linus Torvalds's avatar
Linus Torvalds committed
126 127
}

128 129 130 131 132 133 134 135
static unsigned long tce_get_pseries(struct iommu_table *tbl, long index)
{
	u64 *tcep;

	tcep = ((u64 *)tbl->it_base) + index;

	return *tcep;
}
Linus Torvalds's avatar
Linus Torvalds committed
136

137 138 139 140
static void tce_free_pSeriesLP(struct iommu_table*, long, long);
static void tce_freemulti_pSeriesLP(struct iommu_table*, long, long);

static int tce_build_pSeriesLP(struct iommu_table *tbl, long tcenum,
Linus Torvalds's avatar
Linus Torvalds committed
141
				long npages, unsigned long uaddr,
142 143
				enum dma_data_direction direction,
				struct dma_attrs *attrs)
Linus Torvalds's avatar
Linus Torvalds committed
144
{
145
	u64 rc = 0;
146 147
	u64 proto_tce, tce;
	u64 rpn;
148 149
	int ret = 0;
	long tcenum_start = tcenum, npages_start = npages;
Linus Torvalds's avatar
Linus Torvalds committed
150

151 152
	rpn = (virt_to_abs(uaddr)) >> TCE_SHIFT;
	proto_tce = TCE_PCI_READ;
Linus Torvalds's avatar
Linus Torvalds committed
153
	if (direction != DMA_TO_DEVICE)
154
		proto_tce |= TCE_PCI_WRITE;
Linus Torvalds's avatar
Linus Torvalds committed
155 156

	while (npages--) {
157 158 159
		tce = proto_tce | (rpn & TCE_RPN_MASK) << TCE_RPN_SHIFT;
		rc = plpar_tce_put((u64)tbl->it_index, (u64)tcenum << 12, tce);

160 161 162 163 164 165 166
		if (unlikely(rc == H_NOT_ENOUGH_RESOURCES)) {
			ret = (int)rc;
			tce_free_pSeriesLP(tbl, tcenum_start,
			                   (npages_start - (npages + 1)));
			break;
		}

Linus Torvalds's avatar
Linus Torvalds committed
167
		if (rc && printk_ratelimit()) {
168 169 170 171
			printk("tce_build_pSeriesLP: plpar_tce_put failed. rc=%lld\n", rc);
			printk("\tindex   = 0x%llx\n", (u64)tbl->it_index);
			printk("\ttcenum  = 0x%llx\n", (u64)tcenum);
			printk("\ttce val = 0x%llx\n", tce );
Linus Torvalds's avatar
Linus Torvalds committed
172 173
			show_stack(current, (unsigned long *)__get_SP());
		}
174

Linus Torvalds's avatar
Linus Torvalds committed
175
		tcenum++;
176
		rpn++;
Linus Torvalds's avatar
Linus Torvalds committed
177
	}
178
	return ret;
Linus Torvalds's avatar
Linus Torvalds committed
179 180
}

181
static DEFINE_PER_CPU(u64 *, tce_page);
Linus Torvalds's avatar
Linus Torvalds committed
182

183
static int tce_buildmulti_pSeriesLP(struct iommu_table *tbl, long tcenum,
Linus Torvalds's avatar
Linus Torvalds committed
184
				     long npages, unsigned long uaddr,
185 186
				     enum dma_data_direction direction,
				     struct dma_attrs *attrs)
Linus Torvalds's avatar
Linus Torvalds committed
187
{
188
	u64 rc = 0;
189 190 191
	u64 proto_tce;
	u64 *tcep;
	u64 rpn;
Linus Torvalds's avatar
Linus Torvalds committed
192
	long l, limit;
193 194
	long tcenum_start = tcenum, npages_start = npages;
	int ret = 0;
Linus Torvalds's avatar
Linus Torvalds committed
195

196
	if (npages == 1) {
197 198
		return tce_build_pSeriesLP(tbl, tcenum, npages, uaddr,
		                           direction, attrs);
199
	}
Linus Torvalds's avatar
Linus Torvalds committed
200 201 202 203 204 205 206

	tcep = __get_cpu_var(tce_page);

	/* This is safe to do since interrupts are off when we're called
	 * from iommu_alloc{,_sg}()
	 */
	if (!tcep) {
207
		tcep = (u64 *)__get_free_page(GFP_ATOMIC);
Linus Torvalds's avatar
Linus Torvalds committed
208
		/* If allocation fails, fall back to the loop implementation */
209
		if (!tcep) {
210
			return tce_build_pSeriesLP(tbl, tcenum, npages, uaddr,
211
					    direction, attrs);
212
		}
Linus Torvalds's avatar
Linus Torvalds committed
213 214 215
		__get_cpu_var(tce_page) = tcep;
	}

216 217
	rpn = (virt_to_abs(uaddr)) >> TCE_SHIFT;
	proto_tce = TCE_PCI_READ;
Linus Torvalds's avatar
Linus Torvalds committed
218
	if (direction != DMA_TO_DEVICE)
219
		proto_tce |= TCE_PCI_WRITE;
Linus Torvalds's avatar
Linus Torvalds committed
220 221 222 223 224 225 226

	/* We can map max one pageful of TCEs at a time */
	do {
		/*
		 * Set up the page with TCE data, looping through and setting
		 * the values.
		 */
227
		limit = min_t(long, npages, 4096/TCE_ENTRY_SIZE);
Linus Torvalds's avatar
Linus Torvalds committed
228 229

		for (l = 0; l < limit; l++) {
230 231
			tcep[l] = proto_tce | (rpn & TCE_RPN_MASK) << TCE_RPN_SHIFT;
			rpn++;
Linus Torvalds's avatar
Linus Torvalds committed
232 233 234 235 236 237 238 239 240 241 242
		}

		rc = plpar_tce_put_indirect((u64)tbl->it_index,
					    (u64)tcenum << 12,
					    (u64)virt_to_abs(tcep),
					    limit);

		npages -= limit;
		tcenum += limit;
	} while (npages > 0 && !rc);

243 244 245 246 247 248 249
	if (unlikely(rc == H_NOT_ENOUGH_RESOURCES)) {
		ret = (int)rc;
		tce_freemulti_pSeriesLP(tbl, tcenum_start,
		                        (npages_start - (npages + limit)));
		return ret;
	}

Linus Torvalds's avatar
Linus Torvalds committed
250
	if (rc && printk_ratelimit()) {
251 252 253 254
		printk("tce_buildmulti_pSeriesLP: plpar_tce_put failed. rc=%lld\n", rc);
		printk("\tindex   = 0x%llx\n", (u64)tbl->it_index);
		printk("\tnpages  = 0x%llx\n", (u64)npages);
		printk("\ttce[0] val = 0x%llx\n", tcep[0]);
Linus Torvalds's avatar
Linus Torvalds committed
255 256
		show_stack(current, (unsigned long *)__get_SP());
	}
257
	return ret;
Linus Torvalds's avatar
Linus Torvalds committed
258 259 260 261 262 263 264
}

static void tce_free_pSeriesLP(struct iommu_table *tbl, long tcenum, long npages)
{
	u64 rc;

	while (npages--) {
265
		rc = plpar_tce_put((u64)tbl->it_index, (u64)tcenum << 12, 0);
Linus Torvalds's avatar
Linus Torvalds committed
266 267

		if (rc && printk_ratelimit()) {
268 269 270
			printk("tce_free_pSeriesLP: plpar_tce_put failed. rc=%lld\n", rc);
			printk("\tindex   = 0x%llx\n", (u64)tbl->it_index);
			printk("\ttcenum  = 0x%llx\n", (u64)tcenum);
Linus Torvalds's avatar
Linus Torvalds committed
271 272 273 274 275 276 277 278 279 280 281 282
			show_stack(current, (unsigned long *)__get_SP());
		}

		tcenum++;
	}
}


static void tce_freemulti_pSeriesLP(struct iommu_table *tbl, long tcenum, long npages)
{
	u64 rc;

283
	rc = plpar_tce_stuff((u64)tbl->it_index, (u64)tcenum << 12, 0, npages);
Linus Torvalds's avatar
Linus Torvalds committed
284 285 286

	if (rc && printk_ratelimit()) {
		printk("tce_freemulti_pSeriesLP: plpar_tce_stuff failed\n");
287 288 289
		printk("\trc      = %lld\n", rc);
		printk("\tindex   = 0x%llx\n", (u64)tbl->it_index);
		printk("\tnpages  = 0x%llx\n", (u64)npages);
Linus Torvalds's avatar
Linus Torvalds committed
290 291 292 293
		show_stack(current, (unsigned long *)__get_SP());
	}
}

294 295 296 297 298 299 300 301
static unsigned long tce_get_pSeriesLP(struct iommu_table *tbl, long tcenum)
{
	u64 rc;
	unsigned long tce_ret;

	rc = plpar_tce_get((u64)tbl->it_index, (u64)tcenum << 12, &tce_ret);

	if (rc && printk_ratelimit()) {
302 303 304
		printk("tce_get_pSeriesLP: plpar_tce_get failed. rc=%lld\n", rc);
		printk("\tindex   = 0x%llx\n", (u64)tbl->it_index);
		printk("\ttcenum  = 0x%llx\n", (u64)tcenum);
305 306 307 308 309 310
		show_stack(current, (unsigned long *)__get_SP());
	}

	return tce_ret;
}

Lucas De Marchi's avatar
Lucas De Marchi committed
311
/* this is compatible with cells for the device tree property */
312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456
struct dynamic_dma_window_prop {
	__be32	liobn;		/* tce table number */
	__be64	dma_base;	/* address hi,lo */
	__be32	tce_shift;	/* ilog2(tce_page_size) */
	__be32	window_shift;	/* ilog2(tce_window_size) */
};

struct direct_window {
	struct device_node *device;
	const struct dynamic_dma_window_prop *prop;
	struct list_head list;
};

/* Dynamic DMA Window support */
struct ddw_query_response {
	u32 windows_available;
	u32 largest_available_block;
	u32 page_size;
	u32 migration_capable;
};

struct ddw_create_response {
	u32 liobn;
	u32 addr_hi;
	u32 addr_lo;
};

static LIST_HEAD(direct_window_list);
/* prevents races between memory on/offline and window creation */
static DEFINE_SPINLOCK(direct_window_list_lock);
/* protects initializing window twice for same device */
static DEFINE_MUTEX(direct_window_init_mutex);
#define DIRECT64_PROPNAME "linux,direct64-ddr-window-info"

static int tce_clearrange_multi_pSeriesLP(unsigned long start_pfn,
					unsigned long num_pfn, const void *arg)
{
	const struct dynamic_dma_window_prop *maprange = arg;
	int rc;
	u64 tce_size, num_tce, dma_offset, next;
	u32 tce_shift;
	long limit;

	tce_shift = be32_to_cpu(maprange->tce_shift);
	tce_size = 1ULL << tce_shift;
	next = start_pfn << PAGE_SHIFT;
	num_tce = num_pfn << PAGE_SHIFT;

	/* round back to the beginning of the tce page size */
	num_tce += next & (tce_size - 1);
	next &= ~(tce_size - 1);

	/* covert to number of tces */
	num_tce |= tce_size - 1;
	num_tce >>= tce_shift;

	do {
		/*
		 * Set up the page with TCE data, looping through and setting
		 * the values.
		 */
		limit = min_t(long, num_tce, 512);
		dma_offset = next + be64_to_cpu(maprange->dma_base);

		rc = plpar_tce_stuff((u64)be32_to_cpu(maprange->liobn),
					     dma_offset,
					     0, limit);
		num_tce -= limit;
	} while (num_tce > 0 && !rc);

	return rc;
}

static int tce_setrange_multi_pSeriesLP(unsigned long start_pfn,
					unsigned long num_pfn, const void *arg)
{
	const struct dynamic_dma_window_prop *maprange = arg;
	u64 *tcep, tce_size, num_tce, dma_offset, next, proto_tce, liobn;
	u32 tce_shift;
	u64 rc = 0;
	long l, limit;

	local_irq_disable();	/* to protect tcep and the page behind it */
	tcep = __get_cpu_var(tce_page);

	if (!tcep) {
		tcep = (u64 *)__get_free_page(GFP_ATOMIC);
		if (!tcep) {
			local_irq_enable();
			return -ENOMEM;
		}
		__get_cpu_var(tce_page) = tcep;
	}

	proto_tce = TCE_PCI_READ | TCE_PCI_WRITE;

	liobn = (u64)be32_to_cpu(maprange->liobn);
	tce_shift = be32_to_cpu(maprange->tce_shift);
	tce_size = 1ULL << tce_shift;
	next = start_pfn << PAGE_SHIFT;
	num_tce = num_pfn << PAGE_SHIFT;

	/* round back to the beginning of the tce page size */
	num_tce += next & (tce_size - 1);
	next &= ~(tce_size - 1);

	/* covert to number of tces */
	num_tce |= tce_size - 1;
	num_tce >>= tce_shift;

	/* We can map max one pageful of TCEs at a time */
	do {
		/*
		 * Set up the page with TCE data, looping through and setting
		 * the values.
		 */
		limit = min_t(long, num_tce, 4096/TCE_ENTRY_SIZE);
		dma_offset = next + be64_to_cpu(maprange->dma_base);

		for (l = 0; l < limit; l++) {
			tcep[l] = proto_tce | next;
			next += tce_size;
		}

		rc = plpar_tce_put_indirect(liobn,
					    dma_offset,
					    (u64)virt_to_abs(tcep),
					    limit);

		num_tce -= limit;
	} while (num_tce > 0 && !rc);

	/* error cleanup: caller will clear whole range */

	local_irq_enable();
	return rc;
}

static int tce_setrange_multi_pSeriesLP_walk(unsigned long start_pfn,
		unsigned long num_pfn, void *arg)
{
	return tce_setrange_multi_pSeriesLP(start_pfn, num_pfn, arg);
}


457
#ifdef CONFIG_PCI
Linus Torvalds's avatar
Linus Torvalds committed
458 459
static void iommu_table_setparms(struct pci_controller *phb,
				 struct device_node *dn,
460
				 struct iommu_table *tbl)
Linus Torvalds's avatar
Linus Torvalds committed
461 462
{
	struct device_node *node;
463
	const unsigned long *basep, *sw_inval;
464
	const u32 *sizep;
Linus Torvalds's avatar
Linus Torvalds committed
465

466
	node = phb->dn;
Linus Torvalds's avatar
Linus Torvalds committed
467

468 469
	basep = of_get_property(node, "linux,tce-base", NULL);
	sizep = of_get_property(node, "linux,tce-size", NULL);
Linus Torvalds's avatar
Linus Torvalds committed
470 471 472 473 474 475 476
	if (basep == NULL || sizep == NULL) {
		printk(KERN_ERR "PCI_DMA: iommu_table_setparms: %s has "
				"missing tce entries !\n", dn->full_name);
		return;
	}

	tbl->it_base = (unsigned long)__va(*basep);
477

478
	if (!is_kdump_kernel())
479
		memset((void *)tbl->it_base, 0, *sizep);
Linus Torvalds's avatar
Linus Torvalds committed
480 481

	tbl->it_busno = phb->bus->number;
482

Linus Torvalds's avatar
Linus Torvalds committed
483
	/* Units of tce entries */
484
	tbl->it_offset = phb->dma_window_base_cur >> IOMMU_PAGE_SHIFT;
485

Linus Torvalds's avatar
Linus Torvalds committed
486
	/* Test if we are going over 2GB of DMA space */
487 488
	if (phb->dma_window_base_cur + phb->dma_window_size > 0x80000000ul) {
		udbg_printf("PCI_DMA: Unexpected number of IOAs under this PHB.\n");
489
		panic("PCI_DMA: Unexpected number of IOAs under this PHB.\n");
490
	}
491

Linus Torvalds's avatar
Linus Torvalds committed
492 493 494
	phb->dma_window_base_cur += phb->dma_window_size;

	/* Set the tce table size - measured in entries */
495
	tbl->it_size = phb->dma_window_size >> IOMMU_PAGE_SHIFT;
Linus Torvalds's avatar
Linus Torvalds committed
496 497 498 499

	tbl->it_index = 0;
	tbl->it_blocksize = 16;
	tbl->it_type = TCE_PCI;
500 501 502 503 504 505 506 507 508 509 510 511 512 513

	sw_inval = of_get_property(node, "linux,tce-sw-invalidate-info", NULL);
	if (sw_inval) {
		/*
		 * This property contains information on how to
		 * invalidate the TCE entry.  The first property is
		 * the base MMIO address used to invalidate entries.
		 * The second property tells us the format of the TCE
		 * invalidate (whether it needs to be shifted) and
		 * some magic routing info to add to our invalidate
		 * command.
		 */
		tbl->it_index = (unsigned long) ioremap(sw_inval[0], 8);
		tbl->it_busno = sw_inval[1]; /* overload this with magic */
514
		tbl->it_type = TCE_PCI_SWINV_CREATE | TCE_PCI_SWINV_FREE;
515
	}
Linus Torvalds's avatar
Linus Torvalds committed
516 517 518 519 520 521 522 523 524 525
}

/*
 * iommu_table_setparms_lpar
 *
 * Function: On pSeries LPAR systems, return TCE table info, given a pci bus.
 */
static void iommu_table_setparms_lpar(struct pci_controller *phb,
				      struct device_node *dn,
				      struct iommu_table *tbl,
526
				      const void *dma_window)
Linus Torvalds's avatar
Linus Torvalds committed
527
{
528 529 530
	unsigned long offset, size;

	of_parse_dma_window(dn, dma_window, &tbl->it_index, &offset, &size);
Linus Torvalds's avatar
Linus Torvalds committed
531

532
	tbl->it_busno = phb->bus->number;
Linus Torvalds's avatar
Linus Torvalds committed
533 534 535
	tbl->it_base   = 0;
	tbl->it_blocksize  = 16;
	tbl->it_type = TCE_PCI;
536 537
	tbl->it_offset = offset >> IOMMU_PAGE_SHIFT;
	tbl->it_size = size >> IOMMU_PAGE_SHIFT;
Linus Torvalds's avatar
Linus Torvalds committed
538 539
}

540
static void pci_dma_bus_setup_pSeries(struct pci_bus *bus)
Linus Torvalds's avatar
Linus Torvalds committed
541
{
542
	struct device_node *dn;
Linus Torvalds's avatar
Linus Torvalds committed
543
	struct iommu_table *tbl;
544 545 546 547
	struct device_node *isa_dn, *isa_dn_orig;
	struct device_node *tmp;
	struct pci_dn *pci;
	int children;
Linus Torvalds's avatar
Linus Torvalds committed
548

549
	dn = pci_bus_to_OF_node(bus);
550

551
	pr_debug("pci_dma_bus_setup_pSeries: setting up bus %s\n", dn->full_name);
552 553 554 555 556 557 558

	if (bus->self) {
		/* This is not a root bus, any setup will be done for the
		 * device-side of the bridge in iommu_dev_setup_pSeries().
		 */
		return;
	}
559
	pci = PCI_DN(dn);
560 561 562

	/* Check if the ISA bus on the system is under
	 * this PHB.
Linus Torvalds's avatar
Linus Torvalds committed
563
	 */
564
	isa_dn = isa_dn_orig = of_find_node_by_type(NULL, "isa");
Linus Torvalds's avatar
Linus Torvalds committed
565

566 567 568 569 570
	while (isa_dn && isa_dn != dn)
		isa_dn = isa_dn->parent;

	if (isa_dn_orig)
		of_node_put(isa_dn_orig);
Linus Torvalds's avatar
Linus Torvalds committed
571

572
	/* Count number of direct PCI children of the PHB. */
573
	for (children = 0, tmp = dn->child; tmp; tmp = tmp->sibling)
574
		children++;
Linus Torvalds's avatar
Linus Torvalds committed
575

576
	pr_debug("Children: %d\n", children);
Linus Torvalds's avatar
Linus Torvalds committed
577

578 579 580 581 582
	/* Calculate amount of DMA window per slot. Each window must be
	 * a power of two (due to pci_alloc_consistent requirements).
	 *
	 * Keep 256MB aside for PHBs with ISA.
	 */
Linus Torvalds's avatar
Linus Torvalds committed
583

584 585 586 587 588 589
	if (!isa_dn) {
		/* No ISA/IDE - just set window size and return */
		pci->phb->dma_window_size = 0x80000000ul; /* To be divided */

		while (pci->phb->dma_window_size * children > 0x80000000ul)
			pci->phb->dma_window_size >>= 1;
590
		pr_debug("No ISA/IDE, window size is 0x%llx\n",
591
			 pci->phb->dma_window_size);
592 593 594
		pci->phb->dma_window_base_cur = 0;

		return;
Linus Torvalds's avatar
Linus Torvalds committed
595
	}
596 597 598 599 600 601 602 603 604

	/* If we have ISA, then we probably have an IDE
	 * controller too. Allocate a 128MB table but
	 * skip the first 128MB to avoid stepping on ISA
	 * space.
	 */
	pci->phb->dma_window_size = 0x8000000ul;
	pci->phb->dma_window_base_cur = 0x8000000ul;

605
	tbl = kzalloc_node(sizeof(struct iommu_table), GFP_KERNEL,
606
			   pci->phb->node);
607 608

	iommu_table_setparms(pci->phb, dn, tbl);
609
	pci->iommu_table = iommu_init_table(tbl, pci->phb->node);
610 611 612 613 614 615

	/* Divide the rest (1.75GB) among the children */
	pci->phb->dma_window_size = 0x80000000ul;
	while (pci->phb->dma_window_size * children > 0x70000000ul)
		pci->phb->dma_window_size >>= 1;

616
	pr_debug("ISA/IDE, window size is 0x%llx\n", pci->phb->dma_window_size);
Linus Torvalds's avatar
Linus Torvalds committed
617 618 619
}


620
static void pci_dma_bus_setup_pSeriesLP(struct pci_bus *bus)
Linus Torvalds's avatar
Linus Torvalds committed
621 622 623
{
	struct iommu_table *tbl;
	struct device_node *dn, *pdn;
624
	struct pci_dn *ppci;
625
	const void *dma_window = NULL;
Linus Torvalds's avatar
Linus Torvalds committed
626 627 628

	dn = pci_bus_to_OF_node(bus);

629 630
	pr_debug("pci_dma_bus_setup_pSeriesLP: setting up bus %s\n",
		 dn->full_name);
631

Linus Torvalds's avatar
Linus Torvalds committed
632 633
	/* Find nearest ibm,dma-window, walking up the device tree */
	for (pdn = dn; pdn != NULL; pdn = pdn->parent) {
634
		dma_window = of_get_property(pdn, "ibm,dma-window", NULL);
Linus Torvalds's avatar
Linus Torvalds committed
635 636 637 638 639
		if (dma_window != NULL)
			break;
	}

	if (dma_window == NULL) {
640
		pr_debug("  no ibm,dma-window property !\n");
Linus Torvalds's avatar
Linus Torvalds committed
641 642 643
		return;
	}

644
	ppci = PCI_DN(pdn);
645

646 647
	pr_debug("  parent is %s, iommu_table: 0x%p\n",
		 pdn->full_name, ppci->iommu_table);
648

649
	if (!ppci->iommu_table) {
650
		tbl = kzalloc_node(sizeof(struct iommu_table), GFP_KERNEL,
651
				   ppci->phb->node);
652
		iommu_table_setparms_lpar(ppci->phb, pdn, tbl, dma_window);
653
		ppci->iommu_table = iommu_init_table(tbl, ppci->phb->node);
654
		pr_debug("  created table: %p\n", ppci->iommu_table);
Linus Torvalds's avatar
Linus Torvalds committed
655 656 657 658
	}
}


659
static void pci_dma_dev_setup_pSeries(struct pci_dev *dev)
Linus Torvalds's avatar
Linus Torvalds committed
660
{
661
	struct device_node *dn;
662
	struct iommu_table *tbl;
Linus Torvalds's avatar
Linus Torvalds committed
663

664
	pr_debug("pci_dma_dev_setup_pSeries: %s\n", pci_name(dev));
Linus Torvalds's avatar
Linus Torvalds committed
665

666
	dn = dev->dev.of_node;
Linus Torvalds's avatar
Linus Torvalds committed
667

668 669 670 671 672
	/* If we're the direct child of a root bus, then we need to allocate
	 * an iommu table ourselves. The bus setup code should have setup
	 * the window sizes already.
	 */
	if (!dev->bus->self) {
673 674
		struct pci_controller *phb = PCI_DN(dn)->phb;

675
		pr_debug(" --> first child, no bridge. Allocating iommu table.\n");
676
		tbl = kzalloc_node(sizeof(struct iommu_table), GFP_KERNEL,
677 678
				   phb->node);
		iommu_table_setparms(phb, dn, tbl);
679
		PCI_DN(dn)->iommu_table = iommu_init_table(tbl, phb->node);
680
		set_iommu_table_base(&dev->dev, PCI_DN(dn)->iommu_table);
681 682 683 684 685 686 687
		return;
	}

	/* If this device is further down the bus tree, search upwards until
	 * an already allocated iommu table is found and use that.
	 */

688
	while (dn && PCI_DN(dn) && PCI_DN(dn)->iommu_table == NULL)
Linus Torvalds's avatar
Linus Torvalds committed
689 690
		dn = dn->parent;

691
	if (dn && PCI_DN(dn))
692
		set_iommu_table_base(&dev->dev, PCI_DN(dn)->iommu_table);
693 694 695
	else
		printk(KERN_WARNING "iommu: Device %s has no iommu table\n",
		       pci_name(dev));
Linus Torvalds's avatar
Linus Torvalds committed
696 697
}

698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713
static int __read_mostly disable_ddw;

static int __init disable_ddw_setup(char *str)
{
	disable_ddw = 1;
	printk(KERN_INFO "ppc iommu: disabling ddw.\n");

	return 0;
}

early_param("disable_ddw", disable_ddw_setup);

static void remove_ddw(struct device_node *np)
{
	struct dynamic_dma_window_prop *dwp;
	struct property *win64;
714
	const u32 *ddw_avail;
715 716 717
	u64 liobn;
	int len, ret;

718
	ddw_avail = of_get_property(np, "ibm,ddw-applicable", &len);
719
	win64 = of_find_property(np, DIRECT64_PROPNAME, NULL);
720
	if (!win64)
721 722
		return;

723
	if (!ddw_avail || len < 3 * sizeof(u32) || win64->length < sizeof(*dwp))
724 725
		goto delprop;

726 727 728 729 730 731 732 733 734 735 736 737 738
	dwp = win64->value;
	liobn = (u64)be32_to_cpu(dwp->liobn);

	/* clear the whole window, note the arg is in kernel pages */
	ret = tce_clearrange_multi_pSeriesLP(0,
		1ULL << (be32_to_cpu(dwp->window_shift) - PAGE_SHIFT), dwp);
	if (ret)
		pr_warning("%s failed to clear tces in window.\n",
			 np->full_name);
	else
		pr_debug("%s successfully cleared tces in window.\n",
			 np->full_name);

739
	ret = rtas_call(ddw_avail[2], 1, 1, NULL, liobn);
740 741 742
	if (ret)
		pr_warning("%s: failed to remove direct window: rtas returned "
			"%d to ibm,remove-pe-dma-window(%x) %llx\n",
743
			np->full_name, ret, ddw_avail[2], liobn);
744 745 746
	else
		pr_debug("%s: successfully removed direct window: rtas returned "
			"%d to ibm,remove-pe-dma-window(%x) %llx\n",
747
			np->full_name, ret, ddw_avail[2], liobn);
748

749
delprop:
750
	ret = prom_remove_property(np, win64);
751
	if (ret)
752
		pr_warning("%s: failed to remove direct window property: %d\n",
753 754
			np->full_name, ret);
}
755

756
static u64 find_existing_ddw(struct device_node *pdn)
757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775
{
	struct direct_window *window;
	const struct dynamic_dma_window_prop *direct64;
	u64 dma_addr = 0;

	spin_lock(&direct_window_list_lock);
	/* check if we already created a window and dupe that config if so */
	list_for_each_entry(window, &direct_window_list, list) {
		if (window->device == pdn) {
			direct64 = window->prop;
			dma_addr = direct64->dma_base;
			break;
		}
	}
	spin_unlock(&direct_window_list_lock);

	return dma_addr;
}

776
static int find_existing_ddw_windows(void)
777 778
{
	int len;
779
	struct device_node *pdn;
780 781 782
	struct direct_window *window;
	const struct dynamic_dma_window_prop *direct64;

783 784 785 786 787 788 789 790 791 792 793
	if (!firmware_has_feature(FW_FEATURE_LPAR))
		return 0;

	for_each_node_with_property(pdn, DIRECT64_PROPNAME) {
		direct64 = of_get_property(pdn, DIRECT64_PROPNAME, &len);
		if (!direct64)
			continue;

		window = kzalloc(sizeof(*window), GFP_KERNEL);
		if (!window || len < sizeof(struct dynamic_dma_window_prop)) {
			kfree(window);
794
			remove_ddw(pdn);
795
			continue;
796
		}
797 798 799 800 801 802

		window->device = pdn;
		window->prop = direct64;
		spin_lock(&direct_window_list_lock);
		list_add(&window->list, &direct_window_list);
		spin_unlock(&direct_window_list_lock);
803 804
	}

805
	return 0;
806
}
807
machine_arch_initcall(pseries, find_existing_ddw_windows);
808

809
static int query_ddw(struct pci_dev *dev, const u32 *ddw_avail,
810 811
			struct ddw_query_response *query)
{
812
	struct eeh_dev *edev;
813 814 815 816 817 818 819 820 821 822
	u32 cfg_addr;
	u64 buid;
	int ret;

	/*
	 * Get the config address and phb buid of the PE window.
	 * Rely on eeh to retrieve this for us.
	 * Retrieve them from the pci device, not the node with the
	 * dma-window property
	 */
823 824 825 826 827 828
	edev = pci_dev_to_eeh_dev(dev);
	cfg_addr = edev->config_addr;
	if (edev->pe_config_addr)
		cfg_addr = edev->pe_config_addr;
	buid = edev->phb->buid;

829
	ret = rtas_call(ddw_avail[0], 3, 5, (u32 *)query,
830 831
		  cfg_addr, BUID_HI(buid), BUID_LO(buid));
	dev_info(&dev->dev, "ibm,query-pe-dma-windows(%x) %x %x %x"
832
		" returned %d\n", ddw_avail[0], cfg_addr, BUID_HI(buid),
833 834 835 836
		BUID_LO(buid), ret);
	return ret;
}

837
static int create_ddw(struct pci_dev *dev, const u32 *ddw_avail,
838 839 840
			struct ddw_create_response *create, int page_shift,
			int window_shift)
{
841
	struct eeh_dev *edev;
842 843 844 845 846 847 848 849 850 851
	u32 cfg_addr;
	u64 buid;
	int ret;

	/*
	 * Get the config address and phb buid of the PE window.
	 * Rely on eeh to retrieve this for us.
	 * Retrieve them from the pci device, not the node with the
	 * dma-window property
	 */
852 853 854 855 856
	edev = pci_dev_to_eeh_dev(dev);
	cfg_addr = edev->config_addr;
	if (edev->pe_config_addr)
		cfg_addr = edev->pe_config_addr;
	buid = edev->phb->buid;
857 858 859

	do {
		/* extra outputs are LIOBN and dma-addr (hi, lo) */
860
		ret = rtas_call(ddw_avail[1], 5, 4, (u32 *)create, cfg_addr,
861 862 863 864
				BUID_HI(buid), BUID_LO(buid), page_shift, window_shift);
	} while (rtas_busy_delay(ret));
	dev_info(&dev->dev,
		"ibm,create-pe-dma-window(%x) %x %x %x %x %x returned %d "
865
		"(liobn = 0x%x starting addr = %x %x)\n", ddw_avail[1],
866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890
		 cfg_addr, BUID_HI(buid), BUID_LO(buid), page_shift,
		 window_shift, ret, create->liobn, create->addr_hi, create->addr_lo);

	return ret;
}

/*
 * If the PE supports dynamic dma windows, and there is space for a table
 * that can map all pages in a linear offset, then setup such a table,
 * and record the dma-offset in the struct device.
 *
 * dev: the pci device we are checking
 * pdn: the parent pe node with the ibm,dma_window property
 * Future: also check if we can remap the base window for our base page size
 *
 * returns the dma offset for use by dma_set_mask
 */
static u64 enable_ddw(struct pci_dev *dev, struct device_node *pdn)
{
	int len, ret;
	struct ddw_query_response query;
	struct ddw_create_response create;
	int page_shift;
	u64 dma_addr, max_addr;
	struct device_node *dn;
891
	const u32 *uninitialized_var(ddw_avail);
892
	struct direct_window *window;
893
	struct property *win64;
894 895 896 897
	struct dynamic_dma_window_prop *ddwprop;

	mutex_lock(&direct_window_init_mutex);

898
	dma_addr = find_existing_ddw(pdn);
899 900 901 902 903 904 905 906 907 908 909
	if (dma_addr != 0)
		goto out_unlock;

	/*
	 * the ibm,ddw-applicable property holds the tokens for:
	 * ibm,query-pe-dma-window
	 * ibm,create-pe-dma-window
	 * ibm,remove-pe-dma-window
	 * for the given node in that order.
	 * the property is actually in the parent, not the PE
	 */
910 911
	ddw_avail = of_get_property(pdn, "ibm,ddw-applicable", &len);
	if (!ddw_avail || len < 3 * sizeof(u32))
912 913 914 915 916 917 918 919 920
		goto out_unlock;

       /*
	 * Query if there is a second window of size to map the
	 * whole partition.  Query returns number of windows, largest
	 * block assigned to PE (partition endpoint), and two bitmasks
	 * of page sizes: supported and supported for migrate-dma.
	 */
	dn = pci_device_to_OF_node(dev);
921
	ret = query_ddw(dev, ddw_avail, &query);
922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962
	if (ret != 0)
		goto out_unlock;

	if (query.windows_available == 0) {
		/*
		 * no additional windows are available for this device.
		 * We might be able to reallocate the existing window,
		 * trading in for a larger page size.
		 */
		dev_dbg(&dev->dev, "no free dynamic windows");
		goto out_unlock;
	}
	if (query.page_size & 4) {
		page_shift = 24; /* 16MB */
	} else if (query.page_size & 2) {
		page_shift = 16; /* 64kB */
	} else if (query.page_size & 1) {
		page_shift = 12; /* 4kB */
	} else {
		dev_dbg(&dev->dev, "no supported direct page size in mask %x",
			  query.page_size);
		goto out_unlock;
	}
	/* verify the window * number of ptes will map the partition */
	/* check largest block * page size > max memory hotplug addr */
	max_addr = memory_hotplug_max();
	if (query.largest_available_block < (max_addr >> page_shift)) {
		dev_dbg(&dev->dev, "can't map partiton max 0x%llx with %u "
			  "%llu-sized pages\n", max_addr,  query.largest_available_block,
			  1ULL << page_shift);
		goto out_unlock;
	}
	len = order_base_2(max_addr);
	win64 = kzalloc(sizeof(struct property), GFP_KERNEL);
	if (!win64) {
		dev_info(&dev->dev,
			"couldn't allocate property for 64bit dma window\n");
		goto out_unlock;
	}
	win64->name = kstrdup(DIRECT64_PROPNAME, GFP_KERNEL);
	win64->value = ddwprop = kmalloc(sizeof(*ddwprop), GFP_KERNEL);
963
	win64->length = sizeof(*ddwprop);
964 965 966 967 968 969
	if (!win64->name || !win64->value) {
		dev_info(&dev->dev,
			"couldn't allocate property name and value\n");
		goto out_free_prop;
	}

970
	ret = create_ddw(dev, ddw_avail, &create, page_shift, len);
971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990
	if (ret != 0)
		goto out_free_prop;

	ddwprop->liobn = cpu_to_be32(create.liobn);
	ddwprop->dma_base = cpu_to_be64(of_read_number(&create.addr_hi, 2));
	ddwprop->tce_shift = cpu_to_be32(page_shift);
	ddwprop->window_shift = cpu_to_be32(len);

	dev_dbg(&dev->dev, "created tce table LIOBN 0x%x for %s\n",
		  create.liobn, dn->full_name);

	window = kzalloc(sizeof(*window), GFP_KERNEL);
	if (!window)
		goto out_clear_window;

	ret = walk_system_ram_range(0, memblock_end_of_DRAM() >> PAGE_SHIFT,
			win64->value, tce_setrange_multi_pSeriesLP_walk);
	if (ret) {
		dev_info(&dev->dev, "failed to map direct window for %s: %d\n",
			 dn->full_name, ret);
991
		goto out_free_window;
992 993 994 995 996 997
	}

	ret = prom_add_property(pdn, win64);
	if (ret) {
		dev_err(&dev->dev, "unable to add dma window property for %s: %d",
			 pdn->full_name, ret);
998
		goto out_free_window;
999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009
	}

	window->device = pdn;
	window->prop = ddwprop;
	spin_lock(&direct_window_list_lock);
	list_add(&window->list, &direct_window_list);
	spin_unlock(&direct_window_list_lock);

	dma_addr = of_read_number(&create.addr_hi, 2);
	goto out_unlock;

1010 1011 1012
out_free_window:
	kfree(window);

1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025
out_clear_window:
	remove_ddw(pdn);

out_free_prop:
	kfree(win64->name);
	kfree(win64->value);
	kfree(win64);

out_unlock:
	mutex_unlock(&direct_window_init_mutex);
	return dma_addr;
}

1026
static void pci_dma_dev_setup_pSeriesLP(struct pci_dev *dev)
Linus Torvalds's avatar
Linus Torvalds committed
1027 1028 1029
{
	struct device_node *pdn, *dn;
	struct iommu_table *tbl;
1030
	const void *dma_window = NULL;
1031
	struct pci_dn *pci;
Linus Torvalds's avatar
Linus Torvalds committed
1032

1033
	pr_debug("pci_dma_dev_setup_pSeriesLP: %s\n", pci_name(dev));
1034

Linus Torvalds's avatar
Linus Torvalds committed
1035
	/* dev setup for LPAR is a little tricky, since the device tree might
Lucas De Marchi's avatar
Lucas De Marchi committed
1036
	 * contain the dma-window properties per-device and not necessarily
Linus Torvalds's avatar
Linus Torvalds committed
1037 1038 1039 1040 1041
	 * for the bus. So we need to search upwards in the tree until we
	 * either hit a dma-window property, OR find a parent with a table
	 * already allocated.
	 */
	dn = pci_device_to_OF_node(dev);
1042
	pr_debug("  node is %s\n", dn->full_name);
1043

1044
	for (pdn = dn; pdn && PCI_DN(pdn) && !PCI_DN(pdn)->iommu_table;
1045
	     pdn = pdn->parent) {
1046
		dma_window = of_get_property(pdn, "ibm,dma-window", NULL);
Linus Torvalds's avatar
Linus Torvalds committed
1047 1048 1049 1050
		if (dma_window)
			break;
	}

1051 1052 1053 1054 1055 1056
	if (!pdn || !PCI_DN(pdn)) {
		printk(KERN_WARNING "pci_dma_dev_setup_pSeriesLP: "
		       "no DMA window found for pci dev=%s dn=%s\n",
				 pci_name(dev), dn? dn->full_name : "<null>");
		return;
	}
1057
	pr_debug("  parent is %s\n", pdn->full_name);
1058

1059
	pci = PCI_DN(pdn);
1060
	if (!pci->iommu_table) {
1061
		tbl = kzalloc_node(sizeof(struct iommu_table), GFP_KERNEL,
1062
				   pci->phb->node);
1063
		iommu_table_setparms_lpar(pci->phb, pdn, tbl, dma_window);
1064
		pci->iommu_table = iommu_init_table(tbl, pci->phb->node);
1065
		pr_debug("  created table: %p\n", pci->iommu_table);
1066
	} else {
1067
		pr_debug("  found DMA window, table: %p\n", pci->iommu_table);
Linus Torvalds's avatar
Linus Torvalds committed
1068 1069
	}

1070
	set_iommu_table_base(&dev->dev, pci->iommu_table);
Linus Torvalds's avatar
Linus Torvalds committed
1071
}
1072 1073 1074 1075 1076 1077 1078 1079 1080

static int dma_set_mask_pSeriesLP(struct device *dev, u64 dma_mask)
{
	bool ddw_enabled = false;
	struct device_node *pdn, *dn;
	struct pci_dev *pdev;
	const void *dma_window = NULL;
	u64 dma_offset;

1081
	if (!dev->dma_mask)
1082 1083
		return -EIO;

1084 1085 1086
	if (!dev_is_pci(dev))
		goto check_mask;

1087 1088
	pdev = to_pci_dev(dev);

1089 1090 1091 1092 1093 1094 1095
	/* only attempt to use a new window if 64-bit DMA is requested */
	if (!disable_ddw && dma_mask == DMA_BIT_MASK(64)) {
		dn = pci_device_to_OF_node(pdev);
		dev_dbg(dev, "node is %s\n", dn->full_name);

		/*
		 * the device tree might contain the dma-window properties
Lucas De Marchi's avatar
Lucas De Marchi committed
1096
		 * per-device and not necessarily for the bus. So we need to
1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116
		 * search upwards in the tree until we either hit a dma-window
		 * property, OR find a parent with a table already allocated.
		 */
		for (pdn = dn; pdn && PCI_DN(pdn) && !PCI_DN(pdn)->iommu_table;
				pdn = pdn->parent) {
			dma_window = of_get_property(pdn, "ibm,dma-window", NULL);
			if (dma_window)
				break;
		}
		if (pdn && PCI_DN(pdn)) {
			dma_offset = enable_ddw(pdev, pdn);
			if (dma_offset != 0) {
				dev_info(dev, "Using 64-bit direct DMA at offset %llx\n", dma_offset);
				set_dma_offset(dev, dma_offset);
				set_dma_ops(dev, &dma_direct_ops);
				ddw_enabled = true;
			}
		}
	}

1117 1118 1119
	/* fall back on iommu ops, restore table pointer with ops */
	if (!ddw_enabled && get_dma_ops(dev) != &dma_iommu_ops) {
		dev_info(dev, "Restoring 32-bit DMA via iommu\n");
1120
		set_dma_ops(dev, &dma_iommu_ops);
1121
		pci_dma_dev_setup_pSeriesLP(pdev);
1122 1123
	}

1124 1125 1126 1127
check_mask:
	if (!dma_supported(dev, dma_mask))
		return -EIO;

1128 1129 1130 1131
	*dev->dma_mask = dma_mask;
	return 0;
}

1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153
static u64 dma_get_required_mask_pSeriesLP(struct device *dev)
{
	if (!dev->dma_mask)
		return 0;

	if (!disable_ddw && dev_is_pci(dev)) {
		struct pci_dev *pdev = to_pci_dev(dev);
		struct device_node *dn;

		dn = pci_device_to_OF_node(pdev);

		/* search upwards for ibm,dma-window */
		for (; dn && PCI_DN(dn) && !PCI_DN(dn)->iommu_table;
				dn = dn->parent)
			if (of_get_property(dn, "ibm,dma-window", NULL))
				break;
		/* if there is a ibm,ddw-applicable property require 64 bits */
		if (dn && PCI_DN(dn) &&
				of_get_property(dn, "ibm,ddw-applicable", NULL))
			return DMA_BIT_MASK(64);
	}

1154
	return dma_iommu_ops.get_required_mask(dev);
1155 1156
}

1157 1158 1159 1160 1161
#else  /* CONFIG_PCI */
#define pci_dma_bus_setup_pSeries	NULL
#define pci_dma_dev_setup_pSeries	NULL
#define pci_dma_bus_setup_pSeriesLP	NULL
#define pci_dma_dev_setup_pSeriesLP	NULL
1162
#define dma_set_mask_pSeriesLP		NULL
1163
#define dma_get_required_mask_pSeriesLP	NULL
1164 1165
#endif /* !CONFIG_PCI */

1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205
static int iommu_mem_notifier(struct notifier_block *nb, unsigned long action,
		void *data)
{
	struct direct_window *window;
	struct memory_notify *arg = data;
	int ret = 0;

	switch (action) {
	case MEM_GOING_ONLINE:
		spin_lock(&direct_window_list_lock);
		list_for_each_entry(window, &direct_window_list, list) {
			ret |= tce_setrange_multi_pSeriesLP(arg->start_pfn,
					arg->nr_pages, window->prop);
			/* XXX log error */
		}
		spin_unlock(&direct_window_list_lock);
		break;
	case MEM_CANCEL_ONLINE:
	case MEM_OFFLINE:
		spin_lock(&direct_window_list_lock);
		list_for_each_entry(window, &direct_window_list, list) {
			ret |= tce_clearrange_multi_pSeriesLP(arg->start_pfn,
					arg->nr_pages, window->prop);
			/* XXX log error */
		}
		spin_unlock(&direct_window_list_lock);
		break;
	default:
		break;
	}
	if (ret && action != MEM_CANCEL_ONLINE)
		return NOTIFY_BAD;

	return NOTIFY_OK;
}

static struct notifier_block iommu_mem_nb = {
	.notifier_call = iommu_mem_notifier,
};

1206 1207 1208 1209 1210
static int iommu_reconfig_notifier(struct notifier_block *nb, unsigned long action, void *node)
{
	int err = NOTIFY_OK;
	struct device_node *np = node;
	struct pci_dn *pci = PCI_DN(np);
1211
	struct direct_window *window;
1212 1213 1214

	switch (action) {
	case PSERIES_RECONFIG_REMOVE:
1215
		if (pci && pci->iommu_table)
1216
			iommu_free_table(pci->iommu_table, np->full_name);
1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236

		spin_lock(&direct_window_list_lock);
		list_for_each_entry(window, &direct_window_list, list) {
			if (window->device == np) {
				list_del(&window->list);
				kfree(window);
				break;
			}
		}
		spin_unlock(&direct_window_list_lock);

		/*
		 * Because the notifier runs after isolation of the
		 * slot, we are guaranteed any DMA window has already
		 * been revoked and the TCEs have been marked invalid,
		 * so we don't need a call to remove_ddw(np). However,
		 * if an additional notifier action is added before the
		 * isolate call, we should update this code for
		 * completeness with such a call.
		 */
1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247
		break;
	default:
		err = NOTIFY_DONE;
		break;
	}
	return err;
}

static struct notifier_block iommu_reconfig_nb = {
	.notifier_call = iommu_reconfig_notifier,
};
Linus Torvalds's avatar
Linus Torvalds committed
1248 1249 1250 1251

/* These are called very early. */
void iommu_init_early_pSeries(void)
{
1252
	if (of_chosen && of_get_property(of_chosen, "linux,iommu-off", NULL))
Linus Torvalds's avatar
Linus Torvalds committed
1253 1254
		return;

1255
	if (firmware_has_feature(FW_FEATURE_LPAR)) {
1256
		if (firmware_has_feature(FW_FEATURE_MULTITCE)) {
Linus Torvalds's avatar
Linus Torvalds committed
1257 1258 1259 1260 1261 1262
			ppc_md.tce_build = tce_buildmulti_pSeriesLP;
			ppc_md.tce_free	 = tce_freemulti_pSeriesLP;
		} else {
			ppc_md.tce_build = tce_build_pSeriesLP;
			ppc_md.tce_free	 = tce_free_pSeriesLP;
		}
1263
		ppc_md.tce_get   = tce_get_pSeriesLP;
1264 1265
		ppc_md.pci_dma_bus_setup = pci_dma_bus_setup_pSeriesLP;
		ppc_md.pci_dma_dev_setup = pci_dma_dev_setup_pSeriesLP;
1266
		ppc_md.dma_set_mask = dma_set_mask_pSeriesLP;
1267
		ppc_md.dma_get_required_mask = dma_get_required_mask_pSeriesLP;
Linus Torvalds's avatar
Linus Torvalds committed
1268 1269 1270
	} else {
		ppc_md.tce_build = tce_build_pSeries;
		ppc_md.tce_free  = tce_free_pSeries;
1271
		ppc_md.tce_get   = tce_get_pseries;
1272 1273
		ppc_md.pci_dma_bus_setup = pci_dma_bus_setup_pSeries;
		ppc_md.pci_dma_dev_setup = pci_dma_dev_setup_pSeries;
Linus Torvalds's avatar
Linus Torvalds committed
1274 1275 1276 1277
	}


	pSeries_reconfig_notifier_register(&iommu_reconfig_nb);
1278
	register_memory_notifier(&iommu_mem_nb);
Linus Torvalds's avatar
Linus Torvalds committed
1279

1280
	set_pci_dma_ops(&dma_iommu_ops);
Linus Torvalds's avatar
Linus Torvalds committed
1281 1282
}

1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296
static int __init disable_multitce(char *str)
{
	if (strcmp(str, "off") == 0 &&
	    firmware_has_feature(FW_FEATURE_LPAR) &&
	    firmware_has_feature(FW_FEATURE_MULTITCE)) {
		printk(KERN_INFO "Disabling MULTITCE firmware feature\n");
		ppc_md.tce_build = tce_build_pSeriesLP;
		ppc_md.tce_free	 = tce_free_pSeriesLP;
		powerpc_firmware_features &= ~FW_FEATURE_MULTITCE;
	}
	return 1;
}

__setup("multitce=", disable_multitce);