Commit 95001ee9 authored by Linus Torvalds's avatar Linus Torvalds

Merge master.kernel.org:/pub/scm/linux/kernel/git/davem/sparc-2.6

parents 63906e41 0dc46106
......@@ -33,6 +33,14 @@ config DEBUG_BOOTMEM
depends on DEBUG_KERNEL
bool "Debug BOOTMEM initialization"
config DEBUG_PAGEALLOC
bool "Page alloc debugging"
depends on DEBUG_KERNEL && !SOFTWARE_SUSPEND
help
Unmap pages from the kernel linear mapping after free_pages().
This results in a large slowdown, but helps to find certain types
of memory corruptions.
config MCOUNT
bool
depends on STACK_DEBUG
......
......@@ -135,6 +135,28 @@ void __init device_scan(void)
cpu_data(0).clock_tick = prom_getintdefault(cpu_node,
"clock-frequency",
0);
cpu_data(0).dcache_size = prom_getintdefault(cpu_node,
"dcache-size",
16 * 1024);
cpu_data(0).dcache_line_size =
prom_getintdefault(cpu_node, "dcache-line-size", 32);
cpu_data(0).icache_size = prom_getintdefault(cpu_node,
"icache-size",
16 * 1024);
cpu_data(0).icache_line_size =
prom_getintdefault(cpu_node, "icache-line-size", 32);
cpu_data(0).ecache_size = prom_getintdefault(cpu_node,
"ecache-size",
4 * 1024 * 1024);
cpu_data(0).ecache_line_size =
prom_getintdefault(cpu_node, "ecache-line-size", 64);
printk("CPU[0]: Caches "
"D[sz(%d):line_sz(%d)] "
"I[sz(%d):line_sz(%d)] "
"E[sz(%d):line_sz(%d)]\n",
cpu_data(0).dcache_size, cpu_data(0).dcache_line_size,
cpu_data(0).icache_size, cpu_data(0).icache_line_size,
cpu_data(0).ecache_size, cpu_data(0).ecache_line_size);
}
#endif
......
......@@ -9,17 +9,7 @@
#include <asm/pgtable.h>
#include <asm/mmu.h>
#if PAGE_SHIFT == 13
#define SZ_BITS _PAGE_SZ8K
#elif PAGE_SHIFT == 16
#define SZ_BITS _PAGE_SZ64K
#elif PAGE_SHIFT == 19
#define SZ_BITS _PAGE_SZ512K
#elif PAGE_SHIFT == 22
#define SZ_BITS _PAGE_SZ4MB
#endif
#define VALID_SZ_BITS (_PAGE_VALID | SZ_BITS)
#define VALID_SZ_BITS (_PAGE_VALID | _PAGE_SZBITS)
#define VPTE_BITS (_PAGE_CP | _PAGE_CV | _PAGE_P )
#define VPTE_SHIFT (PAGE_SHIFT - 3)
......@@ -163,7 +153,6 @@ sparc64_vpte_continue:
stxa %g4, [%g1 + %g1] ASI_DMMU ! Restore previous TAG_ACCESS
retry ! Load PTE once again
#undef SZ_BITS
#undef VALID_SZ_BITS
#undef VPTE_SHIFT
#undef VPTE_BITS
......
......@@ -71,7 +71,7 @@
from_tl1_trap:
rdpr %tl, %g5 ! For TL==3 test
CREATE_VPTE_OFFSET1(%g4, %g6) ! Create VPTE offset
be,pn %xcc, 3f ! Yep, special processing
be,pn %xcc, kvmap ! Yep, special processing
CREATE_VPTE_OFFSET2(%g4, %g6) ! Create VPTE offset
cmp %g5, 4 ! Last trap level?
be,pn %xcc, longpath ! Yep, cannot risk VPTE miss
......@@ -83,9 +83,9 @@ from_tl1_trap:
nop ! Delay-slot
9: stxa %g5, [%g0] ASI_DTLB_DATA_IN ! Reload TLB
retry ! Trap return
3: brlz,pt %g4, 9b ! Kernel virtual map?
xor %g2, %g4, %g5 ! Finish bit twiddles
ba,a,pt %xcc, kvmap ! Yep, go check for obp/vmalloc
nop
nop
nop
/* DTLB ** ICACHE line 3: winfixups+real_faults */
longpath:
......
......@@ -30,159 +30,6 @@
.text
.align 32
.globl sparc64_vpte_patchme1
.globl sparc64_vpte_patchme2
/*
* On a second level vpte miss, check whether the original fault is to the OBP
* range (note that this is only possible for instruction miss, data misses to
* obp range do not use vpte). If so, go back directly to the faulting address.
* This is because we want to read the tpc, otherwise we have no way of knowing
* the 8k aligned faulting address if we are using >8k kernel pagesize. This
* also ensures no vpte range addresses are dropped into tlb while obp is
* executing (see inherit_locked_prom_mappings() rant).
*/
sparc64_vpte_nucleus:
/* Note that kvmap below has verified that the address is
* in the range MODULES_VADDR --> VMALLOC_END already. So
* here we need only check if it is an OBP address or not.
*/
sethi %hi(LOW_OBP_ADDRESS), %g5
cmp %g4, %g5
blu,pn %xcc, sparc64_vpte_patchme1
mov 0x1, %g5
sllx %g5, 32, %g5
cmp %g4, %g5
blu,pn %xcc, obp_iaddr_patch
nop
/* These two instructions are patched by paginig_init(). */
sparc64_vpte_patchme1:
sethi %hi(0), %g5
sparc64_vpte_patchme2:
or %g5, %lo(0), %g5
/* With kernel PGD in %g5, branch back into dtlb_backend. */
ba,pt %xcc, sparc64_kpte_continue
andn %g1, 0x3, %g1 /* Finish PMD offset adjustment. */
vpte_noent:
/* Restore previous TAG_ACCESS, %g5 is zero, and we will
* skip over the trap instruction so that the top level
* TLB miss handler will thing this %g5 value is just an
* invalid PTE, thus branching to full fault processing.
*/
mov TLB_SFSR, %g1
stxa %g4, [%g1 + %g1] ASI_DMMU
done
.globl obp_iaddr_patch
obp_iaddr_patch:
/* These two instructions patched by inherit_prom_mappings(). */
sethi %hi(0), %g5
or %g5, %lo(0), %g5
/* Behave as if we are at TL0. */
wrpr %g0, 1, %tl
rdpr %tpc, %g4 /* Find original faulting iaddr */
srlx %g4, 13, %g4 /* Throw out context bits */
sllx %g4, 13, %g4 /* g4 has vpn + ctx0 now */
/* Restore previous TAG_ACCESS. */
mov TLB_SFSR, %g1
stxa %g4, [%g1 + %g1] ASI_IMMU
/* Get PMD offset. */
srlx %g4, 23, %g6
and %g6, 0x7ff, %g6
sllx %g6, 2, %g6
/* Load PMD, is it valid? */
lduwa [%g5 + %g6] ASI_PHYS_USE_EC, %g5
brz,pn %g5, longpath
sllx %g5, 11, %g5
/* Get PTE offset. */
srlx %g4, 13, %g6
and %g6, 0x3ff, %g6
sllx %g6, 3, %g6
/* Load PTE. */
ldxa [%g5 + %g6] ASI_PHYS_USE_EC, %g5
brgez,pn %g5, longpath
nop
/* TLB load and return from trap. */
stxa %g5, [%g0] ASI_ITLB_DATA_IN
retry
.globl obp_daddr_patch
obp_daddr_patch:
/* These two instructions patched by inherit_prom_mappings(). */
sethi %hi(0), %g5
or %g5, %lo(0), %g5
/* Get PMD offset. */
srlx %g4, 23, %g6
and %g6, 0x7ff, %g6
sllx %g6, 2, %g6
/* Load PMD, is it valid? */
lduwa [%g5 + %g6] ASI_PHYS_USE_EC, %g5
brz,pn %g5, longpath
sllx %g5, 11, %g5
/* Get PTE offset. */
srlx %g4, 13, %g6
and %g6, 0x3ff, %g6
sllx %g6, 3, %g6
/* Load PTE. */
ldxa [%g5 + %g6] ASI_PHYS_USE_EC, %g5
brgez,pn %g5, longpath
nop
/* TLB load and return from trap. */
stxa %g5, [%g0] ASI_DTLB_DATA_IN
retry
/*
* On a first level data miss, check whether this is to the OBP range (note
* that such accesses can be made by prom, as well as by kernel using
* prom_getproperty on "address"), and if so, do not use vpte access ...
* rather, use information saved during inherit_prom_mappings() using 8k
* pagesize.
*/
.align 32
kvmap:
sethi %hi(MODULES_VADDR), %g5
cmp %g4, %g5
blu,pn %xcc, longpath
mov (VMALLOC_END >> 24), %g5
sllx %g5, 24, %g5
cmp %g4, %g5
bgeu,pn %xcc, longpath
nop
kvmap_check_obp:
sethi %hi(LOW_OBP_ADDRESS), %g5
cmp %g4, %g5
blu,pn %xcc, kvmap_vmalloc_addr
mov 0x1, %g5
sllx %g5, 32, %g5
cmp %g4, %g5
blu,pn %xcc, obp_daddr_patch
nop
kvmap_vmalloc_addr:
/* If we get here, a vmalloc addr was accessed, load kernel VPTE. */
ldxa [%g3 + %g6] ASI_N, %g5
brgez,pn %g5, longpath
nop
/* PTE is valid, load into TLB and return from trap. */
stxa %g5, [%g0] ASI_DTLB_DATA_IN ! Reload TLB
retry
/* This is trivial with the new code... */
.globl do_fpdis
do_fpdis:
......@@ -525,14 +372,13 @@ cheetah_plus_patch_fpdis:
*
* DATA 0: [low 32-bits] Address of function to call, jmp to this
* [high 32-bits] MMU Context Argument 0, place in %g5
* DATA 1: Address Argument 1, place in %g6
* DATA 1: Address Argument 1, place in %g1
* DATA 2: Address Argument 2, place in %g7
*
* With this method we can do most of the cross-call tlb/cache
* flushing very quickly.
*
* Current CPU's IRQ worklist table is locked into %g1,
* don't touch.
* Current CPU's IRQ worklist table is locked into %g6, don't touch.
*/
.text
.align 32
......@@ -1006,13 +852,14 @@ cheetah_plus_dcpe_trap_vector:
nop
do_cheetah_plus_data_parity:
ba,pt %xcc, etrap
rdpr %pil, %g2
wrpr %g0, 15, %pil
ba,pt %xcc, etrap_irq
rd %pc, %g7
mov 0x0, %o0
call cheetah_plus_parity_error
add %sp, PTREGS_OFF, %o1
ba,pt %xcc, rtrap
clr %l6
ba,a,pt %xcc, rtrap_irq
cheetah_plus_dcpe_trap_vector_tl1:
membar #Sync
......@@ -1036,13 +883,14 @@ cheetah_plus_icpe_trap_vector:
nop
do_cheetah_plus_insn_parity:
ba,pt %xcc, etrap
rdpr %pil, %g2
wrpr %g0, 15, %pil
ba,pt %xcc, etrap_irq
rd %pc, %g7
mov 0x1, %o0
call cheetah_plus_parity_error
add %sp, PTREGS_OFF, %o1
ba,pt %xcc, rtrap
clr %l6
ba,a,pt %xcc, rtrap_irq
cheetah_plus_icpe_trap_vector_tl1:
membar #Sync
......@@ -1075,6 +923,10 @@ do_dcpe_tl1:
nop
wrpr %g1, %tl ! Restore original trap level
do_dcpe_tl1_nonfatal: /* Ok we may use interrupt globals safely. */
sethi %hi(dcache_parity_tl1_occurred), %g2
lduw [%g2 + %lo(dcache_parity_tl1_occurred)], %g1
add %g1, 1, %g1
stw %g1, [%g2 + %lo(dcache_parity_tl1_occurred)]
/* Reset D-cache parity */
sethi %hi(1 << 16), %g1 ! D-cache size
mov (1 << 5), %g2 ! D-cache line size
......@@ -1121,6 +973,10 @@ do_icpe_tl1:
nop
wrpr %g1, %tl ! Restore original trap level
do_icpe_tl1_nonfatal: /* Ok we may use interrupt globals safely. */
sethi %hi(icache_parity_tl1_occurred), %g2
lduw [%g2 + %lo(icache_parity_tl1_occurred)], %g1
add %g1, 1, %g1
stw %g1, [%g2 + %lo(icache_parity_tl1_occurred)]
/* Flush I-cache */
sethi %hi(1 << 15), %g1 ! I-cache size
mov (1 << 5), %g2 ! I-cache line size
......
......@@ -80,15 +80,165 @@ sparc_ramdisk_image64:
.xword 0
.word _end
/* We must be careful, 32-bit OpenBOOT will get confused if it
* tries to save away a register window to a 64-bit kernel
* stack address. Flush all windows, disable interrupts,
* remap if necessary, jump onto kernel trap table, then kernel
* stack, or else we die.
/* PROM cif handler code address is in %o4. */
sparc64_boot:
1: rd %pc, %g7
set 1b, %g1
cmp %g1, %g7
be,pn %xcc, sparc64_boot_after_remap
mov %o4, %l7
/* We need to remap the kernel. Use position independant
* code to remap us to KERNBASE.
*
* PROM entry point is on %o4
* SILO can invoke us with 32-bit address masking enabled,
* so make sure that's clear.
*/
sparc64_boot:
rdpr %pstate, %g1
andn %g1, PSTATE_AM, %g1
wrpr %g1, 0x0, %pstate
ba,a,pt %xcc, 1f
.globl prom_finddev_name, prom_chosen_path
.globl prom_getprop_name, prom_mmu_name
.globl prom_callmethod_name, prom_translate_name
.globl prom_map_name, prom_unmap_name, prom_mmu_ihandle_cache
.globl prom_boot_mapped_pc, prom_boot_mapping_mode
.globl prom_boot_mapping_phys_high, prom_boot_mapping_phys_low
prom_finddev_name:
.asciz "finddevice"
prom_chosen_path:
.asciz "/chosen"
prom_getprop_name:
.asciz "getprop"
prom_mmu_name:
.asciz "mmu"
prom_callmethod_name:
.asciz "call-method"
prom_translate_name:
.asciz "translate"
prom_map_name:
.asciz "map"
prom_unmap_name:
.asciz "unmap"
.align 4
prom_mmu_ihandle_cache:
.word 0
prom_boot_mapped_pc:
.word 0
prom_boot_mapping_mode:
.word 0
.align 8
prom_boot_mapping_phys_high:
.xword 0
prom_boot_mapping_phys_low:
.xword 0
1:
rd %pc, %l0
mov (1b - prom_finddev_name), %l1
mov (1b - prom_chosen_path), %l2
mov (1b - prom_boot_mapped_pc), %l3
sub %l0, %l1, %l1
sub %l0, %l2, %l2
sub %l0, %l3, %l3
stw %l0, [%l3]
sub %sp, (192 + 128), %sp
/* chosen_node = prom_finddevice("/chosen") */
stx %l1, [%sp + 2047 + 128 + 0x00] ! service, "finddevice"
mov 1, %l3
stx %l3, [%sp + 2047 + 128 + 0x08] ! num_args, 1
stx %l3, [%sp + 2047 + 128 + 0x10] ! num_rets, 1
stx %l2, [%sp + 2047 + 128 + 0x18] ! arg1, "/chosen"
stx %g0, [%sp + 2047 + 128 + 0x20] ! ret1
call %l7
add %sp, (2047 + 128), %o0 ! argument array
ldx [%sp + 2047 + 128 + 0x20], %l4 ! chosen device node
mov (1b - prom_getprop_name), %l1
mov (1b - prom_mmu_name), %l2
mov (1b - prom_mmu_ihandle_cache), %l5
sub %l0, %l1, %l1
sub %l0, %l2, %l2
sub %l0, %l5, %l5
/* prom_mmu_ihandle_cache = prom_getint(chosen_node, "mmu") */
stx %l1, [%sp + 2047 + 128 + 0x00] ! service, "getprop"
mov 4, %l3
stx %l3, [%sp + 2047 + 128 + 0x08] ! num_args, 4
mov 1, %l3
stx %l3, [%sp + 2047 + 128 + 0x10] ! num_rets, 1
stx %l4, [%sp + 2047 + 128 + 0x18] ! arg1, chosen_node
stx %l2, [%sp + 2047 + 128 + 0x20] ! arg2, "mmu"
stx %l5, [%sp + 2047 + 128 + 0x28] ! arg3, &prom_mmu_ihandle_cache
mov 4, %l3
stx %l3, [%sp + 2047 + 128 + 0x30] ! arg4, sizeof(arg3)
stx %g0, [%sp + 2047 + 128 + 0x38] ! ret1
call %l7
add %sp, (2047 + 128), %o0 ! argument array
mov (1b - prom_callmethod_name), %l1
mov (1b - prom_translate_name), %l2
sub %l0, %l1, %l1
sub %l0, %l2, %l2
lduw [%l5], %l5 ! prom_mmu_ihandle_cache
stx %l1, [%sp + 2047 + 128 + 0x00] ! service, "call-method"
mov 3, %l3
stx %l3, [%sp + 2047 + 128 + 0x08] ! num_args, 3
mov 5, %l3
stx %l3, [%sp + 2047 + 128 + 0x10] ! num_rets, 5
stx %l2, [%sp + 2047 + 128 + 0x18] ! arg1: "translate"
stx %l5, [%sp + 2047 + 128 + 0x20] ! arg2: prom_mmu_ihandle_cache
srlx %l0, 22, %l3
sllx %l3, 22, %l3
stx %l3, [%sp + 2047 + 128 + 0x28] ! arg3: vaddr, our PC
stx %g0, [%sp + 2047 + 128 + 0x30] ! res1
stx %g0, [%sp + 2047 + 128 + 0x38] ! res2
stx %g0, [%sp + 2047 + 128 + 0x40] ! res3
stx %g0, [%sp + 2047 + 128 + 0x48] ! res4
stx %g0, [%sp + 2047 + 128 + 0x50] ! res5
call %l7
add %sp, (2047 + 128), %o0 ! argument array
ldx [%sp + 2047 + 128 + 0x40], %l1 ! translation mode
mov (1b - prom_boot_mapping_mode), %l4
sub %l0, %l4, %l4
stw %l1, [%l4]
mov (1b - prom_boot_mapping_phys_high), %l4
sub %l0, %l4, %l4
ldx [%sp + 2047 + 128 + 0x48], %l2 ! physaddr high
stx %l2, [%l4 + 0x0]
ldx [%sp + 2047 + 128 + 0x50], %l3 ! physaddr low
stx %l3, [%l4 + 0x8]
/* Leave service as-is, "call-method" */
mov 7, %l3
stx %l3, [%sp + 2047 + 128 + 0x08] ! num_args, 7
mov 1, %l3
stx %l3, [%sp + 2047 + 128 + 0x10] ! num_rets, 1
mov (1b - prom_map_name), %l3
sub %l0, %l3, %l3
stx %l3, [%sp + 2047 + 128 + 0x18] ! arg1: "map"
/* Leave arg2 as-is, prom_mmu_ihandle_cache */
mov -1, %l3
stx %l3, [%sp + 2047 + 128 + 0x28] ! arg3: mode (-1 default)
sethi %hi(8 * 1024 * 1024), %l3
stx %l3, [%sp + 2047 + 128 + 0x30] ! arg4: size (8MB)
sethi %hi(KERNBASE), %l3
stx %l3, [%sp + 2047 + 128 + 0x38] ! arg5: vaddr (KERNBASE)
stx %g0, [%sp + 2047 + 128 + 0x40] ! arg6: empty
mov (1b - prom_boot_mapping_phys_low), %l3
sub %l0, %l3, %l3
ldx [%l3], %l3
stx %l3, [%sp + 2047 + 128 + 0x48] ! arg7: phys addr
call %l7
add %sp, (2047 + 128), %o0 ! argument array
add %sp, (192 + 128), %sp
sparc64_boot_after_remap:
BRANCH_IF_CHEETAH_BASE(g1,g7,cheetah_boot)
BRANCH_IF_CHEETAH_PLUS_OR_FOLLOWON(g1,g7,cheetah_plus_boot)
ba,pt %xcc, spitfire_boot
......@@ -125,185 +275,7 @@ cheetah_generic_boot:
stxa %g0, [%g3] ASI_IMMU
membar #Sync
wrpr %g0, (PSTATE_PRIV|PSTATE_PEF|PSTATE_IE), %pstate
wr %g0, 0, %fprs
/* Just like for Spitfire, we probe itlb-2 for a mapping which
* matches our current %pc. We take the physical address in
* that mapping and use it to make our own.
*/
/* %g5 holds the tlb data */
sethi %uhi(_PAGE_VALID | _PAGE_SZ4MB), %g5
sllx %g5, 32, %g5
or %g5, (_PAGE_CP | _PAGE_CV | _PAGE_P | _PAGE_L | _PAGE_W | _PAGE_G), %g5
/* Put PADDR tlb data mask into %g3. */
sethi %uhi(_PAGE_PADDR), %g3
or %g3, %ulo(_PAGE_PADDR), %g3
sllx %g3, 32, %g3
sethi %hi(_PAGE_PADDR), %g7
or %g7, %lo(_PAGE_PADDR), %g7
or %g3, %g7, %g3
set 2 << 16, %l0 /* TLB entry walker. */
set 0x1fff, %l2 /* Page mask. */
rd %pc, %l3
andn %l3, %l2, %g2 /* vaddr comparator */
1: ldxa [%l0] ASI_ITLB_TAG_READ, %g1
membar #Sync
andn %g1, %l2, %g1
cmp %g1, %g2
be,pn %xcc, cheetah_got_tlbentry
nop
and %l0, (127 << 3), %g1
cmp %g1, (127 << 3)
blu,pt %xcc, 1b
add %l0, (1 << 3), %l0
/* Search the small TLB. OBP never maps us like that but
* newer SILO can.
*/
clr %l0
1: ldxa [%l0] ASI_ITLB_TAG_READ, %g1
membar #Sync
andn %g1, %l2, %g1
cmp %g1, %g2
be,pn %xcc, cheetah_got_tlbentry
nop
cmp %l0, (15 << 3)
blu,pt %xcc, 1b
add %l0, (1 << 3), %l0
/* BUG() if we get here... */
ta 0x5
cheetah_got_tlbentry:
ldxa [%l0] ASI_ITLB_DATA_ACCESS, %g0
ldxa [%l0] ASI_ITLB_DATA_ACCESS, %g1
membar #Sync
and %g1, %g3, %g1
set 0x5fff, %l0
andn %g1, %l0, %g1
or %g5, %g1, %g5
/* Clear out any KERNBASE area entries. */
set 2 << 16, %l0
sethi %hi(KERNBASE), %g3
sethi %hi(KERNBASE<<1), %g7
mov TLB_TAG_ACCESS, %l7
/* First, check ITLB */
1: ldxa [%l0] ASI_ITLB_TAG_READ, %g1
membar #Sync
andn %g1, %l2, %g1
cmp %g1, %g3
blu,pn %xcc, 2f
cmp %g1, %g7
bgeu,pn %xcc, 2f
nop
stxa %g0, [%l7] ASI_IMMU