Commit a9c9147e authored by Russell King's avatar Russell King
Browse files

ARM: dma-mapping: provide per-cpu type map/unmap functions


Signed-off-by: default avatarRussell King <rmk+kernel@arm.linux.org.uk>
Tested-By: default avatarSantosh Shilimkar <santosh.shilimkar@ti.com>
parent 93f1d629
......@@ -213,6 +213,9 @@ struct cpu_cache_fns {
void (*coherent_user_range)(unsigned long, unsigned long);
void (*flush_kern_dcache_area)(void *, size_t);
void (*dma_map_area)(const void *, size_t, int);
void (*dma_unmap_area)(const void *, size_t, int);
void (*dma_inv_range)(const void *, const void *);
void (*dma_clean_range)(const void *, const void *);
void (*dma_flush_range)(const void *, const void *);
......@@ -244,6 +247,8 @@ extern struct cpu_cache_fns cpu_cache;
* is visible to DMA, or data written by DMA to system memory is
* visible to the CPU.
*/
#define dmac_map_area cpu_cache.dma_map_area
#define dmac_unmap_area cpu_cache.dma_unmap_area
#define dmac_inv_range cpu_cache.dma_inv_range
#define dmac_clean_range cpu_cache.dma_clean_range
#define dmac_flush_range cpu_cache.dma_flush_range
......@@ -270,10 +275,14 @@ extern void __cpuc_flush_dcache_area(void *, size_t);
* is visible to DMA, or data written by DMA to system memory is
* visible to the CPU.
*/
#define dmac_map_area __glue(_CACHE,_dma_map_area)
#define dmac_unmap_area __glue(_CACHE,_dma_unmap_area)
#define dmac_inv_range __glue(_CACHE,_dma_inv_range)
#define dmac_clean_range __glue(_CACHE,_dma_clean_range)
#define dmac_flush_range __glue(_CACHE,_dma_flush_range)
extern void dmac_map_area(const void *, size_t, int);
extern void dmac_unmap_area(const void *, size_t, int);
extern void dmac_inv_range(const void *, const void *);
extern void dmac_clean_range(const void *, const void *);
extern void dmac_flush_range(const void *, const void *);
......
......@@ -12,6 +12,7 @@
*/
#include <linux/sched.h>
#include <linux/mm.h>
#include <linux/dma-mapping.h>
#include <asm/mach/arch.h>
#include <asm/thread_info.h>
#include <asm/memory.h>
......@@ -112,5 +113,9 @@ int main(void)
#ifdef MULTI_PABORT
DEFINE(PROCESSOR_PABT_FUNC, offsetof(struct processor, _prefetch_abort));
#endif
BLANK();
DEFINE(DMA_BIDIRECTIONAL, DMA_BIDIRECTIONAL);
DEFINE(DMA_TO_DEVICE, DMA_TO_DEVICE);
DEFINE(DMA_FROM_DEVICE, DMA_FROM_DEVICE);
return 0;
}
......@@ -205,6 +205,30 @@ ENTRY(fa_dma_flush_range)
mcr p15, 0, r0, c7, c10, 4 @ drain write buffer
mov pc, lr
/*
* dma_map_area(start, size, dir)
* - start - kernel virtual start address
* - size - size of region
* - dir - DMA direction
*/
ENTRY(fa_dma_map_area)
add r1, r1, r0
cmp r2, #DMA_TO_DEVICE
beq fa_dma_clean_range
bcs fa_dma_inv_range
b fa_dma_flush_range
ENDPROC(fa_dma_map_area)
/*
* dma_unmap_area(start, size, dir)
* - start - kernel virtual start address
* - size - size of region
* - dir - DMA direction
*/
ENTRY(fa_dma_unmap_area)
mov pc, lr
ENDPROC(fa_dma_unmap_area)
__INITDATA
.type fa_cache_fns, #object
......@@ -215,6 +239,8 @@ ENTRY(fa_cache_fns)
.long fa_coherent_kern_range
.long fa_coherent_user_range
.long fa_flush_kern_dcache_area
.long fa_dma_map_area
.long fa_dma_unmap_area
.long fa_dma_inv_range
.long fa_dma_clean_range
.long fa_dma_flush_range
......
......@@ -121,6 +121,28 @@ ENTRY(v3_dma_flush_range)
ENTRY(v3_dma_clean_range)
mov pc, lr
/*
* dma_unmap_area(start, size, dir)
* - start - kernel virtual start address
* - size - size of region
* - dir - DMA direction
*/
ENTRY(v3_dma_unmap_area)
teq r2, #DMA_TO_DEVICE
bne v3_dma_inv_range
/* FALLTHROUGH */
/*
* dma_map_area(start, size, dir)
* - start - kernel virtual start address
* - size - size of region
* - dir - DMA direction
*/
ENTRY(v3_dma_map_area)
mov pc, lr
ENDPROC(v3_dma_unmap_area)
ENDPROC(v3_dma_map_area)
__INITDATA
.type v3_cache_fns, #object
......@@ -131,6 +153,8 @@ ENTRY(v3_cache_fns)
.long v3_coherent_kern_range
.long v3_coherent_user_range
.long v3_flush_kern_dcache_area
.long v3_dma_map_area
.long v3_dma_unmap_area
.long v3_dma_inv_range
.long v3_dma_clean_range
.long v3_dma_flush_range
......
......@@ -133,6 +133,28 @@ ENTRY(v4_dma_flush_range)
ENTRY(v4_dma_clean_range)
mov pc, lr
/*
* dma_unmap_area(start, size, dir)
* - start - kernel virtual start address
* - size - size of region
* - dir - DMA direction
*/
ENTRY(v4_dma_unmap_area)
teq r2, #DMA_TO_DEVICE
bne v4_dma_inv_range
/* FALLTHROUGH */
/*
* dma_map_area(start, size, dir)
* - start - kernel virtual start address
* - size - size of region
* - dir - DMA direction
*/
ENTRY(v4_dma_map_area)
mov pc, lr
ENDPROC(v4_dma_unmap_area)
ENDPROC(v4_dma_map_area)
__INITDATA
.type v4_cache_fns, #object
......@@ -143,6 +165,8 @@ ENTRY(v4_cache_fns)
.long v4_coherent_kern_range
.long v4_coherent_user_range
.long v4_flush_kern_dcache_area
.long v4_dma_map_area
.long v4_dma_unmap_area
.long v4_dma_inv_range
.long v4_dma_clean_range
.long v4_dma_flush_range
......
......@@ -216,6 +216,30 @@ ENTRY(v4wb_dma_clean_range)
.globl v4wb_dma_flush_range
.set v4wb_dma_flush_range, v4wb_coherent_kern_range
/*
* dma_map_area(start, size, dir)
* - start - kernel virtual start address
* - size - size of region
* - dir - DMA direction
*/
ENTRY(v4wb_dma_map_area)
add r1, r1, r0
cmp r2, #DMA_TO_DEVICE
beq v4wb_dma_clean_range
bcs v4wb_dma_inv_range
b v4wb_dma_flush_range
ENDPROC(v4wb_dma_map_area)
/*
* dma_unmap_area(start, size, dir)
* - start - kernel virtual start address
* - size - size of region
* - dir - DMA direction
*/
ENTRY(v4wb_dma_unmap_area)
mov pc, lr
ENDPROC(v4wb_dma_unmap_area)
__INITDATA
.type v4wb_cache_fns, #object
......@@ -226,6 +250,8 @@ ENTRY(v4wb_cache_fns)
.long v4wb_coherent_kern_range
.long v4wb_coherent_user_range
.long v4wb_flush_kern_dcache_area
.long v4wb_dma_map_area
.long v4wb_dma_unmap_area
.long v4wb_dma_inv_range
.long v4wb_dma_clean_range
.long v4wb_dma_flush_range
......
......@@ -172,6 +172,29 @@ ENTRY(v4wt_dma_clean_range)
.globl v4wt_dma_flush_range
.equ v4wt_dma_flush_range, v4wt_dma_inv_range
/*
* dma_unmap_area(start, size, dir)
* - start - kernel virtual start address
* - size - size of region
* - dir - DMA direction
*/
ENTRY(v4wt_dma_unmap_area)
add r1, r1, r0
teq r2, #DMA_TO_DEVICE
bne v4wt_dma_inv_range
/* FALLTHROUGH */
/*
* dma_map_area(start, size, dir)
* - start - kernel virtual start address
* - size - size of region
* - dir - DMA direction
*/
ENTRY(v4wt_dma_map_area)
mov pc, lr
ENDPROC(v4wt_dma_unmap_area)
ENDPROC(v4wt_dma_map_area)
__INITDATA
.type v4wt_cache_fns, #object
......@@ -182,6 +205,8 @@ ENTRY(v4wt_cache_fns)
.long v4wt_coherent_kern_range
.long v4wt_coherent_user_range
.long v4wt_flush_kern_dcache_area
.long v4wt_dma_map_area
.long v4wt_dma_unmap_area
.long v4wt_dma_inv_range
.long v4wt_dma_clean_range
.long v4wt_dma_flush_range
......
......@@ -263,6 +263,30 @@ ENTRY(v6_dma_flush_range)
mcr p15, 0, r0, c7, c10, 4 @ drain write buffer
mov pc, lr
/*
* dma_map_area(start, size, dir)
* - start - kernel virtual start address
* - size - size of region
* - dir - DMA direction
*/
ENTRY(v6_dma_map_area)
add r1, r1, r0
cmp r2, #DMA_TO_DEVICE
beq v6_dma_clean_range
bcs v6_dma_inv_range
b v6_dma_flush_range
ENDPROC(v6_dma_map_area)
/*
* dma_unmap_area(start, size, dir)
* - start - kernel virtual start address
* - size - size of region
* - dir - DMA direction
*/
ENTRY(v6_dma_unmap_area)
mov pc, lr
ENDPROC(v6_dma_unmap_area)
__INITDATA
.type v6_cache_fns, #object
......@@ -273,6 +297,8 @@ ENTRY(v6_cache_fns)
.long v6_coherent_kern_range
.long v6_coherent_user_range
.long v6_flush_kern_dcache_area
.long v6_dma_map_area
.long v6_dma_unmap_area
.long v6_dma_inv_range
.long v6_dma_clean_range
.long v6_dma_flush_range
......
......@@ -271,6 +271,30 @@ ENTRY(v7_dma_flush_range)
mov pc, lr
ENDPROC(v7_dma_flush_range)
/*
* dma_map_area(start, size, dir)
* - start - kernel virtual start address
* - size - size of region
* - dir - DMA direction
*/
ENTRY(v7_dma_map_area)
add r1, r1, r0
cmp r2, #DMA_TO_DEVICE
beq v7_dma_clean_range
bcs v7_dma_inv_range
b v7_dma_flush_range
ENDPROC(v7_dma_map_area)
/*
* dma_unmap_area(start, size, dir)
* - start - kernel virtual start address
* - size - size of region
* - dir - DMA direction
*/
ENTRY(v7_dma_unmap_area)
mov pc, lr
ENDPROC(v7_dma_unmap_area)
__INITDATA
.type v7_cache_fns, #object
......@@ -281,6 +305,8 @@ ENTRY(v7_cache_fns)
.long v7_coherent_kern_range
.long v7_coherent_user_range
.long v7_flush_kern_dcache_area
.long v7_dma_map_area
.long v7_dma_unmap_area
.long v7_dma_inv_range
.long v7_dma_clean_range
.long v7_dma_flush_range
......
......@@ -406,35 +406,31 @@ EXPORT_SYMBOL(dma_free_coherent);
*/
static void dma_cache_maint(const void *start, size_t size, int direction)
{
void (*inner_op)(const void *, const void *);
void (*outer_op)(unsigned long, unsigned long);
BUG_ON(!virt_addr_valid(start) || !virt_addr_valid(start + size - 1));
switch (direction) {
case DMA_FROM_DEVICE: /* invalidate only */
inner_op = dmac_inv_range;
outer_op = outer_inv_range;
break;
case DMA_TO_DEVICE: /* writeback only */
inner_op = dmac_clean_range;
outer_op = outer_clean_range;
break;
case DMA_BIDIRECTIONAL: /* writeback and invalidate */
inner_op = dmac_flush_range;
outer_op = outer_flush_range;
break;
default:
BUG();
}
inner_op(start, start + size);
outer_op(__pa(start), __pa(start) + size);
}
void ___dma_single_cpu_to_dev(const void *kaddr, size_t size,
enum dma_data_direction dir)
{
BUG_ON(!virt_addr_valid(kaddr) || !virt_addr_valid(kaddr + size - 1));
dmac_map_area(kaddr, size, dir);
dma_cache_maint(kaddr, size, dir);
}
EXPORT_SYMBOL(___dma_single_cpu_to_dev);
......@@ -442,12 +438,15 @@ EXPORT_SYMBOL(___dma_single_cpu_to_dev);
void ___dma_single_dev_to_cpu(const void *kaddr, size_t size,
enum dma_data_direction dir)
{
/* nothing to do */
BUG_ON(!virt_addr_valid(kaddr) || !virt_addr_valid(kaddr + size - 1));
dmac_unmap_area(kaddr, size, dir);
}
EXPORT_SYMBOL(___dma_single_dev_to_cpu);
static void dma_cache_maint_page(struct page *page, unsigned long offset,
size_t size, void (*op)(const void *, const void *))
size_t size, enum dma_data_direction dir,
void (*op)(const void *, size_t, int))
{
/*
* A single sg entry may refer to multiple physically contiguous
......@@ -471,12 +470,12 @@ static void dma_cache_maint_page(struct page *page, unsigned long offset,
vaddr = kmap_high_get(page);
if (vaddr) {
vaddr += offset;
op(vaddr, vaddr + len);
op(vaddr, len, dir);
kunmap_high(page);
}
} else {
vaddr = page_address(page) + offset;
op(vaddr, vaddr + len);
op(vaddr, len, dir);
}
offset = 0;
page++;
......@@ -488,27 +487,23 @@ void ___dma_page_cpu_to_dev(struct page *page, unsigned long off,
size_t size, enum dma_data_direction dir)
{
unsigned long paddr;
void (*inner_op)(const void *, const void *);
void (*outer_op)(unsigned long, unsigned long);
switch (direction) {
case DMA_FROM_DEVICE: /* invalidate only */
inner_op = dmac_inv_range;
outer_op = outer_inv_range;
break;
case DMA_TO_DEVICE: /* writeback only */
inner_op = dmac_clean_range;
outer_op = outer_clean_range;
break;
case DMA_BIDIRECTIONAL: /* writeback and invalidate */
inner_op = dmac_flush_range;
outer_op = outer_flush_range;
break;
default:
BUG();
}
dma_cache_maint_page(page, off, size, inner_op);
dma_cache_maint_page(page, off, size, dir, dmac_map_area);
paddr = page_to_phys(page) + off;
outer_op(paddr, paddr + size);
......@@ -518,7 +513,7 @@ EXPORT_SYMBOL(___dma_page_cpu_to_dev);
void ___dma_page_dev_to_cpu(struct page *page, unsigned long off,
size_t size, enum dma_data_direction dir)
{
/* nothing to do */
dma_cache_maint_page(page, off, size, dir, dmac_unmap_area);
}
EXPORT_SYMBOL(___dma_page_dev_to_cpu);
......
......@@ -330,6 +330,30 @@ ENTRY(arm1020_dma_flush_range)
mcr p15, 0, ip, c7, c10, 4 @ drain WB
mov pc, lr
/*
* dma_map_area(start, size, dir)
* - start - kernel virtual start address
* - size - size of region
* - dir - DMA direction
*/
ENTRY(arm1020_dma_map_area)
add r1, r1, r0
cmp r2, #DMA_TO_DEVICE
beq arm1020_dma_clean_range
bcs arm1020_dma_inv_range
b arm1020_dma_flush_range
ENDPROC(arm1020_dma_map_area)
/*
* dma_unmap_area(start, size, dir)
* - start - kernel virtual start address
* - size - size of region
* - dir - DMA direction
*/
ENTRY(arm1020_dma_unmap_area)
mov pc, lr
ENDPROC(arm1020_dma_unmap_area)
ENTRY(arm1020_cache_fns)
.long arm1020_flush_kern_cache_all
.long arm1020_flush_user_cache_all
......@@ -337,6 +361,8 @@ ENTRY(arm1020_cache_fns)
.long arm1020_coherent_kern_range
.long arm1020_coherent_user_range
.long arm1020_flush_kern_dcache_area
.long arm1020_dma_map_area
.long arm1020_dma_unmap_area
.long arm1020_dma_inv_range
.long arm1020_dma_clean_range
.long arm1020_dma_flush_range
......
......@@ -316,6 +316,30 @@ ENTRY(arm1020e_dma_flush_range)
mcr p15, 0, ip, c7, c10, 4 @ drain WB
mov pc, lr
/*
* dma_map_area(start, size, dir)
* - start - kernel virtual start address
* - size - size of region
* - dir - DMA direction
*/
ENTRY(arm1020e_dma_map_area)
add r1, r1, r0
cmp r2, #DMA_TO_DEVICE
beq arm1020e_dma_clean_range
bcs arm1020e_dma_inv_range
b arm1020e_dma_flush_range
ENDPROC(arm1020e_dma_map_area)
/*
* dma_unmap_area(start, size, dir)
* - start - kernel virtual start address
* - size - size of region
* - dir - DMA direction
*/
ENTRY(arm1020e_dma_unmap_area)
mov pc, lr
ENDPROC(arm1020e_dma_unmap_area)
ENTRY(arm1020e_cache_fns)
.long arm1020e_flush_kern_cache_all
.long arm1020e_flush_user_cache_all
......@@ -323,6 +347,8 @@ ENTRY(arm1020e_cache_fns)
.long arm1020e_coherent_kern_range
.long arm1020e_coherent_user_range
.long arm1020e_flush_kern_dcache_area
.long arm1020e_dma_map_area
.long arm1020e_dma_unmap_area
.long arm1020e_dma_inv_range
.long arm1020e_dma_clean_range
.long arm1020e_dma_flush_range
......
......@@ -305,6 +305,30 @@ ENTRY(arm1022_dma_flush_range)
mcr p15, 0, ip, c7, c10, 4 @ drain WB
mov pc, lr
/*
* dma_map_area(start, size, dir)
* - start - kernel virtual start address
* - size - size of region
* - dir - DMA direction
*/
ENTRY(arm1022_dma_map_area)
add r1, r1, r0
cmp r2, #DMA_TO_DEVICE
beq arm1022_dma_clean_range
bcs arm1022_dma_inv_range
b arm1022_dma_flush_range
ENDPROC(arm1022_dma_map_area)
/*
* dma_unmap_area(start, size, dir)
* - start - kernel virtual start address
* - size - size of region
* - dir - DMA direction
*/
ENTRY(arm1022_dma_unmap_area)
mov pc, lr
ENDPROC(arm1022_dma_unmap_area)
ENTRY(arm1022_cache_fns)
.long arm1022_flush_kern_cache_all
.long arm1022_flush_user_cache_all
......@@ -312,6 +336,8 @@ ENTRY(arm1022_cache_fns)
.long arm1022_coherent_kern_range
.long arm1022_coherent_user_range
.long arm1022_flush_kern_dcache_area
.long arm1022_dma_map_area
.long arm1022_dma_unmap_area
.long arm1022_dma_inv_range
.long arm1022_dma_clean_range
.long arm1022_dma_flush_range
......
......@@ -299,6 +299,30 @@ ENTRY(arm1026_dma_flush_range)
mcr p15, 0, ip, c7, c10, 4 @ drain WB
mov pc, lr
/*
* dma_map_area(start, size, dir)
* - start - kernel virtual start address
* - size - size of region
* - dir - DMA direction
*/
ENTRY(arm1026_dma_map_area)
add r1, r1, r0
cmp r2, #DMA_TO_DEVICE
beq arm1026_dma_clean_range
bcs arm1026_dma_inv_range
b arm1026_dma_flush_range
ENDPROC(arm1026_dma_map_area)
/*
* dma_unmap_area(start, size, dir)
* - start - kernel virtual start address
* - size - size of region
* - dir - DMA direction
*/
ENTRY(arm1026_dma_unmap_area)
mov pc, lr
ENDPROC(arm1026_dma_unmap_area)
ENTRY(arm1026_cache_fns)
.long arm1026_flush_kern_cache_all
.long arm1026_flush_user_cache_all
......@@ -306,6 +330,8 @@ ENTRY(arm1026_cache_fns)
.long arm1026_coherent_kern_range
.long arm1026_coherent_user_range
.long arm1026_flush_kern_dcache_area
.long arm1026_dma_map_area
.long arm1026_dma_unmap_area
.long arm1026_dma_inv_range
.long arm1026_dma_clean_range
.long arm1026_dma_flush_range
......
......@@ -288,6 +288,30 @@ ENTRY(arm920_dma_flush_range)
mcr p15, 0, r0, c7, c10, 4 @ drain WB
mov pc, lr
/*
* dma_map_area(start, size, dir)
* - start - kernel virtual start address
* - size - size of region
* - dir - DMA direction
*/
ENTRY(arm920_dma_map_area)
add r1, r1, r0
cmp r2, #DMA_TO_DEVICE
beq arm920_dma_clean_range
bcs arm920_dma_inv_range
b arm920_dma_flush_range
ENDPROC(arm920_dma_map_area)
/*
* dma_unmap_area(start, size, dir)
* - start - kernel virtual start address