[deliverable/linux.git] / arch / arm / mm / cache-v7.S

/*
 *  linux/arch/arm/mm/cache-v7.S
 *
 *  Copyright (C) 2001 Deep Blue Solutions Ltd.
 *  Copyright (C) 2005 ARM Ltd.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 *  This is the "shell" of the ARMv7 processor support.
 */
#include <linux/linkage.h>
#include <linux/init.h>
#include <asm/assembler.h>
#include <asm/errno.h>
#include <asm/unwind.h>

#include "proc-macros.S"

/*
 * The secondary kernel init calls v7_flush_dcache_all before it enables
 * the L1; however, the L1 comes out of reset in an undefined state, so
 * the clean + invalidate performed by v7_flush_dcache_all causes a bunch
 * of cache lines with uninitialized data and uninitialized tags to get
 * written out to memory, which does really unpleasant things to the main
 * processor.  We fix this by performing an invalidate, rather than a
 * clean + invalidate, before jumping into the kernel.
 *
 * This function is cloned from arch/arm/mach-tegra/headsmp.S, and needs
 * to be called for both secondary cores startup and primary core resume
 * procedures.
 */
ENTRY(v7_invalidate_l1)
       mov     r0, #0
       mcr     p15, 2, r0, c0, c0, 0
       mrc     p15, 1, r0, c0, c0, 0

       movw    r1, #0x7fff
       and     r2, r1, r0, lsr #13

       movw    r1, #0x3ff

       and     r3, r1, r0, lsr #3      @ NumWays - 1
       add     r2, r2, #1              @ NumSets

       and     r0, r0, #0x7
       add     r0, r0, #4      @ SetShift

       clz     r1, r3          @ WayShift
       add     r4, r3, #1      @ NumWays
1:     sub     r2, r2, #1      @ NumSets--
       mov     r3, r4          @ Temp = NumWays
2:     subs    r3, r3, #1      @ Temp--
       mov     r5, r3, lsl r1
       mov     r6, r2, lsl r0
       orr     r5, r5, r6      @ Reg = (Temp<<WayShift)|(NumSets<<SetShift)
       mcr     p15, 0, r5, c7, c6, 2
       bgt     2b
       cmp     r2, #0
       bgt     1b
       dsb     st
       isb
       ret     lr
ENDPROC(v7_invalidate_l1)

/*
 *	v7_flush_icache_all()
 *
 *	Flush the whole I-cache.
 *
 *	Registers:
 *	r0 - set to 0
 */
ENTRY(v7_flush_icache_all)
	mov	r0, #0
	ALT_SMP(mcr	p15, 0, r0, c7, c1, 0)		@ invalidate I-cache inner shareable
	ALT_UP(mcr	p15, 0, r0, c7, c5, 0)		@ I+BTB cache invalidate
	ret	lr
ENDPROC(v7_flush_icache_all)

 /*
 *     v7_flush_dcache_louis()
 *
 *     Flush the D-cache up to the Level of Unification Inner Shareable
 *
 *     Corrupted registers: r0-r7, r9-r11 (r6 only in Thumb mode)
 */

ENTRY(v7_flush_dcache_louis)
	dmb					@ ensure ordering with previous memory accesses
	mrc	p15, 1, r0, c0, c0, 1		@ read clidr, r0 = clidr
ALT_SMP(mov	r3, r0, lsr #20)		@ move LoUIS into position
ALT_UP(	mov	r3, r0, lsr #26)		@ move LoUU into position
	ands	r3, r3, #7 << 1 		@ extract LoU*2 field from clidr
	bne	start_flush_levels		@ LoU != 0, start flushing
#ifdef CONFIG_ARM_ERRATA_643719
ALT_SMP(mrc	p15, 0, r2, c0, c0, 0)		@ read main ID register
ALT_UP(	ret	lr)				@ LoUU is zero, so nothing to do
	movw	r1, #:lower16:(0x410fc090 >> 4)	@ ID of ARM Cortex A9 r0p?
	movt	r1, #:upper16:(0x410fc090 >> 4)
	teq	r1, r2, lsr #4			@ test for errata affected core and if so...
	moveq	r3, #1 << 1			@   fix LoUIS value
	beq	start_flush_levels		@   start flushing cache levels
#endif
	ret	lr
ENDPROC(v7_flush_dcache_louis)

/*
 *	v7_flush_dcache_all()
 *
 *	Flush the whole D-cache.
 *
 *	Corrupted registers: r0-r7, r9-r11 (r6 only in Thumb mode)
 *
 *	- mm    - mm_struct describing address space
 */
ENTRY(v7_flush_dcache_all)
	dmb					@ ensure ordering with previous memory accesses
	mrc	p15, 1, r0, c0, c0, 1		@ read clidr
	mov	r3, r0, lsr #23			@ move LoC into position
	ands	r3, r3, #7 << 1			@ extract LoC*2 from clidr
	beq	finished			@ if loc is 0, then no need to clean
start_flush_levels:
	mov	r10, #0				@ start clean at cache level 0
flush_levels:
	add	r2, r10, r10, lsr #1		@ work out 3x current cache level
	mov	r1, r0, lsr r2			@ extract cache type bits from clidr
	and	r1, r1, #7			@ mask of the bits for current cache only
	cmp	r1, #2				@ see what cache we have at this level
	blt	skip				@ skip if no cache, or just i-cache
#ifdef CONFIG_PREEMPT
	save_and_disable_irqs_notrace r9	@ make cssr&csidr read atomic
#endif
	mcr	p15, 2, r10, c0, c0, 0		@ select current cache level in cssr
	isb					@ isb to sych the new cssr&csidr
	mrc	p15, 1, r1, c0, c0, 0		@ read the new csidr
#ifdef CONFIG_PREEMPT
	restore_irqs_notrace r9
#endif
	and	r2, r1, #7			@ extract the length of the cache lines
	add	r2, r2, #4			@ add 4 (line length offset)
	movw	r4, #0x3ff
	ands	r4, r4, r1, lsr #3		@ find maximum number on the way size
	clz	r5, r4				@ find bit position of way size increment
	movw	r7, #0x7fff
	ands	r7, r7, r1, lsr #13		@ extract max number of the index size
loop1:
	mov	r9, r7				@ create working copy of max index
loop2:
 ARM(	orr	r11, r10, r4, lsl r5	)	@ factor way and cache number into r11
 THUMB(	lsl	r6, r4, r5		)
 THUMB(	orr	r11, r10, r6		)	@ factor way and cache number into r11
 ARM(	orr	r11, r11, r9, lsl r2	)	@ factor index number into r11
 THUMB(	lsl	r6, r9, r2		)
 THUMB(	orr	r11, r11, r6		)	@ factor index number into r11
	mcr	p15, 0, r11, c7, c14, 2		@ clean & invalidate by set/way
	subs	r9, r9, #1			@ decrement the index
	bge	loop2
	subs	r4, r4, #1			@ decrement the way
	bge	loop1
skip:
	add	r10, r10, #2			@ increment cache number
	cmp	r3, r10
	bgt	flush_levels
finished:
	mov	r10, #0				@ swith back to cache level 0
	mcr	p15, 2, r10, c0, c0, 0		@ select current cache level in cssr
	dsb	st
	isb
	ret	lr
ENDPROC(v7_flush_dcache_all)

/*
 *	v7_flush_cache_all()
 *
 *	Flush the entire cache system.
 *  The data cache flush is now achieved using atomic clean / invalidates
 *  working outwards from L1 cache. This is done using Set/Way based cache
 *  maintenance instructions.
 *  The instruction cache can still be invalidated back to the point of
 *  unification in a single instruction.
 *
 */
ENTRY(v7_flush_kern_cache_all)
 ARM(	stmfd	sp!, {r4-r5, r7, r9-r11, lr}	)
 THUMB(	stmfd	sp!, {r4-r7, r9-r11, lr}	)
	bl	v7_flush_dcache_all
	mov	r0, #0
	ALT_SMP(mcr	p15, 0, r0, c7, c1, 0)	@ invalidate I-cache inner shareable
	ALT_UP(mcr	p15, 0, r0, c7, c5, 0)	@ I+BTB cache invalidate
 ARM(	ldmfd	sp!, {r4-r5, r7, r9-r11, lr}	)
 THUMB(	ldmfd	sp!, {r4-r7, r9-r11, lr}	)
	ret	lr
ENDPROC(v7_flush_kern_cache_all)

 /*
 *     v7_flush_kern_cache_louis(void)
 *
 *     Flush the data cache up to Level of Unification Inner Shareable.
 *     Invalidate the I-cache to the point of unification.
 */
ENTRY(v7_flush_kern_cache_louis)
 ARM(	stmfd	sp!, {r4-r5, r7, r9-r11, lr}	)
 THUMB(	stmfd	sp!, {r4-r7, r9-r11, lr}	)
	bl	v7_flush_dcache_louis
	mov	r0, #0
	ALT_SMP(mcr	p15, 0, r0, c7, c1, 0)	@ invalidate I-cache inner shareable
	ALT_UP(mcr	p15, 0, r0, c7, c5, 0)	@ I+BTB cache invalidate
 ARM(	ldmfd	sp!, {r4-r5, r7, r9-r11, lr}	)
 THUMB(	ldmfd	sp!, {r4-r7, r9-r11, lr}	)
	ret	lr
ENDPROC(v7_flush_kern_cache_louis)

/*
 *	v7_flush_cache_all()
 *
 *	Flush all TLB entries in a particular address space
 *
 *	- mm    - mm_struct describing address space
 */
ENTRY(v7_flush_user_cache_all)
	/*FALLTHROUGH*/

/*
 *	v7_flush_cache_range(start, end, flags)
 *
 *	Flush a range of TLB entries in the specified address space.
 *
 *	- start - start address (may not be aligned)
 *	- end   - end address (exclusive, may not be aligned)
 *	- flags	- vm_area_struct flags describing address space
 *
 *	It is assumed that:
 *	- we have a VIPT cache.
 */
ENTRY(v7_flush_user_cache_range)
	ret	lr
ENDPROC(v7_flush_user_cache_all)
ENDPROC(v7_flush_user_cache_range)

/*
 *	v7_coherent_kern_range(start,end)
 *
 *	Ensure that the I and D caches are coherent within specified
 *	region.  This is typically used when code has been written to
 *	a memory region, and will be executed.
 *
 *	- start   - virtual start address of region
 *	- end     - virtual end address of region
 *
 *	It is assumed that:
 *	- the Icache does not read data from the write buffer
 */
ENTRY(v7_coherent_kern_range)
	/* FALLTHROUGH */

/*
 *	v7_coherent_user_range(start,end)
 *
 *	Ensure that the I and D caches are coherent within specified
 *	region.  This is typically used when code has been written to
 *	a memory region, and will be executed.
 *
 *	- start   - virtual start address of region
 *	- end     - virtual end address of region
 *
 *	It is assumed that:
 *	- the Icache does not read data from the write buffer
 */
ENTRY(v7_coherent_user_range)
 UNWIND(.fnstart		)
	dcache_line_size r2, r3
	sub	r3, r2, #1
	bic	r12, r0, r3
#ifdef CONFIG_ARM_ERRATA_764369
	ALT_SMP(W(dsb))
	ALT_UP(W(nop))
#endif
1:
 USER(	mcr	p15, 0, r12, c7, c11, 1	)	@ clean D line to the point of unification
	add	r12, r12, r2
	cmp	r12, r1
	blo	1b
	dsb	ishst
	icache_line_size r2, r3
	sub	r3, r2, #1
	bic	r12, r0, r3
2:
 USER(	mcr	p15, 0, r12, c7, c5, 1	)	@ invalidate I line
	add	r12, r12, r2
	cmp	r12, r1
	blo	2b
	mov	r0, #0
	ALT_SMP(mcr	p15, 0, r0, c7, c1, 6)	@ invalidate BTB Inner Shareable
	ALT_UP(mcr	p15, 0, r0, c7, c5, 6)	@ invalidate BTB
	dsb	ishst
	isb
	ret	lr

/*
 * Fault handling for the cache operation above. If the virtual address in r0
 * isn't mapped, fail with -EFAULT.
 */
9001:
#ifdef CONFIG_ARM_ERRATA_775420
	dsb
#endif
	mov	r0, #-EFAULT
	ret	lr
 UNWIND(.fnend		)
ENDPROC(v7_coherent_kern_range)
ENDPROC(v7_coherent_user_range)

/*
 *	v7_flush_kern_dcache_area(void *addr, size_t size)
 *
 *	Ensure that the data held in the page kaddr is written back
 *	to the page in question.
 *
 *	- addr	- kernel address
 *	- size	- region size
 */
ENTRY(v7_flush_kern_dcache_area)
	dcache_line_size r2, r3
	add	r1, r0, r1
	sub	r3, r2, #1
	bic	r0, r0, r3
#ifdef CONFIG_ARM_ERRATA_764369
	ALT_SMP(W(dsb))
	ALT_UP(W(nop))
#endif
1:
	mcr	p15, 0, r0, c7, c14, 1		@ clean & invalidate D line / unified line
	add	r0, r0, r2
	cmp	r0, r1
	blo	1b
	dsb	st
	ret	lr
ENDPROC(v7_flush_kern_dcache_area)

/*
 *	v7_dma_inv_range(start,end)
 *
 *	Invalidate the data cache within the specified region; we will
 *	be performing a DMA operation in this region and we want to
 *	purge old data in the cache.
 *
 *	- start   - virtual start address of region
 *	- end     - virtual end address of region
 */
v7_dma_inv_range:
	dcache_line_size r2, r3
	sub	r3, r2, #1
	tst	r0, r3
	bic	r0, r0, r3
#ifdef CONFIG_ARM_ERRATA_764369
	ALT_SMP(W(dsb))
	ALT_UP(W(nop))
#endif
	mcrne	p15, 0, r0, c7, c14, 1		@ clean & invalidate D / U line

	tst	r1, r3
	bic	r1, r1, r3
	mcrne	p15, 0, r1, c7, c14, 1		@ clean & invalidate D / U line
1:
	mcr	p15, 0, r0, c7, c6, 1		@ invalidate D / U line
	add	r0, r0, r2
	cmp	r0, r1
	blo	1b
	dsb	st
	ret	lr
ENDPROC(v7_dma_inv_range)

/*
 *	v7_dma_clean_range(start,end)
 *	- start   - virtual start address of region
 *	- end     - virtual end address of region
 */
v7_dma_clean_range:
	dcache_line_size r2, r3
	sub	r3, r2, #1
	bic	r0, r0, r3
#ifdef CONFIG_ARM_ERRATA_764369
	ALT_SMP(W(dsb))
	ALT_UP(W(nop))
#endif
1:
	mcr	p15, 0, r0, c7, c10, 1		@ clean D / U line
	add	r0, r0, r2
	cmp	r0, r1
	blo	1b
	dsb	st
	ret	lr
ENDPROC(v7_dma_clean_range)

/*
 *	v7_dma_flush_range(start,end)
 *	- start   - virtual start address of region
 *	- end     - virtual end address of region
 */
ENTRY(v7_dma_flush_range)
	dcache_line_size r2, r3
	sub	r3, r2, #1
	bic	r0, r0, r3
#ifdef CONFIG_ARM_ERRATA_764369
	ALT_SMP(W(dsb))
	ALT_UP(W(nop))
#endif
1:
	mcr	p15, 0, r0, c7, c14, 1		@ clean & invalidate D / U line
	add	r0, r0, r2
	cmp	r0, r1
	blo	1b
	dsb	st
	ret	lr
ENDPROC(v7_dma_flush_range)

/*
 *	dma_map_area(start, size, dir)
 *	- start	- kernel virtual start address
 *	- size	- size of region
 *	- dir	- DMA direction
 */
ENTRY(v7_dma_map_area)
	add	r1, r1, r0
	teq	r2, #DMA_FROM_DEVICE
	beq	v7_dma_inv_range
	b	v7_dma_clean_range
ENDPROC(v7_dma_map_area)

/*
 *	dma_unmap_area(start, size, dir)
 *	- start	- kernel virtual start address
 *	- size	- size of region
 *	- dir	- DMA direction
 */
ENTRY(v7_dma_unmap_area)
	add	r1, r1, r0
	teq	r2, #DMA_TO_DEVICE
	bne	v7_dma_inv_range
	ret	lr
ENDPROC(v7_dma_unmap_area)

	__INITDATA

	@ define struct cpu_cache_fns (see <asm/cacheflush.h> and proc-macros.S)
	define_cache_functions v7
Commit	Line	Data
bbe88886 CM	1	/*
	2	* linux/arch/arm/mm/cache-v7.S
	3	*
	4	* Copyright (C) 2001 Deep Blue Solutions Ltd.
	5	* Copyright (C) 2005 ARM Ltd.
	6	*
	7	* This program is free software; you can redistribute it and/or modify
	8	* it under the terms of the GNU General Public License version 2 as
	9	* published by the Free Software Foundation.
	10	*
	11	* This is the "shell" of the ARMv7 processor support.
	12	*/
	13	#include <linux/linkage.h>
	14	#include <linux/init.h>
	15	#include <asm/assembler.h>
c5102f59	16	#include <asm/errno.h>
32cfb1b1	17	#include <asm/unwind.h>
bbe88886 CM	18
	19	#include "proc-macros.S"
	20
c08e20d2 DN	21	/*
	22	* The secondary kernel init calls v7_flush_dcache_all before it enables
	23	* the L1; however, the L1 comes out of reset in an undefined state, so
	24	* the clean + invalidate performed by v7_flush_dcache_all causes a bunch
	25	* of cache lines with uninitialized data and uninitialized tags to get
	26	* written out to memory, which does really unpleasant things to the main
	27	* processor. We fix this by performing an invalidate, rather than a
	28	* clean + invalidate, before jumping into the kernel.
	29	*
	30	* This function is cloned from arch/arm/mach-tegra/headsmp.S, and needs
	31	* to be called for both secondary cores startup and primary core resume
	32	* procedures.
	33	*/
	34	ENTRY(v7_invalidate_l1)
	35	mov r0, #0
	36	mcr p15, 2, r0, c0, c0, 0
	37	mrc p15, 1, r0, c0, c0, 0
	38
5aca3708	39	movw r1, #0x7fff
c08e20d2 DN	40	and r2, r1, r0, lsr #13
c08e20d2 DN	41
5aca3708	42	movw r1, #0x3ff
c08e20d2 DN	43
	44	and r3, r1, r0, lsr #3 @ NumWays - 1
	45	add r2, r2, #1 @ NumSets
	46
	47	and r0, r0, #0x7
	48	add r0, r0, #4 @ SetShift
	49
	50	clz r1, r3 @ WayShift
	51	add r4, r3, #1 @ NumWays
	52	1: sub r2, r2, #1 @ NumSets--
	53	mov r3, r4 @ Temp = NumWays
	54	2: subs r3, r3, #1 @ Temp--
	55	mov r5, r3, lsl r1
	56	mov r6, r2, lsl r0
	57	orr r5, r5, r6 @ Reg = (Temp<<WayShift)\|(NumSets<<SetShift)
	58	mcr p15, 0, r5, c7, c6, 2
	59	bgt 2b
	60	cmp r2, #0
	61	bgt 1b
9581960a	62	dsb st
c08e20d2	63	isb
6ebbf2ce	64	ret lr
c08e20d2 DN	65	ENDPROC(v7_invalidate_l1)
c08e20d2 DN	66
81d11955 TL	67	/*
	68	* v7_flush_icache_all()
	69	*
	70	* Flush the whole I-cache.
	71	*
	72	* Registers:
	73	* r0 - set to 0
	74	*/
	75	ENTRY(v7_flush_icache_all)
	76	mov r0, #0
	77	ALT_SMP(mcr p15, 0, r0, c7, c1, 0) @ invalidate I-cache inner shareable
	78	ALT_UP(mcr p15, 0, r0, c7, c5, 0) @ I+BTB cache invalidate
6ebbf2ce	79	ret lr
81d11955 TL	80	ENDPROC(v7_flush_icache_all)
81d11955 TL	81
031bd879 LP	82	/*
	83	* v7_flush_dcache_louis()
	84	*
	85	* Flush the D-cache up to the Level of Unification Inner Shareable
	86	*
	87	* Corrupted registers: r0-r7, r9-r11 (r6 only in Thumb mode)
	88	*/
	89
	90	ENTRY(v7_flush_dcache_louis)
	91	dmb @ ensure ordering with previous memory accesses
	92	mrc p15, 1, r0, c0, c0, 1 @ read clidr, r0 = clidr
47b8484e RK	93	ALT_SMP(mov r3, r0, lsr #20) @ move LoUIS into position
	94	ALT_UP( mov r3, r0, lsr #26) @ move LoUU into position
	95	ands r3, r3, #7 << 1 @ extract LoU*2 field from clidr
d3cd451d	96	bne start_flush_levels @ LoU != 0, start flushing
69155794	97	#ifdef CONFIG_ARM_ERRATA_643719
d3cd451d RK	98	ALT_SMP(mrc p15, 0, r2, c0, c0, 0) @ read main ID register
d3cd451d RK	99	ALT_UP( ret lr) @ LoUU is zero, so nothing to do
aaf4b5d9 RK	100	movw r1, #:lower16:(0x410fc090 >> 4) @ ID of ARM Cortex A9 r0p?
	101	movt r1, #:upper16:(0x410fc090 >> 4)
	102	teq r1, r2, lsr #4 @ test for errata affected core and if so...
d3cd451d RK	103	moveq r3, #1 << 1 @ fix LoUIS value
d3cd451d RK	104	beq start_flush_levels @ start flushing cache levels
69155794	105	#endif
d3cd451d	106	ret lr
031bd879 LP	107	ENDPROC(v7_flush_dcache_louis)
031bd879 LP	108
bbe88886 CM	109	/*
	110	* v7_flush_dcache_all()
	111	*
	112	* Flush the whole D-cache.
	113	*
347c8b70	114	* Corrupted registers: r0-r7, r9-r11 (r6 only in Thumb mode)
bbe88886 CM	115	*
	116	* - mm - mm_struct describing address space
	117	*/
	118	ENTRY(v7_flush_dcache_all)
c30c2f99	119	dmb @ ensure ordering with previous memory accesses
bbe88886	120	mrc p15, 1, r0, c0, c0, 1 @ read clidr
47b8484e RK	121	mov r3, r0, lsr #23 @ move LoC into position
47b8484e RK	122	ands r3, r3, #7 << 1 @ extract LoC*2 from clidr
bbe88886	123	beq finished @ if loc is 0, then no need to clean
cd8b24d9	124	start_flush_levels:
bbe88886	125	mov r10, #0 @ start clean at cache level 0
3287be8c	126	flush_levels:
bbe88886 CM	127	add r2, r10, r10, lsr #1 @ work out 3x current cache level
	128	mov r1, r0, lsr r2 @ extract cache type bits from clidr
	129	and r1, r1, #7 @ mask of the bits for current cache only
	130	cmp r1, #2 @ see what cache we have at this level
	131	blt skip @ skip if no cache, or just i-cache
b46c0f74	132	#ifdef CONFIG_PREEMPT
8e43a905	133	save_and_disable_irqs_notrace r9 @ make cssr&csidr read atomic
b46c0f74	134	#endif
bbe88886 CM	135	mcr p15, 2, r10, c0, c0, 0 @ select current cache level in cssr
	136	isb @ isb to sych the new cssr&csidr
	137	mrc p15, 1, r1, c0, c0, 0 @ read the new csidr
b46c0f74 SB	138	#ifdef CONFIG_PREEMPT
	139	restore_irqs_notrace r9
	140	#endif
bbe88886 CM	141	and r2, r1, #7 @ extract the length of the cache lines
bbe88886 CM	142	add r2, r2, #4 @ add 4 (line length offset)
5aca3708	143	movw r4, #0x3ff
bbe88886 CM	144	ands r4, r4, r1, lsr #3 @ find maximum number on the way size
bbe88886 CM	145	clz r5, r4 @ find bit position of way size increment
5aca3708	146	movw r7, #0x7fff
bbe88886	147	ands r7, r7, r1, lsr #13 @ extract max number of the index size
3287be8c	148	loop1:
70f665fe	149	mov r9, r7 @ create working copy of max index
3287be8c	150	loop2:
70f665fe LP	151	ARM( orr r11, r10, r4, lsl r5 ) @ factor way and cache number into r11
70f665fe LP	152	THUMB( lsl r6, r4, r5 )
347c8b70	153	THUMB( orr r11, r10, r6 ) @ factor way and cache number into r11
70f665fe LP	154	ARM( orr r11, r11, r9, lsl r2 ) @ factor index number into r11
70f665fe LP	155	THUMB( lsl r6, r9, r2 )
347c8b70	156	THUMB( orr r11, r11, r6 ) @ factor index number into r11
bbe88886	157	mcr p15, 0, r11, c7, c14, 2 @ clean & invalidate by set/way
70f665fe	158	subs r9, r9, #1 @ decrement the index
bbe88886	159	bge loop2
70f665fe	160	subs r4, r4, #1 @ decrement the way
3287be8c	161	bge loop1
bbe88886 CM	162	skip:
	163	add r10, r10, #2 @ increment cache number
	164	cmp r3, r10
3287be8c	165	bgt flush_levels
bbe88886 CM	166	finished:
	167	mov r10, #0 @ swith back to cache level 0
	168	mcr p15, 2, r10, c0, c0, 0 @ select current cache level in cssr
9581960a	169	dsb st
bbe88886	170	isb
6ebbf2ce	171	ret lr
93ed3970	172	ENDPROC(v7_flush_dcache_all)
bbe88886 CM	173
	174	/*
	175	* v7_flush_cache_all()
	176	*
	177	* Flush the entire cache system.
	178	* The data cache flush is now achieved using atomic clean / invalidates
	179	* working outwards from L1 cache. This is done using Set/Way based cache
25985edc	180	* maintenance instructions.
bbe88886 CM	181	* The instruction cache can still be invalidated back to the point of
	182	* unification in a single instruction.
	183	*
	184	*/
	185	ENTRY(v7_flush_kern_cache_all)
347c8b70 CM	186	ARM( stmfd sp!, {r4-r5, r7, r9-r11, lr} )
347c8b70 CM	187	THUMB( stmfd sp!, {r4-r7, r9-r11, lr} )
bbe88886 CM	188	bl v7_flush_dcache_all
bbe88886 CM	189	mov r0, #0
f00ec48f RK	190	ALT_SMP(mcr p15, 0, r0, c7, c1, 0) @ invalidate I-cache inner shareable
f00ec48f RK	191	ALT_UP(mcr p15, 0, r0, c7, c5, 0) @ I+BTB cache invalidate
347c8b70 CM	192	ARM( ldmfd sp!, {r4-r5, r7, r9-r11, lr} )
347c8b70 CM	193	THUMB( ldmfd sp!, {r4-r7, r9-r11, lr} )
6ebbf2ce	194	ret lr
93ed3970	195	ENDPROC(v7_flush_kern_cache_all)
bbe88886	196
031bd879 LP	197	/*
	198	* v7_flush_kern_cache_louis(void)
	199	*
	200	* Flush the data cache up to Level of Unification Inner Shareable.
	201	* Invalidate the I-cache to the point of unification.
	202	*/
	203	ENTRY(v7_flush_kern_cache_louis)
	204	ARM( stmfd sp!, {r4-r5, r7, r9-r11, lr} )
	205	THUMB( stmfd sp!, {r4-r7, r9-r11, lr} )
	206	bl v7_flush_dcache_louis
	207	mov r0, #0
	208	ALT_SMP(mcr p15, 0, r0, c7, c1, 0) @ invalidate I-cache inner shareable
	209	ALT_UP(mcr p15, 0, r0, c7, c5, 0) @ I+BTB cache invalidate
	210	ARM( ldmfd sp!, {r4-r5, r7, r9-r11, lr} )
	211	THUMB( ldmfd sp!, {r4-r7, r9-r11, lr} )
6ebbf2ce	212	ret lr
031bd879 LP	213	ENDPROC(v7_flush_kern_cache_louis)
031bd879 LP	214
bbe88886 CM	215	/*
	216	* v7_flush_cache_all()
	217	*
	218	* Flush all TLB entries in a particular address space
	219	*
	220	* - mm - mm_struct describing address space
	221	*/
	222	ENTRY(v7_flush_user_cache_all)
	223	/FALLTHROUGH/
	224
	225	/*
	226	* v7_flush_cache_range(start, end, flags)
	227	*
	228	* Flush a range of TLB entries in the specified address space.
	229	*
	230	* - start - start address (may not be aligned)
	231	* - end - end address (exclusive, may not be aligned)
	232	* - flags - vm_area_struct flags describing address space
	233	*
	234	* It is assumed that:
	235	* - we have a VIPT cache.
	236	*/
	237	ENTRY(v7_flush_user_cache_range)
6ebbf2ce	238	ret lr
93ed3970 CM	239	ENDPROC(v7_flush_user_cache_all)
93ed3970 CM	240	ENDPROC(v7_flush_user_cache_range)
bbe88886 CM	241
	242	/*
	243	* v7_coherent_kern_range(start,end)
	244	*
	245	* Ensure that the I and D caches are coherent within specified
	246	* region. This is typically used when code has been written to
	247	* a memory region, and will be executed.
	248	*
	249	* - start - virtual start address of region
	250	* - end - virtual end address of region
	251	*
	252	* It is assumed that:
	253	* - the Icache does not read data from the write buffer
	254	*/
	255	ENTRY(v7_coherent_kern_range)
	256	/* FALLTHROUGH */
	257
	258	/*
	259	* v7_coherent_user_range(start,end)
	260	*
	261	* Ensure that the I and D caches are coherent within specified
	262	* region. This is typically used when code has been written to
	263	* a memory region, and will be executed.
	264	*
	265	* - start - virtual start address of region
	266	* - end - virtual end address of region
	267	*
	268	* It is assumed that:
	269	* - the Icache does not read data from the write buffer
	270	*/
	271	ENTRY(v7_coherent_user_range)
32cfb1b1	272	UNWIND(.fnstart )
bbe88886 CM	273	dcache_line_size r2, r3
bbe88886 CM	274	sub r3, r2, #1
da30e0ac	275	bic r12, r0, r3
f630c1bd WD	276	#ifdef CONFIG_ARM_ERRATA_764369
	277	ALT_SMP(W(dsb))
	278	ALT_UP(W(nop))
	279	#endif
32cfb1b1	280	1:
da30e0ac CM	281	USER( mcr p15, 0, r12, c7, c11, 1 ) @ clean D line to the point of unification
	282	add r12, r12, r2
	283	cmp r12, r1
	284	blo 1b
6abdd491	285	dsb ishst
da30e0ac CM	286	icache_line_size r2, r3
	287	sub r3, r2, #1
	288	bic r12, r0, r3
32cfb1b1	289	2:
da30e0ac CM	290	USER( mcr p15, 0, r12, c7, c5, 1 ) @ invalidate I line
	291	add r12, r12, r2
	292	cmp r12, r1
	293	blo 2b
bbe88886	294	mov r0, #0
f00ec48f RK	295	ALT_SMP(mcr p15, 0, r0, c7, c1, 6) @ invalidate BTB Inner Shareable
f00ec48f RK	296	ALT_UP(mcr p15, 0, r0, c7, c5, 6) @ invalidate BTB
6abdd491	297	dsb ishst
bbe88886	298	isb
6ebbf2ce	299	ret lr
32cfb1b1 CM	300
	301	/*
	302	* Fault handling for the cache operation above. If the virtual address in r0
c5102f59	303	* isn't mapped, fail with -EFAULT.
32cfb1b1 CM	304	*/
32cfb1b1 CM	305	9001:
7253b85c SH	306	#ifdef CONFIG_ARM_ERRATA_775420
	307	dsb
	308	#endif
c5102f59	309	mov r0, #-EFAULT
6ebbf2ce	310	ret lr
32cfb1b1	311	UNWIND(.fnend )
93ed3970 CM	312	ENDPROC(v7_coherent_kern_range)
93ed3970 CM	313	ENDPROC(v7_coherent_user_range)
bbe88886 CM	314
bbe88886 CM	315	/*
2c9b9c84	316	* v7_flush_kern_dcache_area(void *addr, size_t size)
bbe88886 CM	317	*
	318	* Ensure that the data held in the page kaddr is written back
	319	* to the page in question.
	320	*
2c9b9c84 RK	321	* - addr - kernel address
2c9b9c84 RK	322	* - size - region size
bbe88886	323	*/
2c9b9c84	324	ENTRY(v7_flush_kern_dcache_area)
bbe88886	325	dcache_line_size r2, r3
2c9b9c84	326	add r1, r0, r1
a248b13b WD	327	sub r3, r2, #1
a248b13b WD	328	bic r0, r0, r3
f630c1bd WD	329	#ifdef CONFIG_ARM_ERRATA_764369
	330	ALT_SMP(W(dsb))
	331	ALT_UP(W(nop))
	332	#endif
bbe88886 CM	333	1:
	334	mcr p15, 0, r0, c7, c14, 1 @ clean & invalidate D line / unified line
	335	add r0, r0, r2
	336	cmp r0, r1
	337	blo 1b
9581960a	338	dsb st
6ebbf2ce	339	ret lr
2c9b9c84	340	ENDPROC(v7_flush_kern_dcache_area)
bbe88886 CM	341
	342	/*
	343	* v7_dma_inv_range(start,end)
	344	*
	345	* Invalidate the data cache within the specified region; we will
	346	* be performing a DMA operation in this region and we want to
	347	* purge old data in the cache.
	348	*
	349	* - start - virtual start address of region
	350	* - end - virtual end address of region
	351	*/
702b94bf	352	v7_dma_inv_range:
bbe88886 CM	353	dcache_line_size r2, r3
	354	sub r3, r2, #1
	355	tst r0, r3
	356	bic r0, r0, r3
f630c1bd WD	357	#ifdef CONFIG_ARM_ERRATA_764369
	358	ALT_SMP(W(dsb))
	359	ALT_UP(W(nop))
	360	#endif
bbe88886 CM	361	mcrne p15, 0, r0, c7, c14, 1 @ clean & invalidate D / U line
	362
	363	tst r1, r3
	364	bic r1, r1, r3
	365	mcrne p15, 0, r1, c7, c14, 1 @ clean & invalidate D / U line
	366	1:
	367	mcr p15, 0, r0, c7, c6, 1 @ invalidate D / U line
	368	add r0, r0, r2
	369	cmp r0, r1
	370	blo 1b
9581960a	371	dsb st
6ebbf2ce	372	ret lr
93ed3970	373	ENDPROC(v7_dma_inv_range)
bbe88886 CM	374
	375	/*
	376	* v7_dma_clean_range(start,end)
	377	* - start - virtual start address of region
	378	* - end - virtual end address of region
	379	*/
702b94bf	380	v7_dma_clean_range:
bbe88886 CM	381	dcache_line_size r2, r3
	382	sub r3, r2, #1
	383	bic r0, r0, r3
f630c1bd WD	384	#ifdef CONFIG_ARM_ERRATA_764369
	385	ALT_SMP(W(dsb))
	386	ALT_UP(W(nop))
	387	#endif
bbe88886 CM	388	1:
	389	mcr p15, 0, r0, c7, c10, 1 @ clean D / U line
	390	add r0, r0, r2
	391	cmp r0, r1
	392	blo 1b
9581960a	393	dsb st
6ebbf2ce	394	ret lr
93ed3970	395	ENDPROC(v7_dma_clean_range)
bbe88886 CM	396
	397	/*
	398	* v7_dma_flush_range(start,end)
	399	* - start - virtual start address of region
	400	* - end - virtual end address of region
	401	*/
	402	ENTRY(v7_dma_flush_range)
	403	dcache_line_size r2, r3
	404	sub r3, r2, #1
	405	bic r0, r0, r3
f630c1bd WD	406	#ifdef CONFIG_ARM_ERRATA_764369
	407	ALT_SMP(W(dsb))
	408	ALT_UP(W(nop))
	409	#endif
bbe88886 CM	410	1:
	411	mcr p15, 0, r0, c7, c14, 1 @ clean & invalidate D / U line
	412	add r0, r0, r2
	413	cmp r0, r1
	414	blo 1b
9581960a	415	dsb st
6ebbf2ce	416	ret lr
93ed3970	417	ENDPROC(v7_dma_flush_range)
bbe88886	418
a9c9147e RK	419	/*
	420	* dma_map_area(start, size, dir)
	421	* - start - kernel virtual start address
	422	* - size - size of region
	423	* - dir - DMA direction
	424	*/
	425	ENTRY(v7_dma_map_area)
	426	add r1, r1, r0
2ffe2da3 RK	427	teq r2, #DMA_FROM_DEVICE
	428	beq v7_dma_inv_range
	429	b v7_dma_clean_range
a9c9147e RK	430	ENDPROC(v7_dma_map_area)
	431
	432	/*
	433	* dma_unmap_area(start, size, dir)
	434	* - start - kernel virtual start address
	435	* - size - size of region
	436	* - dir - DMA direction
	437	*/
	438	ENTRY(v7_dma_unmap_area)
2ffe2da3 RK	439	add r1, r1, r0
	440	teq r2, #DMA_TO_DEVICE
	441	bne v7_dma_inv_range
6ebbf2ce	442	ret lr
a9c9147e RK	443	ENDPROC(v7_dma_unmap_area)
a9c9147e RK	444
bbe88886 CM	445	__INITDATA
bbe88886 CM	446
455a01ec DM	447	@ define struct cpu_cache_fns (see <asm/cacheflush.h> and proc-macros.S)
455a01ec DM	448	define_cache_functions v7