patch-2.4.20 linux-2.4.20/arch/sparc64/mm/ultra.S

• Lines: 462
• Date: Thu Nov 28 15:53:12 2002
• Orig file: linux-2.4.19/arch/sparc64/mm/ultra.S
• Orig date: Fri Aug 2 17:39:43 2002

diff -urN linux-2.4.19/arch/sparc64/mm/ultra.S linux-2.4.20/arch/sparc64/mm/ultra.S
@@ -13,21 +13,12 @@
 #include <asm/pil.h>
 #include <asm/head.h>
 
-	/* Basically, all this madness has to do with the
-	 * fact that Cheetah does not support IMMU flushes
-	 * out of the secondary context.  Someone needs to
-	 * throw a south lake birthday party for the folks
+	/* Basically, most of the Spitfire vs. Cheetah madness
+	 * has to do with the fact that Cheetah does not support
+	 * IMMU flushes out of the secondary context.  Someone needs
+	 * to throw a south lake birthday party for the folks
 	 * in Microelectronics who refused to fix this shit.
 	 */
-#define BRANCH_IF_CHEETAH(tmp1, tmp2, label)		\
-	rdpr	%ver, %tmp1;				\
-	sethi	%hi(0x003e0014), %tmp2;			\
-	srlx	%tmp1, 32, %tmp1;			\
-	or	%tmp2, %lo(0x003e0014), %tmp2;		\
-	cmp	%tmp1, %tmp2;				\
-	be,pn	%icc, label;				\
-	 nop;						\
-	nop;
 
 	/* This file is meant to be read efficiently by the CPU, not humans.
 	 * Staraj sie tego nikomu nie pierdolnac...
@@ -36,9 +27,7 @@
 	.align		32
 	.globl		__flush_tlb_page, __flush_tlb_mm, __flush_tlb_range
 __flush_tlb_page: /* %o0=(ctx & TAG_CONTEXT_BITS), %o1=page&PAGE_MASK, %o2=SECONDARY_CONTEXT */
-/*IC1*/	BRANCH_IF_CHEETAH(g2, g3, __cheetah_flush_tlb_page)
-__spitfire_flush_tlb_page:
-/*IC2*/	ldxa		[%o2] ASI_DMMU, %g2
+	ldxa		[%o2] ASI_DMMU, %g2
 	cmp		%g2, %o0
 	bne,pn		%icc, __spitfire_flush_tlb_page_slow
 	 or		%o1, 0x10, %g3
@@ -46,27 +35,17 @@
 	stxa		%g0, [%g3] ASI_IMMU_DEMAP
 	retl
 	 flush		%g6
-__cheetah_flush_tlb_page:
-/*IC3*/	rdpr		%pstate, %g5
-	andn		%g5, PSTATE_IE, %g2
-	wrpr		%g2, 0x0, %pstate
-	wrpr		%g0, 1, %tl
-	mov		PRIMARY_CONTEXT, %o2
-	ldxa		[%o2] ASI_DMMU, %g2
-	stxa		%o0, [%o2] ASI_DMMU
-	stxa		%g0, [%o1] ASI_DMMU_DEMAP
-/*IC4*/	stxa		%g0, [%o1] ASI_IMMU_DEMAP
-	stxa		%g2, [%o2] ASI_DMMU
-	flush		%g6
-	wrpr		%g0, 0, %tl
-	retl
-	 wrpr		%g5, 0x0, %pstate
 	nop
 	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+
 __flush_tlb_mm: /* %o0=(ctx & TAG_CONTEXT_BITS), %o1=SECONDARY_CONTEXT */
-/*IC5*/	BRANCH_IF_CHEETAH(g2, g3, __cheetah_flush_tlb_mm)
-__spitfire_flush_tlb_mm:
-/*IC6*/	ldxa		[%o1] ASI_DMMU, %g2
+	ldxa		[%o1] ASI_DMMU, %g2
 	cmp		%g2, %o0
 	bne,pn		%icc, __spitfire_flush_tlb_mm_slow
 	 mov		0x50, %g3
@@ -74,30 +53,20 @@
 	stxa		%g0, [%g3] ASI_IMMU_DEMAP
 	retl
 	 flush		%g6
-__cheetah_flush_tlb_mm:
-/*IC7*/	rdpr		%pstate, %g5
-	andn		%g5, PSTATE_IE, %g2
-	wrpr		%g2, 0x0, %pstate
-	wrpr		%g0, 1, %tl
-	mov		PRIMARY_CONTEXT, %o2
-	mov		0x40, %g3
-	ldxa		[%o2] ASI_DMMU, %g2
-	stxa		%o0, [%o2] ASI_DMMU
-/*IC8*/	stxa		%g0, [%g3] ASI_DMMU_DEMAP
-	stxa		%g0, [%g3] ASI_IMMU_DEMAP
-	stxa		%g2, [%o2] ASI_DMMU
-	flush		%g6
-	wrpr		%g0, 0, %tl
-	retl
-	 wrpr		%g5, 0x0, %pstate
 	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+	nop
+
 __flush_tlb_range: /* %o0=(ctx&TAG_CONTEXT_BITS), %o1=start&PAGE_MASK, %o2=SECONDARY_CONTEXT,
 		    * %o3=end&PAGE_MASK, %o4=PAGE_SIZE, %o5=(end - start)
 		    */
-/*IC9*/	BRANCH_IF_CHEETAH(g2, g3, __cheetah_flush_tlb_range)
-__spitfire_flush_tlb_range:
 #define TLB_MAGIC	207 /* Students, do you know how I calculated this?  -DaveM */
-/*IC10*/cmp		%o5, %o4
+	cmp		%o5, %o4
 	bleu,pt		%xcc, __flush_tlb_page
 	 srlx		%o5, PAGE_SHIFT, %g5
 	cmp		%g5, TLB_MAGIC
@@ -106,7 +75,7 @@
 	ldxa		[%o2] ASI_DMMU, %g2
 	cmp		%g2, %o0
 __spitfire_flush_tlb_range_page_by_page:
-/*IC11*/bne,pn		%icc, __spitfire_flush_tlb_range_pbp_slow
+	bne,pn		%icc, __spitfire_flush_tlb_range_pbp_slow
 	 sub		%o5, %o4, %o5
 1:	stxa		%g0, [%g5 + %o5] ASI_DMMU_DEMAP
 	stxa		%g0, [%g5 + %o5] ASI_IMMU_DEMAP
@@ -115,10 +84,9 @@
 	retl
 	 flush		%g6
 __spitfire_flush_tlb_range_constant_time: /* %o0=ctx, %o1=start, %o3=end */
-/*IC12*/rdpr		%pstate, %g1
+	rdpr		%pstate, %g1
 	wrpr		%g1, PSTATE_IE, %pstate
 	mov		TLB_TAG_ACCESS, %g3
-	/* XXX Spitfire dependency... */
 	mov		((SPITFIRE_HIGHEST_LOCKED_TLBENT-1) << 3), %g2
 
 	/* Spitfire Errata #32 workaround. */
@@ -130,7 +98,7 @@
 	and		%o4, TAG_CONTEXT_BITS, %o5
 	cmp		%o5, %o0
 	bne,pt		%icc, 2f
-/*IC13*/ andn		%o4, TAG_CONTEXT_BITS, %o4
+	 andn		%o4, TAG_CONTEXT_BITS, %o4
 	cmp		%o4, %o1
 	blu,pt		%xcc, 2f
 	 cmp		%o4, %o3
@@ -138,7 +106,7 @@
 2:	 ldxa		[%g2] ASI_DTLB_TAG_READ, %o4
 	and		%o4, TAG_CONTEXT_BITS, %o5
 	cmp		%o5, %o0
-/*IC14*/andn		%o4, TAG_CONTEXT_BITS, %o4
+	andn		%o4, TAG_CONTEXT_BITS, %o4
 	bne,pt		%icc, 3f
 	 cmp		%o4, %o1
 	blu,pt		%xcc, 3f
@@ -146,7 +114,7 @@
 	blu,pn		%xcc, 5f
 	 nop
 3:	brnz,pt		%g2, 1b
-/*IC15*/ sub		%g2, (1 << 3), %g2
+	 sub		%g2, (1 << 3), %g2
 	retl
 	 wrpr		%g1, 0x0, %pstate
 4:	stxa		%g0, [%g3] ASI_IMMU
@@ -162,7 +130,7 @@
 	 nop
 
 5:	stxa		%g0, [%g3] ASI_DMMU
-/*IC16*/stxa		%g0, [%g2] ASI_DTLB_DATA_ACCESS
+	stxa		%g0, [%g2] ASI_DTLB_DATA_ACCESS
 	flush		%g6
 
 	/* Spitfire Errata #32 workaround. */
@@ -173,33 +141,6 @@
 	ba,pt		%xcc, 3b
 	 nop
 
-	.align		32
-__cheetah_flush_tlb_range:
-	cmp		%o5, %o4
-	bleu,pt		%xcc, __cheetah_flush_tlb_page
-	 nop
-/*IC17*/rdpr		%pstate, %g5
-	andn		%g5, PSTATE_IE, %g2
-	wrpr		%g2, 0x0, %pstate
-	wrpr		%g0, 1, %tl
-	mov		PRIMARY_CONTEXT, %o2
-	sub		%o5, %o4, %o5
-	ldxa		[%o2] ASI_DMMU, %g2
-	stxa		%o0, [%o2] ASI_DMMU
-
-/*IC18*/
-1:	stxa		%g0, [%o1 + %o5] ASI_DMMU_DEMAP
-	stxa		%g0, [%o1 + %o5] ASI_IMMU_DEMAP
-	membar		#Sync
-	brnz,pt		%o5, 1b
-	 sub		%o5, %o4, %o5
-
-	stxa		%g2, [%o2] ASI_DMMU
-	flush		%g6
-	wrpr		%g0, 0, %tl
-	retl
-/*IC19*/ wrpr		%g5, 0x0, %pstate
-
 __spitfire_flush_tlb_mm_slow:
 	rdpr		%pstate, %g1
 	wrpr		%g1, PSTATE_IE, %pstate
@@ -208,7 +149,7 @@
 	stxa		%g0, [%g3] ASI_IMMU_DEMAP
 	flush		%g6
 	stxa		%g2, [%o1] ASI_DMMU
-/*IC18*/flush		%g6
+	flush		%g6
 	retl
 	 wrpr		%g1, 0, %pstate
 
@@ -218,7 +159,7 @@
 	stxa		%o0, [%o2] ASI_DMMU
 	stxa		%g0, [%g3] ASI_DMMU_DEMAP
 	stxa		%g0, [%g3] ASI_IMMU_DEMAP
-/*IC20*/flush		%g6
+	flush		%g6
 	stxa		%g2, [%o2] ASI_DMMU
 	flush		%g6
 	retl
@@ -228,7 +169,7 @@
 	rdpr		%pstate, %g1
 	wrpr		%g1, PSTATE_IE, %pstate
 	stxa		%o0, [%o2] ASI_DMMU
-/*IC21*/
+
 2:	stxa		%g0, [%g5 + %o5] ASI_DMMU_DEMAP
 	stxa		%g0, [%g5 + %o5] ASI_IMMU_DEMAP
 	brnz,pt		%o5, 2b
@@ -237,7 +178,7 @@
 	stxa		%g2, [%o2] ASI_DMMU
 	flush		%g6
 	retl
-/*IC22*/ wrpr		%g1, 0x0, %pstate
+	 wrpr		%g1, 0x0, %pstate
 
 /*
  * The following code flushes one page_size worth.
@@ -301,30 +242,6 @@
 	flush		%g6
 	ba,a,pt		%xcc, 3b
 
-	.align		64
-	.globl		__flush_dcache_page
-__flush_dcache_page:	/* %o0=kaddr, %o1=flush_icache */
-	sub		%o0, %g4, %o0
-	
-	rdpr		%ver, %g1
-	sethi		%hi(0x003e0014), %g2
-	srlx		%g1, 32, %g1
-	or		%g2, %lo(0x003e0014), %g2
-	cmp		%g1, %g2
-	bne,pt		%icc, flush_dcpage_spitfire
-	 nop
-
-flush_dcpage_cheetah:
-	sethi		%hi(PAGE_SIZE), %o4
-1:	subcc		%o4, (1 << 5), %o4
-	stxa		%g0, [%o0 + %o4] ASI_DCACHE_INVALIDATE
-	bne,pt		%icc, 1b
-	 nop
-	membar		#Sync
-	/* I-cache flush never needed on Cheetah, see callers. */
-	retl
-	 nop
-
 #if (PAGE_SHIFT == 13)
 #define DTAG_MASK 0x3
 #elif (PAGE_SHIFT == 16)
@@ -335,7 +252,10 @@
 #define DTAG_MASK 0x3ff
 #endif
 
-flush_dcpage_spitfire:
+	.align		64
+	.globl		__flush_dcache_page
+__flush_dcache_page:	/* %o0=kaddr, %o1=flush_icache */
+	sub		%o0, %g4, %o0
 	clr		%o4
 	srlx		%o0, 11, %o0
 	sethi		%hi(1 << 14), %o2
@@ -424,6 +344,119 @@
 1:	retl
 	 nop
 
+	/* Cheetah specific versions, patched at boot time.  */
+__cheetah_flush_tlb_page: /* 14 insns */
+	rdpr		%pstate, %g5
+	andn		%g5, PSTATE_IE, %g2
+	wrpr		%g2, 0x0, %pstate
+	wrpr		%g0, 1, %tl
+	mov		PRIMARY_CONTEXT, %o2
+	ldxa		[%o2] ASI_DMMU, %g2
+	stxa		%o0, [%o2] ASI_DMMU
+	stxa		%g0, [%o1] ASI_DMMU_DEMAP
+	stxa		%g0, [%o1] ASI_IMMU_DEMAP
+	stxa		%g2, [%o2] ASI_DMMU
+	flush		%g6
+	wrpr		%g0, 0, %tl
+	retl
+	 wrpr		%g5, 0x0, %pstate
+
+__cheetah_flush_tlb_mm: /* 15 insns */
+	rdpr		%pstate, %g5
+	andn		%g5, PSTATE_IE, %g2
+	wrpr		%g2, 0x0, %pstate
+	wrpr		%g0, 1, %tl
+	mov		PRIMARY_CONTEXT, %o2
+	mov		0x40, %g3
+	ldxa		[%o2] ASI_DMMU, %g2
+	stxa		%o0, [%o2] ASI_DMMU
+	stxa		%g0, [%g3] ASI_DMMU_DEMAP
+	stxa		%g0, [%g3] ASI_IMMU_DEMAP
+	stxa		%g2, [%o2] ASI_DMMU
+	flush		%g6
+	wrpr		%g0, 0, %tl
+	retl
+	 wrpr		%g5, 0x0, %pstate
+
+__cheetah_flush_tlb_range: /* 20 insns */
+	cmp		%o5, %o4
+	blu,pt		%xcc, 9f
+	 rdpr		%pstate, %g5
+	andn		%g5, PSTATE_IE, %g2
+	wrpr		%g2, 0x0, %pstate
+	wrpr		%g0, 1, %tl
+	mov		PRIMARY_CONTEXT, %o2
+	sub		%o5, %o4, %o5
+	ldxa		[%o2] ASI_DMMU, %g2
+	stxa		%o0, [%o2] ASI_DMMU
+1:	stxa		%g0, [%o1 + %o5] ASI_DMMU_DEMAP
+	stxa		%g0, [%o1 + %o5] ASI_IMMU_DEMAP
+	membar		#Sync
+	brnz,pt		%o5, 1b
+	 sub		%o5, %o4, %o5
+	stxa		%g2, [%o2] ASI_DMMU
+	flush		%g6
+	wrpr		%g0, 0, %tl
+9:	retl
+	 wrpr		%g5, 0x0, %pstate
+
+flush_dcpage_cheetah: /* 9 insns */
+	sub		%o0, %g4, %o0
+	sethi		%hi(PAGE_SIZE), %o4
+1:	subcc		%o4, (1 << 5), %o4
+	stxa		%g0, [%o0 + %o4] ASI_DCACHE_INVALIDATE
+	membar		#Sync
+	bne,pt		%icc, 1b
+	 nop
+	retl		/* I-cache flush never needed on Cheetah, see callers. */
+	 nop
+
+cheetah_patch_one:
+1:	lduw		[%o1], %g1
+	stw		%g1, [%o0]
+	flush		%o0
+	subcc		%o2, 1, %o2
+	add		%o1, 4, %o1
+	bne,pt		%icc, 1b
+	 add		%o0, 4, %o0
+	retl
+	 nop
+
+	.globl		cheetah_patch_cachetlbops
+cheetah_patch_cachetlbops:
+	save		%sp, -128, %sp
+
+	sethi		%hi(__flush_tlb_page), %o0
+	or		%o0, %lo(__flush_tlb_page), %o0
+	sethi		%hi(__cheetah_flush_tlb_page), %o1
+	or		%o1, %lo(__cheetah_flush_tlb_page), %o1
+	call		cheetah_patch_one
+	 mov		14, %o2
+
+	sethi		%hi(__flush_tlb_mm), %o0
+	or		%o0, %lo(__flush_tlb_mm), %o0
+	sethi		%hi(__cheetah_flush_tlb_mm), %o1
+	or		%o1, %lo(__cheetah_flush_tlb_mm), %o1
+	call		cheetah_patch_one
+	 mov		15, %o2
+
+	sethi		%hi(__flush_tlb_range), %o0
+	or		%o0, %lo(__flush_tlb_range), %o0
+	sethi		%hi(__cheetah_flush_tlb_range), %o1
+	or		%o1, %lo(__cheetah_flush_tlb_range), %o1
+	call		cheetah_patch_one
+	 mov		20, %o2
+
+	sethi		%hi(__flush_dcache_page), %o0
+	or		%o0, %lo(__flush_dcache_page), %o0
+	sethi		%hi(flush_dcpage_cheetah), %o1
+	or		%o1, %lo(flush_dcpage_cheetah), %o1
+	call		cheetah_patch_one
+	 mov		9, %o2
+
+	ret
+	 restore
+
 #ifdef CONFIG_SMP
 	/* These are all called by the slaves of a cross call, at
 	 * trap level 1, with interrupts fully disabled.
@@ -522,9 +555,9 @@
 	sethi		%hi(PAGE_SIZE), %g3
 1:	subcc		%g3, (1 << 5), %g3
 	stxa		%g0, [%g1 + %g3] ASI_DCACHE_INVALIDATE
+	membar		#Sync
 	bne,pt		%icc, 1b
 	 nop
-	membar		#Sync
 	retry
 	nop
 
@@ -587,10 +620,8 @@
 	.text
 
 	/* These two are not performance critical... */
-	.globl		xcall_flush_tlb_all
-xcall_flush_tlb_all:
-	BRANCH_IF_CHEETAH(g2, g3, __cheetah_xcall_flush_tlb_all)
-__spitfire_xcall_flush_tlb_all:
+	.globl		xcall_flush_tlb_all_spitfire
+xcall_flush_tlb_all_spitfire:
 	/* Spitfire Errata #32 workaround. */
 	sethi		%hi(errata32_hwbug), %g4
 	stx		%g0, [%g4 + %lo(errata32_hwbug)]
@@ -632,16 +663,15 @@
 	flush		%g6
 	retry
 
-__cheetah_xcall_flush_tlb_all:
+	.globl		xcall_flush_tlb_all_cheetah
+xcall_flush_tlb_all_cheetah:
 	mov		0x80, %g2
 	stxa		%g0, [%g2] ASI_DMMU_DEMAP
 	stxa		%g0, [%g2] ASI_IMMU_DEMAP
 	retry
 
-	.globl		xcall_flush_cache_all
-xcall_flush_cache_all:
-	BRANCH_IF_CHEETAH(g2, g3, __cheetah_xcall_flush_cache_all)
-__spitfire_xcall_flush_cache_all:
+	.globl		xcall_flush_cache_all_spitfire
+xcall_flush_cache_all_spitfire:
 	sethi		%hi(16383), %g2
 	or		%g2, %lo(16383), %g2
 	clr		%g3
@@ -654,13 +684,6 @@
 	flush		%g6
 	retry
 
-	/* Cheetah's caches are fully coherent in the sense that
-	 * caches are flushed here.  We need to verify this and
-	 * really just not even send out the xcall at the top level.
-	 */
-__cheetah_xcall_flush_cache_all:
-	retry
-
 	/* These just get rescheduled to PIL vectors. */
 	.globl		xcall_call_function
 xcall_call_function: