/* tlb-miss.S: TLB miss handlers
   *
   * Copyright (C) 2004 Red Hat, Inc. All Rights Reserved.
   * Written by David Howells (dhowells@redhat.com)
   *
   * This program is free software; you can redistribute it and/or
   * modify it under the terms of the GNU General Public License
   * as published by the Free Software Foundation; either version
   * 2 of the License, or (at your option) any later version.
   */
  
  #include <linux/sys.h>
  #include <linux/linkage.h>
  #include <asm/page.h>
  #include <asm/pgtable.h>
  #include <asm/spr-regs.h>
  
  	.section	.text..tlbmiss
  	.balign		4
  
  	.globl		__entry_insn_mmu_miss
  __entry_insn_mmu_miss:
  	break
  	nop
  
  	.globl		__entry_insn_mmu_exception
  __entry_insn_mmu_exception:
  	break
  	nop
  
  	.globl		__entry_data_mmu_miss
  __entry_data_mmu_miss:
  	break
  	nop
  
  	.globl		__entry_data_mmu_exception
  __entry_data_mmu_exception:
  	break
  	nop
  
  ###############################################################################
  #
  # handle a lookup failure of one sort or another in a kernel TLB handler
  # On entry:
  #   GR29 - faulting address
  #   SCR2 - saved CCR
  #
  ###############################################################################
  	.type		__tlb_kernel_fault,@function
  __tlb_kernel_fault:
  	# see if we're supposed to re-enable single-step mode upon return
  	sethi.p		%hi(__break_tlb_miss_return_break),gr30
  	setlo		%lo(__break_tlb_miss_return_break),gr30
  	movsg		pcsr,gr31
  
  	subcc		gr31,gr30,gr0,icc0
  	beq		icc0,#0,__tlb_kernel_fault_sstep
  
  	movsg		scr2,gr30
  	movgs		gr30,ccr
  	movgs		gr29,scr2			/* save EAR0 value */
  	sethi.p		%hi(__kernel_current_task),gr29
  	setlo		%lo(__kernel_current_task),gr29
  	ldi.p		@(gr29,#0),gr29			/* restore GR29 */
  
  	bra		__entry_kernel_handle_mmu_fault
  
  	# we've got to re-enable single-stepping
  __tlb_kernel_fault_sstep:
  	sethi.p		%hi(__break_tlb_miss_real_return_info),gr30
  	setlo		%lo(__break_tlb_miss_real_return_info),gr30
  	lddi		@(gr30,0),gr30
  	movgs		gr30,pcsr
  	movgs		gr31,psr
  
  	movsg		scr2,gr30
  	movgs		gr30,ccr
  	movgs		gr29,scr2			/* save EAR0 value */
  	sethi.p		%hi(__kernel_current_task),gr29
  	setlo		%lo(__kernel_current_task),gr29
  	ldi.p		@(gr29,#0),gr29			/* restore GR29 */
  	bra		__entry_kernel_handle_mmu_fault_sstep
  
  	.size		__tlb_kernel_fault, .-__tlb_kernel_fault
  
  ###############################################################################
  #
  # handle a lookup failure of one sort or another in a user TLB handler
  # On entry:
  #   GR28 - faulting address
  #   SCR2 - saved CCR
  #
  ###############################################################################
  	.type		__tlb_user_fault,@function
  __tlb_user_fault:
  	# see if we're supposed to re-enable single-step mode upon return
  	sethi.p		%hi(__break_tlb_miss_return_break),gr30
  	setlo		%lo(__break_tlb_miss_return_break),gr30
  	movsg		pcsr,gr31
  	subcc		gr31,gr30,gr0,icc0
  	beq		icc0,#0,__tlb_user_fault_sstep
  
  	movsg		scr2,gr30
  	movgs		gr30,ccr
  	bra		__entry_uspace_handle_mmu_fault
  
  	# we've got to re-enable single-stepping
  __tlb_user_fault_sstep:
  	sethi.p		%hi(__break_tlb_miss_real_return_info),gr30
  	setlo		%lo(__break_tlb_miss_real_return_info),gr30
  	lddi		@(gr30,0),gr30
  	movgs		gr30,pcsr
  	movgs		gr31,psr
  	movsg		scr2,gr30
  	movgs		gr30,ccr
  	bra		__entry_uspace_handle_mmu_fault_sstep
  
  	.size		__tlb_user_fault, .-__tlb_user_fault
  
  ###############################################################################
  #
  # Kernel instruction TLB miss handler
  # On entry:
  #   GR1   - kernel stack pointer
  #   GR28  - saved exception frame pointer
  #   GR29  - faulting address
  #   GR31  - EAR0 ^ SCR0
  #   SCR0  - base of virtual range covered by cached PGE from last ITLB miss (or 0xffffffff)
  #   DAMR3 - mapped page directory
  #   DAMR4 - mapped page table as matched by SCR0
  #
  ###############################################################################
  	.globl		__entry_kernel_insn_tlb_miss
  	.type		__entry_kernel_insn_tlb_miss,@function
  __entry_kernel_insn_tlb_miss:
  #if 0
  	sethi.p		%hi(0xe1200004),gr30
  	setlo		%lo(0xe1200004),gr30
  	st		gr0,@(gr30,gr0)
  	sethi.p		%hi(0xffc00100),gr30
  	setlo		%lo(0xffc00100),gr30
  	sth		gr30,@(gr30,gr0)
  	membar
  #endif
  
  	movsg		ccr,gr30			/* save CCR */
  	movgs		gr30,scr2
  
  	# see if the cached page table mapping is appropriate
  	srlicc.p	gr31,#26,gr0,icc0
  	setlos		0x3ffc,gr30
  	srli.p		gr29,#12,gr31			/* use EAR0[25:14] as PTE index */
  	bne		icc0,#0,__itlb_k_PTD_miss
  
  __itlb_k_PTD_mapped:
  	# access the PTD with EAR0[25:14]
  	# - DAMLR4 points to the virtual address of the appropriate page table
  	# - the PTD holds 4096 PTEs
  	# - the PTD must be accessed uncached
  	# - the PTE must be marked accessed if it was valid
  	#
  	and		gr31,gr30,gr31
  	movsg		damlr4,gr30
  	add		gr30,gr31,gr31
  	ldi		@(gr31,#0),gr30			/* fetch the PTE */
  	andicc		gr30,#_PAGE_PRESENT,gr0,icc0
  	ori.p		gr30,#_PAGE_ACCESSED,gr30
  	beq		icc0,#0,__tlb_kernel_fault	/* jump if PTE invalid */
  	sti.p		gr30,@(gr31,#0)			/* update the PTE */
  	andi		gr30,#~_PAGE_ACCESSED,gr30
  
  	# we're using IAMR1 as an extra TLB entry
  	# - punt the entry here (if valid) to the real TLB and then replace with the new PTE
  	# - need to check DAMR1 lest we cause an multiple-DAT-hit exception
  	# - IAMPR1 has no WP bit, and we mustn't lose WP information
  	movsg		iampr1,gr31
  	andicc		gr31,#xAMPRx_V,gr0,icc0
  	setlos.p	0xfffff000,gr31
  	beq		icc0,#0,__itlb_k_nopunt		/* punt not required */
  
  	movsg		iamlr1,gr31
  	movgs		gr31,tplr			/* set TPLR.CXN */
  	tlbpr		gr31,gr0,#4,#0			/* delete matches from TLB, IAMR1, DAMR1 */
  
  	movsg		dampr1,gr31
  	ori		gr31,#xAMPRx_V,gr31		/* entry was invalidated by tlbpr #4 */
  	movgs		gr31,tppr
  	movsg		iamlr1,gr31			/* set TPLR.CXN */
  	movgs		gr31,tplr
  	tlbpr		gr31,gr0,#2,#0			/* save to the TLB */
  	movsg		tpxr,gr31			/* check the TLB write error flag */
  	andicc.p	gr31,#TPXR_E,gr0,icc0
  	setlos		#0xfffff000,gr31
  	bne		icc0,#0,__tlb_kernel_fault
  
  __itlb_k_nopunt:
  
  	# assemble the new TLB entry
  	and		gr29,gr31,gr29
  	movsg		cxnr,gr31
  	or		gr29,gr31,gr29
  	movgs		gr29,iamlr1			/* xAMLR = address | context number */
  	movgs		gr30,iampr1
  	movgs		gr29,damlr1
  	movgs		gr30,dampr1
  
  	# return, restoring registers
  	movsg		scr2,gr30
  	movgs		gr30,ccr
  	sethi.p		%hi(__kernel_current_task),gr29
  	setlo		%lo(__kernel_current_task),gr29
  	ldi		@(gr29,#0),gr29
  	rett		#0
  	beq		icc0,#3,0			/* prevent icache prefetch */
  
  	# the PTE we want wasn't in the PTD we have mapped, so we need to go looking for a more
  	# appropriate page table and map that instead
  	#   - access the PGD with EAR0[31:26]
  	#   - DAMLR3 points to the virtual address of the page directory
  	#   - the PGD holds 64 PGEs and each PGE/PME points to a set of page tables
  __itlb_k_PTD_miss:
  	srli		gr29,#26,gr31			/* calculate PGE offset */
  	slli		gr31,#8,gr31			/* and clear bottom bits */
  
  	movsg		damlr3,gr30
  	ld		@(gr31,gr30),gr30		/* access the PGE */
  
  	andicc.p	gr30,#_PAGE_PRESENT,gr0,icc0
  	andicc		gr30,#xAMPRx_SS,gr0,icc1
  
  	# map this PTD instead and record coverage address
  	ori.p		gr30,#xAMPRx_L|xAMPRx_SS_16Kb|xAMPRx_S|xAMPRx_C|xAMPRx_V,gr30
  	beq		icc0,#0,__tlb_kernel_fault	/* jump if PGE not present */
  	slli.p		gr31,#18,gr31
  	bne		icc1,#0,__itlb_k_bigpage
  	movgs		gr30,dampr4
  	movgs		gr31,scr0
  
  	# we can now resume normal service
  	setlos		0x3ffc,gr30
  	srli.p		gr29,#12,gr31			/* use EAR0[25:14] as PTE index */
  	bra		__itlb_k_PTD_mapped
  
  __itlb_k_bigpage:
  	break
  	nop
  
  	.size		__entry_kernel_insn_tlb_miss, .-__entry_kernel_insn_tlb_miss
  
  ###############################################################################
  #
  # Kernel data TLB miss handler
  # On entry:
  #   GR1   - kernel stack pointer
  #   GR28  - saved exception frame pointer
  #   GR29  - faulting address
  #   GR31  - EAR0 ^ SCR1
  #   SCR1  - base of virtual range covered by cached PGE from last DTLB miss (or 0xffffffff)
  #   DAMR3 - mapped page directory
  #   DAMR5 - mapped page table as matched by SCR1
  #
  ###############################################################################
  	.globl		__entry_kernel_data_tlb_miss
  	.type		__entry_kernel_data_tlb_miss,@function
  __entry_kernel_data_tlb_miss:
  #if 0
  	sethi.p		%hi(0xe1200004),gr30
  	setlo		%lo(0xe1200004),gr30
  	st		gr0,@(gr30,gr0)
  	sethi.p		%hi(0xffc00100),gr30
  	setlo		%lo(0xffc00100),gr30
  	sth		gr30,@(gr30,gr0)
  	membar
  #endif
  
  	movsg		ccr,gr30			/* save CCR */
  	movgs		gr30,scr2
  
  	# see if the cached page table mapping is appropriate
  	srlicc.p	gr31,#26,gr0,icc0
  	setlos		0x3ffc,gr30
  	srli.p		gr29,#12,gr31			/* use EAR0[25:14] as PTE index */
  	bne		icc0,#0,__dtlb_k_PTD_miss
  
  __dtlb_k_PTD_mapped:
  	# access the PTD with EAR0[25:14]
  	# - DAMLR5 points to the virtual address of the appropriate page table
  	# - the PTD holds 4096 PTEs
  	# - the PTD must be accessed uncached
  	# - the PTE must be marked accessed if it was valid
  	#
  	and		gr31,gr30,gr31
  	movsg		damlr5,gr30
  	add		gr30,gr31,gr31
  	ldi		@(gr31,#0),gr30			/* fetch the PTE */
  	andicc		gr30,#_PAGE_PRESENT,gr0,icc0
  	ori.p		gr30,#_PAGE_ACCESSED,gr30
  	beq		icc0,#0,__tlb_kernel_fault	/* jump if PTE invalid */
  	sti.p		gr30,@(gr31,#0)			/* update the PTE */
  	andi		gr30,#~_PAGE_ACCESSED,gr30
  
  	# we're using DAMR1 as an extra TLB entry
  	# - punt the entry here (if valid) to the real TLB and then replace with the new PTE
  	# - need to check IAMR1 lest we cause an multiple-DAT-hit exception
  	movsg		dampr1,gr31
  	andicc		gr31,#xAMPRx_V,gr0,icc0
  	setlos.p	0xfffff000,gr31
  	beq		icc0,#0,__dtlb_k_nopunt		/* punt not required */
  
  	movsg		damlr1,gr31
  	movgs		gr31,tplr			/* set TPLR.CXN */
  	tlbpr		gr31,gr0,#4,#0			/* delete matches from TLB, IAMR1, DAMR1 */
  
  	movsg		dampr1,gr31
  	ori		gr31,#xAMPRx_V,gr31		/* entry was invalidated by tlbpr #4 */
  	movgs		gr31,tppr
  	movsg		damlr1,gr31			/* set TPLR.CXN */
  	movgs		gr31,tplr
  	tlbpr		gr31,gr0,#2,#0			/* save to the TLB */
  	movsg		tpxr,gr31			/* check the TLB write error flag */
  	andicc.p	gr31,#TPXR_E,gr0,icc0
  	setlos		#0xfffff000,gr31
  	bne		icc0,#0,__tlb_kernel_fault
  
  __dtlb_k_nopunt:
  
  	# assemble the new TLB entry
  	and		gr29,gr31,gr29
  	movsg		cxnr,gr31
  	or		gr29,gr31,gr29
  	movgs		gr29,iamlr1			/* xAMLR = address | context number */
  	movgs		gr30,iampr1
  	movgs		gr29,damlr1
  	movgs		gr30,dampr1
  
  	# return, restoring registers
  	movsg		scr2,gr30
  	movgs		gr30,ccr
  	sethi.p		%hi(__kernel_current_task),gr29
  	setlo		%lo(__kernel_current_task),gr29
  	ldi		@(gr29,#0),gr29
  	rett		#0
  	beq		icc0,#3,0			/* prevent icache prefetch */
  
  	# the PTE we want wasn't in the PTD we have mapped, so we need to go looking for a more
  	# appropriate page table and map that instead
  	#   - access the PGD with EAR0[31:26]
  	#   - DAMLR3 points to the virtual address of the page directory
  	#   - the PGD holds 64 PGEs and each PGE/PME points to a set of page tables
  __dtlb_k_PTD_miss:
  	srli		gr29,#26,gr31			/* calculate PGE offset */
  	slli		gr31,#8,gr31			/* and clear bottom bits */
  
  	movsg		damlr3,gr30
  	ld		@(gr31,gr30),gr30		/* access the PGE */
  
  	andicc.p	gr30,#_PAGE_PRESENT,gr0,icc0
  	andicc		gr30,#xAMPRx_SS,gr0,icc1
  
  	# map this PTD instead and record coverage address
  	ori.p		gr30,#xAMPRx_L|xAMPRx_SS_16Kb|xAMPRx_S|xAMPRx_C|xAMPRx_V,gr30
  	beq		icc0,#0,__tlb_kernel_fault	/* jump if PGE not present */
  	slli.p		gr31,#18,gr31
  	bne		icc1,#0,__dtlb_k_bigpage
  	movgs		gr30,dampr5
  	movgs		gr31,scr1
  
  	# we can now resume normal service
  	setlos		0x3ffc,gr30
  	srli.p		gr29,#12,gr31			/* use EAR0[25:14] as PTE index */
  	bra		__dtlb_k_PTD_mapped
  
  __dtlb_k_bigpage:
  	break
  	nop
  
  	.size		__entry_kernel_data_tlb_miss, .-__entry_kernel_data_tlb_miss
  
  ###############################################################################
  #
  # Userspace instruction TLB miss handler (with PGE prediction)
  # On entry:
  #   GR28  - faulting address
  #   GR31  - EAR0 ^ SCR0
  #   SCR0  - base of virtual range covered by cached PGE from last ITLB miss (or 0xffffffff)
  #   DAMR3 - mapped page directory
  #   DAMR4 - mapped page table as matched by SCR0
  #
  ###############################################################################
  	.globl		__entry_user_insn_tlb_miss
  	.type		__entry_user_insn_tlb_miss,@function
  __entry_user_insn_tlb_miss:
  #if 0
  	sethi.p		%hi(0xe1200004),gr30
  	setlo		%lo(0xe1200004),gr30
  	st		gr0,@(gr30,gr0)
  	sethi.p		%hi(0xffc00100),gr30
  	setlo		%lo(0xffc00100),gr30
  	sth		gr30,@(gr30,gr0)
  	membar
  #endif
  
  	movsg		ccr,gr30			/* save CCR */
  	movgs		gr30,scr2
  
  	# see if the cached page table mapping is appropriate
  	srlicc.p	gr31,#26,gr0,icc0
  	setlos		0x3ffc,gr30
  	srli.p		gr28,#12,gr31			/* use EAR0[25:14] as PTE index */
  	bne		icc0,#0,__itlb_u_PTD_miss
  
  __itlb_u_PTD_mapped:
  	# access the PTD with EAR0[25:14]
  	# - DAMLR4 points to the virtual address of the appropriate page table
  	# - the PTD holds 4096 PTEs
  	# - the PTD must be accessed uncached
  	# - the PTE must be marked accessed if it was valid
  	#
  	and		gr31,gr30,gr31
  	movsg		damlr4,gr30
  	add		gr30,gr31,gr31
  	ldi		@(gr31,#0),gr30			/* fetch the PTE */
  	andicc		gr30,#_PAGE_PRESENT,gr0,icc0
  	ori.p		gr30,#_PAGE_ACCESSED,gr30
  	beq		icc0,#0,__tlb_user_fault	/* jump if PTE invalid */
  	sti.p		gr30,@(gr31,#0)			/* update the PTE */
  	andi		gr30,#~_PAGE_ACCESSED,gr30
  
  	# we're using IAMR1/DAMR1 as an extra TLB entry
  	# - punt the entry here (if valid) to the real TLB and then replace with the new PTE
  	movsg		dampr1,gr31
  	andicc		gr31,#xAMPRx_V,gr0,icc0
  	setlos.p	0xfffff000,gr31
  	beq		icc0,#0,__itlb_u_nopunt		/* punt not required */
  
  	movsg		dampr1,gr31
  	movgs		gr31,tppr
  	movsg		damlr1,gr31			/* set TPLR.CXN */
  	movgs		gr31,tplr
  	tlbpr		gr31,gr0,#2,#0			/* save to the TLB */
  	movsg		tpxr,gr31			/* check the TLB write error flag */
  	andicc.p	gr31,#TPXR_E,gr0,icc0
  	setlos		#0xfffff000,gr31
  	bne		icc0,#0,__tlb_user_fault
  
  __itlb_u_nopunt:
  
  	# assemble the new TLB entry
  	and		gr28,gr31,gr28
  	movsg		cxnr,gr31
  	or		gr28,gr31,gr28
  	movgs		gr28,iamlr1			/* xAMLR = address | context number */
  	movgs		gr30,iampr1
  	movgs		gr28,damlr1
  	movgs		gr30,dampr1
  
  	# return, restoring registers
  	movsg		scr2,gr30
  	movgs		gr30,ccr
  	rett		#0
  	beq		icc0,#3,0			/* prevent icache prefetch */
  
  	# the PTE we want wasn't in the PTD we have mapped, so we need to go looking for a more
  	# appropriate page table and map that instead
  	#   - access the PGD with EAR0[31:26]
  	#   - DAMLR3 points to the virtual address of the page directory
  	#   - the PGD holds 64 PGEs and each PGE/PME points to a set of page tables
  __itlb_u_PTD_miss:
  	srli		gr28,#26,gr31			/* calculate PGE offset */
  	slli		gr31,#8,gr31			/* and clear bottom bits */
  
  	movsg		damlr3,gr30
  	ld		@(gr31,gr30),gr30		/* access the PGE */
  
  	andicc.p	gr30,#_PAGE_PRESENT,gr0,icc0
  	andicc		gr30,#xAMPRx_SS,gr0,icc1
  
  	# map this PTD instead and record coverage address
  	ori.p		gr30,#xAMPRx_L|xAMPRx_SS_16Kb|xAMPRx_S|xAMPRx_C|xAMPRx_V,gr30
  	beq		icc0,#0,__tlb_user_fault	/* jump if PGE not present */
  	slli.p		gr31,#18,gr31
  	bne		icc1,#0,__itlb_u_bigpage
  	movgs		gr30,dampr4
  	movgs		gr31,scr0
  
  	# we can now resume normal service
  	setlos		0x3ffc,gr30
  	srli.p		gr28,#12,gr31			/* use EAR0[25:14] as PTE index */
  	bra		__itlb_u_PTD_mapped
  
  __itlb_u_bigpage:
  	break
  	nop
  
  	.size		__entry_user_insn_tlb_miss, .-__entry_user_insn_tlb_miss
  
  ###############################################################################
  #
  # Userspace data TLB miss handler
  # On entry:
  #   GR28  - faulting address
  #   GR31  - EAR0 ^ SCR1
  #   SCR1  - base of virtual range covered by cached PGE from last DTLB miss (or 0xffffffff)
  #   DAMR3 - mapped page directory
  #   DAMR5 - mapped page table as matched by SCR1
  #
  ###############################################################################
  	.globl		__entry_user_data_tlb_miss
  	.type		__entry_user_data_tlb_miss,@function
  __entry_user_data_tlb_miss:
  #if 0
  	sethi.p		%hi(0xe1200004),gr30
  	setlo		%lo(0xe1200004),gr30
  	st		gr0,@(gr30,gr0)
  	sethi.p		%hi(0xffc00100),gr30
  	setlo		%lo(0xffc00100),gr30
  	sth		gr30,@(gr30,gr0)
  	membar
  #endif
  
  	movsg		ccr,gr30			/* save CCR */
  	movgs		gr30,scr2
  
  	# see if the cached page table mapping is appropriate
  	srlicc.p	gr31,#26,gr0,icc0
  	setlos		0x3ffc,gr30
  	srli.p		gr28,#12,gr31			/* use EAR0[25:14] as PTE index */
  	bne		icc0,#0,__dtlb_u_PTD_miss
  
  __dtlb_u_PTD_mapped:
  	# access the PTD with EAR0[25:14]
  	# - DAMLR5 points to the virtual address of the appropriate page table
  	# - the PTD holds 4096 PTEs
  	# - the PTD must be accessed uncached
  	# - the PTE must be marked accessed if it was valid
  	#
  	and		gr31,gr30,gr31
  	movsg		damlr5,gr30
  
  __dtlb_u_using_iPTD:
  	add		gr30,gr31,gr31
  	ldi		@(gr31,#0),gr30			/* fetch the PTE */
  	andicc		gr30,#_PAGE_PRESENT,gr0,icc0
  	ori.p		gr30,#_PAGE_ACCESSED,gr30
  	beq		icc0,#0,__tlb_user_fault	/* jump if PTE invalid */
  	sti.p		gr30,@(gr31,#0)			/* update the PTE */
  	andi		gr30,#~_PAGE_ACCESSED,gr30
  
  	# we're using DAMR1 as an extra TLB entry
  	# - punt the entry here (if valid) to the real TLB and then replace with the new PTE
  	movsg		dampr1,gr31
  	andicc		gr31,#xAMPRx_V,gr0,icc0
  	setlos.p	0xfffff000,gr31
  	beq		icc0,#0,__dtlb_u_nopunt		/* punt not required */
  
  	movsg		dampr1,gr31
  	movgs		gr31,tppr
  	movsg		damlr1,gr31			/* set TPLR.CXN */
  	movgs		gr31,tplr
  	tlbpr		gr31,gr0,#2,#0			/* save to the TLB */
  	movsg		tpxr,gr31			/* check the TLB write error flag */
  	andicc.p	gr31,#TPXR_E,gr0,icc0
  	setlos		#0xfffff000,gr31
  	bne		icc0,#0,__tlb_user_fault
  
  __dtlb_u_nopunt:
  
  	# assemble the new TLB entry
  	and		gr28,gr31,gr28
  	movsg		cxnr,gr31
  	or		gr28,gr31,gr28
  	movgs		gr28,iamlr1			/* xAMLR = address | context number */
  	movgs		gr30,iampr1
  	movgs		gr28,damlr1
  	movgs		gr30,dampr1
  
  	# return, restoring registers
  	movsg		scr2,gr30
  	movgs		gr30,ccr
  	rett		#0
  	beq		icc0,#3,0			/* prevent icache prefetch */
  
  	# the PTE we want wasn't in the PTD we have mapped, so we need to go looking for a more
  	# appropriate page table and map that instead
  	#   - first of all, check the insn PGE cache - we may well get a hit there
  	#   - access the PGD with EAR0[31:26]
  	#   - DAMLR3 points to the virtual address of the page directory
  	#   - the PGD holds 64 PGEs and each PGE/PME points to a set of page tables
  __dtlb_u_PTD_miss:
  	movsg		scr0,gr31			/* consult the insn-PGE-cache key */
  	xor		gr28,gr31,gr31
  	srlicc		gr31,#26,gr0,icc0
  	srli		gr28,#12,gr31			/* use EAR0[25:14] as PTE index */
  	bne		icc0,#0,__dtlb_u_iPGE_miss
  
  	# what we're looking for is covered by the insn-PGE-cache
  	setlos		0x3ffc,gr30
  	and		gr31,gr30,gr31
  	movsg		damlr4,gr30
  	bra		__dtlb_u_using_iPTD
  
  __dtlb_u_iPGE_miss:
  	srli		gr28,#26,gr31			/* calculate PGE offset */
  	slli		gr31,#8,gr31			/* and clear bottom bits */
  
  	movsg		damlr3,gr30
  	ld		@(gr31,gr30),gr30		/* access the PGE */
  
  	andicc.p	gr30,#_PAGE_PRESENT,gr0,icc0
  	andicc		gr30,#xAMPRx_SS,gr0,icc1
  
  	# map this PTD instead and record coverage address
  	ori.p		gr30,#xAMPRx_L|xAMPRx_SS_16Kb|xAMPRx_S|xAMPRx_C|xAMPRx_V,gr30
  	beq		icc0,#0,__tlb_user_fault	/* jump if PGE not present */
  	slli.p		gr31,#18,gr31
  	bne		icc1,#0,__dtlb_u_bigpage
  	movgs		gr30,dampr5
  	movgs		gr31,scr1
  
  	# we can now resume normal service
  	setlos		0x3ffc,gr30
  	srli.p		gr28,#12,gr31			/* use EAR0[25:14] as PTE index */
  	bra		__dtlb_u_PTD_mapped
  
  __dtlb_u_bigpage:
  	break
  	nop
  
  	.size		__entry_user_data_tlb_miss, .-__entry_user_data_tlb_miss