/* $OpenBSD: tlbhandler.S,v 1.16 2007/05/25 20:58:39 miod Exp $ */ /* * Copyright (c) 1995-2004 Opsycon AB (www.opsycon.se / www.opsycon.com) * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ /* * This code handles TLB exceptions and updates. */ #include #include #include #include #include #include #include #define HAIRY_R5000_ERRATA #include "assym.h" .set mips3 .set noreorder /* Default reorder mode */ /*---------------------------------------------------------------- tlb_miss * Low level TLB exception handler. TLB and XTLB share some * code for the moment and is copied down at the same time. * This code must be PIC and not more than 64 instructions * for both TLB and XTLB handling or it will overflow the * available storage. If the startup code finds out that it * is larger, the trampoline code is copied instead of panicing. */ /***************************** Start of code copied to exception vector */ .globl tlb_miss /* 0xffffffff80000000 */ .set noat .ent tlb_miss, 0 tlb_miss: #ifdef HAIRY_R5000_ERRATA /* * R5000 errata: edge cases can trigger a TLB miss exception * instead of an invalid TLB exception. Servicing the TLB miss * exception would cause a duplicate TLB to be inserted, which * causes the processor operation to become unpredictable (but * very bad for the kernel's health, ten times out of nine). * * More details about the problem can be found in: * http://www.linux-mips.org/archives/linux-mips/2000-02/msg00040.html * * We work around the issue by checking for an existing TLB entry, * and handling this as an invalid TLB exception (as it was intended * to be), in this case. */ tlbp mfc0 k1, COP_0_TLB_INDEX bltz k1, 1f # missing! nop j k_tlb_inv nop 1: #endif PTR_L k1, curprocpaddr dmfc0 k0, COP_0_BAD_VADDR bltz k0, _k_miss # kernel address space PTR_SRL k0, k0, SEGSHIFT - LOGREGSZ PTR_L k1, PCB_SEGTAB(k1) andi k0, k0, (PMAP_SEGTABSIZE - 1) << LOGREGSZ PTR_ADDU k1, k1, k0 PTR_L k1, 0(k1) # get pointer to page table dmfc0 k0, COP_0_BAD_VADDR PTR_SRL k0, k0, PGSHIFT - 2 andi k0, k0, ((NPTEPG/2) - 1) << 3 beqz k1, _inv_seg # invalid segment map PTR_ADDU k1, k1, k0 # index into segment map lw k0, 0(k1) # get page PTE tlb_load: lw k1, 4(k1) dsll k0, k0, 34 dsrl k0, k0, 34 dmtc0 k0, COP_0_TLB_LO0 dsll k1, k1, 34 dsrl k1, k1, 34 dmtc0 k1, COP_0_TLB_LO1 nop # RM7000 needs 4 nops nop nop nop tlbwr # update TLB nop nop nop nop eret # RM7000 need 4 for JTLB usage. .end tlb_miss .globl e_tlb_miss e_tlb_miss: /*---------------------------------------------------------------- xtlb_miss * Low level XTLB exception handler. */ .globl xtlb_miss /* 0xffffffff80000080 */ .set noat .ent xtlb_miss, 0 xtlb_miss: #ifdef HAIRY_R5000_ERRATA /* See errata comments in tlb_miss above */ tlbp mfc0 k1, COP_0_TLB_INDEX bltz k1, 1f # missing! nop j k_tlb_inv nop 1: #endif dmfc0 k0, COP_0_BAD_VADDR bltz k0, _k_miss # kernel address space PTR_SRL k0, k0, SEGSHIFT sltiu k1, k0, PMAP_SEGTABSIZE beqz k1, _inv_seg # wrong if outside pm_segtab PTR_SLL k0, k0, LOGREGSZ PTR_L k1, curprocpaddr PTR_L k1, PCB_SEGTAB(k1) PTR_ADDU k1, k1, k0 PTR_L k1, 0(k1) # get pointer to page table dmfc0 k0, COP_0_BAD_VADDR PTR_SRL k0, k0, PGSHIFT - 2 andi k0, k0, ((NPTEPG/2) - 1) << 3 beqz k1, _inv_seg PTR_ADDU k1, k1, k0 b tlb_load # rest is same as 'tlb_miss' lw k0, 0(k1) _inv_seg: j tlb_miss_nopt # No page table for this segment. nop _k_miss: j k_tlb_miss # kernel tlbmiss. dmfc0 k0, COP_0_BAD_VADDR # must reload. .end xtlb_miss .globl e_xtlb_miss e_xtlb_miss: .set at /***************************** End of code copied to exception vector */ .globl tlb_miss_nopt .ent tlb_miss_nopt, 0 tlb_miss_nopt: .set noat mfc0 k0, COP_0_STATUS_REG andi k0, SR_KSU_USER bne k0, zero, go_u_general nop j k_general nop .end tlb_miss_nopt .set at /* * Trampolines copied to exception vectors when code is too big. */ .globl tlb_miss_tramp .ent tlb_miss_tramp, 0 tlb_miss_tramp: .set noat LA k0, tlb_miss jr k0 nop .end tlb_miss_tramp .set at .globl e_tlb_miss_tramp e_tlb_miss_tramp: .globl xtlb_miss_tramp .ent xtlb_miss_tramp, 0 xtlb_miss_tramp: .set noat LA k0, xtlb_miss jr k0 nop .end xtlb_miss_tramp .set at .globl e_xtlb_miss_tramp e_xtlb_miss_tramp: /*---------------------------------------------------------------- k_tlb_inv * Handle a TLB invalid exception from kernel mode in kernel * space. This happens when we have a TLB match but an invalid * entry. Try to reload. */ NLEAF(k_tlb_inv, 0) .set noat LA k1, (VM_MIN_KERNEL_ADDRESS) # compute index dmfc0 k0, COP_0_BAD_VADDR # get the fault address PTR_SUBU k0, k0, k1 lw k1, Sysmapsize # index within range? PTR_SRL k0, k0, PGSHIFT sltu k1, k0, k1 beq k1, zero, sys_stk_chk # No. check for valid stack nop PTR_L k1, Sysmap PTR_SLL k0, k0, 2 # compute offset from index tlbp # Probe the invalid entry PTR_ADDU k1, k1, k0 and k0, k0, 4 # check even/odd page bne k0, zero, k_tlb_inv_odd nop mfc0 k0, COP_0_TLB_INDEX blez k0, sys_stk_chk # probe fail or index 0! lw k0, 0(k1) # get PTE entry dsll k0, k0, 34 # get rid of "wired" bit dsrl k0, k0, 34 dmtc0 k0, COP_0_TLB_LO0 # load PTE entry and k0, k0, PG_V # check for valid entry beq k0, zero, go_k_general # PTE invalid lw k0, 4(k1) # get odd PTE entry dsll k0, k0, 34 dsrl k0, k0, 34 dmtc0 k0, COP_0_TLB_LO1 # load PTE entry nop nop nop nop tlbwi # write TLB nop nop nop nop eret k_tlb_inv_odd: mfc0 k0, COP_0_TLB_INDEX blez k0, sys_stk_chk # probe fail or index 0! lw k0, 0(k1) # get PTE entry dsll k0, k0, 34 # get rid of wired bit dsrl k0, k0, 34 dmtc0 k0, COP_0_TLB_LO1 # save PTE entry and k0, k0, PG_V # check for valid entry beq k0, zero, go_k_general # PTE invalid lw k0, -4(k1) # get even PTE entry dsll k0, k0, 34 dsrl k0, k0, 34 dmtc0 k0, COP_0_TLB_LO0 # save PTE entry nop nop nop nop tlbwi # update TLB nop nop nop nop eret END(k_tlb_inv) /*---------------------------------------------------------------- k_tlb_miss * * Handle a TLB miss exception from kernel mode in kernel space. * We must check that this is coming from kernel mode. If not * it's a bad address from user mode so handle properly. * Load up the correct entry contents from the kernel map. * k0 has bad address. */ NLEAF(k_tlb_miss, 0) .set noat mfc0 k1, COP_0_STATUS_REG andi k1, SR_KSU_USER bne k1, zero, go_u_general LA k1, (VM_MIN_KERNEL_ADDRESS) # compute index PTR_SUBU k0, k0, k1 lw k1, Sysmapsize # index within range? PTR_SRL k0, k0, PGSHIFT sltu k1, k0, k1 beq k1, zero, sys_stk_chk # No. check for valid stack PTR_SRL k0, k0, 1 PTR_L k1, Sysmap PTR_SLL k0, k0, 3 # compute offset from index PTR_ADDU k1, k1, k0 lw k0, 0(k1) # get PTE entry lw k1, 4(k1) # get odd PTE entry dsll k0, k0, 34 # get rid of "wired" bit dsrl k0, k0, 34 dmtc0 k0, COP_0_TLB_LO0 # load PTE entry dsll k1, k1, 34 dsrl k1, k1, 34 dmtc0 k1, COP_0_TLB_LO1 # load PTE entry nop nop nop nop tlbwr # write TLB nop nop nop nop eret sys_stk_chk: PTR_L k1, curprocpaddr PTR_SUBU k0, sp, k1 # check to see if we have a sltiu k0, 2048 # valid kernel stack beqz k0, go_k_general # yes, handle. nop LA a0, start-FRAMESZ(CF_SZ)-4*REGSZ # set sp to a valid place #ifdef __mips_n64 mfc0 a4, COP_0_STATUS_REG mfc0 a5, COP_0_CAUSE_REG move a6, sp #else mfc0 a2, COP_0_STATUS_REG mfc0 a3, COP_0_CAUSE_REG REG_S a2, CF_ARGSZ+0*REGSZ(sp) REG_S a3, CF_ARGSZ+1*REGSZ(sp) PTR_S sp, CF_ARGSZ+2*REGSZ(a0) #endif move sp, a0 dmfc0 a1, COP_0_EXC_PC move a2, ra LA a0, 1f jal printf dmfc0 a3, COP_0_BAD_VADDR LA sp, start-FRAMESZ(CF_SZ) # set sp to a valid place #ifdef DDB LA a0, 2f jal trapDump nop #endif PANIC("kernel stack overflow") /*noreturn*/ go_k_general: j k_general nop go_u_general: j u_general nop .data 1: .asciiz "\rktlbmiss: PC %p RA %p ADR %p\nSR %p CR %p SP %p\n" 2: .asciiz "stack ovf" .text .set at END(k_tlb_miss) /*---------------------------------------------------------------- tlb_write_i * Write the given entry into the TLB at the given index. */ LEAF(tlb_write_indexed, 0) mfc0 v1, COP_0_STATUS_REG # Save the status register. ori v0, v1, SR_INT_ENAB xori v0, v0, SR_INT_ENAB mtc0 v0, COP_0_STATUS_REG # Disable interrupts ITLBNOPFIX ld a2, 16(a1) ld a3, 24(a1) dmfc0 ta0, COP_0_TLB_HI # Save the current PID. dmtc0 a2, COP_0_TLB_LO0 # Set up entry low0. dmtc0 a3, COP_0_TLB_LO1 # Set up entry low1. ld a2, 0(a1) ld a3, 8(a1) mtc0 a0, COP_0_TLB_INDEX # Set the index. dmtc0 a2, COP_0_TLB_PG_MASK # Set up entry mask. dmtc0 a3, COP_0_TLB_HI # Set up entry high. nop nop nop nop tlbwi # Write the TLB nop nop # Delay for effect nop nop dmtc0 ta0, COP_0_TLB_HI # Restore the PID. nop dmtc0 zero, COP_0_TLB_PG_MASK # Default mask value. mtc0 v1, COP_0_STATUS_REG # Restore the status register ITLBNOPFIX j ra nop END(tlb_write_indexed) /*---------------------------------------------------------------- tlb_flush * Flush the "random" entries from the TLB. * Uses "wired" register to determine what register to start with. * Arg "tlbsize" is the number of entries to flush. */ LEAF(tlb_flush, 0) mfc0 v1, COP_0_STATUS_REG # Save the status register. ori v0, v1, SR_INT_ENAB xori v0, v0, SR_INT_ENAB mtc0 v0, COP_0_STATUS_REG # Disable interrupts ITLBNOPFIX mfc0 ta1, COP_0_TLB_WIRED LA v0, KSEG0_BASE # invalid address dmfc0 ta0, COP_0_TLB_HI # Save the PID dmtc0 v0, COP_0_TLB_HI # Mark entry high as invalid dmtc0 zero, COP_0_TLB_LO0 # Zero out low entry0. dmtc0 zero, COP_0_TLB_LO1 # Zero out low entry1. mtc0 zero, COP_0_TLB_PG_MASK # Zero out mask entry. /* * Align the starting value (ta1) and the upper bound (a0). */ 1: mtc0 ta1, COP_0_TLB_INDEX # Set the index register. addu ta1, ta1, 1 # Increment index. nop nop nop tlbwi # Write the TLB entry. nop nop bne ta1, a0, 1b nop dmtc0 ta0, COP_0_TLB_HI # Restore the PID mtc0 v1, COP_0_STATUS_REG # Restore the status register ITLBNOPFIX j ra nop END(tlb_flush) /*--------------------------------------------------------------- tlb_flush_addr * Flush any TLB entries for the given address and TLB PID. */ LEAF(tlb_flush_addr, 0) mfc0 v1, COP_0_STATUS_REG # Save the status register. ori v0, v1, SR_INT_ENAB xori v0, v0, SR_INT_ENAB mtc0 v0, COP_0_STATUS_REG # Disable interrupts ITLBNOPFIX li v0, (PG_HVPN | PG_ASID) and a0, a0, v0 # Make sure valid hi value. dmfc0 ta0, COP_0_TLB_HI # Get current PID dmtc0 a0, COP_0_TLB_HI # look for addr & PID nop nop nop nop tlbp # Probe for the entry. nop nop # Delay for effect nop LA ta1, KSEG0_BASE # Load invalid entry. mfc0 v0, COP_0_TLB_INDEX # See what we got bltz v0, 1f # index < 0 => !found nop dmtc0 ta1, COP_0_TLB_HI # Mark entry high as invalid dmtc0 zero, COP_0_TLB_LO0 # Zero out low entry. dmtc0 zero, COP_0_TLB_LO1 # Zero out low entry. nop nop nop nop tlbwi nop nop nop nop 1: dmtc0 ta0, COP_0_TLB_HI # restore PID mtc0 v1, COP_0_STATUS_REG # Restore the status register ITLBNOPFIX j ra nop END(tlb_flush_addr) /*---------------------------------------------------------------- tlb_update * Update the TLB if highreg is found; otherwise, enter the data. */ LEAF(tlb_update, 0) mfc0 v1, COP_0_STATUS_REG # Save the status register. ori v0, v1, SR_INT_ENAB xori v0, v0, SR_INT_ENAB mtc0 v0, COP_0_STATUS_REG # Disable interrupts ITLBNOPFIX and ta1, a0, 0x1000 # ta1 = Even/Odd flag li v0, (PG_HVPN | PG_ASID) and a0, a0, v0 dmfc0 ta0, COP_0_TLB_HI # Save current PID dmtc0 a0, COP_0_TLB_HI # Init high reg and a2, a1, PG_G # Copy global bit nop nop nop tlbp # Probe for the entry. dsll a1, a1, 34 dsrl a1, a1, 34 bne ta1, zero, 2f # Decide even odd mfc0 v0, COP_0_TLB_INDEX # See what we got # EVEN bltz v0, 1f # index < 0 => !found nop tlbr # update, read entry first nop nop nop dmtc0 a1, COP_0_TLB_LO0 # init low reg0. nop nop nop nop tlbwi # update slot found b 4f li v0, 1 1: mtc0 zero, COP_0_TLB_PG_MASK # init mask. dmtc0 a0, COP_0_TLB_HI # init high reg. dmtc0 a1, COP_0_TLB_LO0 # init low reg0. dmtc0 a2, COP_0_TLB_LO1 # init low reg1. nop nop nop nop tlbwr # enter into a random slot b 4f li v0, 0 # ODD 2: nop bltz v0, 3f # index < 0 => !found nop tlbr # read the entry first nop nop nop dmtc0 a1, COP_0_TLB_LO1 # init low reg1. nop nop nop nop tlbwi # update slot found b 4f li v0, 1 3: mtc0 zero, COP_0_TLB_PG_MASK # init mask. dmtc0 a0, COP_0_TLB_HI # init high reg. dmtc0 a2, COP_0_TLB_LO0 # init low reg0. dmtc0 a1, COP_0_TLB_LO1 # init low reg1. nop nop nop nop tlbwr # enter into a random slot nop li v0, 0 4: # Make sure pipeline nop # advances before we nop # use the tlb. dmtc0 ta0, COP_0_TLB_HI # restore PID mtc0 v1, COP_0_STATUS_REG # Restore the status register ITLBNOPFIX j ra nop END(tlb_update) /*---------------------------------------------------------------- tlb_read * Read the TLB entry. */ LEAF(tlb_read, 0) mfc0 v1, COP_0_STATUS_REG # Save the status register. ori v0, v1, SR_INT_ENAB xori v0, v0, SR_INT_ENAB mtc0 v0, COP_0_STATUS_REG # Disable interrupts ITLBNOPFIX dmfc0 v0, COP_0_TLB_HI # Get current PID mtc0 a0, COP_0_TLB_INDEX # Set the index register nop nop nop nop tlbr # Read from the TLB nop nop nop dmfc0 ta0, COP_0_TLB_PG_MASK # fetch the hi entry dmfc0 ta1, COP_0_TLB_HI # fetch the hi entry dmfc0 ta2, COP_0_TLB_LO0 # See what we got dmfc0 ta3, COP_0_TLB_LO1 # See what we got dmtc0 v0, COP_0_TLB_HI # restore PID nop nop nop # wait for PID active mtc0 v1, COP_0_STATUS_REG # Restore the status register ITLBNOPFIX sd ta0, 0(a1) sd ta1, 8(a1) sd ta2, 16(a1) j ra sd ta3, 24(a1) END(tlb_read) /*---------------------------------------------------------------- tlb_get_pid * Read the tlb pid value. */ LEAF(tlb_get_pid, 0) dmfc0 v0, COP_0_TLB_HI # get PID li v1, VMTLB_PID # mask off PID j ra and v0, v0, v1 # mask off PID END(tlb_get_pid) /*---------------------------------------------------------------- tlb_set_pid * Write the given pid into the TLB pid reg. */ LEAF(tlb_set_pid, 0) dmtc0 a0, COP_0_TLB_HI # Write the hi reg value j ra nop END(tlb_set_pid) /*---------------------------------------------------------------- tlb_get_wired * Get the value from the TLB wired reg. */ LEAF(tlb_get_wired, 0) mfc0 v0, COP_0_TLB_WIRED j ra nop END(tlb_get_wired) /*---------------------------------------------------------------- tlb_set_wired * Write the given value into the TLB wired reg. */ LEAF(tlb_set_wired, 0) mtc0 a0, COP_0_TLB_WIRED j ra nop END(tlb_set_wired)