/*-
* Copyright (c) 2015-2016 Ruslan Bukin
* All rights reserved.
*
* Portions of this software were developed by SRI International and the
* University of Cambridge Computer Laboratory under DARPA/AFRL contract
* FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme.
*
* Portions of this software were developed by the University of Cambridge
* Computer Laboratory as part of the CTSRD Project, with support from the
* UK Higher Education Innovation Fund (HEIF).
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include
__FBSDID("$FreeBSD$");
#include "assym.s"
#include
#include
.macro save_registers el
addi sp, sp, -(TF_SIZE)
sd ra, (TF_RA)(sp)
sd tp, (TF_TP)(sp)
.if \el == 0 /* We came from userspace. Load our pcpu */
sd gp, (TF_GP)(sp)
ld gp, (TF_SIZE)(sp)
.endif
sd t0, (TF_T + 0 * 8)(sp)
sd t1, (TF_T + 1 * 8)(sp)
sd t2, (TF_T + 2 * 8)(sp)
sd t3, (TF_T + 3 * 8)(sp)
sd t4, (TF_T + 4 * 8)(sp)
sd t5, (TF_T + 5 * 8)(sp)
sd t6, (TF_T + 6 * 8)(sp)
sd s0, (TF_S + 0 * 8)(sp)
sd s1, (TF_S + 1 * 8)(sp)
sd s2, (TF_S + 2 * 8)(sp)
sd s3, (TF_S + 3 * 8)(sp)
sd s4, (TF_S + 4 * 8)(sp)
sd s5, (TF_S + 5 * 8)(sp)
sd s6, (TF_S + 6 * 8)(sp)
sd s7, (TF_S + 7 * 8)(sp)
sd s8, (TF_S + 8 * 8)(sp)
sd s9, (TF_S + 9 * 8)(sp)
sd s10, (TF_S + 10 * 8)(sp)
sd s11, (TF_S + 11 * 8)(sp)
sd a0, (TF_A + 0 * 8)(sp)
sd a1, (TF_A + 1 * 8)(sp)
sd a2, (TF_A + 2 * 8)(sp)
sd a3, (TF_A + 3 * 8)(sp)
sd a4, (TF_A + 4 * 8)(sp)
sd a5, (TF_A + 5 * 8)(sp)
sd a6, (TF_A + 6 * 8)(sp)
sd a7, (TF_A + 7 * 8)(sp)
#if 0
/* XXX: temporary test: spin if stack is not kernel one */
.if \el == 1 /* kernel */
mv t0, sp
srli t0, t0, 63
1:
beqz t0, 1b
.endif
#endif
.if \el == 1
/* Store kernel sp */
sd sp, (TF_SP)(sp)
.else
/* Store user sp */
csrr t0, sscratch
sd t0, (TF_SP)(sp)
.endif
li t0, 0
csrw sscratch, t0
csrr t0, sepc
sd t0, (TF_SEPC)(sp)
csrr t0, sstatus
sd t0, (TF_SSTATUS)(sp)
csrr t0, sbadaddr
sd t0, (TF_SBADADDR)(sp)
csrr t0, scause
sd t0, (TF_SCAUSE)(sp)
.endm
.macro load_registers el
ld t0, (TF_SSTATUS)(sp)
.if \el == 0
/* Ensure user interrupts will be enabled on eret. */
ori t0, t0, SSTATUS_PIE
.else
/*
* Disable interrupts for supervisor mode exceptions.
* For user mode exceptions we have already done this
* in do_ast.
*/
li t1, ~SSTATUS_IE
and t0, t0, t1
.endif
csrw sstatus, t0
ld t0, (TF_SEPC)(sp)
csrw sepc, t0
.if \el == 0
/* We go to userspace. Load user sp */
ld t0, (TF_SP)(sp)
csrw sscratch, t0
/* And store our pcpu */
sd gp, (TF_SIZE)(sp)
ld gp, (TF_GP)(sp)
.endif
ld ra, (TF_RA)(sp)
ld tp, (TF_TP)(sp)
ld t0, (TF_T + 0 * 8)(sp)
ld t1, (TF_T + 1 * 8)(sp)
ld t2, (TF_T + 2 * 8)(sp)
ld t3, (TF_T + 3 * 8)(sp)
ld t4, (TF_T + 4 * 8)(sp)
ld t5, (TF_T + 5 * 8)(sp)
ld t6, (TF_T + 6 * 8)(sp)
ld s0, (TF_S + 0 * 8)(sp)
ld s1, (TF_S + 1 * 8)(sp)
ld s2, (TF_S + 2 * 8)(sp)
ld s3, (TF_S + 3 * 8)(sp)
ld s4, (TF_S + 4 * 8)(sp)
ld s5, (TF_S + 5 * 8)(sp)
ld s6, (TF_S + 6 * 8)(sp)
ld s7, (TF_S + 7 * 8)(sp)
ld s8, (TF_S + 8 * 8)(sp)
ld s9, (TF_S + 9 * 8)(sp)
ld s10, (TF_S + 10 * 8)(sp)
ld s11, (TF_S + 11 * 8)(sp)
ld a0, (TF_A + 0 * 8)(sp)
ld a1, (TF_A + 1 * 8)(sp)
ld a2, (TF_A + 2 * 8)(sp)
ld a3, (TF_A + 3 * 8)(sp)
ld a4, (TF_A + 4 * 8)(sp)
ld a5, (TF_A + 5 * 8)(sp)
ld a6, (TF_A + 6 * 8)(sp)
ld a7, (TF_A + 7 * 8)(sp)
addi sp, sp, (TF_SIZE)
.endm
.macro do_ast
/* Disable interrupts */
csrr a4, sstatus
1:
csrci sstatus, SSTATUS_IE
ld a1, PC_CURTHREAD(gp)
lw a2, TD_FLAGS(a1)
li a3, (TDF_ASTPENDING|TDF_NEEDRESCHED)
and a2, a2, a3
beqz a2, 2f
/* Restore interrupts */
andi a4, a4, SSTATUS_IE
csrs sstatus, a4
/* Handle the ast */
mv a0, sp
call _C_LABEL(ast)
/* Re-check for new ast scheduled */
j 1b
2:
.endm
ENTRY(cpu_exception_handler_supervisor)
save_registers 1
mv a0, sp
call _C_LABEL(do_trap_supervisor)
load_registers 1
eret
END(cpu_exception_handler_supervisor)
ENTRY(cpu_exception_handler_user)
csrrw sp, sscratch, sp
save_registers 0
mv a0, sp
call _C_LABEL(do_trap_user)
do_ast
load_registers 0
csrrw sp, sscratch, sp
eret
END(cpu_exception_handler_user)
/*
* Trap handlers
*/
.text
bad_trap:
j bad_trap
user_trap:
/* Save state */
csrrw sp, mscratch, sp
addi sp, sp, -64
sd t0, (8 * 0)(sp)
sd t1, (8 * 1)(sp)
sd t2, (8 * 2)(sp)
sd t3, (8 * 3)(sp)
sd t4, (8 * 4)(sp)
sd t5, (8 * 5)(sp)
sd a0, (8 * 7)(sp)
la t2, _C_LABEL(cpu_exception_handler_user)
csrr t0, mcause
bltz t0, machine_interrupt
j exit_mrts
supervisor_trap:
/* Save state */
csrrw sp, mscratch, sp
addi sp, sp, -64
sd t0, (8 * 0)(sp)
sd t1, (8 * 1)(sp)
sd t2, (8 * 2)(sp)
sd t3, (8 * 3)(sp)
sd t4, (8 * 4)(sp)
sd t5, (8 * 5)(sp)
sd a0, (8 * 7)(sp)
la t2, _C_LABEL(cpu_exception_handler_supervisor)
csrr t0, mcause
bltz t0, machine_interrupt
li t1, EXCP_SMODE_ENV_CALL
beq t0, t1, supervisor_call
j exit_mrts
machine_interrupt:
/* Type of interrupt ? */
csrr t0, mcause
andi t0, t0, 3
li t1, 0
beq t1, t0, software_interrupt
li t1, 1
beq t1, t0, timer_interrupt
li t1, 2
beq t1, t0, htif_interrupt
/* not reached */
1:
j 1b
software_interrupt:
li t0, MIP_MSIP
csrc mip, t0
li t0, MIP_SSIP
csrs mip, t0
/* If PRV1 is PRV_U (user) then serve the trap */
csrr t0, mstatus
li t1, (MSTATUS_PRV_M << MSTATUS_PRV1_SHIFT)
and t0, t0, t1
beqz t0, 1f
/*
* If PRV1 is supervisor and interrupts were enabled,
* then serve the trap.
*/
csrr t0, mstatus
li t1, (SR_IE1 | (MSTATUS_PRV_M << MSTATUS_PRV1_SHIFT))
and t0, t0, t1
li t1, (SR_IE1 | (MSTATUS_PRV_S << MSTATUS_PRV1_SHIFT))
beq t0, t1, 1f
j exit
1:
/* Serve a trap in supervisor mode */
j exit_mrts
timer_interrupt:
/* Disable machine timer interrupts */
li t0, MIE_MTIE
csrc mie, t0
/* Clear machine pending */
li t0, MIP_MTIP
csrc mip, t0
/* Post supervisor timer interrupt */
li t0, MIP_STIP
csrs mip, t0
/* If PRV1 is PRV_U (user) then serve the trap */
csrr t0, mstatus
li t1, (MSTATUS_PRV_M << MSTATUS_PRV1_SHIFT)
and t0, t0, t1
beqz t0, 1f
/*
* If PRV1 is supervisor and interrupts were enabled,
* then serve the trap.
*/
csrr t0, mstatus
li t1, (SR_IE1 | (MSTATUS_PRV_M << MSTATUS_PRV1_SHIFT))
and t0, t0, t1
li t1, (SR_IE1 | (MSTATUS_PRV_S << MSTATUS_PRV1_SHIFT))
beq t0, t1, 1f
j exit
1:
/* Serve a trap in supervisor mode */
j exit_mrts
htif_interrupt:
1:
li t5, 0
csrrw t5, mfromhost, t5
beqz t5, 3f
/* Console PUT intr ? */
mv t1, t5
li t0, 0x101
srli t1, t1, 48
bne t1, t0, 2f
/* Yes */
la t0, console_intr
li t1, 1
sd t1, 0(t0)
/* Check if there is any other pending event */
j 1b
2:
/* Save entry */
la t0, htif_ring
csrr t1, mhartid
li t4, (HTIF_RING_SIZE + 16)
mulw t4, t4, t1
add t0, t0, t4
li t4, (HTIF_RING_SIZE)
add t0, t0, t4 /* t0 == htif_ring_cursor */
ld t1, 0(t0) /* load ptr to cursor */
sd t5, 0(t1) /* put entry */
li t4, 1
sd t4, 8(t1) /* mark used */
ld t4, 16(t1) /* take next */
/* Update cursor */
sd t4, 0(t0)
/* Post supervisor software interrupt */
li t0, MIP_SSIP
csrs mip, t0
/* Check if there is any other pending event */
j 1b
3:
j exit
supervisor_call:
csrr t1, mepc
addi t1, t1, 4 /* Next instruction in t1 */
li t4, ECALL_HTIF_CMD
beq t5, t4, htif_cmd
li t4, ECALL_HTIF_GET_ENTRY
beq t5, t4, htif_get_entry
li t4, ECALL_MTIMECMP
beq t5, t4, set_mtimecmp
li t4, ECALL_CLEAR_PENDING
beq t5, t4, clear_pending
li t4, ECALL_MCPUID_GET
beq t5, t4, mcpuid_get
li t4, ECALL_MIMPID_GET
beq t5, t4, mimpid_get
li t4, ECALL_SEND_IPI
beq t5, t4, send_ipi
li t4, ECALL_CLEAR_IPI
beq t5, t4, clear_ipi
li t4, ECALL_HTIF_LOWPUTC
beq t5, t4, htif_lowputc
li t4, ECALL_MIE_SET
beq t5, t4, mie_set
j exit_next_instr
mie_set:
csrs mie, t6
j exit_next_instr
mcpuid_get:
csrr t6, mcpuid
j exit_next_instr
mimpid_get:
csrr t6, mimpid
j exit_next_instr
send_ipi:
/* CPU mmio base in t6 */
mv t0, t6
li t2, (CSR_IPI * XLEN)
add t0, t0, t2 /* t0 = CSR_IPI */
li t2, 1
sd t2, 0(t0)
j exit_next_instr
clear_ipi:
/* Do only clear if there are no new entries in HTIF ring */
la t0, htif_ring
csrr t2, mhartid
li t4, (HTIF_RING_SIZE + 16)
mulw t4, t4, t2
add t0, t0, t4
li t4, (HTIF_RING_SIZE)
add t0, t0, t4 /* t0 == ptr to htif_ring_cursor */
ld t2, 8(t0) /* load htif_ring_last */
ld t2, 8(t2) /* load used */
bnez t2, 1f
/* Clear supervisor software interrupt pending bit */
li t0, MIP_SSIP
csrc mip, t0
1:
j exit_next_instr
htif_get_entry:
/* Get a htif_ring for current core */
la t0, htif_ring
csrr t2, mhartid
li t4, (HTIF_RING_SIZE + 16)
mulw t4, t4, t2
add t0, t0, t4
li t4, (HTIF_RING_SIZE + 8)
add t0, t0, t4 /* t0 == htif_ring_last */
/* Check for new entries */
li t6, 0 /* preset return value */
ld t2, 0(t0) /* load ptr to last */
ld t4, 8(t2) /* get used */
beqz t4, 1f /* No new entries. Exit */
/* Get one */
ld t6, 0(t2) /* get entry */
li t4, 0
sd t4, 8(t2) /* mark free */
sd t4, 0(t2) /* free entry, just in case */
ld t4, 16(t2) /* take next */
sd t4, 0(t0) /* update ptr to last */
1:
/* Exit. Result is stored in t6 */
j exit_next_instr
htif_cmd:
1:
mv t0, t6
csrrw t0, mtohost, t0
bnez t0, 1b
j exit_next_instr
htif_lowputc:
1:
mv t0, t6
csrrw t0, mtohost, t0
bnez t0, 1b
2:
li t4, 0
csrrw t5, mfromhost, t4
beqz t5, 2b
/* Console PUT intr ? */
mv t2, t5
srli t2, t2, 48
li t3, 0x0101
beq t2, t3, 3f
/* Not a console PUT, so save entry */
la t0, htif_ring
csrr t2, mhartid
li t4, (HTIF_RING_SIZE + 16)
mulw t4, t4, t2
add t0, t0, t4
li t4, (HTIF_RING_SIZE)
add t0, t0, t4 /* t0 == htif_ring_cursor */
ld t2, 0(t0) /* load ptr to cursor */
sd t5, 0(t2) /* put entry */
li t4, 1
sd t4, 8(t2) /* mark used */
ld t4, 16(t2) /* take next */
/* Update cursor */
sd t4, 0(t0)
/* Post supervisor software interrupt */
li t0, MIP_SSIP
csrs mip, t0
/* Wait for console intr again */
j 2b
3:
j exit_next_instr
set_mtimecmp:
csrr t2, stime
add t6, t6, t2
csrw mtimecmp, t6
/* Enable interrupts */
li t0, (MIE_MTIE | MIE_STIE)
csrs mie, t0
j exit_next_instr
clear_pending:
li t0, MIP_STIP
csrc mip, t0
j exit_next_instr
/*
* Trap exit functions
*/
exit_next_instr:
/* Next instruction is in t1 */
csrw mepc, t1
exit:
/* Restore state */
ld t0, (8 * 0)(sp)
ld t1, (8 * 1)(sp)
ld t2, (8 * 2)(sp)
ld t3, (8 * 3)(sp)
ld t4, (8 * 4)(sp)
ld t5, (8 * 5)(sp)
ld a0, (8 * 7)(sp)
addi sp, sp, 64
csrrw sp, mscratch, sp
eret
/*
* Redirect to supervisor
*/
exit_mrts:
/* Setup exception handler */
li t1, KERNBASE
add t2, t2, t1
csrw stvec, t2
/* Restore state */
ld t0, (8 * 0)(sp)
ld t1, (8 * 1)(sp)
ld t2, (8 * 2)(sp)
ld t3, (8 * 3)(sp)
ld t4, (8 * 4)(sp)
ld t5, (8 * 5)(sp)
ld a0, (8 * 7)(sp)
addi sp, sp, 64
csrrw sp, mscratch, sp
/* Redirect to supervisor */
mrts