summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--sys/boot/fdt/dts/riscv/spike.dts17
-rw-r--r--sys/conf/files.riscv4
-rw-r--r--sys/riscv/conf/GENERIC8
-rw-r--r--sys/riscv/htif/htif.c8
-rw-r--r--sys/riscv/htif/htif_block.c8
-rw-r--r--sys/riscv/htif/htif_console.c10
-rw-r--r--sys/riscv/include/intr.h14
-rw-r--r--sys/riscv/include/param.h2
-rw-r--r--sys/riscv/include/pcpu.h14
-rw-r--r--sys/riscv/include/riscvreg.h25
-rw-r--r--sys/riscv/include/smp.h56
-rw-r--r--sys/riscv/riscv/cpufunc_asm.S13
-rw-r--r--sys/riscv/riscv/exception.S179
-rw-r--r--sys/riscv/riscv/genassym.c3
-rw-r--r--sys/riscv/riscv/intr_machdep.c90
-rw-r--r--sys/riscv/riscv/locore.S206
-rw-r--r--sys/riscv/riscv/machdep.c8
-rw-r--r--sys/riscv/riscv/mp_machdep.c453
-rw-r--r--sys/riscv/riscv/pmap.c6
-rw-r--r--sys/riscv/riscv/swtch.S12
20 files changed, 1028 insertions, 108 deletions
diff --git a/sys/boot/fdt/dts/riscv/spike.dts b/sys/boot/fdt/dts/riscv/spike.dts
index 38b92bc7..dc67114 100644
--- a/sys/boot/fdt/dts/riscv/spike.dts
+++ b/sys/boot/fdt/dts/riscv/spike.dts
@@ -43,6 +43,23 @@
#size-cells = <1>;
#interrupt-cells = <1>;
+ cpus {
+ #address-cells = <1>;
+ #size-cells = <0>;
+
+ cpu@0 {
+ device_type = "cpu";
+ compatible = "riscv,rv64i";
+ reg = <0x40002000>;
+ };
+
+ cpu@1 {
+ device_type = "cpu";
+ compatible = "riscv,rv64i";
+ reg = <0x4000a000>;
+ };
+ };
+
aliases {
console0 = &console0;
};
diff --git a/sys/conf/files.riscv b/sys/conf/files.riscv
index 297d6fd..cfce99b 100644
--- a/sys/conf/files.riscv
+++ b/sys/conf/files.riscv
@@ -1,6 +1,7 @@
# $FreeBSD$
crypto/blowfish/bf_enc.c optional crypto | ipsec
crypto/des/des_enc.c optional crypto | ipsec | netsmb
+dev/ofw/ofw_cpu.c optional fdt
kern/kern_clocksource.c standard
kern/subr_dummy_vdso_tc.c standard
libkern/bcmp.c standard
@@ -29,8 +30,9 @@ riscv/riscv/intr_machdep.c standard
riscv/riscv/in_cksum.c optional inet | inet6
riscv/riscv/identcpu.c standard
riscv/riscv/locore.S standard no-obj
-riscv/riscv/minidump_machdep.c standard
riscv/riscv/machdep.c standard
+riscv/riscv/minidump_machdep.c standard
+riscv/riscv/mp_machdep.c optional smp
riscv/riscv/mem.c standard
riscv/riscv/nexus.c standard
riscv/riscv/pmap.c standard
diff --git a/sys/riscv/conf/GENERIC b/sys/riscv/conf/GENERIC
index 2565921..d8c6fdb 100644
--- a/sys/riscv/conf/GENERIC
+++ b/sys/riscv/conf/GENERIC
@@ -70,7 +70,13 @@ options MAC # TrustedBSD MAC Framework
options RACCT # Resource accounting framework
options RACCT_DEFAULT_TO_DISABLED # Set kern.racct.enable=0 by default
options RCTL # Resource limits
-# options SMP
+options SMP
+
+# Uncomment for memory disk
+# options MD_ROOT
+# options MD_ROOT_SIZE=8192 # 8MB ram disk
+# makeoptions MFS_IMAGE=/path/to/img
+# options ROOTDEVNAME=\"ufs:/dev/md0\"
# Debugging support. Always need this:
# options KDB # Enable kernel debugger support.
diff --git a/sys/riscv/htif/htif.c b/sys/riscv/htif/htif.c
index f8143dc..b07b789 100644
--- a/sys/riscv/htif/htif.c
+++ b/sys/riscv/htif/htif.c
@@ -1,5 +1,5 @@
/*-
- * Copyright (c) 2015 Ruslan Bukin <br@bsdpad.com>
+ * Copyright (c) 2015-2016 Ruslan Bukin <br@bsdpad.com>
* All rights reserved.
*
* Portions of this software were developed by SRI International and the
@@ -62,7 +62,7 @@ __FBSDID("$FreeBSD$");
#include "htif.h"
static struct resource_spec htif_spec[] = {
- { SYS_RES_IRQ, 0, RF_ACTIVE },
+ { SYS_RES_IRQ, 0, RF_ACTIVE | RF_SHAREABLE},
{ -1, 0 }
};
@@ -126,9 +126,9 @@ htif_intr(void *arg)
sc = arg;
- htif_handle_entry(sc);
+ csr_clear(sip, SIP_SSIP);
- csr_clear(sip, SIE_SSIE);
+ htif_handle_entry(sc);
return (FILTER_HANDLED);
}
diff --git a/sys/riscv/htif/htif_block.c b/sys/riscv/htif/htif_block.c
index 58804d7..c9d2ffc 100644
--- a/sys/riscv/htif/htif_block.c
+++ b/sys/riscv/htif/htif_block.c
@@ -1,5 +1,5 @@
/*-
- * Copyright (c) 2015 Ruslan Bukin <br@bsdpad.com>
+ * Copyright (c) 2015-2016 Ruslan Bukin <br@bsdpad.com>
* All rights reserved.
*
* Portions of this software were developed by SRI International and the
@@ -117,6 +117,9 @@ htif_blk_intr(void *arg, uint64_t entry)
if (sc->curtag == data) {
sc->cmd_done = 1;
wakeup(&sc->intr_chan);
+ } else {
+ device_printf(sc->dev, "Unexpected tag %d (should be %d)\n",
+ data, sc->curtag);
}
}
@@ -212,6 +215,8 @@ htif_blk_task(void *arg)
HTIF_BLK_UNLOCK(sc);
if (bp->bio_cmd == BIO_READ || bp->bio_cmd == BIO_WRITE) {
+ HTIF_BLK_LOCK(sc);
+
req.offset = (bp->bio_pblkno * sc->disk->d_sectorsize);
req.size = bp->bio_bcount;
paddr = vtophys(bp->bio_data);
@@ -233,7 +238,6 @@ htif_blk_task(void *arg)
htif_command(cmd);
/* Wait for interrupt */
- HTIF_BLK_LOCK(sc);
i = 0;
while (sc->cmd_done == 0) {
msleep(&sc->intr_chan, &sc->sc_mtx, PRIBIO, "intr", hz/2);
diff --git a/sys/riscv/htif/htif_console.c b/sys/riscv/htif/htif_console.c
index b4a4676..10e4349 100644
--- a/sys/riscv/htif/htif_console.c
+++ b/sys/riscv/htif/htif_console.c
@@ -1,5 +1,5 @@
/*-
- * Copyright (c) 2015 Ruslan Bukin <br@bsdpad.com>
+ * Copyright (c) 2015-2016 Ruslan Bukin <br@bsdpad.com>
* All rights reserved.
*
* Portions of this software were developed by SRI International and the
@@ -89,6 +89,7 @@ CONSOLE_DRIVER(riscv);
#define MAX_BURST_LEN 1
#define QUEUE_SIZE 256
#define CONSOLE_DEFAULT_ID 1ul
+#define SPIN_IN_MACHINE_MODE 1
struct queue_entry {
uint64_t data;
@@ -109,7 +110,12 @@ htif_putc(int c)
cmd |= (CONSOLE_DEFAULT_ID << HTIF_DEV_ID_SHIFT);
cmd |= c;
+#ifdef SPIN_IN_MACHINE_MODE
+ machine_command(ECALL_HTIF_LOWPUTC, cmd);
+#else
htif_command(cmd);
+#endif
+
}
static uint8_t
@@ -141,6 +147,7 @@ riscv_putc(int c)
htif_putc(c);
+#ifndef SPIN_IN_MACHINE_MODE
/* Wait for an interrupt */
__asm __volatile(
"li %0, 1\n" /* counter = 1 */
@@ -153,6 +160,7 @@ riscv_putc(int c)
"2:"
: "=&r"(counter), "=&r"(val) : "r"(cc)
);
+#endif
}
#ifdef EARLY_PRINTF
diff --git a/sys/riscv/include/intr.h b/sys/riscv/include/intr.h
index 569f7c9..57d5e5d 100644
--- a/sys/riscv/include/intr.h
+++ b/sys/riscv/include/intr.h
@@ -1,5 +1,5 @@
/*-
- * Copyright (c) 2015 Ruslan Bukin <br@bsdpad.com>
+ * Copyright (c) 2015-2016 Ruslan Bukin <br@bsdpad.com>
* All rights reserved.
*
* Portions of this software were developed by SRI International and the
@@ -51,4 +51,16 @@ typedef unsigned long * riscv_intrcnt_t;
riscv_intrcnt_t riscv_intrcnt_create(const char *);
void riscv_intrcnt_setname(riscv_intrcnt_t, const char *);
+#ifdef SMP
+void riscv_setup_ipihandler(driver_filter_t *);
+void riscv_unmask_ipi(void);
+#endif
+
+enum {
+ IRQ_SOFTWARE,
+ IRQ_TIMER,
+ IRQ_HTIF,
+ NIRQS
+};
+
#endif /* !_MACHINE_INTR_MACHDEP_H_ */
diff --git a/sys/riscv/include/param.h b/sys/riscv/include/param.h
index 2c80058..f22e747 100644
--- a/sys/riscv/include/param.h
+++ b/sys/riscv/include/param.h
@@ -51,7 +51,7 @@
#if defined(SMP) || defined(KLD_MODULE)
#ifndef MAXCPU
-#define MAXCPU 2
+#define MAXCPU 16
#endif
#else
#define MAXCPU 1
diff --git a/sys/riscv/include/pcpu.h b/sys/riscv/include/pcpu.h
index 7dfe23d..8dacc4a 100644
--- a/sys/riscv/include/pcpu.h
+++ b/sys/riscv/include/pcpu.h
@@ -1,7 +1,16 @@
/*-
* Copyright (c) 1999 Luoqi Chen <luoqi@freebsd.org>
+ * Copyright (c) 2015-2016 Ruslan Bukin <br@bsdpad.com>
* All rights reserved.
*
+ * Portions of this software were developed by SRI International and the
+ * University of Cambridge Computer Laboratory under DARPA/AFRL contract
+ * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme.
+ *
+ * Portions of this software were developed by the University of Cambridge
+ * Computer Laboratory as part of the CTSRD Project, with support from the
+ * UK Higher Education Innovation Fund (HEIF).
+ *
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
@@ -36,7 +45,10 @@
#define ALT_STACK_SIZE 128
#define PCPU_MD_FIELDS \
- char __pad[129]
+ uint32_t pc_pending_ipis; /* IPIs pending to this CPU */ \
+ uint64_t pc_sptbr; /* L0 page table base (VA) */ \
+ uint64_t pc_reg; /* CPU MMIO base (PA) */ \
+ char __pad[109]
#ifdef _KERNEL
diff --git a/sys/riscv/include/riscvreg.h b/sys/riscv/include/riscvreg.h
index 3f93428..76cba49 100644
--- a/sys/riscv/include/riscvreg.h
+++ b/sys/riscv/include/riscvreg.h
@@ -1,5 +1,5 @@
/*-
- * Copyright (c) 2015 Ruslan Bukin <br@bsdpad.com>
+ * Copyright (c) 2015-2016 Ruslan Bukin <br@bsdpad.com>
* All rights reserved.
*
* Portions of this software were developed by SRI International and the
@@ -38,12 +38,16 @@
#define _MACHINE_RISCVREG_H_
/* Machine mode requests */
-#define ECALL_MTIMECMP 0x1
-#define ECALL_CLEAR_PENDING 0x2
-#define ECALL_HTIF_CMD 0x3
-#define ECALL_HTIF_GET_ENTRY 0x4
-#define ECALL_MCPUID_GET 0x5
-#define ECALL_MIMPID_GET 0x6
+#define ECALL_MTIMECMP 0x01
+#define ECALL_CLEAR_PENDING 0x02
+#define ECALL_HTIF_CMD 0x03
+#define ECALL_HTIF_GET_ENTRY 0x04
+#define ECALL_MCPUID_GET 0x05
+#define ECALL_MIMPID_GET 0x06
+#define ECALL_SEND_IPI 0x07
+#define ECALL_CLEAR_IPI 0x08
+#define ECALL_HTIF_LOWPUTC 0x09
+#define ECALL_MIE_SET 0x10
#define EXCP_SHIFT 0
#define EXCP_MASK (0xf << EXCP_SHIFT)
@@ -109,9 +113,14 @@
#define SIE_SSIE (1 << 1)
#define SIE_STIE (1 << 5)
-/* Note: sip register is not yet implement in Spike simulator */
+/* Note: sip register has no SIP_STIP bit in Spike simulator */
+#define SIP_SSIP (1 << 1)
#define SIP_STIP (1 << 5)
+#define NCSRS 4096
+#define CSR_IPI 0x783
+#define XLEN 8
+
#define CSR_ZIMM(val) \
(__builtin_constant_p(val) && ((u_long)(val) < 32))
diff --git a/sys/riscv/include/smp.h b/sys/riscv/include/smp.h
index da23dbe..52bdf64 100644
--- a/sys/riscv/include/smp.h
+++ b/sys/riscv/include/smp.h
@@ -1 +1,55 @@
-/* $FreeBSD$ */
+/*-
+ * Copyright (c) 2016 Ruslan Bukin <br@bsdpad.com>
+ * All rights reserved.
+ *
+ * Portions of this software were developed by SRI International and the
+ * University of Cambridge Computer Laboratory under DARPA/AFRL contract
+ * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme.
+ *
+ * Portions of this software were developed by the University of Cambridge
+ * Computer Laboratory as part of the CTSRD Project, with support from the
+ * UK Higher Education Innovation Fund (HEIF).
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _MACHINE_SMP_H_
+#define _MACHINE_SMP_H_
+
+#include <machine/pcb.h>
+
+#define IPI_AST (1 << 0)
+#define IPI_PREEMPT (1 << 1)
+#define IPI_RENDEZVOUS (1 << 2)
+#define IPI_STOP (1 << 3)
+#define IPI_STOP_HARD (1 << 4)
+#define IPI_HARDCLOCK (1 << 5)
+
+void ipi_all_but_self(u_int ipi);
+void ipi_cpu(int cpu, u_int ipi);
+void ipi_selected(cpuset_t cpus, u_int ipi);
+
+extern struct pcb stoppcbs[];
+
+#endif /* !_MACHINE_SMP_H_ */
diff --git a/sys/riscv/riscv/cpufunc_asm.S b/sys/riscv/riscv/cpufunc_asm.S
index 21bce53..1d44c22 100644
--- a/sys/riscv/riscv/cpufunc_asm.S
+++ b/sys/riscv/riscv/cpufunc_asm.S
@@ -1,5 +1,5 @@
/*-
- * Copyright (c) 2015 Ruslan Bukin <br@bsdpad.com>
+ * Copyright (c) 2015-2016 Ruslan Bukin <br@bsdpad.com>
* All rights reserved.
*
* Portions of this software were developed by SRI International and the
@@ -64,7 +64,7 @@ END(riscv_tlb_flushID_SE)
* void riscv_dcache_wb_range(vm_offset_t, vm_size_t)
*/
ENTRY(riscv_dcache_wb_range)
- /* RISCVTODO */
+ sfence.vm
ret
END(riscv_dcache_wb_range)
@@ -72,7 +72,7 @@ END(riscv_dcache_wb_range)
* void riscv_dcache_wbinv_range(vm_offset_t, vm_size_t)
*/
ENTRY(riscv_dcache_wbinv_range)
- /* RISCVTODO */
+ sfence.vm
ret
END(riscv_dcache_wbinv_range)
@@ -80,7 +80,7 @@ END(riscv_dcache_wbinv_range)
* void riscv_dcache_inv_range(vm_offset_t, vm_size_t)
*/
ENTRY(riscv_dcache_inv_range)
- /* RISCVTODO */
+ sfence.vm
ret
END(riscv_dcache_inv_range)
@@ -88,7 +88,8 @@ END(riscv_dcache_inv_range)
* void riscv_idcache_wbinv_range(vm_offset_t, vm_size_t)
*/
ENTRY(riscv_idcache_wbinv_range)
- /* RISCVTODO */
+ fence.i
+ sfence.vm
ret
END(riscv_idcache_wbinv_range)
@@ -96,6 +97,6 @@ END(riscv_idcache_wbinv_range)
* void riscv_icache_sync_range(vm_offset_t, vm_size_t)
*/
ENTRY(riscv_icache_sync_range)
- /* RISCVTODO */
+ fence.i
ret
END(riscv_icache_sync_range)
diff --git a/sys/riscv/riscv/exception.S b/sys/riscv/riscv/exception.S
index 814fcf6..f1f127a 100644
--- a/sys/riscv/riscv/exception.S
+++ b/sys/riscv/riscv/exception.S
@@ -1,5 +1,5 @@
/*-
- * Copyright (c) 2015 Ruslan Bukin <br@bsdpad.com>
+ * Copyright (c) 2015-2016 Ruslan Bukin <br@bsdpad.com>
* All rights reserved.
*
* Portions of this software were developed by SRI International and the
@@ -229,6 +229,7 @@ bad_trap:
j bad_trap
user_trap:
+ /* Save state */
csrrw sp, mscratch, sp
addi sp, sp, -64
sd t0, (8 * 0)(sp)
@@ -282,7 +283,31 @@ machine_interrupt:
j 1b
software_interrupt:
- /* Redirect to supervisor */
+ li t0, MIP_MSIP
+ csrc mip, t0
+ li t0, MIP_SSIP
+ csrs mip, t0
+
+ /* If PRV1 is PRV_U (user) then serve the trap */
+ csrr t0, mstatus
+ li t1, (MSTATUS_PRV_M << MSTATUS_PRV1_SHIFT)
+ and t0, t0, t1
+ beqz t0, 1f
+
+ /*
+ * If PRV1 is supervisor and interrupts were enabled,
+ * then serve the trap.
+ */
+ csrr t0, mstatus
+ li t1, (SR_IE1 | (MSTATUS_PRV_M << MSTATUS_PRV1_SHIFT))
+ and t0, t0, t1
+ li t1, (SR_IE1 | (MSTATUS_PRV_S << MSTATUS_PRV1_SHIFT))
+ beq t0, t1, 1f
+
+ j exit
+
+1:
+ /* Serve a trap in supervisor mode */
j exit_mrts
timer_interrupt:
@@ -294,17 +319,20 @@ timer_interrupt:
li t0, MIP_MTIP
csrc mip, t0
- /* Post supervisor software interrupt */
+ /* Post supervisor timer interrupt */
li t0, MIP_STIP
csrs mip, t0
- /* If PRV1 is PRV_U (user) then serve a trap */
+ /* If PRV1 is PRV_U (user) then serve the trap */
csrr t0, mstatus
li t1, (MSTATUS_PRV_M << MSTATUS_PRV1_SHIFT)
and t0, t0, t1
beqz t0, 1f
- /* If PRV1 is supervisor and interrupts were enabled, then serve a trap */
+ /*
+ * If PRV1 is supervisor and interrupts were enabled,
+ * then serve the trap.
+ */
csrr t0, mstatus
li t1, (SR_IE1 | (MSTATUS_PRV_M << MSTATUS_PRV1_SHIFT))
and t0, t0, t1
@@ -332,25 +360,35 @@ htif_interrupt:
la t0, console_intr
li t1, 1
sd t1, 0(t0)
- j 3f
+
+ /* Check if there is any other pending event */
+ j 1b
2:
/* Save entry */
- la t0, htif_ring_cursor
- beqz t0, 3f /* not initialized */
- ld t0, 0(t0) /* load struct */
- sd t5, 0(t0) /* put entry */
+ la t0, htif_ring
+ csrr t1, mhartid
+ li t4, (HTIF_RING_SIZE + 16)
+ mulw t4, t4, t1
+ add t0, t0, t4
+ li t4, (HTIF_RING_SIZE)
+ add t0, t0, t4 /* t0 == htif_ring_cursor */
+
+ ld t1, 0(t0) /* load ptr to cursor */
+ sd t5, 0(t1) /* put entry */
li t4, 1
- sd t4, 8(t0) /* mark used */
- ld t4, 16(t0) /* take next */
+ sd t4, 8(t1) /* mark used */
+ ld t4, 16(t1) /* take next */
/* Update cursor */
- la t0, htif_ring_cursor
sd t4, 0(t0)
/* Post supervisor software interrupt */
li t0, MIP_SSIP
csrs mip, t0
+ /* Check if there is any other pending event */
+ j 1b
+
3:
j exit
@@ -369,6 +407,18 @@ supervisor_call:
beq t5, t4, mcpuid_get
li t4, ECALL_MIMPID_GET
beq t5, t4, mimpid_get
+ li t4, ECALL_SEND_IPI
+ beq t5, t4, send_ipi
+ li t4, ECALL_CLEAR_IPI
+ beq t5, t4, clear_ipi
+ li t4, ECALL_HTIF_LOWPUTC
+ beq t5, t4, htif_lowputc
+ li t4, ECALL_MIE_SET
+ beq t5, t4, mie_set
+ j exit_next_instr
+
+mie_set:
+ csrs mie, t6
j exit_next_instr
mcpuid_get:
@@ -379,28 +429,111 @@ mimpid_get:
csrr t6, mimpid
j exit_next_instr
+send_ipi:
+ /* CPU mmio base in t6 */
+ mv t0, t6
+ li t2, (CSR_IPI * XLEN)
+ add t0, t0, t2 /* t0 = CSR_IPI */
+ li t2, 1
+ sd t2, 0(t0)
+ j exit_next_instr
+
+clear_ipi:
+ /* Do only clear if there are no new entries in HTIF ring */
+ la t0, htif_ring
+ csrr t2, mhartid
+ li t4, (HTIF_RING_SIZE + 16)
+ mulw t4, t4, t2
+ add t0, t0, t4
+ li t4, (HTIF_RING_SIZE)
+ add t0, t0, t4 /* t0 == ptr to htif_ring_cursor */
+ ld t2, 8(t0) /* load htif_ring_last */
+ ld t2, 8(t2) /* load used */
+ bnez t2, 1f
+
+ /* Clear supervisor software interrupt pending bit */
+ li t0, MIP_SSIP
+ csrc mip, t0
+
+1:
+ j exit_next_instr
+
htif_get_entry:
+ /* Get a htif_ring for current core */
+ la t0, htif_ring
+ csrr t2, mhartid
+ li t4, (HTIF_RING_SIZE + 16)
+ mulw t4, t4, t2
+ add t0, t0, t4
+ li t4, (HTIF_RING_SIZE + 8)
+ add t0, t0, t4 /* t0 == htif_ring_last */
+
+ /* Check for new entries */
li t6, 0 /* preset return value */
- la t0, htif_ring_last
- ld t0, 0(t0) /* load struct */
- ld t4, 8(t0) /* get used */
- beqz t4, 1f
- ld t6, 0(t0) /* get entry */
+ ld t2, 0(t0) /* load ptr to last */
+ ld t4, 8(t2) /* get used */
+ beqz t4, 1f /* No new entries. Exit */
+
+ /* Get one */
+ ld t6, 0(t2) /* get entry */
li t4, 0
- sd t4, 8(t0) /* mark free */
- sd t4, 0(t0) /* free entry, just in case */
- ld t4, 16(t0) /* take next */
- la t0, htif_ring_last
- sd t4, 0(t0)
+ sd t4, 8(t2) /* mark free */
+ sd t4, 0(t2) /* free entry, just in case */
+ ld t4, 16(t2) /* take next */
+ sd t4, 0(t0) /* update ptr to last */
1:
/* Exit. Result is stored in t6 */
j exit_next_instr
htif_cmd:
+1:
mv t0, t6
+ csrrw t0, mtohost, t0
+ bnez t0, 1b
+ j exit_next_instr
+
+htif_lowputc:
1:
+ mv t0, t6
csrrw t0, mtohost, t0
bnez t0, 1b
+
+2:
+ li t4, 0
+ csrrw t5, mfromhost, t4
+ beqz t5, 2b
+
+ /* Console PUT intr ? */
+ mv t2, t5
+ srli t2, t2, 48
+ li t3, 0x0101
+ beq t2, t3, 3f
+
+ /* Not a console PUT, so save entry */
+ la t0, htif_ring
+ csrr t2, mhartid
+ li t4, (HTIF_RING_SIZE + 16)
+ mulw t4, t4, t2
+ add t0, t0, t4
+ li t4, (HTIF_RING_SIZE)
+ add t0, t0, t4 /* t0 == htif_ring_cursor */
+
+ ld t2, 0(t0) /* load ptr to cursor */
+ sd t5, 0(t2) /* put entry */
+ li t4, 1
+ sd t4, 8(t2) /* mark used */
+ ld t4, 16(t2) /* take next */
+ /* Update cursor */
+ sd t4, 0(t0)
+
+ /* Post supervisor software interrupt */
+ li t0, MIP_SSIP
+ csrs mip, t0
+
+ /* Wait for console intr again */
+ j 2b
+
+3:
j exit_next_instr
set_mtimecmp:
diff --git a/sys/riscv/riscv/genassym.c b/sys/riscv/riscv/genassym.c
index c5dec83..81b3e13 100644
--- a/sys/riscv/riscv/genassym.c
+++ b/sys/riscv/riscv/genassym.c
@@ -1,5 +1,5 @@
/*-
- * Copyright (c) 2015 Ruslan Bukin <br@bsdpad.com>
+ * Copyright (c) 2015-2016 Ruslan Bukin <br@bsdpad.com>
* All rights reserved.
*
* Portions of this software were developed by SRI International and the
@@ -75,6 +75,7 @@ ASSYM(PCB_A, offsetof(struct pcb, pcb_a));
ASSYM(SF_UC, offsetof(struct sigframe, sf_uc));
ASSYM(PC_CURPCB, offsetof(struct pcpu, pc_curpcb));
+ASSYM(PC_SPTBR, offsetof(struct pcpu, pc_sptbr));
ASSYM(PC_CURTHREAD, offsetof(struct pcpu, pc_curthread));
ASSYM(TD_PCB, offsetof(struct thread, td_pcb));
diff --git a/sys/riscv/riscv/intr_machdep.c b/sys/riscv/riscv/intr_machdep.c
index c51075c..5ed109f 100644
--- a/sys/riscv/riscv/intr_machdep.c
+++ b/sys/riscv/riscv/intr_machdep.c
@@ -1,5 +1,5 @@
/*-
- * Copyright (c) 2015 Ruslan Bukin <br@bsdpad.com>
+ * Copyright (c) 2015-2016 Ruslan Bukin <br@bsdpad.com>
* All rights reserved.
*
* Portions of this software were developed by SRI International and the
@@ -38,7 +38,9 @@ __FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/bus.h>
+#include <sys/cpuset.h>
#include <sys/interrupt.h>
+#include <sys/smp.h>
#include <machine/clock.h>
#include <machine/cpu.h>
@@ -46,12 +48,9 @@ __FBSDID("$FreeBSD$");
#include <machine/frame.h>
#include <machine/intr.h>
-enum {
- IRQ_SOFTWARE,
- IRQ_TIMER,
- IRQ_HTIF,
- NIRQS
-};
+#ifdef SMP
+#include <machine/smp.h>
+#endif
u_long intrcnt[NIRQS];
size_t sintrcnt = sizeof(intrcnt);
@@ -159,8 +158,12 @@ riscv_setup_intr(const char *name, driver_filter_t *filt,
riscv_unmask_irq((void*)(uintptr_t)irq);
}
- intr_event_add_handler(event, name, filt, handler, arg,
+ error = intr_event_add_handler(event, name, filt, handler, arg,
intr_priority(flags), flags, cookiep);
+ if (error) {
+ printf("Failed to setup intr: %d\n", irq);
+ return (error);
+ }
riscv_intrcnt_setname(riscv_intr_counters[irq],
event->ie_fullname);
@@ -221,3 +224,74 @@ riscv_cpu_intr(struct trapframe *frame)
critical_exit();
}
+
+#ifdef SMP
+void
+riscv_setup_ipihandler(driver_filter_t *filt)
+{
+
+ riscv_setup_intr("ipi", filt, NULL, NULL, IRQ_SOFTWARE,
+ INTR_TYPE_MISC, NULL);
+}
+
+void
+riscv_unmask_ipi(void)
+{
+
+ csr_set(sie, SIE_SSIE);
+}
+
+/* Sending IPI */
+static void
+ipi_send(struct pcpu *pc, int ipi)
+{
+
+ CTR3(KTR_SMP, "%s: cpu=%d, ipi=%x", __func__, pc->pc_cpuid, ipi);
+
+ atomic_set_32(&pc->pc_pending_ipis, ipi);
+ machine_command(ECALL_SEND_IPI, pc->pc_reg);
+
+ CTR1(KTR_SMP, "%s: sent", __func__);
+}
+
+void
+ipi_all_but_self(u_int ipi)
+{
+ cpuset_t other_cpus;
+
+ other_cpus = all_cpus;
+ CPU_CLR(PCPU_GET(cpuid), &other_cpus);
+
+ CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi);
+ ipi_selected(other_cpus, ipi);
+}
+
+void
+ipi_cpu(int cpu, u_int ipi)
+{
+ cpuset_t cpus;
+
+ CPU_ZERO(&cpus);
+ CPU_SET(cpu, &cpus);
+
+ CTR3(KTR_SMP, "%s: cpu: %d, ipi: %x\n", __func__, cpu, ipi);
+ ipi_send(cpuid_to_pcpu[cpu], ipi);
+}
+
+void
+ipi_selected(cpuset_t cpus, u_int ipi)
+{
+ struct pcpu *pc;
+
+ CTR1(KTR_SMP, "ipi_selected: ipi: %x", ipi);
+
+ STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) {
+ if (CPU_ISSET(pc->pc_cpuid, &cpus)) {
+ CTR3(KTR_SMP, "%s: pc: %p, ipi: %x\n", __func__, pc,
+ ipi);
+ ipi_send(pc, ipi);
+ }
+ }
+}
+
+#endif
diff --git a/sys/riscv/riscv/locore.S b/sys/riscv/riscv/locore.S
index 0744167..9f32cac0 100644
--- a/sys/riscv/riscv/locore.S
+++ b/sys/riscv/riscv/locore.S
@@ -1,5 +1,5 @@
/*-
- * Copyright (c) 2015 Ruslan Bukin <br@bsdpad.com>
+ * Copyright (c) 2015-2016 Ruslan Bukin <br@bsdpad.com>
* All rights reserved.
*
* Portions of this software were developed by SRI International and the
@@ -43,8 +43,55 @@
#include <machine/riscvreg.h>
#include <machine/pte.h>
-#define HTIF_RING_SIZE (64)
-#define HTIF_RING_LAST (24 * (HTIF_RING_SIZE - 1))
+#define HTIF_RING_NENTRIES (512)
+#define HTIF_RING_ENTRY_SZ (24)
+#define HTIF_RING_SIZE (HTIF_RING_ENTRY_SZ * HTIF_RING_NENTRIES)
+#define HW_STACK_SIZE (96)
+
+/*
+ * Event queue for each CPU core:
+ *
+ * struct htif_ring {
+ * uint64_t data;
+ * uint64_t used;
+ * uint64_t next;
+ * } htif_ring[HTIF_RING_NENTRIES];
+ * uint64_t htif_ring_cursor;
+ * uint64_t htif_ring_last;
+ */
+
+.macro build_ring
+ la t0, htif_ring
+#ifdef SMP
+ csrr a0, mhartid
+ li s0, (HTIF_RING_SIZE + 16)
+ mulw s0, a0, s0
+ add t0, t0, s0
+#endif
+ li t1, 0
+ sd t1, 0(t0) /* zero data */
+ sd t1, 8(t0) /* zero used */
+ mv t2, t0
+ mv t3, t0
+ li t5, (HTIF_RING_SIZE - HTIF_RING_ENTRY_SZ)
+ li t6, 0
+ add t4, t0, t5
+1:
+ addi t3, t3, 24 /* pointer to next */
+ beq t3, t4, 2f /* finish */
+ sd t3, 16(t2) /* store pointer */
+ addi t2, t2, 24 /* next entry */
+ addi t6, t6, 1 /* counter */
+ j 1b
+2:
+ sd t0, 16(t3) /* last -> first */
+
+ li t2, (HTIF_RING_SIZE)
+ add s0, t0, t2
+ sd t0, 0(s0) /* cursor */
+ sd t0, 8(s0) /* last */
+ /* finish building ring */
+.endm
.globl kernbase
.set kernbase, KERNBASE
@@ -74,33 +121,14 @@ mentry:
.align 8
.globl _start
_start:
- li s11, KERNBASE
+ /* Direct secondary cores to mpentry */
+ csrr a0, mhartid
+ bnez a0, mpentry
- /* Build ring */
- la t0, htif_ring
- li t1, 0
- sd t1, 0(t0) /* zero data */
- sd t1, 8(t0) /* zero used */
- mv t2, t0
- mv t3, t0
- li t5, HTIF_RING_LAST
- li t6, 0
- add t4, t0, t5
-1:
- addi t3, t3, 24 /* pointer to next */
- beq t3, t4, 2f /* finish */
- sd t3, 16(t2) /* store pointer */
- addi t2, t2, 24 /* next entry */
- addi t6, t6, 1 /* counter */
- j 1b
-2:
- sd t0, 16(t3) /* last -> first */
- la t1, htif_ring_cursor
- sd t0, 0(t1)
- la t1, htif_ring_last
- sd t0, 0(t1)
- /* finish building ring */
+ /* Build event queue for current core */
+ build_ring
+ /* Setup machine-mode stack for CPU 0 */
la t0, hardstack_end
csrw mscratch, t0
@@ -129,7 +157,14 @@ _start:
li a6, PTE_SIZE
mulw a5, a5, a6
add t0, s1, a5
+ /* Store it to pagetable_l0 for each cpu */
+ li t1, MAXCPU
+ li t2, PAGE_SIZE
+1:
sd t6, 0(t0)
+ add t0, t0, t2
+ addi t1, t1, -1
+ bnez t1, 1b
/* Level 1 */
la s1, pagetable_l1
@@ -177,8 +212,16 @@ _start:
(MSTATUS_PRV_U << MSTATUS_PRV2_SHIFT));
csrw mstatus, s0
+ /*
+ * Enable machine-mode software interrupts
+ * so we can deliver IPI to this core.
+ */
+ li t0, MIE_MSIE
+ csrs mie, t0
+
/* Exit from machine mode */
la t0, .Lmmu_on
+ li s11, KERNBASE
add t0, t0, s11
csrw mepc, t0
eret
@@ -213,24 +256,16 @@ initstack:
.space (PAGE_SIZE * KSTACK_PAGES)
initstack_end:
hardstack:
- .space (PAGE_SIZE)
+ .space (HW_STACK_SIZE * MAXCPU)
hardstack_end:
.globl htif_ring
htif_ring:
- .space (24 * 1024)
-
- .globl htif_ring_cursor
-htif_ring_cursor:
- .space (8)
-
- .globl htif_ring_last
-htif_ring_last:
- .space (8)
+ .space ((HTIF_RING_SIZE + 16) * MAXCPU)
.globl console_intr
console_intr:
- .space (8)
+ .space (8)
ENTRY(sigcode)
mv a0, sp
@@ -259,7 +294,7 @@ szsigcode:
.align 12
.globl pagetable_l0
pagetable_l0:
- .space PAGE_SIZE
+ .space (PAGE_SIZE * MAXCPU)
pagetable_l1:
.space PAGE_SIZE
pagetable_l2:
@@ -270,4 +305,95 @@ pagetable_end:
init_pt_va:
.quad pagetable_l2 /* XXX: Keep page tables VA */
+#ifndef SMP
+ENTRY(mpentry)
+1:
+ wfi
+ j 1b
+END(mpentry)
+#else
+/*
+ * mpentry(unsigned long)
+ *
+ * Called by a core when it is being brought online.
+ * The data in x0 is passed straight to init_secondary.
+ */
+ENTRY(mpentry)
+ /*
+ * Calculate the offset to __riscv_boot_ap
+ * for current core, cpuid in a0.
+ */
+ li t1, 4
+ mulw t1, t1, a0
+ /* Get pointer */
+ la t0, __riscv_boot_ap
+ add t0, t0, t1
+
+1:
+ /* Wait the kernel to be ready */
+ lw t1, 0(t0)
+ beqz t1, 1b
+
+ /* Setup machine exception vector */
+ la t0, mentry
+ csrw mtvec, t0
+
+ /* Build event queue ring for this core */
+ build_ring
+
+ /* Set page tables base register */
+ la t0, pagetable_l0
+ li t1, PAGE_SIZE
+ mulw t1, t1, a0
+ add t0, t0, t1
+ csrw sptbr, t0
+ /* Page tables END */
+
+ /* Configure mstatus */
+ li s0, ((MSTATUS_VM_SV48 << MSTATUS_VM_SHIFT) | \
+ (MSTATUS_PRV_M << MSTATUS_PRV_SHIFT) | \
+ (MSTATUS_PRV_S << MSTATUS_PRV1_SHIFT) | \
+ (MSTATUS_PRV_U << MSTATUS_PRV2_SHIFT));
+ csrw mstatus, s0
+
+ /* Setup stack for machine mode exceptions */
+ la t0, hardstack_end
+ li t1, HW_STACK_SIZE
+ mulw t1, t1, a0
+ sub t0, t0, t1
+ csrw mscratch, t0
+
+ li t0, 0
+ csrw sscratch, t0
+
+ /*
+ * Enable machine-mode software interrupts
+ * so we can deliver IPI to this core.
+ */
+ li t0, MIE_MSIE
+ csrs mie, t0
+
+ /*
+ * Exit from machine mode and go to
+ * the virtual address space.
+ */
+ la t0, mp_virtdone
+ li s11, KERNBASE
+ add t0, t0, s11
+ csrw mepc, t0
+ eret
+
+mp_virtdone:
+ /* We are now in virtual address space */
+
+ /* Setup stack pointer */
+ la t0, secondary_stacks
+ li t1, (PAGE_SIZE * KSTACK_PAGES)
+ mulw t1, t1, a0
+ add sp, t0, t1
+
+ call init_secondary
+END(mpentry)
+#endif
+
#include "exception.S"
diff --git a/sys/riscv/riscv/machdep.c b/sys/riscv/riscv/machdep.c
index 6fcf4a0..0c6676e 100644
--- a/sys/riscv/riscv/machdep.c
+++ b/sys/riscv/riscv/machdep.c
@@ -1,6 +1,6 @@
/*-
* Copyright (c) 2014 Andrew Turner
- * Copyright (c) 2015 Ruslan Bukin <br@bsdpad.com>
+ * Copyright (c) 2015-2016 Ruslan Bukin <br@bsdpad.com>
* All rights reserved.
*
* Portions of this software were developed by SRI International and the
@@ -93,6 +93,7 @@ __FBSDID("$FreeBSD$");
#endif
struct pcpu __pcpu[MAXCPU];
+extern uint64_t pagetable_l0;
static struct trapframe proc0_tf;
@@ -389,7 +390,12 @@ cpu_est_clockrate(int cpu_id, uint64_t *rate)
void
cpu_pcpu_init(struct pcpu *pcpu, int cpuid, size_t size)
{
+ uint64_t addr;
+ addr = (uint64_t)&pagetable_l0;
+ addr += (cpuid * PAGE_SIZE);
+
+ pcpu->pc_sptbr = addr;
}
void
diff --git a/sys/riscv/riscv/mp_machdep.c b/sys/riscv/riscv/mp_machdep.c
new file mode 100644
index 0000000..9152e70
--- /dev/null
+++ b/sys/riscv/riscv/mp_machdep.c
@@ -0,0 +1,453 @@
+/*-
+ * Copyright (c) 2015 The FreeBSD Foundation
+ * Copyright (c) 2016 Ruslan Bukin <br@bsdpad.com>
+ * All rights reserved.
+ *
+ * Portions of this software were developed by Andrew Turner under
+ * sponsorship from the FreeBSD Foundation.
+ *
+ * Portions of this software were developed by SRI International and the
+ * University of Cambridge Computer Laboratory under DARPA/AFRL contract
+ * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme.
+ *
+ * Portions of this software were developed by the University of Cambridge
+ * Computer Laboratory as part of the CTSRD Project, with support from the
+ * UK Higher Education Innovation Fund (HEIF).
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "opt_kstack_pages.h"
+#include "opt_platform.h"
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/cpu.h>
+#include <sys/kernel.h>
+#include <sys/malloc.h>
+#include <sys/module.h>
+#include <sys/mutex.h>
+#include <sys/proc.h>
+#include <sys/sched.h>
+#include <sys/smp.h>
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+#include <vm/vm_extern.h>
+#include <vm/vm_kern.h>
+
+#include <machine/intr.h>
+#include <machine/smp.h>
+#ifdef VFP
+#include <machine/vfp.h>
+#endif
+
+#ifdef FDT
+#include <dev/ofw/openfirm.h>
+#include <dev/ofw/ofw_cpu.h>
+#endif
+
+boolean_t ofw_cpu_reg(phandle_t node, u_int, cell_t *);
+
+extern struct pcpu __pcpu[];
+
+uint32_t __riscv_boot_ap[MAXCPU];
+
+static enum {
+ CPUS_UNKNOWN,
+#ifdef FDT
+ CPUS_FDT,
+#endif
+} cpu_enum_method;
+
+static device_identify_t riscv64_cpu_identify;
+static device_probe_t riscv64_cpu_probe;
+static device_attach_t riscv64_cpu_attach;
+
+static int ipi_handler(void *);
+
+struct mtx ap_boot_mtx;
+struct pcb stoppcbs[MAXCPU];
+
+#ifdef INVARIANTS
+static uint32_t cpu_reg[MAXCPU][2];
+#endif
+static device_t cpu_list[MAXCPU];
+
+void mpentry(unsigned long cpuid);
+void init_secondary(uint64_t);
+
+uint8_t secondary_stacks[MAXCPU - 1][PAGE_SIZE * KSTACK_PAGES] __aligned(16);
+
+/* Set to 1 once we're ready to let the APs out of the pen. */
+volatile int aps_ready = 0;
+
+/* Temporary variables for init_secondary() */
+void *dpcpu[MAXCPU - 1];
+
+static device_method_t riscv64_cpu_methods[] = {
+ /* Device interface */
+ DEVMETHOD(device_identify, riscv64_cpu_identify),
+ DEVMETHOD(device_probe, riscv64_cpu_probe),
+ DEVMETHOD(device_attach, riscv64_cpu_attach),
+
+ DEVMETHOD_END
+};
+
+static devclass_t riscv64_cpu_devclass;
+static driver_t riscv64_cpu_driver = {
+ "riscv64_cpu",
+ riscv64_cpu_methods,
+ 0
+};
+
+DRIVER_MODULE(riscv64_cpu, cpu, riscv64_cpu_driver, riscv64_cpu_devclass, 0, 0);
+
+static void
+riscv64_cpu_identify(driver_t *driver, device_t parent)
+{
+
+ if (device_find_child(parent, "riscv64_cpu", -1) != NULL)
+ return;
+ if (BUS_ADD_CHILD(parent, 0, "riscv64_cpu", -1) == NULL)
+ device_printf(parent, "add child failed\n");
+}
+
+static int
+riscv64_cpu_probe(device_t dev)
+{
+ u_int cpuid;
+
+ cpuid = device_get_unit(dev);
+ if (cpuid >= MAXCPU || cpuid > mp_maxid)
+ return (EINVAL);
+
+ device_quiet(dev);
+ return (0);
+}
+
+static int
+riscv64_cpu_attach(device_t dev)
+{
+ const uint32_t *reg;
+ size_t reg_size;
+ u_int cpuid;
+ int i;
+
+ cpuid = device_get_unit(dev);
+
+ if (cpuid >= MAXCPU || cpuid > mp_maxid)
+ return (EINVAL);
+ KASSERT(cpu_list[cpuid] == NULL, ("Already have cpu %u", cpuid));
+
+ reg = cpu_get_cpuid(dev, &reg_size);
+ if (reg == NULL)
+ return (EINVAL);
+
+ if (bootverbose) {
+ device_printf(dev, "register <");
+ for (i = 0; i < reg_size; i++)
+ printf("%s%x", (i == 0) ? "" : " ", reg[i]);
+ printf(">\n");
+ }
+
+ /* Set the device to start it later */
+ cpu_list[cpuid] = dev;
+
+ return (0);
+}
+
+static void
+release_aps(void *dummy __unused)
+{
+ int cpu, i;
+
+ if (mp_ncpus == 1)
+ return;
+
+ /* Setup the IPI handler */
+ riscv_setup_ipihandler(ipi_handler);
+
+ atomic_store_rel_int(&aps_ready, 1);
+
+ printf("Release APs\n");
+
+ for (i = 0; i < 2000; i++) {
+ if (smp_started) {
+ for (cpu = 0; cpu <= mp_maxid; cpu++) {
+ if (CPU_ABSENT(cpu))
+ continue;
+ }
+ return;
+ }
+ DELAY(1000);
+ }
+
+ printf("APs not started\n");
+}
+SYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_FIRST, release_aps, NULL);
+
+void
+init_secondary(uint64_t cpu)
+{
+ struct pcpu *pcpup;
+
+ /* Setup the pcpu pointer */
+ pcpup = &__pcpu[cpu];
+ __asm __volatile("mv gp, %0" :: "r"(pcpup));
+
+ /* Spin until the BSP releases the APs */
+ while (!aps_ready)
+ __asm __volatile("wfi");
+
+ /* Initialize curthread */
+ KASSERT(PCPU_GET(idlethread) != NULL, ("no idle thread"));
+ pcpup->pc_curthread = pcpup->pc_idlethread;
+ pcpup->pc_curpcb = pcpup->pc_idlethread->td_pcb;
+
+ /*
+ * Identify current CPU. This is necessary to setup
+ * affinity registers and to provide support for
+ * runtime chip identification.
+ */
+ identify_cpu();
+
+ /* Enable software interrupts */
+ riscv_unmask_ipi();
+
+ /* Start per-CPU event timers. */
+ cpu_initclocks_ap();
+
+#ifdef VFP
+ /* TODO: init FPU */
+#endif
+
+ /* Enable interrupts */
+ intr_enable();
+
+ mtx_lock_spin(&ap_boot_mtx);
+
+ atomic_add_rel_32(&smp_cpus, 1);
+
+ if (smp_cpus == mp_ncpus) {
+ /* enable IPI's, tlb shootdown, freezes etc */
+ atomic_store_rel_int(&smp_started, 1);
+ }
+
+ mtx_unlock_spin(&ap_boot_mtx);
+
+ /* Enter the scheduler */
+ sched_throw(NULL);
+
+ panic("scheduler returned us to init_secondary");
+ /* NOTREACHED */
+}
+
+static int
+ipi_handler(void *arg)
+{
+ u_int ipi_bitmap;
+ u_int cpu, ipi;
+ int bit;
+
+ /*
+ * We have shared interrupt line for both IPI and HTIF,
+ * so we don't really need to clear pending bit here
+ * as it will be cleared later in htif_intr.
+ * But lets assume HTIF is optional part, so do clear
+ * pending bit if there is no new entires in htif_ring.
+ */
+ machine_command(ECALL_CLEAR_IPI, 0);
+
+ cpu = PCPU_GET(cpuid);
+
+ mb();
+
+ ipi_bitmap = atomic_readandclear_int(PCPU_PTR(pending_ipis));
+ if (ipi_bitmap == 0)
+ return (FILTER_HANDLED);
+
+ while ((bit = ffs(ipi_bitmap))) {
+ bit = (bit - 1);
+ ipi = (1 << bit);
+ ipi_bitmap &= ~ipi;
+
+ mb();
+
+ switch (ipi) {
+ case IPI_AST:
+ CTR0(KTR_SMP, "IPI_AST");
+ break;
+ case IPI_PREEMPT:
+ CTR1(KTR_SMP, "%s: IPI_PREEMPT", __func__);
+ sched_preempt(curthread);
+ break;
+ case IPI_RENDEZVOUS:
+ CTR0(KTR_SMP, "IPI_RENDEZVOUS");
+ smp_rendezvous_action();
+ break;
+ case IPI_STOP:
+ case IPI_STOP_HARD:
+ CTR0(KTR_SMP, (ipi == IPI_STOP) ? "IPI_STOP" : "IPI_STOP_HARD");
+ savectx(&stoppcbs[cpu]);
+
+ /* Indicate we are stopped */
+ CPU_SET_ATOMIC(cpu, &stopped_cpus);
+
+ /* Wait for restart */
+ while (!CPU_ISSET(cpu, &started_cpus))
+ cpu_spinwait();
+
+ CPU_CLR_ATOMIC(cpu, &started_cpus);
+ CPU_CLR_ATOMIC(cpu, &stopped_cpus);
+ CTR0(KTR_SMP, "IPI_STOP (restart)");
+ break;
+ case IPI_HARDCLOCK:
+ CTR1(KTR_SMP, "%s: IPI_HARDCLOCK", __func__);
+ hardclockintr();
+ break;
+ default:
+ panic("Unknown IPI %#0x on cpu %d", ipi, curcpu);
+ }
+ }
+
+ return (FILTER_HANDLED);
+}
+
+struct cpu_group *
+cpu_topo(void)
+{
+
+ return (smp_topo_none());
+}
+
+/* Determine if we running MP machine */
+int
+cpu_mp_probe(void)
+{
+
+ return (mp_ncpus > 1);
+}
+
+#ifdef FDT
+static boolean_t
+cpu_init_fdt(u_int id, phandle_t node, u_int addr_size, pcell_t *reg)
+{
+ uint64_t target_cpu;
+ struct pcpu *pcpup;
+
+ /* Check we are able to start this cpu */
+ if (id > mp_maxid)
+ return (0);
+
+ KASSERT(id < MAXCPU, ("Too many CPUs"));
+
+ KASSERT(addr_size == 1 || addr_size == 2, ("Invalid register size"));
+#ifdef INVARIANTS
+ cpu_reg[id][0] = reg[0];
+ if (addr_size == 2)
+ cpu_reg[id][1] = reg[1];
+#endif
+
+ target_cpu = reg[0];
+ if (addr_size == 2) {
+ target_cpu <<= 32;
+ target_cpu |= reg[1];
+ }
+
+ pcpup = &__pcpu[id];
+
+ /* We are already running on cpu 0 */
+ if (id == 0) {
+ pcpup->pc_reg = target_cpu;
+ return (1);
+ }
+
+ pcpu_init(pcpup, id, sizeof(struct pcpu));
+ pcpup->pc_reg = target_cpu;
+
+ dpcpu[id - 1] = (void *)kmem_malloc(kernel_arena, DPCPU_SIZE,
+ M_WAITOK | M_ZERO);
+ dpcpu_init(dpcpu[id - 1], id);
+
+ printf("Starting CPU %u (%lx)\n", id, target_cpu);
+ __riscv_boot_ap[id] = 1;
+
+ CPU_SET(id, &all_cpus);
+
+ return (1);
+}
+#endif
+
+/* Initialize and fire up non-boot processors */
+void
+cpu_mp_start(void)
+{
+
+ mtx_init(&ap_boot_mtx, "ap boot", NULL, MTX_SPIN);
+
+ CPU_SET(0, &all_cpus);
+
+ switch(cpu_enum_method) {
+#ifdef FDT
+ case CPUS_FDT:
+ ofw_cpu_early_foreach(cpu_init_fdt, true);
+ break;
+#endif
+ case CPUS_UNKNOWN:
+ break;
+ }
+}
+
+/* Introduce rest of cores to the world */
+void
+cpu_mp_announce(void)
+{
+}
+
+void
+cpu_mp_setmaxid(void)
+{
+#ifdef FDT
+ int cores;
+
+ cores = ofw_cpu_early_foreach(NULL, false);
+ if (cores > 0) {
+ cores = MIN(cores, MAXCPU);
+ if (bootverbose)
+ printf("Found %d CPUs in the device tree\n", cores);
+ mp_ncpus = cores;
+ mp_maxid = cores - 1;
+ cpu_enum_method = CPUS_FDT;
+ return;
+ }
+#endif
+
+ if (bootverbose)
+ printf("No CPU data, limiting to 1 core\n");
+ mp_ncpus = 1;
+ mp_maxid = 0;
+}
diff --git a/sys/riscv/riscv/pmap.c b/sys/riscv/riscv/pmap.c
index ef25b16..30bdd53 100644
--- a/sys/riscv/riscv/pmap.c
+++ b/sys/riscv/riscv/pmap.c
@@ -13,7 +13,7 @@
* All rights reserved.
* Copyright (c) 2014 The FreeBSD Foundation
* All rights reserved.
- * Copyright (c) 2015 Ruslan Bukin <br@bsdpad.com>
+ * Copyright (c) 2015-2016 Ruslan Bukin <br@bsdpad.com>
* All rights reserved.
*
* This code is derived from software contributed to Berkeley by
@@ -217,8 +217,6 @@ struct msgbuf *msgbufp = NULL;
static struct rwlock_padalign pvh_global_lock;
-extern uint64_t pagetable_l0;
-
/*
* Data for the pv entry allocation mechanism
*/
@@ -3097,7 +3095,7 @@ pmap_activate(struct thread *td)
pn = (td->td_pcb->pcb_l1addr / PAGE_SIZE);
entry = (PTE_VALID | (PTE_TYPE_PTR << PTE_TYPE_S));
entry |= (pn << PTE_PPN0_S);
- pmap_load_store(&pagetable_l0, entry);
+ pmap_load_store((uint64_t *)PCPU_GET(sptbr), entry);
pmap_invalidate_all(pmap);
critical_exit();
diff --git a/sys/riscv/riscv/swtch.S b/sys/riscv/riscv/swtch.S
index 1a9092f..a9f890d 100644
--- a/sys/riscv/riscv/swtch.S
+++ b/sys/riscv/riscv/swtch.S
@@ -1,5 +1,5 @@
/*-
- * Copyright (c) 2015 Ruslan Bukin <br@bsdpad.com>
+ * Copyright (c) 2015-2016 Ruslan Bukin <br@bsdpad.com>
* All rights reserved.
*
* Portions of this software were developed by SRI International and the
@@ -55,13 +55,13 @@ ENTRY(cpu_throw)
sfence.vm
/* Switch to the new pmap */
- la t0, pagetable_l0
ld t1, PCB_L1ADDR(x13) /* Link to next level PN */
srli t1, t1, PAGE_SHIFT /* PN no */
li t2, (PTE_VALID | (PTE_TYPE_PTR << PTE_TYPE_S))
slli t3, t1, PTE_PPN0_S /* (t1 << PTE_PPN0_S) */
or t4, t2, t3
/* Store single level0 PTE entry to position */
+ ld t0, PC_SPTBR(gp)
sd t4, 0(t0)
/* TODO: Invalidate the TLB */
@@ -140,13 +140,13 @@ ENTRY(cpu_switch)
sfence.vm
/* Switch to the new pmap */
- la t0, pagetable_l0
ld t1, PCB_L1ADDR(x13) /* Link to next level PN */
srli t1, t1, PAGE_SHIFT /* PN no */
li t2, (PTE_VALID | (PTE_TYPE_PTR << PTE_TYPE_S))
slli t3, t1, PTE_PPN0_S /* (t1 << PTE_PPN0_S) */
or t4, t2, t3
/* Store single level0 PTE entry to position */
+ ld t0, PC_SPTBR(gp)
sd t4, 0(t0)
/* TODO: Invalidate the TLB */
@@ -156,7 +156,11 @@ ENTRY(cpu_switch)
/* Release the old thread */
sd a2, TD_LOCK(a0)
#if defined(SCHED_ULE) && defined(SMP)
- /* TODO */
+ /* Spin if TD_LOCK points to a blocked_lock */
+ la a2, _C_LABEL(blocked_lock)
+1:
+ ld t0, TD_LOCK(a1)
+ beq t0, a2, 1b
#endif
/* Restore the registers */
OpenPOWER on IntegriCloud