diff options
Diffstat (limited to 'sys')
135 files changed, 6335 insertions, 981 deletions
diff --git a/sys/amd64/amd64/machdep.c b/sys/amd64/amd64/machdep.c index db1b77a..6bf251f 100644 --- a/sys/amd64/amd64/machdep.c +++ b/sys/amd64/amd64/machdep.c @@ -188,6 +188,12 @@ extern char kernphys[]; struct msgbuf *msgbufp; +/* + * Physical address of the EFI System Table. Stashed from the metadata hints + * passed into the kernel and used by the EFI code to call runtime services. + */ +vm_paddr_t efi_systbl; + /* Intel ICH registers */ #define ICH_PMBASE 0x400 #define ICH_SMI_EN ICH_PMBASE + 0x30 @@ -1495,6 +1501,7 @@ native_parse_preload_data(u_int64_t modulep) ksym_end = MD_FETCH(kmdp, MODINFOMD_ESYM, uintptr_t); db_fetch_ksymtab(ksym_start, ksym_end); #endif + efi_systbl = MD_FETCH(kmdp, MODINFOMD_FW_HANDLE, vm_paddr_t); return (kmdp); } @@ -1615,6 +1622,8 @@ hammer_time(u_int64_t modulep, u_int64_t physfree) /* * Use vt(4) by default for UEFI boot (during the sc(4)/vt(4) * transition). + * Once bootblocks have updated, we can test directly for + * efi_systbl != NULL here... */ if (preload_search_info(kmdp, MODINFO_METADATA | MODINFOMD_EFI_MAP) != NULL) diff --git a/sys/arm/allwinner/a20/a20_mp.c b/sys/arm/allwinner/a20/a20_mp.c index 4beac42..27cbb3d 100644 --- a/sys/arm/allwinner/a20/a20_mp.c +++ b/sys/arm/allwinner/a20/a20_mp.c @@ -61,7 +61,7 @@ void platform_mp_init_secondary(void) { - arm_pic_init_secondary(); + intr_pic_init_secondary(); } void diff --git a/sys/arm/altera/socfpga/socfpga_mp.c b/sys/arm/altera/socfpga/socfpga_mp.c index 461d3f4..46977f6 100644 --- a/sys/arm/altera/socfpga/socfpga_mp.c +++ b/sys/arm/altera/socfpga/socfpga_mp.c @@ -87,7 +87,7 @@ void platform_mp_init_secondary(void) { - arm_pic_init_secondary(); + intr_pic_init_secondary(); } void diff --git a/sys/arm/amlogic/aml8726/aml8726_mp.c b/sys/arm/amlogic/aml8726/aml8726_mp.c index 1d7ef03..779a793 100644 --- a/sys/arm/amlogic/aml8726/aml8726_mp.c +++ b/sys/arm/amlogic/aml8726/aml8726_mp.c @@ -351,7 +351,7 @@ platform_mp_init_secondary(void) * each AP. */ - arm_pic_init_secondary(); + intr_pic_init_secondary(); } diff --git a/sys/arm/annapurna/alpine/alpine_machdep_mp.c b/sys/arm/annapurna/alpine/alpine_machdep_mp.c index 4420025..75bc63a 100644 --- a/sys/arm/annapurna/alpine/alpine_machdep_mp.c +++ b/sys/arm/annapurna/alpine/alpine_machdep_mp.c @@ -122,7 +122,7 @@ void platform_mp_init_secondary(void) { - arm_pic_init_secondary(); + intr_pic_init_secondary(); } void diff --git a/sys/arm/arm/exception.S b/sys/arm/arm/exception.S index 5acaf15..9f186a5 100644 --- a/sys/arm/arm/exception.S +++ b/sys/arm/arm/exception.S @@ -410,7 +410,7 @@ ASENTRY_NP(irq_entry) PUSHFRAMEINSVC /* mode stack, build trapframe there. */ adr lr, exception_exit /* Return from handler via standard */ mov r0, sp /* exception exit routine. Pass the */ - b _C_LABEL(arm_irq_handler)/* trapframe to the handler. */ + b _C_LABEL(intr_irq_handler)/* trapframe to the handler. */ END(irq_entry) /* diff --git a/sys/arm/arm/gic.c b/sys/arm/arm/gic.c index f570cb4..c220d62 100644 --- a/sys/arm/arm/gic.c +++ b/sys/arm/arm/gic.c @@ -120,14 +120,14 @@ __FBSDID("$FreeBSD$"); #ifdef ARM_INTRNG static u_int gic_irq_cpu; static int arm_gic_intr(void *); -static int arm_gic_bind(device_t dev, struct arm_irqsrc *isrc); +static int arm_gic_bind(device_t dev, struct intr_irqsrc *isrc); #endif struct arm_gic_softc { device_t gic_dev; #ifdef ARM_INTRNG void * gic_intrhand; - struct arm_irqsrc ** gic_irqs; + struct intr_irqsrc ** gic_irqs; #endif struct resource * gic_res[3]; bus_space_tag_t gic_c_bst; @@ -216,7 +216,7 @@ static void arm_gic_init_secondary(device_t dev) { struct arm_gic_softc *sc = device_get_softc(dev); - struct arm_irqsrc *isrc; + struct intr_irqsrc *isrc; u_int irq; for (irq = 0; irq < sc->nirqs; irq += 4) @@ -250,7 +250,7 @@ arm_gic_init_secondary(device_t dev) isrc = sc->gic_irqs[irq]; if (isrc == NULL || isrc->isrc_handlers == 0) continue; - if (isrc->isrc_flags & ARM_ISRCF_BOUND) { + if (isrc->isrc_flags & INTR_ISRCF_BOUND) { if (CPU_ISSET(PCPU_GET(cpuid), &isrc->isrc_cpu)) gic_irq_unmask(sc, irq); } else { @@ -456,7 +456,7 @@ arm_gic_attach(device_t dev) * Now, when everything is initialized, it's right time to * register interrupt controller to interrupt framefork. */ - if (arm_pic_register(dev, xref) != 0) { + if (intr_pic_register(dev, xref) != 0) { device_printf(dev, "could not register PIC\n"); goto cleanup; } @@ -468,23 +468,23 @@ arm_gic_attach(device_t dev) */ pxref = ofw_bus_find_iparent(ofw_bus_get_node(dev)); if (pxref == 0 || xref == pxref) { - if (arm_pic_claim_root(dev, xref, arm_gic_intr, sc, + if (intr_pic_claim_root(dev, xref, arm_gic_intr, sc, GIC_LAST_SGI - GIC_FIRST_SGI + 1) != 0) { device_printf(dev, "could not set PIC as a root\n"); - arm_pic_unregister(dev, xref); + intr_pic_unregister(dev, xref); goto cleanup; } } else { if (sc->gic_res[2] == NULL) { device_printf(dev, "not root PIC must have defined interrupt\n"); - arm_pic_unregister(dev, xref); + intr_pic_unregister(dev, xref); goto cleanup; } if (bus_setup_intr(dev, sc->gic_res[2], INTR_TYPE_CLK, arm_gic_intr, NULL, sc, &sc->gic_intrhand)) { device_printf(dev, "could not setup irq handler\n"); - arm_pic_unregister(dev, xref); + intr_pic_unregister(dev, xref); goto cleanup; } } @@ -508,7 +508,7 @@ static int arm_gic_intr(void *arg) { struct arm_gic_softc *sc = arg; - struct arm_irqsrc *isrc; + struct intr_irqsrc *isrc; uint32_t irq_active_reg, irq; struct trapframe *tf; @@ -560,7 +560,7 @@ dispatch_irq: #ifdef SMP /* Call EOI for all IPI before dispatch. */ gic_c_write_4(sc, GICC_EOIR, irq_active_reg); - arm_ipi_dispatch(isrc, tf); + intr_ipi_dispatch(isrc, tf); goto next_irq; #else device_printf(sc->gic_dev, "SGI %u on UP system detected\n", @@ -576,7 +576,7 @@ dispatch_irq: if (isrc->isrc_trig == INTR_TRIGGER_EDGE) gic_c_write_4(sc, GICC_EOIR, irq_active_reg); - arm_irq_dispatch(isrc, tf); + intr_irq_dispatch(isrc, tf); next_irq: arm_irq_memory_barrier(irq); @@ -589,7 +589,7 @@ next_irq: } static int -gic_attach_isrc(struct arm_gic_softc *sc, struct arm_irqsrc *isrc, u_int irq) +gic_attach_isrc(struct arm_gic_softc *sc, struct intr_irqsrc *isrc, u_int irq) { const char *name; @@ -609,16 +609,16 @@ gic_attach_isrc(struct arm_gic_softc *sc, struct arm_irqsrc *isrc, u_int irq) name = device_get_nameunit(sc->gic_dev); if (irq <= GIC_LAST_SGI) - arm_irq_set_name(isrc, "%s,i%u", name, irq - GIC_FIRST_SGI); + intr_irq_set_name(isrc, "%s,i%u", name, irq - GIC_FIRST_SGI); else if (irq <= GIC_LAST_PPI) - arm_irq_set_name(isrc, "%s,p%u", name, irq - GIC_FIRST_PPI); + intr_irq_set_name(isrc, "%s,p%u", name, irq - GIC_FIRST_PPI); else - arm_irq_set_name(isrc, "%s,s%u", name, irq - GIC_FIRST_SPI); + intr_irq_set_name(isrc, "%s,s%u", name, irq - GIC_FIRST_SPI); return (0); } static int -gic_detach_isrc(struct arm_gic_softc *sc, struct arm_irqsrc *isrc, u_int irq) +gic_detach_isrc(struct arm_gic_softc *sc, struct intr_irqsrc *isrc, u_int irq) { mtx_lock_spin(&sc->mutex); @@ -630,7 +630,7 @@ gic_detach_isrc(struct arm_gic_softc *sc, struct arm_irqsrc *isrc, u_int irq) isrc->isrc_data = 0; mtx_unlock_spin(&sc->mutex); - arm_irq_set_name(isrc, ""); + intr_irq_set_name(isrc, ""); return (0); } @@ -696,15 +696,15 @@ gic_irq_from_nspc(struct arm_gic_softc *sc, u_int type, u_int num, u_int *irqp) { switch (type) { - case ARM_IRQ_NSPC_PLAIN: + case INTR_IRQ_NSPC_PLAIN: *irqp = num; return (*irqp < sc->nirqs ? 0 : EINVAL); - case ARM_IRQ_NSPC_IRQ: + case INTR_IRQ_NSPC_IRQ: *irqp = num + GIC_FIRST_PPI; return (*irqp < sc->nirqs ? 0 : EINVAL); - case ARM_IRQ_NSPC_IPI: + case INTR_IRQ_NSPC_IPI: *irqp = num + GIC_FIRST_SGI; return (*irqp < GIC_LAST_SGI ? 0 : EINVAL); @@ -714,7 +714,7 @@ gic_irq_from_nspc(struct arm_gic_softc *sc, u_int type, u_int num, u_int *irqp) } static int -gic_map_nspc(struct arm_gic_softc *sc, struct arm_irqsrc *isrc, u_int *irqp) +gic_map_nspc(struct arm_gic_softc *sc, struct intr_irqsrc *isrc, u_int *irqp) { int error; @@ -727,7 +727,7 @@ gic_map_nspc(struct arm_gic_softc *sc, struct arm_irqsrc *isrc, u_int *irqp) #ifdef FDT static int -gic_map_fdt(struct arm_gic_softc *sc, struct arm_irqsrc *isrc, u_int *irqp) +gic_map_fdt(struct arm_gic_softc *sc, struct intr_irqsrc *isrc, u_int *irqp) { u_int irq, tripol; enum intr_trigger trig; @@ -773,7 +773,7 @@ gic_map_fdt(struct arm_gic_softc *sc, struct arm_irqsrc *isrc, u_int *irqp) if (error != 0) return (error); - isrc->isrc_nspc_type = ARM_IRQ_NSPC_PLAIN; + isrc->isrc_nspc_type = INTR_IRQ_NSPC_PLAIN; isrc->isrc_nspc_num = irq; isrc->isrc_trig = trig; isrc->isrc_pol = pol; @@ -784,16 +784,16 @@ gic_map_fdt(struct arm_gic_softc *sc, struct arm_irqsrc *isrc, u_int *irqp) #endif static int -arm_gic_register(device_t dev, struct arm_irqsrc *isrc, boolean_t *is_percpu) +arm_gic_register(device_t dev, struct intr_irqsrc *isrc, boolean_t *is_percpu) { struct arm_gic_softc *sc = device_get_softc(dev); u_int irq; int error; - if (isrc->isrc_type == ARM_ISRCT_NAMESPACE) + if (isrc->isrc_type == INTR_ISRCT_NAMESPACE) error = gic_map_nspc(sc, isrc, &irq); #ifdef FDT - else if (isrc->isrc_type == ARM_ISRCT_FDT) + else if (isrc->isrc_type == INTR_ISRCT_FDT) error = gic_map_fdt(sc, isrc, &irq); #endif else @@ -805,7 +805,7 @@ arm_gic_register(device_t dev, struct arm_irqsrc *isrc, boolean_t *is_percpu) } static void -arm_gic_enable_intr(device_t dev, struct arm_irqsrc *isrc) +arm_gic_enable_intr(device_t dev, struct intr_irqsrc *isrc) { struct arm_gic_softc *sc = device_get_softc(dev); u_int irq = isrc->isrc_data; @@ -820,7 +820,7 @@ arm_gic_enable_intr(device_t dev, struct arm_irqsrc *isrc) * pic_enable_source() and pic_disable_source() should act on * per CPU basis only. Thus, it should be solved here somehow. */ - if (isrc->isrc_flags & ARM_ISRCF_PERCPU) + if (isrc->isrc_flags & INTR_ISRCF_PERCPU) CPU_SET(PCPU_GET(cpuid), &isrc->isrc_cpu); gic_config(sc, irq, isrc->isrc_trig, isrc->isrc_pol); @@ -828,7 +828,7 @@ arm_gic_enable_intr(device_t dev, struct arm_irqsrc *isrc) } static void -arm_gic_enable_source(device_t dev, struct arm_irqsrc *isrc) +arm_gic_enable_source(device_t dev, struct intr_irqsrc *isrc) { struct arm_gic_softc *sc = device_get_softc(dev); u_int irq = isrc->isrc_data; @@ -838,7 +838,7 @@ arm_gic_enable_source(device_t dev, struct arm_irqsrc *isrc) } static void -arm_gic_disable_source(device_t dev, struct arm_irqsrc *isrc) +arm_gic_disable_source(device_t dev, struct intr_irqsrc *isrc) { struct arm_gic_softc *sc = device_get_softc(dev); u_int irq = isrc->isrc_data; @@ -847,7 +847,7 @@ arm_gic_disable_source(device_t dev, struct arm_irqsrc *isrc) } static int -arm_gic_unregister(device_t dev, struct arm_irqsrc *isrc) +arm_gic_unregister(device_t dev, struct intr_irqsrc *isrc) { struct arm_gic_softc *sc = device_get_softc(dev); u_int irq = isrc->isrc_data; @@ -856,7 +856,7 @@ arm_gic_unregister(device_t dev, struct arm_irqsrc *isrc) } static void -arm_gic_pre_ithread(device_t dev, struct arm_irqsrc *isrc) +arm_gic_pre_ithread(device_t dev, struct intr_irqsrc *isrc) { struct arm_gic_softc *sc = device_get_softc(dev); @@ -865,7 +865,7 @@ arm_gic_pre_ithread(device_t dev, struct arm_irqsrc *isrc) } static void -arm_gic_post_ithread(device_t dev, struct arm_irqsrc *isrc) +arm_gic_post_ithread(device_t dev, struct intr_irqsrc *isrc) { arm_irq_memory_barrier(0); @@ -873,7 +873,7 @@ arm_gic_post_ithread(device_t dev, struct arm_irqsrc *isrc) } static void -arm_gic_post_filter(device_t dev, struct arm_irqsrc *isrc) +arm_gic_post_filter(device_t dev, struct intr_irqsrc *isrc) { struct arm_gic_softc *sc = device_get_softc(dev); @@ -886,7 +886,7 @@ arm_gic_post_filter(device_t dev, struct arm_irqsrc *isrc) } static int -arm_gic_bind(device_t dev, struct arm_irqsrc *isrc) +arm_gic_bind(device_t dev, struct intr_irqsrc *isrc) { struct arm_gic_softc *sc = device_get_softc(dev); uint32_t irq = isrc->isrc_data; @@ -895,7 +895,7 @@ arm_gic_bind(device_t dev, struct arm_irqsrc *isrc) return (EINVAL); if (CPU_EMPTY(&isrc->isrc_cpu)) { - gic_irq_cpu = arm_irq_next_cpu(gic_irq_cpu, &all_cpus); + gic_irq_cpu = intr_irq_next_cpu(gic_irq_cpu, &all_cpus); CPU_SETOF(gic_irq_cpu, &isrc->isrc_cpu); } return (gic_bind(sc, irq, &isrc->isrc_cpu)); @@ -903,7 +903,7 @@ arm_gic_bind(device_t dev, struct arm_irqsrc *isrc) #ifdef SMP static void -arm_gic_ipi_send(device_t dev, struct arm_irqsrc *isrc, cpuset_t cpus) +arm_gic_ipi_send(device_t dev, struct intr_irqsrc *isrc, cpuset_t cpus) { struct arm_gic_softc *sc = device_get_softc(dev); uint32_t irq, val = 0, i; @@ -1097,7 +1097,7 @@ arm_get_next_irq(int last_irq) #ifdef SMP void -arm_pic_init_secondary(void) +intr_pic_init_secondary(void) { arm_gic_init_secondary(gic_sc->gic_dev); diff --git a/sys/arm/arm/intr.c b/sys/arm/arm/intr.c index 7ab87cc..2528b8e 100644 --- a/sys/arm/arm/intr.c +++ b/sys/arm/arm/intr.c @@ -70,7 +70,7 @@ typedef void (*mask_fn)(void *); static struct intr_event *intr_events[NIRQ]; -void arm_irq_handler(struct trapframe *); +void intr_irq_handler(struct trapframe *); void (*arm_post_filter)(void *) = NULL; int (*arm_config_irq)(int irq, enum intr_trigger trig, @@ -104,7 +104,7 @@ SYSINIT(intr_init, SI_SUB_INTR, SI_ORDER_FIRST, intr_init, NULL); #ifdef FDT int -arm_fdt_map_irq(phandle_t iparent, pcell_t *intr, int icells) +intr_fdt_map_irq(phandle_t iparent, pcell_t *intr, int icells) { fdt_pic_decode_t intr_decode; phandle_t intr_parent; @@ -179,7 +179,7 @@ dosoftints(void) } void -arm_irq_handler(struct trapframe *frame) +intr_irq_handler(struct trapframe *frame) { struct intr_event *event; int i; @@ -199,67 +199,3 @@ arm_irq_handler(struct trapframe *frame) pmc_hook(PCPU_GET(curthread), PMC_FN_USER_CALLCHAIN, frame); #endif } - -/* - * arm_irq_memory_barrier() - * - * Ensure all writes to device memory have reached devices before proceeding. - * - * This is intended to be called from the post-filter and post-thread routines - * of an interrupt controller implementation. A peripheral device driver should - * use bus_space_barrier() if it needs to ensure a write has reached the - * hardware for some reason other than clearing interrupt conditions. - * - * The need for this function arises from the ARM weak memory ordering model. - * Writes to locations mapped with the Device attribute bypass any caches, but - * are buffered. Multiple writes to the same device will be observed by that - * device in the order issued by the cpu. Writes to different devices may - * appear at those devices in a different order than issued by the cpu. That - * is, if the cpu writes to device A then device B, the write to device B could - * complete before the write to device A. - * - * Consider a typical device interrupt handler which services the interrupt and - * writes to a device status-acknowledge register to clear the interrupt before - * returning. That write is posted to the L2 controller which "immediately" - * places it in a store buffer and automatically drains that buffer. This can - * be less immediate than you'd think... There may be no free slots in the store - * buffers, so an existing buffer has to be drained first to make room. The - * target bus may be busy with other traffic (such as DMA for various devices), - * delaying the drain of the store buffer for some indeterminate time. While - * all this delay is happening, execution proceeds on the CPU, unwinding its way - * out of the interrupt call stack to the point where the interrupt driver code - * is ready to EOI and unmask the interrupt. The interrupt controller may be - * accessed via a faster bus than the hardware whose handler just ran; the write - * to unmask and EOI the interrupt may complete quickly while the device write - * to ack and clear the interrupt source is still lingering in a store buffer - * waiting for access to a slower bus. With the interrupt unmasked at the - * interrupt controller but still active at the device, as soon as interrupts - * are enabled on the core the device re-interrupts immediately: now you've got - * a spurious interrupt on your hands. - * - * The right way to fix this problem is for every device driver to use the - * proper bus_space_barrier() calls in its interrupt handler. For ARM a single - * barrier call at the end of the handler would work. This would have to be - * done to every driver in the system, not just arm-specific drivers. - * - * Another potential fix is to map all device memory as Strongly-Ordered rather - * than Device memory, which takes the store buffers out of the picture. This - * has a pretty big impact on overall system performance, because each strongly - * ordered memory access causes all L2 store buffers to be drained. - * - * A compromise solution is to have the interrupt controller implementation call - * this function to establish a barrier between writes to the interrupt-source - * device and writes to the interrupt controller device. - * - * This takes the interrupt number as an argument, and currently doesn't use it. - * The plan is that maybe some day there is a way to flag certain interrupts as - * "memory barrier safe" and we can avoid this overhead with them. - */ -void -arm_irq_memory_barrier(uintptr_t irq) -{ - - dsb(); - cpu_l2cache_drain_writebuf(); -} - diff --git a/sys/arm/arm/machdep_intr.c b/sys/arm/arm/machdep_intr.c new file mode 100644 index 0000000..014cdc5 --- /dev/null +++ b/sys/arm/arm/machdep_intr.c @@ -0,0 +1,123 @@ +/* $NetBSD: intr.c,v 1.12 2003/07/15 00:24:41 lukem Exp $ */ + +/*- + * Copyright (c) 2004 Olivier Houchard. + * Copyright (c) 1994-1998 Mark Brinicombe. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by Mark Brinicombe + * for the NetBSD Project. + * 4. The name of the company nor the name of the author may be used to + * endorse or promote products derived from this software without specific + * prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * Soft interrupt and other generic interrupt functions. + */ + +#include "opt_platform.h" + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/syslog.h> +#include <sys/kernel.h> +#include <sys/malloc.h> +#include <sys/proc.h> +#include <sys/bus.h> +#include <sys/interrupt.h> +#include <sys/conf.h> +#include <sys/pmc.h> +#include <sys/pmckern.h> + +#include <machine/atomic.h> +#include <machine/bus.h> +#include <machine/intr.h> +#include <machine/cpu.h> + +/* + * arm_irq_memory_barrier() + * + * Ensure all writes to device memory have reached devices before proceeding. + * + * This is intended to be called from the post-filter and post-thread routines + * of an interrupt controller implementation. A peripheral device driver should + * use bus_space_barrier() if it needs to ensure a write has reached the + * hardware for some reason other than clearing interrupt conditions. + * + * The need for this function arises from the ARM weak memory ordering model. + * Writes to locations mapped with the Device attribute bypass any caches, but + * are buffered. Multiple writes to the same device will be observed by that + * device in the order issued by the cpu. Writes to different devices may + * appear at those devices in a different order than issued by the cpu. That + * is, if the cpu writes to device A then device B, the write to device B could + * complete before the write to device A. + * + * Consider a typical device interrupt handler which services the interrupt and + * writes to a device status-acknowledge register to clear the interrupt before + * returning. That write is posted to the L2 controller which "immediately" + * places it in a store buffer and automatically drains that buffer. This can + * be less immediate than you'd think... There may be no free slots in the store + * buffers, so an existing buffer has to be drained first to make room. The + * target bus may be busy with other traffic (such as DMA for various devices), + * delaying the drain of the store buffer for some indeterminate time. While + * all this delay is happening, execution proceeds on the CPU, unwinding its way + * out of the interrupt call stack to the point where the interrupt driver code + * is ready to EOI and unmask the interrupt. The interrupt controller may be + * accessed via a faster bus than the hardware whose handler just ran; the write + * to unmask and EOI the interrupt may complete quickly while the device write + * to ack and clear the interrupt source is still lingering in a store buffer + * waiting for access to a slower bus. With the interrupt unmasked at the + * interrupt controller but still active at the device, as soon as interrupts + * are enabled on the core the device re-interrupts immediately: now you've got + * a spurious interrupt on your hands. + * + * The right way to fix this problem is for every device driver to use the + * proper bus_space_barrier() calls in its interrupt handler. For ARM a single + * barrier call at the end of the handler would work. This would have to be + * done to every driver in the system, not just arm-specific drivers. + * + * Another potential fix is to map all device memory as Strongly-Ordered rather + * than Device memory, which takes the store buffers out of the picture. This + * has a pretty big impact on overall system performance, because each strongly + * ordered memory access causes all L2 store buffers to be drained. + * + * A compromise solution is to have the interrupt controller implementation call + * this function to establish a barrier between writes to the interrupt-source + * device and writes to the interrupt controller device. + * + * This takes the interrupt number as an argument, and currently doesn't use it. + * The plan is that maybe some day there is a way to flag certain interrupts as + * "memory barrier safe" and we can avoid this overhead with them. + */ +void +arm_irq_memory_barrier(uintptr_t irq) +{ + + dsb(); + cpu_l2cache_drain_writebuf(); +} + diff --git a/sys/arm/arm/mp_machdep.c b/sys/arm/arm/mp_machdep.c index 19ff079..60bb598 100644 --- a/sys/arm/arm/mp_machdep.c +++ b/sys/arm/arm/mp_machdep.c @@ -443,12 +443,12 @@ release_aps(void *dummy __unused) return; #ifdef ARM_INTRNG - arm_ipi_set_handler(IPI_RENDEZVOUS, "rendezvous", ipi_rendezvous, NULL, 0); - arm_ipi_set_handler(IPI_AST, "ast", ipi_ast, NULL, 0); - arm_ipi_set_handler(IPI_STOP, "stop", ipi_stop, NULL, 0); - arm_ipi_set_handler(IPI_PREEMPT, "preempt", ipi_preempt, NULL, 0); - arm_ipi_set_handler(IPI_HARDCLOCK, "hardclock", ipi_hardclock, NULL, 0); - arm_ipi_set_handler(IPI_TLB, "tlb", ipi_tlb, NULL, 0); + intr_ipi_set_handler(IPI_RENDEZVOUS, "rendezvous", ipi_rendezvous, NULL, 0); + intr_ipi_set_handler(IPI_AST, "ast", ipi_ast, NULL, 0); + intr_ipi_set_handler(IPI_STOP, "stop", ipi_stop, NULL, 0); + intr_ipi_set_handler(IPI_PREEMPT, "preempt", ipi_preempt, NULL, 0); + intr_ipi_set_handler(IPI_HARDCLOCK, "hardclock", ipi_hardclock, NULL, 0); + intr_ipi_set_handler(IPI_TLB, "tlb", ipi_tlb, NULL, 0); #else #ifdef IPI_IRQ_START diff --git a/sys/arm/arm/nexus.c b/sys/arm/arm/nexus.c index e2deeb0..8dce36c 100644 --- a/sys/arm/arm/nexus.c +++ b/sys/arm/arm/nexus.c @@ -267,7 +267,7 @@ nexus_config_intr(device_t dev, int irq, enum intr_trigger trig, int ret = ENODEV; #ifdef ARM_INTRNG - ret = arm_irq_config(irq, trig, pol); + ret = intr_irq_config(irq, trig, pol); #else if (arm_config_irq) ret = (*arm_config_irq)(irq, trig, pol); @@ -286,7 +286,7 @@ nexus_setup_intr(device_t dev, device_t child, struct resource *res, int flags, for (irq = rman_get_start(res); irq <= rman_get_end(res); irq++) { #ifdef ARM_INTRNG - arm_irq_add_handler(child, filt, intr, arg, irq, flags, + intr_irq_add_handler(child, filt, intr, arg, irq, flags, cookiep); #else arm_setup_irqhandler(device_get_nameunit(child), @@ -302,7 +302,7 @@ nexus_teardown_intr(device_t dev, device_t child, struct resource *r, void *ih) { #ifdef ARM_INTRNG - return (arm_irq_remove_handler(child, rman_get_start(r), ih)); + return (intr_irq_remove_handler(child, rman_get_start(r), ih)); #else return (arm_remove_irqhandler(rman_get_start(r), ih)); #endif @@ -314,7 +314,7 @@ nexus_describe_intr(device_t dev, device_t child, struct resource *irq, void *cookie, const char *descr) { - return (arm_irq_describe(rman_get_start(irq), cookie, descr)); + return (intr_irq_describe(rman_get_start(irq), cookie, descr)); } #ifdef SMP @@ -322,7 +322,7 @@ static int nexus_bind_intr(device_t dev, device_t child, struct resource *irq, int cpu) { - return (arm_irq_bind(rman_get_start(irq), cpu)); + return (intr_irq_bind(rman_get_start(irq), cpu)); } #endif #endif @@ -396,6 +396,6 @@ nexus_ofw_map_intr(device_t dev, device_t child, phandle_t iparent, int icells, pcell_t *intr) { - return (arm_fdt_map_irq(iparent, intr, icells)); + return (intr_fdt_map_irq(iparent, intr, icells)); } #endif diff --git a/sys/arm/freescale/imx/imx6_mp.c b/sys/arm/freescale/imx/imx6_mp.c index 8957c54..3208f67 100644 --- a/sys/arm/freescale/imx/imx6_mp.c +++ b/sys/arm/freescale/imx/imx6_mp.c @@ -69,7 +69,7 @@ void platform_mp_init_secondary(void) { - arm_pic_init_secondary(); + intr_pic_init_secondary(); } void diff --git a/sys/arm/freescale/imx/imx_gpio.c b/sys/arm/freescale/imx/imx_gpio.c index 3c10efa..3c81e28 100644 --- a/sys/arm/freescale/imx/imx_gpio.c +++ b/sys/arm/freescale/imx/imx_gpio.c @@ -101,7 +101,7 @@ struct imx51_gpio_softc { bus_space_handle_t sc_ioh; int gpio_npins; struct gpio_pin gpio_pins[NGPIO]; - struct arm_irqsrc *gpio_pic_irqsrc[NGPIO]; + struct intr_irqsrc *gpio_pic_irqsrc[NGPIO]; }; static struct ofw_compat_data compat_data[] = { @@ -149,7 +149,7 @@ static int imx51_gpio_pin_toggle(device_t, uint32_t pin); * this is teardown_intr */ static void -gpio_pic_disable_intr(device_t dev, struct arm_irqsrc *isrc) +gpio_pic_disable_intr(device_t dev, struct intr_irqsrc *isrc) { struct imx51_gpio_softc *sc; u_int irq; @@ -168,7 +168,7 @@ gpio_pic_disable_intr(device_t dev, struct arm_irqsrc *isrc) * this is mask_intr */ static void -gpio_pic_disable_source(device_t dev, struct arm_irqsrc *isrc) +gpio_pic_disable_source(device_t dev, struct intr_irqsrc *isrc) { struct imx51_gpio_softc *sc; @@ -183,7 +183,7 @@ gpio_pic_disable_source(device_t dev, struct arm_irqsrc *isrc) * this is setup_intr */ static void -gpio_pic_enable_intr(device_t dev, struct arm_irqsrc *isrc) +gpio_pic_enable_intr(device_t dev, struct intr_irqsrc *isrc) { struct imx51_gpio_softc *sc; int icfg; @@ -226,7 +226,7 @@ gpio_pic_enable_intr(device_t dev, struct arm_irqsrc *isrc) * this is unmask_intr */ static void -gpio_pic_enable_source(device_t dev, struct arm_irqsrc *isrc) +gpio_pic_enable_source(device_t dev, struct intr_irqsrc *isrc) { struct imx51_gpio_softc *sc; @@ -238,7 +238,7 @@ gpio_pic_enable_source(device_t dev, struct arm_irqsrc *isrc) } static void -gpio_pic_post_filter(device_t dev, struct arm_irqsrc *isrc) +gpio_pic_post_filter(device_t dev, struct intr_irqsrc *isrc) { struct imx51_gpio_softc *sc; @@ -250,7 +250,7 @@ gpio_pic_post_filter(device_t dev, struct arm_irqsrc *isrc) } static void -gpio_pic_post_ithread(device_t dev, struct arm_irqsrc *isrc) +gpio_pic_post_ithread(device_t dev, struct intr_irqsrc *isrc) { arm_irq_memory_barrier(0); @@ -258,7 +258,7 @@ gpio_pic_post_ithread(device_t dev, struct arm_irqsrc *isrc) } static void -gpio_pic_pre_ithread(device_t dev, struct arm_irqsrc *isrc) +gpio_pic_pre_ithread(device_t dev, struct intr_irqsrc *isrc) { gpio_pic_disable_source(dev, isrc); @@ -268,7 +268,7 @@ gpio_pic_pre_ithread(device_t dev, struct arm_irqsrc *isrc) * intrng calls this to make a new isrc known to us. */ static int -gpio_pic_register(device_t dev, struct arm_irqsrc *isrc, boolean_t *is_percpu) +gpio_pic_register(device_t dev, struct intr_irqsrc *isrc, boolean_t *is_percpu) { struct imx51_gpio_softc *sc; u_int irq, tripol; @@ -320,7 +320,7 @@ gpio_pic_register(device_t dev, struct arm_irqsrc *isrc, boolean_t *is_percpu) tripol); return (ENOTSUP); } - isrc->isrc_nspc_type = ARM_IRQ_NSPC_PLAIN; + isrc->isrc_nspc_type = INTR_IRQ_NSPC_PLAIN; isrc->isrc_nspc_num = irq; /* @@ -337,12 +337,12 @@ gpio_pic_register(device_t dev, struct arm_irqsrc *isrc, boolean_t *is_percpu) isrc->isrc_data = irq; mtx_unlock_spin(&sc->sc_mtx); - arm_irq_set_name(isrc, "%s,%u", device_get_nameunit(sc->dev), irq); + intr_irq_set_name(isrc, "%s,%u", device_get_nameunit(sc->dev), irq); return (0); } static int -gpio_pic_unregister(device_t dev, struct arm_irqsrc *isrc) +gpio_pic_unregister(device_t dev, struct intr_irqsrc *isrc) { struct imx51_gpio_softc *sc; u_int irq; @@ -359,7 +359,7 @@ gpio_pic_unregister(device_t dev, struct arm_irqsrc *isrc) isrc->isrc_data = 0; mtx_unlock_spin(&sc->sc_mtx); - arm_irq_set_name(isrc, ""); + intr_irq_set_name(isrc, ""); return (0); } @@ -378,7 +378,7 @@ gpio_pic_filter(void *arg) if ((interrupts & 0x1) == 0) continue; if (sc->gpio_pic_irqsrc[i]) - arm_irq_dispatch(sc->gpio_pic_irqsrc[i], curthread->td_intr_frame); + intr_irq_dispatch(sc->gpio_pic_irqsrc[i], curthread->td_intr_frame); else device_printf(sc->dev, "spurious interrupt %d\n", i); } @@ -656,7 +656,7 @@ imx51_gpio_attach(device_t dev) } #ifdef ARM_INTRNG - arm_pic_register(dev, OF_xref_from_node(ofw_bus_get_node(dev))); + intr_pic_register(dev, OF_xref_from_node(ofw_bus_get_node(dev))); #endif sc->sc_busdev = gpiobus_attach_bus(dev); diff --git a/sys/arm/freescale/imx/imx_sdhci.c b/sys/arm/freescale/imx/imx_sdhci.c index 6acc86e..17edf07 100644 --- a/sys/arm/freescale/imx/imx_sdhci.c +++ b/sys/arm/freescale/imx/imx_sdhci.c @@ -522,12 +522,19 @@ imx_sdhc_set_clock(struct imx_sdhci_softc *sc, int enable) else freq = sc->baseclk_hz / (2 * divisor); - for (prescale = 2; prescale < freq / prescale / 16;) + for (prescale = 2; freq < sc->baseclk_hz / (prescale * 16);) prescale <<= 1; - for (divisor = 1; freq < freq / prescale / divisor;) + for (divisor = 1; freq < sc->baseclk_hz / (prescale * divisor);) ++divisor; +#ifdef DEBUG + device_printf(sc->dev, + "desired SD freq: %d, actual: %d; base %d prescale %d divisor %d\n", + freq, sc->baseclk_hz / (prescale * divisor), sc->baseclk_hz, + prescale, divisor); +#endif + prescale >>= 1; divisor -= 1; diff --git a/sys/arm/include/intr.h b/sys/arm/include/intr.h index 2bb479a..83a3937 100644 --- a/sys/arm/include/intr.h +++ b/sys/arm/include/intr.h @@ -51,33 +51,33 @@ #ifdef notyet #define INTR_SOLO INTR_MD1 -typedef int arm_irq_filter_t(void *arg, struct trapframe *tf); +typedef int intr_irq_filter_t(void *arg, struct trapframe *tf); #else -typedef int arm_irq_filter_t(void *arg); +typedef int intr_irq_filter_t(void *arg); #endif -#define ARM_ISRC_NAMELEN (MAXCOMLEN + 1) +#define INTR_ISRC_NAMELEN (MAXCOMLEN + 1) -typedef void arm_ipi_filter_t(void *arg); +typedef void intr_ipi_filter_t(void *arg); -enum arm_isrc_type { - ARM_ISRCT_NAMESPACE, - ARM_ISRCT_FDT +enum intr_isrc_type { + INTR_ISRCT_NAMESPACE, + INTR_ISRCT_FDT }; -#define ARM_ISRCF_REGISTERED 0x01 /* registered in a controller */ -#define ARM_ISRCF_PERCPU 0x02 /* per CPU interrupt */ -#define ARM_ISRCF_BOUND 0x04 /* bound to a CPU */ +#define INTR_ISRCF_REGISTERED 0x01 /* registered in a controller */ +#define INTR_ISRCF_PERCPU 0x02 /* per CPU interrupt */ +#define INTR_ISRCF_BOUND 0x04 /* bound to a CPU */ /* Interrupt source definition. */ -struct arm_irqsrc { +struct intr_irqsrc { device_t isrc_dev; /* where isrc is mapped */ intptr_t isrc_xref; /* device reference key */ uintptr_t isrc_data; /* device data for isrc */ u_int isrc_irq; /* unique identificator */ - enum arm_isrc_type isrc_type; /* how is isrc decribed */ + enum intr_isrc_type isrc_type; /* how is isrc decribed */ u_int isrc_flags; - char isrc_name[ARM_ISRC_NAMELEN]; + char isrc_name[INTR_ISRC_NAMELEN]; uint16_t isrc_nspc_type; uint16_t isrc_nspc_num; enum intr_trigger isrc_trig; @@ -87,8 +87,8 @@ struct arm_irqsrc { u_long * isrc_count; u_int isrc_handlers; struct intr_event * isrc_event; - arm_irq_filter_t * isrc_filter; - arm_ipi_filter_t * isrc_ipifilter; + intr_irq_filter_t * isrc_filter; + intr_ipi_filter_t * isrc_ipifilter; void * isrc_arg; #ifdef FDT u_int isrc_ncells; @@ -96,45 +96,45 @@ struct arm_irqsrc { #endif }; -void arm_irq_set_name(struct arm_irqsrc *isrc, const char *fmt, ...) +void intr_irq_set_name(struct intr_irqsrc *isrc, const char *fmt, ...) __printflike(2, 3); -void arm_irq_dispatch(struct arm_irqsrc *isrc, struct trapframe *tf); +void intr_irq_dispatch(struct intr_irqsrc *isrc, struct trapframe *tf); -#define ARM_IRQ_NSPC_NONE 0 -#define ARM_IRQ_NSPC_PLAIN 1 -#define ARM_IRQ_NSPC_IRQ 2 -#define ARM_IRQ_NSPC_IPI 3 +#define INTR_IRQ_NSPC_NONE 0 +#define INTR_IRQ_NSPC_PLAIN 1 +#define INTR_IRQ_NSPC_IRQ 2 +#define INTR_IRQ_NSPC_IPI 3 -u_int arm_namespace_map_irq(device_t dev, uint16_t type, uint16_t num); +u_int intr_namespace_map_irq(device_t dev, uint16_t type, uint16_t num); #ifdef FDT -u_int arm_fdt_map_irq(phandle_t, pcell_t *, u_int); +u_int intr_fdt_map_irq(phandle_t, pcell_t *, u_int); #endif -int arm_pic_register(device_t dev, intptr_t xref); -int arm_pic_unregister(device_t dev, intptr_t xref); -int arm_pic_claim_root(device_t dev, intptr_t xref, arm_irq_filter_t *filter, +int intr_pic_register(device_t dev, intptr_t xref); +int intr_pic_unregister(device_t dev, intptr_t xref); +int intr_pic_claim_root(device_t dev, intptr_t xref, intr_irq_filter_t *filter, void *arg, u_int ipicount); -int arm_irq_add_handler(device_t dev, driver_filter_t, driver_intr_t, void *, +int intr_irq_add_handler(device_t dev, driver_filter_t, driver_intr_t, void *, u_int, int, void **); -int arm_irq_remove_handler(device_t dev, u_int, void *); -int arm_irq_config(u_int, enum intr_trigger, enum intr_polarity); -int arm_irq_describe(u_int, void *, const char *); +int intr_irq_remove_handler(device_t dev, u_int, void *); +int intr_irq_config(u_int, enum intr_trigger, enum intr_polarity); +int intr_irq_describe(u_int, void *, const char *); -u_int arm_irq_next_cpu(u_int current_cpu, cpuset_t *cpumask); +u_int intr_irq_next_cpu(u_int current_cpu, cpuset_t *cpumask); #ifdef SMP -int arm_irq_bind(u_int, int); +int intr_irq_bind(u_int, int); -void arm_ipi_dispatch(struct arm_irqsrc *isrc, struct trapframe *tf); +void intr_ipi_dispatch(struct intr_irqsrc *isrc, struct trapframe *tf); #define AISHF_NOALLOC 0x0001 -int arm_ipi_set_handler(u_int ipi, const char *name, arm_ipi_filter_t *filter, +int intr_ipi_set_handler(u_int ipi, const char *name, intr_ipi_filter_t *filter, void *arg, u_int flags); -void arm_pic_init_secondary(void); +void intr_pic_init_secondary(void); #endif #else /* ARM_INTRNG */ @@ -178,11 +178,11 @@ extern void (*arm_post_filter)(void *); extern int (*arm_config_irq)(int irq, enum intr_trigger trig, enum intr_polarity pol); -void arm_pic_init_secondary(void); +void intr_pic_init_secondary(void); #ifdef FDT int gic_decode_fdt(phandle_t, pcell_t *, int *, int *, int *); -int arm_fdt_map_irq(phandle_t, pcell_t *, int); +int intr_fdt_map_irq(phandle_t, pcell_t *, int); #endif #endif /* ARM_INTRNG */ diff --git a/sys/arm/include/smp.h b/sys/arm/include/smp.h index 1c90431..c50b99d 100644 --- a/sys/arm/include/smp.h +++ b/sys/arm/include/smp.h @@ -16,7 +16,7 @@ enum { IPI_HARDCLOCK, IPI_TLB, IPI_CACHE, - ARM_IPI_COUNT + INTR_IPI_COUNT }; #else #define IPI_AST 0 diff --git a/sys/arm/qemu/virt_mp.c b/sys/arm/qemu/virt_mp.c index 52e8943..46d7632 100644 --- a/sys/arm/qemu/virt_mp.c +++ b/sys/arm/qemu/virt_mp.c @@ -102,7 +102,7 @@ void platform_mp_init_secondary(void) { - arm_pic_init_secondary(); + intr_pic_init_secondary(); } void diff --git a/sys/arm/rockchip/rk30xx_mp.c b/sys/arm/rockchip/rk30xx_mp.c index f07605c..38b6b41 100644 --- a/sys/arm/rockchip/rk30xx_mp.c +++ b/sys/arm/rockchip/rk30xx_mp.c @@ -82,7 +82,7 @@ void platform_mp_init_secondary(void) { - arm_pic_init_secondary(); + intr_pic_init_secondary(); } void diff --git a/sys/arm/samsung/exynos/exynos5_mp.c b/sys/arm/samsung/exynos/exynos5_mp.c index 0f4a792..8eb0d29 100644 --- a/sys/arm/samsung/exynos/exynos5_mp.c +++ b/sys/arm/samsung/exynos/exynos5_mp.c @@ -73,7 +73,7 @@ void platform_mp_init_secondary(void) { - arm_pic_init_secondary(); + intr_pic_init_secondary(); } void diff --git a/sys/arm/ti/aintc.c b/sys/arm/ti/aintc.c index ccac5e7..19f4544 100644 --- a/sys/arm/ti/aintc.c +++ b/sys/arm/ti/aintc.c @@ -158,6 +158,7 @@ static devclass_t ti_aintc_devclass; EARLY_DRIVER_MODULE(aintc, simplebus, ti_aintc_driver, ti_aintc_devclass, 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); +SIMPLEBUS_PNP_INFO(compat_data); int arm_get_next_irq(int last_irq) diff --git a/sys/arm/ti/omap4/omap4_mp.c b/sys/arm/ti/omap4/omap4_mp.c index c32d302..1a095ab 100644 --- a/sys/arm/ti/omap4/omap4_mp.c +++ b/sys/arm/ti/omap4/omap4_mp.c @@ -44,7 +44,7 @@ __FBSDID("$FreeBSD$"); void platform_mp_init_secondary(void) { - arm_pic_init_secondary(); + intr_pic_init_secondary(); } void diff --git a/sys/arm/xilinx/zy7_mp.c b/sys/arm/xilinx/zy7_mp.c index c87cf29..f71740c 100644 --- a/sys/arm/xilinx/zy7_mp.c +++ b/sys/arm/xilinx/zy7_mp.c @@ -49,7 +49,7 @@ void platform_mp_init_secondary(void) { - arm_pic_init_secondary(); + intr_pic_init_secondary(); } void diff --git a/sys/arm64/include/ofw_machdep.h b/sys/arm64/include/ofw_machdep.h index 511fc8d..e9c65ef 100644 --- a/sys/arm64/include/ofw_machdep.h +++ b/sys/arm64/include/ofw_machdep.h @@ -41,4 +41,7 @@ struct mem_region { vm_size_t mr_size; }; +/* FDT follows ePAPR */ +#define OFW_EPAPR + #endif /* _MACHINE_OFW_MACHDEP_H_ */ diff --git a/sys/boot/efi/loader/bootinfo.c b/sys/boot/efi/loader/bootinfo.c index c9dbe8d..0b7b7ef 100644 --- a/sys/boot/efi/loader/bootinfo.c +++ b/sys/boot/efi/loader/bootinfo.c @@ -55,6 +55,8 @@ __FBSDID("$FreeBSD$"); #include <fdt_platform.h> #endif +extern EFI_SYSTEM_TABLE *ST; + static const char howto_switches[] = "aCdrgDmphsv"; static int howto_masks[] = { RB_ASKNAME, RB_CDROM, RB_KDB, RB_DFLTROOT, RB_GDB, RB_MULTIPLE, @@ -250,8 +252,8 @@ bi_add_efi_data_and_exit(struct preloaded_file *kfp, for (retry = 2; retry > 0; retry--) { status = BS->GetMemoryMap(&sz, mm, &efi_mapkey, &mmsz, &mmver); if (EFI_ERROR(status)) { - printf("%s: GetMemoryMap() returned 0x%lx\n", __func__, - (long)status); + printf("%s: GetMemoryMap error %lu\n", __func__, + (unsigned long)(status & ~EFI_ERROR_MASK)); return (EINVAL); } status = BS->ExitBootServices(IH, efi_mapkey); @@ -264,7 +266,8 @@ bi_add_efi_data_and_exit(struct preloaded_file *kfp, return (0); } } - printf("ExitBootServices() returned 0x%lx\n", (long)status); + printf("ExitBootServices error %lu\n", + (unsigned long)(status & ~EFI_ERROR_MASK)); return (EINVAL); } @@ -317,8 +320,8 @@ bi_load_efi_data(struct preloaded_file *kfp) status = BS->AllocatePages(AllocateAnyPages, EfiLoaderData, pages, &addr); if (EFI_ERROR(status)) { - printf("%s: AllocatePages() returned 0x%lx\n", __func__, - (long)status); + printf("%s: AllocatePages error %lu\n", __func__, + (unsigned long)(status & ~EFI_ERROR_MASK)); return (ENOMEM); } @@ -434,6 +437,7 @@ bi_load(char *args, vm_offset_t *modulep, vm_offset_t *kernendp) "device tree blob found!\n"); #endif file_addmetadata(kfp, MODINFOMD_KERNEND, sizeof kernend, &kernend); + file_addmetadata(kfp, MODINFOMD_FW_HANDLE, sizeof ST, &ST); bi_load_efi_data(kfp); diff --git a/sys/conf/Makefile.mips b/sys/conf/Makefile.mips index 09eca75..30f7d1e 100644 --- a/sys/conf/Makefile.mips +++ b/sys/conf/Makefile.mips @@ -42,7 +42,7 @@ TRAMPLOADADDR?=0x807963c0 # We default to the MIPS32 ISA, if none specified in the # kernel configuration file. ARCH_FLAGS?=-march=mips32 -EXTRA_FLAGS=-fno-pic -mno-abicalls -G0 -DKERNLOADADDR=${KERNLOADADDR} +EXTRA_FLAGS=-DKERNLOADADDR=${KERNLOADADDR} HACK_EXTRA_FLAGS=-shared diff --git a/sys/conf/files.arm b/sys/conf/files.arm index 1c57106..292fd54 100644 --- a/sys/conf/files.arm +++ b/sys/conf/files.arm @@ -46,16 +46,17 @@ arm/arm/identcpu.c standard arm/arm/in_cksum.c optional inet | inet6 arm/arm/in_cksum_arm.S optional inet | inet6 arm/arm/intr.c optional !arm_intrng -arm/arm/intrng.c optional arm_intrng +kern/subr_intr.c optional arm_intrng arm/arm/locore.S standard no-obj arm/arm/machdep.c standard +arm/arm/machdep_intr.c standard arm/arm/mem.c optional mem arm/arm/minidump_machdep.c optional mem arm/arm/mp_machdep.c optional smp arm/arm/mpcore_timer.c optional mpcore_timer arm/arm/nexus.c standard arm/arm/physmem.c standard -arm/arm/pic_if.m optional arm_intrng +kern/pic_if.m optional arm_intrng arm/arm/pl190.c optional pl190 arm/arm/pl310.c optional pl310 arm/arm/platform.c optional platform diff --git a/sys/conf/kern.mk b/sys/conf/kern.mk index 56ddbda..0a8161d 100644 --- a/sys/conf/kern.mk +++ b/sys/conf/kern.mk @@ -160,6 +160,7 @@ CFLAGS.gcc+= -mcall-aixdesc # For MIPS we also tell gcc to use floating point emulation # .if ${MACHINE_CPUARCH} == "mips" +CFLAGS+= -fno-pic -mno-abicalls -G0 CFLAGS+= -msoft-float INLINE_LIMIT?= 8000 .endif diff --git a/sys/conf/kmod.mk b/sys/conf/kmod.mk index 50000c5..2b7fab2 100644 --- a/sys/conf/kmod.mk +++ b/sys/conf/kmod.mk @@ -130,7 +130,7 @@ CFLAGS+= -mlongcall -fno-omit-frame-pointer .endif .if ${MACHINE_CPUARCH} == mips -CFLAGS+= -G0 -fno-pic -mno-abicalls -mlong-calls +CFLAGS+= -mlong-calls .endif .if defined(DEBUG) || defined(DEBUG_FLAGS) diff --git a/sys/dev/drm2/i915/intel_iic.c b/sys/dev/drm2/i915/intel_iic.c index c04b27b..aacc954 100644 --- a/sys/dev/drm2/i915/intel_iic.c +++ b/sys/dev/drm2/i915/intel_iic.c @@ -456,7 +456,7 @@ timeout: /* Hardware may not support GMBUS over these pins? Try GPIO bitbanging instead. */ sc->force_bit_dev = true; - error = -IICBUS_TRANSFER(adapter, msgs, num); + error = -IICBUS_TRANSFER(dev_priv->bbbus[unit], msgs, num); out: sx_xunlock(&dev_priv->gmbus_sx); diff --git a/sys/dev/drm2/ttm/ttm_bo.c b/sys/dev/drm2/ttm/ttm_bo.c index 70e17c4..010afe6 100644 --- a/sys/dev/drm2/ttm/ttm_bo.c +++ b/sys/dev/drm2/ttm/ttm_bo.c @@ -1488,21 +1488,21 @@ int ttm_bo_global_init(struct drm_global_reference *ref) struct ttm_bo_global_ref *bo_ref = container_of(ref, struct ttm_bo_global_ref, ref); struct ttm_bo_global *glob = ref->object; - int ret; + int req, ret; int tries; sx_init(&glob->device_list_mutex, "ttmdlm"); mtx_init(&glob->lru_lock, "ttmlru", NULL, MTX_DEF); glob->mem_glob = bo_ref->mem_glob; + req = VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ; tries = 0; retry: - glob->dummy_read_page = vm_page_alloc_contig(NULL, 0, - VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ, + glob->dummy_read_page = vm_page_alloc_contig(NULL, 0, req, 1, 0, VM_MAX_ADDRESS, PAGE_SIZE, 0, VM_MEMATTR_UNCACHEABLE); if (unlikely(glob->dummy_read_page == NULL)) { - if (tries < 1) { - vm_pageout_grow_cache(tries, 0, VM_MAX_ADDRESS); + if (tries < 1 && vm_page_reclaim_contig(req, 1, + 0, VM_MAX_ADDRESS, PAGE_SIZE, 0)) { tries++; goto retry; } diff --git a/sys/dev/drm2/ttm/ttm_page_alloc.c b/sys/dev/drm2/ttm/ttm_page_alloc.c index 75abd10..2a2e916 100644 --- a/sys/dev/drm2/ttm/ttm_page_alloc.c +++ b/sys/dev/drm2/ttm/ttm_page_alloc.c @@ -166,13 +166,9 @@ ttm_vm_page_alloc_dma32(int req, vm_memattr_t memattr) PAGE_SIZE, 0, memattr); if (p != NULL || tries > 2) return (p); - - /* - * Before growing the cache see if this is just a normal - * memory shortage. - */ - VM_WAIT; - vm_pageout_grow_cache(tries, 0, 0xffffffff); + if (!vm_page_reclaim_contig(req, 1, 0, 0xffffffff, + PAGE_SIZE, 0)) + VM_WAIT; } } diff --git a/sys/dev/if_ndis/if_ndis_pci.c b/sys/dev/if_ndis/if_ndis_pci.c index b3085db..bf2f313 100644 --- a/sys/dev/if_ndis/if_ndis_pci.c +++ b/sys/dev/if_ndis/if_ndis_pci.c @@ -298,8 +298,7 @@ ndis_attach_pci(dev) BUS_SPACE_MAXADDR_32BIT,/* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ - BUS_SPACE_MAXSIZE_32BIT, /* maxsize */ - NDIS_NSEG_NEW, /* nsegments */ + DFLTPHYS, NDIS_NSEG_NEW,/* maxsize, nsegments */ BUS_SPACE_MAXSIZE_32BIT,/* maxsegsize */ BUS_DMA_ALLOCNOW, /* flags */ NULL, NULL, /* lockfunc, lockarg */ diff --git a/sys/dev/ioat/ioat.c b/sys/dev/ioat/ioat.c index 12a6fe4..4b6d7df 100644 --- a/sys/dev/ioat/ioat.c +++ b/sys/dev/ioat/ioat.c @@ -736,6 +736,15 @@ ioat_put_dmaengine(bus_dmaengine_t dmaengine) } int +ioat_get_hwversion(bus_dmaengine_t dmaengine) +{ + struct ioat_softc *ioat; + + ioat = to_ioat_softc(dmaengine); + return (ioat->version); +} + +int ioat_set_interrupt_coalesce(bus_dmaengine_t dmaengine, uint16_t delay) { struct ioat_softc *ioat; diff --git a/sys/dev/ioat/ioat.h b/sys/dev/ioat/ioat.h index 5c64af1..e8f47ae 100644 --- a/sys/dev/ioat/ioat.h +++ b/sys/dev/ioat/ioat.h @@ -48,6 +48,14 @@ __FBSDID("$FreeBSD$"); #define DMA_NO_WAIT 0x2 #define DMA_ALL_FLAGS (DMA_INT_EN | DMA_NO_WAIT) +/* + * Hardware revision number. Different hardware revisions support different + * features. For example, 3.2 cannot read from MMIO space, while 3.3 can. + */ +#define IOAT_VER_3_0 0x30 +#define IOAT_VER_3_2 0x32 +#define IOAT_VER_3_3 0x33 + typedef void *bus_dmaengine_t; struct bus_dmadesc; typedef void (*bus_dmaengine_callback_t)(void *arg, int error); @@ -60,6 +68,9 @@ bus_dmaengine_t ioat_get_dmaengine(uint32_t channel_index); /* Release the DMA channel */ void ioat_put_dmaengine(bus_dmaengine_t dmaengine); +/* Check the DMA engine's HW version */ +int ioat_get_hwversion(bus_dmaengine_t dmaengine); + /* * Set interrupt coalescing on a DMA channel. * diff --git a/sys/dev/ioat/ioat_hw.h b/sys/dev/ioat/ioat_hw.h index 1aeee2d..46ffaeb 100644 --- a/sys/dev/ioat/ioat_hw.h +++ b/sys/dev/ioat/ioat_hw.h @@ -46,9 +46,6 @@ __FBSDID("$FreeBSD$"); #define IOAT_CBVER_OFFSET 0x08 -#define IOAT_VER_3_0 0x30 -#define IOAT_VER_3_3 0x33 - #define IOAT_INTRDELAY_OFFSET 0x0C #define IOAT_INTRDELAY_SUPPORTED (1 << 15) /* Reserved. (1 << 14) */ diff --git a/sys/dev/mmc/mmc.c b/sys/dev/mmc/mmc.c index eb76256..a65ff41 100644 --- a/sys/dev/mmc/mmc.c +++ b/sys/dev/mmc/mmc.c @@ -118,7 +118,7 @@ struct mmc_ivars { static SYSCTL_NODE(_hw, OID_AUTO, mmc, CTLFLAG_RD, NULL, "mmc driver"); static int mmc_debug; -SYSCTL_INT(_hw_mmc, OID_AUTO, debug, CTLFLAG_RW, &mmc_debug, 0, "Debug level"); +SYSCTL_INT(_hw_mmc, OID_AUTO, debug, CTLFLAG_RWTUN, &mmc_debug, 0, "Debug level"); /* bus entry points */ static int mmc_acquire_bus(device_t busdev, device_t dev); diff --git a/sys/dev/nand/nand_cdev.c b/sys/dev/nand/nand_cdev.c index b011946..0540955 100644 --- a/sys/dev/nand/nand_cdev.c +++ b/sys/dev/nand/nand_cdev.c @@ -236,10 +236,10 @@ nand_strategy(struct bio *bp) chip = dev->si_drv1; nand_debug(NDBG_CDEV, "Strategy %s on chip %d [%p]\n", - (bp->bio_cmd & BIO_READ) == BIO_READ ? "READ" : "WRITE", + bp->bio_cmd == BIO_READ ? "READ" : "WRITE", chip->num, chip); - if ((bp->bio_cmd & BIO_READ) == BIO_READ) { + if (bp->bio_cmd == BIO_READ) { err = nand_read(chip, bp->bio_offset & 0xffffffff, bp->bio_data, bp->bio_bcount); diff --git a/sys/dev/nand/nand_geom.c b/sys/dev/nand/nand_geom.c index 8786a0a..593b5f5 100644 --- a/sys/dev/nand/nand_geom.c +++ b/sys/dev/nand/nand_geom.c @@ -100,9 +100,9 @@ nand_strategy(struct bio *bp) bp->bio_driver1 = BIO_NAND_STD; nand_debug(NDBG_GEOM, "Strategy %s on chip %d [%p]", - (bp->bio_cmd & BIO_READ) == BIO_READ ? "READ" : - ((bp->bio_cmd & BIO_WRITE) == BIO_WRITE ? "WRITE" : - ((bp->bio_cmd & BIO_DELETE) == BIO_DELETE ? "DELETE" : "UNKNOWN")), + bp->bio_cmd == BIO_READ ? "READ" : + (bp->bio_cmd == BIO_WRITE ? "WRITE" : + (bp->bio_cmd == BIO_DELETE ? "DELETE" : "UNKNOWN")), chip->num, chip); mtx_lock(&chip->qlock); @@ -122,9 +122,9 @@ nand_strategy_raw(struct bio *bp) bp->bio_driver1 = BIO_NAND_RAW; nand_debug(NDBG_GEOM, "Strategy %s on chip %d [%p]", - (bp->bio_cmd & BIO_READ) == BIO_READ ? "READ" : - ((bp->bio_cmd & BIO_WRITE) == BIO_WRITE ? "WRITE" : - ((bp->bio_cmd & BIO_DELETE) == BIO_DELETE ? "DELETE" : "UNKNOWN")), + bp->bio_cmd == BIO_READ ? "READ" : + (bp->bio_cmd == BIO_WRITE ? "WRITE" : + (bp->bio_cmd == BIO_DELETE ? "DELETE" : "UNKNOWN")), chip->num, chip); mtx_lock(&chip->qlock); @@ -320,21 +320,21 @@ nand_io_proc(void *arg, int pending) break; if (bp->bio_driver1 == BIO_NAND_STD) { - if ((bp->bio_cmd & BIO_READ) == BIO_READ) { + if (bp->bio_cmd == BIO_READ) { err = nand_read(chip, bp->bio_offset & 0xffffffff, bp->bio_data, bp->bio_bcount); - } else if ((bp->bio_cmd & BIO_WRITE) == BIO_WRITE) { + } else if (bp->bio_cmd == BIO_WRITE) { err = nand_write(chip, bp->bio_offset & 0xffffffff, bp->bio_data, bp->bio_bcount); } } else if (bp->bio_driver1 == BIO_NAND_RAW) { - if ((bp->bio_cmd & BIO_READ) == BIO_READ) { + if (bp->bio_cmd == BIO_READ) { err = nand_read_raw(chip, bp->bio_offset & 0xffffffff, bp->bio_data, bp->bio_bcount); - } else if ((bp->bio_cmd & BIO_WRITE) == BIO_WRITE) { + } else if (bp->bio_cmd == BIO_WRITE) { err = nand_write_raw(chip, bp->bio_offset & 0xffffffff, bp->bio_data, bp->bio_bcount); @@ -342,7 +342,7 @@ nand_io_proc(void *arg, int pending) } else panic("Unknown access type in bio->bio_driver1\n"); - if ((bp->bio_cmd & BIO_DELETE) == BIO_DELETE) { + if (bp->bio_cmd == BIO_DELETE) { nand_debug(NDBG_GEOM, "Delete on chip%d offset %lld " "length %ld\n", chip->num, bp->bio_offset, bp->bio_bcount); diff --git a/sys/dev/ofw/ofw_bus_subr.c b/sys/dev/ofw/ofw_bus_subr.c index ba2b20e..c286373 100644 --- a/sys/dev/ofw/ofw_bus_subr.c +++ b/sys/dev/ofw/ofw_bus_subr.c @@ -341,6 +341,7 @@ ofw_bus_search_intrmap(void *intr, int intrsz, void *regs, int physsz, uint8_t *uiregs = regs; uint8_t *uiimapmsk = imapmsk; uint8_t *mptr; + pcell_t paddrsz; pcell_t pintrsz; int i, rsz, tsz; @@ -357,19 +358,31 @@ ofw_bus_search_intrmap(void *intr, int intrsz, void *regs, int physsz, mptr = imap; i = imapsz; + paddrsz = 0; while (i > 0) { bcopy(mptr + physsz + intrsz, &parent, sizeof(parent)); +#ifdef OFW_EPAPR + /* + * Find if we need to read the parent address data. Sparc64 + * uses a different encoding that doesn't include this data. + */ + if (OF_getencprop(OF_node_from_xref(parent), + "#address-cells", &paddrsz, sizeof(paddrsz)) == -1) + paddrsz = 0; /* default */ + paddrsz *= sizeof(pcell_t); +#endif + if (OF_searchencprop(OF_node_from_xref(parent), "#interrupt-cells", &pintrsz, sizeof(pintrsz)) == -1) pintrsz = 1; /* default */ pintrsz *= sizeof(pcell_t); /* Compute the map stride size. */ - tsz = physsz + intrsz + sizeof(phandle_t) + pintrsz; + tsz = physsz + intrsz + sizeof(phandle_t) + paddrsz + pintrsz; KASSERT(i >= tsz, ("ofw_bus_search_intrmap: truncated map")); if (bcmp(ref, mptr, physsz + intrsz) == 0) { - bcopy(mptr + physsz + intrsz + sizeof(parent), + bcopy(mptr + physsz + intrsz + sizeof(parent) + paddrsz, result, MIN(rintrsz, pintrsz)); if (iparent != NULL) diff --git a/sys/dev/ofw/ofw_bus_subr.h b/sys/dev/ofw/ofw_bus_subr.h index 708e984..7c4df33 100644 --- a/sys/dev/ofw/ofw_bus_subr.h +++ b/sys/dev/ofw/ofw_bus_subr.h @@ -52,6 +52,10 @@ struct ofw_compat_data { uintptr_t ocd_data; }; +#define SIMPLEBUS_PNP_DESCR "Z:compat;P:private;" +#define SIMPLEBUS_PNP_INFO(t) \ + MODULE_PNP_INFO(SIMPLEBUS_PNP_DESCR, simplebus, t, t, sizeof(t[0]), sizeof(t) / sizeof(t[0])); + /* Generic implementation of ofw_bus_if.m methods and helper routines */ int ofw_bus_gen_setup_devinfo(struct ofw_bus_devinfo *, phandle_t); void ofw_bus_gen_destroy_devinfo(struct ofw_bus_devinfo *); diff --git a/sys/geom/sched/gs_delay.c b/sys/geom/sched/gs_delay.c new file mode 100644 index 0000000..151738d --- /dev/null +++ b/sys/geom/sched/gs_delay.c @@ -0,0 +1,264 @@ +/*- + * Copyright (c) 2015 Netflix, Inc. + * + * Derived from gs_rr.c: + * Copyright (c) 2009-2010 Fabio Checconi + * Copyright (c) 2009-2010 Luigi Rizzo, Universita` di Pisa + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* + * $Id$ + * $FreeBSD$ + * + * A simple scheduler that just delays certain transactions by a certain + * amount. We collect all the transactions that are 'done' and put them on + * a queue. The queue is run through every so often and the transactions that + * have taken longer than the threshold delay are completed. + */ + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/kernel.h> +#include <sys/bio.h> +#include <sys/callout.h> +#include <sys/malloc.h> +#include <sys/module.h> +#include <sys/proc.h> +#include <sys/queue.h> +#include <sys/sbuf.h> +#include <sys/sysctl.h> +#include "gs_scheduler.h" + +/* Useful constants */ +#define BTFRAC_1US 18446744073709ULL /* 2^64 / 1000000 */ + +/* list of scheduler instances */ +LIST_HEAD(g_scheds, g_delay_softc); + +/* + * Per device descriptor, holding the Round Robin list of queues + * accessing the disk, a reference to the geom, and the timer. + */ +struct g_delay_softc { + struct g_geom *sc_geom; + + struct bio_queue_head sc_bioq; /* queue of pending requests */ + struct callout sc_wait; /* timer for completing with delays */ + + /* Statistics */ + int sc_in_flight; /* requests in the driver */ +}; + +/* + * parameters, config and stats + */ +struct g_delay_params { + uint64_t io; + int bypass; /* bypass scheduling */ + int units; /* how many instances */ + int latency; /* How big a latncy are hoping for */ +}; + +static struct g_delay_params me = { + .bypass = 0, + .units = 0, + .latency = 0, + .io = 0, +}; +struct g_delay_params *gs_delay_me = &me; + +SYSCTL_DECL(_kern_geom_sched); +static SYSCTL_NODE(_kern_geom_sched, OID_AUTO, delay, CTLFLAG_RW, 0, + "GEOM_SCHED DELAY stuff"); +SYSCTL_INT(_kern_geom_sched_delay, OID_AUTO, bypass, CTLFLAG_RD, + &me.bypass, 0, "Scheduler bypass"); +SYSCTL_INT(_kern_geom_sched_delay, OID_AUTO, units, CTLFLAG_RD, + &me.units, 0, "Scheduler instances"); +SYSCTL_INT(_kern_geom_sched_delay, OID_AUTO, latency, CTLFLAG_RW, + &me.latency, 0, "Minimum latency for requests, in microseconds (1/hz resolution)"); +SYSCTL_QUAD(_kern_geom_sched_delay, OID_AUTO, io, CTLFLAG_RW, + &me.io, 0, "I/Os delayed\n"); + +static int +g_delay_init_class(void *data, void *priv) +{ + return (0); +} + +static void +g_delay_fini_class(void *data, void *priv) +{ +} + +/* + * Called on a request arrival, timeout or completion. + * Try to serve a request among those queued. + */ +static struct bio * +g_delay_next(void *data, int force) +{ + struct g_delay_softc *sc = data; + struct bio *bp; + struct bintime bt; + + bp = bioq_first(&sc->sc_bioq); + if (bp == NULL) + return (NULL); + + /* + * If the time isn't yet ripe for this bp to be let loose, + * then the time isn't ripe for any of its friends either + * since we insert in-order. Terminate if the bio hasn't + * aged appropriately. Note that there's pathology here + * such that we may be up to one tick early in releasing + * this I/O. We could implement this up to a tick late too + * but choose not to. + */ + getbinuptime(&bt); /* BIO's bio_t0 is uptime */ + if (bintime_cmp(&bp->bio_t0, &bt, >)) + return (NULL); + me.io++; + + /* + * The bp has mellowed enough, let it through and update stats. + * If there's others, we'll catch them next time we get called. + */ + sc->sc_in_flight++; + + bp = bioq_takefirst(&sc->sc_bioq); + return (bp); +} + +/* + * Called when a real request for disk I/O arrives. + * Locate the queue associated with the client. + * If the queue is the one we are anticipating for, reset its timeout; + * if the queue is not in the round robin list, insert it in the list. + * On any error, do not queue the request and return -1, the caller + * will take care of this request. + */ +static int +g_delay_start(void *data, struct bio *bp) +{ + struct g_delay_softc *sc = data; + + if (me.bypass) + return (-1); /* bypass the scheduler */ + + bp->bio_caller1 = sc; + getbinuptime(&bp->bio_t0); /* BIO's bio_t0 is uptime */ + bintime_addx(&bp->bio_t0, BTFRAC_1US * me.latency); + + /* + * Keep the I/Os ordered. Lower layers will reorder as we release them down. + * We rely on this in g_delay_next() so that we delay all things equally. Even + * if we move to multiple queues to push stuff down the stack, we'll want to + * insert in order and let the lower layers do whatever reordering they want. + */ + bioq_insert_tail(&sc->sc_bioq, bp); + + return (0); +} + +static void +g_delay_timeout(void *data) +{ + struct g_delay_softc *sc = data; + + g_sched_lock(sc->sc_geom); + g_sched_dispatch(sc->sc_geom); + g_sched_unlock(sc->sc_geom); + callout_reset(&sc->sc_wait, 1, g_delay_timeout, sc); +} + +/* + * Module glue: allocate descriptor, initialize its fields. + */ +static void * +g_delay_init(struct g_geom *geom) +{ + struct g_delay_softc *sc; + + sc = malloc(sizeof *sc, M_GEOM_SCHED, M_WAITOK | M_ZERO); + sc->sc_geom = geom; + bioq_init(&sc->sc_bioq); + callout_init(&sc->sc_wait, CALLOUT_MPSAFE); + callout_reset(&sc->sc_wait, 1, g_delay_timeout, sc); + me.units++; + + return (sc); +} + +/* + * Module glue -- drain the callout structure, destroy the + * hash table and its element, and free the descriptor. + */ +static void +g_delay_fini(void *data) +{ + struct g_delay_softc *sc = data; + + /* We're force drained before getting here */ + + /* Kick out timers */ + callout_drain(&sc->sc_wait); + me.units--; + free(sc, M_GEOM_SCHED); +} + +/* + * Called when the request under service terminates. + * Start the anticipation timer if needed. + */ +static void +g_delay_done(void *data, struct bio *bp) +{ + struct g_delay_softc *sc = data; + + sc->sc_in_flight--; + + g_sched_dispatch(sc->sc_geom); +} + +static void +g_delay_dumpconf(struct sbuf *sb, const char *indent, struct g_geom *gp, + struct g_consumer *cp, struct g_provider *pp) +{ +} + +static struct g_gsched g_delay = { + .gs_name = "delay", + .gs_priv_size = 0, + .gs_init = g_delay_init, + .gs_fini = g_delay_fini, + .gs_start = g_delay_start, + .gs_done = g_delay_done, + .gs_next = g_delay_next, + .gs_dumpconf = g_delay_dumpconf, + .gs_init_class = g_delay_init_class, + .gs_fini_class = g_delay_fini_class, +}; + +DECLARE_GSCHED_MODULE(delay, &g_delay); diff --git a/sys/kern/kern_proc.c b/sys/kern/kern_proc.c index 8a3b6ca..bbedd9b 100644 --- a/sys/kern/kern_proc.c +++ b/sys/kern/kern_proc.c @@ -248,6 +248,7 @@ proc_init(void *mem, int size, int flags) TAILQ_INIT(&p->p_threads); /* all threads in proc */ EVENTHANDLER_INVOKE(process_init, p); p->p_stats = pstats_alloc(); + p->p_pgrp = NULL; SDT_PROBE3(proc, , init, return, p, size, flags); return (0); } diff --git a/sys/arm/arm/pic_if.m b/sys/kern/pic_if.m index d3002ea..2309f0c 100644 --- a/sys/arm/arm/pic_if.m +++ b/sys/kern/pic_if.m @@ -36,17 +36,17 @@ INTERFACE pic; CODE { - static int null_pic_bind(device_t dev, struct arm_irqsrc *isrc) + static int null_pic_bind(device_t dev, struct intr_irqsrc *isrc) { return (EOPNOTSUPP); } - static void null_pic_disable_intr(device_t dev, struct arm_irqsrc *isrc) + static void null_pic_disable_intr(device_t dev, struct intr_irqsrc *isrc) { return; } - static void null_pic_enable_intr(device_t dev, struct arm_irqsrc *isrc) + static void null_pic_enable_intr(device_t dev, struct intr_irqsrc *isrc) { return; } @@ -64,53 +64,53 @@ CODE { METHOD int register { device_t dev; - struct arm_irqsrc *isrc; + struct intr_irqsrc *isrc; boolean_t *is_percpu; }; METHOD int unregister { device_t dev; - struct arm_irqsrc *isrc; + struct intr_irqsrc *isrc; }; METHOD void disable_intr { device_t dev; - struct arm_irqsrc *isrc; + struct intr_irqsrc *isrc; } DEFAULT null_pic_disable_intr; METHOD void disable_source { device_t dev; - struct arm_irqsrc *isrc; + struct intr_irqsrc *isrc; }; METHOD void enable_source { device_t dev; - struct arm_irqsrc *isrc; + struct intr_irqsrc *isrc; }; METHOD void enable_intr { device_t dev; - struct arm_irqsrc *isrc; + struct intr_irqsrc *isrc; } DEFAULT null_pic_enable_intr; METHOD void pre_ithread { device_t dev; - struct arm_irqsrc *isrc; + struct intr_irqsrc *isrc; }; METHOD void post_ithread { device_t dev; - struct arm_irqsrc *isrc; + struct intr_irqsrc *isrc; }; METHOD void post_filter { device_t dev; - struct arm_irqsrc *isrc; + struct intr_irqsrc *isrc; }; METHOD int bind { device_t dev; - struct arm_irqsrc *isrc; + struct intr_irqsrc *isrc; } DEFAULT null_pic_bind; METHOD void init_secondary { @@ -119,6 +119,6 @@ METHOD void init_secondary { METHOD void ipi_send { device_t dev; - struct arm_irqsrc *isrc; + struct intr_irqsrc *isrc; cpuset_t cpus; } DEFAULT null_pic_ipi_send; diff --git a/sys/arm/arm/intrng.c b/sys/kern/subr_intr.c index c30ec68..813f9ec 100644 --- a/sys/arm/arm/intrng.c +++ b/sys/kern/subr_intr.c @@ -32,7 +32,7 @@ __FBSDID("$FreeBSD$"); /* - * ARM Interrupt Framework + * New-style Interrupt Framework * * TODO: - to support IPI (PPI) enabling on other CPUs if already started * - to complete things for removable PICs @@ -82,33 +82,33 @@ __FBSDID("$FreeBSD$"); #endif MALLOC_DECLARE(M_INTRNG); -MALLOC_DEFINE(M_INTRNG, "intrng", "ARM interrupt handling"); +MALLOC_DEFINE(M_INTRNG, "intr", "intr interrupt handling"); -/* Main ARM interrupt handler called from assembler -> 'hidden' for C code. */ -void arm_irq_handler(struct trapframe *tf); +/* Main interrupt handler called from assembler -> 'hidden' for C code. */ +void intr_irq_handler(struct trapframe *tf); /* Root interrupt controller stuff. */ -static struct arm_irqsrc *irq_root_isrc; +static struct intr_irqsrc *irq_root_isrc; static device_t irq_root_dev; -static arm_irq_filter_t *irq_root_filter; +static intr_irq_filter_t *irq_root_filter; static void *irq_root_arg; static u_int irq_root_ipicount; /* Interrupt controller definition. */ -struct arm_pic { - SLIST_ENTRY(arm_pic) pic_next; +struct intr_pic { + SLIST_ENTRY(intr_pic) pic_next; intptr_t pic_xref; /* hardware identification */ device_t pic_dev; }; static struct mtx pic_list_lock; -static SLIST_HEAD(, arm_pic) pic_list; +static SLIST_HEAD(, intr_pic) pic_list; -static struct arm_pic *pic_lookup(device_t dev, intptr_t xref); +static struct intr_pic *pic_lookup(device_t dev, intptr_t xref); /* Interrupt source definition. */ static struct mtx isrc_table_lock; -static struct arm_irqsrc *irq_sources[NIRQ]; +static struct intr_irqsrc *irq_sources[NIRQ]; u_int irq_next_free; #define IRQ_INVALID nitems(irq_sources) @@ -116,7 +116,7 @@ u_int irq_next_free; #ifdef SMP static boolean_t irq_assign_cpu = FALSE; -static struct arm_irqsrc ipi_sources[ARM_IPI_COUNT]; +static struct intr_irqsrc ipi_sources[INTR_IPI_COUNT]; static u_int ipi_next_num; #endif @@ -125,7 +125,7 @@ static u_int ipi_next_num; * - MAXCPU counters for each IPI counters for SMP. */ #ifdef SMP -#define INTRCNT_COUNT (NIRQ * 2 + ARM_IPI_COUNT * MAXCPU) +#define INTRCNT_COUNT (NIRQ * 2 + INTR_IPI_COUNT * MAXCPU) #else #define INTRCNT_COUNT (NIRQ * 2) #endif @@ -141,14 +141,14 @@ static u_int intrcnt_index; * Interrupt framework initialization routine. */ static void -arm_irq_init(void *dummy __unused) +intr_irq_init(void *dummy __unused) { SLIST_INIT(&pic_list); - mtx_init(&pic_list_lock, "arm pic list", NULL, MTX_DEF); - mtx_init(&isrc_table_lock, "arm isrc table", NULL, MTX_DEF); + mtx_init(&pic_list_lock, "intr pic list", NULL, MTX_DEF); + mtx_init(&isrc_table_lock, "intr isrc table", NULL, MTX_DEF); } -SYSINIT(arm_irq_init, SI_SUB_INTR, SI_ORDER_FIRST, arm_irq_init, NULL); +SYSINIT(intr_irq_init, SI_SUB_INTR, SI_ORDER_FIRST, intr_irq_init, NULL); static void intrcnt_setname(const char *name, int index) @@ -162,7 +162,7 @@ intrcnt_setname(const char *name, int index) * Update name for interrupt source with interrupt event. */ static void -intrcnt_updatename(struct arm_irqsrc *isrc) +intrcnt_updatename(struct intr_irqsrc *isrc) { /* QQQ: What about stray counter name? */ @@ -174,7 +174,7 @@ intrcnt_updatename(struct arm_irqsrc *isrc) * Virtualization for interrupt source interrupt counter increment. */ static inline void -isrc_increment_count(struct arm_irqsrc *isrc) +isrc_increment_count(struct intr_irqsrc *isrc) { /* @@ -189,7 +189,7 @@ isrc_increment_count(struct arm_irqsrc *isrc) * Virtualization for interrupt source interrupt stray counter increment. */ static inline void -isrc_increment_straycount(struct arm_irqsrc *isrc) +isrc_increment_straycount(struct intr_irqsrc *isrc) { isrc->isrc_count[1]++; @@ -199,7 +199,7 @@ isrc_increment_straycount(struct arm_irqsrc *isrc) * Virtualization for interrupt source interrupt name update. */ static void -isrc_update_name(struct arm_irqsrc *isrc, const char *name) +isrc_update_name(struct intr_irqsrc *isrc, const char *name) { char str[INTRNAME_LEN]; @@ -223,7 +223,7 @@ isrc_update_name(struct arm_irqsrc *isrc, const char *name) * Virtualization for interrupt source interrupt counters setup. */ static void -isrc_setup_counters(struct arm_irqsrc *isrc) +isrc_setup_counters(struct intr_irqsrc *isrc) { u_int index; @@ -242,7 +242,7 @@ isrc_setup_counters(struct arm_irqsrc *isrc) * Virtualization for interrupt source IPI counter increment. */ static inline void -isrc_increment_ipi_count(struct arm_irqsrc *isrc, u_int cpu) +isrc_increment_ipi_count(struct intr_irqsrc *isrc, u_int cpu) { isrc->isrc_count[cpu]++; @@ -252,7 +252,7 @@ isrc_increment_ipi_count(struct arm_irqsrc *isrc, u_int cpu) * Virtualization for interrupt source IPI counters setup. */ static void -isrc_setup_ipi_counters(struct arm_irqsrc *isrc, const char *name) +isrc_setup_ipi_counters(struct intr_irqsrc *isrc, const char *name) { u_int index, i; char str[INTRNAME_LEN]; @@ -273,11 +273,11 @@ isrc_setup_ipi_counters(struct arm_irqsrc *isrc, const char *name) #endif /* - * Main ARM interrupt dispatch handler. It's called straight + * Main interrupt dispatch handler. It's called straight * from the assembler, where CPU interrupt is served. */ void -arm_irq_handler(struct trapframe *tf) +intr_irq_handler(struct trapframe *tf) { struct trapframe * oldframe; struct thread * td; @@ -295,12 +295,12 @@ arm_irq_handler(struct trapframe *tf) } /* - * ARM interrupt controller dispatch function for interrupts. It should + * interrupt controller dispatch function for interrupts. It should * be called straight from the interrupt controller, when associated interrupt * source is learned. */ void -arm_irq_dispatch(struct arm_irqsrc *isrc, struct trapframe *tf) +intr_irq_dispatch(struct intr_irqsrc *isrc, struct trapframe *tf) { KASSERT(isrc != NULL, ("%s: no source", __func__)); @@ -331,15 +331,15 @@ arm_irq_dispatch(struct arm_irqsrc *isrc, struct trapframe *tf) /* * Allocate interrupt source. */ -static struct arm_irqsrc * +static struct intr_irqsrc * isrc_alloc(u_int type, u_int extsize) { - struct arm_irqsrc *isrc; + struct intr_irqsrc *isrc; isrc = malloc(sizeof(*isrc) + extsize, M_INTRNG, M_WAITOK | M_ZERO); isrc->isrc_irq = IRQ_INVALID; /* just to be safe */ isrc->isrc_type = type; - isrc->isrc_nspc_type = ARM_IRQ_NSPC_NONE; + isrc->isrc_nspc_type = INTR_IRQ_NSPC_NONE; isrc->isrc_trig = INTR_TRIGGER_CONFORM; isrc->isrc_pol = INTR_POLARITY_CONFORM; CPU_ZERO(&isrc->isrc_cpu); @@ -350,19 +350,19 @@ isrc_alloc(u_int type, u_int extsize) * Free interrupt source. */ static void -isrc_free(struct arm_irqsrc *isrc) +isrc_free(struct intr_irqsrc *isrc) { free(isrc, M_INTRNG); } void -arm_irq_set_name(struct arm_irqsrc *isrc, const char *fmt, ...) +intr_irq_set_name(struct intr_irqsrc *isrc, const char *fmt, ...) { va_list ap; va_start(ap, fmt); - vsnprintf(isrc->isrc_name, ARM_ISRC_NAMELEN, fmt, ap); + vsnprintf(isrc->isrc_name, INTR_ISRC_NAMELEN, fmt, ap); va_end(ap); } @@ -377,7 +377,7 @@ arm_irq_set_name(struct arm_irqsrc *isrc, const char *fmt, ...) * constantly... */ static int -isrc_alloc_irq_locked(struct arm_irqsrc *isrc) +isrc_alloc_irq_locked(struct intr_irqsrc *isrc) { u_int maxirqs, irq; @@ -403,7 +403,7 @@ found: isrc->isrc_irq = irq; irq_sources[irq] = isrc; - arm_irq_set_name(isrc, "irq%u", irq); + intr_irq_set_name(isrc, "irq%u", irq); isrc_setup_counters(isrc); irq_next_free = irq + 1; @@ -416,7 +416,7 @@ found: * Free unique interrupt number (resource handle) from interrupt source. */ static int -isrc_free_irq(struct arm_irqsrc *isrc) +isrc_free_irq(struct intr_irqsrc *isrc) { u_int maxirqs; @@ -442,7 +442,7 @@ isrc_free_irq(struct arm_irqsrc *isrc) /* * Lookup interrupt source by interrupt number (resource handle). */ -static struct arm_irqsrc * +static struct intr_irqsrc * isrc_lookup(u_int irq) { @@ -454,11 +454,11 @@ isrc_lookup(u_int irq) /* * Lookup interrupt source by namespace description. */ -static struct arm_irqsrc * +static struct intr_irqsrc * isrc_namespace_lookup(device_t dev, uint16_t type, uint16_t num) { u_int irq; - struct arm_irqsrc *isrc; + struct intr_irqsrc *isrc; mtx_assert(&isrc_table_lock, MA_OWNED); @@ -477,12 +477,12 @@ isrc_namespace_lookup(device_t dev, uint16_t type, uint16_t num) * associated with mapped interrupt source. */ u_int -arm_namespace_map_irq(device_t dev, uint16_t type, uint16_t num) +intr_namespace_map_irq(device_t dev, uint16_t type, uint16_t num) { - struct arm_irqsrc *isrc, *new_isrc; + struct intr_irqsrc *isrc, *new_isrc; int error; - new_isrc = isrc_alloc(ARM_ISRCT_NAMESPACE, 0); + new_isrc = isrc_alloc(INTR_ISRCT_NAMESPACE, 0); mtx_lock(&isrc_table_lock); isrc = isrc_namespace_lookup(dev, type, num); @@ -511,18 +511,18 @@ arm_namespace_map_irq(device_t dev, uint16_t type, uint16_t num) /* * Lookup interrupt source by FDT description. */ -static struct arm_irqsrc * +static struct intr_irqsrc * isrc_fdt_lookup(intptr_t xref, pcell_t *cells, u_int ncells) { u_int irq, cellsize; - struct arm_irqsrc *isrc; + struct intr_irqsrc *isrc; mtx_assert(&isrc_table_lock, MA_OWNED); cellsize = ncells * sizeof(*cells); for (irq = 0; irq < nitems(irq_sources); irq++) { isrc = irq_sources[irq]; - if (isrc != NULL && isrc->isrc_type == ARM_ISRCT_FDT && + if (isrc != NULL && isrc->isrc_type == INTR_ISRCT_FDT && isrc->isrc_xref == xref && isrc->isrc_ncells == ncells && memcmp(isrc->isrc_cells, cells, cellsize) == 0) return (isrc); @@ -536,9 +536,9 @@ isrc_fdt_lookup(intptr_t xref, pcell_t *cells, u_int ncells) * associated with mapped interrupt source. */ u_int -arm_fdt_map_irq(phandle_t node, pcell_t *cells, u_int ncells) +intr_fdt_map_irq(phandle_t node, pcell_t *cells, u_int ncells) { - struct arm_irqsrc *isrc, *new_isrc; + struct intr_irqsrc *isrc, *new_isrc; u_int cellsize; intptr_t xref; int error; @@ -546,7 +546,7 @@ arm_fdt_map_irq(phandle_t node, pcell_t *cells, u_int ncells) xref = (intptr_t)node; /* It's so simple for now. */ cellsize = ncells * sizeof(*cells); - new_isrc = isrc_alloc(ARM_ISRCT_FDT, cellsize); + new_isrc = isrc_alloc(INTR_ISRCT_FDT, cellsize); mtx_lock(&isrc_table_lock); isrc = isrc_fdt_lookup(xref, cells, ncells); @@ -576,13 +576,13 @@ arm_fdt_map_irq(phandle_t node, pcell_t *cells, u_int ncells) * Register interrupt source into interrupt controller. */ static int -isrc_register(struct arm_irqsrc *isrc) +isrc_register(struct intr_irqsrc *isrc) { - struct arm_pic *pic; + struct intr_pic *pic; boolean_t is_percpu; int error; - if (isrc->isrc_flags & ARM_ISRCF_REGISTERED) + if (isrc->isrc_flags & INTR_ISRCF_REGISTERED) return (0); if (isrc->isrc_dev == NULL) { @@ -597,9 +597,9 @@ isrc_register(struct arm_irqsrc *isrc) return (error); mtx_lock(&isrc_table_lock); - isrc->isrc_flags |= ARM_ISRCF_REGISTERED; + isrc->isrc_flags |= INTR_ISRCF_REGISTERED; if (is_percpu) - isrc->isrc_flags |= ARM_ISRCF_PERCPU; + isrc->isrc_flags |= INTR_ISRCF_PERCPU; isrc_update_name(isrc, NULL); mtx_unlock(&isrc_table_lock); return (0); @@ -610,8 +610,8 @@ isrc_register(struct arm_irqsrc *isrc) * Setup filter into interrupt source. */ static int -iscr_setup_filter(struct arm_irqsrc *isrc, const char *name, - arm_irq_filter_t *filter, void *arg, void **cookiep) +iscr_setup_filter(struct intr_irqsrc *isrc, const char *name, + intr_irq_filter_t *filter, void *arg, void **cookiep) { if (filter == NULL) @@ -640,9 +640,9 @@ iscr_setup_filter(struct arm_irqsrc *isrc, const char *name, * Interrupt source pre_ithread method for MI interrupt framework. */ static void -arm_isrc_pre_ithread(void *arg) +intr_isrc_pre_ithread(void *arg) { - struct arm_irqsrc *isrc = arg; + struct intr_irqsrc *isrc = arg; PIC_PRE_ITHREAD(isrc->isrc_dev, isrc); } @@ -651,9 +651,9 @@ arm_isrc_pre_ithread(void *arg) * Interrupt source post_ithread method for MI interrupt framework. */ static void -arm_isrc_post_ithread(void *arg) +intr_isrc_post_ithread(void *arg) { - struct arm_irqsrc *isrc = arg; + struct intr_irqsrc *isrc = arg; PIC_POST_ITHREAD(isrc->isrc_dev, isrc); } @@ -662,9 +662,9 @@ arm_isrc_post_ithread(void *arg) * Interrupt source post_filter method for MI interrupt framework. */ static void -arm_isrc_post_filter(void *arg) +intr_isrc_post_filter(void *arg) { - struct arm_irqsrc *isrc = arg; + struct intr_irqsrc *isrc = arg; PIC_POST_FILTER(isrc->isrc_dev, isrc); } @@ -673,10 +673,10 @@ arm_isrc_post_filter(void *arg) * Interrupt source assign_cpu method for MI interrupt framework. */ static int -arm_isrc_assign_cpu(void *arg, int cpu) +intr_isrc_assign_cpu(void *arg, int cpu) { #ifdef SMP - struct arm_irqsrc *isrc = arg; + struct intr_irqsrc *isrc = arg; int error; if (isrc->isrc_dev != irq_root_dev) @@ -685,10 +685,10 @@ arm_isrc_assign_cpu(void *arg, int cpu) mtx_lock(&isrc_table_lock); if (cpu == NOCPU) { CPU_ZERO(&isrc->isrc_cpu); - isrc->isrc_flags &= ~ARM_ISRCF_BOUND; + isrc->isrc_flags &= ~INTR_ISRCF_BOUND; } else { CPU_SETOF(cpu, &isrc->isrc_cpu); - isrc->isrc_flags |= ARM_ISRCF_BOUND; + isrc->isrc_flags |= INTR_ISRCF_BOUND; } /* @@ -716,14 +716,14 @@ arm_isrc_assign_cpu(void *arg, int cpu) * Create interrupt event for interrupt source. */ static int -isrc_event_create(struct arm_irqsrc *isrc) +isrc_event_create(struct intr_irqsrc *isrc) { struct intr_event *ie; int error; error = intr_event_create(&ie, isrc, 0, isrc->isrc_irq, - arm_isrc_pre_ithread, arm_isrc_post_ithread, arm_isrc_post_filter, - arm_isrc_assign_cpu, "%s:", isrc->isrc_name); + intr_isrc_pre_ithread, intr_isrc_post_ithread, intr_isrc_post_filter, + intr_isrc_assign_cpu, "%s:", isrc->isrc_name); if (error) return (error); @@ -747,7 +747,7 @@ isrc_event_create(struct arm_irqsrc *isrc) * Destroy interrupt event for interrupt source. */ static void -isrc_event_destroy(struct arm_irqsrc *isrc) +isrc_event_destroy(struct intr_irqsrc *isrc) { struct intr_event *ie; @@ -764,7 +764,7 @@ isrc_event_destroy(struct arm_irqsrc *isrc) * Add handler to interrupt source. */ static int -isrc_add_handler(struct arm_irqsrc *isrc, const char *name, +isrc_add_handler(struct intr_irqsrc *isrc, const char *name, driver_filter_t filter, driver_intr_t handler, void *arg, enum intr_type flags, void **cookiep) { @@ -790,10 +790,10 @@ isrc_add_handler(struct arm_irqsrc *isrc, const char *name, /* * Lookup interrupt controller locked. */ -static struct arm_pic * +static struct intr_pic * pic_lookup_locked(device_t dev, intptr_t xref) { - struct arm_pic *pic; + struct intr_pic *pic; mtx_assert(&pic_list_lock, MA_OWNED); @@ -809,10 +809,10 @@ pic_lookup_locked(device_t dev, intptr_t xref) /* * Lookup interrupt controller. */ -static struct arm_pic * +static struct intr_pic * pic_lookup(device_t dev, intptr_t xref) { - struct arm_pic *pic; + struct intr_pic *pic; mtx_lock(&pic_list_lock); pic = pic_lookup_locked(dev, xref); @@ -824,10 +824,10 @@ pic_lookup(device_t dev, intptr_t xref) /* * Create interrupt controller. */ -static struct arm_pic * +static struct intr_pic * pic_create(device_t dev, intptr_t xref) { - struct arm_pic *pic; + struct intr_pic *pic; mtx_lock(&pic_list_lock); pic = pic_lookup_locked(dev, xref); @@ -850,7 +850,7 @@ pic_create(device_t dev, intptr_t xref) static void pic_destroy(device_t dev, intptr_t xref) { - struct arm_pic *pic; + struct intr_pic *pic; mtx_lock(&pic_list_lock); pic = pic_lookup_locked(dev, xref); @@ -858,7 +858,7 @@ pic_destroy(device_t dev, intptr_t xref) mtx_unlock(&pic_list_lock); return; } - SLIST_REMOVE(&pic_list, pic, arm_pic, pic_next); + SLIST_REMOVE(&pic_list, pic, intr_pic, pic_next); mtx_unlock(&pic_list_lock); free(pic, M_INTRNG); @@ -868,9 +868,9 @@ pic_destroy(device_t dev, intptr_t xref) * Register interrupt controller. */ int -arm_pic_register(device_t dev, intptr_t xref) +intr_pic_register(device_t dev, intptr_t xref) { - struct arm_pic *pic; + struct intr_pic *pic; pic = pic_create(dev, xref); if (pic == NULL) @@ -887,7 +887,7 @@ arm_pic_register(device_t dev, intptr_t xref) * Unregister interrupt controller. */ int -arm_pic_unregister(device_t dev, intptr_t xref) +intr_pic_unregister(device_t dev, intptr_t xref) { panic("%s: not implemented", __func__); @@ -906,7 +906,7 @@ arm_pic_unregister(device_t dev, intptr_t xref) * an interrupts property and thus no explicit interrupt parent." */ int -arm_pic_claim_root(device_t dev, intptr_t xref, arm_irq_filter_t *filter, +intr_pic_claim_root(device_t dev, intptr_t xref, intr_irq_filter_t *filter, void *arg, u_int ipicount) { int error; @@ -924,14 +924,14 @@ arm_pic_claim_root(device_t dev, intptr_t xref, arm_irq_filter_t *filter, /* * Only one interrupt controllers could be on the root for now. * Note that we further suppose that there is not threaded interrupt - * routine (handler) on the root. See arm_irq_handler(). + * routine (handler) on the root. See intr_irq_handler(). */ if (irq_root_dev != NULL) { device_printf(dev, "another root already set\n"); return (EBUSY); } - rootirq = arm_namespace_map_irq(device_get_parent(dev), 0, 0); + rootirq = intr_namespace_map_irq(device_get_parent(dev), 0, 0); if (rootirq == IRQ_INVALID) { device_printf(dev, "failed to map an irq for the root pic\n"); return (ENOMEM); @@ -941,9 +941,9 @@ arm_pic_claim_root(device_t dev, intptr_t xref, arm_irq_filter_t *filter, irq_root_isrc = isrc_lookup(rootirq); /* XXX "register" with the PIC. We are the "pic" here, so fake it. */ - irq_root_isrc->isrc_flags |= ARM_ISRCF_REGISTERED; + irq_root_isrc->isrc_flags |= INTR_ISRCF_REGISTERED; - error = arm_irq_add_handler(device_get_parent(dev), + error = intr_irq_add_handler(device_get_parent(dev), (void*)filter, NULL, arg, rootirq, INTR_TYPE_CLK, NULL); if (error != 0) { device_printf(dev, "failed to install root pic handler\n"); @@ -959,11 +959,11 @@ arm_pic_claim_root(device_t dev, intptr_t xref, arm_irq_filter_t *filter, } int -arm_irq_add_handler(device_t dev, driver_filter_t filt, driver_intr_t hand, +intr_irq_add_handler(device_t dev, driver_filter_t filt, driver_intr_t hand, void *arg, u_int irq, int flags, void **cookiep) { const char *name; - struct arm_irqsrc *isrc; + struct intr_irqsrc *isrc; int error; name = device_get_nameunit(dev); @@ -977,7 +977,7 @@ arm_irq_add_handler(device_t dev, driver_filter_t filt, driver_intr_t hand, * chained, MI interrupt framework is called only in leaf controller. * * Note that root interrupt controller routine is served as well, - * however in arm_irq_handler(), i.e. main system dispatch routine. + * however in intr_irq_handler(), i.e. main system dispatch routine. */ if (flags & INTR_SOLO && hand != NULL) { debugf("irq %u cannot solo on %s\n", irq, name); @@ -999,7 +999,7 @@ arm_irq_add_handler(device_t dev, driver_filter_t filt, driver_intr_t hand, #ifdef INTR_SOLO if (flags & INTR_SOLO) { - error = iscr_setup_filter(isrc, name, (arm_irq_filter_t *)filt, + error = iscr_setup_filter(isrc, name, (intr_irq_filter_t *)filt, arg, cookiep); debugf("irq %u setup filter error %d on %s\n", irq, error, name); @@ -1024,9 +1024,9 @@ arm_irq_add_handler(device_t dev, driver_filter_t filt, driver_intr_t hand, } int -arm_irq_remove_handler(device_t dev, u_int irq, void *cookie) +intr_irq_remove_handler(device_t dev, u_int irq, void *cookie) { - struct arm_irqsrc *isrc; + struct intr_irqsrc *isrc; int error; isrc = isrc_lookup(irq); @@ -1066,9 +1066,9 @@ arm_irq_remove_handler(device_t dev, u_int irq, void *cookie) } int -arm_irq_config(u_int irq, enum intr_trigger trig, enum intr_polarity pol) +intr_irq_config(u_int irq, enum intr_trigger trig, enum intr_polarity pol) { - struct arm_irqsrc *isrc; + struct intr_irqsrc *isrc; isrc = isrc_lookup(irq); if (isrc == NULL) @@ -1090,9 +1090,9 @@ arm_irq_config(u_int irq, enum intr_trigger trig, enum intr_polarity pol) } int -arm_irq_describe(u_int irq, void *cookie, const char *descr) +intr_irq_describe(u_int irq, void *cookie, const char *descr) { - struct arm_irqsrc *isrc; + struct intr_irqsrc *isrc; int error; isrc = isrc_lookup(irq); @@ -1120,16 +1120,16 @@ arm_irq_describe(u_int irq, void *cookie, const char *descr) #ifdef SMP int -arm_irq_bind(u_int irq, int cpu) +intr_irq_bind(u_int irq, int cpu) { - struct arm_irqsrc *isrc; + struct intr_irqsrc *isrc; isrc = isrc_lookup(irq); if (isrc == NULL || isrc->isrc_handlers == 0) return (EINVAL); if (isrc->isrc_filter != NULL) - return (arm_isrc_assign_cpu(isrc, cpu)); + return (intr_isrc_assign_cpu(isrc, cpu)); return (intr_event_bind(isrc->isrc_event, cpu)); } @@ -1139,7 +1139,7 @@ arm_irq_bind(u_int irq, int cpu) * For now just returns the next CPU according to round-robin. */ u_int -arm_irq_next_cpu(u_int last_cpu, cpuset_t *cpumask) +intr_irq_next_cpu(u_int last_cpu, cpuset_t *cpumask) { if (!irq_assign_cpu || mp_ncpus == 1) @@ -1158,9 +1158,9 @@ arm_irq_next_cpu(u_int last_cpu, cpuset_t *cpumask) * CPUs once the AP's have been launched. */ static void -arm_irq_shuffle(void *arg __unused) +intr_irq_shuffle(void *arg __unused) { - struct arm_irqsrc *isrc; + struct intr_irqsrc *isrc; u_int i; if (mp_ncpus == 1) @@ -1171,15 +1171,15 @@ arm_irq_shuffle(void *arg __unused) for (i = 0; i < NIRQ; i++) { isrc = irq_sources[i]; if (isrc == NULL || isrc->isrc_handlers == 0 || - isrc->isrc_flags & ARM_ISRCF_PERCPU) + isrc->isrc_flags & INTR_ISRCF_PERCPU) continue; if (isrc->isrc_event != NULL && - isrc->isrc_flags & ARM_ISRCF_BOUND && + isrc->isrc_flags & INTR_ISRCF_BOUND && isrc->isrc_event->ie_cpu != CPU_FFS(&isrc->isrc_cpu) - 1) panic("%s: CPU inconsistency", __func__); - if ((isrc->isrc_flags & ARM_ISRCF_BOUND) == 0) + if ((isrc->isrc_flags & INTR_ISRCF_BOUND) == 0) CPU_ZERO(&isrc->isrc_cpu); /* start again */ /* @@ -1192,11 +1192,11 @@ arm_irq_shuffle(void *arg __unused) } mtx_unlock(&isrc_table_lock); } -SYSINIT(arm_irq_shuffle, SI_SUB_SMP, SI_ORDER_SECOND, arm_irq_shuffle, NULL); +SYSINIT(intr_irq_shuffle, SI_SUB_SMP, SI_ORDER_SECOND, intr_irq_shuffle, NULL); #else u_int -arm_irq_next_cpu(u_int current_cpu, cpuset_t *cpumask) +intr_irq_next_cpu(u_int current_cpu, cpuset_t *cpumask) { return (PCPU_GET(cpuid)); @@ -1209,91 +1209,28 @@ dosoftints(void) { } -/* - * arm_irq_memory_barrier() - * - * Ensure all writes to device memory have reached devices before proceeding. - * - * This is intended to be called from the post-filter and post-thread routines - * of an interrupt controller implementation. A peripheral device driver should - * use bus_space_barrier() if it needs to ensure a write has reached the - * hardware for some reason other than clearing interrupt conditions. - * - * The need for this function arises from the ARM weak memory ordering model. - * Writes to locations mapped with the Device attribute bypass any caches, but - * are buffered. Multiple writes to the same device will be observed by that - * device in the order issued by the cpu. Writes to different devices may - * appear at those devices in a different order than issued by the cpu. That - * is, if the cpu writes to device A then device B, the write to device B could - * complete before the write to device A. - * - * Consider a typical device interrupt handler which services the interrupt and - * writes to a device status-acknowledge register to clear the interrupt before - * returning. That write is posted to the L2 controller which "immediately" - * places it in a store buffer and automatically drains that buffer. This can - * be less immediate than you'd think... There may be no free slots in the store - * buffers, so an existing buffer has to be drained first to make room. The - * target bus may be busy with other traffic (such as DMA for various devices), - * delaying the drain of the store buffer for some indeterminate time. While - * all this delay is happening, execution proceeds on the CPU, unwinding its way - * out of the interrupt call stack to the point where the interrupt driver code - * is ready to EOI and unmask the interrupt. The interrupt controller may be - * accessed via a faster bus than the hardware whose handler just ran; the write - * to unmask and EOI the interrupt may complete quickly while the device write - * to ack and clear the interrupt source is still lingering in a store buffer - * waiting for access to a slower bus. With the interrupt unmasked at the - * interrupt controller but still active at the device, as soon as interrupts - * are enabled on the core the device re-interrupts immediately: now you've got - * a spurious interrupt on your hands. - * - * The right way to fix this problem is for every device driver to use the - * proper bus_space_barrier() calls in its interrupt handler. For ARM a single - * barrier call at the end of the handler would work. This would have to be - * done to every driver in the system, not just arm-specific drivers. - * - * Another potential fix is to map all device memory as Strongly-Ordered rather - * than Device memory, which takes the store buffers out of the picture. This - * has a pretty big impact on overall system performance, because each strongly - * ordered memory access causes all L2 store buffers to be drained. - * - * A compromise solution is to have the interrupt controller implementation call - * this function to establish a barrier between writes to the interrupt-source - * device and writes to the interrupt controller device. - * - * This takes the interrupt number as an argument, and currently doesn't use it. - * The plan is that maybe some day there is a way to flag certain interrupts as - * "memory barrier safe" and we can avoid this overhead with them. - */ -void -arm_irq_memory_barrier(uintptr_t irq) -{ - - dsb(); - cpu_l2cache_drain_writebuf(); -} - #ifdef SMP /* * Lookup IPI source. */ -static struct arm_irqsrc * -arm_ipi_lookup(u_int ipi) +static struct intr_irqsrc * +intr_ipi_lookup(u_int ipi) { - if (ipi >= ARM_IPI_COUNT) + if (ipi >= INTR_IPI_COUNT) panic("%s: no such IPI %u", __func__, ipi); return (&ipi_sources[ipi]); } /* - * ARM interrupt controller dispatch function for IPIs. It should + * interrupt controller dispatch function for IPIs. It should * be called straight from the interrupt controller, when associated * interrupt source is learned. Or from anybody who has an interrupt * source mapped. */ void -arm_ipi_dispatch(struct arm_irqsrc *isrc, struct trapframe *tf) +intr_ipi_dispatch(struct intr_irqsrc *isrc, struct trapframe *tf) { void *arg; @@ -1315,18 +1252,18 @@ arm_ipi_dispatch(struct arm_irqsrc *isrc, struct trapframe *tf) * Not SMP coherent. */ static int -ipi_map(struct arm_irqsrc *isrc, u_int ipi) +ipi_map(struct intr_irqsrc *isrc, u_int ipi) { boolean_t is_percpu; int error; - if (ipi >= ARM_IPI_COUNT) + if (ipi >= INTR_IPI_COUNT) panic("%s: no such IPI %u", __func__, ipi); KASSERT(irq_root_dev != NULL, ("%s: no root attached", __func__)); - isrc->isrc_type = ARM_ISRCT_NAMESPACE; - isrc->isrc_nspc_type = ARM_IRQ_NSPC_IPI; + isrc->isrc_type = INTR_ISRCT_NAMESPACE; + isrc->isrc_nspc_type = INTR_IRQ_NSPC_IPI; isrc->isrc_nspc_num = ipi_next_num; error = PIC_REGISTER(irq_root_dev, isrc, &is_percpu); @@ -1347,21 +1284,21 @@ ipi_map(struct arm_irqsrc *isrc, u_int ipi) * Note that there could be more ways how to send and receive IPIs * on a platform like fast interrupts for example. In that case, * one can call this function with ASIF_NOALLOC flag set and then - * call arm_ipi_dispatch() when appropriate. + * call intr_ipi_dispatch() when appropriate. * * Not SMP coherent. */ int -arm_ipi_set_handler(u_int ipi, const char *name, arm_ipi_filter_t *filter, +intr_ipi_set_handler(u_int ipi, const char *name, intr_ipi_filter_t *filter, void *arg, u_int flags) { - struct arm_irqsrc *isrc; + struct intr_irqsrc *isrc; int error; if (filter == NULL) return(EINVAL); - isrc = arm_ipi_lookup(ipi); + isrc = intr_ipi_lookup(ipi); if (isrc->isrc_ipifilter != NULL) return (EEXIST); @@ -1391,9 +1328,9 @@ arm_ipi_set_handler(u_int ipi, const char *name, arm_ipi_filter_t *filter, void pic_ipi_send(cpuset_t cpus, u_int ipi) { - struct arm_irqsrc *isrc; + struct intr_irqsrc *isrc; - isrc = arm_ipi_lookup(ipi); + isrc = intr_ipi_lookup(ipi); KASSERT(irq_root_dev != NULL, ("%s: no root attached", __func__)); PIC_IPI_SEND(irq_root_dev, isrc, cpus); @@ -1403,7 +1340,7 @@ pic_ipi_send(cpuset_t cpus, u_int ipi) * Init interrupt controller on another CPU. */ void -arm_pic_init_secondary(void) +intr_pic_init_secondary(void) { /* @@ -1421,7 +1358,7 @@ arm_pic_init_secondary(void) DB_SHOW_COMMAND(irqs, db_show_irqs) { u_int i, irqsum; - struct arm_irqsrc *isrc; + struct intr_irqsrc *isrc; #ifdef SMP for (i = 0; i <= mp_maxid; i++) { @@ -1430,8 +1367,8 @@ DB_SHOW_COMMAND(irqs, db_show_irqs) pc = pcpu_find(i); if (pc != NULL) { - for (ipisum = 0, ipi = 0; ipi < ARM_IPI_COUNT; ipi++) { - isrc = arm_ipi_lookup(ipi); + for (ipisum = 0, ipi = 0; ipi < INTR_IPI_COUNT; ipi++) { + isrc = intr_ipi_lookup(ipi); if (isrc->isrc_count != NULL) ipisum += isrc->isrc_count[i]; } @@ -1449,7 +1386,7 @@ DB_SHOW_COMMAND(irqs, db_show_irqs) db_printf("irq%-3u <%s>: cpu %02lx%s cnt %lu\n", i, isrc->isrc_name, isrc->isrc_cpu.__bits[0], - isrc->isrc_flags & ARM_ISRCF_BOUND ? " (bound)" : "", + isrc->isrc_flags & INTR_ISRCF_BOUND ? " (bound)" : "", isrc->isrc_count[0]); irqsum += isrc->isrc_count[0]; } diff --git a/sys/mips/include/pmap.h b/sys/mips/include/pmap.h index 8392d8a..0354119 100644 --- a/sys/mips/include/pmap.h +++ b/sys/mips/include/pmap.h @@ -178,7 +178,6 @@ void *pmap_kenter_temporary(vm_paddr_t pa, int i); void pmap_kenter_temporary_free(vm_paddr_t pa); void pmap_flush_pvcache(vm_page_t m); int pmap_emulate_modified(pmap_t pmap, vm_offset_t va); -void pmap_grow_direct_page_cache(void); void pmap_page_set_memattr(vm_page_t, vm_memattr_t); #endif /* _KERNEL */ diff --git a/sys/mips/mips/pmap.c b/sys/mips/mips/pmap.c index 7bf3ee0..0827ad1 100644 --- a/sys/mips/mips/pmap.c +++ b/sys/mips/mips/pmap.c @@ -166,6 +166,7 @@ static pv_entry_t pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, static vm_page_t pmap_alloc_direct_page(unsigned int index, int req); static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, vm_page_t mpte); +static void pmap_grow_direct_page(int req); static int pmap_remove_pte(struct pmap *pmap, pt_entry_t *ptq, vm_offset_t va, pd_entry_t pde); static void pmap_remove_page(struct pmap *pmap, vm_offset_t va); @@ -1040,14 +1041,16 @@ pmap_pinit0(pmap_t pmap) bzero(&pmap->pm_stats, sizeof pmap->pm_stats); } -void -pmap_grow_direct_page_cache() +static void +pmap_grow_direct_page(int req) { #ifdef __mips_n64 VM_WAIT; #else - vm_pageout_grow_cache(3, 0, MIPS_KSEG0_LARGEST_PHYS); + if (!vm_page_reclaim_contig(req, 1, 0, MIPS_KSEG0_LARGEST_PHYS, + PAGE_SIZE, 0)) + VM_WAIT; #endif } @@ -1077,13 +1080,15 @@ pmap_pinit(pmap_t pmap) { vm_offset_t ptdva; vm_page_t ptdpg; - int i; + int i, req_class; /* * allocate the page directory page */ - while ((ptdpg = pmap_alloc_direct_page(NUSERPGTBLS, VM_ALLOC_NORMAL)) == NULL) - pmap_grow_direct_page_cache(); + req_class = VM_ALLOC_NORMAL; + while ((ptdpg = pmap_alloc_direct_page(NUSERPGTBLS, req_class)) == + NULL) + pmap_grow_direct_page(req_class); ptdva = MIPS_PHYS_TO_DIRECT(VM_PAGE_TO_PHYS(ptdpg)); pmap->pm_segtab = (pd_entry_t *)ptdva; @@ -1107,15 +1112,17 @@ _pmap_allocpte(pmap_t pmap, unsigned ptepindex, u_int flags) { vm_offset_t pageva; vm_page_t m; + int req_class; /* * Find or fabricate a new pagetable page */ - if ((m = pmap_alloc_direct_page(ptepindex, VM_ALLOC_NORMAL)) == NULL) { + req_class = VM_ALLOC_NORMAL; + if ((m = pmap_alloc_direct_page(ptepindex, req_class)) == NULL) { if ((flags & PMAP_ENTER_NOSLEEP) == 0) { PMAP_UNLOCK(pmap); rw_wunlock(&pvh_global_lock); - pmap_grow_direct_page_cache(); + pmap_grow_direct_page(req_class); rw_wlock(&pvh_global_lock); PMAP_LOCK(pmap); } @@ -1241,9 +1248,10 @@ pmap_growkernel(vm_offset_t addr) vm_page_t nkpg; pd_entry_t *pde, *pdpe; pt_entry_t *pte; - int i; + int i, req_class; mtx_assert(&kernel_map->system_mtx, MA_OWNED); + req_class = VM_ALLOC_INTERRUPT; addr = roundup2(addr, NBSEG); if (addr - 1 >= kernel_map->max_offset) addr = kernel_map->max_offset; @@ -1252,7 +1260,7 @@ pmap_growkernel(vm_offset_t addr) #ifdef __mips_n64 if (*pdpe == 0) { /* new intermediate page table entry */ - nkpg = pmap_alloc_direct_page(nkpt, VM_ALLOC_INTERRUPT); + nkpg = pmap_alloc_direct_page(nkpt, req_class); if (nkpg == NULL) panic("pmap_growkernel: no memory to grow kernel"); *pdpe = (pd_entry_t)MIPS_PHYS_TO_DIRECT(VM_PAGE_TO_PHYS(nkpg)); @@ -1272,8 +1280,13 @@ pmap_growkernel(vm_offset_t addr) /* * This index is bogus, but out of the way */ - nkpg = pmap_alloc_direct_page(nkpt, VM_ALLOC_INTERRUPT); - if (!nkpg) + nkpg = pmap_alloc_direct_page(nkpt, req_class); +#ifndef __mips_n64 + if (nkpg == NULL && vm_page_reclaim_contig(req_class, 1, + 0, MIPS_KSEG0_LARGEST_PHYS, PAGE_SIZE, 0)) + nkpg = pmap_alloc_direct_page(nkpt, req_class); +#endif + if (nkpg == NULL) panic("pmap_growkernel: no memory to grow kernel"); nkpt++; *pde = (pd_entry_t)MIPS_PHYS_TO_DIRECT(VM_PAGE_TO_PHYS(nkpg)); diff --git a/sys/mips/mips/uma_machdep.c b/sys/mips/mips/uma_machdep.c index e47c502..b4006e1 100644 --- a/sys/mips/mips/uma_machdep.c +++ b/sys/mips/mips/uma_machdep.c @@ -53,11 +53,16 @@ uma_small_alloc(uma_zone_t zone, vm_size_t bytes, u_int8_t *flags, int wait) for (;;) { m = vm_page_alloc_freelist(VM_FREELIST_DIRECT, pflags); +#ifndef __mips_n64 + if (m == NULL && vm_page_reclaim_contig(pflags, 1, + 0, MIPS_KSEG0_LARGEST_PHYS, PAGE_SIZE, 0)) + continue; +#endif if (m == NULL) { if (wait & M_NOWAIT) return (NULL); else - pmap_grow_direct_page_cache(); + VM_WAIT; } else break; } diff --git a/sys/modules/geom/geom_sched/Makefile b/sys/modules/geom/geom_sched/Makefile index 5937fa0..72ee42a 100644 --- a/sys/modules/geom/geom_sched/Makefile +++ b/sys/modules/geom/geom_sched/Makefile @@ -1,5 +1,5 @@ # $FreeBSD$ -SUBDIR= gs_sched gsched_rr +SUBDIR= gs_sched gsched_rr gsched_delay .include <bsd.subdir.mk> diff --git a/sys/modules/geom/geom_sched/gsched_delay/Makefile b/sys/modules/geom/geom_sched/gsched_delay/Makefile new file mode 100644 index 0000000..4d4381d --- /dev/null +++ b/sys/modules/geom/geom_sched/gsched_delay/Makefile @@ -0,0 +1,7 @@ +# $FreeBSD$ + +KMOD= gsched_delay +SRCS= gs_delay.c + +# ../Makefile.inc automatically included +.include <bsd.kmod.mk> diff --git a/sys/net/if.c b/sys/net/if.c index 177f356..b88c05e 100644 --- a/sys/net/if.c +++ b/sys/net/if.c @@ -126,7 +126,7 @@ SX_SYSINIT(ifdescr_sx, &ifdescr_sx, "ifnet descr"); void (*bridge_linkstate_p)(struct ifnet *ifp); void (*ng_ether_link_state_p)(struct ifnet *ifp, int state); -void (*lagg_linkstate_p)(struct ifnet *ifp); +void (*lagg_linkstate_p)(struct ifnet *ifp, int state); /* These are external hooks for CARP. */ void (*carp_linkstate_p)(struct ifnet *ifp); void (*carp_demote_adj_p)(int, char *); @@ -1984,8 +1984,6 @@ if_unroute(struct ifnet *ifp, int flag, int fam) if (ifp->if_carp) (*carp_linkstate_p)(ifp); - if (ifp->if_lagg) - (*lagg_linkstate_p)(ifp); rt_ifmsg(ifp); } @@ -2007,8 +2005,6 @@ if_route(struct ifnet *ifp, int flag, int fam) pfctlinput(PRC_IFUP, ifa->ifa_addr); if (ifp->if_carp) (*carp_linkstate_p)(ifp); - if (ifp->if_lagg) - (*lagg_linkstate_p)(ifp); rt_ifmsg(ifp); #ifdef INET6 in6_if_up(ifp); @@ -2023,27 +2019,17 @@ int (*vlan_tag_p)(struct ifnet *, uint16_t *); int (*vlan_setcookie_p)(struct ifnet *, void *); void *(*vlan_cookie_p)(struct ifnet *); -void -if_link_state_change(struct ifnet *ifp, int link_state) -{ - - return if_link_state_change_cond(ifp, link_state, 0); -} - /* * Handle a change in the interface link state. To avoid LORs * between driver lock and upper layer locks, as well as possible * recursions, we post event to taskqueue, and all job * is done in static do_link_state_change(). - * - * If the current link state matches link_state and force isn't - * specified no action is taken. */ void -if_link_state_change_cond(struct ifnet *ifp, int link_state, int force) +if_link_state_change(struct ifnet *ifp, int link_state) { - - if (ifp->if_link_state == link_state && !force) + /* Return if state hasn't changed. */ + if (ifp->if_link_state == link_state) return; ifp->if_link_state = link_state; @@ -2071,7 +2057,7 @@ do_link_state_change(void *arg, int pending) if (ifp->if_bridge) (*bridge_linkstate_p)(ifp); if (ifp->if_lagg) - (*lagg_linkstate_p)(ifp); + (*lagg_linkstate_p)(ifp, link_state); if (IS_DEFAULT_VNET(curvnet)) devctl_notify("IFNET", ifp->if_xname, diff --git a/sys/net/if_lagg.c b/sys/net/if_lagg.c index 6ca3f22..730a044 100644 --- a/sys/net/if_lagg.c +++ b/sys/net/if_lagg.c @@ -106,7 +106,7 @@ static int lagg_port_create(struct lagg_softc *, struct ifnet *); static int lagg_port_destroy(struct lagg_port *, int); static struct mbuf *lagg_input(struct ifnet *, struct mbuf *); static void lagg_linkstate(struct lagg_softc *); -static void lagg_port_state(struct ifnet *); +static void lagg_port_state(struct ifnet *, int); static int lagg_port_ioctl(struct ifnet *, u_long, caddr_t); static int lagg_port_output(struct ifnet *, struct mbuf *, const struct sockaddr *, struct route *); @@ -1774,12 +1774,7 @@ lagg_linkstate(struct lagg_softc *sc) break; } } - - /* - * Force state change to ensure ifnet_link_event is generated allowing - * protocols to notify other nodes of potential address move. - */ - if_link_state_change_cond(sc->sc_ifp, new_link, 1); + if_link_state_change(sc->sc_ifp, new_link); /* Update if_baudrate to reflect the max possible speed */ switch (sc->sc_proto) { @@ -1802,7 +1797,7 @@ lagg_linkstate(struct lagg_softc *sc) } static void -lagg_port_state(struct ifnet *ifp) +lagg_port_state(struct ifnet *ifp, int state) { struct lagg_port *lp = (struct lagg_port *)ifp->if_lagg; struct lagg_softc *sc = NULL; @@ -1818,7 +1813,7 @@ lagg_port_state(struct ifnet *ifp) LAGG_WUNLOCK(sc); } -static struct lagg_port * +struct lagg_port * lagg_link_active(struct lagg_softc *sc, struct lagg_port *lp) { struct lagg_port *lp_next, *rval = NULL; diff --git a/sys/net/if_lagg.h b/sys/net/if_lagg.h index ea59762..195ac3a 100644 --- a/sys/net/if_lagg.h +++ b/sys/net/if_lagg.h @@ -281,7 +281,7 @@ struct lagg_port { #define LAGG_UNLOCK_ASSERT(_sc) rm_assert(&(_sc)->sc_mtx, RA_UNLOCKED) extern struct mbuf *(*lagg_input_p)(struct ifnet *, struct mbuf *); -extern void (*lagg_linkstate_p)(struct ifnet *); +extern void (*lagg_linkstate_p)(struct ifnet *, int ); int lagg_enqueue(struct ifnet *, struct mbuf *); diff --git a/sys/net/if_var.h b/sys/net/if_var.h index 2cbd76c..5911cec 100644 --- a/sys/net/if_var.h +++ b/sys/net/if_var.h @@ -500,7 +500,6 @@ struct ifmultiaddr * void if_free(struct ifnet *); void if_initname(struct ifnet *, const char *, int); void if_link_state_change(struct ifnet *, int); -void if_link_state_change_cond(struct ifnet *, int, int); int if_printf(struct ifnet *, const char *, ...) __printflike(2, 3); void if_ref(struct ifnet *); void if_rele(struct ifnet *); diff --git a/sys/netinet/if_ether.c b/sys/netinet/if_ether.c index 6ca4844..931883d 100644 --- a/sys/netinet/if_ether.c +++ b/sys/netinet/if_ether.c @@ -107,7 +107,6 @@ VNET_PCPUSTAT_SYSUNINIT(arpstat); #endif /* VIMAGE */ static VNET_DEFINE(int, arp_maxhold) = 1; -static VNET_DEFINE(int, arp_on_link) = 1; #define V_arpt_keep VNET(arpt_keep) #define V_arpt_down VNET(arpt_down) @@ -115,7 +114,6 @@ static VNET_DEFINE(int, arp_on_link) = 1; #define V_arp_maxtries VNET(arp_maxtries) #define V_arp_proxyall VNET(arp_proxyall) #define V_arp_maxhold VNET(arp_maxhold) -#define V_arp_on_link VNET(arp_on_link) SYSCTL_INT(_net_link_ether_inet, OID_AUTO, max_age, CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(arpt_keep), 0, @@ -138,9 +136,6 @@ SYSCTL_INT(_net_link_ether_inet, OID_AUTO, max_log_per_second, CTLFLAG_RW, &arp_maxpps, 0, "Maximum number of remotely triggered ARP messages that can be " "logged per second"); -SYSCTL_INT(_net_link_ether_inet, OID_AUTO, arp_on_link, CTLFLAG_VNET | CTLFLAG_RW, - &VNET_NAME(arp_on_link), 0, - "Send gratuitous ARP's on interface link up events"); #define ARP_LOG(pri, ...) do { \ if (ppsratecheck(&arp_lastlog, &arp_curpps, arp_maxpps)) \ @@ -161,7 +156,6 @@ static void arp_mark_lle_reachable(struct llentry *la); static void arp_iflladdr(void *arg __unused, struct ifnet *ifp); static eventhandler_tag iflladdr_tag; -static eventhandler_tag ifnet_link_event_tag; static const struct netisr_handler arp_nh = { .nh_name = "arp", @@ -1190,99 +1184,43 @@ arp_ifinit(struct ifnet *ifp, struct ifaddr *ifa) if (ntohl(dst_in->sin_addr.s_addr) == INADDR_ANY) return; - arp_announce_addr(ifp, &dst_in->sin_addr, IF_LLADDR(ifp)); + arp_announce_ifaddr(ifp, dst_in->sin_addr, IF_LLADDR(ifp)); arp_add_ifa_lle(ifp, dst); } -void __noinline -arp_announce_addr(struct ifnet *ifp, const struct in_addr *addr, u_char *enaddr) +void +arp_announce_ifaddr(struct ifnet *ifp, struct in_addr addr, u_char *enaddr) { - if (ntohl(addr->s_addr) != INADDR_ANY) - arprequest(ifp, addr, addr, enaddr); + if (ntohl(addr.s_addr) != INADDR_ANY) + arprequest(ifp, &addr, &addr, enaddr); } /* - * Send gratuitous ARPs for all interfaces addresses to notify other nodes of - * changes. - * - * This is a noop if the interface isn't up or has been flagged for no ARP. + * Sends gratuitous ARPs for each ifaddr to notify other + * nodes about the address change. */ -void __noinline -arp_announce(struct ifnet *ifp) +static __noinline void +arp_handle_ifllchange(struct ifnet *ifp) { - int i, cnt, entries; - u_char *lladdr; struct ifaddr *ifa; - struct in_addr *addr, *head; - if (!(ifp->if_flags & IFF_UP) || (ifp->if_flags & IFF_NOARP) || - ifp->if_addr == NULL) - return; - - entries = 8; - cnt = 0; - head = malloc(sizeof(*addr) * entries, M_TEMP, M_NOWAIT); - if (head == NULL) { - log(LOG_INFO, "arp_announce: malloc %d entries failed\n", - entries); - return; - } - - /* Take a copy then process to avoid locking issues. */ - IF_ADDR_RLOCK(ifp); TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { - if (ifa->ifa_addr->sa_family != AF_INET) - continue; - - if (cnt == entries) { - addr = (struct in_addr *)realloc(head, sizeof(*addr) * - (entries + 8), M_TEMP, M_NOWAIT); - if (addr == NULL) { - log(LOG_INFO, "arp_announce: realloc to %d " - "entries failed\n", entries + 8); - /* Process what we have. */ - break; - } - entries += 8; - head = addr; - } - - addr = head + cnt; - bcopy(IFA_IN(ifa), addr, sizeof(*addr)); - cnt++; + if (ifa->ifa_addr->sa_family == AF_INET) + arp_ifinit(ifp, ifa); } - IF_ADDR_RUNLOCK(ifp); - - if (cnt > 0) { - lladdr = IF_LLADDR(ifp); - for (i = 0; i < cnt; i++) { - arp_announce_addr(ifp, head + i, lladdr); - } - } - free(head, M_TEMP); -} - -/* - * A handler for interface linkstate change events. - */ -static void -arp_ifnet_link_event(void *arg __unused, struct ifnet *ifp, int linkstate) -{ - - if (linkstate == LINK_STATE_UP && V_arp_on_link) - arp_announce(ifp); } /* - * A handler for interface link layer address change events. + * A handler for interface link layer address change event. */ static __noinline void arp_iflladdr(void *arg __unused, struct ifnet *ifp) { - arp_announce(ifp); + if ((ifp->if_flags & IFF_UP) != 0) + arp_handle_ifllchange(ifp); } static void @@ -1290,12 +1228,8 @@ arp_init(void) { netisr_register(&arp_nh); - - if (IS_DEFAULT_VNET(curvnet)) { + if (IS_DEFAULT_VNET(curvnet)) iflladdr_tag = EVENTHANDLER_REGISTER(iflladdr_event, arp_iflladdr, NULL, EVENTHANDLER_PRI_ANY); - ifnet_link_event_tag = EVENTHANDLER_REGISTER(ifnet_link_event, - arp_ifnet_link_event, 0, EVENTHANDLER_PRI_ANY); - } } SYSINIT(arp, SI_SUB_PROTO_DOMAIN, SI_ORDER_ANY, arp_init, 0); diff --git a/sys/netinet/if_ether.h b/sys/netinet/if_ether.h index ce517e4..1583ca5 100644 --- a/sys/netinet/if_ether.h +++ b/sys/netinet/if_ether.h @@ -119,8 +119,7 @@ int arpresolve(struct ifnet *ifp, int is_gw, struct mbuf *m, void arprequest(struct ifnet *, const struct in_addr *, const struct in_addr *, u_char *); void arp_ifinit(struct ifnet *, struct ifaddr *); -void arp_announce(struct ifnet *); -void arp_announce_addr(struct ifnet *, const struct in_addr *addr, u_char *); +void arp_announce_ifaddr(struct ifnet *, struct in_addr addr, u_char *); #endif #endif diff --git a/sys/netinet/in_var.h b/sys/netinet/in_var.h index 10997306..f21ddf4 100644 --- a/sys/netinet/in_var.h +++ b/sys/netinet/in_var.h @@ -129,9 +129,6 @@ extern struct rmlock in_ifaddr_lock; #define IN_IFADDR_WLOCK_ASSERT() rm_assert(&in_ifaddr_lock, RA_WLOCKED) #define IN_IFADDR_WUNLOCK() rm_wunlock(&in_ifaddr_lock) -#define IFA_IN(ifa) \ - (&((struct sockaddr_in *)ifa->ifa_addr)->sin_addr) - /* * Macro for finding the internet address structure (in_ifaddr) * corresponding to one of our IP addresses (in_addr). diff --git a/sys/netinet/ip_carp.c b/sys/netinet/ip_carp.c index da0ceab..7855af2 100644 --- a/sys/netinet/ip_carp.c +++ b/sys/netinet/ip_carp.c @@ -1009,12 +1009,13 @@ static void carp_send_arp(struct carp_softc *sc) { struct ifaddr *ifa; + struct in_addr addr; CARP_FOREACH_IFA(sc, ifa) { if (ifa->ifa_addr->sa_family != AF_INET) continue; - arp_announce_addr(sc->sc_carpdev, IFA_IN(ifa), - LLADDR(&sc->sc_addr)); + addr = ((struct sockaddr_in *)ifa->ifa_addr)->sin_addr; + arp_announce_ifaddr(sc->sc_carpdev, addr, LLADDR(&sc->sc_addr)); } } @@ -1036,16 +1037,18 @@ carp_iamatch(struct ifaddr *ifa, uint8_t **enaddr) static void carp_send_na(struct carp_softc *sc) { + static struct in6_addr mcast = IN6ADDR_LINKLOCAL_ALLNODES_INIT; struct ifaddr *ifa; + struct in6_addr *in6; CARP_FOREACH_IFA(sc, ifa) { - if (ifa->ifa_addr->sa_family != AF_INET6 || - IFA_ND6_NA_UNSOLICITED_SKIP(ifa)) + if (ifa->ifa_addr->sa_family != AF_INET6) continue; - nd6_na_output_unsolicited_addr(sc->sc_carpdev, IFA_IN6(ifa), - IFA_ND6_NA_BASE_FLAGS(sc->sc_carpdev, ifa)); - DELAY(nd6_na_unsolicited_addr_delay(ifa)); + in6 = IFA_IN6(ifa); + nd6_na_output(sc->sc_carpdev, &mcast, in6, + ND_NA_FLAG_OVERRIDE, 1, NULL); + DELAY(1000); /* XXX */ } } diff --git a/sys/netinet6/in6.c b/sys/netinet6/in6.c index c2f4397..aedaf7f 100644 --- a/sys/netinet6/in6.c +++ b/sys/netinet6/in6.c @@ -114,7 +114,7 @@ VNET_DECLARE(int, icmp6_nodeinfo_oldmcprefix); #define V_icmp6_nodeinfo_oldmcprefix VNET(icmp6_nodeinfo_oldmcprefix) /* - * Definitions of some constant IP6 addresses. + * Definitions of some costant IP6 addresses. */ const struct in6_addr in6addr_any = IN6ADDR_ANY_INIT; const struct in6_addr in6addr_loopback = IN6ADDR_LOOPBACK_INIT; diff --git a/sys/netinet6/in6_var.h b/sys/netinet6/in6_var.h index 54fa1f5..007fd66 100644 --- a/sys/netinet6/in6_var.h +++ b/sys/netinet6/in6_var.h @@ -399,16 +399,6 @@ struct in6_rrenumreq { #define IA6_SIN6(ia) (&((ia)->ia_addr)) #define IA6_DSTSIN6(ia) (&((ia)->ia_dstaddr)) #define IFA_IN6(x) (&((struct sockaddr_in6 *)((x)->ifa_addr))->sin6_addr) -#define IFA_IN6_FLAGS(ifa) ((struct in6_ifaddr *)ifa)->ia6_flags -#define IFA_ND6_NA_BASE_FLAGS(ifp, ifa) \ - (IFA_IN6_FLAGS(ifa) & IN6_IFF_ANYCAST ? 0 : ND_NA_FLAG_OVERRIDE) | \ - ((V_ip6_forwarding && !(ND_IFINFO(ifp)->flags & ND6_IFF_ACCEPT_RTADV && \ - V_ip6_norbit_raif)) ? ND_NA_FLAG_ROUTER : 0) -#define IFA_ND6_NA_UNSOLICITED_SKIP(ifa) \ - (IFA_IN6_FLAGS(ifa) & (IN6_IFF_DUPLICATED | IN6_IFF_DEPRECATED | \ - IN6_IFF_TENTATIVE)) != 0 -#define IN6_MAX_ANYCAST_DELAY_TIME_MS 1000000 -#define IN6_BROADCAST_DELAY_TIME_MS 1000 #define IFA_DSTIN6(x) (&((struct sockaddr_in6 *)((x)->ifa_dstaddr))->sin6_addr) #define IFPR_IN6(x) (&((struct sockaddr_in6 *)((x)->ifpr_prefix))->sin6_addr) diff --git a/sys/netinet6/nd6.c b/sys/netinet6/nd6.c index d2a457d..7ca172c 100644 --- a/sys/netinet6/nd6.c +++ b/sys/netinet6/nd6.c @@ -38,7 +38,6 @@ __FBSDID("$FreeBSD$"); #include <sys/param.h> #include <sys/systm.h> #include <sys/callout.h> -#include <sys/random.h> #include <sys/malloc.h> #include <sys/mbuf.h> #include <sys/socket.h> @@ -103,12 +102,8 @@ VNET_DEFINE(int, nd6_maxnudhint) = 0; /* max # of subsequent upper * layer hints */ static VNET_DEFINE(int, nd6_maxqueuelen) = 1; /* max pkts cached in unresolved * ND entries */ - -static VNET_DEFINE(int, nd6_on_link) = 1; /* Send unsolicited ND's on link up */ - #define V_nd6_maxndopt VNET(nd6_maxndopt) #define V_nd6_maxqueuelen VNET(nd6_maxqueuelen) -#define V_nd6_on_link VNET(nd6_on_link) #ifdef ND6_DEBUG VNET_DEFINE(int, nd6_debug) = 1; @@ -117,7 +112,6 @@ VNET_DEFINE(int, nd6_debug) = 0; #endif static eventhandler_tag lle_event_eh; -static eventhandler_tag ifnet_link_event_eh; /* for debugging? */ #if 0 @@ -202,14 +196,6 @@ nd6_lle_event(void *arg __unused, struct llentry *lle, int evt) type == RTM_ADD ? RTF_UP: 0), 0, RT_DEFAULT_FIB); } -static void -nd6_ifnet_link_event(void *arg __unused, struct ifnet *ifp, int linkstate) -{ - - if (linkstate == LINK_STATE_UP && V_nd6_on_link) - nd6_na_output_unsolicited(ifp); -} - void nd6_init(void) { @@ -225,12 +211,9 @@ nd6_init(void) nd6_slowtimo, curvnet); nd6_dad_init(); - if (IS_DEFAULT_VNET(curvnet)) { + if (IS_DEFAULT_VNET(curvnet)) lle_event_eh = EVENTHANDLER_REGISTER(lle_event, nd6_lle_event, NULL, EVENTHANDLER_PRI_ANY); - ifnet_link_event_eh = EVENTHANDLER_REGISTER(ifnet_link_event, - nd6_ifnet_link_event, NULL, EVENTHANDLER_PRI_ANY); - } } #ifdef VIMAGE @@ -240,10 +223,8 @@ nd6_destroy() callout_drain(&V_nd6_slowtimo_ch); callout_drain(&V_nd6_timer_ch); - if (IS_DEFAULT_VNET(curvnet)) { + if (IS_DEFAULT_VNET(curvnet)) EVENTHANDLER_DEREGISTER(lle_event, lle_event_eh); - EVENTHANDLER_DEREGISTER(ifnet_link_event, ifnet_link_event_eh); - } } #endif @@ -2476,18 +2457,13 @@ static int nd6_sysctl_prlist(SYSCTL_HANDLER_ARGS); SYSCTL_DECL(_net_inet6_icmp6); #endif SYSCTL_NODE(_net_inet6_icmp6, ICMPV6CTL_ND6_DRLIST, nd6_drlist, - CTLFLAG_RD, nd6_sysctl_drlist, "List default routers"); + CTLFLAG_RD, nd6_sysctl_drlist, ""); SYSCTL_NODE(_net_inet6_icmp6, ICMPV6CTL_ND6_PRLIST, nd6_prlist, - CTLFLAG_RD, nd6_sysctl_prlist, "List prefixes"); + CTLFLAG_RD, nd6_sysctl_prlist, ""); SYSCTL_INT(_net_inet6_icmp6, ICMPV6CTL_ND6_MAXQLEN, nd6_maxqueuelen, - CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(nd6_maxqueuelen), 1, - "Max packets cached in unresolved ND entries"); + CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(nd6_maxqueuelen), 1, ""); SYSCTL_INT(_net_inet6_icmp6, OID_AUTO, nd6_gctimer, - CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(nd6_gctimer), (60 * 60 * 24), - "Interface in seconds between garbage collection passes"); -SYSCTL_INT(_net_inet6_icmp6, OID_AUTO, nd6_on_link, CTLFLAG_VNET | CTLFLAG_RW, - &VNET_NAME(nd6_on_link), 0, - "Send unsolicited neighbor discovery on interface link up events"); + CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(nd6_gctimer), (60 * 60 * 24), ""); static int nd6_sysctl_drlist(SYSCTL_HANDLER_ARGS) diff --git a/sys/netinet6/nd6.h b/sys/netinet6/nd6.h index 710a1ad..8a0a56e 100644 --- a/sys/netinet6/nd6.h +++ b/sys/netinet6/nd6.h @@ -398,10 +398,6 @@ void nd6_init(void); #ifdef VIMAGE void nd6_destroy(void); #endif -void nd6_na_output_unsolicited(struct ifnet *); -void nd6_na_output_unsolicited_addr(struct ifnet *, const struct in6_addr *, - u_long); -int nd6_na_unsolicited_addr_delay(struct ifaddr *); struct nd_ifinfo *nd6_ifattach(struct ifnet *); void nd6_ifdetach(struct nd_ifinfo *); int nd6_is_addr_neighbor(const struct sockaddr_in6 *, struct ifnet *); diff --git a/sys/netinet6/nd6_nbr.c b/sys/netinet6/nd6_nbr.c index 07ad0cc..bf43fb6 100644 --- a/sys/netinet6/nd6_nbr.c +++ b/sys/netinet6/nd6_nbr.c @@ -124,16 +124,20 @@ nd6_ns_input(struct mbuf *m, int off, int icmp6len) struct in6_addr saddr6 = ip6->ip6_src; struct in6_addr daddr6 = ip6->ip6_dst; struct in6_addr taddr6; + struct in6_addr myaddr6; char *lladdr = NULL; struct ifaddr *ifa = NULL; - u_long flags; int lladdrlen = 0; - int proxy = 0; + int anycast = 0, proxy = 0, tentative = 0; int tlladdr; + int rflag; union nd_opts ndopts; struct sockaddr_dl proxydl; char ip6bufs[INET6_ADDRSTRLEN], ip6bufd[INET6_ADDRSTRLEN]; + rflag = (V_ip6_forwarding) ? ND_NA_FLAG_ROUTER : 0; + if (ND_IFINFO(ifp)->flags & ND6_IFF_ACCEPT_RTADV && V_ip6_norbit_raif) + rflag = 0; #ifndef PULLDOWN_TEST IP6_EXTHDR_CHECK(m, off, icmp6len,); nd_ns = (struct nd_neighbor_solicit *)((caddr_t)ip6 + off); @@ -225,7 +229,10 @@ nd6_ns_input(struct mbuf *m, int off, int icmp6len) * In implementation, we add target link-layer address by default. * We do not add one in MUST NOT cases. */ - tlladdr = !IN6_IS_ADDR_MULTICAST(&daddr6); + if (!IN6_IS_ADDR_MULTICAST(&daddr6)) + tlladdr = 0; + else + tlladdr = 1; /* * Target address (taddr6) must be either: @@ -282,6 +289,9 @@ nd6_ns_input(struct mbuf *m, int off, int icmp6len) */ goto freeit; } + myaddr6 = *IFA_IN6(ifa); + anycast = ((struct in6_ifaddr *)ifa)->ia6_flags & IN6_IFF_ANYCAST; + tentative = ((struct in6_ifaddr *)ifa)->ia6_flags & IN6_IFF_TENTATIVE; if (((struct in6_ifaddr *)ifa)->ia6_flags & IN6_IFF_DUPLICATED) goto freeit; @@ -293,7 +303,7 @@ nd6_ns_input(struct mbuf *m, int off, int icmp6len) goto bad; } - if (IN6_ARE_ADDR_EQUAL(IFA_IN6(ifa), &saddr6)) { + if (IN6_ARE_ADDR_EQUAL(&myaddr6, &saddr6)) { nd6log((LOG_INFO, "nd6_ns_input: duplicate IP6 address %s\n", ip6_sprintf(ip6bufs, &saddr6))); goto freeit; @@ -311,7 +321,7 @@ nd6_ns_input(struct mbuf *m, int off, int icmp6len) * * The processing is defined in RFC 2462. */ - if (IFA_IN6_FLAGS(ifa) & IN6_IFF_TENTATIVE) { + if (tentative) { /* * If source address is unspecified address, it is for * duplicate address detection. @@ -325,10 +335,6 @@ nd6_ns_input(struct mbuf *m, int off, int icmp6len) goto freeit; } - flags = IFA_ND6_NA_BASE_FLAGS(ifp, ifa); - if (proxy || !tlladdr) - flags &= ~ND_NA_FLAG_OVERRIDE; - /* * If the source address is unspecified address, entries must not * be created or updated. @@ -343,16 +349,20 @@ nd6_ns_input(struct mbuf *m, int off, int icmp6len) in6_all = in6addr_linklocal_allnodes; if (in6_setscope(&in6_all, ifp, NULL) != 0) goto bad; - nd6_na_output_fib(ifp, &in6_all, &taddr6, flags, tlladdr, - proxy ? (struct sockaddr *)&proxydl : NULL, M_GETFIB(m)); + nd6_na_output_fib(ifp, &in6_all, &taddr6, + ((anycast || proxy || !tlladdr) ? 0 : ND_NA_FLAG_OVERRIDE) | + rflag, tlladdr, proxy ? (struct sockaddr *)&proxydl : NULL, + M_GETFIB(m)); goto freeit; } nd6_cache_lladdr(ifp, &saddr6, lladdr, lladdrlen, ND_NEIGHBOR_SOLICIT, 0); - nd6_na_output_fib(ifp, &saddr6, &taddr6, flags | ND_NA_FLAG_SOLICITED, - tlladdr, proxy ? (struct sockaddr *)&proxydl : NULL, M_GETFIB(m)); + nd6_na_output_fib(ifp, &saddr6, &taddr6, + ((anycast || proxy || !tlladdr) ? 0 : ND_NA_FLAG_OVERRIDE) | + rflag | ND_NA_FLAG_SOLICITED, tlladdr, + proxy ? (struct sockaddr *)&proxydl : NULL, M_GETFIB(m)); freeit: if (ifa != NULL) ifa_free(ifa); @@ -1587,111 +1597,3 @@ nd6_dad_na_input(struct ifaddr *ifa) nd6_dad_rele(dp); } } - -/* - * Send unsolicited neighbor advertisements for all interface addresses to - * notify other nodes of changes. - * - * This is a noop if the interface isn't up. - */ -void __noinline -nd6_na_output_unsolicited(struct ifnet *ifp) -{ - int i, cnt, entries; - struct ifaddr *ifa; - struct ann { - struct in6_addr addr; - u_long flags; - int delay; - } *ann1, *head; - - if (!(ifp->if_flags & IFF_UP)) - return; - - entries = 8; - cnt = 0; - head = malloc(sizeof(struct ann) * entries, M_TEMP, M_WAITOK); - - /* Take a copy then process to avoid locking issues. */ - IF_ADDR_RLOCK(ifp); - TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { - if (ifa->ifa_addr->sa_family != AF_INET6 || - IFA_ND6_NA_UNSOLICITED_SKIP(ifa)) - continue; - - if (cnt == entries) { - ann1 = (struct ann*)realloc(head, sizeof(struct ann) * - (entries + 8), M_TEMP, M_NOWAIT); - if (ann1 == NULL) { - log(LOG_INFO, "nd6_announce: realloc to %d " - "entries failed\n", entries + 8); - /* Process what we have. */ - break; - } - entries += 8; - head = ann1; - } - - ann1 = head + cnt; - bcopy(IFA_IN6(ifa), &ann1->addr, sizeof(ann1->addr)); - ann1->flags = IFA_ND6_NA_BASE_FLAGS(ifp, ifa); - ann1->delay = nd6_na_unsolicited_addr_delay(ifa); - cnt++; - } - IF_ADDR_RUNLOCK(ifp); - - for (i = 0; i < cnt;) { - ann1 = head + i; - nd6_na_output_unsolicited_addr(ifp, &ann1->addr, ann1->flags); - i++; - if (i == cnt) - break; - /* XXX DELAY needs to be done in taskqueue to avoid stalling. */ - //DELAY(ann1->delay); - } - free(head, M_TEMP); -} - -/* - * Return the delay required for announcements of the address as per RFC 4861. - */ -int -nd6_na_unsolicited_addr_delay(struct ifaddr *ifa) -{ - - if (IFA_IN6_FLAGS(ifa) & IN6_IFF_ANYCAST) { - /* - * Random value between 0 and MAX_ANYCAST_DELAY_TIME - * as per section 7.2.7. - */ - return (random() % IN6_MAX_ANYCAST_DELAY_TIME_MS); - } - - /* Small delay as per section 7.2.6. */ - return (IN6_BROADCAST_DELAY_TIME_MS); -} - -/* - * Send an unsolicited neighbor advertisement for an address to notify other - * nodes of changes. - */ -void __noinline -nd6_na_output_unsolicited_addr(struct ifnet *ifp, const struct in6_addr *addr, - u_long flags) -{ - int error; - struct in6_addr mcast; - - mcast = in6addr_linklocal_allnodes; - if ((error = in6_setscope(&mcast, ifp, NULL)) != 0) { - /* - * This shouldn't by possible as the only error is for loopback - * address which we're not using. - */ - log(LOG_INFO, "in6_setscope: on mcast failed: %d\n", error); - return; - } - nd6_na_output(ifp, &mcast, addr, flags, 1, NULL); -} - - diff --git a/sys/powerpc/conf/GENERIC64 b/sys/powerpc/conf/GENERIC64 index def7012..847101f 100644 --- a/sys/powerpc/conf/GENERIC64 +++ b/sys/powerpc/conf/GENERIC64 @@ -69,6 +69,7 @@ options SYSVSHM #SYSV-style shared memory options SYSVMSG #SYSV-style message queues options SYSVSEM #SYSV-style semaphores options _KPOSIX_PRIORITY_SCHEDULING #Posix P1003_1B real-time extensions +options PRINTF_BUFR_SIZE=128 # Prevent printf output being interspersed. options HWPMC_HOOKS # Necessary kernel hooks for hwpmc(4) options AUDIT # Security event auditing options CAPABILITY_MODE # Capsicum capability mode diff --git a/sys/powerpc/pseries/phyp_llan.c b/sys/powerpc/pseries/phyp_llan.c index 04e359d..e35124c 100644 --- a/sys/powerpc/pseries/phyp_llan.c +++ b/sys/powerpc/pseries/phyp_llan.c @@ -87,6 +87,8 @@ struct llan_softc { cell_t unit; uint8_t mac_address[8]; + struct ifmedia media; + int irqid; struct resource *irq; void *irq_cookie; @@ -116,6 +118,8 @@ static void llan_intr(void *xsc); static void llan_init(void *xsc); static void llan_start(struct ifnet *ifp); static int llan_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data); +static void llan_media_status(struct ifnet *ifp, struct ifmediareq *ifmr); +static int llan_media_change(struct ifnet *ifp); static void llan_rx_load_cb(void *xsc, bus_dma_segment_t *segs, int nsegs, int err); static int llan_add_rxbuf(struct llan_softc *sc, struct llan_xfer *rx); @@ -220,16 +224,46 @@ llan_attach(device_t dev) sc->ifp->if_ioctl = llan_ioctl; sc->ifp->if_init = llan_init; + ifmedia_init(&sc->media, IFM_IMASK, llan_media_change, + llan_media_status); + ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); + ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO); + IFQ_SET_MAXLEN(&sc->ifp->if_snd, LLAN_MAX_TX_PACKETS); sc->ifp->if_snd.ifq_drv_maxlen = LLAN_MAX_TX_PACKETS; IFQ_SET_READY(&sc->ifp->if_snd); ether_ifattach(sc->ifp, &sc->mac_address[2]); + /* We don't have link state reporting, so make it always up */ + if_link_state_change(sc->ifp, LINK_STATE_UP); + + return (0); +} + +static int +llan_media_change(struct ifnet *ifp) +{ + struct llan_softc *sc = ifp->if_softc; + + if (IFM_TYPE(sc->media.ifm_media) != IFM_ETHER) + return (EINVAL); + + if (IFM_SUBTYPE(sc->media.ifm_media) != IFM_AUTO) + return (EINVAL); + return (0); } static void +llan_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) +{ + + ifmr->ifm_status = IFM_AVALID | IFM_ACTIVE | IFM_UNKNOWN | IFM_FDX; + ifmr->ifm_active = IFM_ETHER; +} + +static void llan_rx_load_cb(void *xsc, bus_dma_segment_t *segs, int nsegs, int err) { struct llan_softc *sc = xsc; @@ -501,6 +535,10 @@ llan_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) llan_set_multicast(sc); mtx_unlock(&sc->io_lock); break; + case SIOCGIFMEDIA: + case SIOCSIFMEDIA: + err = ifmedia_ioctl(ifp, (struct ifreq *)data, &sc->media, cmd); + break; case SIOCSIFFLAGS: default: err = ether_ioctl(ifp, cmd, data); diff --git a/sys/riscv/include/_align.h b/sys/riscv/include/_align.h new file mode 100644 index 0000000..3844133 --- /dev/null +++ b/sys/riscv/include/_align.h @@ -0,0 +1,44 @@ +/*- + * Copyright (c) 1990 The Regents of the University of California. + * All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * William Jolitz. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * from: @(#)param.h 5.8 (Berkeley) 6/28/91 + * $FreeBSD$ + */ + +#ifndef _MACHINE__ALIGN_H_ +#define _MACHINE__ALIGN_H_ + +/* + * Round p (pointer or byte index) up to a correctly-aligned value + * for all data types (int, long, ...). The result is unsigned int + * and must be cast to any desired pointer type. + */ +#define _ALIGNBYTES (sizeof(long long) - 1) +#define _ALIGN(p) (((u_long)(p) + _ALIGNBYTES) & ~_ALIGNBYTES) + +#endif /* !_MACHINE__ALIGN_H_ */ diff --git a/sys/riscv/include/_bus.h b/sys/riscv/include/_bus.h new file mode 100644 index 0000000..f11991f --- /dev/null +++ b/sys/riscv/include/_bus.h @@ -0,0 +1,46 @@ +/*- + * Copyright (c) 2005 M. Warner Losh. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions, and the following disclaimer, + * without modification, immediately at the beginning of the file. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _MACHINE__BUS_H_ +#define _MACHINE__BUS_H_ + +/* + * Addresses (in bus space). + */ +typedef u_long bus_addr_t; +typedef u_long bus_size_t; + +/* + * Access methods for bus space. + */ +typedef u_long bus_space_handle_t; +typedef struct bus_space *bus_space_tag_t; + +#endif /* !_MACHINE__BUS_H_ */ diff --git a/sys/riscv/include/_inttypes.h b/sys/riscv/include/_inttypes.h new file mode 100644 index 0000000..df1af0b --- /dev/null +++ b/sys/riscv/include/_inttypes.h @@ -0,0 +1,213 @@ +/*- + * Copyright (c) 2001 The NetBSD Foundation, Inc. + * All rights reserved. + * + * This code is derived from software contributed to The NetBSD Foundation + * by Klaus Klein. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * From: $NetBSD: int_fmtio.h,v 1.4 2008/04/28 20:23:36 martin Exp $ + * $FreeBSD$ + */ + +#ifndef _MACHINE__INTTYPES_H_ +#define _MACHINE__INTTYPES_H_ + +/* + * Macros for format specifiers. + */ + +/* fprintf(3) macros for signed integers. */ + +#define PRId8 "d" /* int8_t */ +#define PRId16 "d" /* int16_t */ +#define PRId32 "d" /* int32_t */ +#define PRId64 "ld" /* int64_t */ +#define PRIdLEAST8 "d" /* int_least8_t */ +#define PRIdLEAST16 "d" /* int_least16_t */ +#define PRIdLEAST32 "d" /* int_least32_t */ +#define PRIdLEAST64 "ld" /* int_least64_t */ +#define PRIdFAST8 "d" /* int_fast8_t */ +#define PRIdFAST16 "d" /* int_fast16_t */ +#define PRIdFAST32 "d" /* int_fast32_t */ +#define PRIdFAST64 "ld" /* int_fast64_t */ +#define PRIdMAX "jd" /* intmax_t */ +#define PRIdPTR "ld" /* intptr_t */ + +#define PRIi8 "i" /* int8_t */ +#define PRIi16 "i" /* int16_t */ +#define PRIi32 "i" /* int32_t */ +#define PRIi64 "li" /* int64_t */ +#define PRIiLEAST8 "i" /* int_least8_t */ +#define PRIiLEAST16 "i" /* int_least16_t */ +#define PRIiLEAST32 "i" /* int_least32_t */ +#define PRIiLEAST64 "li" /* int_least64_t */ +#define PRIiFAST8 "i" /* int_fast8_t */ +#define PRIiFAST16 "i" /* int_fast16_t */ +#define PRIiFAST32 "i" /* int_fast32_t */ +#define PRIiFAST64 "li" /* int_fast64_t */ +#define PRIiMAX "ji" /* intmax_t */ +#define PRIiPTR "li" /* intptr_t */ + +/* fprintf(3) macros for unsigned integers. */ + +#define PRIo8 "o" /* uint8_t */ +#define PRIo16 "o" /* uint16_t */ +#define PRIo32 "o" /* uint32_t */ +#define PRIo64 "lo" /* uint64_t */ +#define PRIoLEAST8 "o" /* uint_least8_t */ +#define PRIoLEAST16 "o" /* uint_least16_t */ +#define PRIoLEAST32 "o" /* uint_least32_t */ +#define PRIoLEAST64 "lo" /* uint_least64_t */ +#define PRIoFAST8 "o" /* uint_fast8_t */ +#define PRIoFAST16 "o" /* uint_fast16_t */ +#define PRIoFAST32 "o" /* uint_fast32_t */ +#define PRIoFAST64 "lo" /* uint_fast64_t */ +#define PRIoMAX "jo" /* uintmax_t */ +#define PRIoPTR "lo" /* uintptr_t */ + +#define PRIu8 "u" /* uint8_t */ +#define PRIu16 "u" /* uint16_t */ +#define PRIu32 "u" /* uint32_t */ +#define PRIu64 "lu" /* uint64_t */ +#define PRIuLEAST8 "u" /* uint_least8_t */ +#define PRIuLEAST16 "u" /* uint_least16_t */ +#define PRIuLEAST32 "u" /* uint_least32_t */ +#define PRIuLEAST64 "lu" /* uint_least64_t */ +#define PRIuFAST8 "u" /* uint_fast8_t */ +#define PRIuFAST16 "u" /* uint_fast16_t */ +#define PRIuFAST32 "u" /* uint_fast32_t */ +#define PRIuFAST64 "lu" /* uint_fast64_t */ +#define PRIuMAX "ju" /* uintmax_t */ +#define PRIuPTR "lu" /* uintptr_t */ + +#define PRIx8 "x" /* uint8_t */ +#define PRIx16 "x" /* uint16_t */ +#define PRIx32 "x" /* uint32_t */ +#define PRIx64 "lx" /* uint64_t */ +#define PRIxLEAST8 "x" /* uint_least8_t */ +#define PRIxLEAST16 "x" /* uint_least16_t */ +#define PRIxLEAST32 "x" /* uint_least32_t */ +#define PRIxLEAST64 "lx" /* uint_least64_t */ +#define PRIxFAST8 "x" /* uint_fast8_t */ +#define PRIxFAST16 "x" /* uint_fast16_t */ +#define PRIxFAST32 "x" /* uint_fast32_t */ +#define PRIxFAST64 "lx" /* uint_fast64_t */ +#define PRIxMAX "jx" /* uintmax_t */ +#define PRIxPTR "lx" /* uintptr_t */ + +#define PRIX8 "X" /* uint8_t */ +#define PRIX16 "X" /* uint16_t */ +#define PRIX32 "X" /* uint32_t */ +#define PRIX64 "lX" /* uint64_t */ +#define PRIXLEAST8 "X" /* uint_least8_t */ +#define PRIXLEAST16 "X" /* uint_least16_t */ +#define PRIXLEAST32 "X" /* uint_least32_t */ +#define PRIXLEAST64 "lX" /* uint_least64_t */ +#define PRIXFAST8 "X" /* uint_fast8_t */ +#define PRIXFAST16 "X" /* uint_fast16_t */ +#define PRIXFAST32 "X" /* uint_fast32_t */ +#define PRIXFAST64 "lX" /* uint_fast64_t */ +#define PRIXMAX "jX" /* uintmax_t */ +#define PRIXPTR "lX" /* uintptr_t */ + +/* fscanf(3) macros for signed integers. */ + +#define SCNd8 "hhd" /* int8_t */ +#define SCNd16 "hd" /* int16_t */ +#define SCNd32 "d" /* int32_t */ +#define SCNd64 "ld" /* int64_t */ +#define SCNdLEAST8 "hhd" /* int_least8_t */ +#define SCNdLEAST16 "hd" /* int_least16_t */ +#define SCNdLEAST32 "d" /* int_least32_t */ +#define SCNdLEAST64 "ld" /* int_least64_t */ +#define SCNdFAST8 "d" /* int_fast8_t */ +#define SCNdFAST16 "d" /* int_fast16_t */ +#define SCNdFAST32 "d" /* int_fast32_t */ +#define SCNdFAST64 "ld" /* int_fast64_t */ +#define SCNdMAX "jd" /* intmax_t */ +#define SCNdPTR "ld" /* intptr_t */ + +#define SCNi8 "hhi" /* int8_t */ +#define SCNi16 "hi" /* int16_t */ +#define SCNi32 "i" /* int32_t */ +#define SCNi64 "li" /* int64_t */ +#define SCNiLEAST8 "hhi" /* int_least8_t */ +#define SCNiLEAST16 "hi" /* int_least16_t */ +#define SCNiLEAST32 "i" /* int_least32_t */ +#define SCNiLEAST64 "li" /* int_least64_t */ +#define SCNiFAST8 "i" /* int_fast8_t */ +#define SCNiFAST16 "i" /* int_fast16_t */ +#define SCNiFAST32 "i" /* int_fast32_t */ +#define SCNiFAST64 "li" /* int_fast64_t */ +#define SCNiMAX "ji" /* intmax_t */ +#define SCNiPTR "li" /* intptr_t */ + +/* fscanf(3) macros for unsigned integers. */ + +#define SCNo8 "hho" /* uint8_t */ +#define SCNo16 "ho" /* uint16_t */ +#define SCNo32 "o" /* uint32_t */ +#define SCNo64 "lo" /* uint64_t */ +#define SCNoLEAST8 "hho" /* uint_least8_t */ +#define SCNoLEAST16 "ho" /* uint_least16_t */ +#define SCNoLEAST32 "o" /* uint_least32_t */ +#define SCNoLEAST64 "lo" /* uint_least64_t */ +#define SCNoFAST8 "o" /* uint_fast8_t */ +#define SCNoFAST16 "o" /* uint_fast16_t */ +#define SCNoFAST32 "o" /* uint_fast32_t */ +#define SCNoFAST64 "lo" /* uint_fast64_t */ +#define SCNoMAX "jo" /* uintmax_t */ +#define SCNoPTR "lo" /* uintptr_t */ + +#define SCNu8 "hhu" /* uint8_t */ +#define SCNu16 "hu" /* uint16_t */ +#define SCNu32 "u" /* uint32_t */ +#define SCNu64 "lu" /* uint64_t */ +#define SCNuLEAST8 "hhu" /* uint_least8_t */ +#define SCNuLEAST16 "hu" /* uint_least16_t */ +#define SCNuLEAST32 "u" /* uint_least32_t */ +#define SCNuLEAST64 "lu" /* uint_least64_t */ +#define SCNuFAST8 "u" /* uint_fast8_t */ +#define SCNuFAST16 "u" /* uint_fast16_t */ +#define SCNuFAST32 "u" /* uint_fast32_t */ +#define SCNuFAST64 "lu" /* uint_fast64_t */ +#define SCNuMAX "ju" /* uintmax_t */ +#define SCNuPTR "lu" /* uintptr_t */ + +#define SCNx8 "hhx" /* uint8_t */ +#define SCNx16 "hx" /* uint16_t */ +#define SCNx32 "x" /* uint32_t */ +#define SCNx64 "lx" /* uint64_t */ +#define SCNxLEAST8 "hhx" /* uint_least8_t */ +#define SCNxLEAST16 "hx" /* uint_least16_t */ +#define SCNxLEAST32 "x" /* uint_least32_t */ +#define SCNxLEAST64 "lx" /* uint_least64_t */ +#define SCNxFAST8 "x" /* uint_fast8_t */ +#define SCNxFAST16 "x" /* uint_fast16_t */ +#define SCNxFAST32 "x" /* uint_fast32_t */ +#define SCNxFAST64 "lx" /* uint_fast64_t */ +#define SCNxMAX "jx" /* uintmax_t */ +#define SCNxPTR "lx" /* uintptr_t */ + +#endif /* !_MACHINE__INTTYPES_H_ */ diff --git a/sys/riscv/include/_limits.h b/sys/riscv/include/_limits.h new file mode 100644 index 0000000..39f0bcf --- /dev/null +++ b/sys/riscv/include/_limits.h @@ -0,0 +1,85 @@ +/*- + * Copyright (c) 1988, 1993 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)limits.h 8.3 (Berkeley) 1/4/94 + * $FreeBSD$ + */ + +#ifndef _MACHINE__LIMITS_H_ +#define _MACHINE__LIMITS_H_ + +/* + * According to ANSI (section 2.2.4.2), the values below must be usable by + * #if preprocessing directives. Additionally, the expression must have the + * same type as would an expression that is an object of the corresponding + * type converted according to the integral promotions. The subtraction for + * INT_MIN, etc., is so the value is not unsigned; e.g., 0x80000000 is an + * unsigned int for 32-bit two's complement ANSI compilers (section 3.1.3.2). + */ + +#define __CHAR_BIT 8 /* number of bits in a char */ + +#define __SCHAR_MAX 0x7f /* max value for a signed char */ +#define __SCHAR_MIN (-0x7f - 1) /* min value for a signed char */ + +#define __UCHAR_MAX 0xff /* max value for an unsigned char */ + +#define __USHRT_MAX 0xffff /* max value for an unsigned short */ +#define __SHRT_MAX 0x7fff /* max value for a short */ +#define __SHRT_MIN (-0x7fff - 1) /* min value for a short */ + +#define __UINT_MAX 0xffffffff /* max value for an unsigned int */ +#define __INT_MAX 0x7fffffff /* max value for an int */ +#define __INT_MIN (-0x7fffffff - 1) /* min value for an int */ + +#define __ULONG_MAX 0xffffffffffffffffUL /* max for an unsigned long */ +#define __LONG_MAX 0x7fffffffffffffffL /* max for a long */ +#define __LONG_MIN (-0x7fffffffffffffffL - 1) /* min for a long */ + +/* Long longs have the same size but not the same type as longs. */ + /* max for an unsigned long long */ +#define __ULLONG_MAX 0xffffffffffffffffULL +#define __LLONG_MAX 0x7fffffffffffffffLL /* max for a long long */ +#define __LLONG_MIN (-0x7fffffffffffffffLL - 1) /* min for a long long */ + +#define __SSIZE_MAX __LONG_MAX /* max value for a ssize_t */ + +#define __SIZE_T_MAX __ULONG_MAX /* max value for a size_t */ + +#define __OFF_MAX __LONG_MAX /* max value for an off_t */ +#define __OFF_MIN __LONG_MIN /* min value for an off_t */ + +/* Quads and longs are the same size. Ensure they stay in sync. */ +#define __UQUAD_MAX (__ULONG_MAX) /* max value for a uquad_t */ +#define __QUAD_MAX (__LONG_MAX) /* max value for a quad_t */ +#define __QUAD_MIN (__LONG_MIN) /* min value for a quad_t */ + +#define __LONG_BIT 64 +#define __WORD_BIT 32 + +/* Minimum signal stack size. */ +#define __MINSIGSTKSZ (1024 * 4) + +#endif /* !_MACHINE__LIMITS_H_ */ diff --git a/sys/riscv/include/_stdint.h b/sys/riscv/include/_stdint.h new file mode 100644 index 0000000..32e5b6f --- /dev/null +++ b/sys/riscv/include/_stdint.h @@ -0,0 +1,158 @@ +/*- + * Copyright (c) 2001, 2002 Mike Barcroft <mike@FreeBSD.org> + * Copyright (c) 2001 The NetBSD Foundation, Inc. + * All rights reserved. + * + * This code is derived from software contributed to The NetBSD Foundation + * by Klaus Klein. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _MACHINE__STDINT_H_ +#define _MACHINE__STDINT_H_ + +#if !defined(__cplusplus) || defined(__STDC_CONSTANT_MACROS) + +#define INT8_C(c) (c) +#define INT16_C(c) (c) +#define INT32_C(c) (c) +#define INT64_C(c) (c ## L) + +#define UINT8_C(c) (c) +#define UINT16_C(c) (c) +#define UINT32_C(c) (c ## U) +#define UINT64_C(c) (c ## UL) + +#define INTMAX_C(c) INT64_C(c) +#define UINTMAX_C(c) UINT64_C(c) + +#endif /* !defined(__cplusplus) || defined(__STDC_CONSTANT_MACROS) */ + +#if !defined(__cplusplus) || defined(__STDC_LIMIT_MACROS) + +/* + * ISO/IEC 9899:1999 + * 7.18.2.1 Limits of exact-width integer types + */ +/* Minimum values of exact-width signed integer types. */ +#define INT8_MIN (-0x7f-1) +#define INT16_MIN (-0x7fff-1) +#define INT32_MIN (-0x7fffffff-1) +#define INT64_MIN (-0x7fffffffffffffffL-1) + +/* Maximum values of exact-width signed integer types. */ +#define INT8_MAX 0x7f +#define INT16_MAX 0x7fff +#define INT32_MAX 0x7fffffff +#define INT64_MAX 0x7fffffffffffffffL + +/* Maximum values of exact-width unsigned integer types. */ +#define UINT8_MAX 0xff +#define UINT16_MAX 0xffff +#define UINT32_MAX 0xffffffffU +#define UINT64_MAX 0xffffffffffffffffUL + +/* + * ISO/IEC 9899:1999 + * 7.18.2.2 Limits of minimum-width integer types + */ +/* Minimum values of minimum-width signed integer types. */ +#define INT_LEAST8_MIN INT8_MIN +#define INT_LEAST16_MIN INT16_MIN +#define INT_LEAST32_MIN INT32_MIN +#define INT_LEAST64_MIN INT64_MIN + +/* Maximum values of minimum-width signed integer types. */ +#define INT_LEAST8_MAX INT8_MAX +#define INT_LEAST16_MAX INT16_MAX +#define INT_LEAST32_MAX INT32_MAX +#define INT_LEAST64_MAX INT64_MAX + +/* Maximum values of minimum-width unsigned integer types. */ +#define UINT_LEAST8_MAX UINT8_MAX +#define UINT_LEAST16_MAX UINT16_MAX +#define UINT_LEAST32_MAX UINT32_MAX +#define UINT_LEAST64_MAX UINT64_MAX + +/* + * ISO/IEC 9899:1999 + * 7.18.2.3 Limits of fastest minimum-width integer types + */ +/* Minimum values of fastest minimum-width signed integer types. */ +#define INT_FAST8_MIN INT32_MIN +#define INT_FAST16_MIN INT32_MIN +#define INT_FAST32_MIN INT32_MIN +#define INT_FAST64_MIN INT64_MIN + +/* Maximum values of fastest minimum-width signed integer types. */ +#define INT_FAST8_MAX INT32_MAX +#define INT_FAST16_MAX INT32_MAX +#define INT_FAST32_MAX INT32_MAX +#define INT_FAST64_MAX INT64_MAX + +/* Maximum values of fastest minimum-width unsigned integer types. */ +#define UINT_FAST8_MAX UINT32_MAX +#define UINT_FAST16_MAX UINT32_MAX +#define UINT_FAST32_MAX UINT32_MAX +#define UINT_FAST64_MAX UINT64_MAX + +/* + * ISO/IEC 9899:1999 + * 7.18.2.4 Limits of integer types capable of holding object pointers + */ +#define INTPTR_MIN INT64_MIN +#define INTPTR_MAX INT64_MAX +#define UINTPTR_MAX UINT64_MAX + +/* + * ISO/IEC 9899:1999 + * 7.18.2.5 Limits of greatest-width integer types + */ +#define INTMAX_MIN INT64_MIN +#define INTMAX_MAX INT64_MAX +#define UINTMAX_MAX UINT64_MAX + +/* + * ISO/IEC 9899:1999 + * 7.18.3 Limits of other integer types + */ +/* Limits of ptrdiff_t. */ +#define PTRDIFF_MIN INT64_MIN +#define PTRDIFF_MAX INT64_MAX + +/* Limits of sig_atomic_t. */ +#define SIG_ATOMIC_MIN INT32_MIN +#define SIG_ATOMIC_MAX INT32_MAX + +/* Limit of size_t. */ +#define SIZE_MAX UINT64_MAX + +/* Limits of wint_t. */ +#define WINT_MIN INT32_MIN +#define WINT_MAX INT32_MAX + +#endif /* !defined(__cplusplus) || defined(__STDC_LIMIT_MACROS) */ + +#endif /* !_MACHINE__STDINT_H_ */ diff --git a/sys/riscv/include/_types.h b/sys/riscv/include/_types.h new file mode 100644 index 0000000..733d47c --- /dev/null +++ b/sys/riscv/include/_types.h @@ -0,0 +1,114 @@ +/*- + * Copyright (c) 2002 Mike Barcroft <mike@FreeBSD.org> + * Copyright (c) 1990, 1993 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * From: @(#)ansi.h 8.2 (Berkeley) 1/4/94 + * From: @(#)types.h 8.3 (Berkeley) 1/5/94 + * $FreeBSD$ + */ + +#ifndef _MACHINE__TYPES_H_ +#define _MACHINE__TYPES_H_ + +#ifndef _SYS_CDEFS_H_ +#error this file needs sys/cdefs.h as a prerequisite +#endif + +/* + * Basic types upon which most other types are built. + */ +typedef signed char __int8_t; +typedef unsigned char __uint8_t; +typedef short __int16_t; +typedef unsigned short __uint16_t; +typedef int __int32_t; +typedef unsigned int __uint32_t; +typedef long __int64_t; +typedef unsigned long __uint64_t; + +/* + * Standard type definitions. + */ +typedef __int32_t __clock_t; /* clock()... */ +typedef __int64_t __critical_t; +typedef double __double_t; +typedef float __float_t; +typedef __int64_t __intfptr_t; +typedef __int64_t __intmax_t; +typedef __int64_t __intptr_t; +typedef __int32_t __int_fast8_t; +typedef __int32_t __int_fast16_t; +typedef __int32_t __int_fast32_t; +typedef __int64_t __int_fast64_t; +typedef __int8_t __int_least8_t; +typedef __int16_t __int_least16_t; +typedef __int32_t __int_least32_t; +typedef __int64_t __int_least64_t; +typedef __int64_t __ptrdiff_t; /* ptr1 - ptr2 */ +typedef __int64_t __register_t; +typedef __int64_t __segsz_t; /* segment size (in pages) */ +typedef __uint64_t __size_t; /* sizeof() */ +typedef __int64_t __ssize_t; /* byte count or error */ +typedef __int64_t __time_t; /* time()... */ +typedef __uint64_t __uintfptr_t; +typedef __uint64_t __uintmax_t; +typedef __uint64_t __uintptr_t; +typedef __uint32_t __uint_fast8_t; +typedef __uint32_t __uint_fast16_t; +typedef __uint32_t __uint_fast32_t; +typedef __uint64_t __uint_fast64_t; +typedef __uint8_t __uint_least8_t; +typedef __uint16_t __uint_least16_t; +typedef __uint32_t __uint_least32_t; +typedef __uint64_t __uint_least64_t; +typedef __uint64_t __u_register_t; +typedef __uint64_t __vm_offset_t; +typedef __int64_t __vm_ooffset_t; +typedef __uint64_t __vm_paddr_t; +typedef __uint64_t __vm_pindex_t; +typedef __uint64_t __vm_size_t; +typedef int ___wchar_t; + +#define __WCHAR_MIN __INT_MIN /* min value for a wchar_t */ +#define __WCHAR_MAX __INT_MAX /* max value for a wchar_t */ + +/* + * Unusual type definitions. + */ +#ifdef __GNUCLIKE_BUILTIN_VARARGS +typedef __builtin_va_list __va_list; /* internally known to gcc */ +#else +typedef char * __va_list; +#endif /* __GNUCLIKE_BUILTIN_VARARGS */ +#if defined(__GNUCLIKE_BUILTIN_VAALIST) && !defined(__GNUC_VA_LIST) \ + && !defined(__NO_GNUC_VA_LIST) +#define __GNUC_VA_LIST +typedef __va_list __gnuc_va_list; /* compatibility w/GNU headers*/ +#endif + +#endif /* !_MACHINE__TYPES_H_ */ diff --git a/sys/riscv/include/asm.h b/sys/riscv/include/asm.h new file mode 100644 index 0000000..fb0c844 --- /dev/null +++ b/sys/riscv/include/asm.h @@ -0,0 +1,68 @@ +/*- + * Copyright (c) 2015 Ruslan Bukin <br@bsdpad.com> + * All rights reserved. + * + * Portions of this software were developed by SRI International and the + * University of Cambridge Computer Laboratory under DARPA/AFRL contract + * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. + * + * Portions of this software were developed by the University of Cambridge + * Computer Laboratory as part of the CTSRD Project, with support from the + * UK Higher Education Innovation Fund (HEIF). + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _MACHINE_ASM_H_ +#define _MACHINE_ASM_H_ + +#undef __FBSDID +#if !defined(lint) && !defined(STRIP_FBSDID) +#define __FBSDID(s) .ident s +#else +#define __FBSDID(s) /* nothing */ +#endif /* not lint and not STRIP_FBSDID */ + +#define _C_LABEL(x) x + +#define ENTRY(sym) \ + .text; .globl sym; .type sym,@function; .align 2; sym: +#define END(sym) .size sym, . - sym + +#define EENTRY(sym) \ + .globl sym; sym: +#define EEND(sym) + +#define WEAK_REFERENCE(sym, alias) \ + .weak alias; \ + .set alias,sym + +#define SET_FAULT_HANDLER(handler, tmp) \ + la tmp, pcpup; \ + ld tmp, 0(tmp); \ + ld tmp, PC_CURTHREAD(tmp); \ + ld tmp, TD_PCB(tmp); /* Load the pcb */ \ + sd handler, PCB_ONFAULT(tmp) /* Set the handler */ + +#endif /* _MACHINE_ASM_H_ */ diff --git a/sys/riscv/include/atomic.h b/sys/riscv/include/atomic.h new file mode 100644 index 0000000..f86570a --- /dev/null +++ b/sys/riscv/include/atomic.h @@ -0,0 +1,452 @@ +/*- + * Copyright (c) 2015 Ruslan Bukin <br@bsdpad.com> + * All rights reserved. + * + * Portions of this software were developed by SRI International and the + * University of Cambridge Computer Laboratory under DARPA/AFRL contract + * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. + * + * Portions of this software were developed by the University of Cambridge + * Computer Laboratory as part of the CTSRD Project, with support from the + * UK Higher Education Innovation Fund (HEIF). + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _MACHINE_ATOMIC_H_ +#define _MACHINE_ATOMIC_H_ + +#define fence() __asm __volatile("fence" ::: "memory"); +#define mb() fence() +#define rmb() fence() +#define wmb() fence() + +#define ATOMIC_ACQ_REL(NAME, WIDTH) \ +static __inline void \ +atomic_##NAME##_acq_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\ +{ \ + atomic_##NAME##_##WIDTH(p, v); \ + fence(); \ +} \ + \ +static __inline void \ +atomic_##NAME##_rel_##WIDTH(__volatile uint##WIDTH##_t *p, uint##WIDTH##_t v)\ +{ \ + fence(); \ + atomic_##NAME##_##WIDTH(p, v); \ +} + +static __inline void +atomic_add_32(volatile uint32_t *p, uint32_t val) +{ + + __asm __volatile("amoadd.w zero, %1, %0" + : "+A" (*p) + : "r" (val) + : "memory"); +} + +static __inline void +atomic_subtract_32(volatile uint32_t *p, uint32_t val) +{ + + __asm __volatile("amoadd.w zero, %1, %0" + : "+A" (*p) + : "r" (-val) + : "memory"); +} + +static __inline void +atomic_set_32(volatile uint32_t *p, uint32_t val) +{ + + __asm __volatile("amoor.w zero, %1, %0" + : "+A" (*p) + : "r" (val) + : "memory"); +} + +static __inline void +atomic_clear_32(volatile uint32_t *p, uint32_t val) +{ + + __asm __volatile("amoand.w zero, %1, %0" + : "+A" (*p) + : "r" (~val) + : "memory"); +} + +static __inline int +atomic_cmpset_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval) +{ + uint32_t tmp; + int res; + + res = 0; + + __asm __volatile( + "0:" + "li %1, 1\n" /* Preset to fail */ + "lr.w %0, %2\n" + "bne %0, %z3, 1f\n" + "sc.w %1, %z4, %2\n" + "bnez %1, 0b\n" + "1:" + : "=&r" (tmp), "=&r" (res), "+A" (*p) + : "rJ" (cmpval), "rJ" (newval) + : "memory"); + + return (!res); +} + +static __inline uint32_t +atomic_fetchadd_32(volatile uint32_t *p, uint32_t val) +{ + uint32_t ret; + + __asm __volatile("amoadd.w %0, %2, %1" + : "=&r" (ret), "+A" (*p) + : "r" (val) + : "memory"); + + return (ret); +} + +static __inline uint32_t +atomic_readandclear_32(volatile uint32_t *p) +{ + uint32_t ret; + uint32_t val; + + val = 0; + + __asm __volatile("amoswap.w %0, %2, %1" + : "=&r"(ret), "+A" (*p) + : "r" (val) + : "memory"); + + return (ret); +} + +#define atomic_add_int atomic_add_32 +#define atomic_clear_int atomic_clear_32 +#define atomic_cmpset_int atomic_cmpset_32 +#define atomic_fetchadd_int atomic_fetchadd_32 +#define atomic_readandclear_int atomic_readandclear_32 +#define atomic_set_int atomic_set_32 +#define atomic_subtract_int atomic_subtract_32 + +ATOMIC_ACQ_REL(set, 32) +ATOMIC_ACQ_REL(clear, 32) +ATOMIC_ACQ_REL(add, 32) +ATOMIC_ACQ_REL(subtract, 32) + +static __inline int +atomic_cmpset_acq_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval) +{ + int res; + + res = atomic_cmpset_32(p, cmpval, newval); + + fence(); + + return (res); +} + +static __inline int +atomic_cmpset_rel_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval) +{ + + fence(); + + return (atomic_cmpset_32(p, cmpval, newval)); +} + +static __inline uint32_t +atomic_load_acq_32(volatile uint32_t *p) +{ + uint32_t ret; + + ret = *p; + + fence(); + + return (ret); +} + +static __inline void +atomic_store_rel_32(volatile uint32_t *p, uint32_t val) +{ + + fence(); + + *p = val; +} + +#define atomic_add_acq_int atomic_add_acq_32 +#define atomic_clear_acq_int atomic_clear_acq_32 +#define atomic_cmpset_acq_int atomic_cmpset_acq_32 +#define atomic_load_acq_int atomic_load_acq_32 +#define atomic_set_acq_int atomic_set_acq_32 +#define atomic_subtract_acq_int atomic_subtract_acq_32 + +#define atomic_add_rel_int atomic_add_rel_32 +#define atomic_clear_rel_int atomic_add_rel_32 +#define atomic_cmpset_rel_int atomic_cmpset_rel_32 +#define atomic_set_rel_int atomic_set_rel_32 +#define atomic_subtract_rel_int atomic_subtract_rel_32 +#define atomic_store_rel_int atomic_store_rel_32 + +static __inline void +atomic_add_64(volatile uint64_t *p, uint64_t val) +{ + + __asm __volatile("amoadd.d zero, %1, %0" + : "+A" (*p) + : "r" (val) + : "memory"); +} + +static __inline void +atomic_subtract_64(volatile uint64_t *p, uint64_t val) +{ + + __asm __volatile("amoadd.d zero, %1, %0" + : "+A" (*p) + : "r" (-val) + : "memory"); +} + +static __inline void +atomic_set_64(volatile uint64_t *p, uint64_t val) +{ + + __asm __volatile("amoor.d zero, %1, %0" + : "+A" (*p) + : "r" (val) + : "memory"); +} + +static __inline void +atomic_clear_64(volatile uint64_t *p, uint64_t val) +{ + + __asm __volatile("amoand.d zero, %1, %0" + : "+A" (*p) + : "r" (~val) + : "memory"); +} + +static __inline int +atomic_cmpset_64(volatile uint64_t *p, uint64_t cmpval, uint64_t newval) +{ + uint64_t tmp; + int res; + + res = 0; + + __asm __volatile( + "0:" + "li %1, 1\n" /* Preset to fail */ + "lr.d %0, %2\n" + "bne %0, %z3, 1f\n" + "sc.d %1, %z4, %2\n" + "bnez %1, 0b\n" + "1:" + : "=&r" (tmp), "=&r" (res), "+A" (*p) + : "rJ" (cmpval), "rJ" (newval) + : "memory"); + + return (!res); +} + +static __inline uint64_t +atomic_fetchadd_64(volatile uint64_t *p, uint64_t val) +{ + uint64_t ret; + + __asm __volatile("amoadd.d %0, %2, %1" + : "=&r" (ret), "+A" (*p) + : "r" (val) + : "memory"); + + return (ret); +} + +static __inline uint64_t +atomic_readandclear_64(volatile uint64_t *p) +{ + uint64_t ret; + uint64_t val; + + val = 0; + + __asm __volatile("amoswap.d %0, %2, %1" + : "=&r"(ret), "+A" (*p) + : "r" (val) + : "memory"); + + return (ret); +} + +static __inline uint64_t +atomic_swap_64(volatile uint64_t *p, uint64_t val) +{ + uint64_t old; + + __asm __volatile("amoswap.d %0, %2, %1" + : "=&r"(old), "+A" (*p) + : "r" (val) + : "memory"); + + return (old); +} + +#define atomic_add_long atomic_add_64 +#define atomic_clear_long atomic_clear_64 +#define atomic_cmpset_long atomic_cmpset_64 +#define atomic_fetchadd_long atomic_fetchadd_64 +#define atomic_readandclear_long atomic_readandclear_64 +#define atomic_set_long atomic_set_64 +#define atomic_subtract_long atomic_subtract_64 + +#define atomic_add_ptr atomic_add_64 +#define atomic_clear_ptr atomic_clear_64 +#define atomic_cmpset_ptr atomic_cmpset_64 +#define atomic_fetchadd_ptr atomic_fetchadd_64 +#define atomic_readandclear_ptr atomic_readandclear_64 +#define atomic_set_ptr atomic_set_64 +#define atomic_subtract_ptr atomic_subtract_64 + +ATOMIC_ACQ_REL(set, 64) +ATOMIC_ACQ_REL(clear, 64) +ATOMIC_ACQ_REL(add, 64) +ATOMIC_ACQ_REL(subtract, 64) + +static __inline int +atomic_cmpset_acq_64(volatile uint64_t *p, uint64_t cmpval, uint64_t newval) +{ + int res; + + res = atomic_cmpset_64(p, cmpval, newval); + + fence(); + + return (res); +} + +static __inline int +atomic_cmpset_rel_64(volatile uint64_t *p, uint64_t cmpval, uint64_t newval) +{ + + fence(); + + return (atomic_cmpset_64(p, cmpval, newval)); +} + +static __inline uint64_t +atomic_load_acq_64(volatile uint64_t *p) +{ + uint64_t ret; + + ret = *p; + + fence(); + + return (ret); +} + +static __inline void +atomic_store_rel_64(volatile uint64_t *p, uint64_t val) +{ + + fence(); + + *p = val; +} + +#define atomic_add_acq_int atomic_add_acq_32 +#define atomic_clear_acq_int atomic_clear_acq_32 +#define atomic_cmpset_acq_int atomic_cmpset_acq_32 + +#define atomic_add_acq_long atomic_add_acq_64 +#define atomic_clear_acq_long atomic_add_acq_64 +#define atomic_cmpset_acq_long atomic_cmpset_acq_64 +#define atomic_load_acq_long atomic_load_acq_64 +#define atomic_set_acq_long atomic_set_acq_64 +#define atomic_subtract_acq_long atomic_subtract_acq_64 + +#define atomic_add_acq_ptr atomic_add_acq_64 +#define atomic_clear_acq_ptr atomic_add_acq_64 +#define atomic_cmpset_acq_ptr atomic_cmpset_acq_64 +#define atomic_load_acq_ptr atomic_load_acq_64 +#define atomic_set_acq_ptr atomic_set_acq_64 +#define atomic_subtract_acq_ptr atomic_subtract_acq_64 + +static __inline void +atomic_thread_fence_acq(void) +{ + + fence(); +} + +static __inline void +atomic_thread_fence_rel(void) +{ + + fence(); +} + +static __inline void +atomic_thread_fence_acq_rel(void) +{ + + fence(); +} + +static __inline void +atomic_thread_fence_seq_cst(void) +{ + + fence(); +} + +#define atomic_add_rel_long atomic_add_rel_64 +#define atomic_clear_rel_long atomic_clear_rel_64 + +#define atomic_add_rel_long atomic_add_rel_64 +#define atomic_clear_rel_long atomic_clear_rel_64 +#define atomic_cmpset_rel_long atomic_cmpset_rel_64 +#define atomic_set_rel_long atomic_set_rel_64 +#define atomic_subtract_rel_long atomic_subtract_rel_64 +#define atomic_store_rel_long atomic_store_rel_64 + +#define atomic_add_rel_ptr atomic_add_rel_64 +#define atomic_clear_rel_ptr atomic_clear_rel_64 +#define atomic_cmpset_rel_ptr atomic_cmpset_rel_64 +#define atomic_set_rel_ptr atomic_set_rel_64 +#define atomic_subtract_rel_ptr atomic_subtract_rel_64 +#define atomic_store_rel_ptr atomic_store_rel_64 + +#endif /* _MACHINE_ATOMIC_H_ */ diff --git a/sys/riscv/include/bus.h b/sys/riscv/include/bus.h new file mode 100644 index 0000000..8aaf1d3 --- /dev/null +++ b/sys/riscv/include/bus.h @@ -0,0 +1,469 @@ +/* $NetBSD: bus.h,v 1.11 2003/07/28 17:35:54 thorpej Exp $ */ + +/*- + * Copyright (c) 1996, 1997, 1998, 2001 The NetBSD Foundation, Inc. + * All rights reserved. + * + * This code is derived from software contributed to The NetBSD Foundation + * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, + * NASA Ames Research Center. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/*- + * Copyright (c) 1996 Charles M. Hannum. All rights reserved. + * Copyright (c) 1996 Christopher G. Demetriou. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by Christopher G. Demetriou + * for the NetBSD Project. + * 4. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * From: sys/arm/include/bus.h + * + * $FreeBSD$ + */ + +#ifndef _MACHINE_BUS_H_ +#define _MACHINE_BUS_H_ + +#include <machine/_bus.h> + +#define BUS_SPACE_ALIGNED_POINTER(p, t) ALIGNED_POINTER(p, t) + +#define BUS_SPACE_MAXADDR_24BIT 0xFFFFFFUL +#define BUS_SPACE_MAXADDR_32BIT 0xFFFFFFFFUL +#define BUS_SPACE_MAXSIZE_24BIT 0xFFFFFFUL +#define BUS_SPACE_MAXSIZE_32BIT 0xFFFFFFFFUL + +#define BUS_SPACE_MAXADDR 0xFFFFFFFFFFFFFFFFUL +#define BUS_SPACE_MAXSIZE 0xFFFFFFFFFFFFFFFFUL + +#define BUS_SPACE_MAP_CACHEABLE 0x01 +#define BUS_SPACE_MAP_LINEAR 0x02 +#define BUS_SPACE_MAP_PREFETCHABLE 0x04 + +#define BUS_SPACE_UNRESTRICTED (~0) + +#define BUS_SPACE_BARRIER_READ 0x01 +#define BUS_SPACE_BARRIER_WRITE 0x02 + + +struct bus_space { + /* cookie */ + void *bs_cookie; + + /* mapping/unmapping */ + int (*bs_map) (void *, bus_addr_t, bus_size_t, + int, bus_space_handle_t *); + void (*bs_unmap) (void *, bus_space_handle_t, bus_size_t); + int (*bs_subregion) (void *, bus_space_handle_t, + bus_size_t, bus_size_t, bus_space_handle_t *); + + /* allocation/deallocation */ + int (*bs_alloc) (void *, bus_addr_t, bus_addr_t, + bus_size_t, bus_size_t, bus_size_t, int, + bus_addr_t *, bus_space_handle_t *); + void (*bs_free) (void *, bus_space_handle_t, + bus_size_t); + + /* get kernel virtual address */ + /* barrier */ + void (*bs_barrier) (void *, bus_space_handle_t, + bus_size_t, bus_size_t, int); + + /* read single */ + u_int8_t (*bs_r_1) (void *, bus_space_handle_t, bus_size_t); + u_int16_t (*bs_r_2) (void *, bus_space_handle_t, bus_size_t); + u_int32_t (*bs_r_4) (void *, bus_space_handle_t, bus_size_t); + u_int64_t (*bs_r_8) (void *, bus_space_handle_t, bus_size_t); + + /* read multiple */ + void (*bs_rm_1) (void *, bus_space_handle_t, bus_size_t, + u_int8_t *, bus_size_t); + void (*bs_rm_2) (void *, bus_space_handle_t, bus_size_t, + u_int16_t *, bus_size_t); + void (*bs_rm_4) (void *, bus_space_handle_t, + bus_size_t, u_int32_t *, bus_size_t); + void (*bs_rm_8) (void *, bus_space_handle_t, + bus_size_t, u_int64_t *, bus_size_t); + + /* read region */ + void (*bs_rr_1) (void *, bus_space_handle_t, + bus_size_t, u_int8_t *, bus_size_t); + void (*bs_rr_2) (void *, bus_space_handle_t, + bus_size_t, u_int16_t *, bus_size_t); + void (*bs_rr_4) (void *, bus_space_handle_t, + bus_size_t, u_int32_t *, bus_size_t); + void (*bs_rr_8) (void *, bus_space_handle_t, + bus_size_t, u_int64_t *, bus_size_t); + + /* write single */ + void (*bs_w_1) (void *, bus_space_handle_t, + bus_size_t, u_int8_t); + void (*bs_w_2) (void *, bus_space_handle_t, + bus_size_t, u_int16_t); + void (*bs_w_4) (void *, bus_space_handle_t, + bus_size_t, u_int32_t); + void (*bs_w_8) (void *, bus_space_handle_t, + bus_size_t, u_int64_t); + + /* write multiple */ + void (*bs_wm_1) (void *, bus_space_handle_t, + bus_size_t, const u_int8_t *, bus_size_t); + void (*bs_wm_2) (void *, bus_space_handle_t, + bus_size_t, const u_int16_t *, bus_size_t); + void (*bs_wm_4) (void *, bus_space_handle_t, + bus_size_t, const u_int32_t *, bus_size_t); + void (*bs_wm_8) (void *, bus_space_handle_t, + bus_size_t, const u_int64_t *, bus_size_t); + + /* write region */ + void (*bs_wr_1) (void *, bus_space_handle_t, + bus_size_t, const u_int8_t *, bus_size_t); + void (*bs_wr_2) (void *, bus_space_handle_t, + bus_size_t, const u_int16_t *, bus_size_t); + void (*bs_wr_4) (void *, bus_space_handle_t, + bus_size_t, const u_int32_t *, bus_size_t); + void (*bs_wr_8) (void *, bus_space_handle_t, + bus_size_t, const u_int64_t *, bus_size_t); + + /* set multiple */ + void (*bs_sm_1) (void *, bus_space_handle_t, + bus_size_t, u_int8_t, bus_size_t); + void (*bs_sm_2) (void *, bus_space_handle_t, + bus_size_t, u_int16_t, bus_size_t); + void (*bs_sm_4) (void *, bus_space_handle_t, + bus_size_t, u_int32_t, bus_size_t); + void (*bs_sm_8) (void *, bus_space_handle_t, + bus_size_t, u_int64_t, bus_size_t); + + /* set region */ + void (*bs_sr_1) (void *, bus_space_handle_t, + bus_size_t, u_int8_t, bus_size_t); + void (*bs_sr_2) (void *, bus_space_handle_t, + bus_size_t, u_int16_t, bus_size_t); + void (*bs_sr_4) (void *, bus_space_handle_t, + bus_size_t, u_int32_t, bus_size_t); + void (*bs_sr_8) (void *, bus_space_handle_t, + bus_size_t, u_int64_t, bus_size_t); + + /* copy */ + void (*bs_c_1) (void *, bus_space_handle_t, bus_size_t, + bus_space_handle_t, bus_size_t, bus_size_t); + void (*bs_c_2) (void *, bus_space_handle_t, bus_size_t, + bus_space_handle_t, bus_size_t, bus_size_t); + void (*bs_c_4) (void *, bus_space_handle_t, bus_size_t, + bus_space_handle_t, bus_size_t, bus_size_t); + void (*bs_c_8) (void *, bus_space_handle_t, bus_size_t, + bus_space_handle_t, bus_size_t, bus_size_t); + + /* read single stream */ + u_int8_t (*bs_r_1_s) (void *, bus_space_handle_t, bus_size_t); + u_int16_t (*bs_r_2_s) (void *, bus_space_handle_t, bus_size_t); + u_int32_t (*bs_r_4_s) (void *, bus_space_handle_t, bus_size_t); + u_int64_t (*bs_r_8_s) (void *, bus_space_handle_t, bus_size_t); + + /* read multiple stream */ + void (*bs_rm_1_s) (void *, bus_space_handle_t, bus_size_t, + u_int8_t *, bus_size_t); + void (*bs_rm_2_s) (void *, bus_space_handle_t, bus_size_t, + u_int16_t *, bus_size_t); + void (*bs_rm_4_s) (void *, bus_space_handle_t, + bus_size_t, u_int32_t *, bus_size_t); + void (*bs_rm_8_s) (void *, bus_space_handle_t, + bus_size_t, u_int64_t *, bus_size_t); + + /* read region stream */ + void (*bs_rr_1_s) (void *, bus_space_handle_t, + bus_size_t, u_int8_t *, bus_size_t); + void (*bs_rr_2_s) (void *, bus_space_handle_t, + bus_size_t, u_int16_t *, bus_size_t); + void (*bs_rr_4_s) (void *, bus_space_handle_t, + bus_size_t, u_int32_t *, bus_size_t); + void (*bs_rr_8_s) (void *, bus_space_handle_t, + bus_size_t, u_int64_t *, bus_size_t); + + /* write single stream */ + void (*bs_w_1_s) (void *, bus_space_handle_t, + bus_size_t, u_int8_t); + void (*bs_w_2_s) (void *, bus_space_handle_t, + bus_size_t, u_int16_t); + void (*bs_w_4_s) (void *, bus_space_handle_t, + bus_size_t, u_int32_t); + void (*bs_w_8_s) (void *, bus_space_handle_t, + bus_size_t, u_int64_t); + + /* write multiple stream */ + void (*bs_wm_1_s) (void *, bus_space_handle_t, + bus_size_t, const u_int8_t *, bus_size_t); + void (*bs_wm_2_s) (void *, bus_space_handle_t, + bus_size_t, const u_int16_t *, bus_size_t); + void (*bs_wm_4_s) (void *, bus_space_handle_t, + bus_size_t, const u_int32_t *, bus_size_t); + void (*bs_wm_8_s) (void *, bus_space_handle_t, + bus_size_t, const u_int64_t *, bus_size_t); + + /* write region stream */ + void (*bs_wr_1_s) (void *, bus_space_handle_t, + bus_size_t, const u_int8_t *, bus_size_t); + void (*bs_wr_2_s) (void *, bus_space_handle_t, + bus_size_t, const u_int16_t *, bus_size_t); + void (*bs_wr_4_s) (void *, bus_space_handle_t, + bus_size_t, const u_int32_t *, bus_size_t); + void (*bs_wr_8_s) (void *, bus_space_handle_t, + bus_size_t, const u_int64_t *, bus_size_t); +}; + + +/* + * Utility macros; INTERNAL USE ONLY. + */ +#define __bs_c(a,b) __CONCAT(a,b) +#define __bs_opname(op,size) __bs_c(__bs_c(__bs_c(bs_,op),_),size) + +#define __bs_rs(sz, t, h, o) \ + (*(t)->__bs_opname(r,sz))((t)->bs_cookie, h, o) +#define __bs_ws(sz, t, h, o, v) \ + (*(t)->__bs_opname(w,sz))((t)->bs_cookie, h, o, v) +#define __bs_nonsingle(type, sz, t, h, o, a, c) \ + (*(t)->__bs_opname(type,sz))((t)->bs_cookie, h, o, a, c) +#define __bs_set(type, sz, t, h, o, v, c) \ + (*(t)->__bs_opname(type,sz))((t)->bs_cookie, h, o, v, c) +#define __bs_copy(sz, t, h1, o1, h2, o2, cnt) \ + (*(t)->__bs_opname(c,sz))((t)->bs_cookie, h1, o1, h2, o2, cnt) + +#define __bs_opname_s(op,size) __bs_c(__bs_c(__bs_c(__bs_c(bs_,op),_),size),_s) +#define __bs_rs_s(sz, t, h, o) \ + (*(t)->__bs_opname_s(r,sz))((t)->bs_cookie, h, o) +#define __bs_ws_s(sz, t, h, o, v) \ + (*(t)->__bs_opname_s(w,sz))((t)->bs_cookie, h, o, v) +#define __bs_nonsingle_s(type, sz, t, h, o, a, c) \ + (*(t)->__bs_opname_s(type,sz))((t)->bs_cookie, h, o, a, c) + + +/* + * Mapping and unmapping operations. + */ +#define bus_space_map(t, a, s, c, hp) \ + (*(t)->bs_map)((t)->bs_cookie, (a), (s), (c), (hp)) +#define bus_space_unmap(t, h, s) \ + (*(t)->bs_unmap)((t)->bs_cookie, (h), (s)) +#define bus_space_subregion(t, h, o, s, hp) \ + (*(t)->bs_subregion)((t)->bs_cookie, (h), (o), (s), (hp)) + + +/* + * Allocation and deallocation operations. + */ +#define bus_space_alloc(t, rs, re, s, a, b, c, ap, hp) \ + (*(t)->bs_alloc)((t)->bs_cookie, (rs), (re), (s), (a), (b), \ + (c), (ap), (hp)) +#define bus_space_free(t, h, s) \ + (*(t)->bs_free)((t)->bs_cookie, (h), (s)) + +/* + * Bus barrier operations. + */ +#define bus_space_barrier(t, h, o, l, f) \ + (*(t)->bs_barrier)((t)->bs_cookie, (h), (o), (l), (f)) + + + +/* + * Bus read (single) operations. + */ +#define bus_space_read_1(t, h, o) __bs_rs(1,(t),(h),(o)) +#define bus_space_read_2(t, h, o) __bs_rs(2,(t),(h),(o)) +#define bus_space_read_4(t, h, o) __bs_rs(4,(t),(h),(o)) +#define bus_space_read_8(t, h, o) __bs_rs(8,(t),(h),(o)) + +#define bus_space_read_stream_1(t, h, o) __bs_rs_s(1,(t), (h), (o)) +#define bus_space_read_stream_2(t, h, o) __bs_rs_s(2,(t), (h), (o)) +#define bus_space_read_stream_4(t, h, o) __bs_rs_s(4,(t), (h), (o)) +#define bus_space_read_stream_8(t, h, o) __bs_rs_s(8,8,(t),(h),(o)) + +/* + * Bus read multiple operations. + */ +#define bus_space_read_multi_1(t, h, o, a, c) \ + __bs_nonsingle(rm,1,(t),(h),(o),(a),(c)) +#define bus_space_read_multi_2(t, h, o, a, c) \ + __bs_nonsingle(rm,2,(t),(h),(o),(a),(c)) +#define bus_space_read_multi_4(t, h, o, a, c) \ + __bs_nonsingle(rm,4,(t),(h),(o),(a),(c)) +#define bus_space_read_multi_8(t, h, o, a, c) \ + __bs_nonsingle(rm,8,(t),(h),(o),(a),(c)) + +#define bus_space_read_multi_stream_1(t, h, o, a, c) \ + __bs_nonsingle_s(rm,1,(t),(h),(o),(a),(c)) +#define bus_space_read_multi_stream_2(t, h, o, a, c) \ + __bs_nonsingle_s(rm,2,(t),(h),(o),(a),(c)) +#define bus_space_read_multi_stream_4(t, h, o, a, c) \ + __bs_nonsingle_s(rm,4,(t),(h),(o),(a),(c)) +#define bus_space_read_multi_stream_8(t, h, o, a, c) \ + __bs_nonsingle_s(rm,8,(t),(h),(o),(a),(c)) + + +/* + * Bus read region operations. + */ +#define bus_space_read_region_1(t, h, o, a, c) \ + __bs_nonsingle(rr,1,(t),(h),(o),(a),(c)) +#define bus_space_read_region_2(t, h, o, a, c) \ + __bs_nonsingle(rr,2,(t),(h),(o),(a),(c)) +#define bus_space_read_region_4(t, h, o, a, c) \ + __bs_nonsingle(rr,4,(t),(h),(o),(a),(c)) +#define bus_space_read_region_8(t, h, o, a, c) \ + __bs_nonsingle(rr,8,(t),(h),(o),(a),(c)) + +#define bus_space_read_region_stream_1(t, h, o, a, c) \ + __bs_nonsingle_s(rr,1,(t),(h),(o),(a),(c)) +#define bus_space_read_region_stream_2(t, h, o, a, c) \ + __bs_nonsingle_s(rr,2,(t),(h),(o),(a),(c)) +#define bus_space_read_region_stream_4(t, h, o, a, c) \ + __bs_nonsingle_s(rr,4,(t),(h),(o),(a),(c)) +#define bus_space_read_region_stream_8(t, h, o, a, c) \ + __bs_nonsingle_s(rr,8,(t),(h),(o),(a),(c)) + + +/* + * Bus write (single) operations. + */ +#define bus_space_write_1(t, h, o, v) __bs_ws(1,(t),(h),(o),(v)) +#define bus_space_write_2(t, h, o, v) __bs_ws(2,(t),(h),(o),(v)) +#define bus_space_write_4(t, h, o, v) __bs_ws(4,(t),(h),(o),(v)) +#define bus_space_write_8(t, h, o, v) __bs_ws(8,(t),(h),(o),(v)) + +#define bus_space_write_stream_1(t, h, o, v) __bs_ws_s(1,(t),(h),(o),(v)) +#define bus_space_write_stream_2(t, h, o, v) __bs_ws_s(2,(t),(h),(o),(v)) +#define bus_space_write_stream_4(t, h, o, v) __bs_ws_s(4,(t),(h),(o),(v)) +#define bus_space_write_stream_8(t, h, o, v) __bs_ws_s(8,(t),(h),(o),(v)) + + +/* + * Bus write multiple operations. + */ +#define bus_space_write_multi_1(t, h, o, a, c) \ + __bs_nonsingle(wm,1,(t),(h),(o),(a),(c)) +#define bus_space_write_multi_2(t, h, o, a, c) \ + __bs_nonsingle(wm,2,(t),(h),(o),(a),(c)) +#define bus_space_write_multi_4(t, h, o, a, c) \ + __bs_nonsingle(wm,4,(t),(h),(o),(a),(c)) +#define bus_space_write_multi_8(t, h, o, a, c) \ + __bs_nonsingle(wm,8,(t),(h),(o),(a),(c)) + +#define bus_space_write_multi_stream_1(t, h, o, a, c) \ + __bs_nonsingle_s(wm,1,(t),(h),(o),(a),(c)) +#define bus_space_write_multi_stream_2(t, h, o, a, c) \ + __bs_nonsingle_s(wm,2,(t),(h),(o),(a),(c)) +#define bus_space_write_multi_stream_4(t, h, o, a, c) \ + __bs_nonsingle_s(wm,4,(t),(h),(o),(a),(c)) +#define bus_space_write_multi_stream_8(t, h, o, a, c) \ + __bs_nonsingle_s(wm,8,(t),(h),(o),(a),(c)) + + +/* + * Bus write region operations. + */ +#define bus_space_write_region_1(t, h, o, a, c) \ + __bs_nonsingle(wr,1,(t),(h),(o),(a),(c)) +#define bus_space_write_region_2(t, h, o, a, c) \ + __bs_nonsingle(wr,2,(t),(h),(o),(a),(c)) +#define bus_space_write_region_4(t, h, o, a, c) \ + __bs_nonsingle(wr,4,(t),(h),(o),(a),(c)) +#define bus_space_write_region_8(t, h, o, a, c) \ + __bs_nonsingle(wr,8,(t),(h),(o),(a),(c)) + +#define bus_space_write_region_stream_1(t, h, o, a, c) \ + __bs_nonsingle_s(wr,1,(t),(h),(o),(a),(c)) +#define bus_space_write_region_stream_2(t, h, o, a, c) \ + __bs_nonsingle_s(wr,2,(t),(h),(o),(a),(c)) +#define bus_space_write_region_stream_4(t, h, o, a, c) \ + __bs_nonsingle_s(wr,4,(t),(h),(o),(a),(c)) +#define bus_space_write_region_stream_8(t, h, o, a, c) \ + __bs_nonsingle_s(wr,8,(t),(h),(o),(a),(c)) + + +/* + * Set multiple operations. + */ +#define bus_space_set_multi_1(t, h, o, v, c) \ + __bs_set(sm,1,(t),(h),(o),(v),(c)) +#define bus_space_set_multi_2(t, h, o, v, c) \ + __bs_set(sm,2,(t),(h),(o),(v),(c)) +#define bus_space_set_multi_4(t, h, o, v, c) \ + __bs_set(sm,4,(t),(h),(o),(v),(c)) +#define bus_space_set_multi_8(t, h, o, v, c) \ + __bs_set(sm,8,(t),(h),(o),(v),(c)) + + +/* + * Set region operations. + */ +#define bus_space_set_region_1(t, h, o, v, c) \ + __bs_set(sr,1,(t),(h),(o),(v),(c)) +#define bus_space_set_region_2(t, h, o, v, c) \ + __bs_set(sr,2,(t),(h),(o),(v),(c)) +#define bus_space_set_region_4(t, h, o, v, c) \ + __bs_set(sr,4,(t),(h),(o),(v),(c)) +#define bus_space_set_region_8(t, h, o, v, c) \ + __bs_set(sr,8,(t),(h),(o),(v),(c)) + + +/* + * Copy operations. + */ +#define bus_space_copy_region_1(t, h1, o1, h2, o2, c) \ + __bs_copy(1, t, h1, o1, h2, o2, c) +#define bus_space_copy_region_2(t, h1, o1, h2, o2, c) \ + __bs_copy(2, t, h1, o1, h2, o2, c) +#define bus_space_copy_region_4(t, h1, o1, h2, o2, c) \ + __bs_copy(4, t, h1, o1, h2, o2, c) +#define bus_space_copy_region_8(t, h1, o1, h2, o2, c) \ + __bs_copy(8, t, h1, o1, h2, o2, c) + +#include <machine/bus_dma.h> + +#endif /* _MACHINE_BUS_H_ */ diff --git a/sys/riscv/include/bus_dma.h b/sys/riscv/include/bus_dma.h new file mode 100644 index 0000000..b07d0f7 --- /dev/null +++ b/sys/riscv/include/bus_dma.h @@ -0,0 +1,8 @@ +/* $FreeBSD$ */ + +#ifndef _MACHINE_BUS_DMA_H_ +#define _MACHINE_BUS_DMA_H_ + +#include <sys/bus_dma.h> + +#endif /* !_MACHINE_BUS_DMA_H_ */ diff --git a/sys/riscv/include/clock.h b/sys/riscv/include/clock.h new file mode 100644 index 0000000..da23dbe --- /dev/null +++ b/sys/riscv/include/clock.h @@ -0,0 +1 @@ +/* $FreeBSD$ */ diff --git a/sys/riscv/include/counter.h b/sys/riscv/include/counter.h new file mode 100644 index 0000000..9d56cce --- /dev/null +++ b/sys/riscv/include/counter.h @@ -0,0 +1,93 @@ +/*- + * Copyright (c) 2012 Konstantin Belousov <kib@FreeBSD.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _MACHINE_COUNTER_H_ +#define _MACHINE_COUNTER_H_ + +#include <sys/pcpu.h> +#ifdef INVARIANTS +#include <sys/proc.h> +#endif + +#define counter_enter() critical_enter() +#define counter_exit() critical_exit() + +#ifdef IN_SUBR_COUNTER_C +static inline uint64_t +counter_u64_read_one(uint64_t *p, int cpu) +{ + + return (*(uint64_t *)((char *)p + sizeof(struct pcpu) * cpu)); +} + +static inline uint64_t +counter_u64_fetch_inline(uint64_t *p) +{ + uint64_t r; + int i; + + r = 0; + for (i = 0; i < mp_ncpus; i++) + r += counter_u64_read_one((uint64_t *)p, i); + + return (r); +} + +/* XXXKIB might interrupt increment */ +static void +counter_u64_zero_one_cpu(void *arg) +{ + + *((uint64_t *)((char *)arg + sizeof(struct pcpu) * + PCPU_GET(cpuid))) = 0; +} + +static inline void +counter_u64_zero_inline(counter_u64_t c) +{ + + smp_rendezvous(smp_no_rendevous_barrier, counter_u64_zero_one_cpu, + smp_no_rendevous_barrier, c); +} +#endif + +#define counter_u64_add_protected(c, inc) do { \ + CRITICAL_ASSERT(curthread); \ + *(uint64_t *)zpcpu_get(c) += (inc); \ +} while (0) + +static inline void +counter_u64_add(counter_u64_t c, int64_t inc) +{ + + counter_enter(); + counter_u64_add_protected(c, inc); + counter_exit(); +} + +#endif /* ! _MACHINE_COUNTER_H_ */ diff --git a/sys/riscv/include/cpu.h b/sys/riscv/include/cpu.h new file mode 100644 index 0000000..c0a55a7 --- /dev/null +++ b/sys/riscv/include/cpu.h @@ -0,0 +1,95 @@ +/*- + * Copyright (c) 2015 Ruslan Bukin <br@bsdpad.com> + * All rights reserved. + * + * Portions of this software were developed by SRI International and the + * University of Cambridge Computer Laboratory under DARPA/AFRL contract + * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. + * + * Portions of this software were developed by the University of Cambridge + * Computer Laboratory as part of the CTSRD Project, with support from the + * UK Higher Education Innovation Fund (HEIF). + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _MACHINE_CPU_H_ +#define _MACHINE_CPU_H_ + +#include <machine/atomic.h> +#include <machine/frame.h> + +#define TRAPF_PC(tfp) ((tfp)->tf_ra) +#define TRAPF_USERMODE(tfp) (((tfp)->tf_sepc & (1ul << 63)) == 0) + +#define cpu_getstack(td) ((td)->td_frame->tf_sp) +#define cpu_setstack(td, sp) ((td)->td_frame->tf_sp = (sp)) +#define cpu_spinwait() /* nothing */ + +#ifdef _KERNEL + +/* + * 0x0000 CPU ID unimplemented + * 0x0001 UC Berkeley Rocket repo + * 0x0002Â0x7FFE Reserved for open-source repos + * 0x7FFF Reserved for extension + * 0x8000 Reserved for anonymous source + * 0x8001Â0xFFFE Reserved for proprietary implementations + * 0xFFFF Reserved for extension + */ + +#define CPU_IMPL_SHIFT 0 +#define CPU_IMPL_MASK (0xffff << CPU_IMPL_SHIFT) +#define CPU_IMPL(mimpid) ((mimpid & CPU_IMPL_MASK) >> CPU_IMPL_SHIFT) +#define CPU_IMPL_UNIMPLEMEN 0x0 +#define CPU_IMPL_UCB_ROCKET 0x1 + +#define CPU_PART_SHIFT 62 +#define CPU_PART_MASK (0x3ul << CPU_PART_SHIFT) +#define CPU_PART(mcpuid) ((mcpuid & CPU_PART_MASK) >> CPU_PART_SHIFT) +#define CPU_PART_RV32I 0x0 +#define CPU_PART_RV32E 0x1 +#define CPU_PART_RV64I 0x2 +#define CPU_PART_RV128I 0x3 + +extern char btext[]; +extern char etext[]; + +void cpu_halt(void) __dead2; +void cpu_reset(void) __dead2; +void fork_trampoline(void); +void identify_cpu(void); +void swi_vm(void *v); + +static __inline uint64_t +get_cyclecount(void) +{ + + /* TODO: This is bogus */ + return (1); +} + +#endif + +#endif /* !_MACHINE_CPU_H_ */ diff --git a/sys/riscv/include/cpufunc.h b/sys/riscv/include/cpufunc.h new file mode 100644 index 0000000..4ee68dc --- /dev/null +++ b/sys/riscv/include/cpufunc.h @@ -0,0 +1,123 @@ +/*- + * Copyright (c) 2015 Ruslan Bukin <br@bsdpad.com> + * All rights reserved. + * + * Portions of this software were developed by SRI International and the + * University of Cambridge Computer Laboratory under DARPA/AFRL contract + * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. + * + * Portions of this software were developed by the University of Cambridge + * Computer Laboratory as part of the CTSRD Project, with support from the + * UK Higher Education Innovation Fund (HEIF). + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _MACHINE_CPUFUNC_H_ +#define _MACHINE_CPUFUNC_H_ + +#ifdef _KERNEL + +#include <machine/riscvreg.h> + +static __inline void +breakpoint(void) +{ + + __asm("ebreak"); +} + +static __inline register_t +intr_disable(void) +{ + uint64_t ret; + + __asm __volatile( + "csrrci %0, sstatus, 1" + : "=&r" (ret) + ); + + return (ret & SSTATUS_IE); +} + +static __inline void +intr_restore(register_t s) +{ + + __asm __volatile( + "csrs sstatus, %0" + :: "r" (s) + ); +} + +static __inline void +intr_enable(void) +{ + + __asm __volatile( + "csrsi sstatus, 1" + ); +} + +static __inline register_t +machine_command(uint64_t cmd, uint64_t arg) +{ + uint64_t res; + + __asm __volatile( + "mv t5, %2\n" + "mv t6, %1\n" + "ecall\n" + "mv %0, t6" : "=&r"(res) : "r"(arg), "r"(cmd) + ); + + return (res); +} + +#define cpu_nullop() riscv_nullop() +#define cpufunc_nullop() riscv_nullop() +#define cpu_setttb(a) riscv_setttb(a) + +#define cpu_tlb_flushID() riscv_tlb_flushID() +#define cpu_tlb_flushID_SE(e) riscv_tlb_flushID_SE(e) + +#define cpu_dcache_wbinv_range(a, s) riscv_dcache_wbinv_range((a), (s)) +#define cpu_dcache_inv_range(a, s) riscv_dcache_inv_range((a), (s)) +#define cpu_dcache_wb_range(a, s) riscv_dcache_wb_range((a), (s)) + +#define cpu_idcache_wbinv_range(a, s) riscv_idcache_wbinv_range((a), (s)) +#define cpu_icache_sync_range(a, s) riscv_icache_sync_range((a), (s)) + +void riscv_nullop(void); +void riscv_setttb(vm_offset_t); +void riscv_tlb_flushID(void); +void riscv_tlb_flushID_SE(vm_offset_t); +void riscv_icache_sync_range(vm_offset_t, vm_size_t); +void riscv_idcache_wbinv_range(vm_offset_t, vm_size_t); +void riscv_dcache_wbinv_range(vm_offset_t, vm_size_t); +void riscv_dcache_inv_range(vm_offset_t, vm_size_t); +void riscv_dcache_wb_range(vm_offset_t, vm_size_t); + +#endif /* _KERNEL */ +#endif /* _MACHINE_CPUFUNC_H_ */ diff --git a/sys/riscv/include/db_machdep.h b/sys/riscv/include/db_machdep.h new file mode 100644 index 0000000..a9890d9 --- /dev/null +++ b/sys/riscv/include/db_machdep.h @@ -0,0 +1,47 @@ +/*- + * Copyright (c) 2015 Ruslan Bukin <br@bsdpad.com> + * All rights reserved. + * + * Portions of this software were developed by SRI International and the + * University of Cambridge Computer Laboratory under DARPA/AFRL contract + * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. + * + * Portions of this software were developed by the University of Cambridge + * Computer Laboratory as part of the CTSRD Project, with support from the + * UK Higher Education Innovation Fund (HEIF). + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _MACHINE_DB_MACHDEP_H_ +#define _MACHINE_DB_MACHDEP_H_ + +#include <machine/riscvreg.h> +#include <machine/frame.h> +#include <machine/trap.h> + +typedef vm_offset_t db_addr_t; +typedef long db_expr_t; + +#endif /* !_MACHINE_DB_MACHDEP_H_ */ diff --git a/sys/riscv/include/dump.h b/sys/riscv/include/dump.h new file mode 100644 index 0000000..182a89e --- /dev/null +++ b/sys/riscv/include/dump.h @@ -0,0 +1,82 @@ +/*- + * Copyright (c) 2014 EMC Corp. + * Author: Conrad Meyer <conrad.meyer@isilon.com> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _MACHINE_DUMP_H_ +#define _MACHINE_DUMP_H_ + +#define KERNELDUMP_ARCH_VERSION KERNELDUMP_RISCV_VERSION +#define EM_VALUE EM_RISCV + +/* 20 phys_avail entry pairs correspond to 10 pa's */ +#define DUMPSYS_MD_PA_NPAIRS 10 +#define DUMPSYS_NUM_AUX_HDRS 0 + +static inline void +dumpsys_pa_init(void) +{ + + dumpsys_gen_pa_init(); +} + +static inline struct dump_pa * +dumpsys_pa_next(struct dump_pa *p) +{ + + return (dumpsys_gen_pa_next(p)); +} + +static inline void +dumpsys_wbinv_all(void) +{ + + dumpsys_gen_wbinv_all(); +} + +static inline void +dumpsys_unmap_chunk(vm_paddr_t pa, size_t s, void *va) +{ + + dumpsys_gen_unmap_chunk(pa, s, va); +} + +static inline int +dumpsys_write_aux_headers(struct dumperinfo *di) +{ + + return (dumpsys_gen_write_aux_headers(di)); +} + +static inline int +dumpsys(struct dumperinfo *di) +{ + + return (dumpsys_generic(di)); +} + +#endif /* !_MACHINE_DUMP_H_ */ diff --git a/sys/riscv/include/elf.h b/sys/riscv/include/elf.h new file mode 100644 index 0000000..3a89577 --- /dev/null +++ b/sys/riscv/include/elf.h @@ -0,0 +1,105 @@ +/*- + * Copyright (c) 1996-1997 John D. Polstra. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _MACHINE_ELF_H_ +#define _MACHINE_ELF_H_ + +/* + * ELF definitions for the RISC-V architecture. + */ + +#include <sys/elf32.h> /* Definitions common to all 32 bit architectures. */ +#include <sys/elf64.h> /* Definitions common to all 64 bit architectures. */ + +#define __ELF_WORD_SIZE 64 /* Used by <sys/elf_generic.h> */ +#include <sys/elf_generic.h> + +/* + * Auxiliary vector entries for passing information to the interpreter. + */ + +typedef struct { /* Auxiliary vector entry on initial stack */ + int a_type; /* Entry type. */ + union { + int a_val; /* Integer value. */ + } a_un; +} Elf32_Auxinfo; + +typedef struct { /* Auxiliary vector entry on initial stack */ + long a_type; /* Entry type. */ + union { + long a_val; /* Integer value. */ + void *a_ptr; /* Address. */ + void (*a_fcn)(void); /* Function pointer (not used). */ + } a_un; +} Elf64_Auxinfo; + +__ElfType(Auxinfo); + +#define ELF_ARCH EM_RISCV + +#define ELF_MACHINE_OK(x) ((x) == (ELF_ARCH)) + +/* Values for a_type. */ +#define AT_NULL 0 /* Terminates the vector. */ +#define AT_IGNORE 1 /* Ignored entry. */ +#define AT_EXECFD 2 /* File descriptor of program to load. */ +#define AT_PHDR 3 /* Program header of program already loaded. */ +#define AT_PHENT 4 /* Size of each program header entry. */ +#define AT_PHNUM 5 /* Number of program header entries. */ +#define AT_PAGESZ 6 /* Page size in bytes. */ +#define AT_BASE 7 /* Interpreter's base address. */ +#define AT_FLAGS 8 /* Flags (unused). */ +#define AT_ENTRY 9 /* Where interpreter should transfer control. */ +#define AT_NOTELF 10 /* Program is not ELF ?? */ +#define AT_UID 11 /* Real uid. */ +#define AT_EUID 12 /* Effective uid. */ +#define AT_GID 13 /* Real gid. */ +#define AT_EGID 14 /* Effective gid. */ +#define AT_EXECPATH 15 /* Path to the executable. */ +#define AT_CANARY 16 /* Canary for SSP */ +#define AT_CANARYLEN 17 /* Length of the canary. */ +#define AT_OSRELDATE 18 /* OSRELDATE. */ +#define AT_NCPUS 19 /* Number of CPUs. */ +#define AT_PAGESIZES 20 /* Pagesizes. */ +#define AT_PAGESIZESLEN 21 /* Number of pagesizes. */ +#define AT_TIMEKEEP 22 /* Pointer to timehands. */ +#define AT_STACKPROT 23 /* Initial stack protection. */ + +#define AT_COUNT 24 /* Count of defined aux entry types. */ + +/* Define "machine" characteristics */ +#define ELF_TARG_CLASS ELFCLASS64 +#define ELF_TARG_DATA ELFDATA2LSB +#define ELF_TARG_MACH EM_RISCV +#define ELF_TARG_VER 1 + +/* TODO: set correct value */ +#define ET_DYN_LOAD_ADDR 0x100000 + +#endif /* !_MACHINE_ELF_H_ */ diff --git a/sys/riscv/include/endian.h b/sys/riscv/include/endian.h new file mode 100644 index 0000000..25516fa --- /dev/null +++ b/sys/riscv/include/endian.h @@ -0,0 +1,124 @@ +/*- + * Copyright (c) 2001 David E. O'Brien + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)endian.h 8.1 (Berkeley) 6/10/93 + * $NetBSD: endian.h,v 1.7 1999/08/21 05:53:51 simonb Exp $ + * $FreeBSD$ + */ + +#ifndef _MACHINE_ENDIAN_H_ +#define _MACHINE_ENDIAN_H_ + +#include <sys/_types.h> + +/* + * Definitions for byte order, according to byte significance from low + * address to high. + */ +#define _LITTLE_ENDIAN 1234 /* LSB first: i386, vax */ +#define _BIG_ENDIAN 4321 /* MSB first: 68000, ibm, net */ +#define _PDP_ENDIAN 3412 /* LSB first in word, MSW first in long */ + +#define _BYTE_ORDER _LITTLE_ENDIAN + +#if __BSD_VISIBLE +#define LITTLE_ENDIAN _LITTLE_ENDIAN +#define BIG_ENDIAN _BIG_ENDIAN +#define PDP_ENDIAN _PDP_ENDIAN +#define BYTE_ORDER _BYTE_ORDER +#endif + +#define _QUAD_HIGHWORD 1 +#define _QUAD_LOWWORD 0 +#define __ntohl(x) (__bswap32(x)) +#define __ntohs(x) (__bswap16(x)) +#define __htonl(x) (__bswap32(x)) +#define __htons(x) (__bswap16(x)) + +static __inline __uint64_t +__bswap64(__uint64_t _x) +{ + __uint64_t ret; + + ret = (_x >> 56); + ret |= ((_x >> 40) & 0xff00); + ret |= ((_x >> 24) & 0xff0000); + ret |= ((_x >> 8) & 0xff000000); + ret |= ((_x << 8) & ((__uint64_t)0xff << 32)); + ret |= ((_x << 24) & ((__uint64_t)0xff << 40)); + ret |= ((_x << 40) & ((__uint64_t)0xff << 48)); + ret |= (_x << 56); + + return (ret); +} + +static __inline __uint32_t +__bswap32_var(__uint32_t _x) +{ + + return ((_x >> 24) | ((_x >> 8) & 0xff00) | ((_x << 8) & 0xff0000) | + ((_x << 24) & 0xff000000)); +} + +static __inline __uint16_t +__bswap16_var(__uint16_t _x) +{ + __uint32_t ret; + + ret = ((_x >> 8) | ((_x << 8) & 0xff00)); + + return ((__uint16_t)ret); +} + +#ifdef __OPTIMIZE__ + +#define __bswap32_constant(x) \ + ((((x) & 0xff000000U) >> 24) | \ + (((x) & 0x00ff0000U) >> 8) | \ + (((x) & 0x0000ff00U) << 8) | \ + (((x) & 0x000000ffU) << 24)) + +#define __bswap16_constant(x) \ + ((((x) & 0xff00) >> 8) | \ + (((x) & 0x00ff) << 8)) + +#define __bswap16(x) \ + ((__uint16_t)(__builtin_constant_p(x) ? \ + __bswap16_constant(x) : \ + __bswap16_var(x))) + +#define __bswap32(x) \ + ((__uint32_t)(__builtin_constant_p(x) ? \ + __bswap32_constant(x) : \ + __bswap32_var(x))) + +#else +#define __bswap16(x) __bswap16_var(x) +#define __bswap32(x) __bswap32_var(x) + +#endif /* __OPTIMIZE__ */ +#endif /* !_MACHINE_ENDIAN_H_ */ diff --git a/sys/riscv/include/exec.h b/sys/riscv/include/exec.h new file mode 100644 index 0000000..da23dbe --- /dev/null +++ b/sys/riscv/include/exec.h @@ -0,0 +1 @@ +/* $FreeBSD$ */ diff --git a/sys/riscv/include/float.h b/sys/riscv/include/float.h new file mode 100644 index 0000000..0d6abe1 --- /dev/null +++ b/sys/riscv/include/float.h @@ -0,0 +1,111 @@ +/*- + * Copyright (c) 1989 Regents of the University of California. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * from: @(#)float.h 7.1 (Berkeley) 5/8/90 + * $FreeBSD$ + */ + +#ifndef _MACHINE_FLOAT_H_ +#define _MACHINE_FLOAT_H_ + +#include <sys/cdefs.h> + +__BEGIN_DECLS +extern int __flt_rounds(void); +__END_DECLS + +#define FLT_RADIX 2 /* b */ +#define FLT_ROUNDS __flt_rounds() +#if __ISO_C_VISIBLE >= 1999 +#define FLT_EVAL_METHOD 0 +#define DECIMAL_DIG 17 /* max precision in decimal digits */ +#endif + +#define FLT_MANT_DIG 24 /* p */ +#define FLT_EPSILON 1.19209290E-07F /* b**(1-p) */ +#define FLT_DIG 6 /* floor((p-1)*log10(b))+(b == 10) */ +#define FLT_MIN_EXP (-125) /* emin */ +#define FLT_MIN 1.17549435E-38F /* b**(emin-1) */ +#define FLT_MIN_10_EXP (-37) /* ceil(log10(b**(emin-1))) */ +#define FLT_MAX_EXP 128 /* emax */ +#define FLT_MAX 3.40282347E+38F /* (1-b**(-p))*b**emax */ +#define FLT_MAX_10_EXP 38 /* floor(log10((1-b**(-p))*b**emax)) */ +#if __ISO_C_VISIBLE >= 2011 +#define FLT_TRUE_MIN 1.40129846E-45F /* b**(emin-p) */ +#define FLT_DECIMAL_DIG 9 /* ceil(1+p*log10(b)) */ +#define FLT_HAS_SUBNORM 1 +#endif /* __ISO_C_VISIBLE >= 2011 */ + +#define DBL_MANT_DIG 53 +#define DBL_EPSILON 2.2204460492503131E-16 +#define DBL_DIG 15 +#define DBL_MIN_EXP (-1021) +#define DBL_MIN 2.2250738585072014E-308 +#define DBL_MIN_10_EXP (-307) +#define DBL_MAX_EXP 1024 +#define DBL_MAX 1.7976931348623157E+308 +#define DBL_MAX_10_EXP 308 +#if __ISO_C_VISIBLE >= 2011 +#define DBL_TRUE_MIN 4.9406564584124654E-324 +#define DBL_DECIMAL_DIG 17 +#define DBL_HAS_SUBNORM 1 +#endif /* __ISO_C_VISIBLE >= 2011 */ + +#define LDBL_MANT_DIG DBL_MANT_DIG +#define LDBL_EPSILON ((long double)DBL_EPSILON) +#define LDBL_DIG DBL_DIG +#define LDBL_MIN_EXP DBL_MIN_EXP +#define LDBL_MIN ((long double)DBL_MIN) +#define LDBL_MIN_10_EXP DBL_MIN_10_EXP +#define LDBL_MAX_EXP DBL_MAX_EXP +#define LDBL_MAX ((long double)DBL_MAX) +#define LDBL_MAX_10_EXP DBL_MAX_10_EXP +#if __ISO_C_VISIBLE >= 2011 +#define LDBL_TRUE_MIN ((long double)DBL_TRUE_MIN) +#define LDBL_DECIMAL_DIG DBL_DECIMAL_DIG +#define LDBL_HAS_SUBNORM DBL_HAS_SUBNORM +#endif /* __ISO_C_VISIBLE >= 2011 */ + +#if 0 /* RISCVTODO */ +#define LDBL_MANT_DIG 113 +#define LDBL_EPSILON 1.925929944387235853055977942584927319E-34L +#define LDBL_DIG 33 +#define LDBL_MIN_EXP (-16381) +#define LDBL_MIN 3.362103143112093506262677817321752603E-4932L +#define LDBL_MIN_10_EXP (-4931) +#define LDBL_MAX_EXP (+16384) +#define LDBL_MAX 1.189731495357231765085759326628007016E+4932L +#define LDBL_MAX_10_EXP (+4932) +#if __ISO_C_VISIBLE >= 2011 +#define LDBL_TRUE_MIN 6.475175119438025110924438958227646552E-4966L +#define LDBL_DECIMAL_DIG 36 +#define LDBL_HAS_SUBNORM 1 +#endif /* __ISO_C_VISIBLE >= 2011 */ +#endif + +#endif /* _MACHINE_FLOAT_H_ */ diff --git a/sys/riscv/include/floatingpoint.h b/sys/riscv/include/floatingpoint.h new file mode 100644 index 0000000..da2e005 --- /dev/null +++ b/sys/riscv/include/floatingpoint.h @@ -0,0 +1,3 @@ +/* $FreeBSD$ */ + +#include <machine/ieeefp.h> diff --git a/sys/riscv/include/frame.h b/sys/riscv/include/frame.h new file mode 100644 index 0000000..e74d926 --- /dev/null +++ b/sys/riscv/include/frame.h @@ -0,0 +1,72 @@ +/*- + * Copyright (c) 2015 Ruslan Bukin <br@bsdpad.com> + * All rights reserved. + * + * Portions of this software were developed by SRI International and the + * University of Cambridge Computer Laboratory under DARPA/AFRL contract + * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. + * + * Portions of this software were developed by the University of Cambridge + * Computer Laboratory as part of the CTSRD Project, with support from the + * UK Higher Education Innovation Fund (HEIF). + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _MACHINE_FRAME_H_ +#define _MACHINE_FRAME_H_ + +#ifndef LOCORE + +#include <sys/signal.h> +#include <sys/ucontext.h> + +/* + * NOTE: keep this structure in sync with struct reg and struct mcontext. + */ +struct trapframe { + uint64_t tf_ra; + uint64_t tf_sp; + uint64_t tf_gp; + uint64_t tf_tp; + uint64_t tf_t[7]; + uint64_t tf_s[12]; + uint64_t tf_a[8]; + uint64_t tf_sepc; + uint64_t tf_sstatus; + uint64_t tf_sbadaddr; + uint64_t tf_scause; +}; + +/* + * Signal frame. Pushed onto user stack before calling sigcode. + */ +struct sigframe { + siginfo_t sf_si; /* actual saved siginfo */ + ucontext_t sf_uc; /* actual saved ucontext */ +}; + +#endif /* !LOCORE */ + +#endif /* !_MACHINE_FRAME_H_ */ diff --git a/sys/riscv/include/ieeefp.h b/sys/riscv/include/ieeefp.h new file mode 100644 index 0000000..f7d44f3 --- /dev/null +++ b/sys/riscv/include/ieeefp.h @@ -0,0 +1,8 @@ +/* $FreeBSD$ */ + +#ifndef _MACHINE_IEEEFP_H_ +#define _MACHINE_IEEEFP_H_ + +/* TODO */ + +#endif /* _MACHINE_IEEEFP_H_ */ diff --git a/sys/riscv/include/in_cksum.h b/sys/riscv/include/in_cksum.h new file mode 100644 index 0000000..522ba00 --- /dev/null +++ b/sys/riscv/include/in_cksum.h @@ -0,0 +1,52 @@ +/*- + * Copyright (c) 1990 The Regents of the University of California. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * from tahoe: in_cksum.c 1.2 86/01/05 + * from: @(#)in_cksum.c 1.3 (Berkeley) 1/19/91 + * from: Id: in_cksum.c,v 1.8 1995/12/03 18:35:19 bde Exp + * $FreeBSD$ + */ + +#ifndef _MACHINE_IN_CKSUM_H_ +#define _MACHINE_IN_CKSUM_H_ 1 + +#include <sys/cdefs.h> + +#ifdef _KERNEL +#define in_cksum(m, len) in_cksum_skip(m, len, 0) +u_short in_addword(u_short sum, u_short b); +u_short in_cksum_skip(struct mbuf *m, int len, int skip); +u_int do_cksum(const void *, int); +#if defined(IPVERSION) && (IPVERSION == 4) +u_int in_cksum_hdr(const struct ip *); +#endif + +u_short in_pseudo(u_int sum, u_int b, u_int c); + +#endif /* _KERNEL */ +#endif /* _MACHINE_IN_CKSUM_H_ */ diff --git a/sys/riscv/include/intr.h b/sys/riscv/include/intr.h new file mode 100644 index 0000000..569f7c9 --- /dev/null +++ b/sys/riscv/include/intr.h @@ -0,0 +1,54 @@ +/*- + * Copyright (c) 2015 Ruslan Bukin <br@bsdpad.com> + * All rights reserved. + * + * Portions of this software were developed by SRI International and the + * University of Cambridge Computer Laboratory under DARPA/AFRL contract + * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. + * + * Portions of this software were developed by the University of Cambridge + * Computer Laboratory as part of the CTSRD Project, with support from the + * UK Higher Education Innovation Fund (HEIF). + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _MACHINE_INTR_MACHDEP_H_ +#define _MACHINE_INTR_MACHDEP_H_ + +struct trapframe; + +void riscv_init_interrupts(void); +int riscv_teardown_intr(void *); +int riscv_config_intr(u_int, enum intr_trigger, enum intr_polarity); +int riscv_setup_intr(const char *, driver_filter_t *, driver_intr_t *, + void *, int, int, void **); +void riscv_cpu_intr(struct trapframe *); + +typedef unsigned long * riscv_intrcnt_t; + +riscv_intrcnt_t riscv_intrcnt_create(const char *); +void riscv_intrcnt_setname(riscv_intrcnt_t, const char *); + +#endif /* !_MACHINE_INTR_MACHDEP_H_ */ diff --git a/sys/riscv/include/kdb.h b/sys/riscv/include/kdb.h new file mode 100644 index 0000000..0a1e781 --- /dev/null +++ b/sys/riscv/include/kdb.h @@ -0,0 +1,58 @@ +/*- + * Copyright (c) 2004 Marcel Moolenaar + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _MACHINE_KDB_H_ +#define _MACHINE_KDB_H_ + +#include <machine/cpufunc.h> + +#define KDB_STOPPEDPCB(pc) &stoppcbs[pc->pc_cpuid] + +static __inline void +kdb_cpu_clear_singlestep(void) +{ +} + +static __inline void +kdb_cpu_set_singlestep(void) +{ +} + +static __inline void +kdb_cpu_sync_icache(unsigned char *addr, size_t size) +{ + + cpu_icache_sync_range((vm_offset_t)addr, size); +} + +static __inline void +kdb_cpu_trap(int type, int code) +{ +} + +#endif /* _MACHINE_KDB_H_ */ diff --git a/sys/riscv/include/machdep.h b/sys/riscv/include/machdep.h new file mode 100644 index 0000000..033f7d6 --- /dev/null +++ b/sys/riscv/include/machdep.h @@ -0,0 +1,51 @@ +/*- + * Copyright (c) 2015 Ruslan Bukin <br@bsdpad.com> + * All rights reserved. + * + * Portions of this software were developed by SRI International and the + * University of Cambridge Computer Laboratory under DARPA/AFRL contract + * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. + * + * Portions of this software were developed by the University of Cambridge + * Computer Laboratory as part of the CTSRD Project, with support from the + * UK Higher Education Innovation Fund (HEIF). + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _MACHINE_MACHDEP_H_ +#define _MACHINE_MACHDEP_H_ + +struct riscv_bootparams { + vm_offset_t kern_l1pt; /* Kernel L1 base */ + vm_offset_t kern_stack; +}; + +extern vm_paddr_t physmap[]; +extern u_int physmap_idx; + +vm_offset_t fake_preload_metadata(struct riscv_bootparams *rbp); +void initriscv(struct riscv_bootparams *); + +#endif /* _MACHINE_MACHDEP_H_ */ diff --git a/sys/riscv/include/md_var.h b/sys/riscv/include/md_var.h new file mode 100644 index 0000000..8545591 --- /dev/null +++ b/sys/riscv/include/md_var.h @@ -0,0 +1,49 @@ +/*- + * Copyright (c) 1995 Bruce D. Evans. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the author nor the names of contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * from: FreeBSD: src/sys/i386/include/md_var.h,v 1.40 2001/07/12 + * $FreeBSD$ + */ + +#ifndef _MACHINE_MD_VAR_H_ +#define _MACHINE_MD_VAR_H_ + +extern long Maxmem; +extern char sigcode[]; +extern int szsigcode; +extern uint64_t *vm_page_dump; +extern int vm_page_dump_size; + +struct dumperinfo; + +void busdma_swi(void); +void dump_add_page(vm_paddr_t); +void dump_drop_page(vm_paddr_t); +int minidumpsys(struct dumperinfo *); + +#endif /* !_MACHINE_MD_VAR_H_ */ diff --git a/sys/riscv/include/memdev.h b/sys/riscv/include/memdev.h new file mode 100644 index 0000000..f16ba16 --- /dev/null +++ b/sys/riscv/include/memdev.h @@ -0,0 +1,40 @@ +/*- + * Copyright (c) 2004 Mark R V Murray + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer + * in this position and unchanged. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _MACHINE_MEMDEV_H_ +#define _MACHINE_MEMDEV_H_ + +#define CDEV_MINOR_MEM 0 +#define CDEV_MINOR_KMEM 1 + +d_open_t memopen; +d_read_t memrw; +#define memioctl (d_ioctl_t *)NULL +#define memmmap (d_mmap_t *)NULL + +#endif /* _MACHINE_MEMDEV_H_ */ diff --git a/sys/riscv/include/minidump.h b/sys/riscv/include/minidump.h new file mode 100644 index 0000000..314040e --- /dev/null +++ b/sys/riscv/include/minidump.h @@ -0,0 +1,48 @@ +/*- + * Copyright (c) 2006 Peter Wemm + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * From i386: FreeBSD: 157909 2006-04-21 04:28:43Z peter + * $FreeBSD$ + */ + +#ifndef _MACHINE_MINIDUMP_H_ +#define _MACHINE_MINIDUMP_H_ 1 + +#define MINIDUMP_MAGIC "minidump FreeBSD/riscv" +#define MINIDUMP_VERSION 1 + +struct minidumphdr { + char magic[24]; + uint32_t version; + uint32_t msgbufsize; + uint32_t bitmapsize; + uint32_t pmapsize; + uint64_t kernbase; + uint64_t dmapphys; + uint64_t dmapbase; + uint64_t dmapend; +}; + +#endif /* _MACHINE_MINIDUMP_H_ */ diff --git a/sys/riscv/include/ofw_machdep.h b/sys/riscv/include/ofw_machdep.h new file mode 100644 index 0000000..ac5d9d0 --- /dev/null +++ b/sys/riscv/include/ofw_machdep.h @@ -0,0 +1,39 @@ +/*- + * Copyright (c) 2001 by Thomas Moestl <tmm@FreeBSD.org>. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE + * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _MACHINE_OFW_MACHDEP_H_ +#define _MACHINE_OFW_MACHDEP_H_ + +#include <vm/vm.h> + +typedef uint32_t cell_t; +struct mem_region { + vm_offset_t mr_start; + vm_size_t mr_size; +}; + +#endif /* _MACHINE_OFW_MACHDEP_H_ */ diff --git a/sys/riscv/include/param.h b/sys/riscv/include/param.h new file mode 100644 index 0000000..2c80058 --- /dev/null +++ b/sys/riscv/include/param.h @@ -0,0 +1,108 @@ +/*- + * Copyright (c) 1990 The Regents of the University of California. + * All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * William Jolitz. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * from: @(#)param.h 5.8 (Berkeley) 6/28/91 + * $FreeBSD$ + */ + +#ifndef _MACHINE_PARAM_H_ +#define _MACHINE_PARAM_H_ + +/* + * Machine dependent constants for RISC-V. + */ + +#include <machine/_align.h> + +#define STACKALIGNBYTES (16 - 1) +#define STACKALIGN(p) ((uint64_t)(p) & ~STACKALIGNBYTES) + +#ifndef MACHINE +#define MACHINE "riscv" +#endif +#ifndef MACHINE_ARCH +#define MACHINE_ARCH "riscv64" +#endif + +#if defined(SMP) || defined(KLD_MODULE) +#ifndef MAXCPU +#define MAXCPU 2 +#endif +#else +#define MAXCPU 1 +#endif /* SMP || KLD_MODULE */ + +#ifndef MAXMEMDOM +#define MAXMEMDOM 1 +#endif + +#define ALIGNBYTES _ALIGNBYTES +#define ALIGN(p) _ALIGN(p) +/* + * ALIGNED_POINTER is a boolean macro that checks whether an address + * is valid to fetch data elements of type t from on this architecture. + * This does not reflect the optimal alignment, just the possibility + * (within reasonable limits). + */ +#define ALIGNED_POINTER(p, t) ((((u_long)(p)) & (sizeof(t) - 1)) == 0) + +/* + * CACHE_LINE_SIZE is the compile-time maximum cache line size for an + * architecture. It should be used with appropriate caution. + */ +#define CACHE_LINE_SHIFT 6 +#define CACHE_LINE_SIZE (1 << CACHE_LINE_SHIFT) + +#define PAGE_SHIFT 12 +#define PAGE_SIZE (1 << PAGE_SHIFT) /* Page size */ +#define PAGE_MASK (PAGE_SIZE - 1) + +#define MAXPAGESIZES 1 /* maximum number of supported page sizes */ + +#ifndef KSTACK_PAGES +#define KSTACK_PAGES 4 /* pages of kernel stack (with pcb) */ +#endif + +#define KSTACK_GUARD_PAGES 1 /* pages of kstack guard; 0 disables */ +#define PCPU_PAGES 1 + +/* + * Mach derived conversion macros + */ +#define round_page(x) (((unsigned long)(x) + PAGE_MASK) & ~PAGE_MASK) +#define trunc_page(x) ((unsigned long)(x) & ~PAGE_MASK) + +#define atop(x) ((unsigned long)(x) >> PAGE_SHIFT) +#define ptoa(x) ((unsigned long)(x) << PAGE_SHIFT) + +#define riscv_btop(x) ((unsigned long)(x) >> PAGE_SHIFT) +#define riscv_ptob(x) ((unsigned long)(x) << PAGE_SHIFT) + +#define pgtok(x) ((unsigned long)(x) * (PAGE_SIZE / 1024)) + +#endif /* !_MACHINE_PARAM_H_ */ diff --git a/sys/riscv/include/pcb.h b/sys/riscv/include/pcb.h new file mode 100644 index 0000000..a6cecb7 --- /dev/null +++ b/sys/riscv/include/pcb.h @@ -0,0 +1,64 @@ +/*- + * Copyright (c) 2015 Ruslan Bukin <br@bsdpad.com> + * All rights reserved. + * + * Portions of this software were developed by SRI International and the + * University of Cambridge Computer Laboratory under DARPA/AFRL contract + * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. + * + * Portions of this software were developed by the University of Cambridge + * Computer Laboratory as part of the CTSRD Project, with support from the + * UK Higher Education Innovation Fund (HEIF). + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _MACHINE_PCB_H_ +#define _MACHINE_PCB_H_ + +#ifndef LOCORE + +struct trapframe; + +struct pcb { + uint64_t pcb_ra; + uint64_t pcb_sp; + uint64_t pcb_gp; + uint64_t pcb_tp; + uint64_t pcb_t[7]; + uint64_t pcb_s[12]; + uint64_t pcb_a[8]; + uint64_t pcb_sepc; + vm_offset_t pcb_l1addr; /* L1 page tables base address */ + vm_offset_t pcb_onfault; +}; + +#ifdef _KERNEL +void makectx(struct trapframe *tf, struct pcb *pcb); +int savectx(struct pcb *pcb) __returns_twice; +#endif + +#endif /* !LOCORE */ + +#endif /* !_MACHINE_PCB_H_ */ diff --git a/sys/riscv/include/pcpu.h b/sys/riscv/include/pcpu.h new file mode 100644 index 0000000..c60a954 --- /dev/null +++ b/sys/riscv/include/pcpu.h @@ -0,0 +1,74 @@ +/*- + * Copyright (c) 1999 Luoqi Chen <luoqi@freebsd.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * from: FreeBSD: src/sys/i386/include/globaldata.h,v 1.27 2001/04/27 + * $FreeBSD$ + */ + +#ifndef _MACHINE_PCPU_H_ +#define _MACHINE_PCPU_H_ + +#include <machine/cpu.h> +#include <machine/cpufunc.h> + +#define ALT_STACK_SIZE 128 + +#define PCPU_MD_FIELDS \ + char __pad[129] + +#ifdef _KERNEL + +struct pcb; +struct pcpu; +extern struct pcpu *pcpup; + +static inline struct pcpu * +get_pcpu(void) +{ + + return (pcpup); +} + +static inline struct thread * +get_curthread(void) +{ + struct thread *td; + + td = (struct thread *)*(uint64_t *)pcpup; + + return (td); +} + +#define curthread get_curthread() + +#define PCPU_GET(member) (get_pcpu()->pc_ ## member) +#define PCPU_ADD(member, value) (get_pcpu()->pc_ ## member += (value)) +#define PCPU_INC(member) PCPU_ADD(member, 1) +#define PCPU_PTR(member) (&get_pcpu()->pc_ ## member) +#define PCPU_SET(member,value) (get_pcpu()->pc_ ## member = (value)) + +#endif /* _KERNEL */ + +#endif /* !_MACHINE_PCPU_H_ */ diff --git a/sys/riscv/include/pmap.h b/sys/riscv/include/pmap.h new file mode 100644 index 0000000..e109de2 --- /dev/null +++ b/sys/riscv/include/pmap.h @@ -0,0 +1,160 @@ +/*- + * Copyright (c) 1991 Regents of the University of California. + * All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * the Systems Programming Group of the University of Utah Computer + * Science Department and William Jolitz of UUNET Technologies Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _MACHINE_PMAP_H_ +#define _MACHINE_PMAP_H_ + +#include <machine/pte.h> + +#ifndef LOCORE + +#include <sys/queue.h> +#include <sys/_lock.h> +#include <sys/_mutex.h> + +#ifdef _KERNEL + +#define vtophys(va) pmap_kextract((vm_offset_t)(va)) + +#endif + +#define pmap_page_get_memattr(m) ((m)->md.pv_memattr) +#define pmap_page_is_write_mapped(m) (((m)->aflags & PGA_WRITEABLE) != 0) +void pmap_page_set_memattr(vm_page_t m, vm_memattr_t ma); + +/* + * Pmap stuff + */ + +struct md_page { + TAILQ_HEAD(,pv_entry) pv_list; + int pv_gen; + vm_memattr_t pv_memattr; +}; + +/* + * This structure is used to hold a virtual<->physical address + * association and is used mostly by bootstrap code + */ +struct pv_addr { + SLIST_ENTRY(pv_addr) pv_list; + vm_offset_t pv_va; + vm_paddr_t pv_pa; +}; + + +struct pmap { + struct mtx pm_mtx; + struct pmap_statistics pm_stats; /* pmap statictics */ + pd_entry_t *pm_l1; + TAILQ_HEAD(,pv_chunk) pm_pvchunk; /* list of mappings in pmap */ +}; + +typedef struct pv_entry { + vm_offset_t pv_va; /* virtual address for mapping */ + TAILQ_ENTRY(pv_entry) pv_next; +} *pv_entry_t; + +/* + * pv_entries are allocated in chunks per-process. This avoids the + * need to track per-pmap assignments. + */ +#define _NPCM 3 +#define _NPCPV 168 +struct pv_chunk { + struct pmap * pc_pmap; + TAILQ_ENTRY(pv_chunk) pc_list; + uint64_t pc_map[_NPCM]; /* bitmap; 1 = free */ + TAILQ_ENTRY(pv_chunk) pc_lru; + struct pv_entry pc_pventry[_NPCPV]; +}; + +typedef struct pmap *pmap_t; + +#ifdef _KERNEL +extern struct pmap kernel_pmap_store; +#define kernel_pmap (&kernel_pmap_store) +#define pmap_kernel() kernel_pmap + +#define PMAP_ASSERT_LOCKED(pmap) \ + mtx_assert(&(pmap)->pm_mtx, MA_OWNED) +#define PMAP_LOCK(pmap) mtx_lock(&(pmap)->pm_mtx) +#define PMAP_LOCK_ASSERT(pmap, type) \ + mtx_assert(&(pmap)->pm_mtx, (type)) +#define PMAP_LOCK_DESTROY(pmap) mtx_destroy(&(pmap)->pm_mtx) +#define PMAP_LOCK_INIT(pmap) mtx_init(&(pmap)->pm_mtx, "pmap", \ + NULL, MTX_DEF | MTX_DUPOK) +#define PMAP_OWNED(pmap) mtx_owned(&(pmap)->pm_mtx) +#define PMAP_MTX(pmap) (&(pmap)->pm_mtx) +#define PMAP_TRYLOCK(pmap) mtx_trylock(&(pmap)->pm_mtx) +#define PMAP_UNLOCK(pmap) mtx_unlock(&(pmap)->pm_mtx) + +#define PHYS_AVAIL_SIZE 10 +extern vm_paddr_t phys_avail[]; +extern vm_paddr_t dump_avail[]; +extern vm_offset_t virtual_avail; +extern vm_offset_t virtual_end; + +/* + * Macros to test if a mapping is mappable with an L1 Section mapping + * or an L2 Large Page mapping. + */ +#define L1_MAPPABLE_P(va, pa, size) \ + ((((va) | (pa)) & L1_OFFSET) == 0 && (size) >= L1_SIZE) + +void pmap_bootstrap(vm_offset_t, vm_paddr_t, vm_size_t); +void pmap_kenter_device(vm_offset_t, vm_size_t, vm_paddr_t); +vm_paddr_t pmap_kextract(vm_offset_t va); +void pmap_kremove(vm_offset_t); +void pmap_kremove_device(vm_offset_t, vm_size_t); + +void *pmap_mapdev(vm_offset_t, vm_size_t); +void *pmap_mapbios(vm_paddr_t, vm_size_t); +void pmap_unmapdev(vm_offset_t, vm_size_t); +void pmap_unmapbios(vm_offset_t, vm_size_t); + +boolean_t pmap_map_io_transient(vm_page_t *, vm_offset_t *, int, boolean_t); +void pmap_unmap_io_transient(vm_page_t *, vm_offset_t *, int, boolean_t); + +bool pmap_get_tables(pmap_t, vm_offset_t, pd_entry_t **, pd_entry_t **, + pt_entry_t **); + +#define pmap_page_is_mapped(m) (!TAILQ_EMPTY(&(m)->md.pv_list)) + +#endif /* _KERNEL */ + +#endif /* !LOCORE */ + +#endif /* !_MACHINE_PMAP_H_ */ diff --git a/sys/riscv/include/pmc_mdep.h b/sys/riscv/include/pmc_mdep.h new file mode 100644 index 0000000..10dc5c8 --- /dev/null +++ b/sys/riscv/include/pmc_mdep.h @@ -0,0 +1,65 @@ +/*- + * Copyright (c) 2009 Rui Paulo <rpaulo@FreeBSD.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _MACHINE_PMC_MDEP_H_ +#define _MACHINE_PMC_MDEP_H_ + +#define PMC_MDEP_CLASS_INDEX_RISCV 1 +/* + * On the RISC-V platform we don't support any PMCs yet. + */ +#include <dev/hwpmc/hwpmc_riscv.h> + +union pmc_md_op_pmcallocate { + uint64_t __pad[4]; +}; + +/* Logging */ +#define PMCLOG_READADDR PMCLOG_READ64 +#define PMCLOG_EMITADDR PMCLOG_EMIT64 + +#ifdef _KERNEL +union pmc_md_pmc { + struct pmc_md_riscv_pmc pm_riscv; +}; + +#define PMC_IN_KERNEL_STACK(S,START,END) \ + ((S) >= (START) && (S) < (END)) +#define PMC_IN_KERNEL(va) INKERNEL((va)) +#define PMC_IN_USERSPACE(va) ((va) <= VM_MAXUSER_ADDRESS) +#define PMC_TRAPFRAME_TO_PC(TF) ((TF)->tf_ra) +#define PMC_TRAPFRAME_TO_FP(TF) (0) /* stub */ + +/* + * Prototypes + */ +struct pmc_mdep *pmc_riscv_initialize(void); +void pmc_riscv_finalize(struct pmc_mdep *_md); +#endif /* _KERNEL */ + +#endif /* !_MACHINE_PMC_MDEP_H_ */ diff --git a/sys/riscv/include/proc.h b/sys/riscv/include/proc.h new file mode 100644 index 0000000..6732681 --- /dev/null +++ b/sys/riscv/include/proc.h @@ -0,0 +1,60 @@ +/*- + * Copyright (c) 1991 Regents of the University of California. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * from: @(#)proc.h 7.1 (Berkeley) 5/15/91 + * from: FreeBSD: src/sys/i386/include/proc.h,v 1.11 2001/06/29 + * $FreeBSD$ + */ + +#ifndef _MACHINE_PROC_H_ +#define _MACHINE_PROC_H_ + +struct mdthread { + int md_spinlock_count; /* (k) */ + register_t md_saved_sstatus_ie; /* (k) */ +}; + +struct mdproc { + int dummy; +}; + +#define KINFO_PROC_SIZE 1088 + +#ifdef _KERNEL + +#define MAXARGS 8 +struct syscall_args { + u_int code; + struct sysent *callp; + register_t args[MAXARGS]; + int narg; +}; + +#endif + +#endif /* !_MACHINE_PROC_H_ */ diff --git a/sys/riscv/include/profile.h b/sys/riscv/include/profile.h new file mode 100644 index 0000000..c2ec9b7 --- /dev/null +++ b/sys/riscv/include/profile.h @@ -0,0 +1,78 @@ +/*- + * Copyright (c) 1994, 1995, 1996 Carnegie-Mellon University. + * All rights reserved. + * + * Author: Chris G. Demetriou + * + * Permission to use, copy, modify and distribute this software and + * its documentation is hereby granted, provided that both the copyright + * notice and this permission notice appear in all copies of the + * software, derivative works or modified versions, and any portions + * thereof, and that both notices appear in supporting documentation. + * + * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" + * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND + * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. + * + * Carnegie Mellon requests users of this software to return to + * + * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU + * School of Computer Science + * Carnegie Mellon University + * Pittsburgh PA 15213-3890 + * + * any improvements or extensions that they make and grant Carnegie the + * rights to redistribute these changes. + * + * from: NetBSD: profile.h,v 1.9 1997/04/06 08:47:37 cgd Exp + * from: FreeBSD: src/sys/alpha/include/profile.h,v 1.4 1999/12/29 + * $FreeBSD$ + */ + +#ifndef _MACHINE_PROFILE_H_ +#define _MACHINE_PROFILE_H_ + +#if !defined(_KERNEL) && !defined(_SYS_CDEFS_H_) +#error this file needs sys/cdefs.h as a prerequisite +#endif + +#define FUNCTION_ALIGNMENT 32 + +typedef u_long fptrdiff_t; + +#ifdef _KERNEL + +#include <machine/cpufunc.h> + +#define _MCOUNT_DECL void mcount +#define MCOUNT + +#define MCOUNT_DECL(s) register_t s; +#define MCOUNT_ENTER(s) {s = intr_disable(); } +#define MCOUNT_EXIT(s) {intr_restore(s); } + +void bintr(void); +void btrap(void); +void eintr(void); +void user(void); + +#define MCOUNT_FROMPC_USER(pc) \ + ((pc < (uintfptr_t)VM_MAXUSER_ADDRESS) ? (uintfptr_t)user : pc) + +#define MCOUNT_FROMPC_INTR(pc) \ + ((pc >= (uintfptr_t)btrap && pc < (uintfptr_t)eintr) ? \ + ((pc >= (uintfptr_t)bintr) ? (uintfptr_t)bintr : \ + (uintfptr_t)btrap) : ~0UL) + +void mcount(uintfptr_t frompc, uintfptr_t selfpc); + +#else /* !_KERNEL */ + +typedef unsigned long long uintfptr_t; + +#define _MCOUNT_DECL void mcount +#define MCOUNT + +#endif /* _KERNEL */ + +#endif /* !_MACHINE_PROFILE_H_ */ diff --git a/sys/riscv/include/psl.h b/sys/riscv/include/psl.h new file mode 100644 index 0000000..da23dbe --- /dev/null +++ b/sys/riscv/include/psl.h @@ -0,0 +1 @@ +/* $FreeBSD$ */ diff --git a/sys/riscv/include/pte.h b/sys/riscv/include/pte.h new file mode 100644 index 0000000..1331779 --- /dev/null +++ b/sys/riscv/include/pte.h @@ -0,0 +1,100 @@ +/*- + * Copyright (c) 2014 Andrew Turner + * Copyright (c) 2015 Ruslan Bukin <br@bsdpad.com> + * All rights reserved. + * + * Portions of this software were developed by SRI International and the + * University of Cambridge Computer Laboratory under DARPA/AFRL contract + * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. + * + * Portions of this software were developed by the University of Cambridge + * Computer Laboratory as part of the CTSRD Project, with support from the + * UK Higher Education Innovation Fund (HEIF). + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _MACHINE_PTE_H_ +#define _MACHINE_PTE_H_ + +#ifndef LOCORE +typedef uint64_t pd_entry_t; /* page directory entry */ +typedef uint64_t pt_entry_t; /* page table entry */ +#endif + +/* Level 0 table, 512GiB per entry */ +#define L0_SHIFT 39 + +/* Level 1 table, 1GiB per entry */ +#define L1_SHIFT 30 +#define L1_SIZE (1 << L1_SHIFT) +#define L1_OFFSET (L1_SIZE - 1) + +/* Level 2 table, 2MiB per entry */ +#define L2_SHIFT 21 +#define L2_SIZE (1 << L2_SHIFT) +#define L2_OFFSET (L2_SIZE - 1) + +/* Level 3 table, 4KiB per entry */ +#define L3_SHIFT 12 +#define L3_SIZE (1 << L3_SHIFT) +#define L3_OFFSET (L3_SIZE - 1) + +#define Ln_ENTRIES (1 << 9) +#define Ln_ADDR_MASK (Ln_ENTRIES - 1) + +/* Bits 9:7 are reserved for software */ +#define PTE_SW_MANAGED (1 << 8) +#define PTE_SW_WIRED (1 << 7) +#define PTE_DIRTY (1 << 6) /* Virtual page is written */ +#define PTE_REF (1 << 5) /* Virtual page is referenced */ +#define PTE_VALID (1 << 0) /* Virtual page is valid */ +#define PTE_TYPE_S 1 +#define PTE_TYPE_M (0xf << PTE_TYPE_S) +#define PTE_TYPE_PTR 0 +#define PTE_TYPE_PTR_G 1 +#define PTE_TYPE_SROURX 2 /* Supervisor read-only, user read-execute page. */ +#define PTE_TYPE_SRWURWX 3 /* Supervisor read-write, user read-write-execute page. */ +#define PTE_TYPE_SURO 4 /* Supervisor and user read-only page. */ +#define PTE_TYPE_SURW 5 /* Supervisor and user read-write page. */ +#define PTE_TYPE_SURX 6 /* Supervisor and user read-execute page. */ +#define PTE_TYPE_SURWX 7 /* Supervisor and User Read Write Execute */ +#define PTE_TYPE_SRO 8 /* Supervisor read-only page. */ +#define PTE_TYPE_SRW 9 /* Supervisor read-write page. */ +#define PTE_TYPE_SRX 10 /* Supervisor read-execute page. */ +#define PTE_TYPE_SRWX 11 /* Supervisor read-write-execute page. */ +#define PTE_TYPE_SRO_G 12 /* Supervisor read-only page--global mapping. */ +#define PTE_TYPE_SRW_G 13 /* Supervisor read-write page--global mapping. */ +#define PTE_TYPE_SRX_G 14 /* Supervisor read-execute page--global mapping. */ +#define PTE_TYPE_SRWX_G 15 /* Supervisor Read Write Execute Global */ + +#define PTE_PPN0_S 10 +#define PTE_PPN1_S 19 +#define PTE_PPN2_S 28 +#define PTE_PPN3_S 37 +#define PTE_SIZE 8 + +#endif /* !_MACHINE_PTE_H_ */ + +/* End of pte.h */ diff --git a/sys/riscv/include/ptrace.h b/sys/riscv/include/ptrace.h new file mode 100644 index 0000000..da23dbe --- /dev/null +++ b/sys/riscv/include/ptrace.h @@ -0,0 +1 @@ +/* $FreeBSD$ */ diff --git a/sys/riscv/include/reg.h b/sys/riscv/include/reg.h new file mode 100644 index 0000000..35d79f1 --- /dev/null +++ b/sys/riscv/include/reg.h @@ -0,0 +1,72 @@ +/*- + * Copyright (c) 2015 Ruslan Bukin <br@bsdpad.com> + * All rights reserved. + * + * Portions of this software were developed by SRI International and the + * University of Cambridge Computer Laboratory under DARPA/AFRL contract + * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. + * + * Portions of this software were developed by the University of Cambridge + * Computer Laboratory as part of the CTSRD Project, with support from the + * UK Higher Education Innovation Fund (HEIF). + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _MACHINE_REG_H_ +#define _MACHINE_REG_H_ + +struct reg { + uint64_t ra; /* return address */ + uint64_t sp; /* stack pointer */ + uint64_t gp; /* global pointer */ + uint64_t tp; /* thread pointer */ + uint64_t t[7]; /* temporaries */ + uint64_t s[12]; /* saved registers */ + uint64_t a[8]; /* function arguments */ + uint64_t sepc; /* exception program counter */ + uint64_t sstatus; /* status register */ +}; + +struct fpreg { + int dummy; +}; + +struct dbreg { + int dummy; +}; + +#ifdef _KERNEL +/* + * XXX these interfaces are MI, so they should be declared in a MI place. + */ +int fill_regs(struct thread *, struct reg *); +int set_regs(struct thread *, struct reg *); +int fill_fpregs(struct thread *, struct fpreg *); +int set_fpregs(struct thread *, struct fpreg *); +int fill_dbregs(struct thread *, struct dbreg *); +int set_dbregs(struct thread *, struct dbreg *); +#endif + +#endif /* !_MACHINE_REG_H_ */ diff --git a/sys/riscv/include/reloc.h b/sys/riscv/include/reloc.h new file mode 100644 index 0000000..da23dbe --- /dev/null +++ b/sys/riscv/include/reloc.h @@ -0,0 +1 @@ +/* $FreeBSD$ */ diff --git a/sys/riscv/include/resource.h b/sys/riscv/include/resource.h new file mode 100644 index 0000000..723d145 --- /dev/null +++ b/sys/riscv/include/resource.h @@ -0,0 +1,46 @@ +/*- + * Copyright 1998 Massachusetts Institute of Technology + * + * Permission to use, copy, modify, and distribute this software and + * its documentation for any purpose and without fee is hereby + * granted, provided that both the above copyright notice and this + * permission notice appear in all copies, that both the above + * copyright notice and this permission notice appear in all + * supporting documentation, and that the name of M.I.T. not be used + * in advertising or publicity pertaining to distribution of the + * software without specific, written prior permission. M.I.T. makes + * no representations about the suitability of this software for any + * purpose. It is provided "as is" without express or implied + * warranty. + * + * THIS SOFTWARE IS PROVIDED BY M.I.T. ``AS IS''. M.I.T. DISCLAIMS + * ALL EXPRESS OR IMPLIED WARRANTIES WITH REGARD TO THIS SOFTWARE, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. IN NO EVENT + * SHALL M.I.T. BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF + * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _MACHINE_RESOURCE_H_ +#define _MACHINE_RESOURCE_H_ 1 + +/* + * Definitions of resource types for Intel Architecture machines + * with support for legacy ISA devices and drivers. + */ + +#define SYS_RES_IRQ 1 /* interrupt lines */ +#define SYS_RES_DRQ 2 /* isa dma lines */ +#define SYS_RES_MEMORY 3 /* i/o memory */ +#define SYS_RES_IOPORT 4 /* i/o ports */ +#define SYS_RES_GPIO 5 /* general purpose i/o */ + +#endif /* !_MACHINE_RESOURCE_H_ */ diff --git a/sys/riscv/include/riscvreg.h b/sys/riscv/include/riscvreg.h new file mode 100644 index 0000000..7b08085 --- /dev/null +++ b/sys/riscv/include/riscvreg.h @@ -0,0 +1,153 @@ +/*- + * Copyright (c) 2015 Ruslan Bukin <br@bsdpad.com> + * All rights reserved. + * + * Portions of this software were developed by SRI International and the + * University of Cambridge Computer Laboratory under DARPA/AFRL contract + * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. + * + * Portions of this software were developed by the University of Cambridge + * Computer Laboratory as part of the CTSRD Project, with support from the + * UK Higher Education Innovation Fund (HEIF). + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _MACHINE_RISCVREG_H_ +#define _MACHINE_RISCVREG_H_ + +/* Machine mode requests */ +#define ECALL_MTIMECMP 0x1 +#define ECALL_CLEAR_PENDING 0x2 +#define ECALL_HTIF_CMD 0x3 +#define ECALL_HTIF_GET_ENTRY 0x4 +#define ECALL_MCPUID_GET 0x5 +#define ECALL_MIMPID_GET 0x6 + +#define EXCP_SHIFT 0 +#define EXCP_MASK (0xf << EXCP_SHIFT) +#define EXCP_INSTR_ADDR_MISALIGNED 0 +#define EXCP_INSTR_ACCESS_FAULT 1 +#define EXCP_INSTR_ILLEGAL 2 +#define EXCP_INSTR_BREAKPOINT 3 +#define EXCP_RESERVED_0 4 +#define EXCP_LOAD_ACCESS_FAULT 5 +#define EXCP_AMO_ADDR_MISALIGNED 6 +#define EXCP_STORE_ACCESS_FAULT 7 +#define EXCP_ENV_CALL 8 +#define EXCP_RESERVED_1 9 +#define EXCP_INTR (1 << 31) +#define EXCP_INTR_SOFTWARE 0 +#define EXCP_INTR_TIMER 1 +#define EXCP_INTR_HTIF 2 + +#define SSTATUS_IE (1 << 0) +#define SSTATUS_PIE (1 << 3) +#define SSTATUS_PS (1 << 4) + +#define MSTATUS_MPRV (1 << 16) +#define MSTATUS_PRV_SHIFT 1 +#define MSTATUS_PRV1_SHIFT 4 +#define MSTATUS_PRV2_SHIFT 7 +#define MSTATUS_PRV_MASK (0x3 << MSTATUS_PRV_SHIFT) +#define MSTATUS_PRV_U 0 /* user */ +#define MSTATUS_PRV_S 1 /* supervisor */ +#define MSTATUS_PRV_H 2 /* hypervisor */ +#define MSTATUS_PRV_M 3 /* machine */ + +#define MSTATUS_VM_SHIFT 17 +#define MSTATUS_VM_MASK 0x1f +#define MSTATUS_VM_MBARE 0 +#define MSTATUS_VM_MBB 1 +#define MSTATUS_VM_MBBID 2 +#define MSTATUS_VM_SV32 8 +#define MSTATUS_VM_SV39 9 +#define MSTATUS_VM_SV48 10 + +#define MIE_SSIE (1 << 1) +#define MIE_HSIE (1 << 2) +#define MIE_MSIE (1 << 3) +#define MIE_STIE (1 << 5) +#define MIE_HTIE (1 << 6) +#define MIE_MTIE (1 << 7) + +#define MIP_SSIP (1 << 1) +#define MIP_HSIP (1 << 2) +#define MIP_MSIP (1 << 3) +#define MIP_STIP (1 << 5) +#define MIP_HTIP (1 << 6) +#define MIP_MTIP (1 << 7) + +#define SR_IE (1 << 0) +#define SR_IE1 (1 << 3) +#define SR_IE2 (1 << 6) +#define SR_IE3 (1 << 9) + +#define SIE_SSIE (1 << 1) +#define SIE_STIE (1 << 5) + +/* Note: sip register is not yet implement in Spike simulator */ +#define SIP_STIP (1 << 5) + +#define CSR_ZIMM(val) \ + (__builtin_constant_p(val) && ((u_long)(val) < 32)) + +#define csr_swap(csr, val) \ +({ if (CSR_ZIMM(val)) \ + __asm __volatile("csrrwi %0, " #csr ", %1" \ + : "=r" (val) : "i" (val)); \ + else \ + __asm __volatile("csrrw %0, " #csr ", %1" \ + : "=r" (val) : "r" (val)); \ + val; \ +}) + +#define csr_write(csr, val) \ +({ if (CSR_ZIMM(val)) \ + __asm __volatile("csrwi " #csr ", %0" :: "i" (val)); \ + else \ + __asm __volatile("csrw " #csr ", %0" :: "r" (val)); \ +}) + +#define csr_set(csr, val) \ +({ if (CSR_ZIMM(val)) \ + __asm __volatile("csrsi " #csr ", %0" :: "i" (val)); \ + else \ + __asm __volatile("csrs " #csr ", %0" :: "r" (val)); \ +}) + +#define csr_clear(csr, val) \ +({ if (CSR_ZIMM(val)) \ + __asm __volatile("csrci " #csr ", %0" :: "i" (val)); \ + else \ + __asm __volatile("csrc " #csr ", %0" :: "r" (val)); \ +}) + +#define csr_read(csr) \ +({ u_long val; \ + __asm __volatile("csrr %0, " #csr : "=r" (val)); \ + val; \ +}) + +#endif /* !_MACHINE_RISCVREG_H_ */ diff --git a/sys/riscv/include/runq.h b/sys/riscv/include/runq.h new file mode 100644 index 0000000..eaeb824 --- /dev/null +++ b/sys/riscv/include/runq.h @@ -0,0 +1,46 @@ +/*- + * Copyright (c) 2001 Jake Burkholder <jake@FreeBSD.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _MACHINE_RUNQ_H_ +#define _MACHINE_RUNQ_H_ + +#define RQB_LEN (1) /* Number of priority status words. */ +#define RQB_L2BPW (6) /* Log2(sizeof(rqb_word_t) * NBBY)). */ +#define RQB_BPW (1<<RQB_L2BPW) /* Bits in an rqb_word_t. */ + +#define RQB_BIT(pri) (1ul << ((pri) & (RQB_BPW - 1))) +#define RQB_WORD(pri) ((pri) >> RQB_L2BPW) + +#define RQB_FFS(word) (ffsl(word) - 1) + +/* + * Type of run queue status word. + */ +typedef unsigned long rqb_word_t; + +#endif diff --git a/sys/riscv/include/setjmp.h b/sys/riscv/include/setjmp.h new file mode 100644 index 0000000..0d6c87e --- /dev/null +++ b/sys/riscv/include/setjmp.h @@ -0,0 +1,63 @@ +/*- + * Copyright (c) 2015 Ruslan Bukin <br@bsdpad.com> + * All rights reserved. + * + * Portions of this software were developed by SRI International and the + * University of Cambridge Computer Laboratory under DARPA/AFRL contract + * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. + * + * Portions of this software were developed by the University of Cambridge + * Computer Laboratory as part of the CTSRD Project, with support from the + * UK Higher Education Innovation Fund (HEIF). + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _MACHINE_SETJMP_H_ +#define _MACHINE_SETJMP_H_ + +#include <sys/cdefs.h> + +#define _JBLEN 32 /* sp, ra, [f]s0-11, magic val, sigmask */ +#define _JB_SIGMASK 21 + +#ifdef __ASSEMBLER__ +#define _JB_MAGIC__SETJMP 0xbe87fd8a2910af00 +#define _JB_MAGIC_SETJMP 0xbe87fd8a2910af01 +#endif /* !__ASSEMBLER__ */ + +#ifndef __ASSEMBLER__ +/* + * jmp_buf and sigjmp_buf are encapsulated in different structs to force + * compile-time diagnostics for mismatches. The structs are the same + * internally to avoid some run-time errors for mismatches. + */ +#if __BSD_VISIBLE || __POSIX_VISIBLE || __XSI_VISIBLE +typedef struct _sigjmp_buf { long _sjb[_JBLEN + 1]; } sigjmp_buf[1]; +#endif + +typedef struct _jmp_buf { long _jb[_JBLEN + 1]; } jmp_buf[1]; +#endif /* __ASSEMBLER__ */ + +#endif /* !_MACHINE_SETJMP_H_ */ diff --git a/sys/riscv/include/sf_buf.h b/sys/riscv/include/sf_buf.h new file mode 100644 index 0000000..59f9009 --- /dev/null +++ b/sys/riscv/include/sf_buf.h @@ -0,0 +1,51 @@ +/*- + * Copyright (c) 2003, 2005 Alan L. Cox <alc@cs.rice.edu> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _MACHINE_SF_BUF_H_ +#define _MACHINE_SF_BUF_H_ + +/* + * On this machine, the only purpose for which sf_buf is used is to implement + * an opaque pointer required by the machine-independent parts of the kernel. + * That pointer references the vm_page that is "mapped" by the sf_buf. The + * actual mapping is provided by the direct virtual-to-physical mapping. + */ +static inline vm_offset_t +sf_buf_kva(struct sf_buf *sf) +{ + + return (PHYS_TO_DMAP(VM_PAGE_TO_PHYS((vm_page_t)sf))); +} + +static inline vm_page_t +sf_buf_page(struct sf_buf *sf) +{ + + return ((vm_page_t)sf); +} +#endif /* !_MACHINE_SF_BUF_H_ */ diff --git a/sys/riscv/include/signal.h b/sys/riscv/include/signal.h new file mode 100644 index 0000000..6c8ac5c --- /dev/null +++ b/sys/riscv/include/signal.h @@ -0,0 +1,50 @@ +/*- + * Copyright (c) 1986, 1989, 1991, 1993 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)signal.h 8.1 (Berkeley) 6/11/93 + * from: FreeBSD: src/sys/i386/include/signal.h,v 1.13 2000/11/09 + * from: FreeBSD: src/sys/sparc64/include/signal.h,v 1.6 2001/09/30 18:52:17 + * $FreeBSD$ + */ + +#ifndef _MACHINE_SIGNAL_H_ +#define _MACHINE_SIGNAL_H_ + +#include <sys/cdefs.h> + +typedef long sig_atomic_t; + +#if __BSD_VISIBLE + +struct sigcontext { + int _dummy; +}; + +#endif + +#endif /* !_MACHINE_SIGNAL_H_ */ diff --git a/sys/riscv/include/smp.h b/sys/riscv/include/smp.h new file mode 100644 index 0000000..da23dbe --- /dev/null +++ b/sys/riscv/include/smp.h @@ -0,0 +1 @@ +/* $FreeBSD$ */ diff --git a/sys/riscv/include/stdarg.h b/sys/riscv/include/stdarg.h new file mode 100644 index 0000000..19a375d --- /dev/null +++ b/sys/riscv/include/stdarg.h @@ -0,0 +1,79 @@ +/*- + * Copyright (c) 2002 David E. O'Brien. All rights reserved. + * Copyright (c) 1991, 1993 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * @(#)stdarg.h 8.1 (Berkeley) 6/10/93 + * $FreeBSD$ + */ + +#ifndef _MACHINE_STDARG_H_ +#define _MACHINE_STDARG_H_ + +#include <sys/cdefs.h> +#include <sys/_types.h> + +#ifndef _VA_LIST_DECLARED +#define _VA_LIST_DECLARED +typedef __va_list va_list; +#endif + +#ifdef __GNUCLIKE_BUILTIN_STDARG + +#define va_start(ap, last) \ + __builtin_va_start((ap), (last)) + +#define va_arg(ap, type) \ + __builtin_va_arg((ap), type) + +#if __ISO_C_VISIBLE >= 1999 +#define va_copy(dest, src) \ + __builtin_va_copy((dest), (src)) +#endif + +#define va_end(ap) \ + __builtin_va_end(ap) + +#elif defined(lint) +/* Provide a fake implementation for lint's benefit */ +#define __va_size(type) \ + (((sizeof(type) + sizeof(long) - 1) / sizeof(long)) * sizeof(long)) +#define va_start(ap, last) \ + ((ap) = (va_list)&(last) + __va_size(last)) +#define va_copy(dst, src) \ + ((dst) = (src)) +#define va_arg(ap, type) \ + (*(type *)((ap) += __va_size(type), (ap) - __va_size(type))) +#define va_end(ap) + +#else /* !__GNUCLIKE_BUILTIN_STDARG */ + +#error no support for your compiler + +#endif /* __GNUCLIKE_BUILTIN_STDARG */ + +#endif /* !_MACHINE_STDARG_H_ */ diff --git a/sys/riscv/include/sysarch.h b/sys/riscv/include/sysarch.h new file mode 100644 index 0000000..e957497 --- /dev/null +++ b/sys/riscv/include/sysarch.h @@ -0,0 +1,47 @@ +/*- + * Copyright (c) 1993 The Regents of the University of California. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * from: FreeBSD: src/sys/i386/include/sysarch.h,v 1.14 2000/09/21 + * $FreeBSD$ + */ + +/* + * Architecture specific syscalls (RISC-V) + */ +#ifndef _MACHINE_SYSARCH_H_ +#define _MACHINE_SYSARCH_H_ + +#ifndef _KERNEL + +__BEGIN_DECLS +int sysarch(int _number, void *_args); +__END_DECLS + +#endif + +#endif /* !_MACHINE_SYSARCH_H_ */ diff --git a/sys/riscv/include/trap.h b/sys/riscv/include/trap.h new file mode 100644 index 0000000..da23dbe --- /dev/null +++ b/sys/riscv/include/trap.h @@ -0,0 +1 @@ +/* $FreeBSD$ */ diff --git a/sys/riscv/include/ucontext.h b/sys/riscv/include/ucontext.h new file mode 100644 index 0000000..4fbf346 --- /dev/null +++ b/sys/riscv/include/ucontext.h @@ -0,0 +1,70 @@ +/*- + * Copyright (c) 2015 Ruslan Bukin <br@bsdpad.com> + * All rights reserved. + * + * Portions of this software were developed by SRI International and the + * University of Cambridge Computer Laboratory under DARPA/AFRL contract + * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. + * + * Portions of this software were developed by the University of Cambridge + * Computer Laboratory as part of the CTSRD Project, with support from the + * UK Higher Education Innovation Fund (HEIF). + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _MACHINE_UCONTEXT_H_ +#define _MACHINE_UCONTEXT_H_ + +struct gpregs { + __register_t gp_ra; + __register_t gp_sp; + __register_t gp_gp; + __register_t gp_tp; + __register_t gp_t[7]; + __register_t gp_s[12]; + __register_t gp_a[8]; + __register_t gp_sepc; + __register_t gp_sstatus; +}; + +struct fpregs { + __uint128_t fp_x[32]; + __uint64_t fp_fcsr; + int fp_flags; + int pad; +}; + +struct __mcontext { + struct gpregs mc_gpregs; + struct fpregs mc_fpregs; + int mc_flags; +#define _MC_FP_VALID 0x1 /* Set when mc_fpregs has valid data */ + int mc_pad; + __uint64_t mc_spare[8]; /* Space for expansion */ +}; + +typedef struct __mcontext mcontext_t; + +#endif /* !_MACHINE_UCONTEXT_H_ */ diff --git a/sys/riscv/include/vdso.h b/sys/riscv/include/vdso.h new file mode 100644 index 0000000..5a8f7f5 --- /dev/null +++ b/sys/riscv/include/vdso.h @@ -0,0 +1,34 @@ +/*- + * Copyright 2012 Konstantin Belousov <kib@FreeBSD.ORG>. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _MACHINE_VDSO_H_ +#define _MACHINE_VDSO_H_ + +#define VDSO_TIMEHANDS_MD \ + uint32_t th_res[8]; + +#endif /* !_MACHINE_VDSO_H_ */ diff --git a/sys/riscv/include/vm.h b/sys/riscv/include/vm.h new file mode 100644 index 0000000..182313e --- /dev/null +++ b/sys/riscv/include/vm.h @@ -0,0 +1,39 @@ +/*- + * Copyright (c) 2009 Alan L. Cox <alc@cs.rice.edu> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _MACHINE_VM_H_ +#define _MACHINE_VM_H_ + +/* Memory attribute configuration. */ +#define VM_MEMATTR_DEVICE 0 +#define VM_MEMATTR_UNCACHEABLE 1 +#define VM_MEMATTR_WRITE_BACK 2 + +#define VM_MEMATTR_DEFAULT VM_MEMATTR_WRITE_BACK + +#endif /* !_MACHINE_VM_H_ */ diff --git a/sys/riscv/include/vmparam.h b/sys/riscv/include/vmparam.h new file mode 100644 index 0000000..08d3c3b --- /dev/null +++ b/sys/riscv/include/vmparam.h @@ -0,0 +1,239 @@ +/*- + * Copyright (c) 1990 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 1994 John S. Dyson + * All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * William Jolitz. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * from: @(#)vmparam.h 5.9 (Berkeley) 5/12/91 + * from: FreeBSD: src/sys/i386/include/vmparam.h,v 1.33 2000/03/30 + * $FreeBSD$ + */ + +#ifndef _MACHINE_VMPARAM_H_ +#define _MACHINE_VMPARAM_H_ + +/* + * Virtual memory related constants, all in bytes + */ +#ifndef MAXTSIZ +#define MAXTSIZ (32*1024*1024) /* max text size */ +#endif +#ifndef DFLDSIZ +#define DFLDSIZ (128*1024*1024) /* initial data size limit */ +#endif +#ifndef MAXDSIZ +#define MAXDSIZ (128*1024*1024) /* max data size */ +#endif +#ifndef DFLSSIZ +#define DFLSSIZ (2*1024*1024) /* initial stack size limit */ +#endif +#ifndef MAXSSIZ +#define MAXSSIZ (8*1024*1024) /* max stack size */ +#endif +#ifndef SGROWSIZ +#define SGROWSIZ (128*1024) /* amount to grow stack */ +#endif + +/* + * The physical address space is sparsely populated. + */ +#define VM_PHYSSEG_SPARSE + +/* + * The number of PHYSSEG entries must be one greater than the number + * of phys_avail entries because the phys_avail entry that spans the + * largest physical address that is accessible by ISA DMA is split + * into two PHYSSEG entries. + */ +#define VM_PHYSSEG_MAX 64 + +/* + * Create two free page pools: VM_FREEPOOL_DEFAULT is the default pool + * from which physical pages are allocated and VM_FREEPOOL_DIRECT is + * the pool from which physical pages for small UMA objects are + * allocated. + */ +#define VM_NFREEPOOL 2 +#define VM_FREEPOOL_DEFAULT 0 +#define VM_FREEPOOL_DIRECT 1 + +/* + * Create two free page lists: VM_FREELIST_DEFAULT is for physical + * pages that are above the largest physical address that is + * accessible by ISA DMA and VM_FREELIST_ISADMA is for physical pages + * that are below that address. + */ +#define VM_NFREELIST 2 +#define VM_FREELIST_DEFAULT 0 +#define VM_FREELIST_ISADMA 1 + +/* + * An allocation size of 16MB is supported in order to optimize the + * use of the direct map by UMA. Specifically, a cache line contains + * at most four TTEs, collectively mapping 16MB of physical memory. + * By reducing the number of distinct 16MB "pages" that are used by UMA, + * the physical memory allocator reduces the likelihood of both 4MB + * page TLB misses and cache misses caused by 4MB page TLB misses. + */ +#define VM_NFREEORDER 12 + +/* + * Enable superpage reservations: 1 level. + */ +#ifndef VM_NRESERVLEVEL +#define VM_NRESERVLEVEL 1 +#endif + +/* + * Level 0 reservations consist of 512 pages. + */ +#ifndef VM_LEVEL_0_ORDER +#define VM_LEVEL_0_ORDER 9 +#endif + +/** + * Address space layout. + * + * RISC-V implements up to a 48 bit virtual address space. The address space is + * split into 2 regions at each end of the 64 bit address space, with an + * out of range "hole" in the middle. + * + * We limit the size of the two spaces to 39 bits each. + * + * Upper region: 0xffffffffffffffff + * 0xffffffffc0000000 + * + * Hole: 0xffffffffbfffffff + * 0x0000000080000000 + * + * Lower region: 0x000000007fffffff + * 0x0000000000000000 + * + * We use the upper region for the kernel, and the lower region for userland. + * + * We define some interesting address constants: + * + * VM_MIN_ADDRESS and VM_MAX_ADDRESS define the start and end of the entire + * 64 bit address space, mostly just for convenience. + * + * VM_MIN_KERNEL_ADDRESS and VM_MAX_KERNEL_ADDRESS define the start and end of + * mappable kernel virtual address space. + * + * VM_MIN_USER_ADDRESS and VM_MAX_USER_ADDRESS define the start and end of the + * user address space. + */ +#define VM_MIN_ADDRESS (0x0000000000000000UL) +#define VM_MAX_ADDRESS (0xffffffffffffffffUL) + +/* 256 MiB of kernel addresses */ +#define VM_MIN_KERNEL_ADDRESS (0xffffffffc0000000UL) +#define VM_MAX_KERNEL_ADDRESS (0xffffffffcfffffffUL) + +/* Direct Map for 512 MiB of PA: 0x0 - 0x1fffffff */ +#define DMAP_MIN_ADDRESS (0xffffffffd0000000UL) +#define DMAP_MAX_ADDRESS (0xffffffffefffffffUL) + +#define DMAP_MIN_PHYSADDR (0x0000000000000000UL) +#define DMAP_MAX_PHYSADDR (DMAP_MAX_ADDRESS - DMAP_MIN_ADDRESS) + +/* True if pa is in the dmap range */ +#define PHYS_IN_DMAP(pa) ((pa) <= DMAP_MAX_PHYSADDR) +/* True if va is in the dmap range */ +#define VIRT_IN_DMAP(va) ((va) >= DMAP_MIN_ADDRESS && \ + (va) <= DMAP_MAX_ADDRESS) + +#define PHYS_TO_DMAP(pa) \ +({ \ + KASSERT(PHYS_IN_DMAP(pa), \ + ("%s: PA out of range, PA: 0x%lx", __func__, \ + (vm_paddr_t)(pa))); \ + (pa) | DMAP_MIN_ADDRESS; \ +}) + +#define DMAP_TO_PHYS(va) \ +({ \ + KASSERT(VIRT_IN_DMAP(va), \ + ("%s: VA out of range, VA: 0x%lx", __func__, \ + (vm_offset_t)(va))); \ + (va) & ~DMAP_MIN_ADDRESS; \ +}) + +#define VM_MIN_USER_ADDRESS (0x0000000000000000UL) +#define VM_MAX_USER_ADDRESS (0x0000000080000000UL) + +#define VM_MINUSER_ADDRESS (VM_MIN_USER_ADDRESS) +#define VM_MAXUSER_ADDRESS (VM_MAX_USER_ADDRESS) + +#define KERNBASE (VM_MIN_KERNEL_ADDRESS) +#define USRSTACK (VM_MAX_USER_ADDRESS) +#define KERNENTRY (0x200) + +/* + * How many physical pages per kmem arena virtual page. + */ +#ifndef VM_KMEM_SIZE_SCALE +#define VM_KMEM_SIZE_SCALE (3) +#endif + +/* + * Optional floor (in bytes) on the size of the kmem arena. + */ +#ifndef VM_KMEM_SIZE_MIN +#define VM_KMEM_SIZE_MIN (16 * 1024 * 1024) +#endif + +/* + * Optional ceiling (in bytes) on the size of the kmem arena: 60% of the + * kernel map. + */ +#ifndef VM_KMEM_SIZE_MAX +#define VM_KMEM_SIZE_MAX ((VM_MAX_KERNEL_ADDRESS - \ + VM_MIN_KERNEL_ADDRESS + 1) * 3 / 5) +#endif + +/* + * Initial pagein size of beginning of executable file. + */ +#ifndef VM_INITIAL_PAGEIN +#define VM_INITIAL_PAGEIN 16 +#endif + +/* + * RISCVTODO + * #define UMA_MD_SMALL_ALLOC + */ + +extern u_int tsb_kernel_ldd_phys; +extern vm_offset_t vm_max_kernel_address; +extern vm_offset_t init_pt_va; + +#define ZERO_REGION_SIZE (64 * 1024) /* 64KB */ + +#endif /* !_MACHINE_VMPARAM_H_ */ diff --git a/sys/sys/efi.h b/sys/sys/efi.h index 4c5d937..a4ffeb0 100644 --- a/sys/sys/efi.h +++ b/sys/sys/efi.h @@ -151,4 +151,5 @@ struct efi_systbl { uint64_t st_cfgtbl; }; +extern vm_paddr_t efi_systbl; #endif /* _SYS_EFI_H_ */ diff --git a/sys/sys/linker.h b/sys/sys/linker.h index 7e562a6..5248bfb 100644 --- a/sys/sys/linker.h +++ b/sys/sys/linker.h @@ -215,6 +215,7 @@ void *linker_hwpmc_list_objects(void); #define MODINFOMD_SHDR 0x0009 /* section header table */ #define MODINFOMD_CTORS_ADDR 0x000a /* address of .ctors */ #define MODINFOMD_CTORS_SIZE 0x000b /* size of .ctors */ +#define MODINFOMD_FW_HANDLE 0x000c /* Firmware dependent handle */ #define MODINFOMD_NOCOPY 0x8000 /* don't copy this metadata to the kernel */ #define MODINFOMD_DEPLIST (0x4001 | MODINFOMD_NOCOPY) /* depends on */ diff --git a/sys/sys/proc.h b/sys/sys/proc.h index 90effa6..cb94318 100644 --- a/sys/sys/proc.h +++ b/sys/sys/proc.h @@ -586,7 +586,6 @@ struct proc { int p_osrel; /* (x) osreldate for the binary (from ELF note, if any) */ char p_comm[MAXCOMLEN + 1]; /* (b) Process name. */ - struct pgrp *p_pgrp; /* (c + e) Pointer to process group. */ struct sysentvec *p_sysent; /* (b) Syscall dispatch info. */ struct pargs *p_args; /* (c) Process arguments. */ rlim_t p_cpulimit; /* (c) Current CPU limit in seconds. */ @@ -599,6 +598,7 @@ struct proc { u_int p_xsig; /* (c) Stop/kill sig. */ /* End area that is copied on creation. */ #define p_endcopy p_xsig + struct pgrp *p_pgrp; /* (c + e) Pointer to process group. */ struct knlist p_klist; /* (c) Knotes attached to this proc. */ int p_numthreads; /* (c) Number of threads. */ struct mdproc p_md; /* Any machine-dependent fields. */ diff --git a/sys/vm/device_pager.c b/sys/vm/device_pager.c index 5473081..a4f0d6e 100644 --- a/sys/vm/device_pager.c +++ b/sys/vm/device_pager.c @@ -347,8 +347,7 @@ old_dev_pager_fault(vm_object_t object, vm_ooffset_t offset, int prot, */ page = vm_page_getfake(paddr, memattr); VM_OBJECT_WLOCK(object); - if (vm_page_replace(page, object, (*mres)->pindex) != *mres) - panic("old_dev_pager_fault: invalid page replacement"); + vm_page_replace_checked(page, object, (*mres)->pindex, *mres); vm_page_lock(*mres); vm_page_free(*mres); vm_page_unlock(*mres); diff --git a/sys/vm/sg_pager.c b/sys/vm/sg_pager.c index 26aa1d3..84bfa49 100644 --- a/sys/vm/sg_pager.c +++ b/sys/vm/sg_pager.c @@ -188,8 +188,7 @@ sg_pager_getpages(vm_object_t object, vm_page_t *m, int count, int *rbehind, page = vm_page_getfake(paddr, memattr); VM_OBJECT_WLOCK(object); TAILQ_INSERT_TAIL(&object->un_pager.sgp.sgp_pglist, page, plinks.q); - if (vm_page_replace(page, object, offset) != m[0]) - panic("sg_pager_getpages: invalid place replacement"); + vm_page_replace_checked(page, object, offset, m[0]); m[0] = page; page->valid = VM_PAGE_BITS_ALL; diff --git a/sys/vm/vm_kern.c b/sys/vm/vm_kern.c index a05c263..f56dbca 100644 --- a/sys/vm/vm_kern.c +++ b/sys/vm/vm_kern.c @@ -181,7 +181,10 @@ retry: if (m == NULL) { VM_OBJECT_WUNLOCK(object); if (tries < ((flags & M_NOWAIT) != 0 ? 1 : 3)) { - vm_pageout_grow_cache(tries, low, high); + if (!vm_page_reclaim_contig(pflags, 1, + low, high, PAGE_SIZE, 0) && + (flags & M_WAITOK) != 0) + VM_WAIT; VM_OBJECT_WLOCK(object); tries++; goto retry; @@ -217,6 +220,7 @@ kmem_alloc_contig(struct vmem *vmem, vm_size_t size, int flags, vm_paddr_t low, vm_offset_t addr, tmp; vm_ooffset_t offset; vm_page_t end_m, m; + u_long npages; int pflags, tries; size = round_page(size); @@ -224,15 +228,18 @@ kmem_alloc_contig(struct vmem *vmem, vm_size_t size, int flags, vm_paddr_t low, return (0); offset = addr - VM_MIN_KERNEL_ADDRESS; pflags = malloc2vm_flags(flags) | VM_ALLOC_NOBUSY | VM_ALLOC_WIRED; + npages = atop(size); VM_OBJECT_WLOCK(object); tries = 0; retry: m = vm_page_alloc_contig(object, OFF_TO_IDX(offset), pflags, - atop(size), low, high, alignment, boundary, memattr); + npages, low, high, alignment, boundary, memattr); if (m == NULL) { VM_OBJECT_WUNLOCK(object); if (tries < ((flags & M_NOWAIT) != 0 ? 1 : 3)) { - vm_pageout_grow_cache(tries, low, high); + if (!vm_page_reclaim_contig(pflags, npages, low, high, + alignment, boundary) && (flags & M_WAITOK) != 0) + VM_WAIT; VM_OBJECT_WLOCK(object); tries++; goto retry; @@ -240,7 +247,7 @@ retry: vmem_free(vmem, addr, size); return (0); } - end_m = m + atop(size); + end_m = m + npages; tmp = addr; for (; m < end_m; m++) { if ((flags & M_ZERO) && (m->flags & PG_ZERO) == 0) diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c index 2e6b56a..d27eb2d 100644 --- a/sys/vm/vm_page.c +++ b/sys/vm/vm_page.c @@ -158,11 +158,14 @@ static struct vnode *vm_page_alloc_init(vm_page_t m); static void vm_page_cache_turn_free(vm_page_t m); static void vm_page_clear_dirty_mask(vm_page_t m, vm_page_bits_t pagebits); static void vm_page_enqueue(uint8_t queue, vm_page_t m); +static void vm_page_free_wakeup(void); static void vm_page_init_fakepg(void *dummy); static int vm_page_insert_after(vm_page_t m, vm_object_t object, vm_pindex_t pindex, vm_page_t mpred); static void vm_page_insert_radixdone(vm_page_t m, vm_object_t object, vm_page_t mpred); +static int vm_page_reclaim_run(int req_class, u_long npages, vm_page_t m_run, + vm_paddr_t high); SYSINIT(vm_page, SI_SUB_VM, SI_ORDER_SECOND, vm_page_init_fakepg, NULL); @@ -2093,6 +2096,592 @@ vm_page_alloc_freelist(int flind, int req) return (m); } +#define VPSC_ANY 0 /* No restrictions. */ +#define VPSC_NORESERV 1 /* Skip reservations; implies VPSC_NOSUPER. */ +#define VPSC_NOSUPER 2 /* Skip superpages. */ + +/* + * vm_page_scan_contig: + * + * Scan vm_page_array[] between the specified entries "m_start" and + * "m_end" for a run of contiguous physical pages that satisfy the + * specified conditions, and return the lowest page in the run. The + * specified "alignment" determines the alignment of the lowest physical + * page in the run. If the specified "boundary" is non-zero, then the + * run of physical pages cannot span a physical address that is a + * multiple of "boundary". + * + * "m_end" is never dereferenced, so it need not point to a vm_page + * structure within vm_page_array[]. + * + * "npages" must be greater than zero. "m_start" and "m_end" must not + * span a hole (or discontiguity) in the physical address space. Both + * "alignment" and "boundary" must be a power of two. + */ +vm_page_t +vm_page_scan_contig(u_long npages, vm_page_t m_start, vm_page_t m_end, + u_long alignment, vm_paddr_t boundary, int options) +{ + struct mtx *m_mtx, *new_mtx; + vm_object_t object; + vm_paddr_t pa; + vm_page_t m, m_run; +#if VM_NRESERVLEVEL > 0 + int level; +#endif + int m_inc, order, run_ext, run_len; + + KASSERT(npages > 0, ("npages is 0")); + KASSERT(powerof2(alignment), ("alignment is not a power of 2")); + KASSERT(powerof2(boundary), ("boundary is not a power of 2")); + m_run = NULL; + run_len = 0; + m_mtx = NULL; + for (m = m_start; m < m_end && run_len < npages; m += m_inc) { + KASSERT((m->flags & (PG_FICTITIOUS | PG_MARKER)) == 0, + ("page %p is PG_FICTITIOUS or PG_MARKER", m)); + + /* + * If the current page would be the start of a run, check its + * physical address against the end, alignment, and boundary + * conditions. If it doesn't satisfy these conditions, either + * terminate the scan or advance to the next page that + * satisfies the failed condition. + */ + if (run_len == 0) { + KASSERT(m_run == NULL, ("m_run != NULL")); + if (m + npages > m_end) + break; + pa = VM_PAGE_TO_PHYS(m); + if ((pa & (alignment - 1)) != 0) { + m_inc = atop(roundup2(pa, alignment) - pa); + continue; + } + if (((pa ^ (pa + ptoa(npages) - 1)) & ~(boundary - + 1)) != 0) { + m_inc = atop(roundup2(pa, boundary) - pa); + continue; + } + } else + KASSERT(m_run != NULL, ("m_run == NULL")); + + /* + * Avoid releasing and reacquiring the same page lock. + */ + new_mtx = vm_page_lockptr(m); + if (m_mtx != new_mtx) { + if (m_mtx != NULL) + mtx_unlock(m_mtx); + m_mtx = new_mtx; + mtx_lock(m_mtx); + } + m_inc = 1; +retry: + if (m->wire_count != 0 || m->hold_count != 0) + run_ext = 0; +#if VM_NRESERVLEVEL > 0 + else if ((level = vm_reserv_level(m)) >= 0 && + (options & VPSC_NORESERV) != 0) { + run_ext = 0; + /* Advance to the end of the reservation. */ + pa = VM_PAGE_TO_PHYS(m); + m_inc = atop(roundup2(pa + 1, vm_reserv_size(level)) - + pa); + } +#endif + else if ((object = m->object) != NULL) { + /* + * The page is considered eligible for relocation if + * and only if it could be laundered or reclaimed by + * the page daemon. + */ + if (!VM_OBJECT_TRYRLOCK(object)) { + mtx_unlock(m_mtx); + VM_OBJECT_RLOCK(object); + mtx_lock(m_mtx); + if (m->object != object) { + /* + * The page may have been freed. + */ + VM_OBJECT_RUNLOCK(object); + goto retry; + } else if (m->wire_count != 0 || + m->hold_count != 0) { + run_ext = 0; + goto unlock; + } + } + KASSERT((m->flags & PG_UNHOLDFREE) == 0, + ("page %p is PG_UNHOLDFREE", m)); + /* Don't care: PG_NODUMP, PG_WINATCFLS, PG_ZERO. */ + if (object->type != OBJT_DEFAULT && + object->type != OBJT_SWAP && + object->type != OBJT_VNODE) + run_ext = 0; + else if ((m->flags & PG_CACHED) != 0 || + m != vm_page_lookup(object, m->pindex)) { + /* + * The page is cached or recently converted + * from cached to free. + */ +#if VM_NRESERVLEVEL > 0 + if (level >= 0) { + /* + * The page is reserved. Extend the + * current run by one page. + */ + run_ext = 1; + } else +#endif + if ((order = m->order) < VM_NFREEORDER) { + /* + * The page is enqueued in the + * physical memory allocator's cache/ + * free page queues. Moreover, it is + * the first page in a power-of-two- + * sized run of contiguous cache/free + * pages. Add these pages to the end + * of the current run, and jump + * ahead. + */ + run_ext = 1 << order; + m_inc = 1 << order; + } else + run_ext = 0; +#if VM_NRESERVLEVEL > 0 + } else if ((options & VPSC_NOSUPER) != 0 && + (level = vm_reserv_level_iffullpop(m)) >= 0) { + run_ext = 0; + /* Advance to the end of the superpage. */ + pa = VM_PAGE_TO_PHYS(m); + m_inc = atop(roundup2(pa + 1, + vm_reserv_size(level)) - pa); +#endif + } else if (object->memattr == VM_MEMATTR_DEFAULT && + m->queue != PQ_NONE && !vm_page_busied(m)) { + /* + * The page is allocated but eligible for + * relocation. Extend the current run by one + * page. + */ + KASSERT(pmap_page_get_memattr(m) == + VM_MEMATTR_DEFAULT, + ("page %p has an unexpected memattr", m)); + KASSERT((m->oflags & (VPO_SWAPINPROG | + VPO_SWAPSLEEP | VPO_UNMANAGED)) == 0, + ("page %p has unexpected oflags", m)); + /* Don't care: VPO_NOSYNC. */ + run_ext = 1; + } else + run_ext = 0; +unlock: + VM_OBJECT_RUNLOCK(object); +#if VM_NRESERVLEVEL > 0 + } else if (level >= 0) { + /* + * The page is reserved but not yet allocated. In + * other words, it is still cached or free. Extend + * the current run by one page. + */ + run_ext = 1; +#endif + } else if ((order = m->order) < VM_NFREEORDER) { + /* + * The page is enqueued in the physical memory + * allocator's cache/free page queues. Moreover, it + * is the first page in a power-of-two-sized run of + * contiguous cache/free pages. Add these pages to + * the end of the current run, and jump ahead. + */ + run_ext = 1 << order; + m_inc = 1 << order; + } else { + /* + * Skip the page for one of the following reasons: (1) + * It is enqueued in the physical memory allocator's + * cache/free page queues. However, it is not the + * first page in a run of contiguous cache/free pages. + * (This case rarely occurs because the scan is + * performed in ascending order.) (2) It is not + * reserved, and it is transitioning from free to + * allocated. (Conversely, the transition from + * allocated to free for managed pages is blocked by + * the page lock.) (3) It is allocated but not + * contained by an object and not wired, e.g., + * allocated by Xen's balloon driver. + */ + run_ext = 0; + } + + /* + * Extend or reset the current run of pages. + */ + if (run_ext > 0) { + if (run_len == 0) + m_run = m; + run_len += run_ext; + } else { + if (run_len > 0) { + m_run = NULL; + run_len = 0; + } + } + } + if (m_mtx != NULL) + mtx_unlock(m_mtx); + if (run_len >= npages) + return (m_run); + return (NULL); +} + +/* + * vm_page_reclaim_run: + * + * Try to relocate each of the allocated virtual pages within the + * specified run of physical pages to a new physical address. Free the + * physical pages underlying the relocated virtual pages. A virtual page + * is relocatable if and only if it could be laundered or reclaimed by + * the page daemon. Whenever possible, a virtual page is relocated to a + * physical address above "high". + * + * Returns 0 if every physical page within the run was already free or + * just freed by a successful relocation. Otherwise, returns a non-zero + * value indicating why the last attempt to relocate a virtual page was + * unsuccessful. + * + * "req_class" must be an allocation class. + */ +static int +vm_page_reclaim_run(int req_class, u_long npages, vm_page_t m_run, + vm_paddr_t high) +{ + struct mtx *m_mtx, *new_mtx; + struct spglist free; + vm_object_t object; + vm_paddr_t pa; + vm_page_t m, m_end, m_new; + int error, order, req; + + KASSERT((req_class & VM_ALLOC_CLASS_MASK) == req_class, + ("req_class is not an allocation class")); + SLIST_INIT(&free); + error = 0; + m = m_run; + m_end = m_run + npages; + m_mtx = NULL; + for (; error == 0 && m < m_end; m++) { + KASSERT((m->flags & (PG_FICTITIOUS | PG_MARKER)) == 0, + ("page %p is PG_FICTITIOUS or PG_MARKER", m)); + + /* + * Avoid releasing and reacquiring the same page lock. + */ + new_mtx = vm_page_lockptr(m); + if (m_mtx != new_mtx) { + if (m_mtx != NULL) + mtx_unlock(m_mtx); + m_mtx = new_mtx; + mtx_lock(m_mtx); + } +retry: + if (m->wire_count != 0 || m->hold_count != 0) + error = EBUSY; + else if ((object = m->object) != NULL) { + /* + * The page is relocated if and only if it could be + * laundered or reclaimed by the page daemon. + */ + if (!VM_OBJECT_TRYWLOCK(object)) { + mtx_unlock(m_mtx); + VM_OBJECT_WLOCK(object); + mtx_lock(m_mtx); + if (m->object != object) { + /* + * The page may have been freed. + */ + VM_OBJECT_WUNLOCK(object); + goto retry; + } else if (m->wire_count != 0 || + m->hold_count != 0) { + error = EBUSY; + goto unlock; + } + } + KASSERT((m->flags & PG_UNHOLDFREE) == 0, + ("page %p is PG_UNHOLDFREE", m)); + /* Don't care: PG_NODUMP, PG_WINATCFLS, PG_ZERO. */ + if (object->type != OBJT_DEFAULT && + object->type != OBJT_SWAP && + object->type != OBJT_VNODE) + error = EINVAL; + else if ((m->flags & PG_CACHED) != 0 || + m != vm_page_lookup(object, m->pindex)) { + /* + * The page is cached or recently converted + * from cached to free. + */ + VM_OBJECT_WUNLOCK(object); + goto cached; + } else if (object->memattr != VM_MEMATTR_DEFAULT) + error = EINVAL; + else if (m->queue != PQ_NONE && !vm_page_busied(m)) { + KASSERT(pmap_page_get_memattr(m) == + VM_MEMATTR_DEFAULT, + ("page %p has an unexpected memattr", m)); + KASSERT((m->oflags & (VPO_SWAPINPROG | + VPO_SWAPSLEEP | VPO_UNMANAGED)) == 0, + ("page %p has unexpected oflags", m)); + /* Don't care: VPO_NOSYNC. */ + if (m->valid != 0) { + /* + * First, try to allocate a new page + * that is above "high". Failing + * that, try to allocate a new page + * that is below "m_run". Allocate + * the new page between the end of + * "m_run" and "high" only as a last + * resort. + */ + req = req_class | VM_ALLOC_NOOBJ; + if ((m->flags & PG_NODUMP) != 0) + req |= VM_ALLOC_NODUMP; + if (trunc_page(high) != + ~(vm_paddr_t)PAGE_MASK) { + m_new = vm_page_alloc_contig( + NULL, 0, req, 1, + round_page(high), + ~(vm_paddr_t)0, + PAGE_SIZE, 0, + VM_MEMATTR_DEFAULT); + } else + m_new = NULL; + if (m_new == NULL) { + pa = VM_PAGE_TO_PHYS(m_run); + m_new = vm_page_alloc_contig( + NULL, 0, req, 1, + 0, pa - 1, PAGE_SIZE, 0, + VM_MEMATTR_DEFAULT); + } + if (m_new == NULL) { + pa += ptoa(npages); + m_new = vm_page_alloc_contig( + NULL, 0, req, 1, + pa, high, PAGE_SIZE, 0, + VM_MEMATTR_DEFAULT); + } + if (m_new == NULL) { + error = ENOMEM; + goto unlock; + } + KASSERT(m_new->wire_count == 0, + ("page %p is wired", m)); + + /* + * Replace "m" with the new page. For + * vm_page_replace(), "m" must be busy + * and dequeued. Finally, change "m" + * as if vm_page_free() was called. + */ + if (object->ref_count != 0) + pmap_remove_all(m); + m_new->aflags = m->aflags; + KASSERT(m_new->oflags == VPO_UNMANAGED, + ("page %p is managed", m)); + m_new->oflags = m->oflags & VPO_NOSYNC; + pmap_copy_page(m, m_new); + m_new->valid = m->valid; + m_new->dirty = m->dirty; + m->flags &= ~PG_ZERO; + vm_page_xbusy(m); + vm_page_remque(m); + vm_page_replace_checked(m_new, object, + m->pindex, m); + m->valid = 0; + vm_page_undirty(m); + + /* + * The new page must be deactivated + * before the object is unlocked. + */ + new_mtx = vm_page_lockptr(m_new); + if (m_mtx != new_mtx) { + mtx_unlock(m_mtx); + m_mtx = new_mtx; + mtx_lock(m_mtx); + } + vm_page_deactivate(m_new); + } else { + m->flags &= ~PG_ZERO; + vm_page_remque(m); + vm_page_remove(m); + KASSERT(m->dirty == 0, + ("page %p is dirty", m)); + } + SLIST_INSERT_HEAD(&free, m, plinks.s.ss); + } else + error = EBUSY; +unlock: + VM_OBJECT_WUNLOCK(object); + } else { +cached: + mtx_lock(&vm_page_queue_free_mtx); + order = m->order; + if (order < VM_NFREEORDER) { + /* + * The page is enqueued in the physical memory + * allocator's cache/free page queues. + * Moreover, it is the first page in a power- + * of-two-sized run of contiguous cache/free + * pages. Jump ahead to the last page within + * that run, and continue from there. + */ + m += (1 << order) - 1; + } +#if VM_NRESERVLEVEL > 0 + else if (vm_reserv_is_page_free(m)) + order = 0; +#endif + mtx_unlock(&vm_page_queue_free_mtx); + if (order == VM_NFREEORDER) + error = EINVAL; + } + } + if (m_mtx != NULL) + mtx_unlock(m_mtx); + if ((m = SLIST_FIRST(&free)) != NULL) { + mtx_lock(&vm_page_queue_free_mtx); + do { + SLIST_REMOVE_HEAD(&free, plinks.s.ss); + vm_phys_freecnt_adj(m, 1); +#if VM_NRESERVLEVEL > 0 + if (!vm_reserv_free_page(m)) +#else + if (true) +#endif + vm_phys_free_pages(m, 0); + } while ((m = SLIST_FIRST(&free)) != NULL); + vm_page_zero_idle_wakeup(); + vm_page_free_wakeup(); + mtx_unlock(&vm_page_queue_free_mtx); + } + return (error); +} + +#define NRUNS 16 + +CTASSERT(powerof2(NRUNS)); + +#define RUN_INDEX(count) ((count) & (NRUNS - 1)) + +#define MIN_RECLAIM 8 + +/* + * vm_page_reclaim_contig: + * + * Reclaim allocated, contiguous physical memory satisfying the specified + * conditions by relocating the virtual pages using that physical memory. + * Returns true if reclamation is successful and false otherwise. Since + * relocation requires the allocation of physical pages, reclamation may + * fail due to a shortage of cache/free pages. When reclamation fails, + * callers are expected to perform VM_WAIT before retrying a failed + * allocation operation, e.g., vm_page_alloc_contig(). + * + * The caller must always specify an allocation class through "req". + * + * allocation classes: + * VM_ALLOC_NORMAL normal process request + * VM_ALLOC_SYSTEM system *really* needs a page + * VM_ALLOC_INTERRUPT interrupt time request + * + * The optional allocation flags are ignored. + * + * "npages" must be greater than zero. Both "alignment" and "boundary" + * must be a power of two. + */ +bool +vm_page_reclaim_contig(int req, u_long npages, vm_paddr_t low, vm_paddr_t high, + u_long alignment, vm_paddr_t boundary) +{ + vm_paddr_t curr_low; + vm_page_t m_run, m_runs[NRUNS]; + u_long count, reclaimed; + int error, i, options, req_class; + + KASSERT(npages > 0, ("npages is 0")); + KASSERT(powerof2(alignment), ("alignment is not a power of 2")); + KASSERT(powerof2(boundary), ("boundary is not a power of 2")); + req_class = req & VM_ALLOC_CLASS_MASK; + + /* + * The page daemon is allowed to dig deeper into the free page list. + */ + if (curproc == pageproc && req_class != VM_ALLOC_INTERRUPT) + req_class = VM_ALLOC_SYSTEM; + + /* + * Return if the number of cached and free pages cannot satisfy the + * requested allocation. + */ + count = vm_cnt.v_free_count + vm_cnt.v_cache_count; + if (count < npages + vm_cnt.v_free_reserved || (count < npages + + vm_cnt.v_interrupt_free_min && req_class == VM_ALLOC_SYSTEM) || + (count < npages && req_class == VM_ALLOC_INTERRUPT)) + return (false); + + /* + * Scan up to three times, relaxing the restrictions ("options") on + * the reclamation of reservations and superpages each time. + */ + for (options = VPSC_NORESERV;;) { + /* + * Find the highest runs that satisfy the given constraints + * and restrictions, and record them in "m_runs". + */ + curr_low = low; + count = 0; + for (;;) { + m_run = vm_phys_scan_contig(npages, curr_low, high, + alignment, boundary, options); + if (m_run == NULL) + break; + curr_low = VM_PAGE_TO_PHYS(m_run) + ptoa(npages); + m_runs[RUN_INDEX(count)] = m_run; + count++; + } + + /* + * Reclaim the highest runs in LIFO (descending) order until + * the number of reclaimed pages, "reclaimed", is at least + * MIN_RECLAIM. Reset "reclaimed" each time because each + * reclamation is idempotent, and runs will (likely) recur + * from one scan to the next as restrictions are relaxed. + */ + reclaimed = 0; + for (i = 0; count > 0 && i < NRUNS; i++) { + count--; + m_run = m_runs[RUN_INDEX(count)]; + error = vm_page_reclaim_run(req_class, npages, m_run, + high); + if (error == 0) { + reclaimed += npages; + if (reclaimed >= MIN_RECLAIM) + return (true); + } + } + + /* + * Either relax the restrictions on the next scan or return if + * the last scan had no restrictions. + */ + if (options == VPSC_NORESERV) + options = VPSC_NOSUPER; + else if (options == VPSC_NOSUPER) + options = VPSC_ANY; + else if (options == VPSC_ANY) + return (reclaimed != 0); + } +} + /* * vm_wait: (also see VM_WAIT macro) * diff --git a/sys/vm/vm_page.h b/sys/vm/vm_page.h index 9abd31c..972603a 100644 --- a/sys/vm/vm_page.h +++ b/sys/vm/vm_page.h @@ -474,6 +474,8 @@ vm_page_t vm_page_prev(vm_page_t m); boolean_t vm_page_ps_is_valid(vm_page_t m); void vm_page_putfake(vm_page_t m); void vm_page_readahead_finish(vm_page_t m); +bool vm_page_reclaim_contig(int req, u_long npages, vm_paddr_t low, + vm_paddr_t high, u_long alignment, vm_paddr_t boundary); void vm_page_reference(vm_page_t m); void vm_page_remove (vm_page_t); int vm_page_rename (vm_page_t, vm_object_t, vm_pindex_t); @@ -482,6 +484,8 @@ vm_page_t vm_page_replace(vm_page_t mnew, vm_object_t object, void vm_page_requeue(vm_page_t m); void vm_page_requeue_locked(vm_page_t m); int vm_page_sbusied(vm_page_t m); +vm_page_t vm_page_scan_contig(u_long npages, vm_page_t m_start, + vm_page_t m_end, u_long alignment, vm_paddr_t boundary, int options); void vm_page_set_valid_range(vm_page_t m, int base, int size); int vm_page_sleep_if_busy(vm_page_t m, const char *msg); vm_offset_t vm_page_startup(vm_offset_t vaddr); @@ -678,5 +682,20 @@ vm_page_undirty(vm_page_t m) m->dirty = 0; } +static inline void +vm_page_replace_checked(vm_page_t mnew, vm_object_t object, vm_pindex_t pindex, + vm_page_t mold) +{ + vm_page_t mret; + + mret = vm_page_replace(mnew, object, pindex); + KASSERT(mret == mold, + ("invalid page replacement, mold=%p, mret=%p", mold, mret)); + + /* Unused if !INVARIANTS. */ + (void)mold; + (void)mret; +} + #endif /* _KERNEL */ #endif /* !_VM_PAGE_ */ diff --git a/sys/vm/vm_pageout.c b/sys/vm/vm_pageout.c index 4997c82..b7f6887 100644 --- a/sys/vm/vm_pageout.c +++ b/sys/vm/vm_pageout.c @@ -237,8 +237,6 @@ SYSCTL_INT(_vm, OID_AUTO, max_wired, CTLFLAG_RW, &vm_page_max_wired, 0, "System-wide limit to wired page count"); static boolean_t vm_pageout_fallback_object_lock(vm_page_t, vm_page_t *); -static boolean_t vm_pageout_launder(struct vm_pagequeue *pq, int, vm_paddr_t, - vm_paddr_t); #if !defined(NO_SWAPPING) static void vm_pageout_map_deactivate_pages(vm_map_t, long); static void vm_pageout_object_deactivate_pages(pmap_t, vm_object_t, long); @@ -595,170 +593,6 @@ vm_pageout_flush(vm_page_t *mc, int count, int flags, int mreq, int *prunlen, return (numpagedout); } -static boolean_t -vm_pageout_launder(struct vm_pagequeue *pq, int tries, vm_paddr_t low, - vm_paddr_t high) -{ - struct mount *mp; - struct vnode *vp; - vm_object_t object; - vm_paddr_t pa; - vm_page_t m, m_tmp, next; - int lockmode; - - vm_pagequeue_lock(pq); - TAILQ_FOREACH_SAFE(m, &pq->pq_pl, plinks.q, next) { - if ((m->flags & PG_MARKER) != 0) - continue; - pa = VM_PAGE_TO_PHYS(m); - if (pa < low || pa + PAGE_SIZE > high) - continue; - if (!vm_pageout_page_lock(m, &next) || m->hold_count != 0) { - vm_page_unlock(m); - continue; - } - object = m->object; - if ((!VM_OBJECT_TRYWLOCK(object) && - (!vm_pageout_fallback_object_lock(m, &next) || - m->hold_count != 0)) || vm_page_busied(m)) { - vm_page_unlock(m); - VM_OBJECT_WUNLOCK(object); - continue; - } - vm_page_test_dirty(m); - if (m->dirty == 0 && object->ref_count != 0) - pmap_remove_all(m); - if (m->dirty != 0) { - vm_page_unlock(m); - if (tries == 0 || (object->flags & OBJ_DEAD) != 0) { - VM_OBJECT_WUNLOCK(object); - continue; - } - if (object->type == OBJT_VNODE) { - vm_pagequeue_unlock(pq); - vp = object->handle; - vm_object_reference_locked(object); - VM_OBJECT_WUNLOCK(object); - (void)vn_start_write(vp, &mp, V_WAIT); - lockmode = MNT_SHARED_WRITES(vp->v_mount) ? - LK_SHARED : LK_EXCLUSIVE; - vn_lock(vp, lockmode | LK_RETRY); - VM_OBJECT_WLOCK(object); - vm_object_page_clean(object, 0, 0, OBJPC_SYNC); - VM_OBJECT_WUNLOCK(object); - VOP_UNLOCK(vp, 0); - vm_object_deallocate(object); - vn_finished_write(mp); - return (TRUE); - } else if (object->type == OBJT_SWAP || - object->type == OBJT_DEFAULT) { - vm_pagequeue_unlock(pq); - m_tmp = m; - vm_pageout_flush(&m_tmp, 1, VM_PAGER_PUT_SYNC, - 0, NULL, NULL); - VM_OBJECT_WUNLOCK(object); - return (TRUE); - } - } else { - /* - * Dequeue here to prevent lock recursion in - * vm_page_cache(). - */ - vm_page_dequeue_locked(m); - vm_page_cache(m); - vm_page_unlock(m); - } - VM_OBJECT_WUNLOCK(object); - } - vm_pagequeue_unlock(pq); - return (FALSE); -} - -/* - * Increase the number of cached pages. The specified value, "tries", - * determines which categories of pages are cached: - * - * 0: All clean, inactive pages within the specified physical address range - * are cached. Will not sleep. - * 1: The vm_lowmem handlers are called. All inactive pages within - * the specified physical address range are cached. May sleep. - * 2: The vm_lowmem handlers are called. All inactive and active pages - * within the specified physical address range are cached. May sleep. - */ -void -vm_pageout_grow_cache(int tries, vm_paddr_t low, vm_paddr_t high) -{ - int actl, actmax, inactl, inactmax, dom, initial_dom; - static int start_dom = 0; - - if (tries > 0) { - /* - * Decrease registered cache sizes. The vm_lowmem handlers - * may acquire locks and/or sleep, so they can only be invoked - * when "tries" is greater than zero. - */ - SDT_PROBE0(vm, , , vm__lowmem_cache); - EVENTHANDLER_INVOKE(vm_lowmem, 0); - - /* - * We do this explicitly after the caches have been drained - * above. - */ - uma_reclaim(); - } - - /* - * Make the next scan start on the next domain. - */ - initial_dom = atomic_fetchadd_int(&start_dom, 1) % vm_ndomains; - - inactl = 0; - inactmax = vm_cnt.v_inactive_count; - actl = 0; - actmax = tries < 2 ? 0 : vm_cnt.v_active_count; - dom = initial_dom; - - /* - * Scan domains in round-robin order, first inactive queues, - * then active. Since domain usually owns large physically - * contiguous chunk of memory, it makes sense to completely - * exhaust one domain before switching to next, while growing - * the pool of contiguous physical pages. - * - * Do not even start launder a domain which cannot contain - * the specified address range, as indicated by segments - * constituting the domain. - */ -again_inact: - if (inactl < inactmax) { - if (vm_phys_domain_intersects(vm_dom[dom].vmd_segs, - low, high) && - vm_pageout_launder(&vm_dom[dom].vmd_pagequeues[PQ_INACTIVE], - tries, low, high)) { - inactl++; - goto again_inact; - } - if (++dom == vm_ndomains) - dom = 0; - if (dom != initial_dom) - goto again_inact; - } -again_act: - if (actl < actmax) { - if (vm_phys_domain_intersects(vm_dom[dom].vmd_segs, - low, high) && - vm_pageout_launder(&vm_dom[dom].vmd_pagequeues[PQ_ACTIVE], - tries, low, high)) { - actl++; - goto again_act; - } - if (++dom == vm_ndomains) - dom = 0; - if (dom != initial_dom) - goto again_act; - } -} - #if !defined(NO_SWAPPING) /* * vm_pageout_object_deactivate_pages diff --git a/sys/vm/vm_pageout.h b/sys/vm/vm_pageout.h index 9c8da8b..2d0b961 100644 --- a/sys/vm/vm_pageout.h +++ b/sys/vm/vm_pageout.h @@ -101,7 +101,6 @@ extern void vm_waitpfault(void); #ifdef _KERNEL int vm_pageout_flush(vm_page_t *, int, int, int, int *, boolean_t *); -void vm_pageout_grow_cache(int, vm_paddr_t, vm_paddr_t); void vm_pageout_oom(int shortage); #endif #endif /* _VM_VM_PAGEOUT_H_ */ diff --git a/sys/vm/vm_phys.c b/sys/vm/vm_phys.c index d26b8b5..38799f2 100644 --- a/sys/vm/vm_phys.c +++ b/sys/vm/vm_phys.c @@ -170,6 +170,9 @@ static struct vm_domain_policy vm_default_policy = static vm_page_t vm_phys_alloc_domain_pages(int domain, int flind, int pool, int order); +static vm_page_t vm_phys_alloc_seg_contig(struct vm_phys_seg *seg, + u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment, + vm_paddr_t boundary); static void _vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end, int domain); static void vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end); static int vm_phys_paddr_to_segind(vm_paddr_t pa); @@ -1163,6 +1166,56 @@ vm_phys_free_contig(vm_page_t m, u_long npages) } /* + * Scan physical memory between the specified addresses "low" and "high" for a + * run of contiguous physical pages that satisfy the specified conditions, and + * return the lowest page in the run. The specified "alignment" determines + * the alignment of the lowest physical page in the run. If the specified + * "boundary" is non-zero, then the run of physical pages cannot span a + * physical address that is a multiple of "boundary". + * + * "npages" must be greater than zero. Both "alignment" and "boundary" must + * be a power of two. + */ +vm_page_t +vm_phys_scan_contig(u_long npages, vm_paddr_t low, vm_paddr_t high, + u_long alignment, vm_paddr_t boundary, int options) +{ + vm_paddr_t pa_end; + vm_page_t m_end, m_run, m_start; + struct vm_phys_seg *seg; + int segind; + + KASSERT(npages > 0, ("npages is 0")); + KASSERT(powerof2(alignment), ("alignment is not a power of 2")); + KASSERT(powerof2(boundary), ("boundary is not a power of 2")); + if (low >= high) + return (NULL); + for (segind = 0; segind < vm_phys_nsegs; segind++) { + seg = &vm_phys_segs[segind]; + if (seg->start >= high) + break; + if (low >= seg->end) + continue; + if (low <= seg->start) + m_start = seg->first_page; + else + m_start = &seg->first_page[atop(low - seg->start)]; + if (high < seg->end) + pa_end = high; + else + pa_end = seg->end; + if (pa_end - VM_PAGE_TO_PHYS(m_start) < ptoa(npages)) + continue; + m_end = &seg->first_page[atop(pa_end - seg->start)]; + m_run = vm_page_scan_contig(npages, m_start, m_end, + alignment, boundary, options); + if (m_run != NULL) + return (m_run); + } + return (NULL); +} + +/* * Set the pool for a contiguous, power of two-sized set of physical pages. */ void @@ -1300,93 +1353,123 @@ vm_page_t vm_phys_alloc_contig(u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment, vm_paddr_t boundary) { - struct vm_freelist *fl; - struct vm_phys_seg *seg; - vm_paddr_t pa, pa_last, size; - vm_page_t m, m_ret; - u_long npages_end; - int domain, flind, oind, order, pind; + vm_paddr_t pa_end, pa_start; + vm_page_t m_run; struct vm_domain_iterator vi; + struct vm_phys_seg *seg; + int domain, segind; + KASSERT(npages > 0, ("npages is 0")); + KASSERT(powerof2(alignment), ("alignment is not a power of 2")); + KASSERT(powerof2(boundary), ("boundary is not a power of 2")); mtx_assert(&vm_page_queue_free_mtx, MA_OWNED); - size = npages << PAGE_SHIFT; - KASSERT(size != 0, - ("vm_phys_alloc_contig: size must not be 0")); - KASSERT((alignment & (alignment - 1)) == 0, - ("vm_phys_alloc_contig: alignment must be a power of 2")); - KASSERT((boundary & (boundary - 1)) == 0, - ("vm_phys_alloc_contig: boundary must be a power of 2")); - /* Compute the queue that is the best fit for npages. */ - for (order = 0; (1 << order) < npages; order++); - + if (low >= high) + return (NULL); vm_policy_iterator_init(&vi); - restartdom: if (vm_domain_iterator_run(&vi, &domain) != 0) { vm_policy_iterator_finish(&vi); return (NULL); } + m_run = NULL; + for (segind = 0; segind < vm_phys_nsegs; segind++) { + seg = &vm_phys_segs[segind]; + if (seg->start >= high) + break; + if (low >= seg->end || seg->domain != domain) + continue; + if (low <= seg->start) + pa_start = seg->start; + else + pa_start = low; + if (high < seg->end) + pa_end = high; + else + pa_end = seg->end; + if (pa_end - pa_start < ptoa(npages)) + continue; + m_run = vm_phys_alloc_seg_contig(seg, npages, low, high, + alignment, boundary); + if (m_run != NULL) + break; + } + if (m_run == NULL && !vm_domain_iterator_isdone(&vi)) + goto restartdom; + vm_policy_iterator_finish(&vi); + return (m_run); +} - for (flind = 0; flind < vm_nfreelists; flind++) { - for (oind = min(order, VM_NFREEORDER - 1); oind < VM_NFREEORDER; oind++) { - for (pind = 0; pind < VM_NFREEPOOL; pind++) { - fl = &vm_phys_free_queues[domain][flind][pind][0]; - TAILQ_FOREACH(m_ret, &fl[oind].pl, plinks.q) { - /* - * A free list may contain physical pages - * from one or more segments. - */ - seg = &vm_phys_segs[m_ret->segind]; - if (seg->start > high || - low >= seg->end) - continue; - - /* - * Is the size of this allocation request - * larger than the largest block size? - */ - if (order >= VM_NFREEORDER) { - /* - * Determine if a sufficient number - * of subsequent blocks to satisfy - * the allocation request are free. - */ - pa = VM_PAGE_TO_PHYS(m_ret); - pa_last = pa + size; - for (;;) { - pa += 1 << (PAGE_SHIFT + VM_NFREEORDER - 1); - if (pa >= pa_last) - break; - if (pa < seg->start || - pa >= seg->end) - break; - m = &seg->first_page[atop(pa - seg->start)]; - if (m->order != VM_NFREEORDER - 1) - break; - } - /* If not, continue to the next block. */ - if (pa < pa_last) - continue; - } +/* + * Allocate a run of contiguous physical pages from the free list for the + * specified segment. + */ +static vm_page_t +vm_phys_alloc_seg_contig(struct vm_phys_seg *seg, u_long npages, + vm_paddr_t low, vm_paddr_t high, u_long alignment, vm_paddr_t boundary) +{ + struct vm_freelist *fl; + vm_paddr_t pa, pa_end, size; + vm_page_t m, m_ret; + u_long npages_end; + int oind, order, pind; + KASSERT(npages > 0, ("npages is 0")); + KASSERT(powerof2(alignment), ("alignment is not a power of 2")); + KASSERT(powerof2(boundary), ("boundary is not a power of 2")); + mtx_assert(&vm_page_queue_free_mtx, MA_OWNED); + /* Compute the queue that is the best fit for npages. */ + for (order = 0; (1 << order) < npages; order++); + /* Search for a run satisfying the specified conditions. */ + size = npages << PAGE_SHIFT; + for (oind = min(order, VM_NFREEORDER - 1); oind < VM_NFREEORDER; + oind++) { + for (pind = 0; pind < VM_NFREEPOOL; pind++) { + fl = (*seg->free_queues)[pind]; + TAILQ_FOREACH(m_ret, &fl[oind].pl, plinks.q) { + /* + * Is the size of this allocation request + * larger than the largest block size? + */ + if (order >= VM_NFREEORDER) { /* - * Determine if the blocks are within the given range, - * satisfy the given alignment, and do not cross the - * given boundary. + * Determine if a sufficient number of + * subsequent blocks to satisfy the + * allocation request are free. */ pa = VM_PAGE_TO_PHYS(m_ret); - if (pa >= low && - pa + size <= high && - (pa & (alignment - 1)) == 0 && - ((pa ^ (pa + size - 1)) & ~(boundary - 1)) == 0) - goto done; + pa_end = pa + size; + for (;;) { + pa += 1 << (PAGE_SHIFT + + VM_NFREEORDER - 1); + if (pa >= pa_end || + pa < seg->start || + pa >= seg->end) + break; + m = &seg->first_page[atop(pa - + seg->start)]; + if (m->order != VM_NFREEORDER - + 1) + break; + } + /* If not, go to the next block. */ + if (pa < pa_end) + continue; } + + /* + * Determine if the blocks are within the + * given range, satisfy the given alignment, + * and do not cross the given boundary. + */ + pa = VM_PAGE_TO_PHYS(m_ret); + pa_end = pa + size; + if (pa >= low && pa_end <= high && (pa & + (alignment - 1)) == 0 && ((pa ^ (pa_end - + 1)) & ~(boundary - 1)) == 0) + goto done; } } } - if (!vm_domain_iterator_isdone(&vi)) - goto restartdom; - vm_policy_iterator_finish(&vi); return (NULL); done: for (m = m_ret; m < &m_ret[npages]; m = &m[1 << oind]) { diff --git a/sys/vm/vm_phys.h b/sys/vm/vm_phys.h index 37864db..f9c776d 100644 --- a/sys/vm/vm_phys.h +++ b/sys/vm/vm_phys.h @@ -84,6 +84,8 @@ void vm_phys_free_contig(vm_page_t m, u_long npages); void vm_phys_free_pages(vm_page_t m, int order); void vm_phys_init(void); vm_page_t vm_phys_paddr_to_vm_page(vm_paddr_t pa); +vm_page_t vm_phys_scan_contig(u_long npages, vm_paddr_t low, vm_paddr_t high, + u_long alignment, vm_paddr_t boundary, int options); void vm_phys_set_pool(int pool, vm_page_t m, int order); boolean_t vm_phys_unfree_page(vm_page_t m); boolean_t vm_phys_zero_pages_idle(void); diff --git a/sys/vm/vm_reserv.c b/sys/vm/vm_reserv.c index 8f34b31..8bb1788 100644 --- a/sys/vm/vm_reserv.c +++ b/sys/vm/vm_reserv.c @@ -866,6 +866,35 @@ vm_reserv_init(void) } /* + * Returns true if the given page belongs to a reservation and that page is + * free. Otherwise, returns false. + */ +bool +vm_reserv_is_page_free(vm_page_t m) +{ + vm_reserv_t rv; + + mtx_assert(&vm_page_queue_free_mtx, MA_OWNED); + rv = vm_reserv_from_page(m); + if (rv->object == NULL) + return (false); + return (popmap_is_clear(rv->popmap, m - rv->pages)); +} + +/* + * If the given page belongs to a reservation, returns the level of that + * reservation. Otherwise, returns -1. + */ +int +vm_reserv_level(vm_page_t m) +{ + vm_reserv_t rv; + + rv = vm_reserv_from_page(m); + return (rv->object != NULL ? 0 : -1); +} + +/* * Returns a reservation level if the given page belongs to a fully-populated * reservation and -1 otherwise. */ @@ -1076,6 +1105,23 @@ vm_reserv_rename(vm_page_t m, vm_object_t new_object, vm_object_t old_object, } /* + * Returns the size (in bytes) of a reservation of the specified level. + */ +int +vm_reserv_size(int level) +{ + + switch (level) { + case 0: + return (VM_LEVEL_0_SIZE); + case -1: + return (PAGE_SIZE); + default: + return (0); + } +} + +/* * Allocates the virtual and physical memory required by the reservation * management system's data structures, in particular, the reservation array. */ diff --git a/sys/vm/vm_reserv.h b/sys/vm/vm_reserv.h index 4c142c7..52f6ab2 100644 --- a/sys/vm/vm_reserv.h +++ b/sys/vm/vm_reserv.h @@ -53,6 +53,8 @@ vm_page_t vm_reserv_alloc_page(vm_object_t object, vm_pindex_t pindex, void vm_reserv_break_all(vm_object_t object); boolean_t vm_reserv_free_page(vm_page_t m); void vm_reserv_init(void); +bool vm_reserv_is_page_free(vm_page_t m); +int vm_reserv_level(vm_page_t m); int vm_reserv_level_iffullpop(vm_page_t m); boolean_t vm_reserv_reactivate_page(vm_page_t m); boolean_t vm_reserv_reclaim_contig(u_long npages, vm_paddr_t low, @@ -60,6 +62,7 @@ boolean_t vm_reserv_reclaim_contig(u_long npages, vm_paddr_t low, boolean_t vm_reserv_reclaim_inactive(void); void vm_reserv_rename(vm_page_t m, vm_object_t new_object, vm_object_t old_object, vm_pindex_t old_object_offset); +int vm_reserv_size(int level); vm_paddr_t vm_reserv_startup(vm_offset_t *vaddr, vm_paddr_t end, vm_paddr_t high_water); |
