diff options
author | jhb <jhb@FreeBSD.org> | 2003-11-03 21:53:38 +0000 |
---|---|---|
committer | jhb <jhb@FreeBSD.org> | 2003-11-03 21:53:38 +0000 |
commit | dcec7e1907fe867da713052f8c91d79bbfe68000 (patch) | |
tree | c7dc6abe8f136cde7b0dbbc6ce1152f91bb42ad1 /sys/amd64/amd64/apic_vector.S | |
parent | aac4b7181cbed34861c76ff9d9142c7fdf212008 (diff) | |
download | FreeBSD-src-dcec7e1907fe867da713052f8c91d79bbfe68000.zip FreeBSD-src-dcec7e1907fe867da713052f8c91d79bbfe68000.tar.gz |
New APIC support code:
- The apic interrupt entry points have been rewritten so that each entry
point can serve 32 different vectors. When the entry is executed, it
uses one of the 32-bit ISR registers to determine which vector in its
assigned range was triggered. Thus, the apic code can support 159
different interrupt vectors with only 5 entry points.
- We now always to disable the local APIC to work around an errata in
certain PPros and then re-enable it again if we decide to use the APICs
to route interrupts.
- We no longer map IO APICs or local APICs using special page table
entries. Instead, we just use pmap_mapdev(). We also no longer
export the virtual address of the local APIC as a global symbol to
the rest of the system, but only in local_apic.c. To aid this, the
APIC ID of each CPU is exported as a per-CPU variable.
- Interrupt sources are provided for each intpin on each IO APIC.
Currently, each source is given a unique interrupt vector meaning that
PCI interrupts are not shared on most machines with an I/O APIC.
That mapping for interrupt sources to interrupt vectors is up to the
APIC enumerator driver however.
- We no longer probe to see if we need to use mixed mode to route IRQ 0,
instead we always use mixed mode to route IRQ 0 for now. This can be
disabled via the 'NO_MIXED_MODE' kernel option.
- The npx(4) driver now always probes to see if a built-in FPU is present
since this test can now be performed with the new APIC code. However,
an SMP kernel will panic if there is more than one CPU and a built-in
FPU is not found.
- PCI interrupts are now properly routed when using APICs to route
interrupts, so remove the hack to psuedo-route interrupts when the
intpin register was read.
- The apic.h header was moved to apicreg.h and a new apicvar.h header
that declares the APIs used by the new APIC code was added.
Diffstat (limited to 'sys/amd64/amd64/apic_vector.S')
-rw-r--r-- | sys/amd64/amd64/apic_vector.S | 433 |
1 files changed, 90 insertions, 343 deletions
diff --git a/sys/amd64/amd64/apic_vector.S b/sys/amd64/amd64/apic_vector.S index fb6464e..f1d2520 100644 --- a/sys/amd64/amd64/apic_vector.S +++ b/sys/amd64/amd64/apic_vector.S @@ -1,19 +1,53 @@ -/* +/*- + * Copyright (c) 1989, 1990 William F. Jolitz. + * Copyright (c) 1990 The Regents of the University of California. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Berkeley and its contributors. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * * from: vector.s, 386BSD 0.1 unknown origin * $FreeBSD$ */ -#include <machine/apic.h> -#include <machine/smp.h> +/* + * Interrupt entry points for external interrupts triggered by I/O APICs + * as well as IPI handlers. + */ -/* convert an absolute IRQ# into a bitmask */ -#define IRQ_BIT(irq_num) (1 << (irq_num)) +#include <machine/asmacros.h> +#include <machine/apicreg.h> +#include <machine/smptests.h> -/* make an index into the IO APIC from the IRQ# */ -#define REDTBL_IDX(irq_num) (0x10 + ((irq_num) * 2)) +#include "assym.s" /* - * + * Macros to create and destroy a trap frame. */ #define PUSH_FRAME \ pushl $0 ; /* dummy error code */ \ @@ -23,14 +57,6 @@ pushl %es ; \ pushl %fs -#define PUSH_DUMMY \ - pushfl ; /* eflags */ \ - pushl %cs ; /* cs */ \ - pushl 12(%esp) ; /* original caller eip */ \ - pushl $0 ; /* dummy error code */ \ - pushl $0 ; /* dummy trap type */ \ - subl $11*4,%esp ; - #define POP_FRAME \ popl %fs ; \ popl %es ; \ @@ -38,209 +64,40 @@ popal ; \ addl $4+4,%esp -#define POP_DUMMY \ - addl $16*4,%esp - -#define IOAPICADDR(irq_num) CNAME(int_to_apicintpin) + 16 * (irq_num) + 8 -#define REDIRIDX(irq_num) CNAME(int_to_apicintpin) + 16 * (irq_num) + 12 - -#define MASK_IRQ(irq_num) \ - ICU_LOCK ; /* into critical reg */ \ - testl $IRQ_BIT(irq_num), apic_imen ; \ - jne 7f ; /* masked, don't mask */ \ - orl $IRQ_BIT(irq_num), apic_imen ; /* set the mask bit */ \ - movl IOAPICADDR(irq_num), %ecx ; /* ioapic addr */ \ - movl REDIRIDX(irq_num), %eax ; /* get the index */ \ - movl %eax, (%ecx) ; /* write the index */ \ - movl IOAPIC_WINDOW(%ecx), %eax ; /* current value */ \ - orl $IOART_INTMASK, %eax ; /* set the mask */ \ - movl %eax, IOAPIC_WINDOW(%ecx) ; /* new value */ \ -7: ; /* already masked */ \ - ICU_UNLOCK -/* - * Test to see whether we are handling an edge or level triggered INT. - * Level-triggered INTs must still be masked as we don't clear the source, - * and the EOI cycle would cause redundant INTs to occur. - */ -#define MASK_LEVEL_IRQ(irq_num) \ - testl $IRQ_BIT(irq_num), apic_pin_trigger ; \ - jz 9f ; /* edge, don't mask */ \ - MASK_IRQ(irq_num) ; \ -9: - - -#ifdef APIC_INTR_REORDER -#define EOI_IRQ(irq_num) \ - movl apic_isrbit_location + 8 * (irq_num), %eax ; \ - movl (%eax), %eax ; \ - testl apic_isrbit_location + 4 + 8 * (irq_num), %eax ; \ - jz 9f ; /* not active */ \ - movl $0, lapic+LA_EOI ; \ -9: - -#else -#define EOI_IRQ(irq_num) \ - testl $IRQ_BIT(irq_num), lapic+LA_ISR1; \ - jz 9f ; /* not active */ \ - movl $0, lapic+LA_EOI; \ -9: -#endif - - -/* - * Test to see if the source is currently masked, clear if so. - */ -#define UNMASK_IRQ(irq_num) \ - ICU_LOCK ; /* into critical reg */ \ - testl $IRQ_BIT(irq_num), apic_imen ; \ - je 7f ; /* bit clear, not masked */ \ - andl $~IRQ_BIT(irq_num), apic_imen ;/* clear mask bit */ \ - movl IOAPICADDR(irq_num), %ecx ; /* ioapic addr */ \ - movl REDIRIDX(irq_num), %eax ; /* get the index */ \ - movl %eax, (%ecx) ; /* write the index */ \ - movl IOAPIC_WINDOW(%ecx), %eax ; /* current value */ \ - andl $~IOART_INTMASK, %eax ; /* clear the mask */ \ - movl %eax, IOAPIC_WINDOW(%ecx) ; /* new value */ \ -7: ; /* already unmasked */ \ - ICU_UNLOCK - -/* - * Test to see whether we are handling an edge or level triggered INT. - * Level-triggered INTs have to be unmasked. - */ -#define UNMASK_LEVEL_IRQ(irq_num) \ - testl $IRQ_BIT(irq_num), apic_pin_trigger ; \ - jz 9f ; /* edge, don't unmask */ \ - UNMASK_IRQ(irq_num) ; \ -9: - -/* - * Macros for interrupt entry, call to handler, and exit. - */ - -#define FAST_INTR(irq_num, vec_name) \ - .text ; \ - SUPERALIGN_TEXT ; \ -IDTVEC(vec_name) ; \ - PUSH_FRAME ; \ - movl $KDSEL,%eax ; \ - mov %ax,%ds ; \ - mov %ax,%es ; \ - movl $KPSEL,%eax ; \ - mov %ax,%fs ; \ - FAKE_MCOUNT(13*4(%esp)) ; \ - movl PCPU(CURTHREAD),%ebx ; \ - cmpl $0,TD_CRITNEST(%ebx) ; \ - je 1f ; \ -; \ - movl $1,PCPU(INT_PENDING) ; \ - orl $IRQ_BIT(irq_num),PCPU(FPENDING) ; \ - MASK_LEVEL_IRQ(irq_num) ; \ - movl $0, lapic+LA_EOI ; \ - jmp 10f ; \ -1: ; \ - incl TD_CRITNEST(%ebx) ; \ - incl TD_INTR_NESTING_LEVEL(%ebx) ; \ - pushl intr_unit + (irq_num) * 4 ; \ - call *intr_handler + (irq_num) * 4 ; /* do the work ASAP */ \ - addl $4, %esp ; \ - movl $0, lapic+LA_EOI ; \ - lock ; \ - incl cnt+V_INTR ; /* book-keeping can wait */ \ - movl intr_countp + (irq_num) * 4, %eax ; \ - lock ; \ - incl (%eax) ; \ - decl TD_CRITNEST(%ebx) ; \ - cmpl $0,PCPU(INT_PENDING) ; \ - je 2f ; \ -; \ - call i386_unpend ; \ -2: ; \ - decl TD_INTR_NESTING_LEVEL(%ebx) ; \ -10: ; \ - MEXITCOUNT ; \ - jmp doreti - /* - * Restart a fast interrupt that was held up by a critical section. - * This routine is called from unpend(). unpend() ensures we are - * in a critical section and deals with the interrupt nesting level - * for us. If we previously masked the irq, we have to unmask it. - * - * We have a choice. We can regenerate the irq using the 'int' - * instruction or we can create a dummy frame and call the interrupt - * handler directly. I've chosen to use the dummy-frame method. + * I/O Interrupt Entry Point. Rather than having one entry point for + * each interrupt source, we use one entry point for each 32-bit word + * in the ISR. The handler determines the highest bit set in the ISR, + * translates that into a vector, and passes the vector to the + * lapic_handle_intr() function. */ -#define FAST_UNPEND(irq_num, vec_name) \ +#define ISR_VEC(index, vec_name) \ .text ; \ SUPERALIGN_TEXT ; \ IDTVEC(vec_name) ; \ -; \ - pushl %ebp ; \ - movl %esp, %ebp ; \ - PUSH_DUMMY ; \ - pushl intr_unit + (irq_num) * 4 ; \ - call *intr_handler + (irq_num) * 4 ; /* do the work ASAP */ \ - addl $4, %esp ; \ - lock ; \ - incl cnt+V_INTR ; /* book-keeping can wait */ \ - movl intr_countp + (irq_num) * 4, %eax ; \ - lock ; \ - incl (%eax) ; \ - UNMASK_LEVEL_IRQ(irq_num) ; \ - POP_DUMMY ; \ - popl %ebp ; \ - ret ; \ - - -/* - * Slow, threaded interrupts. - * - * XXX Most of the parameters here are obsolete. Fix this when we're - * done. - * XXX we really shouldn't return via doreti if we just schedule the - * interrupt handler and don't run anything. We could just do an - * iret. FIXME. - */ -#define INTR(irq_num, vec_name, maybe_extra_ipending) \ - .text ; \ - SUPERALIGN_TEXT ; \ -/* _XintrNN: entry point used by IDT/HWIs via _vec[]. */ \ -IDTVEC(vec_name) ; \ PUSH_FRAME ; \ movl $KDSEL, %eax ; /* reload with kernel's data segment */ \ mov %ax, %ds ; \ mov %ax, %es ; \ - movl $KPSEL, %eax ; \ + movl $KPSEL, %eax ; /* reload with per-CPU data segment */ \ mov %ax, %fs ; \ -; \ - maybe_extra_ipending ; \ -; \ - MASK_LEVEL_IRQ(irq_num) ; \ - EOI_IRQ(irq_num) ; \ -; \ - movl PCPU(CURTHREAD),%ebx ; \ - cmpl $0,TD_CRITNEST(%ebx) ; \ - je 1f ; \ - movl $1,PCPU(INT_PENDING) ; \ - orl $IRQ_BIT(irq_num),PCPU(IPENDING) ; \ - jmp 10f ; \ -1: ; \ + movl lapic, %edx ; /* pointer to local APIC */ \ + movl PCPU(CURTHREAD), %ebx ; \ + movl LA_ISR + 16 * (index)(%edx), %eax ; /* load ISR */ \ incl TD_INTR_NESTING_LEVEL(%ebx) ; \ -; \ - FAKE_MCOUNT(13*4(%esp)) ; /* XXX avoid dbl cnt */ \ - cmpl $0,PCPU(INT_PENDING) ; \ - je 9f ; \ - call i386_unpend ; \ -9: ; \ - pushl $irq_num; /* pass the IRQ */ \ - call sched_ithd ; \ - addl $4, %esp ; /* discard the parameter */ \ -; \ + bsrl %eax, %eax ; /* index of highset set bit in ISR */ \ + jz 2f ; \ + addl $(32 * index),%eax ; \ +1: ; \ + FAKE_MCOUNT(13*4(%esp)) ; /* XXX avoid double count */ \ + pushl %eax ; /* pass the IRQ */ \ + call lapic_handle_intr ; \ + addl $4, %esp ; /* discard parameter */ \ decl TD_INTR_NESTING_LEVEL(%ebx) ; \ -10: ; \ MEXITCOUNT ; \ - jmp doreti + jmp doreti ; \ +2: movl $-1, %eax ; /* send a vector of -1 */ \ + jmp 1b /* * Handle "spurious INTerrupts". @@ -257,6 +114,14 @@ IDTVEC(spuriousint) iret +MCOUNT_LABEL(bintr2) + ISR_VEC(1,apic_isr1) + ISR_VEC(2,apic_isr2) + ISR_VEC(3,apic_isr3) + ISR_VEC(4,apic_isr4) + ISR_VEC(5,apic_isr5) +MCOUNT_LABEL(eintr2) + #ifdef SMP /* * Global address space TLB shootdown. @@ -281,7 +146,8 @@ IDTVEC(invltlb) movl %cr3, %eax /* invalidate the TLB */ movl %eax, %cr3 - movl $0, lapic+LA_EOI /* End Of Interrupt to APIC */ + movl lapic, %eax + movl $0, LA_EOI(%eax) /* End Of Interrupt to APIC */ lock incl smp_tlb_wait @@ -313,7 +179,8 @@ IDTVEC(invlpg) movl smp_tlb_addr1, %eax invlpg (%eax) /* invalidate single page */ - movl $0, lapic+LA_EOI /* End Of Interrupt to APIC */ + movl lapic, %eax + movl $0, LA_EOI(%eax) /* End Of Interrupt to APIC */ lock incl smp_tlb_wait @@ -350,7 +217,8 @@ IDTVEC(invlrng) cmpl %eax, %edx jb 1b - movl $0, lapic+LA_EOI /* End Of Interrupt to APIC */ + movl lapic, %eax + movl $0, LA_EOI(%eax) /* End Of Interrupt to APIC */ lock incl smp_tlb_wait @@ -374,21 +242,15 @@ IDTVEC(hardclock) movl $KPSEL, %eax mov %ax, %fs - movl $0, lapic+LA_EOI /* End Of Interrupt to APIC */ + movl lapic, %edx + movl $0, LA_EOI(%edx) /* End Of Interrupt to APIC */ movl PCPU(CURTHREAD),%ebx - cmpl $0,TD_CRITNEST(%ebx) - je 1f - movl $1,PCPU(INT_PENDING) - orl $1,PCPU(SPENDING); - jmp 10f -1: incl TD_INTR_NESTING_LEVEL(%ebx) pushl $0 /* XXX convert trapframe to clockframe */ call forwarded_hardclock addl $4, %esp /* XXX convert clockframe to trapframe */ decl TD_INTR_NESTING_LEVEL(%ebx) -10: MEXITCOUNT jmp doreti @@ -406,23 +268,17 @@ IDTVEC(statclock) movl $KPSEL, %eax mov %ax, %fs - movl $0, lapic+LA_EOI /* End Of Interrupt to APIC */ + movl lapic, %edx + movl $0, LA_EOI(%edx) /* End Of Interrupt to APIC */ FAKE_MCOUNT(13*4(%esp)) movl PCPU(CURTHREAD),%ebx - cmpl $0,TD_CRITNEST(%ebx) - je 1f - movl $1,PCPU(INT_PENDING) - orl $2,PCPU(SPENDING); - jmp 10f -1: incl TD_INTR_NESTING_LEVEL(%ebx) pushl $0 /* XXX convert trapframe to clockframe */ call forwarded_statclock addl $4, %esp /* XXX convert clockframe to trapframe */ decl TD_INTR_NESTING_LEVEL(%ebx) -10: MEXITCOUNT jmp doreti @@ -444,7 +300,8 @@ IDTVEC(cpuast) movl $KPSEL, %eax mov %ax, %fs - movl $0, lapic+LA_EOI /* End Of Interrupt to APIC */ + movl lapic, %edx + movl $0, LA_EOI(%edx) /* End Of Interrupt to APIC */ FAKE_MCOUNT(13*4(%esp)) @@ -476,7 +333,8 @@ IDTVEC(cpustop) movl $KPSEL, %eax mov %ax, %fs - movl $0, lapic+LA_EOI /* End Of Interrupt to APIC */ + movl lapic, %eax + movl $0, LA_EOI(%eax) /* End Of Interrupt to APIC */ movl PCPU(CPUID), %eax imull $PCB_SIZE, %eax @@ -518,111 +376,6 @@ IDTVEC(cpustop) popl %ebp iret -#endif /* SMP */ - -MCOUNT_LABEL(bintr) - FAST_INTR(0,fastintr0) - FAST_INTR(1,fastintr1) - FAST_INTR(2,fastintr2) - FAST_INTR(3,fastintr3) - FAST_INTR(4,fastintr4) - FAST_INTR(5,fastintr5) - FAST_INTR(6,fastintr6) - FAST_INTR(7,fastintr7) - FAST_INTR(8,fastintr8) - FAST_INTR(9,fastintr9) - FAST_INTR(10,fastintr10) - FAST_INTR(11,fastintr11) - FAST_INTR(12,fastintr12) - FAST_INTR(13,fastintr13) - FAST_INTR(14,fastintr14) - FAST_INTR(15,fastintr15) - FAST_INTR(16,fastintr16) - FAST_INTR(17,fastintr17) - FAST_INTR(18,fastintr18) - FAST_INTR(19,fastintr19) - FAST_INTR(20,fastintr20) - FAST_INTR(21,fastintr21) - FAST_INTR(22,fastintr22) - FAST_INTR(23,fastintr23) - FAST_INTR(24,fastintr24) - FAST_INTR(25,fastintr25) - FAST_INTR(26,fastintr26) - FAST_INTR(27,fastintr27) - FAST_INTR(28,fastintr28) - FAST_INTR(29,fastintr29) - FAST_INTR(30,fastintr30) - FAST_INTR(31,fastintr31) -#define CLKINTR_PENDING movl $1,CNAME(clkintr_pending) -/* Threaded interrupts */ - INTR(0,intr0, CLKINTR_PENDING) - INTR(1,intr1,) - INTR(2,intr2,) - INTR(3,intr3,) - INTR(4,intr4,) - INTR(5,intr5,) - INTR(6,intr6,) - INTR(7,intr7,) - INTR(8,intr8,) - INTR(9,intr9,) - INTR(10,intr10,) - INTR(11,intr11,) - INTR(12,intr12,) - INTR(13,intr13,) - INTR(14,intr14,) - INTR(15,intr15,) - INTR(16,intr16,) - INTR(17,intr17,) - INTR(18,intr18,) - INTR(19,intr19,) - INTR(20,intr20,) - INTR(21,intr21,) - INTR(22,intr22,) - INTR(23,intr23,) - INTR(24,intr24,) - INTR(25,intr25,) - INTR(26,intr26,) - INTR(27,intr27,) - INTR(28,intr28,) - INTR(29,intr29,) - INTR(30,intr30,) - INTR(31,intr31,) - - FAST_UNPEND(0,fastunpend0) - FAST_UNPEND(1,fastunpend1) - FAST_UNPEND(2,fastunpend2) - FAST_UNPEND(3,fastunpend3) - FAST_UNPEND(4,fastunpend4) - FAST_UNPEND(5,fastunpend5) - FAST_UNPEND(6,fastunpend6) - FAST_UNPEND(7,fastunpend7) - FAST_UNPEND(8,fastunpend8) - FAST_UNPEND(9,fastunpend9) - FAST_UNPEND(10,fastunpend10) - FAST_UNPEND(11,fastunpend11) - FAST_UNPEND(12,fastunpend12) - FAST_UNPEND(13,fastunpend13) - FAST_UNPEND(14,fastunpend14) - FAST_UNPEND(15,fastunpend15) - FAST_UNPEND(16,fastunpend16) - FAST_UNPEND(17,fastunpend17) - FAST_UNPEND(18,fastunpend18) - FAST_UNPEND(19,fastunpend19) - FAST_UNPEND(20,fastunpend20) - FAST_UNPEND(21,fastunpend21) - FAST_UNPEND(22,fastunpend22) - FAST_UNPEND(23,fastunpend23) - FAST_UNPEND(24,fastunpend24) - FAST_UNPEND(25,fastunpend25) - FAST_UNPEND(26,fastunpend26) - FAST_UNPEND(27,fastunpend27) - FAST_UNPEND(28,fastunpend28) - FAST_UNPEND(29,fastunpend29) - FAST_UNPEND(30,fastunpend30) - FAST_UNPEND(31,fastunpend31) -MCOUNT_LABEL(eintr) - -#ifdef SMP /* * Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU. * @@ -640,7 +393,8 @@ IDTVEC(rendezvous) call smp_rendezvous_action - movl $0, lapic+LA_EOI /* End Of Interrupt to APIC */ + movl lapic, %eax + movl $0, LA_EOI(%eax) /* End Of Interrupt to APIC */ POP_FRAME iret @@ -658,16 +412,9 @@ IDTVEC(lazypmap) mov %ax, %fs call pmap_lazyfix_action - - movl $0, lapic+LA_EOI /* End Of Interrupt to APIC */ + + movl lapic, %eax + movl $0, LA_EOI(%eax) /* End Of Interrupt to APIC */ POP_FRAME iret #endif /* SMP */ - - .data - - .globl apic_pin_trigger -apic_pin_trigger: - .long 0 - - .text |