diff options
-rw-r--r-- | sys/amd64/amd64/apic_vector.S | 105 | ||||
-rw-r--r-- | sys/amd64/amd64/db_trace.c | 7 | ||||
-rw-r--r-- | sys/amd64/amd64/intr_machdep.c | 8 | ||||
-rw-r--r-- | sys/amd64/amd64/local_apic.c | 12 | ||||
-rw-r--r-- | sys/amd64/amd64/mp_machdep.c | 22 | ||||
-rw-r--r-- | sys/amd64/include/apicvar.h | 2 | ||||
-rw-r--r-- | sys/amd64/include/asmacros.h | 60 | ||||
-rw-r--r-- | sys/amd64/include/frame.h | 30 | ||||
-rw-r--r-- | sys/amd64/include/intr_machdep.h | 4 | ||||
-rw-r--r-- | sys/amd64/include/smp.h | 1 | ||||
-rw-r--r-- | sys/amd64/isa/atpic.c | 12 | ||||
-rw-r--r-- | sys/amd64/isa/atpic_vector.S | 22 | ||||
-rw-r--r-- | sys/amd64/isa/icu.h | 2 |
13 files changed, 101 insertions, 186 deletions
diff --git a/sys/amd64/amd64/apic_vector.S b/sys/amd64/amd64/apic_vector.S index 5d7685b..6fccaf2 100644 --- a/sys/amd64/amd64/apic_vector.S +++ b/sys/amd64/amd64/apic_vector.S @@ -42,53 +42,6 @@ #include "assym.s" /* - * Macros to create and destroy a trap frame. - */ -#define PUSH_FRAME \ - subq $TF_RIP,%rsp ; /* skip dummy tf_err and tf_trapno */ \ - testb $SEL_RPL_MASK,TF_CS(%rsp) ; /* come from kernel? */ \ - jz 1f ; /* Yes, dont swapgs again */ \ - swapgs ; \ -1: movq %rdi,TF_RDI(%rsp) ; \ - movq %rsi,TF_RSI(%rsp) ; \ - movq %rdx,TF_RDX(%rsp) ; \ - movq %rcx,TF_RCX(%rsp) ; \ - movq %r8,TF_R8(%rsp) ; \ - movq %r9,TF_R9(%rsp) ; \ - movq %rax,TF_RAX(%rsp) ; \ - movq %rbx,TF_RBX(%rsp) ; \ - movq %rbp,TF_RBP(%rsp) ; \ - movq %r10,TF_R10(%rsp) ; \ - movq %r11,TF_R11(%rsp) ; \ - movq %r12,TF_R12(%rsp) ; \ - movq %r13,TF_R13(%rsp) ; \ - movq %r14,TF_R14(%rsp) ; \ - movq %r15,TF_R15(%rsp) - -#define POP_FRAME \ - movq TF_RDI(%rsp),%rdi ; \ - movq TF_RSI(%rsp),%rsi ; \ - movq TF_RDX(%rsp),%rdx ; \ - movq TF_RCX(%rsp),%rcx ; \ - movq TF_R8(%rsp),%r8 ; \ - movq TF_R9(%rsp),%r9 ; \ - movq TF_RAX(%rsp),%rax ; \ - movq TF_RBX(%rsp),%rbx ; \ - movq TF_RBP(%rsp),%rbp ; \ - movq TF_R10(%rsp),%r10 ; \ - movq TF_R11(%rsp),%r11 ; \ - movq TF_R12(%rsp),%r12 ; \ - movq TF_R13(%rsp),%r13 ; \ - movq TF_R14(%rsp),%r14 ; \ - movq TF_R15(%rsp),%r15 ; \ - testb $SEL_RPL_MASK,TF_CS(%rsp) ; /* come from kernel? */ \ - jz 1f ; /* keep kernel GS.base */ \ - cli ; \ - swapgs ; \ -1: addq $TF_RIP,%rsp /* skip over tf_err, tf_trapno */ - - -/* * I/O Interrupt Entry Point. Rather than having one entry point for * each interrupt source, we use one entry point for each 32-bit word * in the ISR. The handler determines the highest bit set in the ISR, @@ -107,7 +60,7 @@ IDTVEC(vec_name) ; \ jz 2f ; \ addl $(32 * index),%eax ; \ 1: ; \ - movq %rax, %rdi ; /* pass the IRQ */ \ + movl %eax, %edi ; /* pass the IRQ */ \ call lapic_handle_intr ; \ MEXITCOUNT ; \ jmp doreti ; \ @@ -144,10 +97,6 @@ IDTVEC(spuriousint) SUPERALIGN_TEXT IDTVEC(timerint) PUSH_FRAME - - movq lapic, %rdx - movl $0, LA_EOI(%rdx) /* End Of Interrupt to APIC */ - FAKE_MCOUNT(TF_RIP(%rsp)) call lapic_handle_timer @@ -227,8 +176,7 @@ IDTVEC(invlrng) */ .text SUPERALIGN_TEXT -IDTVEC(ipi_intr_bitmap_handler) - +IDTVEC(ipi_intr_bitmap_handler) PUSH_FRAME movq lapic, %rdx @@ -241,27 +189,6 @@ IDTVEC(ipi_intr_bitmap_handler) jmp doreti /* - * Executed by a CPU when it receives an Xcpuast IPI from another CPU, - * - * The other CPU has already executed aston() or need_resched() on our - * current process, so we simply need to ack the interrupt and return - * via doreti to run ast(). - */ - - .text - SUPERALIGN_TEXT -IDTVEC(cpuast) - PUSH_FRAME - - movq lapic, %rdx - movl $0, LA_EOI(%rdx) /* End Of Interrupt to APIC */ - - FAKE_MCOUNT(TF_RIP(%rsp)) - - MEXITCOUNT - jmp doreti - -/* * Executed by a CPU when it receives an Xcpustop IPI from another CPU, * * - Signals its receipt. @@ -276,34 +203,8 @@ IDTVEC(cpustop) movq lapic, %rax movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */ - movl PCPU(CPUID), %eax - imull $PCB_SIZE, %eax - leaq stoppcbs(%rax), %rdi - call savectx /* Save process context */ - - movl PCPU(CPUID), %eax - - lock - btsl %eax, stopped_cpus /* stopped_cpus |= (1<<id) */ -1: - btl %eax, started_cpus /* while (!(started_cpus & (1<<id))) */ - jnc 1b - - lock - btrl %eax, started_cpus /* started_cpus &= ~(1<<id) */ - lock - btrl %eax, stopped_cpus /* stopped_cpus &= ~(1<<id) */ - - test %eax, %eax - jnz 2f - - movq cpustop_restartfunc, %rax - testq %rax, %rax - jz 2f - movq $0, cpustop_restartfunc /* One-shot */ + call cpustop_handler - call *%rax -2: POP_FRAME iretq diff --git a/sys/amd64/amd64/db_trace.c b/sys/amd64/amd64/db_trace.c index 567f66e..d426395 100644 --- a/sys/amd64/amd64/db_trace.c +++ b/sys/amd64/amd64/db_trace.c @@ -318,8 +318,11 @@ db_nextframe(struct amd64_frame **fp, db_addr_t *ip, struct thread *td) strcmp(name, "nmi_calltrap") == 0) frame_type = TRAP; else if (strncmp(name, "Xatpic_intr", 11) == 0 || - strncmp(name, "Xatpic_fastintr", 15) == 0 || - strncmp(name, "Xapic_isr", 9) == 0) + strncmp(name, "Xapic_isr", 9) == 0 || + strcmp(name, "Xtimerint") == 0 || + strcmp(name, "Xipi_intr_bitmap_handler") == 0 || + strcmp(name, "Xcpustop") == 0 || + strcmp(name, "Xrendezvous") == 0) frame_type = INTERRUPT; else if (strcmp(name, "Xfast_syscall") == 0) frame_type = SYSCALL; diff --git a/sys/amd64/amd64/intr_machdep.c b/sys/amd64/amd64/intr_machdep.c index 9092d87..7338bae 100644 --- a/sys/amd64/amd64/intr_machdep.c +++ b/sys/amd64/amd64/intr_machdep.c @@ -149,7 +149,7 @@ intr_config_intr(int vector, enum intr_trigger trig, enum intr_polarity pol) } void -intr_execute_handlers(struct intsrc *isrc, struct intrframe *iframe) +intr_execute_handlers(struct intsrc *isrc, struct trapframe *frame) { struct thread *td; struct intr_event *ie; @@ -197,7 +197,7 @@ intr_execute_handlers(struct intsrc *isrc, struct intrframe *iframe) * Execute fast interrupt handlers directly. * To support clock handlers, if a handler registers * with a NULL argument, then we pass it a pointer to - * an intrframe as its argument. + * a trapframe as its argument. */ td->td_intr_nesting_level++; thread = 0; @@ -208,10 +208,10 @@ intr_execute_handlers(struct intsrc *isrc, struct intrframe *iframe) continue; } CTR4(KTR_INTR, "%s: exec %p(%p) for %s", __func__, - ih->ih_handler, ih->ih_argument == NULL ? iframe : + ih->ih_handler, ih->ih_argument == NULL ? frame : ih->ih_argument, ih->ih_name); if (ih->ih_argument == NULL) - ih->ih_handler(iframe); + ih->ih_handler(frame); else ih->ih_handler(ih->ih_argument); } diff --git a/sys/amd64/amd64/local_apic.c b/sys/amd64/amd64/local_apic.c index 4debf4b..e81ac65 100644 --- a/sys/amd64/amd64/local_apic.c +++ b/sys/amd64/amd64/local_apic.c @@ -46,7 +46,6 @@ __FBSDID("$FreeBSD$"); #include <sys/mutex.h> #include <sys/pcpu.h> #include <sys/smp.h> -#include <sys/proc.h> #include <vm/vm.h> #include <vm/pmap.h> @@ -607,14 +606,13 @@ lapic_eoi(void) } void -lapic_handle_intr(void *cookie, struct intrframe frame) +lapic_handle_intr(int vector, struct trapframe frame) { struct intsrc *isrc; - int vec = (uintptr_t)cookie; - if (vec == -1) + if (vector == -1) panic("Couldn't get vector from ISR!"); - isrc = intr_lookup_source(apic_idt_to_irq(vec)); + isrc = intr_lookup_source(apic_idt_to_irq(vector)); intr_execute_handlers(isrc, &frame); } @@ -623,6 +621,10 @@ lapic_handle_timer(struct clockframe frame) { struct lapic *la; + /* Send EOI first thing. */ + lapic_eoi(); + + /* Look up our local APIC structure for the tick counters. */ la = &lapics[PCPU_GET(apic_id)]; (*la->la_timer_count)++; critical_enter(); diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c index 2e02f2c..c042354 100644 --- a/sys/amd64/amd64/mp_machdep.c +++ b/sys/amd64/amd64/mp_machdep.c @@ -1068,15 +1068,29 @@ ipi_nmi_selected(u_int32_t cpus) } int -ipi_nmi_handler() +ipi_nmi_handler(void) { - int cpu = PCPU_GET(cpuid); int cpumask = PCPU_GET(cpumask); if (!(ipi_nmi_pending & cpumask)) return 1; atomic_clear_int(&ipi_nmi_pending, cpumask); + cpustop_handler(); + return 0; +} + +#endif /* STOP_NMI */ + +/* + * Handle an IPI_STOP by saving our current context and spinning until we + * are resumed. + */ +void +cpustop_handler(void) +{ + int cpu = PCPU_GET(cpuid); + int cpumask = PCPU_GET(cpumask); savectx(&stoppcbs[cpu]); @@ -1094,11 +1108,7 @@ ipi_nmi_handler() cpustop_restartfunc(); cpustop_restartfunc = NULL; } - - return 0; } - -#endif /* STOP_NMI */ /* * This is called once the rest of the system is up and running and we're diff --git a/sys/amd64/include/apicvar.h b/sys/amd64/include/apicvar.h index 10dae1e..ff2dd26 100644 --- a/sys/amd64/include/apicvar.h +++ b/sys/amd64/include/apicvar.h @@ -201,7 +201,7 @@ int lapic_intr_pending(u_int vector); void lapic_ipi_raw(register_t icrlo, u_int dest); void lapic_ipi_vectored(u_int vector, int dest); int lapic_ipi_wait(int delay); -void lapic_handle_intr(void *cookie, struct intrframe frame); +void lapic_handle_intr(int vector, struct trapframe frame); void lapic_handle_timer(struct clockframe frame); void lapic_set_logical_id(u_int apic_id, u_int cluster, u_int cluster_id); int lapic_set_lvt_mask(u_int apic_id, u_int lvt, u_char masked); diff --git a/sys/amd64/include/asmacros.h b/sys/amd64/include/asmacros.h index 7ca8c62..75d15ac 100644 --- a/sys/amd64/include/asmacros.h +++ b/sys/amd64/include/asmacros.h @@ -59,12 +59,6 @@ #define NON_GPROF_ENTRY(name) GEN_ENTRY(name) #define NON_GPROF_RET .byte 0xc3 /* opcode for `ret' */ -#ifdef LOCORE -#define PCPU(member) %gs:PC_ ## member -#define PCPU_ADDR(member, reg) movq %gs:PC_PRVSPACE,reg; \ - addq $PC_ ## member,reg -#endif - #ifdef GPROF /* * __mcount is like [.]mcount except that doesn't require its caller to set @@ -141,6 +135,60 @@ #define IDTVEC(name) ALIGN_TEXT; .globl __CONCAT(X,name); \ .type __CONCAT(X,name),@function; __CONCAT(X,name): +/* + * Macros to create and destroy a trap frame. + */ +#define PUSH_FRAME \ + subq $TF_RIP,%rsp ; /* skip dummy tf_err and tf_trapno */ \ + testb $SEL_RPL_MASK,TF_CS(%rsp) ; /* come from kernel? */ \ + jz 1f ; /* Yes, dont swapgs again */ \ + swapgs ; \ +1: movq %rdi,TF_RDI(%rsp) ; \ + movq %rsi,TF_RSI(%rsp) ; \ + movq %rdx,TF_RDX(%rsp) ; \ + movq %rcx,TF_RCX(%rsp) ; \ + movq %r8,TF_R8(%rsp) ; \ + movq %r9,TF_R9(%rsp) ; \ + movq %rax,TF_RAX(%rsp) ; \ + movq %rbx,TF_RBX(%rsp) ; \ + movq %rbp,TF_RBP(%rsp) ; \ + movq %r10,TF_R10(%rsp) ; \ + movq %r11,TF_R11(%rsp) ; \ + movq %r12,TF_R12(%rsp) ; \ + movq %r13,TF_R13(%rsp) ; \ + movq %r14,TF_R14(%rsp) ; \ + movq %r15,TF_R15(%rsp) + +#define POP_FRAME \ + movq TF_RDI(%rsp),%rdi ; \ + movq TF_RSI(%rsp),%rsi ; \ + movq TF_RDX(%rsp),%rdx ; \ + movq TF_RCX(%rsp),%rcx ; \ + movq TF_R8(%rsp),%r8 ; \ + movq TF_R9(%rsp),%r9 ; \ + movq TF_RAX(%rsp),%rax ; \ + movq TF_RBX(%rsp),%rbx ; \ + movq TF_RBP(%rsp),%rbp ; \ + movq TF_R10(%rsp),%r10 ; \ + movq TF_R11(%rsp),%r11 ; \ + movq TF_R12(%rsp),%r12 ; \ + movq TF_R13(%rsp),%r13 ; \ + movq TF_R14(%rsp),%r14 ; \ + movq TF_R15(%rsp),%r15 ; \ + testb $SEL_RPL_MASK,TF_CS(%rsp) ; /* come from kernel? */ \ + jz 1f ; /* keep kernel GS.base */ \ + cli ; \ + swapgs ; \ +1: addq $TF_RIP,%rsp /* skip over tf_err, tf_trapno */ + +/* + * Access per-CPU data. + */ +#define PCPU(member) %gs:PC_ ## member +#define PCPU_ADDR(member, reg) \ + movq %gs:PC_PRVSPACE, reg ; \ + addq $PC_ ## member, reg + #endif /* LOCORE */ #endif /* !_MACHINE_ASMACROS_H_ */ diff --git a/sys/amd64/include/frame.h b/sys/amd64/include/frame.h index 1996546..2bf9f3b 100644 --- a/sys/amd64/include/frame.h +++ b/sys/amd64/include/frame.h @@ -76,36 +76,6 @@ struct trapframe { register_t tf_ss; }; -/* Interrupt stack frame */ - -struct intrframe { - register_t if_rdi; - register_t if_rsi; - register_t if_rdx; - register_t if_rcx; - register_t if_r8; - register_t if_r9; - register_t if_rax; - register_t if_rbx; - register_t if_rbp; - register_t if_r10; - register_t if_r11; - register_t if_r12; - register_t if_r13; - register_t if_r14; - register_t if_r15; - register_t :64; /* compat with trap frame - trapno */ - register_t :64; /* compat with trap frame - addr */ - register_t :64; /* compat with trap frame - flags */ - register_t :64; /* compat with trap frame - err */ - /* below portion defined in hardware */ - register_t if_rip; - register_t if_cs; - register_t if_rflags; - register_t if_rsp; - register_t if_ss; -}; - /* frame of clock (same as interrupt frame) */ struct clockframe { diff --git a/sys/amd64/include/intr_machdep.h b/sys/amd64/include/intr_machdep.h index d174ed0..4d5743a 100644 --- a/sys/amd64/include/intr_machdep.h +++ b/sys/amd64/include/intr_machdep.h @@ -107,7 +107,7 @@ struct intsrc { u_int is_index; }; -struct intrframe; +struct trapframe; extern struct mtx icu_lock; extern int elcr_found; @@ -121,7 +121,7 @@ int intr_add_handler(const char *name, int vector, driver_intr_t handler, void *arg, enum intr_type flags, void **cookiep); int intr_config_intr(int vector, enum intr_trigger trig, enum intr_polarity pol); -void intr_execute_handlers(struct intsrc *isrc, struct intrframe *iframe); +void intr_execute_handlers(struct intsrc *isrc, struct trapframe *frame); struct intsrc *intr_lookup_source(int vector); int intr_register_source(struct intsrc *isrc); int intr_remove_handler(void *cookie); diff --git a/sys/amd64/include/smp.h b/sys/amd64/include/smp.h index ba967c6..45870fe 100644 --- a/sys/amd64/include/smp.h +++ b/sys/amd64/include/smp.h @@ -46,6 +46,7 @@ inthand_t /* functions in mp_machdep.c */ void cpu_add(u_int apic_id, char boot_cpu); +void cpustop_handler(void); void init_secondary(void); void ipi_selected(u_int cpus, u_int ipi); void ipi_all(u_int ipi); diff --git a/sys/amd64/isa/atpic.c b/sys/amd64/isa/atpic.c index 88d461f..906edda 100644 --- a/sys/amd64/isa/atpic.c +++ b/sys/amd64/isa/atpic.c @@ -45,7 +45,6 @@ __FBSDID("$FreeBSD$"); #include <sys/lock.h> #include <sys/module.h> #include <sys/mutex.h> -#include <sys/proc.h> #include <machine/cpufunc.h> #include <machine/frame.h> @@ -478,19 +477,18 @@ atpic_init(void *dummy __unused) SYSINIT(atpic_init, SI_SUB_INTR, SI_ORDER_SECOND + 1, atpic_init, NULL) void -atpic_handle_intr(void *cookie, struct intrframe iframe) +atpic_handle_intr(u_int vector, struct trapframe frame) { struct intsrc *isrc; - int vec = (uintptr_t)cookie; - KASSERT(vec < NUM_ISA_IRQS, ("unknown int %d\n", vec)); - isrc = &atintrs[vec].at_intsrc; + KASSERT(vector < NUM_ISA_IRQS, ("unknown int %u\n", vector)); + isrc = &atintrs[vector].at_intsrc; /* * If we don't have an event, see if this is a spurious * interrupt. */ - if (isrc->is_event == NULL && (vec == 7 || vec == 15)) { + if (isrc->is_event == NULL && (vector == 7 || vector == 15)) { int port, isr; /* @@ -506,7 +504,7 @@ atpic_handle_intr(void *cookie, struct intrframe iframe) if ((isr & IRQ_MASK(7)) == 0) return; } - intr_execute_handlers(isrc, &iframe); + intr_execute_handlers(isrc, &frame); } #ifdef DEV_ISA diff --git a/sys/amd64/isa/atpic_vector.S b/sys/amd64/isa/atpic_vector.S index 37390a0..45027d8 100644 --- a/sys/amd64/isa/atpic_vector.S +++ b/sys/amd64/isa/atpic_vector.S @@ -47,27 +47,9 @@ .text ; \ SUPERALIGN_TEXT ; \ IDTVEC(vec_name) ; \ - subq $TF_RIP,%rsp ; /* skip dummy tf_err and tf_trapno */ \ - testb $SEL_RPL_MASK,TF_CS(%rsp) ; /* come from kernel? */ \ - jz 1f ; /* Yes, dont swapgs again */ \ - swapgs ; \ -1: movq %rdi,TF_RDI(%rsp) ; \ - movq %rsi,TF_RSI(%rsp) ; \ - movq %rdx,TF_RDX(%rsp) ; \ - movq %rcx,TF_RCX(%rsp) ; \ - movq %r8,TF_R8(%rsp) ; \ - movq %r9,TF_R9(%rsp) ; \ - movq %rax,TF_RAX(%rsp) ; \ - movq %rbx,TF_RBX(%rsp) ; \ - movq %rbp,TF_RBP(%rsp) ; \ - movq %r10,TF_R10(%rsp) ; \ - movq %r11,TF_R11(%rsp) ; \ - movq %r12,TF_R12(%rsp) ; \ - movq %r13,TF_R13(%rsp) ; \ - movq %r14,TF_R14(%rsp) ; \ - movq %r15,TF_R15(%rsp) ; \ + PUSH_FRAME ; \ FAKE_MCOUNT(TF_RIP(%rsp)) ; \ - movq $irq_num, %rdi; /* pass the IRQ */ \ + movl $irq_num, %edi; /* pass the IRQ */ \ call atpic_handle_intr ; \ MEXITCOUNT ; \ jmp doreti diff --git a/sys/amd64/isa/icu.h b/sys/amd64/isa/icu.h index f25fac6..ce01253 100644 --- a/sys/amd64/isa/icu.h +++ b/sys/amd64/isa/icu.h @@ -43,7 +43,7 @@ #define ICU_IMR_OFFSET 1 -void atpic_handle_intr(void *cookie, struct intrframe iframe); +void atpic_handle_intr(u_int vector, struct trapframe frame); void atpic_startup(void); #endif /* !_AMD64_ISA_ICU_H_ */ |