From 769e0f974d8929599ba599ac496510fffc90ff34 Mon Sep 17 00:00:00 2001 From: jasone Date: Thu, 7 Sep 2000 01:33:02 +0000 Subject: Major update to the way synchronization is done in the kernel. Highlights include: * Mutual exclusion is used instead of spl*(). See mutex(9). (Note: The alpha port is still in transition and currently uses both.) * Per-CPU idle processes. * Interrupts are run in their own separate kernel threads and can be preempted (i386 only). Partially contributed by: BSDi (BSD/OS) Submissions by (at least): cp, dfr, dillon, grog, jake, jhb, sheldonh --- sys/alpha/alpha/clock.c | 204 ++++++- sys/alpha/alpha/genassym.c | 19 + sys/alpha/alpha/interrupt.c | 41 +- sys/alpha/alpha/ipl_funcs.c | 12 +- sys/alpha/alpha/locore.s | 50 +- sys/alpha/alpha/machdep.c | 67 ++- sys/alpha/alpha/mem.c | 3 + sys/alpha/alpha/mp_machdep.c | 1115 +++++++++++++++++++++++++++++++++++++++ sys/alpha/alpha/pmap.c | 126 +++-- sys/alpha/alpha/prom.c | 1 - sys/alpha/alpha/support.s | 34 +- sys/alpha/alpha/swtch.s | 45 +- sys/alpha/alpha/synch_machdep.c | 529 +++++++++++++++++++ sys/alpha/alpha/trap.c | 43 +- sys/alpha/alpha/vm_machdep.c | 11 +- sys/alpha/include/asm.h | 6 +- sys/alpha/include/cpu.h | 6 +- sys/alpha/include/cpufunc.h | 28 + sys/alpha/include/globaldata.h | 79 +++ sys/alpha/include/globals.h | 63 +++ sys/alpha/include/ipl.h | 15 + sys/alpha/include/lock.h | 32 +- sys/alpha/include/mutex.h | 563 ++++++++++++++++++++ sys/alpha/include/param.h | 4 + sys/alpha/include/pcb.h | 9 +- sys/alpha/include/pcpu.h | 79 +++ sys/alpha/include/pmap.h | 8 +- sys/alpha/include/proc.h | 8 + sys/alpha/include/rpb.h | 54 +- sys/alpha/include/smp.h | 53 +- 30 files changed, 3111 insertions(+), 196 deletions(-) create mode 100644 sys/alpha/alpha/mp_machdep.c create mode 100644 sys/alpha/alpha/synch_machdep.c create mode 100644 sys/alpha/include/globaldata.h create mode 100644 sys/alpha/include/globals.h create mode 100644 sys/alpha/include/mutex.h create mode 100644 sys/alpha/include/pcpu.h (limited to 'sys/alpha') diff --git a/sys/alpha/alpha/clock.c b/sys/alpha/alpha/clock.c index 88adaa4..500d169 100644 --- a/sys/alpha/alpha/clock.c +++ b/sys/alpha/alpha/clock.c @@ -43,6 +43,8 @@ * @(#)clock.c 8.1 (Berkeley) 6/10/93 */ +#include "opt_clock.h" + #include /* RCS ID & Copyright macro defns */ #include @@ -80,8 +82,23 @@ int disable_rtc_set; /* disable resettodr() if != 0 */ int wall_cmos_clock; /* wall CMOS clock assumed if != 0 */ static int beeping = 0; +#define TIMER_DIV(x) ((timer_freq + (x) / 2) / (x)) + +#ifndef TIMER_FREQ +#define TIMER_FREQ 1193182 +#endif +u_int32_t timer_freq = TIMER_FREQ; +int timer0_max_count; + +static u_int32_t i8254_lastcount; +static u_int32_t i8254_offset; +static int i8254_ticked; +static int clkintr_pending = 0; + extern int cycles_per_sec; +extern int ncpus; +static timecounter_get_t i8254_get_timecount; static timecounter_get_t alpha_get_timecount; static struct timecounter alpha_timecounter = { @@ -95,6 +112,17 @@ static struct timecounter alpha_timecounter = { SYSCTL_OPAQUE(_debug, OID_AUTO, alpha_timecounter, CTLFLAG_RD, &alpha_timecounter, sizeof(alpha_timecounter), "S,timecounter", ""); +static struct timecounter i8254_timecounter = { + i8254_get_timecount, /* get_timecount */ + 0, /* no poll_pps */ + ~0u, /* counter_mask */ + 0, /* frequency */ + "i8254" /* name */ +}; + +SYSCTL_OPAQUE(_debug, OID_AUTO, i8254_timecounter, CTLFLAG_RD, + &i8254_timecounter, sizeof(i8254_timecounter), "S,timecounter", ""); + /* Values for timerX_state: */ #define RELEASED 0 #define RELEASE_PENDING 1 @@ -120,11 +148,14 @@ static u_int32_t max_cycles_per_tick; static u_int32_t last_time; static void handleclock(void* arg); -static u_int32_t calibrate_clocks(u_int32_t firmware_freq); +static void calibrate_clocks(u_int32_t firmware_freq, + u_int32_t *pcc, u_int32_t *timer); +static void set_timer_freq(u_int freq, int intr_freq); void clockattach(device_t dev) { + u_int32_t pcc, freq, delta; /* * Just bookkeeping. @@ -132,7 +163,33 @@ clockattach(device_t dev) if (clockdev) panic("clockattach: multiple clocks"); clockdev = dev; - cycles_per_sec = calibrate_clocks(cycles_per_sec); + + calibrate_clocks(cycles_per_sec, &pcc, &freq); + cycles_per_sec = pcc; + + /* + * Use the calibrated i8254 frequency if it seems reasonable. + * Otherwise use the default, and don't use the calibrated i586 + * frequency. + */ + delta = freq > timer_freq ? freq - timer_freq : timer_freq - freq; + if (delta < timer_freq / 100) { +#ifndef CLK_USE_I8254_CALIBRATION + if (bootverbose) + printf( +"CLK_USE_I8254_CALIBRATION not specified - using default frequency\n"); + freq = timer_freq; +#endif + timer_freq = freq; + } else { + if (bootverbose) + printf( + "%d Hz differs from default of %d Hz by more than 1%%\n", + freq, timer_freq); + } + set_timer_freq(timer_freq, hz); + i8254_timecounter.tc_frequency = timer_freq; + #ifdef EVCNT_COUNTERS evcnt_attach(dev, "intr", &clock_intr_evcnt); #endif @@ -190,8 +247,12 @@ cpu_initclocks() scaled_ticks_per_cycle = ((u_int64_t)hz << FIX_SHIFT) / freq; max_cycles_per_tick = 2*freq / hz; - alpha_timecounter.tc_frequency = freq; - tc_init(&alpha_timecounter); + tc_init(&i8254_timecounter); + + if (ncpus == 1) { + alpha_timecounter.tc_frequency = freq; + tc_init(&alpha_timecounter); + } stathz = 128; platform.clockintr = (void (*) __P((void *))) handleclock; @@ -202,15 +263,36 @@ cpu_initclocks() CLOCK_INIT(clockdev); } -static u_int32_t -calibrate_clocks(u_int32_t firmware_freq) +static int +getit(void) +{ + int high, low; + int s; + + s = splhigh(); + + /* Select timer0 and latch counter value. */ + outb(TIMER_MODE, TIMER_SEL0 | TIMER_LATCH); + + low = inb(TIMER_CNTR0); + high = inb(TIMER_CNTR0); + + splx(s); + return ((high << 8) | low); +} + +static void +calibrate_clocks(u_int32_t firmware_freq, u_int32_t *pcc, u_int32_t *timer) { u_int32_t start_pcc, stop_pcc; + u_int count, prev_count, tot_count; int sec, start_sec; if (bootverbose) printf("Calibrating clock(s) ... "); + set_timer_freq(timer_freq, hz); + /* Read the mc146818A seconds counter. */ if (CLOCK_GETSECS(clockdev, &sec)) goto fail; @@ -224,16 +306,36 @@ calibrate_clocks(u_int32_t firmware_freq) break; } - /* Start keeping track of the PCC. */ + /* Start keeping track of the PCC and i8254. */ + prev_count = getit(); + if (prev_count == 0) + goto fail; + tot_count = 0; + start_pcc = alpha_rpcc(); /* - * Wait for the mc146818A seconds counter to change. + * Wait for the mc146818A seconds counter to change. Read the i8254 + * counter for each iteration since this is convenient and only + * costs a few usec of inaccuracy. The timing of the final reads + * of the counters almost matches the timing of the initial reads, + * so the main cause of inaccuracy is the varying latency from + * inside getit() or rtcin(RTC_STATUSA) to the beginning of the + * rtcin(RTC_SEC) that returns a changed seconds count. The + * maximum inaccuracy from this cause is < 10 usec on 486's. */ start_sec = sec; for (;;) { if (CLOCK_GETSECS(clockdev, &sec)) goto fail; + count = getit(); + if (count == 0) + goto fail; + if (count > prev_count) + tot_count += prev_count - (count - timer0_max_count); + else + tot_count += prev_count - count; + prev_count = count; if (sec != start_sec) break; } @@ -246,29 +348,55 @@ calibrate_clocks(u_int32_t firmware_freq) if (bootverbose) { printf("PCC clock: %u Hz (firmware %u Hz)\n", stop_pcc - start_pcc, firmware_freq); + printf("i8254 clock: %u Hz\n", tot_count); } - return (stop_pcc - start_pcc); + *pcc = stop_pcc - start_pcc; + *timer = tot_count; + return; fail: if (bootverbose) printf("failed, using firmware default of %u Hz\n", firmware_freq); - return (firmware_freq); + + *pcc = firmware_freq; + *timer = 0; + return; +} + +static void +set_timer_freq(u_int freq, int intr_freq) +{ + int new_timer0_max_count; + int s; + + s = splhigh(); + timer_freq = freq; + new_timer0_max_count = TIMER_DIV(intr_freq); + if (new_timer0_max_count != timer0_max_count) { + timer0_max_count = new_timer0_max_count; + outb(TIMER_MODE, TIMER_SEL0 | TIMER_RATEGEN | TIMER_16BIT); + outb(TIMER_CNTR0, timer0_max_count & 0xff); + outb(TIMER_CNTR0, timer0_max_count >> 8); + } + splx(s); } static void handleclock(void* arg) { - u_int32_t now = alpha_rpcc(); - u_int32_t delta = now - last_time; - last_time = now; - - if (delta > max_cycles_per_tick) { - int i, missed_ticks; - missed_ticks = (delta * scaled_ticks_per_cycle) >> FIX_SHIFT; - for (i = 0; i < missed_ticks; i++) - hardclock(arg); + if (timecounter->tc_get_timecount == i8254_get_timecount) { + int s = splhigh(); + if (i8254_ticked) + i8254_ticked = 0; + else { + i8254_offset += timer0_max_count; + i8254_lastcount = 0; + } + clkintr_pending = 0; + splx(s); } + hardclock(arg); setdelayed(); } @@ -433,6 +561,35 @@ resettodr() } static unsigned +i8254_get_timecount(struct timecounter *tc) +{ + u_int count; + u_int high, low; + int s; + + s = splhigh(); + + /* Select timer0 and latch counter value. */ + outb(TIMER_MODE, TIMER_SEL0 | TIMER_LATCH); + + low = inb(TIMER_CNTR0); + high = inb(TIMER_CNTR0); + count = timer0_max_count - ((high << 8) | low); + if (count < i8254_lastcount || + (!i8254_ticked && (clkintr_pending || + ((count < 20) && (inb(IO_ICU1) & 1))) + )) { + i8254_ticked = 1; + i8254_offset += timer0_max_count; + } + i8254_lastcount = count; + count += i8254_offset; + + splx(s); + return (count); +} + +static unsigned alpha_get_timecount(struct timecounter* tc) { return alpha_rpcc(); @@ -477,15 +634,6 @@ sysbeepstop(void *chan) beeping = 0; } -/* - * Frequency of all three count-down timers; (TIMER_FREQ/freq) is the - * appropriate count to generate a frequency of freq hz. - */ -#ifndef TIMER_FREQ -#define TIMER_FREQ 1193182 -#endif -#define TIMER_DIV(x) ((TIMER_FREQ+(x)/2)/(x)) - int sysbeep(int pitch, int period) { diff --git a/sys/alpha/alpha/genassym.c b/sys/alpha/alpha/genassym.c index a67f2d1..066d87b 100644 --- a/sys/alpha/alpha/genassym.c +++ b/sys/alpha/alpha/genassym.c @@ -51,8 +51,11 @@ #include #include #include +#include #include #include +#include +#include #include #include #include @@ -66,6 +69,21 @@ #include #include +#include "opt_smp.h" + +ASSYM(GD_CURPROC, offsetof(struct globaldata, gd_curproc)); +ASSYM(GD_FPCURPROC, offsetof(struct globaldata, gd_fpcurproc)); +ASSYM(GD_CURPCB, offsetof(struct globaldata, gd_curpcb)); +ASSYM(GD_SWITCHTIME, offsetof(struct globaldata, gd_switchtime)); +ASSYM(GD_CPUNO, offsetof(struct globaldata, gd_cpuno)); +ASSYM(GD_IDLEPCBPHYS, offsetof(struct globaldata, gd_idlepcbphys)); +ASSYM(GD_ASTPENDING, offsetof(struct globaldata, gd_astpending)); + +ASSYM(MTX_LOCK, offsetof(struct mtx, mtx_lock)); +ASSYM(MTX_RECURSE, offsetof(struct mtx, mtx_recurse)); +ASSYM(MTX_SAVEIPL, offsetof(struct mtx, mtx_saveipl)); +ASSYM(MTX_UNOWNED, MTX_UNOWNED); + ASSYM(P_ADDR, offsetof(struct proc, p_addr)); ASSYM(P_MD_FLAGS, offsetof(struct proc, p_md.md_flags)); ASSYM(P_MD_PCBPADDR, offsetof(struct proc, p_md.md_pcbpaddr)); @@ -81,6 +99,7 @@ ASSYM(PTESIZE, PTESIZE); ASSYM(U_PCB_ONFAULT, offsetof(struct user, u_pcb.pcb_onfault)); ASSYM(U_PCB_HWPCB_KSP, offsetof(struct user, u_pcb.pcb_hw.apcb_ksp)); ASSYM(U_PCB_CONTEXT, offsetof(struct user, u_pcb.pcb_context)); +ASSYM(U_PCB_SCHEDNEST, offsetof(struct user, u_pcb.pcb_schednest)); ASSYM(PCB_HW, offsetof(struct pcb, pcb_hw)); diff --git a/sys/alpha/alpha/interrupt.c b/sys/alpha/alpha/interrupt.c index deedefe..20f621e 100644 --- a/sys/alpha/alpha/interrupt.c +++ b/sys/alpha/alpha/interrupt.c @@ -33,6 +33,8 @@ * notice. */ +#include "opt_ddb.h" + #include /* RCS ID & Copyright macro defns */ /* __KERNEL_RCSID(0, "$NetBSD: interrupt.c,v 1.23 1998/02/24 07:38:01 thorpej Exp $");*/ @@ -43,12 +45,15 @@ #include #include #include +#include #include #include #include #include #include +#include +#include #ifdef EVCNT_COUNTERS struct evcnt clock_intr_evcnt; /* event counter for clock intrs. */ @@ -56,8 +61,11 @@ struct evcnt clock_intr_evcnt; /* event counter for clock intrs. */ #include #endif +#ifdef DDB +#include +#endif + volatile int mc_expected, mc_received; -u_int32_t intr_nesting_level; static void dummy_perf(unsigned long vector, struct trapframe *framep) @@ -75,13 +83,19 @@ interrupt(a0, a1, a2, framep) unsigned long a0, a1, a2; struct trapframe *framep; { + /* + * Find our per-cpu globals. + */ + globalp = (struct globaldata *) alpha_pal_rdval(); - atomic_add_int(&intr_nesting_level, 1); + atomic_add_int(&PCPU_GET(intr_nesting_level), 1); { struct proc* p = curproc; if (!p) p = &proc0; - if ((caddr_t) framep < (caddr_t) p->p_addr + 1024) + if ((caddr_t) framep < (caddr_t) p->p_addr + 1024) { + mtx_enter(&Giant, MTX_DEF); panic("possible stack overflow\n"); + } } framep->tf_regs[FRAME_TRAPARG_A0] = a0; @@ -89,10 +103,18 @@ interrupt(a0, a1, a2, framep) framep->tf_regs[FRAME_TRAPARG_A2] = a2; switch (a0) { case ALPHA_INTR_XPROC: /* interprocessor interrupt */ - printf("interprocessor interrupt!\n"); + CTR0(KTR_INTR|KTR_SMP, "interprocessor interrupt"); + smp_handle_ipi(framep); /* note: lock not taken */ break; case ALPHA_INTR_CLOCK: /* clock interrupt */ + CTR0(KTR_INTR, "clock interrupt"); + if (PCPU_GET(cpuno) != hwrpb->rpb_primary_cpu_id) { + CTR0(KTR_INTR, "ignoring clock on secondary"); + return; + } + + mtx_enter(&Giant, MTX_DEF); cnt.v_intr++; #ifdef EVCNT_COUNTERS clock_intr_evcnt.ev_count++; @@ -105,24 +127,31 @@ interrupt(a0, a1, a2, framep) if((++schedclk2 & 0x7) == 0) statclock((struct clockframe *)framep); } + mtx_exit(&Giant, MTX_DEF); break; case ALPHA_INTR_ERROR: /* Machine Check or Correctable Error */ + mtx_enter(&Giant, MTX_DEF); a0 = alpha_pal_rdmces(); if (platform.mcheck_handler) (*platform.mcheck_handler)(a0, framep, a1, a2); else machine_check(a0, framep, a1, a2); + mtx_exit(&Giant, MTX_DEF); break; case ALPHA_INTR_DEVICE: /* I/O device interrupt */ + mtx_enter(&Giant, MTX_DEF); cnt.v_intr++; if (platform.iointr) (*platform.iointr)(framep, a1); + mtx_exit(&Giant, MTX_DEF); break; case ALPHA_INTR_PERF: /* interprocessor interrupt */ + mtx_enter(&Giant, MTX_DEF); perf_irq(a1, framep); + mtx_exit(&Giant, MTX_DEF); break; case ALPHA_INTR_PASSIVE: @@ -132,11 +161,12 @@ interrupt(a0, a1, a2, framep) break; default: + mtx_enter(&Giant, MTX_DEF); panic("unexpected interrupt: type 0x%lx vec 0x%lx a2 0x%lx\n", a0, a1, a2); /* NOTREACHED */ } - atomic_subtract_int(&intr_nesting_level, 1); + atomic_subtract_int(&PCPU_GET(intr_nesting_level), 1); } void @@ -204,6 +234,7 @@ fatal: printf(" pid = %d, comm = %s\n", curproc->p_pid, curproc->p_comm); printf("\n"); + kdb_trap(mces, vector, param, ALPHA_KENTRY_MM, framep); panic("machine check"); } diff --git a/sys/alpha/alpha/ipl_funcs.c b/sys/alpha/alpha/ipl_funcs.c index 8c2cb67..6642bce 100644 --- a/sys/alpha/alpha/ipl_funcs.c +++ b/sys/alpha/alpha/ipl_funcs.c @@ -30,9 +30,13 @@ #include #include #include +#include #include #include #include +#include +#include +#include #include #include "sio.h" @@ -129,7 +133,9 @@ do_sir() u_int32_t pend; int i; - atomic_add_int(&intr_nesting_level, 1); + mtx_enter(&Giant, MTX_DEF); + + atomic_add_int(&PCPU_GET(intr_nesting_level), 1); splsoft(); while ((pend = atomic_readandclear(&ipending)) != 0) { for (i = 0; pend && i < 32; i++) { @@ -142,7 +148,9 @@ do_sir() } } } - atomic_subtract_int(&intr_nesting_level, 1); + atomic_subtract_int(&PCPU_GET(intr_nesting_level), 1); + + mtx_exit(&Giant, MTX_DEF); } #define GENSET(name, ptr, bit) \ diff --git a/sys/alpha/alpha/locore.s b/sys/alpha/alpha/locore.s index 221c673..2da1315 100644 --- a/sys/alpha/alpha/locore.s +++ b/sys/alpha/alpha/locore.s @@ -77,7 +77,7 @@ */ #define SWITCH_CONTEXT \ /* Make a note of the context we're running on. */ \ - stq a0, curpcb ; \ + stq a0, GD_CURPCB(globalp); \ \ /* Swap in the new context. */ \ call_pal PAL_OSF1_swpctx @@ -107,6 +107,12 @@ call_pal PAL_OSF1_wrvptptr /* clobbers a0, t0, t8-t11 */ /* + * Initialise globalp. + */ + call_pal PAL_OSF1_rdval /* clobbers t0, t8-t11 */ + mov v0, globalp + + /* * Switch to proc0's PCB, which is at U_PCB off of proc0paddr. */ lda t0,proc0 /* get phys addr of pcb */ @@ -126,18 +132,50 @@ * Note that setregs() is responsible for setting its contents * to 'reasonable' values. */ - lda sp,-(FRAME_SIZE * 8)(sp) /* space for struct trapframe */ + lda sp,-288(sp) /* space for struct trapframe */ mov sp, a0 /* arg is frame ptr */ CALL(mi_startup) /* go to mi_startup()! */ + /* NOTREACHED */ + + END(locorestart) + /* - * Call exception_return, to simulate return from (fake) - * exception to user-land, running process 1, init! + * Secondary processors start executing here. They will have their + * unique value set to point at the per-cpu structure and will + * be executing on their private idle stack. */ - jmp zero, exception_return /* "And that's all she wrote." */ - END(locorestart) + NESTED(smp_init_secondary_glue, 1, 0, ra, 0, 0) + mov pv, globalp + + ldiq a0, ALPHA_PSL_IPL_HIGH /* disable all interrupts */ + call_pal PAL_OSF1_swpipl + + br pv, 1f +1: LDGP(pv) + mov gp, a0 + call_pal PAL_OSF1_wrkgp /* clobbers a0, t0, t8-t11 */ + ldiq a0, -2 /* TBIA */ + call_pal PAL_OSF1_tbi + call_pal PAL_imb + + ldq a0, GD_IDLEPCBPHYS(globalp) /* switch to idle ctx */ + call_pal PAL_OSF1_swpctx + + CALL(smp_init_secondary) /* initialise the rest */ + + /* + * After initialising, we start idling for real. + * We have the kernel lock at this point. + */ + CALL(cpu_switch) /* never returns */ + + call_pal PAL_halt + + END(smp_init_secondary_glue) + /**************************************************************************/ /* diff --git a/sys/alpha/alpha/machdep.c b/sys/alpha/alpha/machdep.c index 598362d..9af4441 100644 --- a/sys/alpha/alpha/machdep.c +++ b/sys/alpha/alpha/machdep.c @@ -97,6 +97,8 @@ #include #include #include +#include +#include #include #include #include @@ -127,6 +129,8 @@ #include #include #include +#include +#include #include #include #include @@ -140,18 +144,17 @@ #include #include -struct proc* curproc; -struct proc* fpcurproc; -struct pcb* curpcb; u_int64_t cycles_per_usec; u_int32_t cycles_per_sec; -int whichqs, whichrtqs, whichidqs; int cold = 1; struct platform platform; alpha_chipset_t chipset; struct bootinfo_kernel bootinfo; -struct timeval switchtime; -int switchticks; + +struct cpuhead cpuhead; + +mtx_t sched_lock; +mtx_t Giant; struct user *proc0paddr; @@ -419,6 +422,14 @@ again: vm_pager_bufferinit(); EVENTHANDLER_REGISTER(shutdown_final, alpha_srm_shutdown, 0, SHUTDOWN_PRI_LAST); + +#ifdef SMP + /* + * OK, enough kmem_alloc/malloc state should be up, lets get on with it! + */ + mp_start(); /* fire up the secondaries */ + mp_announce(); +#endif /* SMP */ } int @@ -978,11 +989,25 @@ alpha_init(pfn, ptb, bim, bip, biv) (struct user *)pmap_steal_memory(UPAGES * PAGE_SIZE); /* + * Setup the global data for the bootstrap cpu. + */ + { + size_t sz = round_page(UPAGES * PAGE_SIZE); + globalp = (struct globaldata *) pmap_steal_memory(sz); + globaldata_init(globalp, alpha_pal_whami(), sz); + alpha_pal_wrval((u_int64_t) globalp); + PCPU_GET(next_asn) = 1; /* 0 used for proc0 pmap */ + } + + /* * Initialize the virtual memory system, and set the * page table base register in proc 0's PCB. */ pmap_bootstrap(ALPHA_PHYS_TO_K0SEG(alpha_ptob(ptb)), hwrpb->rpb_max_asn); + hwrpb->rpb_vptb = VPTBASE; + hwrpb->rpb_checksum = hwrpb_checksum(); + /* * Initialize the rest of proc 0's PCB, and cache its physical @@ -999,6 +1024,29 @@ alpha_init(pfn, ptb, bim, bip, biv) (u_int64_t)proc0paddr + USPACE - sizeof(struct trapframe); proc0.p_md.md_tf = (struct trapframe *)proc0paddr->u_pcb.pcb_hw.apcb_ksp; + PCPU_SET(curproc, &proc0); + + /* + * Get the right value for the boot cpu's idle ptbr. + */ + globalp->gd_idlepcb.apcb_ptbr = proc0.p_addr->u_pcb.pcb_hw.apcb_ptbr; + + /* + * Record all cpus in a list. + */ + SLIST_INIT(&cpuhead); + SLIST_INSERT_HEAD(&cpuhead, GLOBALP, gd_allcpu); + + /* + * Initialise the kernel lock. + */ + mtx_init(&Giant, "Giant", MTX_DEF); + mtx_init(&sched_lock, "sched lock", MTX_SPIN); + + /* + * Enable interrupts on first release (in switch_trampoline). + */ + sched_lock.mtx_saveipl = ALPHA_PSL_IPL_0; /* * Look at arguments passed to us and compute boothowto. @@ -1118,6 +1166,8 @@ alpha_init(pfn, ptb, bim, bip, biv) #endif } + hwrpb_restart_setup(); + alpha_pal_wrfen(0); } @@ -2034,9 +2084,14 @@ SYSCTL_INT(_machdep, CPU_WALLCLOCK, wall_cmos_clock, void alpha_fpstate_check(struct proc *p) { + /* + * For SMP, we should check the fpcurproc of each cpu. + */ +#ifndef SMP if (p->p_addr->u_pcb.pcb_hw.apcb_flags & ALPHA_PCB_FLAGS_FEN) if (p != fpcurproc) panic("alpha_check_fpcurproc: bogus"); +#endif } #define SET_FEN(p) \ diff --git a/sys/alpha/alpha/mem.c b/sys/alpha/alpha/mem.c index 940d827..196ed14 100644 --- a/sys/alpha/alpha/mem.c +++ b/sys/alpha/alpha/mem.c @@ -261,9 +261,12 @@ mem_modevent(module_t mod, int type, void *data) case MOD_LOAD: if (bootverbose) printf("mem: \n"); +/* XXX - ??? */ +#if 0 /* Initialise memory range handling */ if (mem_range_softc.mr_op != NULL) mem_range_softc.mr_op->init(&mem_range_softc); +#endif memdev = make_dev(&mem_cdevsw, 0, UID_ROOT, GID_KMEM, 0640, "mem"); diff --git a/sys/alpha/alpha/mp_machdep.c b/sys/alpha/alpha/mp_machdep.c new file mode 100644 index 0000000..367b57e --- /dev/null +++ b/sys/alpha/alpha/mp_machdep.c @@ -0,0 +1,1115 @@ +/*- + * Copyright (c) 2000 Doug Rabson + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#define CHECKSTATE_USER 0 +#define CHECKSTATE_SYS 1 +#define CHECKSTATE_INTR 2 + +volatile u_int stopped_cpus; +volatile u_int started_cpus; +volatile u_int checkstate_probed_cpus; +volatile u_int checkstate_need_ast; +volatile u_int checkstate_pending_ast; +struct proc* checkstate_curproc[NCPUS]; +int checkstate_cpustate[NCPUS]; +u_long checkstate_pc[NCPUS]; +volatile u_int resched_cpus; +void (*cpustop_restartfunc) __P((void)); +int mp_ncpus; + +int smp_started; +int boot_cpu_id; +u_int32_t all_cpus; + +static struct globaldata *cpuno_to_globaldata[NCPUS]; + +int smp_active = 0; /* are the APs allowed to run? */ +SYSCTL_INT(_machdep, OID_AUTO, smp_active, CTLFLAG_RW, &smp_active, 0, ""); + +/* Is forwarding of a interrupt to the CPU holding the ISR lock enabled ? */ +int forward_irq_enabled = 1; +SYSCTL_INT(_machdep, OID_AUTO, forward_irq_enabled, CTLFLAG_RW, + &forward_irq_enabled, 0, ""); + +/* Enable forwarding of a signal to a process running on a different CPU */ +static int forward_signal_enabled = 1; +SYSCTL_INT(_machdep, OID_AUTO, forward_signal_enabled, CTLFLAG_RW, + &forward_signal_enabled, 0, ""); + +/* Enable forwarding of roundrobin to all other cpus */ +static int forward_roundrobin_enabled = 1; +SYSCTL_INT(_machdep, OID_AUTO, forward_roundrobin_enabled, CTLFLAG_RW, + &forward_roundrobin_enabled, 0, ""); + +/* + * Communicate with a console running on a secondary processor. + * Return 1 on failure. + */ +static int +smp_send_secondary_command(const char *command, int cpuno) +{ + u_int64_t mask = 1L << cpuno; + struct pcs *cpu = LOCATE_PCS(hwrpb, cpuno); + int i, len; + + /* + * Sanity check. + */ + len = strlen(command); + if (len > sizeof(cpu->pcs_buffer.rxbuf)) { + printf("smp_send_secondary_command: command '%s' too long\n", + command); + return 0; + } + + /* + * Wait for the rx bit to clear. + */ + for (i = 0; i < 100000; i++) { + if (!(hwrpb->rpb_rxrdy & mask)) + break; + DELAY(10); + } + if (hwrpb->rpb_rxrdy & mask) + return 0; + + /* + * Write the command into the processor's buffer. + */ + bcopy(command, cpu->pcs_buffer.rxbuf, len); + cpu->pcs_buffer.rxlen = len; + + /* + * Set the bit in the rxrdy mask and let the secondary try to + * handle the command. + */ + atomic_set_64(&hwrpb->rpb_rxrdy, mask); + + /* + * Wait for the rx bit to clear. + */ + for (i = 0; i < 100000; i++) { + if (!(hwrpb->rpb_rxrdy & mask)) + break; + DELAY(10); + } + if (hwrpb->rpb_rxrdy & mask) + return 0; + + return 1; +} + +void +smp_init_secondary(void) +{ + /* + * Record the globaldata pointer in the per-cpu system value. + */ + alpha_pal_wrval((u_int64_t) globalp); + + /* + * Point interrupt/exception vectors to our own. + */ + alpha_pal_wrent(XentInt, ALPHA_KENTRY_INT); + alpha_pal_wrent(XentArith, ALPHA_KENTRY_ARITH); + alpha_pal_wrent(XentMM, ALPHA_KENTRY_MM); + alpha_pal_wrent(XentIF, ALPHA_KENTRY_IF); + alpha_pal_wrent(XentUna, ALPHA_KENTRY_UNA); + alpha_pal_wrent(XentSys, ALPHA_KENTRY_SYS); + + mtx_enter(&Giant, MTX_DEF); + + printf("smp_init_secondary: called\n"); + CTR0(KTR_SMP, "smp_init_secondary"); + + /* + * Add to mask. + */ + smp_started = 1; + if (PCPU_GET(cpuno) + 1 > mp_ncpus) + mp_ncpus = PCPU_GET(cpuno) + 1; + spl0(); + smp_ipi_all(0); + + mtx_exit(&Giant, MTX_DEF); +} + +extern void smp_init_secondary_glue(void); + +static int +smp_start_secondary(int cpuno) +{ + struct pcs *cpu = LOCATE_PCS(hwrpb, cpuno); + struct pcs *bootcpu = LOCATE_PCS(hwrpb, hwrpb->rpb_primary_cpu_id); + struct alpha_pcb *pcb = (struct alpha_pcb *) cpu->pcs_hwpcb; + struct globaldata *globaldata; + int i; + size_t sz; + + if ((cpu->pcs_flags & PCS_PV) == 0) { + printf("smp_start_secondary: cpu %d PALcode invalid\n", cpuno); + return 0; + } + + printf("smp_start_secondary: starting cpu %d\n", cpuno); + + sz = round_page(UPAGES * PAGE_SIZE); + globaldata = malloc(sz, M_TEMP, M_NOWAIT); + if (!globaldata) { + printf("smp_start_secondary: can't allocate memory\n"); + return 0; + } + + globaldata_init(globaldata, cpuno, sz); + + /* + * Copy the idle pcb and setup the address to start executing. + * Use the pcb unique value to point the secondary at its globaldata + * structure. + */ + *pcb = globaldata->gd_idlepcb; + hwrpb->rpb_restart = (u_int64_t) smp_init_secondary_glue; + hwrpb->rpb_restart_val = (u_int64_t) globaldata; + hwrpb->rpb_checksum = hwrpb_checksum(); + + /* + * Tell the cpu to start with the same PALcode as us. + */ + bcopy(&bootcpu->pcs_pal_rev, &cpu->pcs_pal_rev, + sizeof cpu->pcs_pal_rev); + + /* + * Set flags in cpu structure and push out write buffers to + * make sure the secondary sees it. + */ + cpu->pcs_flags |= PCS_CV|PCS_RC; + cpu->pcs_flags &= ~PCS_BIP; + alpha_mb(); + + /* + * Fire it up and hope for the best. + */ + if (!smp_send_secondary_command("START\r\n", cpuno)) { + printf("smp_init_secondary: can't send START command\n"); + free(globaldata, M_TEMP); + return 0; + } + + /* + * Wait for the secondary to set the BIP flag in its structure. + */ + for (i = 0; i < 100000; i++) { + if (cpu->pcs_flags & PCS_BIP) + break; + DELAY(10); + } + if (!(cpu->pcs_flags & PCS_BIP)) { + printf("smp_init_secondary: secondary did not respond\n"); + free(globaldata, M_TEMP); + } + + /* + * It worked (I think). + */ + /* if (bootverbose) */ + printf("smp_init_secondary: cpu %d started\n", cpuno); + + return 1; +} + +/* + * Initialise a struct globaldata. + */ +void +globaldata_init(struct globaldata *globaldata, int cpuno, size_t sz) +{ + bzero(globaldata, sz); + globaldata->gd_idlepcbphys = vtophys((vm_offset_t) &globaldata->gd_idlepcb); + globaldata->gd_idlepcb.apcb_ksp = (u_int64_t) + ((caddr_t) globaldata + sz - sizeof(struct trapframe)); + globaldata->gd_idlepcb.apcb_ptbr = proc0.p_addr->u_pcb.pcb_hw.apcb_ptbr; + globaldata->gd_cpuno = cpuno; + globaldata->gd_other_cpus = all_cpus & ~(1 << cpuno); + globaldata->gd_next_asn = 0; + globaldata->gd_current_asngen = 1; + cpuno_to_globaldata[cpuno] = globaldata; +} + +struct globaldata * +globaldata_find(int cpuno) +{ + return cpuno_to_globaldata[cpuno]; +} + +/* Implementation of simplelocks */ + +/* + * Atomically swap the value of *p with val. Return the old value of *p. + */ +static __inline int +atomic_xchg(volatile u_int *p, u_int val) +{ + u_int32_t oldval, temp; + __asm__ __volatile__ ( + "1:\tldl_l %0,%3\n\t" /* load current value */ + "mov %4,%1\n\t" /* value to store */ + "stl_c %1,%2\n\t" /* attempt to store */ + "beq %1,2f\n\t" /* if the store failed, spin */ + "br 3f\n" /* it worked, exit */ + "2:\tbr 1b\n" /* *p not updated, loop */ + "3:\n" /* it worked */ + : "=&r"(oldval), "=r"(temp), "=m" (*p) + : "m"(*p), "r"(val) + : "memory"); + return oldval; +} + +void +s_lock_init(struct simplelock *lkp) +{ + lkp->lock_data = 0; +} + +void +s_lock(struct simplelock *lkp) +{ + for (;;) { + if (s_lock_try(lkp)) + return; + + /* + * Spin until clear. + */ + while (lkp->lock_data) + ; + } +} + +int +s_lock_try(struct simplelock *lkp) +{ + u_int32_t oldval, temp; + + __asm__ __volatile__ ( + "1:\tldl_l %0,%3\n\t" /* load current value */ + "blbs %0,2f\n" /* if set, give up now */ + "mov 1,%1\n\t" /* value to store */ + "stl_c %1,%2\n\t" /* attempt to store */ + "beq %1,3f\n\t" /* if the store failed, spin */ + "2:" /* exit */ + ".section .text2,\"ax\"\n" /* improve branch prediction */ + "3:\tbr 1b\n" /* *p not updated, loop */ + ".previous\n" + : "=&r"(oldval), "=r"(temp), "=m" (lkp->lock_data) + : "m"(lkp->lock_data) + : "memory"); + + if (!oldval) { + /* + * It was clear, return success. + */ + alpha_mb(); + return 1; + } + return 0; +} + +/* Other stuff */ + +/* lock around the MP rendezvous */ +static struct simplelock smp_rv_lock; + +static void +init_locks(void) +{ + s_lock_init(&smp_rv_lock); +} + +void +mp_start() +{ + int i; + int cpuno = PCPU_GET(cpuno); + + init_locks(); + + if (cpuno + 1 > mp_ncpus) + mp_ncpus = cpuno + 1; + + all_cpus = 1<rpb_pcs_cnt; i++) { + struct pcs *pcsp; + + if (i == cpuno) + continue; + pcsp = (struct pcs *)((char *)hwrpb + hwrpb->rpb_pcs_off + + (i * hwrpb->rpb_pcs_size)); + if ((pcsp->pcs_flags & PCS_PP) != 0) { + all_cpus |= 1<rpb_pcs_cnt; i++) { + struct pcs *pcsp; + + if (i == cpuno) + continue; + pcsp = (struct pcs *)((char *)hwrpb + hwrpb->rpb_pcs_off + + (i * hwrpb->rpb_pcs_size)); + if ((pcsp->pcs_flags & PCS_PP) != 0) { + smp_active = 1; + smp_start_secondary(i); + break; /* only one for now */ + } + } +} + +void +mp_announce() +{ +} + +void +smp_invltlb() +{ +} + +#define GD_TO_INDEX(pc, prof) \ + ((int)(((u_quad_t)((pc) - (prof)->pr_off) * \ + (u_quad_t)((prof)->pr_scale)) >> 16) & ~1) + +extern long cp_time[CPUSTATES]; + +static void +addugd_intr_forwarded(struct proc *p, int id, int *astmap) +{ + int i; + struct uprof *prof; + u_long pc; + + pc = checkstate_pc[id]; + prof = &p->p_stats->p_prof; + if (pc >= prof->pr_off && + (i = GD_TO_INDEX(pc, prof)) < prof->pr_size) { + if ((p->p_flag & P_OWEUPC) == 0) { + prof->pr_addr = pc; + prof->pr_ticks = 1; + p->p_flag |= P_OWEUPC; + } + *astmap |= (1 << id); + } +} + +static void +forwarded_statclock(int id, int pscnt, int *astmap) +{ + struct pstats *pstats; + long rss; + struct rusage *ru; + struct vmspace *vm; + int cpustate; + struct proc *p; +#ifdef GPROF + register struct gmonparam *g; + int i; +#endif + + p = checkstate_curproc[id]; + cpustate = checkstate_cpustate[id]; + + switch (cpustate) { + case CHECKSTATE_USER: + if (p->p_flag & P_PROFIL) + addugd_intr_forwarded(p, id, astmap); + if (pscnt > 1) + return; + p->p_uticks++; + if (p->p_nice > NZERO) + cp_time[CP_NICE]++; + else + cp_time[CP_USER]++; + break; + case CHECKSTATE_SYS: +#ifdef GPROF + /* + * Kernel statistics are just like addugd_intr, only easier. + */ + g = &_gmonparam; + if (g->state == GMON_PROF_ON) { + i = checkstate_pc[id] - g->lowpc; + if (i < g->textsize) { + i /= HISTFRACTION * sizeof(*g->kcount); + g->kcount[i]++; + } + } +#endif + if (pscnt > 1) + return; + + if (!p) + cp_time[CP_IDLE]++; + else { + p->p_sticks++; + cp_time[CP_SYS]++; + } + break; + case CHECKSTATE_INTR: + default: +#ifdef GPROF + /* + * Kernel statistics are just like addugd_intr, only easier. + */ + g = &_gmonparam; + if (g->state == GMON_PROF_ON) { + i = checkstate_pc[id] - g->lowpc; + if (i < g->textsize) { + i /= HISTFRACTION * sizeof(*g->kcount); + g->kcount[i]++; + } + } +#endif + if (pscnt > 1) + return; + if (p) + p->p_iticks++; + cp_time[CP_INTR]++; + } + if (p != NULL) { + schedclock(p); + + /* Update resource usage integrals and maximums. */ + if ((pstats = p->p_stats) != NULL && + (ru = &pstats->p_ru) != NULL && + (vm = p->p_vmspace) != NULL) { + ru->ru_ixrss += pgtok(vm->vm_tsize); + ru->ru_idrss += pgtok(vm->vm_dsize); + ru->ru_isrss += pgtok(vm->vm_ssize); + rss = pgtok(vmspace_resident_count(vm)); + if (ru->ru_maxrss < rss) + ru->ru_maxrss = rss; + } + } +} + +#define BETTER_CLOCK_DIAGNOSTIC + +void +forward_statclock(int pscnt) +{ + int map; + int id; + int i; + + /* Kludge. We don't yet have separate locks for the interrupts + * and the kernel. This means that we cannot let the other processors + * handle complex interrupts while inhibiting them from entering + * the kernel in a non-interrupt context. + * + * What we can do, without changing the locking mechanisms yet, + * is letting the other processors handle a very simple interrupt + * (wich determines the processor states), and do the main + * work ourself. + */ + + CTR1(KTR_SMP, "forward_statclock(%d)", pscnt); + + if (!smp_started || cold || panicstr) + return; + + /* Step 1: Probe state (user, cpu, interrupt, spinlock, idle ) */ + + map = PCPU_GET(other_cpus) & ~stopped_cpus ; + checkstate_probed_cpus = 0; + if (map != 0) + smp_ipi_selected(map, IPI_CHECKSTATE); + + i = 0; + while (checkstate_probed_cpus != map) { + /* spin */ + i++; + if (i == 100000) { +#ifdef BETTER_CLOCK_DIAGNOSTIC + printf("forward_statclock: checkstate %x\n", + checkstate_probed_cpus); +#endif + break; + } + } + + /* + * Step 2: walk through other processors processes, update ticks and + * profiling info. + */ + + map = 0; + for (id = 0; id < mp_ncpus; id++) { + if (id == cpuid) + continue; + if (((1 << id) & checkstate_probed_cpus) == 0) + continue; + forwarded_statclock(id, pscnt, &map); + } + if (map != 0) { + checkstate_need_ast |= map; + smp_ipi_selected(map, IPI_AST); + i = 0; + while ((checkstate_need_ast & map) != 0) { + /* spin */ + i++; + if (i > 100000) { +#ifdef BETTER_CLOCK_DIAGNOSTIC + printf("forward_statclock: dropped ast 0x%x\n", + checkstate_need_ast & map); +#endif + break; + } + } + } +} + +void +forward_hardclock(int pscnt) +{ + int map; + int id; + struct proc *p; + struct pstats *pstats; + int i; + + /* Kludge. We don't yet have separate locks for the interrupts + * and the kernel. This means that we cannot let the other processors + * handle complex interrupts while inhibiting them from entering + * the kernel in a non-interrupt context. + * + * What we can do, without changing the locking mechanisms yet, + * is letting the other processors handle a very simple interrupt + * (wich determines the processor states), and do the main + * work ourself. + */ + + CTR1(KTR_SMP, "forward_hardclock(%d)", pscnt); + + if (!smp_started || cold || panicstr) + return; + + /* Step 1: Probe state (user, cpu, interrupt, spinlock, idle) */ + + map = PCPU_GET(other_cpus) & ~stopped_cpus ; + checkstate_probed_cpus = 0; + if (map != 0) + smp_ipi_selected(map, IPI_CHECKSTATE); + + i = 0; + while (checkstate_probed_cpus != map) { + /* spin */ + i++; + if (i == 100000) { +#ifdef BETTER_CLOCK_DIAGNOSTIC + printf("forward_hardclock: checkstate %x\n", + checkstate_probed_cpus); +#endif + breakpoint(); + break; + } + } + + /* + * Step 2: walk through other processors processes, update virtual + * timer and profiling timer. If stathz == 0, also update ticks and + * profiling info. + */ + + map = 0; + for (id = 0; id < mp_ncpus; id++) { + if (id == cpuid) + continue; + if (((1 << id) & checkstate_probed_cpus) == 0) + continue; + p = checkstate_curproc[id]; + if (p) { + pstats = p->p_stats; + if (checkstate_cpustate[id] == CHECKSTATE_USER && + timevalisset(&pstats->p_timer[ITIMER_VIRTUAL].it_value) && + itimerdecr(&pstats->p_timer[ITIMER_VIRTUAL], tick) == 0) { + psignal(p, SIGVTALRM); + map |= (1 << id); + } + if (timevalisset(&pstats->p_timer[ITIMER_PROF].it_value) && + itimerdecr(&pstats->p_timer[ITIMER_PROF], tick) == 0) { + psignal(p, SIGPROF); + map |= (1 << id); + } + } + if (stathz == 0) { + forwarded_statclock( id, pscnt, &map); + } + } + if (map != 0) { + checkstate_need_ast |= map; + smp_ipi_selected(map, IPI_AST); + i = 0; + while ((checkstate_need_ast & map) != 0) { + /* spin */ + i++; + if (i > 100000) { +#ifdef BETTER_CLOCK_DIAGNOSTIC + printf("forward_hardclock: dropped ast 0x%x\n", + checkstate_need_ast & map); +#endif + break; + } + } + } +} + +void +forward_signal(struct proc *p) +{ + int map; + int id; + int i; + + /* Kludge. We don't yet have separate locks for the interrupts + * and the kernel. This means that we cannot let the other processors + * handle complex interrupts while inhibiting them from entering + * the kernel in a non-interrupt context. + * + * What we can do, without changing the locking mechanisms yet, + * is letting the other processors handle a very simple interrupt + * (wich determines the processor states), and do the main + * work ourself. + */ + + CTR1(KTR_SMP, "forward_signal(%p)", p); + + if (!smp_started || cold || panicstr) + return; + if (!forward_signal_enabled) + return; + while (1) { + if (p->p_stat != SRUN) + return; + id = p->p_oncpu; + if (id == 0xff) + return; + map = (1< 100000) { +#if 0 + printf("forward_signal: dropped ast 0x%x\n", + checkstate_need_ast & map); +#endif + break; + } + } + if (id == p->p_oncpu) + return; + } +} + +void +forward_roundrobin(void) +{ + u_int map; + int i; + + CTR0(KTR_SMP, "forward_roundrobin()"); + + if (!smp_started || cold || panicstr) + return; + if (!forward_roundrobin_enabled) + return; + resched_cpus |= PCPU_GET(other_cpus); + map = PCPU_GET(other_cpus) & ~stopped_cpus ; + smp_ipi_selected(map, IPI_AST); + i = 0; + while ((checkstate_need_ast & map) != 0) { + /* spin */ + i++; + if (i > 100000) { +#if 0 + printf("forward_roundrobin: dropped ast 0x%x\n", + checkstate_need_ast & map); +#endif + break; + } + } +} + +/* + * When called the executing CPU will send an IPI to all other CPUs + * requesting that they halt execution. + * + * Usually (but not necessarily) called with 'other_cpus' as its arg. + * + * - Signals all CPUs in map to stop. + * - Waits for each to stop. + * + * Returns: + * -1: error + * 0: NA + * 1: ok + * + * XXX FIXME: this is not MP-safe, needs a lock to prevent multiple CPUs + * from executing at same time. + */ +int +stop_cpus(u_int map) +{ + int i; + + if (!smp_started) + return 0; + + CTR1(KTR_SMP, "stop_cpus(%x)", map); + + /* send the stop IPI to all CPUs in map */ + smp_ipi_selected(map, IPI_STOP); + + i = 0; + while ((stopped_cpus & map) != map) { + /* spin */ + i++; + if (i == 100000) { + printf("timeout stopping cpus\n"); + break; + } + alpha_mb(); + } + + printf("stopped_cpus=%x\n", stopped_cpus); + + return 1; +} + + +/* + * Called by a CPU to restart stopped CPUs. + * + * Usually (but not necessarily) called with 'stopped_cpus' as its arg. + * + * - Signals all CPUs in map to restart. + * - Waits for each to restart. + * + * Returns: + * -1: error + * 0: NA + * 1: ok + */ +int +restart_cpus(u_int map) +{ + if (!smp_started) + return 0; + + CTR1(KTR_SMP, "restart_cpus(%x)", map); + + started_cpus = map; /* signal other cpus to restart */ + alpha_mb(); + + while ((stopped_cpus & map) != 0) /* wait for each to clear its bit */ + alpha_mb(); + + return 1; +} + +/* + * All-CPU rendezvous. CPUs are signalled, all execute the setup function + * (if specified), rendezvous, execute the action function (if specified), + * rendezvous again, execute the teardown function (if specified), and then + * resume. + * + * Note that the supplied external functions _must_ be reentrant and aware + * that they are running in parallel and in an unknown lock context. + */ +static void (*smp_rv_setup_func)(void *arg); +static void (*smp_rv_action_func)(void *arg); +static void (*smp_rv_teardown_func)(void *arg); +static void *smp_rv_func_arg; +static volatile int smp_rv_waiters[2]; + +void +smp_rendezvous_action(void) +{ + /* setup function */ + if (smp_rv_setup_func != NULL) + smp_rv_setup_func(smp_rv_func_arg); + /* spin on entry rendezvous */ + atomic_add_int(&smp_rv_waiters[0], 1); + while (smp_rv_waiters[0] < mp_ncpus) + ; + /* action function */ + if (smp_rv_action_func != NULL) + smp_rv_action_func(smp_rv_func_arg); + /* spin on exit rendezvous */ + atomic_add_int(&smp_rv_waiters[1], 1); + while (smp_rv_waiters[1] < mp_ncpus) + ; + /* teardown function */ + if (smp_rv_teardown_func != NULL) + smp_rv_teardown_func(smp_rv_func_arg); +} + +void +smp_rendezvous(void (* setup_func)(void *), + void (* action_func)(void *), + void (* teardown_func)(void *), + void *arg) +{ + int s; + + /* disable interrupts on this CPU, save interrupt status */ + s = splhigh(); + + /* obtain rendezvous lock */ + s_lock(&smp_rv_lock); /* XXX sleep here? NOWAIT flag? */ + + /* set static function pointers */ + smp_rv_setup_func = setup_func; + smp_rv_action_func = action_func; + smp_rv_teardown_func = teardown_func; + smp_rv_func_arg = arg; + smp_rv_waiters[0] = 0; + smp_rv_waiters[1] = 0; + + /* signal other processors, which will enter the IPI with interrupts off */ + smp_ipi_all_but_self(IPI_RENDEZVOUS); + + /* call executor function */ + smp_rendezvous_action(); + + /* release lock */ + s_unlock(&smp_rv_lock); + + /* restore interrupt flag */ + splx(s); +} + +/* + * send an IPI to a set of cpus. + */ +void +smp_ipi_selected(u_int32_t cpus, u_int64_t ipi) +{ + struct globaldata *globaldata; + + CTR2(KTR_SMP, "smp_ipi_selected", cpus, ipi); + alpha_mb(); + while (cpus) { + int cpuno = ffs(cpus) - 1; + cpus &= ~(1 << cpuno); + + globaldata = cpuno_to_globaldata[cpuno]; + if (globaldata) { + atomic_set_64(&globaldata->gd_pending_ipis, ipi); + alpha_mb(); + CTR1(KTR_SMP, "calling alpha_pal_wripir(%d)", cpuno); + alpha_pal_wripir(cpuno); + } + } +} + +/* + * send an IPI INTerrupt containing 'vector' to all CPUs, including myself + */ +void +smp_ipi_all(u_int64_t ipi) +{ + smp_ipi_selected(all_cpus, ipi); +} + +/* + * send an IPI to all CPUs EXCEPT myself + */ +void +smp_ipi_all_but_self(u_int64_t ipi) +{ + smp_ipi_selected(PCPU_GET(other_cpus), ipi); +} + +/* + * send an IPI to myself + */ +void +smp_ipi_self(u_int64_t ipi) +{ + smp_ipi_selected(1 << PCPU_GET(cpuno), ipi); +} + +static u_int64_t +atomic_readandclear(u_int64_t* p) +{ + u_int64_t v, temp; + __asm__ __volatile__ ( + "wmb\n" /* ensure pending writes have drained */ + "1:\tldq_l %0,%3\n\t" /* load current value, asserting lock */ + "ldiq %1,0\n\t" /* value to store */ + "stq_c %1,%2\n\t" /* attempt to store */ + "beq %1,2f\n\t" /* if the store failed, spin */ + "br 3f\n" /* it worked, exit */ + "2:\tbr 1b\n" /* *p not updated, loop */ + "3:\tmb\n" /* it worked */ + : "=&r"(v), "=&r"(temp), "=m" (*p) + : "m"(*p) + : "memory"); + return v; +} + +/* + * Handle an IPI sent to this processor. + */ +void +smp_handle_ipi(struct trapframe *frame) +{ + u_int64_t ipis = atomic_readandclear(&PCPU_GET(pending_ipis)); + u_int64_t ipi; + int cpuno = PCPU_GET(cpuno); + + CTR1(KTR_SMP, "smp_handle_ipi(), ipis=%x", ipis); + while (ipis) { + /* + * Find the lowest set bit. + */ + ipi = ipis & ~(ipis - 1); + switch (ipi) { + case IPI_INVLTLB: + break; + + case IPI_RENDEZVOUS: + CTR0(KTR_SMP, "IPI_RENDEZVOUS"); + smp_rendezvous_action(); + break; + + case IPI_AST: + CTR0(KTR_SMP, "IPI_AST"); + atomic_clear_int(&checkstate_need_ast, 1<tf_regs[FRAME_PS] & ALPHA_PSL_USERMODE) + ast(frame); /* XXX */ + break; + + case IPI_CHECKSTATE: + CTR0(KTR_SMP, "IPI_CHECKSTATE"); + if (frame->tf_regs[FRAME_PS] & ALPHA_PSL_USERMODE) + checkstate_cpustate[cpuno] = CHECKSTATE_USER; + else if (PCPU_GET(intr_nesting_level) == 1) + checkstate_cpustate[cpuno] = CHECKSTATE_SYS; + else + checkstate_cpustate[cpuno] = CHECKSTATE_INTR; + checkstate_curproc[cpuno] = PCPU_GET(curproc); + atomic_set_int(&checkstate_probed_cpus, 1<rpb_primary_cpu_id + && hwrpb->rpb_txrdy != 0) { + hwrpb->rpb_txrdy = 0; + alpha_mb(); + } +} + +#if 0 + +/* + * Atomically compare the value stored at *p with cmpval and if the + * two values are equal, update the value of *p with newval. Returns + * zero if the compare failed, nonzero otherwise. + */ +u_int64_t +atomic_cmpset_64(volatile u_int64_t* p, u_int64_t cmpval, u_int64_t newval) +{ + u_int64_t ret, temp; + + + printf("atomic_cmpset_64: *p=%lx, cmpval=%lx, newval=%lx\n", + *p, cmpval, newval); + __asm __volatile ( + "1:\tldq_l %1, %5\n\t" /* load old value */ + "cmpeq %1, %3, %0\n\t" /* compare */ + "beq %0, 2f\n\t" /* exit if not equal */ + "mov %4, %1\n\t" /* value to store */ + "stq_c %1, %2\n\t" /* attempt to store */ + "beq %1, 3f\n\t" /* if it failed, spin */ + "2:\n" /* done */ + ".section .text3,\"ax\"\n" /* improve branch prediction */ + "3:\tbr 1b\n" /* try again */ + ".previous\n" + : "=&r" (ret), "=r" (temp), "=m" (*p) + : "r" (cmpval), "r" (newval), "m" (*p) + : "memory"); + printf("atomic_cmpset_64: *p=%lx\n", *p); + + return ret; +} + +#endif diff --git a/sys/alpha/alpha/pmap.c b/sys/alpha/alpha/pmap.c index 7cdf67e..2a4852f 100644 --- a/sys/alpha/alpha/pmap.c +++ b/sys/alpha/alpha/pmap.c @@ -171,6 +171,7 @@ #include #include +#include #ifndef PMAP_SHPGPERPROC #define PMAP_SHPGPERPROC 200 @@ -325,9 +326,7 @@ vm_offset_t kernel_vm_end; * Data for the ASN allocator */ static int pmap_maxasn; -static int pmap_nextasn = 0; -static u_int pmap_current_asngen = 1; -static pmap_t pmap_active = 0; +static pmap_t pmap_active[NCPUS]; /* * Data for the pv entry allocation mechanism @@ -456,16 +455,13 @@ void pmap_bootstrap(vm_offset_t ptaddr, u_int maxasn) { pt_entry_t newpte; - pt_entry_t* pte; - vm_offset_t va; int i; /* - * Setup ASNs + * Setup ASNs. PCPU_GET(next_asn) and PCPU_GET(current_asngen) are set + * up already. */ - pmap_nextasn = 0; pmap_maxasn = maxasn; - pmap_current_asngen = 1; /* * Allocate a level 1 map for the kernel. @@ -550,27 +546,14 @@ pmap_bootstrap(vm_offset_t ptaddr, u_int maxasn) kernel_pmap = &kernel_pmap_store; kernel_pmap->pm_lev1 = Lev1map; kernel_pmap->pm_count = 1; - kernel_pmap->pm_active = 1; - kernel_pmap->pm_asn = 0; - kernel_pmap->pm_asngen = pmap_current_asngen; - pmap_nextasn = 1; + kernel_pmap->pm_active = ~0; + kernel_pmap->pm_asn[alpha_pal_whami()].asn = 0; + kernel_pmap->pm_asn[alpha_pal_whami()].gen = 1; TAILQ_INIT(&kernel_pmap->pm_pvlist); nklev3 = NKPT; nklev2 = 1; /* - * Reserve some special page table entries/VA space for temporary - * mapping of pages. - */ -#define SYSMAP(c, p, v, n) \ - v = (c)va; va += ((n)*PAGE_SIZE); p = pte; pte += (n); - - va = virtual_avail; - pte = pmap_lev3pte(kernel_pmap, va); - - virtual_avail = va; - - /* * Set up proc0's PCB such that the ptbr points to the right place * and has the kernel pmap's. */ @@ -663,23 +646,43 @@ pmap_init2() static void pmap_invalidate_asn(pmap_t pmap) { - pmap->pm_asngen = 0; + pmap->pm_asn[PCPU_GET(cpuno)].gen = 0; } +struct pmap_invalidate_page_arg { + pmap_t pmap; + vm_offset_t va; +}; + static void -pmap_invalidate_page(pmap_t pmap, vm_offset_t va) +pmap_invalidate_page_action(void *arg) { - if (pmap_isactive(pmap)) { + pmap_t pmap = ((struct pmap_invalidate_page_arg *) arg)->pmap; + vm_offset_t va = ((struct pmap_invalidate_page_arg *) arg)->va; + + if (pmap->pm_active & (1 << PCPU_GET(cpuno))) { ALPHA_TBIS(va); alpha_pal_imb(); /* XXX overkill? */ - } else + } else { pmap_invalidate_asn(pmap); + } } static void -pmap_invalidate_all(pmap_t pmap) +pmap_invalidate_page(pmap_t pmap, vm_offset_t va) +{ + struct pmap_invalidate_page_arg arg; + arg.pmap = pmap; + arg.va = va; + smp_rendezvous(0, pmap_invalidate_page_action, 0, (void *) &arg); +} + +static void +pmap_invalidate_all_action(void *arg) { - if (pmap_isactive(pmap)) { + pmap_t pmap = (pmap_t) arg; + + if (pmap->pm_active & (1 << PCPU_GET(cpuno))) { ALPHA_TBIA(); alpha_pal_imb(); /* XXX overkill? */ } else @@ -687,24 +690,31 @@ pmap_invalidate_all(pmap_t pmap) } static void +pmap_invalidate_all(pmap_t pmap) +{ + smp_rendezvous(0, pmap_invalidate_all_action, 0, (void *) pmap); +} + +static void pmap_get_asn(pmap_t pmap) { - if (pmap->pm_asngen != pmap_current_asngen) { - if (pmap_nextasn > pmap_maxasn) { + if (pmap->pm_asn[PCPU_GET(cpuno)].gen != PCPU_GET(current_asngen)) { + if (PCPU_GET(next_asn) > pmap_maxasn) { /* * Start a new ASN generation. * * Invalidate all per-process mappings and I-cache */ - pmap_nextasn = 0; - pmap_current_asngen++; + PCPU_GET(next_asn) = 0; + PCPU_GET(current_asngen)++; + PCPU_GET(current_asngen) &= (1 << 24) - 1; - if (pmap_current_asngen == 0) { + if (PCPU_GET(current_asngen) == 0) { /* - * Clear the pm_asngen of all pmaps. + * Clear the pm_asn[].gen of all pmaps. * This is safe since it is only called from * pmap_activate after it has deactivated - * the old pmap. + * the old pmap and it only affects this cpu. */ struct proc *p; pmap_t tpmap; @@ -712,11 +722,11 @@ pmap_get_asn(pmap_t pmap) #ifdef PMAP_DIAGNOSTIC printf("pmap_get_asn: generation rollover\n"); #endif - pmap_current_asngen = 1; + PCPU_GET(current_asngen) = 1; LIST_FOREACH(p, &allproc, p_list) { if (p->p_vmspace) { tpmap = vmspace_pmap(p->p_vmspace); - tpmap->pm_asngen = 0; + tpmap->pm_asn[PCPU_GET(cpuno)].gen = 0; } } } @@ -729,8 +739,8 @@ pmap_get_asn(pmap_t pmap) ALPHA_TBIAP(); alpha_pal_imb(); /* XXX overkill? */ } - pmap->pm_asn = pmap_nextasn++; - pmap->pm_asngen = pmap_current_asngen; + pmap->pm_asn[PCPU_GET(cpuno)].asn = PCPU_GET(next_asn)++; + pmap->pm_asn[PCPU_GET(cpuno)].gen = PCPU_GET(current_asngen); } } @@ -1163,13 +1173,17 @@ void pmap_pinit0(pmap) struct pmap *pmap; { + int i; + pmap->pm_lev1 = Lev1map; pmap->pm_flags = 0; pmap->pm_count = 1; pmap->pm_ptphint = NULL; pmap->pm_active = 0; - pmap->pm_asn = 0; - pmap->pm_asngen = 0; + for (i = 0; i < NCPUS; i++) { + pmap->pm_asn[i].asn = 0; + pmap->pm_asn[i].gen = 0; + } TAILQ_INIT(&pmap->pm_pvlist); bzero(&pmap->pm_stats, sizeof pmap->pm_stats); } @@ -1183,6 +1197,7 @@ pmap_pinit(pmap) register struct pmap *pmap; { vm_page_t lev1pg; + int i; /* * allocate object for the ptes @@ -1215,8 +1230,10 @@ pmap_pinit(pmap) pmap->pm_count = 1; pmap->pm_ptphint = NULL; pmap->pm_active = 0; - pmap->pm_asn = 0; - pmap->pm_asngen = 0; + for (i = 0; i < NCPUS; i++) { + pmap->pm_asn[i].asn = 0; + pmap->pm_asn[i].gen = 0; + } TAILQ_INIT(&pmap->pm_pvlist); bzero(&pmap->pm_stats, sizeof pmap->pm_stats); } @@ -2994,21 +3011,22 @@ pmap_activate(struct proc *p) pmap = vmspace_pmap(p->p_vmspace); - if (pmap_active && pmap != pmap_active) { - pmap_active->pm_active = 0; - pmap_active = 0; + if (pmap_active[PCPU_GET(cpuno)] && pmap != pmap_active[PCPU_GET(cpuno)]) { + atomic_clear_32(&pmap_active[PCPU_GET(cpuno)]->pm_active, + 1 << PCPU_GET(cpuno)); + pmap_active[PCPU_GET(cpuno)] = 0; } p->p_addr->u_pcb.pcb_hw.apcb_ptbr = ALPHA_K0SEG_TO_PHYS((vm_offset_t) pmap->pm_lev1) >> PAGE_SHIFT; - if (pmap->pm_asngen != pmap_current_asngen) + if (pmap->pm_asn[PCPU_GET(cpuno)].gen != PCPU_GET(current_asngen)) pmap_get_asn(pmap); - pmap_active = pmap; - pmap->pm_active = 1; /* XXX use bitmap for SMP */ + pmap_active[PCPU_GET(cpuno)] = pmap; + atomic_set_32(&pmap->pm_active, 1 << PCPU_GET(cpuno)); - p->p_addr->u_pcb.pcb_hw.apcb_asn = pmap->pm_asn; + p->p_addr->u_pcb.pcb_hw.apcb_asn = pmap->pm_asn[PCPU_GET(cpuno)].asn; if (p == curproc) { alpha_pal_swpctx((u_long)p->p_md.md_pcbpaddr); @@ -3020,8 +3038,8 @@ pmap_deactivate(struct proc *p) { pmap_t pmap; pmap = vmspace_pmap(p->p_vmspace); - pmap->pm_active = 0; - pmap_active = 0; + atomic_clear_32(&pmap->pm_active, 1 << PCPU_GET(cpuno)); + pmap_active[PCPU_GET(cpuno)] = 0; } vm_offset_t diff --git a/sys/alpha/alpha/prom.c b/sys/alpha/alpha/prom.c index 5880b2c..805539a 100644 --- a/sys/alpha/alpha/prom.c +++ b/sys/alpha/alpha/prom.c @@ -57,7 +57,6 @@ int prom_mapped = 1; /* Is PROM still mapped? */ pt_entry_t rom_pte, saved_pte[1]; /* XXX */ static pt_entry_t *rom_lev1map __P((void)); -extern struct pcb* curpcb; extern pt_entry_t* Lev1map; static void prom_cache_sync __P((void)); diff --git a/sys/alpha/alpha/support.s b/sys/alpha/alpha/support.s index 2e5ff39..2a87327 100644 --- a/sys/alpha/alpha/support.s +++ b/sys/alpha/alpha/support.s @@ -71,7 +71,7 @@ beq t1, fusufault lda t0, fusufault /* trap faults */ - ldq t2, curproc + ldq t2, GD_CURPROC(globalp) ldq t2, P_ADDR(t2) stq t0, U_PCB_ONFAULT(t2) @@ -91,7 +91,7 @@ beq t1, fusufault lda t0, fusufault /* trap faults */ - ldq t2, curproc + ldq t2, GD_CURPROC(globalp) ldq t2, P_ADDR(t2) stq t0, U_PCB_ONFAULT(t2) @@ -116,7 +116,7 @@ beq t1, fusufault lda t0, fusufault /* trap faults */ - ldq t2, curproc + ldq t2, GD_CURPROC(globalp) ldq t2, P_ADDR(t2) stq t0, U_PCB_ONFAULT(t2) @@ -135,7 +135,7 @@ beq t1, fusufault lda t0, fusufault /* trap faults */ - ldq t2, curproc + ldq t2, GD_CURPROC(globalp) ldq t2, P_ADDR(t2) stq t0, U_PCB_ONFAULT(t2) @@ -153,7 +153,7 @@ END(suibyte) LEAF(fusufault, 0) - ldq t0, curproc + ldq t0, GD_CURPROC(globalp) ldq t0, P_ADDR(t0) stq zero, U_PCB_ONFAULT(t0) ldiq v0, -1 @@ -221,13 +221,13 @@ NESTED(copyinstr, 4, 16, ra, 0, 0) beq t1, copyerr /* if it's not, error out. */ lda v0, copyerr /* set up fault handler. */ .set noat - ldq at_reg, curproc + ldq at_reg, GD_CURPROC(globalp) ldq at_reg, P_ADDR(at_reg) stq v0, U_PCB_ONFAULT(at_reg) .set at CALL(copystr) /* do the copy. */ .set noat - ldq at_reg, curproc /* kill the fault handler. */ + ldq at_reg, GD_CURPROC(globalp) /* kill the fault handler. */ ldq at_reg, P_ADDR(at_reg) stq zero, U_PCB_ONFAULT(at_reg) .set at @@ -245,13 +245,13 @@ NESTED(copyoutstr, 4, 16, ra, 0, 0) beq t1, copyerr /* if it's not, error out. */ lda v0, copyerr /* set up fault handler. */ .set noat - ldq at_reg, curproc + ldq at_reg, GD_CURPROC(globalp) ldq at_reg, P_ADDR(at_reg) stq v0, U_PCB_ONFAULT(at_reg) .set at CALL(copystr) /* do the copy. */ .set noat - ldq at_reg, curproc /* kill the fault handler. */ + ldq at_reg, GD_CURPROC(globalp) /* kill the fault handler. */ ldq at_reg, P_ADDR(at_reg) stq zero, U_PCB_ONFAULT(at_reg) .set at @@ -423,13 +423,13 @@ bcopy_da_finish: insql t4,a1,t4 addq a1,a2,a4 ldq_u t6,0(a1) - ldq_u t7,-1(a4) + ldq_u t8,-1(a4) bic t6,t4,t6 - bic t7,t5,t7 + bic t8,t5,t8 and t2,t4,t2 and t3,t5,t3 or t2,t6,t2 - or t3,t7,t3 + or t3,t8,t3 stq_u t3,-1(a4) stq_u t2,0(a1) RET @@ -513,13 +513,13 @@ NESTED(copyin, 3, 16, ra, 0, 0) beq t1, copyerr /* if it's not, error out. */ lda v0, copyerr /* set up fault handler. */ .set noat - ldq at_reg, curproc + ldq at_reg, GD_CURPROC(globalp) ldq at_reg, P_ADDR(at_reg) stq v0, U_PCB_ONFAULT(at_reg) .set at CALL(bcopy) /* do the copy. */ .set noat - ldq at_reg, curproc /* kill the fault handler. */ + ldq at_reg, GD_CURPROC(globalp) /* kill the fault handler. */ ldq at_reg, P_ADDR(at_reg) stq zero, U_PCB_ONFAULT(at_reg) .set at @@ -538,13 +538,13 @@ NESTED(copyout, 3, 16, ra, 0, 0) beq t1, copyerr /* if it's not, error out. */ lda v0, copyerr /* set up fault handler. */ .set noat - ldq at_reg, curproc + ldq at_reg, GD_CURPROC(globalp) ldq at_reg, P_ADDR(at_reg) stq v0, U_PCB_ONFAULT(at_reg) .set at CALL(bcopy) /* do the copy. */ .set noat - ldq at_reg, curproc /* kill the fault handler. */ + ldq at_reg, GD_CURPROC(globalp) /* kill the fault handler. */ ldq at_reg, P_ADDR(at_reg) stq zero, U_PCB_ONFAULT(at_reg) .set at @@ -555,7 +555,7 @@ NESTED(copyout, 3, 16, ra, 0, 0) END(copyout) LEAF(copyerr, 0) - ldq t0, curproc + ldq t0, GD_CURPROC(globalp) ldq t0, P_ADDR(t0) stq zero, U_PCB_ONFAULT(t0) /* reset fault handler. */ ldq ra, (16-8)(sp) /* restore ra. */ diff --git a/sys/alpha/alpha/swtch.s b/sys/alpha/alpha/swtch.s index ee191eb..f457a34 100644 --- a/sys/alpha/alpha/swtch.s +++ b/sys/alpha/alpha/swtch.s @@ -28,7 +28,9 @@ * rights to redistribute these changes. */ +#define _LOCORE #include +#include #include "assym.s" /**************************************************************************/ @@ -39,7 +41,7 @@ */ #define SWITCH_CONTEXT \ /* Make a note of the context we're running on. */ \ - stq a0, curpcb; \ + stq a0, GD_CURPCB(globalp); \ \ /* Swap in the new context. */ \ call_pal PAL_OSF1_swpctx @@ -86,34 +88,16 @@ IMPORT(want_resched, 4) IMPORT(Lev1map, 8) /* - * When no processes are on the runq, cpu_switch branches to idle - * to wait for something to come ready. - * Note: this is really a part of cpu_switch() but defined here for kernel - * profiling. - */ -LEAF(idle, 0) - br pv, Lidle1 -Lidle1: LDGP(pv) - stq zero, switchtime /* zero switchtime.tv_sec */ - stq zero, curproc /* curproc <- NULL for stats */ - mov zero, a0 /* enable all interrupts */ - call_pal PAL_OSF1_swpipl -Lidle2: - CALL(procrunnable) - beq v0, Lidle2 - ldiq a0, ALPHA_PSL_IPL_HIGH /* disable all interrupts */ - call_pal PAL_OSF1_swpipl - jmp zero, sw1 /* jump back into the fray */ - END(idle) - -/* * cpu_switch() * Find the highest priority process and resume it. */ LEAF(cpu_switch, 1) LDGP(pv) /* do an inline savectx(), to save old context */ + ldq a0, GD_CURPROC(globalp) ldq a1, P_ADDR(a0) + ldl t0, sched_lock+MTX_RECURSE /* save sched_lock state */ + stl t0, U_PCB_SCHEDNEST(a1) /* NOTE: ksp is stored by the swpctx */ stq s0, U_PCB_CONTEXT+(0 * 8)(a1) /* store s0 - s6 */ stq s1, U_PCB_CONTEXT+(1 * 8)(a1) @@ -129,16 +113,12 @@ LEAF(cpu_switch, 1) mov a0, s0 /* save old curproc */ mov a1, s1 /* save old U-area */ - CALL(procrunnable) /* anything to run? */ - beq v0, idle /* and if none, go idle */ - ldiq a0, ALPHA_PSL_IPL_HIGH /* disable all interrupts */ call_pal PAL_OSF1_swpipl sw1: br pv, Lcs1 Lcs1: LDGP(pv) - CALL(chooseproc) - beq v0, idle + CALL(chooseproc) /* can't return NULL */ mov v0, s2 ldq s3, P_MD_PCBPADDR(s2) /* save new pcbpaddr */ @@ -194,7 +174,7 @@ Lcs7: * because we might have re-entered cpu_switch() from idle(), * in which case curproc would be NULL. */ - stq s2, curproc /* curproc = p */ + stq s2, GD_CURPROC(globalp) /* curproc = p */ stl zero, want_resched /* we've rescheduled */ /* @@ -212,6 +192,10 @@ Lcs7: ldq s5, U_PCB_CONTEXT+(5 * 8)(t0) ldq s6, U_PCB_CONTEXT+(6 * 8)(t0) ldq ra, U_PCB_CONTEXT+(7 * 8)(t0) /* restore ra */ + ldl t1, U_PCB_SCHEDNEST(t0) + stl t1, sched_lock+MTX_RECURSE /* restore lock */ + ldq t1, GD_CURPROC(globalp) + stq t1, sched_lock+MTX_LOCK ldq a0, U_PCB_CONTEXT+(8 * 8)(t0) /* restore ipl */ and a0, ALPHA_PSL_IPL_MASK, a0 call_pal PAL_OSF1_swpipl @@ -231,6 +215,7 @@ Lcs7: * pointer to the executing process's proc structure. */ LEAF(switch_trampoline, 0) + MTX_EXIT(sched_lock) mov s0, pv mov s1, ra mov s2, a0 @@ -266,7 +251,7 @@ Lchkast: and s1, ALPHA_PSL_USERMODE, t0 /* are we returning to user? */ beq t0, Lrestoreregs /* no: just return */ - ldl t2, astpending /* AST pending? */ + ldl t2, GD_ASTPENDING(globalp) /* AST pending? */ beq t2, Lrestoreregs /* no: return */ /* We've got an AST. Handle it. */ @@ -277,7 +262,7 @@ Lchkast: Lrestoreregs: /* set the hae register if this process has specified a value */ - ldq t0, curproc + ldq t0, GD_CURPROC(globalp) beq t0, Lnohae ldq t1, P_MD_FLAGS(t0) and t1, MDP_HAEUSED diff --git a/sys/alpha/alpha/synch_machdep.c b/sys/alpha/alpha/synch_machdep.c new file mode 100644 index 0000000..a3077e9 --- /dev/null +++ b/sys/alpha/alpha/synch_machdep.c @@ -0,0 +1,529 @@ +/*- + * Copyright (c) 1997, 1998 Berkeley Software Design, Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Berkeley Software Design Inc's name may not be used to endorse or + * promote products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * from BSDI $Id: synch_machdep.c,v 2.3.2.39 2000/04/27 03:10:25 cp Exp $ + * $FreeBSD$ + */ + +#define MTX_STRS /* define common strings */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* All mutii in system (used for debug/panic) */ +mtx_t all_mtx = { MTX_UNOWNED, 0, 0, "All muti queue head", + TAILQ_HEAD_INITIALIZER(all_mtx.mtx_blocked), + { NULL, NULL }, &all_mtx, &all_mtx +#ifdef SMP_DEBUG + , NULL, { NULL, NULL }, NULL, 0 +#endif +}; + +int mtx_cur_cnt; +int mtx_max_cnt; + +extern void _mtx_enter_giant_def(void); +extern void _mtx_exit_giant_def(void); + +static void propagate_priority(struct proc *) __unused; + +#define mtx_unowned(m) ((m)->mtx_lock == MTX_UNOWNED) +#define mtx_owner(m) (mtx_unowned(m) ? NULL \ + : (struct proc *)((m)->mtx_lock & MTX_FLAGMASK)) + +#define RETIP(x) *(((u_int64_t *)(&x)) - 1) +#define SET_PRIO(p, pri) (p)->p_priority = (pri) + +/* + * XXX Temporary, for use from assembly language + */ + +void +_mtx_enter_giant_def(void) +{ + + mtx_enter(&Giant, MTX_DEF); +} + +void +_mtx_exit_giant_def(void) +{ + + mtx_exit(&Giant, MTX_DEF); +} + +static void +propagate_priority(struct proc *p) +{ + int pri = p->p_priority; + mtx_t *m = p->p_blocked; + + for (;;) { + struct proc *p1; + + p = mtx_owner(m); + + if (p == NULL) { + /* + * This really isn't quite right. Really + * ought to bump priority of process that + * next axcquires the mutex. + */ + MPASS(m->mtx_lock == MTX_CONTESTED); + return; + } + MPASS(p->p_magic == P_MAGIC); + if (p->p_priority <= pri) + return; + /* + * If lock holder is actually running just bump priority. + */ + if (TAILQ_NEXT(p, p_procq) == NULL) { + SET_PRIO(p, pri); + return; + } + /* + * If on run queue move to new run queue, and + * quit. Otherwise pick up mutex p is blocked on + */ + if ((m = p->p_blocked) == NULL) { + remrunqueue(p); + SET_PRIO(p, pri); + setrunqueue(p); + return; + } + /* + * Check if the proc needs to be moved up on + * the blocked chain + */ + if ((p1 = TAILQ_PREV(p, rq, p_procq)) == NULL || + p1->p_priority <= pri) + continue; + + /* + * Remove proc from blocked chain + */ + TAILQ_REMOVE(&m->mtx_blocked, p, p_procq); + TAILQ_FOREACH(p1, &m->mtx_blocked, p_procq) { + MPASS(p1->p_magic == P_MAGIC); + if (p1->p_priority > pri) + break; + } + if (p1) + TAILQ_INSERT_BEFORE(p1, p, p_procq); + else + TAILQ_INSERT_TAIL(&m->mtx_blocked, p, p_procq); + CTR4(KTR_LOCK, + "propagate priority: p 0x%x moved before 0x%x on [0x%x] %s", + p, p1, m, m->mtx_description); + } +} + +void +mtx_enter_hard(mtx_t *m, int type, int ipl) +{ + struct proc *p = CURPROC; + + switch (type) { + case MTX_DEF: + if ((m->mtx_lock & MTX_FLAGMASK) == (u_int64_t)p) { + m->mtx_recurse++; + atomic_set_64(&m->mtx_lock, MTX_RECURSE); + CTR1(KTR_LOCK, "mtx_enter: 0x%x recurse", m); + return; + } + CTR3(KTR_LOCK, "mtx_enter: 0x%x contested (lock=%x) [0x%x]", + m, m->mtx_lock, RETIP(m)); + while (!atomic_cmpset_64(&m->mtx_lock, MTX_UNOWNED, + (u_int64_t)p)) { + int v; + struct timeval tv; + struct proc *p1; + + mtx_enter(&sched_lock, MTX_SPIN | MTX_RLIKELY); + /* + * check if the lock has been released while + * waiting for the schedlock. + */ + if ((v = m->mtx_lock) == MTX_UNOWNED) { + mtx_exit(&sched_lock, MTX_SPIN); + continue; + } + /* + * The mutex was marked contested on release. This + * means that there are processes blocked on it. + */ + if (v == MTX_CONTESTED) { + p1 = TAILQ_FIRST(&m->mtx_blocked); + m->mtx_lock = (u_int64_t)p | MTX_CONTESTED; + if (p1->p_priority < p->p_priority) { + SET_PRIO(p, p1->p_priority); + } + mtx_exit(&sched_lock, MTX_SPIN); + return; + } + /* + * If the mutex isn't already contested and + * a failure occurs setting the contested bit the + * mutex was either release or the + * state of the RECURSION bit changed. + */ + if ((v & MTX_CONTESTED) == 0 && + !atomic_cmpset_64(&m->mtx_lock, v, + v | MTX_CONTESTED)) { + mtx_exit(&sched_lock, MTX_SPIN); + continue; + } + + /* We definitely have to sleep for this lock */ + mtx_assert(m, MA_NOTOWNED); + + printf("m->mtx_lock=%lx\n", m->mtx_lock); + +#ifdef notyet + /* + * If we're borrowing an interrupted thread's VM + * context must clean up before going to sleep. + */ + if (p->p_flag & (P_ITHD | P_SITHD)) { + ithd_t *it = (ithd_t *)p; + + if (it->it_interrupted) { + CTR2(KTR_LOCK, + "mtx_enter: 0x%x interrupted 0x%x", + it, it->it_interrupted); + intr_thd_fixup(it); + } + } +#endif + + /* Put us on the list of procs blocked on this mutex */ + if (TAILQ_EMPTY(&m->mtx_blocked)) { + p1 = (struct proc *)(m->mtx_lock & + MTX_FLAGMASK); + LIST_INSERT_HEAD(&p1->p_contested, m, + mtx_contested); + TAILQ_INSERT_TAIL(&m->mtx_blocked, p, p_procq); + } else { + TAILQ_FOREACH(p1, &m->mtx_blocked, p_procq) + if (p1->p_priority > p->p_priority) + break; + if (p1) + TAILQ_INSERT_BEFORE(p1, p, p_procq); + else + TAILQ_INSERT_TAIL(&m->mtx_blocked, p, + p_procq); + } + + p->p_blocked = m; /* Who we're blocked on */ +#ifdef notyet + propagate_priority(p); +#endif + CTR3(KTR_LOCK, "mtx_enter: p 0x%x blocked on [0x%x] %s", + p, m, m->mtx_description); + /* + * cloaned from mi_switch + */ + microtime(&tv); + p->p_runtime += (tv.tv_usec - + PCPU_GET(switchtime.tv_usec)) + + (tv.tv_sec - + PCPU_GET(switchtime.tv_sec)) * + (int64_t)1000000; + PCPU_SET(switchtime.tv_usec, tv.tv_usec); + PCPU_SET(switchtime.tv_sec, tv.tv_sec); + cpu_switch(); + if (PCPU_GET(switchtime.tv_sec) == 0) + microtime(&GLOBALP->gd_switchtime); + PCPU_SET(switchticks, ticks); + CTR3(KTR_LOCK, + "mtx_enter: p 0x%x free from blocked on [0x%x] %s", + p, m, m->mtx_description); + mtx_exit(&sched_lock, MTX_SPIN); + } + alpha_mb(); + return; + case MTX_SPIN: + case MTX_SPIN | MTX_FIRST: + case MTX_SPIN | MTX_TOPHALF: + { + int i = 0; + + if (m->mtx_lock == (u_int64_t)p) { + m->mtx_recurse++; + return; + } + CTR1(KTR_LOCK, "mtx_enter: 0x%x spinning", m); + for (;;) { + if (atomic_cmpset_64(&m->mtx_lock, MTX_UNOWNED, + (u_int64_t)p)) { + alpha_mb(); + break; + } + while (m->mtx_lock != MTX_UNOWNED) { + if (i++ < 1000000) + continue; + if (i++ < 6000000) + DELAY (1); + else + panic("spin lock > 5 seconds"); + } + } + +#ifdef SMP_DEBUG + if (type != MTX_SPIN) + m->mtx_saveipl = 0xbeefface; + else +#endif + m->mtx_saveipl = ipl; + CTR1(KTR_LOCK, "mtx_enter: 0x%x spin done", m); + return; + } + } +} + +void +mtx_exit_hard(mtx_t *m, int type) +{ + struct proc *p, *p1; + mtx_t *m1; + int pri; + + switch (type) { + case MTX_DEF: + case MTX_DEF | MTX_NOSWITCH: + if (m->mtx_recurse != 0) { + if (--(m->mtx_recurse) == 0) + atomic_clear_64(&m->mtx_lock, MTX_RECURSE); + CTR1(KTR_LOCK, "mtx_exit: 0x%x unrecurse", m); + return; + } + mtx_enter(&sched_lock, MTX_SPIN); + CTR1(KTR_LOCK, "mtx_exit: 0x%x contested", m); + p = CURPROC; + p1 = TAILQ_FIRST(&m->mtx_blocked); + MPASS(p->p_magic == P_MAGIC); + MPASS(p1->p_magic == P_MAGIC); + TAILQ_REMOVE(&m->mtx_blocked, p1, p_procq); + if (TAILQ_EMPTY(&m->mtx_blocked)) { + LIST_REMOVE(m, mtx_contested); + atomic_cmpset_64(&m->mtx_lock, m->mtx_lock, + MTX_UNOWNED); + CTR1(KTR_LOCK, "mtx_exit: 0x%x not held", m); + } else + m->mtx_lock = MTX_CONTESTED; + pri = MAXPRI; + LIST_FOREACH(m1, &p->p_contested, mtx_contested) { + int cp = TAILQ_FIRST(&m1->mtx_blocked)->p_priority; + if (cp < pri) + pri = cp; + } + if (pri > p->p_nativepri) + pri = p->p_nativepri; + SET_PRIO(p, pri); + CTR2(KTR_LOCK, "mtx_exit: 0x%x contested setrunqueue 0x%x", + m, p1); + p1->p_blocked = NULL; + setrunqueue(p1); + if ((type & MTX_NOSWITCH) == 0 && p1->p_priority < pri) { +#ifdef notyet + if (p->p_flag & (P_ITHD | P_SITHD)) { + ithd_t *it = (ithd_t *)p; + + if (it->it_interrupted) { + CTR2(KTR_LOCK, + "mtx_exit: 0x%x interruped 0x%x", + it, it->it_interrupted); + intr_thd_fixup(it); + } + } +#endif + setrunqueue(p); + CTR2(KTR_LOCK, "mtx_exit: 0x%x switching out lock=0x%x", + m, m->mtx_lock); + cpu_switch(); + CTR2(KTR_LOCK, "mtx_exit: 0x%x resuming lock=0x%x", + m, m->mtx_lock); + } + mtx_exit(&sched_lock, MTX_SPIN); + return; + case MTX_SPIN: + case MTX_SPIN | MTX_FIRST: + if (m->mtx_recurse != 0) { + m->mtx_recurse--; + return; + } + alpha_mb(); + if (atomic_cmpset_64(&m->mtx_lock, CURTHD, MTX_UNOWNED)) { + MPASS(m->mtx_saveipl != 0xbeefface); + alpha_pal_swpipl(m->mtx_saveipl); + return; + } + panic("unsucuessful release of spin lock"); + case MTX_SPIN | MTX_TOPHALF: + if (m->mtx_recurse != 0) { + m->mtx_recurse--; + return; + } + alpha_mb(); + if (atomic_cmpset_64(&m->mtx_lock, CURTHD, MTX_UNOWNED)) + return; + panic("unsucuessful release of spin lock"); + default: + panic("mtx_exit_hard: unsupported type 0x%x\n", type); + } +} + +#define MV_DESTROY 0 /* validate before destory */ +#define MV_INIT 1 /* validate before init */ + +#ifdef SMP_DEBUG + +int mtx_validate __P((mtx_t *, int)); + +int +mtx_validate(mtx_t *m, int when) +{ + mtx_t *mp; + int i; + int retval = 0; + + if (m == &all_mtx || cold) + return 0; + + mtx_enter(&all_mtx, MTX_DEF); + ASS(kernacc((caddr_t)all_mtx.mtx_next, 4, 1) == 1); + ASS(all_mtx.mtx_next->mtx_prev == &all_mtx); + for (i = 0, mp = all_mtx.mtx_next; mp != &all_mtx; mp = mp->mtx_next) { + if (kernacc((caddr_t)mp->mtx_next, 4, 1) != 1) { + panic("mtx_validate: mp=%p mp->mtx_next=%p", + mp, mp->mtx_next); + } + i++; + if (i > mtx_cur_cnt) { + panic("mtx_validate: too many in chain, known=%d\n", + mtx_cur_cnt); + } + } + ASS(i == mtx_cur_cnt); + switch (when) { + case MV_DESTROY: + for (mp = all_mtx.mtx_next; mp != &all_mtx; mp = mp->mtx_next) + if (mp == m) + break; + ASS(mp == m); + break; + case MV_INIT: + for (mp = all_mtx.mtx_next; mp != &all_mtx; mp = mp->mtx_next) + if (mp == m) { + /* + * Not good. This mutex already exits + */ + retval = 1; +#if 1 + printf("re-initing existing mutex %s\n", + m->mtx_description); + ASS(m->mtx_lock == MTX_UNOWNED); + retval = 1; +#else + panic("re-initing existing mutex %s", + m->mtx_description); +#endif + } + } + mtx_exit(&all_mtx, MTX_DEF); + return (retval); +} +#endif + +void +mtx_init(mtx_t *m, char *t, int flag) +{ + + CTR2(KTR_LOCK, "mtx_init 0x%x (%s)", m, t); +#ifdef SMP_DEBUG + if (mtx_validate(m, MV_INIT)) /* diagnostic and error correction */ + return; +#endif + bzero((void *)m, sizeof *m); + TAILQ_INIT(&m->mtx_blocked); + m->mtx_description = t; + m->mtx_lock = MTX_UNOWNED; + /* Put on all mutex queue */ + mtx_enter(&all_mtx, MTX_DEF); + m->mtx_next = &all_mtx; + m->mtx_prev = all_mtx.mtx_prev; + m->mtx_prev->mtx_next = m; + all_mtx.mtx_prev = m; + if (++mtx_cur_cnt > mtx_max_cnt) + mtx_max_cnt = mtx_cur_cnt; + mtx_exit(&all_mtx, MTX_DEF); + witness_init(m, flag); +} + +void +mtx_destroy(mtx_t *m) +{ + + CTR2(KTR_LOCK, "mtx_destroy 0x%x (%s)", m, m->mtx_description); +#ifdef SMP_DEBUG + if (m->mtx_next == NULL) + panic("mtx_destroy: %p (%s) already destroyed", + m, m->mtx_description); + + if (!mtx_owned(m)) { + ASS(m->mtx_lock == MTX_UNOWNED); + } else { + ASS((m->mtx_lock & (MTX_RECURSE|MTX_CONTESTED)) == 0); + } + mtx_validate(m, MV_DESTROY); /* diagnostic */ +#endif + +#ifdef WITNESS + if (m->mtx_witness) + witness_destroy(m); +#endif /* WITNESS */ + + /* Remove from the all mutex queue */ + mtx_enter(&all_mtx, MTX_DEF); + m->mtx_next->mtx_prev = m->mtx_prev; + m->mtx_prev->mtx_next = m->mtx_next; +#ifdef SMP_DEBUG + m->mtx_next = m->mtx_prev = NULL; +#endif + mtx_cur_cnt--; + mtx_exit(&all_mtx, MTX_DEF); +} diff --git a/sys/alpha/alpha/trap.c b/sys/alpha/alpha/trap.c index a21d532..072c5f3 100644 --- a/sys/alpha/alpha/trap.c +++ b/sys/alpha/alpha/trap.c @@ -35,8 +35,9 @@ #include #include +#include +#include #include -#include #include #include #include @@ -58,6 +59,8 @@ #include #include #include +#include +#include #ifdef KTRACE #include @@ -69,8 +72,6 @@ #endif u_int32_t want_resched; -u_int32_t astpending; -struct proc *fpcurproc; /* current user of the FPU */ void userret __P((struct proc *, u_int64_t, u_quad_t)); @@ -201,6 +202,11 @@ trap(a0, a1, a2, entry, framep) u_quad_t sticks; int user; + /* + * Find our per-cpu globals. + */ + globalp = (struct globaldata *) alpha_pal_rdval(); + cnt.v_trap++; p = curproc; ucode = 0; @@ -233,9 +239,12 @@ trap(a0, a1, a2, entry, framep) * and per-process unaligned-access-handling flags). */ if (user) { - if ((i = unaligned_fixup(a0, a1, a2, p)) == 0) + mtx_enter(&Giant, MTX_DEF); + if ((i = unaligned_fixup(a0, a1, a2, p)) == 0) { + mtx_exit(&Giant, MTX_DEF); goto out; - + } + mtx_exit(&Giant, MTX_DEF); ucode = a0; /* VA */ break; } @@ -259,9 +268,13 @@ trap(a0, a1, a2, entry, framep) * is not requested or if the completion fails. */ if (user) { + mtx_enter(&Giant, MTX_DEF); if (a0 & EXCSUM_SWC) - if (fp_software_completion(a1, p)) + if (fp_software_completion(a1, p)) { + mtx_exit(&Giant, MTX_DEF); goto out; + } + mtx_exit(&Giant, MTX_DEF); i = SIGFPE; ucode = a0; /* exception summary */ break; @@ -364,6 +377,7 @@ trap(a0, a1, a2, entry, framep) vm_prot_t ftype = 0; int rv; + mtx_enter(&Giant, MTX_DEF); /* * If it was caused by fuswintr or suswintr, * just punt. Note that we check the faulting @@ -379,6 +393,7 @@ trap(a0, a1, a2, entry, framep) framep->tf_regs[FRAME_PC] = p->p_addr->u_pcb.pcb_onfault; p->p_addr->u_pcb.pcb_onfault = 0; + mtx_exit(&Giant, MTX_DEF); goto out; } @@ -489,9 +504,11 @@ trap(a0, a1, a2, entry, framep) rv = KERN_INVALID_ADDRESS; } if (rv == KERN_SUCCESS) { + mtx_exit(&Giant, MTX_DEF); goto out; } + mtx_exit(&Giant, MTX_DEF); if (!user) { /* Check for copyin/copyout fault */ if (p != NULL && @@ -573,6 +590,12 @@ syscall(code, framep) u_int64_t args[10]; /* XXX */ u_int hidden = 0, nargs; + /* + * Find our per-cpu globals. + */ + globalp = (struct globaldata *) alpha_pal_rdval(); + mtx_enter(&Giant, MTX_DEF); + framep->tf_regs[FRAME_TRAPARG_A0] = 0; framep->tf_regs[FRAME_TRAPARG_A1] = 0; framep->tf_regs[FRAME_TRAPARG_A2] = 0; @@ -693,6 +716,7 @@ syscall(code, framep) * is not the case, this code will need to be revisited. */ STOPEVENT(p, S_SCX, code); + mtx_exit(&Giant, MTX_DEF); } /* @@ -712,6 +736,7 @@ child_return(p) if (KTRPOINT(p, KTR_SYSRET)) ktrsysret(p->p_tracep, SYS_fork, 0, 0); #endif + mtx_exit(&Giant, MTX_DEF); } /* @@ -725,6 +750,8 @@ ast(framep) register struct proc *p; u_quad_t sticks; + mtx_enter(&Giant, MTX_DEF); + p = curproc; sticks = p->p_sticks; p->p_md.md_tf = framep; @@ -734,7 +761,7 @@ ast(framep) cnt.v_soft++; - astpending = 0; + PCPU_SET(astpending, 0); if (p->p_flag & P_OWEUPC) { p->p_flag &= ~P_OWEUPC; addupc_task(p, p->p_stats->p_prof.pr_addr, @@ -742,6 +769,8 @@ ast(framep) } userret(p, framep->tf_regs[FRAME_PC], sticks); + + mtx_exit(&Giant, MTX_DEF); } /* diff --git a/sys/alpha/alpha/vm_machdep.c b/sys/alpha/alpha/vm_machdep.c index 8baea02..3831d67 100644 --- a/sys/alpha/alpha/vm_machdep.c +++ b/sys/alpha/alpha/vm_machdep.c @@ -84,6 +84,7 @@ #include #include #include +#include #include #include @@ -246,8 +247,10 @@ cpu_exit(p) alpha_fpstate_drop(p); (void) splhigh(); + mtx_enter(&sched_lock, MTX_SPIN); + mtx_exit(&Giant, MTX_DEF); cnt.v_swtch++; - cpu_switch(p); + cpu_switch(); panic("cpu_exit"); } @@ -358,7 +361,7 @@ vunmapbuf(bp) } /* - * Force reset the processor by invalidating the entire address space! + * Reset back to firmware. */ void cpu_reset() @@ -416,7 +419,7 @@ vm_page_zero_idle() return(0); #ifdef SMP - if (try_mplock()) { + if (KLOCK_ENTER(M_TRY)) { #endif s = splvm(); m = vm_page_list_find(PQ_FREE, free_rover, FALSE); @@ -447,7 +450,7 @@ vm_page_zero_idle() free_rover = (free_rover + PQ_PRIME2) & PQ_L2_MASK; splx(s); #ifdef SMP - rel_mplock(); + KLOCK_EXIT; #endif return (1); #ifdef SMP diff --git a/sys/alpha/include/asm.h b/sys/alpha/include/asm.h index b185295..d46eb97 100644 --- a/sys/alpha/include/asm.h +++ b/sys/alpha/include/asm.h @@ -90,6 +90,11 @@ #define sp $30 /* (S) stack pointer */ #define zero $31 /* wired zero */ +/* In the kernel, we use t7 to point at the per-cpu globals. */ +#ifdef _KERNEL +#define globalp $8 +#endif + /* Floating point registers (XXXX VERIFY THIS) */ #define fv0 $f0 /* (T) return value (real) */ #define fv1 $f1 /* (T) return value (imaginary)*/ @@ -266,7 +271,6 @@ _name_:; \ .loc 1 __LINE__; \ bsr ra,exception_save_regs /* jmp/CALL trashes pv/t12 */ - /* * LEAF * Declare a global leaf function. diff --git a/sys/alpha/include/cpu.h b/sys/alpha/include/cpu.h index c9d783b..99eb79e 100644 --- a/sys/alpha/include/cpu.h +++ b/sys/alpha/include/cpu.h @@ -65,7 +65,7 @@ struct clockframe { #define CLKF_BASEPRI(framep) \ (((framep)->cf_tf.tf_regs[FRAME_PS] & ALPHA_PSL_IPL_MASK) == 0) #define CLKF_PC(framep) ((framep)->cf_tf.tf_regs[FRAME_PC]) -#define CLKF_INTR(framep) (intr_nesting_level >= 2) +#define CLKF_INTR(framep) (PCPU_GET(intr_nesting_level) >= 2) /* * Preempt the current process if in interrupt from user mode, @@ -89,9 +89,10 @@ struct clockframe { */ #define signotify(p) aston() -#define aston() (astpending = 1) +#define aston() PCPU_SET(astpending, 1) #ifdef _KERNEL +extern u_int astpending; extern u_int32_t intr_nesting_level; /* bookeeping only; counts sw intrs */ extern u_int32_t want_resched; /* resched() was called */ #endif @@ -132,7 +133,6 @@ struct reg; struct rpb; struct trapframe; -extern struct proc *fpcurproc; extern struct rpb *hwrpb; extern volatile int mc_expected, mc_received; diff --git a/sys/alpha/include/cpufunc.h b/sys/alpha/include/cpufunc.h index e7d37f0..cabfe0f 100644 --- a/sys/alpha/include/cpufunc.h +++ b/sys/alpha/include/cpufunc.h @@ -33,6 +33,7 @@ #include #include +#include #ifdef __GNUC__ @@ -44,6 +45,33 @@ breakpoint(void) #endif +/* + * Bogus interrupt manipulation + */ +static __inline void +disable_intr(void) +{ + alpha_pal_swpipl(ALPHA_PSL_IPL_HIGH); +} + +static __inline void +enable_intr(void) +{ + alpha_pal_swpipl(ALPHA_PSL_IPL_0); +} + +static __inline u_int +save_intr(void) +{ + return alpha_pal_rdps() & ALPHA_PSL_IPL_MASK; +} + +static __inline void +restore_intr(u_int ipl) +{ + alpha_pal_swpipl(ipl); +} + #endif /* _KERNEL */ #endif /* !_MACHINE_CPUFUNC_H_ */ diff --git a/sys/alpha/include/globaldata.h b/sys/alpha/include/globaldata.h new file mode 100644 index 0000000..b246bb1 --- /dev/null +++ b/sys/alpha/include/globaldata.h @@ -0,0 +1,79 @@ +/*- + * Copyright (c) 1999 Luoqi Chen + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _MACHINE_GLOBALDATA_H_ +#define _MACHINE_GLOBALDATA_H_ + +#ifdef _KERNEL + +#include + +/* + * This structure maps out the global data that needs to be kept on a + * per-cpu basis. genassym uses this to generate offsets for the assembler + * code, which also provides external symbols so that C can get at them as + * though they were really globals. This structure is pointed to by + * the per-cpu system value (see alpha_pal_rdval() and alpha_pal_wrval()). + * Inside the kernel, the globally reserved register t7 is used to + * point at the globaldata structure. + */ +struct globaldata { + struct alpha_pcb gd_idlepcb; /* pcb for idling */ + struct proc *gd_curproc; /* current process */ + struct proc *gd_idleproc; /* idle process */ + struct proc *gd_fpcurproc; /* fp state owner */ + struct pcb *gd_curpcb; /* current pcb */ + struct timeval gd_switchtime; + int gd_switchticks; + u_int gd_cpuno; /* this cpu number */ + u_int gd_other_cpus; /* all other cpus */ + int gd_inside_intr; + u_int64_t gd_idlepcbphys; /* pa of gd_idlepcb */ + u_int64_t gd_pending_ipis; /* pending IPI events */ + u_int32_t gd_next_asn; /* next ASN to allocate */ + u_int32_t gd_current_asngen; /* ASN rollover check */ + u_int32_t gd_intr_nesting_level; /* interrupt recursion */ + + u_int gd_astpending; + SLIST_ENTRY(globaldata) gd_allcpu; +#ifdef KTR_PERCPU + volatile int gd_ktr_idx; /* Index into trace table */ + char *gd_ktr_buf; + char gd_ktr_buf_data[0]; +#endif +}; + +SLIST_HEAD(cpuhead, globaldata); +extern struct cpuhead cpuhead; + +void globaldata_init(struct globaldata *pcpu, int cpuno, size_t sz); +struct globaldata *globaldata_find(int cpuno); + +#endif /* _KERNEL */ + +#endif /* !_MACHINE_GLOBALDATA_H_ */ diff --git a/sys/alpha/include/globals.h b/sys/alpha/include/globals.h new file mode 100644 index 0000000..303efdf --- /dev/null +++ b/sys/alpha/include/globals.h @@ -0,0 +1,63 @@ +/*- + * Copyright (c) 1999 Luoqi Chen + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _MACHINE_GLOBALS_H_ +#define _MACHINE_GLOBALS_H_ + +#ifdef _KERNEL + +register struct globaldata *globalp __asm__("$8"); + +#if 1 +#define GLOBALP globalp +#else +#define GLOBALP ((struct globaldata *) alpha_pal_rdval()) +#endif + +#define PCPU_GET(name) (GLOBALP->gd_##name) +#define PCPU_SET(name,value) (GLOBALP->gd_##name = (value)) + +/* + * The following set of macros works for UP kernel as well, but for maximum + * performance we allow the global variables to be accessed directly. On the + * other hand, kernel modules should always use these macros to maintain + * portability between UP and SMP kernels. + */ +#define CURPROC PCPU_GET(curproc) +#define curproc PCPU_GET(curproc) +#define idleproc PCPU_GET(idleproc) +#define curpcb PCPU_GET(curpcb) +#define fpcurproc PCPU_GET(fpcurproc) +#define switchtime PCPU_GET(switchtime) +#define switchticks PCPU_GET(switchticks) +#define cpuid PCPU_GET(cpuno) +#define prevproc PCPU_GET(curproc) /* XXX - until ithreads */ + +#endif /* _KERNEL */ + +#endif /* !_MACHINE_GLOBALS_H_ */ diff --git a/sys/alpha/include/ipl.h b/sys/alpha/include/ipl.h index ea93fbb..2e9b3cc 100644 --- a/sys/alpha/include/ipl.h +++ b/sys/alpha/include/ipl.h @@ -127,4 +127,19 @@ extern void schedsoftclock(void); extern unsigned cpl; /* current priority level mask */ #endif +/* + * Interprocessor interrupts for SMP. + */ +#define IPI_INVLTLB 0x0001 +#define IPI_RENDEZVOUS 0x0002 +#define IPI_AST 0x0004 +#define IPI_CHECKSTATE 0x0008 +#define IPI_STOP 0x0010 + +void smp_ipi_selected(u_int32_t cpus, u_int64_t ipi); +void smp_ipi_all(u_int64_t ipi); +void smp_ipi_all_but_self(u_int64_t ipi); +void smp_ipi_self(u_int64_t ipi); +void smp_handle_ipi(struct trapframe *frame); + #endif /* !_MACHINE_MD_VAR_H_ */ diff --git a/sys/alpha/include/lock.h b/sys/alpha/include/lock.h index c2ae0fa..1066d46 100644 --- a/sys/alpha/include/lock.h +++ b/sys/alpha/include/lock.h @@ -35,10 +35,40 @@ * It is an error to hold one of these locks while a process is sleeping. */ struct simplelock { - volatile int lock_data; + volatile u_int lock_data; }; +/* functions in mp_machdep.c */ +void s_lock_init __P((struct simplelock *)); +void s_lock __P((struct simplelock *)); +int s_lock_try __P((struct simplelock *)); +void ss_lock __P((struct simplelock *)); +void ss_unlock __P((struct simplelock *)); +void s_lock_np __P((struct simplelock *)); +void s_unlock_np __P((struct simplelock *)); + +/* inline simplelock functions */ +static __inline void +s_unlock(struct simplelock *lkp) +{ + alpha_mb(); + lkp->lock_data = 0; +} + +#if !defined(SIMPLELOCK_DEBUG) && NCPUS > 1 +/* + * This set of defines turns on the real functions in i386/isa/apic_ipl.s. + */ +#define simple_lock_init(alp) s_lock_init(alp) +#define simple_lock(alp) s_lock(alp) +#define simple_lock_try(alp) s_lock_try(alp) +#define simple_unlock(alp) s_unlock(alp) + +#endif /* !SIMPLELOCK_DEBUG && NCPUS > 1 */ + #define COM_LOCK() #define COM_UNLOCK() +#define COM_DISABLE_INTR() COM_LOCK() +#define COM_ENABLE_INTR() COM_UNLOCK() #endif /* !_MACHINE_LOCK_H_ */ diff --git a/sys/alpha/include/mutex.h b/sys/alpha/include/mutex.h new file mode 100644 index 0000000..ac13b8c --- /dev/null +++ b/sys/alpha/include/mutex.h @@ -0,0 +1,563 @@ +/*- + * Copyright (c) 1997 Berkeley Software Design, Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Berkeley Software Design Inc's name may not be used to endorse or + * promote products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * from BSDI $Id: mutex.h,v 2.7.2.35 2000/04/27 03:10:26 cp Exp $ + * $FreeBSD$ + */ + + +#ifndef _MACHINE_MUTEX_H_ +#define _MACHINE_MUTEX_H_ + +#ifndef LOCORE + +#include +#include +#include +#include + +/* + * Mutex flags + * + * Types + */ +#define MTX_DEF 0x1 /* Default (spin/sleep) */ +#define MTX_SPIN 0x2 /* Spin only lock */ + +/* Options */ +#define MTX_RLIKELY 0x4 /* (opt) Recursion likely */ +#define MTX_NORECURSE 0x8 /* No recursion possible */ +#define MTX_NOSPIN 0x10 /* Don't spin before sleeping */ +#define MTX_NOSWITCH 0x20 /* Do not switch on release */ +#define MTX_FIRST 0x40 /* First spin lock holder */ +#define MTX_TOPHALF 0x80 /* Interrupts not disabled on spin */ + +/* options that should be passed on to mtx_enter_hard, mtx_exit_hard */ +#define MTX_HARDOPTS (MTX_DEF | MTX_SPIN | MTX_FIRST | MTX_TOPHALF | MTX_NOSWITCH) + +/* Flags/value used in mtx_lock */ +#define MTX_RECURSE 0x01 /* (non-spin) lock held recursively */ +#define MTX_CONTESTED 0x02 /* (non-spin) lock contested */ +#define MTX_FLAGMASK ~(MTX_RECURSE | MTX_CONTESTED) +#define MTX_UNOWNED 0x8 /* Cookie for free mutex */ + +struct proc; /* XXX */ + +/* + * Sleep/spin mutex + */ +struct mtx { + volatile u_int64_t mtx_lock; /* lock owner/gate/flags */ + volatile u_int32_t mtx_recurse; /* number of recursive holds */ + u_int32_t mtx_saveipl; /* saved ipl (for spin locks) */ + char *mtx_description; + TAILQ_HEAD(, proc) mtx_blocked; + LIST_ENTRY(mtx) mtx_contested; + struct mtx *mtx_next; /* all locks in system */ + struct mtx *mtx_prev; +#ifdef SMP_DEBUG + /* If you add anything here, adjust the mtxf_t definition below */ + struct witness *mtx_witness; + LIST_ENTRY(mtx) mtx_held; + char *mtx_file; + int mtx_line; +#endif /* SMP_DEBUG */ +}; + +typedef struct mtx mtx_t; + +/* + * Filler for structs which need to remain the same size + * whether or not SMP_DEBUG is turned on. + */ +typedef struct mtxf { +#ifdef SMP_DEBUG + char mtxf_data[0]; +#else + char mtxf_data[4*sizeof(void *) + sizeof(int)]; +#endif +} mtxf_t; + +#define mp_fixme(string) + +#ifdef _KERNEL +/* Misc */ +#define CURTHD ((u_int64_t)CURPROC) /* Current thread ID */ + +/* Prototypes */ +void mtx_init(mtx_t *m, char *description, int flag); +void mtx_enter_hard(mtx_t *, int type, int ipl); +void mtx_exit_hard(mtx_t *, int type); +void mtx_destroy(mtx_t *m); + +/* Global locks */ +extern mtx_t sched_lock; +extern mtx_t Giant; + +/* + * Used to replace return with an exit Giant and return. + */ + +#define EGAR(a) \ +do { \ + mtx_exit(&Giant, MTX_DEF); \ + return (a); \ +} while (0) + +#define VEGAR \ +do { \ + mtx_exit(&Giant, MTX_DEF); \ + return; \ +} while (0) + +#define DROP_GIANT() \ +do { \ + int _giantcnt; \ + WITNESS_SAVE_DECL(Giant); \ + \ + WITNESS_SAVE(&Giant, Giant); \ + for (_giantcnt = 0; mtx_owned(&Giant); _giantcnt++) \ + mtx_exit(&Giant, MTX_DEF) + +#define PICKUP_GIANT() \ + mtx_assert(&Giant, MA_NOTOWNED); \ + while (_giantcnt--) \ + mtx_enter(&Giant, MTX_DEF); \ + WITNESS_RESTORE(&Giant, Giant); \ +} while (0) + +#define PARTIAL_PICKUP_GIANT() \ + mtx_assert(&Giant, MA_NOTOWNED); \ + while (_giantcnt--) \ + mtx_enter(&Giant, MTX_DEF); \ + WITNESS_RESTORE(&Giant, Giant) + + +/* + * Debugging + */ +#ifndef SMP_DEBUG +#define mtx_assert(m, what) +#else /* SMP_DEBUG */ + +#define MA_OWNED 1 +#define MA_NOTOWNED 2 +#define mtx_assert(m, what) { \ + switch ((what)) { \ + case MA_OWNED: \ + ASS(mtx_owned((m))); \ + break; \ + case MA_NOTOWNED: \ + ASS(!mtx_owned((m))); \ + break; \ + default: \ + panic("unknown mtx_assert at %s:%d", __FILE__, __LINE__); \ + } \ +} + +#ifdef INVARIANTS +#define ASS(ex) MPASS(ex) +#define MPASS(ex) if (!(ex)) panic("Assertion %s failed at %s:%d", \ + #ex, __FILE__, __LINE__) +#define MPASS2(ex, what) if (!(ex)) panic("Assertion %s failed at %s:%d", \ + what, __FILE__, __LINE__) + +#ifdef MTX_STRS +char STR_IEN[] = "fl & 0x200"; +char STR_IDIS[] = "!(fl & 0x200)"; +#else /* MTX_STRS */ +extern char STR_IEN[]; +extern char STR_IDIS[]; +#endif /* MTX_STRS */ +#define ASS_IEN MPASS2((alpha_pal_rdps & ALPHA_PSL_IPL_MASK) + == ALPHA_PSL_IPL_HIGH, STR_IEN) +#define ASS_IDIS MPASS2((alpha_pal_rdps & ALPHA_PSL_IPL_MASK) + != ALPHA_PSL_IPL_HIGH, STR_IDIS) +#endif /* INVARIANTS */ + +#endif /* SMP_DEBUG */ + +#if !defined(SMP_DEBUG) || !defined(INVARIANTS) +#define ASS(ex) +#define MPASS(ex) +#define MPASS2(ex, where) +#define ASS_IEN +#define ASS_IDIS +#endif /* !defined(SMP_DEBUG) || !defined(INVARIANTS) */ + +#ifdef WITNESS +#ifndef SMP_DEBUG +#error WITNESS requires SMP_DEBUG +#endif /* SMP_DEBUG */ +#define WITNESS_ENTER(m, f) \ + if ((m)->mtx_witness != NULL) \ + witness_enter((m), (f), __FILE__, __LINE__) +#define WITNESS_EXIT(m, f) \ + if ((m)->mtx_witness != NULL) \ + witness_exit((m), (f), __FILE__, __LINE__) + +#define WITNESS_SLEEP(check, m) witness_sleep(check, (m), __FILE__, __LINE__) +#define WITNESS_SAVE_DECL(n) \ + char * __CONCAT(n, __wf); \ + int __CONCAT(n, __wl) + +#define WITNESS_SAVE(m, n) \ +do { \ + if ((m)->mtx_witness != NULL) \ + witness_save(m, &__CONCAT(n, __wf), &__CONCAT(n, __wl)); \ +} while (0) + +#define WITNESS_RESTORE(m, n) \ +do { \ + if ((m)->mtx_witness != NULL) \ + witness_restore(m, __CONCAT(n, __wf), __CONCAT(n, __wl)); \ +} while (0) + +void witness_init(mtx_t *, int flag); +void witness_destroy(mtx_t *); +void witness_enter(mtx_t *, int, char *, int); +void witness_try_enter(mtx_t *, int, char *, int); +void witness_exit(mtx_t *, int, char *, int); +void witness_display(void(*)(const char *fmt, ...)); +void witness_list(struct proc *); +int witness_sleep(int, mtx_t *, char *, int); +void witness_save(mtx_t *, char **, int *); +void witness_restore(mtx_t *, char *, int); +#else /* WITNESS */ +#define WITNESS_ENTER(m, flag) +#define WITNESS_EXIT(m, flag) +#define WITNESS_SLEEP(check, m) +#define WITNESS_SAVE_DECL(n) +#define WITNESS_SAVE(m, n) +#define WITNESS_RESTORE(m, n) + +/* + * flag++ is slezoid way of shutting up unused parameter warning + * in mtx_init() + */ +#define witness_init(m, flag) flag++ +#define witness_destroy(m) +#define witness_enter(m, flag, f, l) +#define witness_try_enter(m, flag, f, l ) +#define witness_exit(m, flag, f, l) +#endif /* WITNESS */ + +/* + * Assembly macros (for internal use only) + *-------------------------------------------------------------------------- + */ + +/* + * Get a sleep lock, deal with recursion inline + */ + +#define _V(x) __STRING(x) + +#define _getlock_sleep(mp, tid, type) do { \ + if (atomic_cmpset_64(&(mp)->mtx_lock, MTX_UNOWNED, (tid)) == 0) { \ + if (((mp)->mtx_lock & MTX_FLAGMASK) != (tid)) \ + mtx_enter_hard(mp, (type) & MTX_HARDOPTS, 0); \ + else { \ + if (((mp)->mtx_lock & MTX_RECURSE) == 0) \ + atomic_set_64(&(mp)->mtx_lock, MTX_RECURSE); \ + (mp)->mtx_recurse++; \ + } \ + } else { \ + alpha_mb(); \ + } \ +} while (0) + +/* + * Get a spin lock, handle recusion inline (as the less common case) + */ + +#define _getlock_spin_block(mp, tid, type) do { \ + u_int _ipl = alpha_pal_rdps() & ALPHA_PSL_IPL_MASK; \ + if (atomic_cmpset_64(&(mp)->mtx_lock, MTX_UNOWNED, (tid)) == 0) \ + mtx_enter_hard(mp, (type) & MTX_HARDOPTS, _ipl); \ + else { \ + alpha_mb(); \ + (mp)->mtx_saveipl = _ipl; \ + } \ +} while (0) + +/* + * Get a lock without any recursion handling. Calls the hard enter + * function if we can't get it inline. + */ + +#define _getlock_norecurse(mp, tid, type) do { \ + if (atomic_cmpset_64(&(mp)->mtx_lock, MTX_UNOWNED, (tid)) == 0) \ + mtx_enter_hard((mp), (type) & MTX_HARDOPTS, 0); \ + else \ + alpha_mb(); \ +} while (0) + +/* + * Release a sleep lock assuming we haven't recursed on it, recursion is + * handled in the hard function. + */ + +#define _exitlock_norecurse(mp, tid, type) do { \ + alpha_mb(); \ + if (atomic_cmpset_64(&(mp)->mtx_lock, (tid), MTX_UNOWNED) == 0) \ + mtx_exit_hard((mp), (type) & MTX_HARDOPTS); \ +} while (0) + +/* + * Release a sleep lock when its likely we recursed (the code to + * deal with simple recursion is inline). + */ + +#define _exitlock(mp, tid, type) do { \ + alpha_mb(); \ + if (atomic_cmpset_64(&(mp)->mtx_lock, (tid), MTX_UNOWNED) == 0) {\ + if (((mp)->mtx_lock & MTX_RECURSE) && \ + (--(mp)->mtx_recurse == 0)) \ + atomic_clear_64(&(mp)->mtx_lock, MTX_RECURSE); \ + else \ + mtx_exit_hard((mp), (type) & MTX_HARDOPTS); \ + } \ +} while (0) + +/* + * Release a spin lock (with possible recursion) + */ + +#define _exitlock_spin(mp) do { \ + int _ipl = (mp)->mtx_saveipl; \ + alpha_mb(); \ + if ((mp)->mtx_recurse == 0 || (--(mp)->mtx_recurse) == 0) \ + atomic_cmpset_64(&(mp)->mtx_lock, (mp)->mtx_lock, \ + MTX_UNOWNED); \ + alpha_pal_swpipl(_ipl); \ +} while (0) + +/* + * Externally visible mutex functions + *------------------------------------------------------------------------ + */ + +/* + * Return non-zero if a mutex is already owned by the current thread + */ +#define mtx_owned(m) (((m)->mtx_lock & MTX_FLAGMASK) == CURTHD) + +/* Common strings */ +#ifdef MTX_STRS +char STR_mtx_enter_fmt[] = "GOT %s [%p] at %s:%d r=%d"; +char STR_mtx_bad_type[] = "((type) & (MTX_NORECURSE | MTX_NOSWITCH)) == 0"; +char STR_mtx_exit_fmt[] = "REL %s [%p] at %s:%d r=%d"; +char STR_mtx_owned[] = "mtx_owned(_mpp)"; +char STR_mtx_recurse[] = "_mpp->mtx_recurse == 0"; +char STR_mtx_try_enter_fmt[] = "TRY_ENTER %s [%p] at %s:%d result=%d"; +#else /* MTX_STRS */ +extern char STR_mtx_enter_fmt[]; +extern char STR_mtx_bad_type[]; +extern char STR_mtx_exit_fmt[]; +extern char STR_mtx_owned[]; +extern char STR_mtx_recurse[]; +extern char STR_mtx_try_enter_fmt[]; +#endif /* MTX_STRS */ + +/* + * Get lock 'm', the macro handles the easy (and most common cases) and + * leaves the slow stuff to the mtx_enter_hard() function. + * + * Note: since type is usually a constant much of this code is optimized out + */ +#define mtx_enter(mtxp, type) do { \ + mtx_t * _mpp = mtxp; \ + \ + /* bits only valid on mtx_exit() */ \ + MPASS2(((type) & (MTX_NORECURSE | MTX_NOSWITCH)) == 0, STR_mtx_bad_type); \ + \ + do { \ + if ((type) & MTX_SPIN) { \ + /* \ + * Easy cases of spin locks: \ + * \ + * 1) We already own the lock and will simply \ + * recurse on it (if RLIKELY) \ + * \ + * 2) The lock is free, we just get it \ + */ \ + if ((type) & MTX_RLIKELY) { \ + /* \ + * Check for recursion, if we already \ + * have this lock we just bump the \ + * recursion count. \ + */ \ + if (_mpp->mtx_lock == CURTHD) { \ + _mpp->mtx_recurse++; \ + break; /* Done */ \ + } \ + } \ + \ + if (((type) & MTX_TOPHALF) == 0) \ + /* \ + * If an interrupt thread uses this \ + * we must block interrupts here. \ + */ \ + _getlock_spin_block(_mpp, CURTHD, \ + (type) & MTX_HARDOPTS); \ + else \ + _getlock_norecurse(_mpp, CURTHD, \ + (type) & MTX_HARDOPTS); \ + } else { \ + /* Sleep locks */ \ + if ((type) & MTX_RLIKELY) \ + _getlock_sleep(_mpp, CURTHD, \ + (type) & MTX_HARDOPTS); \ + else \ + _getlock_norecurse(_mpp, CURTHD, \ + (type) & MTX_HARDOPTS); \ + } \ + } while (0); \ + WITNESS_ENTER(_mpp, type); \ + CTR5(KTR_LOCK, STR_mtx_enter_fmt, \ + (_mpp)->mtx_description, (_mpp), __FILE__, __LINE__, \ + (_mpp)->mtx_recurse); \ +} while (0) + +/* + * Attempt to get MTX_DEF lock, return non-zero if lock acquired + * + * XXX DOES NOT HANDLE RECURSION + */ +#ifdef SMP_DEBUG +#define mtx_try_enter(mtxp, type) ({ \ + mtx_t *const _mpp = mtxp; \ + int _rval; \ + \ + _rval = atomic_cmpset_int(&_mpp->mtx_lock, MTX_UNOWNED, CURTHD);\ + if (_rval && (_mpp)->mtx_witness != NULL) { \ + ASS((_mpp)->mtx_recurse == 0); \ + witness_try_enter(_mpp, type, __FILE__, __LINE__); \ + } \ + CTR5(KTR_LOCK, STR_mtx_try_enter_fmt, \ + (_mpp)->mtx_description, (_mpp), __FILE__, __LINE__, \ + _rval); \ + _rval; \ +}) + +#else /* SMP_DEBUG */ + +#define mtx_try_enter(mtxp, type) ({ \ + mtx_t *const _mpp = mtxp; \ + int _rval; \ + \ + _rval = atomic_cmpset_int(&_mpp->mtx_lock, MTX_UNOWNED, CURTHD);\ + CTR5(KTR_LOCK, STR_mtx_try_enter_fmt, \ + (_mpp)->mtx_description, (_mpp), __FILE__, __LINE__, \ + _rval); \ + _rval; \ +}) + +#endif /* SMP_DEBUG */ + +#if 0 +#define mtx_legal2block() ({ \ + register int _l2b; \ + __asm __volatile ( \ +" pushfl;" \ +" popl %%eax;" \ +" andl $0x200, %%eax;" \ + : "=a" (_l2b) \ + : \ + : "cc"); \ + _l2b; \ +}) +#endif + +#define mtx_legal2block() (read_eflags() & 0x200) + +/* + * Release lock m + */ +#define mtx_exit(mtxp, type) do { \ + mtx_t *const _mpp = mtxp; \ + \ + MPASS2(mtx_owned(_mpp), STR_mtx_owned); \ + WITNESS_EXIT(_mpp, type); \ + CTR5(KTR_LOCK, STR_mtx_exit_fmt, \ + (_mpp)->mtx_description, (_mpp), __FILE__, __LINE__, \ + (_mpp)->mtx_recurse); \ + if ((type) & MTX_SPIN) { \ + if ((type) & MTX_NORECURSE) { \ + MPASS2(_mpp->mtx_recurse == 0, STR_mtx_recurse); \ + atomic_cmpset_64(&_mpp->mtx_lock, _mpp->mtx_lock, \ + MTX_UNOWNED); \ + if (((type) & MTX_TOPHALF) == 0) { \ + splx(_mpp->mtx_saveipl); \ + } \ + } else \ + if ((type) & MTX_TOPHALF) \ + _exitlock_norecurse(_mpp, CURTHD, \ + (type) & MTX_HARDOPTS); \ + else \ + _exitlock_spin(_mpp); \ + } else { \ + /* Handle sleep locks */ \ + if ((type) & MTX_RLIKELY) \ + _exitlock(_mpp, CURTHD, (type) & MTX_HARDOPTS); \ + else \ + _exitlock_norecurse(_mpp, CURTHD, \ + (type) & MTX_HARDOPTS); \ + } \ +} while (0) +#endif /* _KERNEL */ + +#else /* !LOCORE */ + +/* + * Simple assembly macros to get and release non-recursive spin locks + */ +#define MTX_ENTER(lck) \ + call_pal PAL_OSF1_rdps; \ + and v0, ALPHA_PSL_IPL_MASK, v0; \ +1: ldq_l a0, lck+MTX_LOCK; \ + cmpeq a0, MTX_UNOWNED, a1; \ + beq a1, 1b; \ + ldq a0, PC_CURPROC(globalp); \ + stq_c a0, lck+MTX_LOCK; \ + beq a0, 1b; \ + mb; \ + stl v0, lck+MTX_SAVEIPL; \ + ldq a0, ALPHA_PSL_IPL_HIGH; \ + call_pal PSL_OSF1_swpipl + +#define MTX_EXIT(lck) \ + mb; \ + ldiq a0, MTX_UNOWNED; \ + stq a0, lck+MTX_LOCK; \ + ldl a0, lck+MTX_SAVEIPL; \ + call_pal PAL_OSF1_swpipl + +#endif /* !LOCORE */ + +#endif /* __MACHINE_MUTEX_H */ diff --git a/sys/alpha/include/param.h b/sys/alpha/include/param.h index 80dce22..742a3f7 100644 --- a/sys/alpha/include/param.h +++ b/sys/alpha/include/param.h @@ -70,7 +70,11 @@ #define OBJFORMAT_NAMES "elf" #define OBJFORMAT_DEFAULT "elf" +#ifdef SMP +#define NCPUS 32 +#else #define NCPUS 1 +#endif /* * Round p (pointer or byte index) up to a correctly-aligned value for all diff --git a/sys/alpha/include/pcb.h b/sys/alpha/include/pcb.h index 3caa144..3bf2586 100644 --- a/sys/alpha/include/pcb.h +++ b/sys/alpha/include/pcb.h @@ -30,7 +30,7 @@ #include #include - +#include #include /* @@ -53,6 +53,7 @@ struct pcb { u_int64_t pcb_fp_control; /* IEEE control word [SW] */ unsigned long pcb_onfault; /* for copy faults [SW] */ unsigned long pcb_accessaddr; /* for [fs]uswintr [SW] */ + u_int32_t pcb_schednest; /* state of sched_lock [SW] */ }; /* @@ -64,3 +65,9 @@ struct md_coredump { struct trapframe md_tf; struct fpreg md_fpstate; }; + +#ifdef _KERNEL +#ifndef curpcb +extern struct pcb *curpcb; /* our current running pcb */ +#endif +#endif diff --git a/sys/alpha/include/pcpu.h b/sys/alpha/include/pcpu.h new file mode 100644 index 0000000..b246bb1 --- /dev/null +++ b/sys/alpha/include/pcpu.h @@ -0,0 +1,79 @@ +/*- + * Copyright (c) 1999 Luoqi Chen + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _MACHINE_GLOBALDATA_H_ +#define _MACHINE_GLOBALDATA_H_ + +#ifdef _KERNEL + +#include + +/* + * This structure maps out the global data that needs to be kept on a + * per-cpu basis. genassym uses this to generate offsets for the assembler + * code, which also provides external symbols so that C can get at them as + * though they were really globals. This structure is pointed to by + * the per-cpu system value (see alpha_pal_rdval() and alpha_pal_wrval()). + * Inside the kernel, the globally reserved register t7 is used to + * point at the globaldata structure. + */ +struct globaldata { + struct alpha_pcb gd_idlepcb; /* pcb for idling */ + struct proc *gd_curproc; /* current process */ + struct proc *gd_idleproc; /* idle process */ + struct proc *gd_fpcurproc; /* fp state owner */ + struct pcb *gd_curpcb; /* current pcb */ + struct timeval gd_switchtime; + int gd_switchticks; + u_int gd_cpuno; /* this cpu number */ + u_int gd_other_cpus; /* all other cpus */ + int gd_inside_intr; + u_int64_t gd_idlepcbphys; /* pa of gd_idlepcb */ + u_int64_t gd_pending_ipis; /* pending IPI events */ + u_int32_t gd_next_asn; /* next ASN to allocate */ + u_int32_t gd_current_asngen; /* ASN rollover check */ + u_int32_t gd_intr_nesting_level; /* interrupt recursion */ + + u_int gd_astpending; + SLIST_ENTRY(globaldata) gd_allcpu; +#ifdef KTR_PERCPU + volatile int gd_ktr_idx; /* Index into trace table */ + char *gd_ktr_buf; + char gd_ktr_buf_data[0]; +#endif +}; + +SLIST_HEAD(cpuhead, globaldata); +extern struct cpuhead cpuhead; + +void globaldata_init(struct globaldata *pcpu, int cpuno, size_t sz); +struct globaldata *globaldata_find(int cpuno); + +#endif /* _KERNEL */ + +#endif /* !_MACHINE_GLOBALDATA_H_ */ diff --git a/sys/alpha/include/pmap.h b/sys/alpha/include/pmap.h index 134c9a2..de59b66 100644 --- a/sys/alpha/include/pmap.h +++ b/sys/alpha/include/pmap.h @@ -174,9 +174,11 @@ struct pmap { TAILQ_HEAD(,pv_entry) pm_pvlist; /* list of mappings in pmap */ int pm_count; /* reference count */ int pm_flags; /* pmap flags */ - int pm_active; /* active flag */ - int pm_asn; /* address space number */ - u_int pm_asngen; /* generation number of pm_asn */ + u_int32_t pm_active; /* active cpus */ + struct { + u_int32_t asn:8; /* address space number */ + u_int32_t gen:24; /* generation number */ + } pm_asn[NCPUS]; struct pmap_statistics pm_stats; /* pmap statistics */ struct vm_page *pm_ptphint; /* pmap ptp hint */ }; diff --git a/sys/alpha/include/proc.h b/sys/alpha/include/proc.h index 502b607..d003816 100644 --- a/sys/alpha/include/proc.h +++ b/sys/alpha/include/proc.h @@ -28,6 +28,12 @@ * rights to redistribute these changes. */ +#ifndef _MACHINE_PROC_H_ +#define _MACHINE_PROC_H_ + +#include +#include + /* * Machine-dependent part of the proc struct for the Alpha. */ @@ -55,3 +61,5 @@ struct mdproc { #define MDP_UAC_SIGBUS 0x0040 /* Deliver SIGBUS upon unaligned access */ #define MDP_UAC_MASK (MDP_UAC_NOPRINT | MDP_UAC_NOFIX | MDP_UAC_SIGBUS) + +#endif /* !_MACHINE_PROC_H_ */ diff --git a/sys/alpha/include/rpb.h b/sys/alpha/include/rpb.h index 1f2f884..0be0775 100644 --- a/sys/alpha/include/rpb.h +++ b/sys/alpha/include/rpb.h @@ -219,7 +219,8 @@ struct rpb { * PCS: Per-CPU information. */ struct pcs { - u_int8_t pcs_hwpcb[128]; /* 0: PAL dependent */ + + u_int64_t pcs_hwpcb[16]; /* 0: PAL dependent */ #define PCS_BIP 0x000001 /* boot in progress */ #define PCS_RC 0x000002 /* restart possible */ @@ -238,12 +239,12 @@ struct pcs { #define PCS_HALT_WARM_BOOT 0x030000 #define PCS_HALT_STAY_HALTED 0x040000 #define PCS_mbz 0xffffffffff000000 /* 24:63 -- must be zero */ - u_int64_t pcs_flags; /* 80: */ + u_int64_t pcs_flags; /* 128: */ - u_int64_t pcs_pal_memsize; /* 88: PAL memory size */ - u_int64_t pcs_pal_scrsize; /* 90: PAL scratch size */ - vm_offset_t pcs_pal_memaddr; /* 98: PAL memory addr */ - vm_offset_t pcs_pal_scraddr; /* A0: PAL scratch addr */ + u_int64_t pcs_pal_memsize; /* 136: PAL memory size */ + u_int64_t pcs_pal_scrsize; /* 144: PAL scratch size */ + vm_offset_t pcs_pal_memaddr; /* 152: PAL memory addr */ + vm_offset_t pcs_pal_scraddr; /* 160: PAL scratch addr */ struct { u_int64_t minorrev : 8, /* alphabetic char 'a' - 'z' */ @@ -261,14 +262,14 @@ struct pcs { sbz1 : 8, compatibility : 16, /* Compatibility revision */ proc_cnt : 16; /* Processor count */ - } pcs_pal_rev; /* A8: */ + } pcs_pal_rev; /* 168: */ #define pcs_minorrev pcs_pal_rev.minorrev #define pcs_majorrev pcs_pal_rev.majorrev #define pcs_pal_type pcs_pal_rev.pal_type #define pcs_compatibility pcs_pal_rev.compatibility #define pcs_proc_cnt pcs_pal_rev.proc_cnt - u_int64_t pcs_proc_type; /* B0: processor type */ + u_int64_t pcs_proc_type; /* 176: processor type */ #define PCS_PROC_MAJOR 0x00000000ffffffff #define PCS_PROC_MAJORSHIFT 0 @@ -288,23 +289,23 @@ struct pcs { /* Minor number interpretation is processor specific. See cpu.c. */ - u_int64_t pcs_proc_var; /* B8: processor variation. */ + u_int64_t pcs_proc_var; /* 184: processor variation. */ #define PCS_VAR_VAXFP 0x0000000000000001 /* VAX FP support */ #define PCS_VAR_IEEEFP 0x0000000000000002 /* IEEE FP support */ #define PCS_VAR_PE 0x0000000000000004 /* Primary Eligible */ #define PCS_VAR_RESERVED 0xfffffffffffffff8 /* Reserved */ - char pcs_proc_revision[8]; /* C0: only first 4 valid */ - char pcs_proc_sn[16]; /* C8: only first 10 valid */ - vm_offset_t pcs_machcheck; /* D8: mach chk phys addr. */ - u_int64_t pcs_machcheck_len; /* E0: length in bytes */ - vm_offset_t pcs_halt_pcbb; /* E8: phys addr of halt PCB */ - vm_offset_t pcs_halt_pc; /* F0: halt PC */ - u_int64_t pcs_halt_ps; /* F8: halt PS */ - u_int64_t pcs_halt_r25; /* 100: halt argument list */ - u_int64_t pcs_halt_r26; /* 108: halt return addr list */ - u_int64_t pcs_halt_r27; /* 110: halt procedure value */ + char pcs_proc_revision[8]; /* 192: only first 4 valid */ + char pcs_proc_sn[16]; /* 200: only first 10 valid */ + vm_offset_t pcs_machcheck; /* 216: mach chk phys addr. */ + u_int64_t pcs_machcheck_len; /* 224: length in bytes */ + vm_offset_t pcs_halt_pcbb; /* 232: pa of halt PCB */ + vm_offset_t pcs_halt_pc; /* 240: halt PC */ + u_int64_t pcs_halt_ps; /* 248: halt PS */ + u_int64_t pcs_halt_r25; /* 256: halt argument list */ + u_int64_t pcs_halt_r26; /* 264: halt ra list */ + u_int64_t pcs_halt_r27; /* 272: halt procedure value */ #define PCS_HALT_RESERVED 0 #define PCS_HALT_POWERUP 1 @@ -315,17 +316,22 @@ struct pcs { #define PCS_HALT_DOUBLE_ERROR_ABORT 6 #define PCS_HALT_SCBB 7 #define PCS_HALT_PTBR 8 /* 9-FF: reserved */ - u_int64_t pcs_halt_reason; /* 118: */ + u_int64_t pcs_halt_reason; /* 280: */ - u_int64_t pcs_reserved_soft; /* 120: preserved software */ - u_int64_t pcs_buffer[21]; /* 128: console buffers */ + u_int64_t pcs_reserved_soft; /* 288: preserved software */ + struct { + u_int32_t rxlen; + u_int32_t txlen; + char rxbuf[80]; + char txbuf[80]; + } pcs_buffer; /* 296: console buffers */ #define PALvar_reserved 0 #define PALvar_OpenVMS 1 #define PALvar_OSF1 2 - u_int64_t pcs_palrevisions[16]; /* 1D0: PALcode revisions */ + u_int64_t pcs_palrevisions[16]; /* 464: PALcode revisions */ - u_int64_t pcs_reserved_arch[6]; /* 250: reserved arch */ + u_int64_t pcs_reserved_arch[6]; /* 592: reserved arch */ }; /* diff --git a/sys/alpha/include/smp.h b/sys/alpha/include/smp.h index 48d6737..00aec6a 100644 --- a/sys/alpha/include/smp.h +++ b/sys/alpha/include/smp.h @@ -1,10 +1,57 @@ /* + * ---------------------------------------------------------------------------- + * "THE BEER-WARE LICENSE" (Revision 42): + * wrote this file. As long as you retain this notice you + * can do whatever you want with this stuff. If we meet some day, and you think + * this stuff is worth it, you can buy me a beer in return. Poul-Henning Kamp + * ---------------------------------------------------------------------------- + * * $FreeBSD$ + * */ + #ifndef _MACHINE_SMP_H_ #define _MACHINE_SMP_H_ -#define get_mplock() { } -#define rel_mplock() { } +#ifdef _KERNEL + +#include +#include +#include + +#ifndef LOCORE + +#define BETTER_CLOCK /* unconditional on alpha */ + +/* global data in mp_machdep.c */ +extern volatile u_int checkstate_probed_cpus; +extern volatile u_int checkstate_need_ast; +extern volatile u_int resched_cpus; +extern void (*cpustop_restartfunc) __P((void)); + +extern int smp_active; +extern int mp_ncpus; +extern u_int all_cpus; +extern u_int started_cpus; +extern u_int stopped_cpus; + +/* functions in mp_machdep.c */ +void mp_start(void); +void mp_announce(void); +void smp_invltlb(void); +void forward_statclock(int pscnt); +void forward_hardclock(int pscnt); +void forward_signal(struct proc *); +void forward_roundrobin(void); +int stop_cpus(u_int); +int restart_cpus(u_int); +void smp_rendezvous_action(void); +void smp_rendezvous(void (*)(void *), + void (*)(void *), + void (*)(void *), + void *arg); +void smp_init_secondary(void); -#endif +#endif /* !LOCORE */ +#endif /* _KERNEL */ +#endif /* _MACHINE_SMP_H_ */ -- cgit v1.1