diff options
| author | jhb <jhb@FreeBSD.org> | 2011-04-07 21:32:25 +0000 |
|---|---|---|
| committer | jhb <jhb@FreeBSD.org> | 2011-04-07 21:32:25 +0000 |
| commit | ab2321a07e3bc40ee9bfad5ecd684d8715648d9c (patch) | |
| tree | 9b6f8bbe6756f8f3b066dea53d3d9d597035bea0 /sys/amd64/amd64/exception.S | |
| parent | 4808c79fbecb248236d730dd83919073dff1d727 (diff) | |
| download | FreeBSD-src-ab2321a07e3bc40ee9bfad5ecd684d8715648d9c.zip FreeBSD-src-ab2321a07e3bc40ee9bfad5ecd684d8715648d9c.tar.gz | |
If a system call does not request a full interrupt return, use a fast
path via the sysretq instruction to return from the system call. This was
removed in 190620 and not quite fully restored in 195486. This resolves
most of the performance regression in system call microbenchmarks between
7 and 8 on amd64.
Reviewed by: kib
MFC after: 1 week
Diffstat (limited to 'sys/amd64/amd64/exception.S')
| -rw-r--r-- | sys/amd64/amd64/exception.S | 33 |
1 files changed, 33 insertions, 0 deletions
diff --git a/sys/amd64/amd64/exception.S b/sys/amd64/amd64/exception.S index f2339f8..220cf23 100644 --- a/sys/amd64/amd64/exception.S +++ b/sys/amd64/amd64/exception.S @@ -339,6 +339,9 @@ IDTVEC(prot) * and the new privilige level. We are still running on the old user stack * pointer. We have to juggle a few things around to find our stack etc. * swapgs gives us access to our PCPU space only. + * + * We do not support invoking this from a custom %cs or %ss (e.g. using + * entries from an LDT). */ IDTVEC(fast_syscall) swapgs @@ -380,6 +383,36 @@ IDTVEC(fast_syscall) movq %rsp,%rdi call syscall movq PCPU(CURPCB),%rax + testq $PCB_FULL_IRET,PCB_FLAGS(%rax) + jne 3f +1: /* Check for and handle AST's on return to userland. */ + cli + movq PCPU(CURTHREAD),%rax + testl $TDF_ASTPENDING | TDF_NEEDRESCHED,TD_FLAGS(%rax) + je 2f + sti + movq %rsp, %rdi + call ast + jmp 1b +2: /* Restore preserved registers. */ + MEXITCOUNT + movq TF_RDI(%rsp),%rdi /* bonus; preserve arg 1 */ + movq TF_RSI(%rsp),%rsi /* bonus: preserve arg 2 */ + movq TF_RDX(%rsp),%rdx /* return value 2 */ + movq TF_RAX(%rsp),%rax /* return value 1 */ + movq TF_RBX(%rsp),%rbx /* C preserved */ + movq TF_RBP(%rsp),%rbp /* C preserved */ + movq TF_R12(%rsp),%r12 /* C preserved */ + movq TF_R13(%rsp),%r13 /* C preserved */ + movq TF_R14(%rsp),%r14 /* C preserved */ + movq TF_R15(%rsp),%r15 /* C preserved */ + movq TF_RFLAGS(%rsp),%r11 /* original %rflags */ + movq TF_RIP(%rsp),%rcx /* original %rip */ + movq TF_RSP(%rsp),%r9 /* user stack pointer */ + movq %r9,%rsp /* original %rsp */ + swapgs + sysretq +3: /* Requested full context restore, use doreti for that. */ MEXITCOUNT jmp doreti |
