From ab2321a07e3bc40ee9bfad5ecd684d8715648d9c Mon Sep 17 00:00:00 2001
From: jhb <jhb@FreeBSD.org>
Date: Thu, 7 Apr 2011 21:32:25 +0000
Subject: If a system call does not request a full interrupt return, use a fast
 path via the sysretq instruction to return from the system call.  This was
 removed in 190620 and not quite fully restored in 195486.  This resolves most
 of the performance regression in system call microbenchmarks between 7 and 8
 on amd64.

Reviewed by:	kib
MFC after:	1 week
---
 sys/amd64/amd64/exception.S | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)

(limited to 'sys/amd64/amd64')

diff --git a/sys/amd64/amd64/exception.S b/sys/amd64/amd64/exception.S
index f2339f8..220cf23 100644
--- a/sys/amd64/amd64/exception.S
+++ b/sys/amd64/amd64/exception.S
@@ -339,6 +339,9 @@ IDTVEC(prot)
  * and the new privilige level.  We are still running on the old user stack
  * pointer.  We have to juggle a few things around to find our stack etc.
  * swapgs gives us access to our PCPU space only.
+ *
+ * We do not support invoking this from a custom %cs or %ss (e.g. using
+ * entries from an LDT).
  */
 IDTVEC(fast_syscall)
 	swapgs
@@ -380,6 +383,36 @@ IDTVEC(fast_syscall)
 	movq	%rsp,%rdi
 	call	syscall
 	movq	PCPU(CURPCB),%rax
+	testq	$PCB_FULL_IRET,PCB_FLAGS(%rax)
+	jne	3f
+1:	/* Check for and handle AST's on return to userland. */
+	cli
+	movq	PCPU(CURTHREAD),%rax
+	testl	$TDF_ASTPENDING | TDF_NEEDRESCHED,TD_FLAGS(%rax)
+	je	2f
+	sti
+	movq	%rsp, %rdi
+	call	ast
+	jmp	1b
+2:	/* Restore preserved registers. */
+	MEXITCOUNT
+	movq	TF_RDI(%rsp),%rdi	/* bonus; preserve arg 1 */
+	movq	TF_RSI(%rsp),%rsi	/* bonus: preserve arg 2 */
+	movq	TF_RDX(%rsp),%rdx	/* return value 2 */
+	movq	TF_RAX(%rsp),%rax	/* return value 1 */
+	movq	TF_RBX(%rsp),%rbx	/* C preserved */
+	movq	TF_RBP(%rsp),%rbp	/* C preserved */
+	movq	TF_R12(%rsp),%r12	/* C preserved */
+	movq	TF_R13(%rsp),%r13	/* C preserved */
+	movq	TF_R14(%rsp),%r14	/* C preserved */
+	movq	TF_R15(%rsp),%r15	/* C preserved */
+	movq	TF_RFLAGS(%rsp),%r11	/* original %rflags */
+	movq	TF_RIP(%rsp),%rcx	/* original %rip */
+	movq	TF_RSP(%rsp),%r9	/* user stack pointer */
+	movq	%r9,%rsp		/* original %rsp */
+	swapgs
+	sysretq
+3:	/* Requested full context restore, use doreti for that. */
 	MEXITCOUNT
 	jmp	doreti
 
-- 
cgit v1.1