Perform amd64-specific microoptimizations for native syscall entry

sequence. The effect is ~1% on the microbenchmark.

In particular, do not restore registers which are preserved by the
C calling sequence. Align the jump target. Avoid unneeded memory
accesses by calculating some data in syscall entry trampoline.

Reviewed by:	jhb
Approved by:	re (bz)
MFC after:	2 weeks

3 files changed, 13 insertions, 19 deletions

diff --git a/sys/amd64/amd64/exception.S b/sys/amd64/amd64/exception.S
index c537a4f..b5b84ed 100644
--- a/sys/amd64/amd64/exception.S
+++ b/sys/amd64/amd64/exception.S
@@ -380,8 +380,11 @@ IDTVEC(fast_syscall)
 	movl	$TF_HASSEGS,TF_FLAGS(%rsp)
 	cld
 	FAKE_MCOUNT(TF_RIP(%rsp))
-	movq	%rsp,%rdi
-	call	syscall
+	movq	PCPU(CURTHREAD),%rdi
+	movq	%rsp,TD_FRAME(%rdi)
+	movl	TF_RFLAGS(%rsp),%esi
+	andl	$PSL_T,%esi
+	call	amd64_syscall
 1:	movq	PCPU(CURPCB),%rax
 	/* Disable interrupts before testing PCB_FULL_IRET. */
 	cli
@@ -396,17 +399,12 @@ IDTVEC(fast_syscall)
 	call	ast
 	jmp	1b
 2:	/* Restore preserved registers. */
+	.align	16
 	MEXITCOUNT
 	movq	TF_RDI(%rsp),%rdi	/* bonus; preserve arg 1 */
 	movq	TF_RSI(%rsp),%rsi	/* bonus: preserve arg 2 */
 	movq	TF_RDX(%rsp),%rdx	/* return value 2 */
 	movq	TF_RAX(%rsp),%rax	/* return value 1 */
-	movq	TF_RBX(%rsp),%rbx	/* C preserved */
-	movq	TF_RBP(%rsp),%rbp	/* C preserved */
-	movq	TF_R12(%rsp),%r12	/* C preserved */
-	movq	TF_R13(%rsp),%r13	/* C preserved */
-	movq	TF_R14(%rsp),%r14	/* C preserved */
-	movq	TF_R15(%rsp),%r15	/* C preserved */
 	movq	TF_RFLAGS(%rsp),%r11	/* original %rflags */
 	movq	TF_RIP(%rsp),%rcx	/* original %rip */
 	movq	TF_RSP(%rsp),%r9	/* user stack pointer */
diff --git a/sys/amd64/amd64/genassym.c b/sys/amd64/amd64/genassym.c
index 1c9abd5..d133223 100644
--- a/sys/amd64/amd64/genassym.c
+++ b/sys/amd64/amd64/genassym.c
@@ -87,6 +87,7 @@ ASSYM(TD_PCB, offsetof(struct thread, td_pcb));
 ASSYM(TD_PFLAGS, offsetof(struct thread, td_pflags));
 ASSYM(TD_PROC, offsetof(struct thread, td_proc));
 ASSYM(TD_TID, offsetof(struct thread, td_tid));
+ASSYM(TD_FRAME, offsetof(struct thread, td_frame));
 
 ASSYM(TDF_ASTPENDING, TDF_ASTPENDING);
 ASSYM(TDF_NEEDRESCHED, TDF_NEEDRESCHED);
diff --git a/sys/amd64/amd64/trap.c b/sys/amd64/amd64/trap.c
index a36a6be..bd75209 100644
--- a/sys/amd64/amd64/trap.c
+++ b/sys/amd64/amd64/trap.c
@@ -885,41 +885,36 @@ cpu_fetch_syscall_args(struct thread *td, struct syscall_args *sa)
 
 #include "../../kern/subr_syscall.c"
 
+void amd64_syscall(struct thread *td, int traced);
 /*
  *	syscall -	system call request C handler
  *
  *	A system call is essentially treated as a trap.
  */
 void
-syscall(struct trapframe *frame)
+amd64_syscall(struct thread *td, int traced)
 {
-	struct thread *td;
 	struct syscall_args sa;
-	register_t orig_tf_rflags;
 	int error;
 	ksiginfo_t ksi;
 
 #ifdef DIAGNOSTIC
-	if (ISPL(frame->tf_cs) != SEL_UPL) {
+	if (ISPL(td->td_frame->tf_cs) != SEL_UPL) {
 		panic("syscall");
 		/* NOT REACHED */
 	}
 #endif
-	orig_tf_rflags = frame->tf_rflags;
-	td = curthread;
-	td->td_frame = frame;
-
 	error = syscallenter(td, &sa);
 
 	/*
 	 * Traced syscall.
 	 */
-	if (orig_tf_rflags & PSL_T) {
-		frame->tf_rflags &= ~PSL_T;
+	if (__predict_false(traced)) {
+		td->td_frame->tf_rflags &= ~PSL_T;
 		ksiginfo_init_trap(&ksi);
 		ksi.ksi_signo = SIGTRAP;
 		ksi.ksi_code = TRAP_TRACE;
-		ksi.ksi_addr = (void *)frame->tf_rip;
+		ksi.ksi_addr = (void *)td->td_frame->tf_rip;
 		trapsignal(td, &ksi);
 	}