Change the syscall stub branch orders so that the static branch prediction

will assume that syscalls will succeed rather than fail.

4 files changed, 51 insertions, 12 deletions

diff --git a/lib/libc/amd64/SYS.h b/lib/libc/amd64/SYS.h
index 46aeb47..844fe6a 100644
--- a/lib/libc/amd64/SYS.h
+++ b/lib/libc/amd64/SYS.h
@@ -41,26 +41,23 @@
 #include <machine/asm.h>
 
 #ifdef PIC
-#define	SYSCALL(x)	2: movq PIC_GOT(HIDENAME(cerror)),%rcx;		\
-			jmp *%rcx;					\
-			ENTRY(__CONCAT(__sys_,x));			\
+#define	RSYSCALL(x)	ENTRY(__CONCAT(__sys_,x));			\
 			.weak CNAME(x);					\
 			.set CNAME(x),CNAME(__CONCAT(__sys_,x));	\
 			.weak CNAME(__CONCAT(_,x));			\
 			.set CNAME(__CONCAT(_,x)),CNAME(__CONCAT(__sys_,x)); \
-			mov __CONCAT($SYS_,x),%rax; KERNCALL; jb 2b
+			mov __CONCAT($SYS_,x),%rax; KERNCALL; jb 2f; ret; \
+			2: movq PIC_GOT(HIDENAME(cerror)),%rcx; jmp *%rcx
 #else
-#define	SYSCALL(x)	2: jmp HIDENAME(cerror);			\
-			ENTRY(__CONCAT(__sys_,x));			\
+#define	RSYSCALL(x)	ENTRY(__CONCAT(__sys_,x));			\
 			.weak CNAME(x);					\
 			.set CNAME(x),CNAME(__CONCAT(__sys_,x));	\
 			.weak CNAME(__CONCAT(_,x));			\
 			.set CNAME(__CONCAT(_,x)),CNAME(__CONCAT(__sys_,x)); \
-			mov __CONCAT($SYS_,x),%rax; KERNCALL; jb 2b
+			mov __CONCAT($SYS_,x),%rax; KERNCALL; jb 2f; ret; \
+			2: jmp HIDENAME(cerror)
 #endif
 
-#define	RSYSCALL(x)	SYSCALL(x); ret
-
 #define	PSEUDO(x)	ENTRY(__CONCAT(__sys_,x));			\
 			.weak CNAME(__CONCAT(_,x));			\
 			.set CNAME(__CONCAT(_,x)),CNAME(__CONCAT(__sys_,x)); \
diff --git a/lib/libc/amd64/sys/pipe.S b/lib/libc/amd64/sys/pipe.S
index 87bda41..4baf20c 100644
--- a/lib/libc/amd64/sys/pipe.S
+++ b/lib/libc/amd64/sys/pipe.S
@@ -42,8 +42,22 @@ __FBSDID("$FreeBSD$");
 
 #include "SYS.h"
 
-SYSCALL(pipe)
+	.weak	_pipe
+	.set	_pipe,__sys_pipe
+	.weak	pipe
+	.set	pipe,__sys_pipe
+ENTRY(__sys_pipe)
+	mov	$SYS_pipe,%rax
+	KERNCALL
+	jb	1f
 	movl	%eax,(%rdi)	/* %rdi is preserved by syscall */
 	movl	%edx,4(%rdi)
 	movq	$0,%rax
 	ret
+1:
+#ifdef PIC
+	movq	PIC_GOT(HIDENAME(cerror)),%rdx
+	jmp	*%rdx
+#else
+	jmp	HIDENAME(cerror)
+#endif
diff --git a/lib/libc/amd64/sys/reboot.S b/lib/libc/amd64/sys/reboot.S
index 450a65e..0ed4329 100644
--- a/lib/libc/amd64/sys/reboot.S
+++ b/lib/libc/amd64/sys/reboot.S
@@ -42,5 +42,19 @@ __FBSDID("$FreeBSD$");
 
 #include "SYS.h"
 
-SYSCALL(reboot)
+	.weak	_reboot
+	.set	_reboot,__sys_reboot
+	.weak	reboot
+	.set	reboot,__sys_reboot
+ENTRY(__sys_reboot)
+	mov	$SYS_reboot,%rax
+	KERNCALL
+	jb	1f
 	iretq
+1:
+#ifdef PIC
+	movq	PIC_GOT(HIDENAME(cerror)),%rdx
+	jmp	*%rdx
+#else
+	jmp	HIDENAME(cerror)
+#endif
diff --git a/lib/libc/amd64/sys/setlogin.S b/lib/libc/amd64/sys/setlogin.S
index cb355d2..439e417 100644
--- a/lib/libc/amd64/sys/setlogin.S
+++ b/lib/libc/amd64/sys/setlogin.S
@@ -44,7 +44,14 @@ __FBSDID("$FreeBSD$");
 
 .globl	CNAME(_logname_valid)		/* in _getlogin() */
 
-SYSCALL(setlogin)
+	.weak	_setlogin
+	.set	_setlogin,__sys_setlogin
+	.weak	setlogin
+	.set	setlogin,__sys_setlogin
+ENTRY(__sys_setlogin)
+	mov	$SYS_setlogin,%rax
+	KERNCALL
+	jb	1f
 #ifdef PIC
 	movq	PIC_GOT(CNAME(_logname_valid)),%rdx
 	movl	$0,(%rdx)
@@ -52,3 +59,10 @@ SYSCALL(setlogin)
 	movl	$0,CNAME(_logname_valid)(%rip)
 #endif
 	ret				/* setlogin(name) */
+1:
+#ifdef PIC
+	movq	PIC_GOT(HIDENAME(cerror)),%rdx
+	jmp	*%rdx
+#else
+	jmp	HIDENAME(cerror)
+#endif