Extend struct sysvec with new method sv_schedtail, which is used for an

explicit process at fork trampoline path instead of eventhadler(schedtail)
invocation for each child process.

Remove eventhandler(schedtail) code and change linux ABI to use newly added
sysvec method.

While here replace explicit comparing of module sysentvec structure with the
newly created process sysentvec to detect the linux ABI.

Discussed with:	kib

MFC after:	2 Week

21 files changed, 28 insertions, 14 deletions

diff --git a/sys/amd64/amd64/elf_machdep.c b/sys/amd64/amd64/elf_machdep.c
index 4344731..38a11c68 100644
--- a/sys/amd64/amd64/elf_machdep.c
+++ b/sys/amd64/amd64/elf_machdep.c
@@ -81,6 +81,7 @@ struct sysentvec elf64_freebsd_sysvec = {
 	.sv_syscallnames = syscallnames,
 	.sv_shared_page_base = SHAREDPAGE,
 	.sv_shared_page_len = PAGE_SIZE,
+	.sv_schedtail	= NULL,
 };
 INIT_SYSENTVEC(elf64_sysvec, &elf64_freebsd_sysvec);
 
diff --git a/sys/amd64/linux32/linux32_sysvec.c b/sys/amd64/linux32/linux32_sysvec.c
index 712370d..12153ee 100644
--- a/sys/amd64/linux32/linux32_sysvec.c
+++ b/sys/amd64/linux32/linux32_sysvec.c
@@ -128,7 +128,6 @@ static void	linux32_fixlimit(struct rlimit *rl, int which);
 static boolean_t linux32_trans_osrel(const Elf_Note *note, int32_t *osrel);
 
 static eventhandler_tag linux_exit_tag;
-static eventhandler_tag linux_schedtail_tag;
 static eventhandler_tag linux_exec_tag;
 
 /*
@@ -1063,6 +1062,7 @@ struct sysentvec elf_linux_sysvec = {
 	.sv_set_syscall_retval = cpu_set_syscall_retval,
 	.sv_fetch_syscall_args = linux32_fetch_syscall_args,
 	.sv_syscallnames = NULL,
+	.sv_schedtail	= linux_schedtail,
 };
 
 static char GNU_ABI_VENDOR[] = "GNU";
@@ -1156,8 +1156,6 @@ linux_elf_modevent(module_t mod, int type, void *data)
 			mtx_init(&futex_mtx, "ftllk", NULL, MTX_DEF);
 			linux_exit_tag = EVENTHANDLER_REGISTER(process_exit,
 			    linux_proc_exit, NULL, 1000);
-			linux_schedtail_tag = EVENTHANDLER_REGISTER(schedtail,
-			    linux_schedtail, NULL, 1000);
 			linux_exec_tag = EVENTHANDLER_REGISTER(process_exec,
 			    linux_proc_exec, NULL, 1000);
 			linux_szplatform = roundup(strlen(linux_platform) + 1,
@@ -1189,7 +1187,6 @@ linux_elf_modevent(module_t mod, int type, void *data)
 			sx_destroy(&emul_shared_lock);
 			mtx_destroy(&futex_mtx);
 			EVENTHANDLER_DEREGISTER(process_exit, linux_exit_tag);
-			EVENTHANDLER_DEREGISTER(schedtail, linux_schedtail_tag);
 			EVENTHANDLER_DEREGISTER(process_exec, linux_exec_tag);
 			linux_osd_jail_deregister();
 			if (bootverbose)
diff --git a/sys/arm/arm/elf_machdep.c b/sys/arm/arm/elf_machdep.c
index fcc332c..5ea5615 100644
--- a/sys/arm/arm/elf_machdep.c
+++ b/sys/arm/arm/elf_machdep.c
@@ -78,6 +78,7 @@ struct sysentvec elf32_freebsd_sysvec = {
 	.sv_set_syscall_retval = cpu_set_syscall_retval,
 	.sv_fetch_syscall_args = NULL, /* XXXKIB */
 	.sv_syscallnames = syscallnames,
+	.sv_schedtail	= NULL,
 };
 
 static Elf32_Brandinfo freebsd_brand_info = {
diff --git a/sys/compat/ia32/ia32_sysvec.c b/sys/compat/ia32/ia32_sysvec.c
index 058e739..a7790ef 100644
--- a/sys/compat/ia32/ia32_sysvec.c
+++ b/sys/compat/ia32/ia32_sysvec.c
@@ -149,6 +149,7 @@ struct sysentvec ia32_freebsd_sysvec = {
 	.sv_syscallnames = freebsd32_syscallnames,
 	.sv_shared_page_base = FREEBSD32_SHAREDPAGE,
 	.sv_shared_page_len = PAGE_SIZE,
+	.sv_schedtail	= NULL,
 };
 INIT_SYSENTVEC(elf_ia32_sysvec, &ia32_freebsd_sysvec);
 
diff --git a/sys/compat/linux/linux_emul.c b/sys/compat/linux/linux_emul.c
index 69f1d08..639d403 100644
--- a/sys/compat/linux/linux_emul.c
+++ b/sys/compat/linux/linux_emul.c
@@ -300,13 +300,16 @@ linux_proc_exec(void *arg __unused, struct proc *p, struct image_params *imgp)
 }
 
 void
-linux_schedtail(void *arg __unused, struct proc *p)
+linux_schedtail(struct thread *td)
 {
 	struct linux_emuldata *em;
+	struct proc *p;
 	int error = 0;
 	int *child_set_tid;
 
-	if (__predict_true(p->p_sysent != &elf_linux_sysvec))
+	p = td->td_proc;
+
+	if (SV_PROC_ABI(p) != SV_ABI_LINUX)
 		return;
 
 	/* find the emuldata */
diff --git a/sys/compat/linux/linux_emul.h b/sys/compat/linux/linux_emul.h
index d517bfd..3acde64 100644
--- a/sys/compat/linux/linux_emul.h
+++ b/sys/compat/linux/linux_emul.h
@@ -82,7 +82,7 @@ struct linux_emuldata	*em_find(struct proc *, int locked);
 
 int	linux_proc_init(struct thread *, pid_t, int);
 void	linux_proc_exit(void *, struct proc *);
-void	linux_schedtail(void *, struct proc *);
+void	linux_schedtail(struct thread *);
 void	linux_proc_exec(void *, struct proc *, struct image_params *);
 void	linux_kill_threads(struct thread *, int);
 
diff --git a/sys/compat/svr4/svr4_sysvec.c b/sys/compat/svr4/svr4_sysvec.c
index b6ff153..561a838 100644
--- a/sys/compat/svr4/svr4_sysvec.c
+++ b/sys/compat/svr4/svr4_sysvec.c
@@ -195,6 +195,7 @@ struct sysentvec svr4_sysvec = {
 	.sv_set_syscall_retval = cpu_set_syscall_retval,
 	.sv_fetch_syscall_args = cpu_fetch_syscall_args,
 	.sv_syscallnames = NULL,
+	.sv_schedtail	= NULL,
 };
 
 const char      svr4_emul_path[] = "/compat/svr4";
diff --git a/sys/i386/i386/elf_machdep.c b/sys/i386/i386/elf_machdep.c
index df3f48f..a782445 100644
--- a/sys/i386/i386/elf_machdep.c
+++ b/sys/i386/i386/elf_machdep.c
@@ -78,6 +78,7 @@ struct sysentvec elf32_freebsd_sysvec = {
 	.sv_set_syscall_retval = cpu_set_syscall_retval,
 	.sv_fetch_syscall_args = cpu_fetch_syscall_args,
 	.sv_syscallnames = syscallnames,
+	.sv_schedtail	= NULL,
 };
 
 static Elf32_Brandinfo freebsd_brand_info = {
diff --git a/sys/i386/ibcs2/ibcs2_sysvec.c b/sys/i386/ibcs2/ibcs2_sysvec.c
index d69a244..5b6f680 100644
--- a/sys/i386/ibcs2/ibcs2_sysvec.c
+++ b/sys/i386/ibcs2/ibcs2_sysvec.c
@@ -90,6 +90,7 @@ struct sysentvec ibcs2_svr3_sysvec = {
 	.sv_set_syscall_retval = cpu_set_syscall_retval,
 	.sv_fetch_syscall_args = cpu_fetch_syscall_args,
 	.sv_syscallnames = NULL,
+	.sv_schedtail	= NULL,
 };
 
 static int
diff --git a/sys/i386/linux/linux_sysvec.c b/sys/i386/linux/linux_sysvec.c
index c0d9c23..62901d4 100644
--- a/sys/i386/linux/linux_sysvec.c
+++ b/sys/i386/linux/linux_sysvec.c
@@ -112,7 +112,6 @@ static int linux_szplatform;
 const char *linux_platform;
 
 static eventhandler_tag linux_exit_tag;
-static eventhandler_tag linux_schedtail_tag;
 static eventhandler_tag linux_exec_tag;
 
 /*
@@ -998,6 +997,7 @@ struct sysentvec linux_sysvec = {
 	.sv_set_syscall_retval = cpu_set_syscall_retval,
 	.sv_fetch_syscall_args = linux_fetch_syscall_args,
 	.sv_syscallnames = NULL,
+	.sv_schedtail	= linux_schedtail,
 };
 
 struct sysentvec elf_linux_sysvec = {
@@ -1032,6 +1032,7 @@ struct sysentvec elf_linux_sysvec = {
 	.sv_set_syscall_retval = cpu_set_syscall_retval,
 	.sv_fetch_syscall_args = linux_fetch_syscall_args,
 	.sv_syscallnames = NULL,
+	.sv_schedtail	= linux_schedtail,
 };
 
 static char GNU_ABI_VENDOR[] = "GNU";
@@ -1125,8 +1126,6 @@ linux_elf_modevent(module_t mod, int type, void *data)
 			mtx_init(&futex_mtx, "ftllk", NULL, MTX_DEF);
 			linux_exit_tag = EVENTHANDLER_REGISTER(process_exit, linux_proc_exit,
 			      NULL, 1000);
-			linux_schedtail_tag = EVENTHANDLER_REGISTER(schedtail, linux_schedtail,
-			      NULL, 1000);
 			linux_exec_tag = EVENTHANDLER_REGISTER(process_exec, linux_proc_exec,
 			      NULL, 1000);
 			linux_get_machine(&linux_platform);
@@ -1159,7 +1158,6 @@ linux_elf_modevent(module_t mod, int type, void *data)
 			sx_destroy(&emul_shared_lock);
 			mtx_destroy(&futex_mtx);
 			EVENTHANDLER_DEREGISTER(process_exit, linux_exit_tag);
-			EVENTHANDLER_DEREGISTER(schedtail, linux_schedtail_tag);
 			EVENTHANDLER_DEREGISTER(process_exec, linux_exec_tag);
 			linux_osd_jail_deregister();
 			if (bootverbose)
diff --git a/sys/ia64/ia64/elf_machdep.c b/sys/ia64/ia64/elf_machdep.c
index 87fcd4f..05cb641 100644
--- a/sys/ia64/ia64/elf_machdep.c
+++ b/sys/ia64/ia64/elf_machdep.c
@@ -85,6 +85,7 @@ struct sysentvec elf64_freebsd_sysvec = {
 	.sv_set_syscall_retval = cpu_set_syscall_retval,
 	.sv_fetch_syscall_args = cpu_fetch_syscall_args,
 	.sv_syscallnames = syscallnames,
+	.sv_schedtail	= NULL,
 };
 
 static Elf64_Brandinfo freebsd_brand_info = {
diff --git a/sys/kern/imgact_aout.c b/sys/kern/imgact_aout.c
index 6710135..a783c41 100644
--- a/sys/kern/imgact_aout.c
+++ b/sys/kern/imgact_aout.c
@@ -93,6 +93,7 @@ struct sysentvec aout_sysvec = {
 	.sv_set_syscall_retval = cpu_set_syscall_retval,
 	.sv_fetch_syscall_args = cpu_fetch_syscall_args,
 	.sv_syscallnames = syscallnames,
+	.sv_schedtail	= NULL,
 };
 
 static int
diff --git a/sys/kern/init_main.c b/sys/kern/init_main.c
index 9250e79..eef0808 100644
--- a/sys/kern/init_main.c
+++ b/sys/kern/init_main.c
@@ -378,6 +378,7 @@ struct sysentvec null_sysvec = {
 	.sv_set_syscall_retval = null_set_syscall_retval,
 	.sv_fetch_syscall_args = null_fetch_syscall_args,
 	.sv_syscallnames = NULL,
+	.sv_schedtail	= NULL,
 };
 
 /*
diff --git a/sys/kern/kern_fork.c b/sys/kern/kern_fork.c
index 483103b..a29dc17 100644
--- a/sys/kern/kern_fork.c
+++ b/sys/kern/kern_fork.c
@@ -67,6 +67,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/unistd.h>	
 #include <sys/sdt.h>
 #include <sys/sx.h>
+#include <sys/sysent.h>
 #include <sys/signalvar.h>
 
 #include <security/audit/audit.h>
@@ -895,7 +896,8 @@ fork_exit(void (*callout)(void *, struct trapframe *), void *arg,
 	}
 	mtx_assert(&Giant, MA_NOTOWNED);
 
-	EVENTHANDLER_INVOKE(schedtail, p);
+	if (p->p_sysent->sv_schedtail != NULL)
+		(p->p_sysent->sv_schedtail)(td);
 }
 
 /*
diff --git a/sys/mips/mips/elf64_machdep.c b/sys/mips/mips/elf64_machdep.c
index 41b6691..9fa31fa 100644
--- a/sys/mips/mips/elf64_machdep.c
+++ b/sys/mips/mips/elf64_machdep.c
@@ -82,6 +82,7 @@ struct sysentvec elf64_freebsd_sysvec = {
 	.sv_set_syscall_retval = cpu_set_syscall_retval,
 	.sv_fetch_syscall_args = NULL, /* XXXKIB */
 	.sv_syscallnames = NULL,
+	.sv_schedtail	= NULL,
 };
 
 static Elf64_Brandinfo freebsd_brand_gnutools_info64 = {
diff --git a/sys/mips/mips/elf_machdep.c b/sys/mips/mips/elf_machdep.c
index fc1bc15..41611e3 100644
--- a/sys/mips/mips/elf_machdep.c
+++ b/sys/mips/mips/elf_machdep.c
@@ -82,6 +82,7 @@ struct sysentvec elf64_freebsd_sysvec = {
 	.sv_set_syscall_retval = cpu_set_syscall_retval,
 	.sv_fetch_syscall_args = NULL, /* XXXKIB */
 	.sv_syscallnames = syscallnames,
+	.sv_schedtail	= NULL,
 };
 
 static Elf64_Brandinfo freebsd_brand_info = {
@@ -137,6 +138,7 @@ struct sysentvec elf32_freebsd_sysvec = {
 	.sv_set_syscall_retval = cpu_set_syscall_retval,
 	.sv_fetch_syscall_args = NULL, /* XXXKIB */
 	.sv_syscallnames = syscallnames,
+	.sv_schedtail	= NULL,
 };
 
 static Elf32_Brandinfo freebsd_brand_info = {
diff --git a/sys/powerpc/powerpc/elf32_machdep.c b/sys/powerpc/powerpc/elf32_machdep.c
index e35bd19..dbe58df 100644
--- a/sys/powerpc/powerpc/elf32_machdep.c
+++ b/sys/powerpc/powerpc/elf32_machdep.c
@@ -106,6 +106,7 @@ struct sysentvec elf32_freebsd_sysvec = {
 	.sv_fetch_syscall_args = cpu_fetch_syscall_args,
 	.sv_shared_page_base = FREEBSD32_SHAREDPAGE,
 	.sv_shared_page_len = PAGE_SIZE,
+	.sv_schedtail	= NULL,
 };
 INIT_SYSENTVEC(elf32_sysvec, &elf32_freebsd_sysvec);
 
diff --git a/sys/powerpc/powerpc/elf64_machdep.c b/sys/powerpc/powerpc/elf64_machdep.c
index af5b12d..0c41a8d 100644
--- a/sys/powerpc/powerpc/elf64_machdep.c
+++ b/sys/powerpc/powerpc/elf64_machdep.c
@@ -82,6 +82,7 @@ struct sysentvec elf64_freebsd_sysvec = {
 	.sv_syscallnames = syscallnames,
 	.sv_shared_page_base = SHAREDPAGE,
 	.sv_shared_page_len = PAGE_SIZE,
+	.sv_schedtail	= NULL,
 };
 INIT_SYSENTVEC(elf64_sysvec, &elf64_freebsd_sysvec);
 
diff --git a/sys/sparc64/sparc64/elf_machdep.c b/sys/sparc64/sparc64/elf_machdep.c
index 2e58c8c..386c0fa 100644
--- a/sys/sparc64/sparc64/elf_machdep.c
+++ b/sys/sparc64/sparc64/elf_machdep.c
@@ -93,6 +93,7 @@ static struct sysentvec elf64_freebsd_sysvec = {
 	.sv_set_syscall_retval = cpu_set_syscall_retval,
 	.sv_fetch_syscall_args = cpu_fetch_syscall_args,
 	.sv_syscallnames = syscallnames,
+	.sv_schedtail	= NULL,
 };
 
 static Elf64_Brandinfo freebsd_brand_info = {
diff --git a/sys/sys/eventhandler.h b/sys/sys/eventhandler.h
index 8fa0cea..a5e7bec 100644
--- a/sys/sys/eventhandler.h
+++ b/sys/sys/eventhandler.h
@@ -250,6 +250,4 @@ typedef void (*uma_zone_chfn)(void *);
 EVENTHANDLER_DECLARE(nmbclusters_change, uma_zone_chfn);
 EVENTHANDLER_DECLARE(maxsockets_change, uma_zone_chfn);
 
-typedef void(*schedtail_fn)(void *, struct proc *);
-EVENTHANDLER_DECLARE(schedtail, schedtail_fn);
 #endif /* SYS_EVENTHANDLER_H */
diff --git a/sys/sys/sysent.h b/sys/sys/sysent.h
index 6ba071c..90bce21 100644
--- a/sys/sys/sysent.h
+++ b/sys/sys/sysent.h
@@ -125,6 +125,7 @@ struct sysentvec {
 	vm_offset_t	sv_shared_page_len;
 	vm_offset_t	sv_sigcode_base;
 	void		*sv_shared_page_obj;
+	void		(*sv_schedtail)(struct thread *);
 };
 
 #define	SV_ILP32	0x000100