MFC r289563,r291481,r292537,r292538,r292542,r292543,r292544 and r292834:

Update the LinuxKPI:
- Add more functions and types.
- Implement ACCESS_ONCE(), WRITE_ONCE() and READ_ONCE().
- Implement sleepable RCU mechanism using shared exclusive locks.
- Minor workqueue cleanup:
  - Make some functions global instead of inline to ease debugging.
  - Fix some minor style issues.
- In the zero delay case in queue_delayed_work() use the return value
  from taskqueue_enqueue() instead of reading "ta_pending" unlocked and
  also ensure the callout is stopped before proceeding.
- Implement drain_workqueue() function.
- Reduce memory consumption when allocating kobject strings in the
  LinuxKPI. Compute string length before allocating memory instead of
  using fixed size allocations. Make kobject_set_name_vargs() global
  instead of inline to save some bytes when compiling.

Sponsored by:	Mellanox Technologies

7 files changed, 260 insertions, 93 deletions

diff --git a/sys/ofed/include/linux/compiler.h b/sys/ofed/include/linux/compiler.h
index 9b1a5ad..c7a3d28 100644
--- a/sys/ofed/include/linux/compiler.h
+++ b/sys/ofed/include/linux/compiler.h
@@ -2,7 +2,8 @@
  * Copyright (c) 2010 Isilon Systems, Inc.
  * Copyright (c) 2010 iX Systems, Inc.
  * Copyright (c) 2010 Panasas, Inc.
- * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd.
+ * Copyright (c) 2013-2015 Mellanox Technologies, Ltd.
+ * Copyright (c) 2015 François Tigeot
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -62,5 +63,28 @@
 #define typeof(x)			__typeof(x)
 
 #define	uninitialized_var(x)		x = x
+#define	__read_mostly __attribute__((__section__(".data.read_mostly")))
+#define	__always_unused			__unused
+#define	__must_check			__result_use_check
+
+#define	__printf(a,b)			__printflike(a,b)
+
+#define	barrier()			__asm__ __volatile__("": : :"memory")
+
+#define	ACCESS_ONCE(x)			(*(volatile __typeof(x) *)&(x))
+  
+#define	WRITE_ONCE(x,v) do {		\
+	barrier();			\
+	ACCESS_ONCE(x) = (v);		\
+	barrier();			\
+} while (0)
+
+#define	READ_ONCE(x) ({			\
+	__typeof(x) __var;		\
+	barrier();			\
+	__var = ACCESS_ONCE(x);		\
+	barrier();			\
+	__var;				\
+})
 
 #endif	/* _LINUX_COMPILER_H_ */
diff --git a/sys/ofed/include/linux/file.h b/sys/ofed/include/linux/file.h
index b76a408..1f72822 100644
--- a/sys/ofed/include/linux/file.h
+++ b/sys/ofed/include/linux/file.h
@@ -2,7 +2,7 @@
  * Copyright (c) 2010 Isilon Systems, Inc.
  * Copyright (c) 2010 iX Systems, Inc.
  * Copyright (c) 2010 Panasas, Inc.
- * Copyright (c) 2013 Mellanox Technologies, Ltd.
+ * Copyright (c) 2013-2015 Mellanox Technologies, Ltd.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -119,6 +119,21 @@ get_unused_fd(void)
 	return fd;
 }
 
+static inline int
+get_unused_fd_flags(int flags)
+{
+	struct file *file;
+	int error;
+	int fd;
+
+	error = falloc(curthread, &file, &fd, flags);
+	if (error)
+		return -error;
+	/* drop the extra reference */
+	fdrop(file, curthread);
+	return fd;
+}
+
 static inline struct linux_file *
 alloc_file(int mode, const struct file_operations *fops)
 {
diff --git a/sys/ofed/include/linux/kobject.h b/sys/ofed/include/linux/kobject.h
index 159f071..437d32f 100644
--- a/sys/ofed/include/linux/kobject.h
+++ b/sys/ofed/include/linux/kobject.h
@@ -87,29 +87,7 @@ kobject_get(struct kobject *kobj)
 	return kobj;
 }
 
-static inline int
-kobject_set_name_vargs(struct kobject *kobj, const char *fmt, va_list args)
-{
-	char *old;
-	char *name;
-
-	old = kobj->name;
-
-	if (old && !fmt)
-		return 0;
-
-	name = kzalloc(MAXPATHLEN, GFP_KERNEL);
-	if (!name)
-		return -ENOMEM;
-	vsnprintf(name, MAXPATHLEN, fmt, args);
-	kobj->name = name;
-	kfree(old);
-	for (; *name != '\0'; name++)
-		if (*name == '/')
-			*name = '!';
-	return (0);
-}
-
+int	kobject_set_name_vargs(struct kobject *kobj, const char *fmt, va_list);
 int	kobject_add(struct kobject *kobj, struct kobject *parent,
 	    const char *fmt, ...);
 
diff --git a/sys/ofed/include/linux/linux_compat.c b/sys/ofed/include/linux/linux_compat.c
index e9cc0c6..ef44c6d 100644
--- a/sys/ofed/include/linux/linux_compat.c
+++ b/sys/ofed/include/linux/linux_compat.c
@@ -64,6 +64,8 @@
 
 #include <vm/vm_pager.h>
 
+#include <linux/workqueue.h>
+
 MALLOC_DEFINE(M_KMALLOC, "linux", "Linux kmalloc compat");
 
 #include <linux/rbtree.h>
@@ -90,7 +92,50 @@ panic_cmp(struct rb_node *one, struct rb_node *two)
 }
 
 RB_GENERATE(linux_root, rb_node, __entry, panic_cmp);
- 
+
+int
+kobject_set_name_vargs(struct kobject *kobj, const char *fmt, va_list args)
+{
+	va_list tmp_va;
+	int len;
+	char *old;
+	char *name;
+	char dummy;
+
+	old = kobj->name;
+
+	if (old && fmt == NULL)
+		return (0);
+
+	/* compute length of string */
+	va_copy(tmp_va, args);
+	len = vsnprintf(&dummy, 0, fmt, tmp_va);
+	va_end(tmp_va);
+
+	/* account for zero termination */
+	len++;
+
+	/* check for error */
+	if (len < 1)
+		return (-EINVAL);
+
+	/* allocate memory for string */
+	name = kzalloc(len, GFP_KERNEL);
+	if (name == NULL)
+		return (-ENOMEM);
+	vsnprintf(name, len, fmt, args);
+	kobj->name = name;
+
+	/* free old string */
+	kfree(old);
+
+	/* filter new string */
+	for (; *name != '\0'; name++)
+		if (*name == '/')
+			*name = '!';
+	return (0);
+}
+
 int
 kobject_set_name(struct kobject *kobj, const char *fmt, ...)
 {
@@ -882,6 +927,50 @@ linux_completion_done(struct completion *c)
 	return (isdone);
 }
 
+void
+linux_delayed_work_fn(void *arg)
+{
+	struct delayed_work *work;
+
+	work = arg;
+	taskqueue_enqueue(work->work.taskqueue, &work->work.work_task);
+}
+
+void
+linux_work_fn(void *context, int pending)
+{
+	struct work_struct *work;
+
+	work = context;
+	work->fn(work);
+}
+
+void
+linux_flush_fn(void *context, int pending)
+{
+}
+
+struct workqueue_struct *
+linux_create_workqueue_common(const char *name, int cpus)
+{
+	struct workqueue_struct *wq;
+
+	wq = kmalloc(sizeof(*wq), M_WAITOK);
+	wq->taskqueue = taskqueue_create(name, M_WAITOK,
+	    taskqueue_thread_enqueue,  &wq->taskqueue);
+	atomic_set(&wq->draining, 0);
+	taskqueue_start_threads(&wq->taskqueue, cpus, PWAIT, "%s", name);
+
+	return (wq);
+}
+
+void
+destroy_workqueue(struct workqueue_struct *wq)
+{
+	taskqueue_free(wq->taskqueue);
+	kfree(wq);
+}
+
 static void
 linux_compat_init(void *arg)
 {
diff --git a/sys/ofed/include/linux/srcu.h b/sys/ofed/include/linux/srcu.h
new file mode 100644
index 0000000..c20215b
--- /dev/null
+++ b/sys/ofed/include/linux/srcu.h
@@ -0,0 +1,72 @@
+/*-
+ * Copyright (c) 2015 Mellanox Technologies, Ltd.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice unmodified, this list of conditions, and the following
+ *    disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+#ifndef	_LINUX_SRCU_H_
+#define	_LINUX_SRCU_H_
+
+#include <sys/param.h>
+#include <sys/lock.h>
+#include <sys/sx.h>
+
+struct srcu_struct {
+	struct sx sx;
+};
+
+static inline int
+init_srcu_struct(struct srcu_struct *srcu)
+{
+	sx_init(&srcu->sx, "SleepableRCU");
+	return (0);
+}
+
+static inline void
+cleanup_srcu_struct(struct srcu_struct *srcu)
+{
+	sx_destroy(&srcu->sx);
+}
+
+static inline int
+srcu_read_lock(struct srcu_struct *srcu)
+{
+	sx_slock(&srcu->sx);
+	return (0);
+}
+
+static inline void
+srcu_read_unlock(struct srcu_struct *srcu, int key)
+{
+	sx_sunlock(&srcu->sx);
+}
+
+static inline void
+synchronize_srcu(struct srcu_struct *srcu)
+{
+	sx_xlock(&srcu->sx);
+	sx_xunlock(&srcu->sx);
+}
+
+#endif					/* _LINUX_SRCU_H_ */
diff --git a/sys/ofed/include/linux/types.h b/sys/ofed/include/linux/types.h
index 9fff0ec..b5def55 100644
--- a/sys/ofed/include/linux/types.h
+++ b/sys/ofed/include/linux/types.h
@@ -36,8 +36,6 @@
 #include <linux/compiler.h>
 #include <asm/types.h>
 
-#define	__read_mostly __attribute__((__section__(".data.read_mostly")))
-
 #ifndef __bitwise__
 #ifdef __CHECKER__
 #define __bitwise__ __attribute__((bitwise))
diff --git a/sys/ofed/include/linux/workqueue.h b/sys/ofed/include/linux/workqueue.h
index 38cd2fe..41ca80f 100644
--- a/sys/ofed/include/linux/workqueue.h
+++ b/sys/ofed/include/linux/workqueue.h
@@ -2,7 +2,7 @@
  * Copyright (c) 2010 Isilon Systems, Inc.
  * Copyright (c) 2010 iX Systems, Inc.
  * Copyright (c) 2010 Panasas, Inc.
- * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd.
+ * Copyright (c) 2013-2015 Mellanox Technologies, Ltd.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -34,10 +34,13 @@
 #include <linux/timer.h>
 #include <linux/slab.h>
 
+#include <asm/atomic.h>
+
 #include <sys/taskqueue.h>
 
 struct workqueue_struct {
 	struct taskqueue	*taskqueue;
+	atomic_t		draining;
 };
 
 struct work_struct {
@@ -46,11 +49,19 @@ struct work_struct {
 	void			(*fn)(struct work_struct *);
 };
 
+typedef __typeof(((struct work_struct *)0)->fn) work_func_t;
+
 struct delayed_work {
 	struct work_struct	work;
 	struct callout		timer;
 };
 
+extern void linux_work_fn(void *, int);
+extern void linux_flush_fn(void *, int);
+extern void linux_delayed_work_fn(void *);
+extern struct workqueue_struct *linux_create_workqueue_common(const char *, int);
+extern void destroy_workqueue(struct workqueue_struct *);
+
 static inline struct delayed_work *
 to_delayed_work(struct work_struct *work)
 {
@@ -58,21 +69,11 @@ to_delayed_work(struct work_struct *work)
  	return container_of(work, struct delayed_work, work);
 }
 
-
-static inline void
-_work_fn(void *context, int pending)
-{
-	struct work_struct *work;
-
-	work = context;
-	work->fn(work);
-}
-
 #define	INIT_WORK(work, func) 	 					\
 do {									\
 	(work)->fn = (func);						\
 	(work)->taskqueue = NULL;					\
-	TASK_INIT(&(work)->work_task, 0, _work_fn, (work));		\
+	TASK_INIT(&(work)->work_task, 0, linux_work_fn, (work));		\
 } while (0)
 
 #define	INIT_DELAYED_WORK(_work, func)					\
@@ -81,7 +82,7 @@ do {									\
 	callout_init(&(_work)->timer, CALLOUT_MPSAFE);			\
 } while (0)
 
-#define	INIT_DEFERRABLE_WORK	INIT_DELAYED_WORK
+#define	INIT_DEFERRABLE_WORK(...) INIT_DELAYED_WORK(__VA_ARGS__)
 
 #define	schedule_work(work)						\
 do {									\
@@ -91,20 +92,15 @@ do {									\
 
 #define	flush_scheduled_work()	flush_taskqueue(taskqueue_thread)
 
-static inline int queue_work (struct workqueue_struct *q, struct work_struct *work)
+static inline int
+queue_work(struct workqueue_struct *wq, struct work_struct *work)
 {
-	(work)->taskqueue = (q)->taskqueue;
-	/* Return opposite val to align with Linux logic */
-        return !taskqueue_enqueue((q)->taskqueue, &(work)->work_task);
-}
-
-static inline void
-_delayed_work_fn(void *arg)
-{
-	struct delayed_work *work;
-
-	work = arg;
-	taskqueue_enqueue(work->work.taskqueue, &work->work.work_task);
+	work->taskqueue = wq->taskqueue;
+	/* Check for draining */
+	if (atomic_read(&wq->draining) != 0)
+		return (!work->work_task.ta_pending);
+	/* Return opposite value to align with Linux logic */
+	return (!taskqueue_enqueue(wq->taskqueue, &work->work_task));
 }
 
 static inline int
@@ -113,57 +109,44 @@ queue_delayed_work(struct workqueue_struct *wq, struct delayed_work *work,
 {
 	int pending;
 
-	pending = work->work.work_task.ta_pending;
 	work->work.taskqueue = wq->taskqueue;
-	if (delay != 0)
-		callout_reset(&work->timer, delay, _delayed_work_fn, work);
-	else
-		_delayed_work_fn((void *)work);
-
+	if (atomic_read(&wq->draining) != 0) {
+	  	pending = work->work.work_task.ta_pending;
+	} else if (delay != 0) {
+		pending = work->work.work_task.ta_pending;
+		callout_reset(&work->timer, delay, linux_delayed_work_fn, work);
+	} else {
+		callout_stop(&work->timer);
+		pending = taskqueue_enqueue(work->work.taskqueue,
+		    &work->work.work_task);
+	}
 	return (!pending);
 }
 
-static inline bool schedule_delayed_work(struct delayed_work *dwork,
-                                         unsigned long delay)
-{
-        struct workqueue_struct wq;
-        wq.taskqueue = taskqueue_thread;
-        return queue_delayed_work(&wq, dwork, delay);
-}
-
-static inline struct workqueue_struct *
-_create_workqueue_common(char *name, int cpus)
+static inline bool
+schedule_delayed_work(struct delayed_work *dwork,
+    unsigned long delay)
 {
-	struct workqueue_struct *wq;
+	struct workqueue_struct wq;
 
-	wq = kmalloc(sizeof(*wq), M_WAITOK);
-	wq->taskqueue = taskqueue_create((name), M_WAITOK,
-	    taskqueue_thread_enqueue,  &wq->taskqueue);
-	taskqueue_start_threads(&wq->taskqueue, cpus, PWAIT, "%s", name);
-
-	return (wq);
+	wq.taskqueue = taskqueue_thread;
+	atomic_set(&wq.draining, 0);
+	return (queue_delayed_work(&wq, dwork, delay));
 }
 
-
 #define	create_singlethread_workqueue(name)				\
-	_create_workqueue_common(name, 1)
+	linux_create_workqueue_common(name, 1)
 
 #define	create_workqueue(name)						\
-	_create_workqueue_common(name, MAXCPU)
+	linux_create_workqueue_common(name, MAXCPU)
 
-static inline void
-destroy_workqueue(struct workqueue_struct *wq)
-{
-	taskqueue_free(wq->taskqueue);
-	kfree(wq);
-}
+#define	alloc_ordered_workqueue(name, flags)				\
+	linux_create_workqueue_common(name, 1)
 
-#define	flush_workqueue(wq)	flush_taskqueue((wq)->taskqueue)
+#define	alloc_workqueue(name, flags, max_active)			\
+	linux_create_workqueue_common(name, max_active)
 
-static inline void
-_flush_fn(void *context, int pending)
-{
-}
+#define	flush_workqueue(wq)	flush_taskqueue((wq)->taskqueue)
 
 static inline void
 flush_taskqueue(struct taskqueue *tq)
@@ -171,12 +154,20 @@ flush_taskqueue(struct taskqueue *tq)
 	struct task flushtask;
 
 	PHOLD(curproc);
-	TASK_INIT(&flushtask, 0, _flush_fn, NULL);
+	TASK_INIT(&flushtask, 0, linux_flush_fn, NULL);
 	taskqueue_enqueue(tq, &flushtask);
 	taskqueue_drain(tq, &flushtask);
 	PRELE(curproc);
 }
 
+static inline void
+drain_workqueue(struct workqueue_struct *wq)
+{
+	atomic_inc(&wq->draining);
+	flush_taskqueue(wq->taskqueue);
+	atomic_dec(&wq->draining);
+}
+
 static inline int
 cancel_work_sync(struct work_struct *work)
 {
@@ -213,7 +204,7 @@ cancel_delayed_work_sync(struct delayed_work *work)
 
 static inline bool
 mod_delayed_work(struct workqueue_struct *wq, struct delayed_work *dwork,
-		                      unsigned long delay)
+    unsigned long delay)
 {
 	cancel_delayed_work(dwork);
 	queue_delayed_work(wq, dwork, delay);