Merge remote-tracking branch 'origin/stable/10' into devel

132 files changed, 31376 insertions, 1873 deletions

diff --git a/contrib/netbsd-tests/kernel/t_lockf.c b/contrib/netbsd-tests/kernel/t_lockf.c
index d937301..702e612 100644
--- a/contrib/netbsd-tests/kernel/t_lockf.c
+++ b/contrib/netbsd-tests/kernel/t_lockf.c
@@ -102,6 +102,9 @@ trylocks(int id)
 		(void)fcntl(fd, F_SETLKW, &fl);
 
 		if (usleep(sleeptime) < 0) 
+#if defined(__FreeBSD__)
+		  if (errno != EINTR)
+#endif
 		  err(1, "usleep");
 	}
 	printf("%d: done\n", id);
diff --git a/contrib/netbsd-tests/lib/libc/regex/debug.c b/contrib/netbsd-tests/lib/libc/regex/debug.c
index fd2e467..3fc6d5b 100644
--- a/contrib/netbsd-tests/lib/libc/regex/debug.c
+++ b/contrib/netbsd-tests/lib/libc/regex/debug.c
@@ -45,16 +45,18 @@
 
 #include "test_regex.h"
 
+#ifdef __NetBSD__
 static void s_print(struct re_guts *, FILE *);
 static char *regchar(int);
+#endif
 
+#ifdef __NetBSD__
 /*
  * regprint - print a regexp for debugging
  */
 void
 regprint(regex_t *r, FILE *d)
 {
-#ifdef __NetBSD__
 	struct re_guts *g = r->re_g;
 	int c;
 	int last;
@@ -116,7 +118,6 @@ regprint(regex_t *r, FILE *d)
 				}
 			fprintf(d, "\n");
 		}
-#endif
 }
 
 /*
@@ -126,15 +127,11 @@ static void
 s_print(struct re_guts *g, FILE *d)
 {
 	sop *s;
-#ifdef __NetBSD__
 	cset *cs;
-#endif
 	int done = 0;
 	sop opnd;
 	int col = 0;
-#ifdef __NetBSD__
 	ssize_t last;
-#endif
 	sopno offset = 2;
 #	define	GAP()	{	if (offset % 5 == 0) { \
 					if (col > 40) { \
@@ -181,7 +178,6 @@ s_print(struct re_guts *g, FILE *d)
 			break;
 		case OANYOF:
 			fprintf(d, "[(%ld)", (long)opnd);
-#ifdef __NetBSD__
 			cs = &g->sets[opnd];
 			last = -1;
 			for (size_t i = 0; i < g->csetsize+1; i++)	/* +1 flushes */
@@ -198,7 +194,6 @@ s_print(struct re_guts *g, FILE *d)
 						last = -1;
 					}
 				}
-#endif
 			fprintf(d, "]");
 			break;
 		case OBACK_:
@@ -254,11 +249,7 @@ s_print(struct re_guts *g, FILE *d)
 			fprintf(d, ">");
 			break;
 		default:
-#ifdef __FreeBSD__
-			fprintf(d, "!%ld(%ld)!", OP(*s), opnd);
-#else
 			fprintf(d, "!%d(%d)!", OP(*s), opnd);
-#endif
 			break;
 		}
 		if (!done)
@@ -280,3 +271,10 @@ regchar(int ch)
 		sprintf(buf, "\\%o", ch);
 	return(buf);
 }
+#else
+void
+regprint(regex_t *r, FILE *d)
+{
+
+}
+#endif
diff --git a/contrib/netbsd-tests/lib/libc/rpc/t_rpc.c b/contrib/netbsd-tests/lib/libc/rpc/t_rpc.c
index 67bc8c7..a9715cf 100644
--- a/contrib/netbsd-tests/lib/libc/rpc/t_rpc.c
+++ b/contrib/netbsd-tests/lib/libc/rpc/t_rpc.c
@@ -73,8 +73,14 @@ onehost(const char *host, const char *transp)
 
 	tv.tv_sec = 1;
 	tv.tv_usec = 0;
+#ifdef __FreeBSD__
+	if (clnt_call(clnt, RPCBPROC_NULL, (xdrproc_t)xdr_void, NULL,
+	    (xdrproc_t)xdr_void, NULL, tv)
+	    != RPC_SUCCESS)
+#else
 	if (clnt_call(clnt, RPCBPROC_NULL, xdr_void, NULL, xdr_void, NULL, tv)
 	    != RPC_SUCCESS)
+#endif
 		ERRX(EXIT_FAILURE, "clnt_call (%s)", clnt_sperror(clnt, ""));
 	clnt_control(clnt, CLGET_SVC_ADDR, (char *) &addr);
 	reply(NULL, &addr, NULL);
diff --git a/contrib/netbsd-tests/lib/libc/stdio/t_fopen.c b/contrib/netbsd-tests/lib/libc/stdio/t_fopen.c
index d9e2dd6..4083b80 100644
--- a/contrib/netbsd-tests/lib/libc/stdio/t_fopen.c
+++ b/contrib/netbsd-tests/lib/libc/stdio/t_fopen.c
@@ -303,6 +303,7 @@ ATF_TC_BODY(fopen_perm, tc)
 	ATF_REQUIRE_ERRNO(EACCES, fopen("/bin/ls", "w+") == NULL);
 }
 
+#ifdef __NetBSD__
 ATF_TC(fopen_regular);
 ATF_TC_HEAD(fopen_regular, tc)
 {
@@ -335,6 +336,7 @@ ATF_TC_BODY(fopen_regular, tc)
 		}
 	}
 }
+#endif
 
 ATF_TC_WITH_CLEANUP(fopen_seek);
 ATF_TC_HEAD(fopen_seek, tc)
diff --git a/contrib/netbsd-tests/lib/libcrypt/t_crypt.c b/contrib/netbsd-tests/lib/libcrypt/t_crypt.c
index 1a192cb..a5a4dcb 100644
--- a/contrib/netbsd-tests/lib/libcrypt/t_crypt.c
+++ b/contrib/netbsd-tests/lib/libcrypt/t_crypt.c
@@ -124,6 +124,10 @@ ATF_TC_HEAD(crypt_salts, tc)
 ATF_TC_BODY(crypt_salts, tc)
 {
 	for (size_t i = 0; tests[i].hash; i++) {
+#if defined(__FreeBSD__)
+		if (22 <= i)
+			atf_tc_expect_fail("Old-style/bad inputs fail on FreeBSD");
+#endif
 		char *hash = crypt(tests[i].pw, tests[i].hash);
 		if (!hash) {
 			ATF_CHECK_MSG(0, "Test %zu NULL\n", i);
diff --git a/etc/mtree/BSD.tests.dist b/etc/mtree/BSD.tests.dist
index 459a1a7..b73ab33 100644
--- a/etc/mtree/BSD.tests.dist
+++ b/etc/mtree/BSD.tests.dist
@@ -197,8 +197,12 @@
         file
         ..
         kern
+            acct
+            ..
             execve
             ..
+            pipe
+            ..
         ..
         kqueue
         ..
diff --git a/lib/libc/tests/gen/Makefile b/lib/libc/tests/gen/Makefile
index f9a0bd4..91097ef 100644
--- a/lib/libc/tests/gen/Makefile
+++ b/lib/libc/tests/gen/Makefile
@@ -4,8 +4,15 @@
 
 TESTSDIR=	${TESTSBASE}/lib/libc/gen
 
-ATF_TESTS_C=		arc4random_test
+ATF_TESTS_C+=		arc4random_test
+ATF_TESTS_C+=		fmtcheck2_test
+ATF_TESTS_C+=		fmtmsg_test
+ATF_TESTS_C+=		fnmatch2_test
 ATF_TESTS_C+=		fpclassify2_test
+ATF_TESTS_C+=		ftw_test
+ATF_TESTS_C+=		popen_test
+ATF_TESTS_C+=		posix_spawn_test
+ATF_TESTS_C+=		wordexp_test
 
 # TODO: t_closefrom, t_cpuset, t_fmtcheck, t_randomid, t_sleep
 # TODO: t_siginfo (fixes require further inspection)
@@ -55,7 +62,25 @@ DPADD.nice_test+=	${LIBPTHREAD}
 LDADD.syslog_test+=	-lpthread
 DPADD.syslog_test+=	${LIBPTHREAD}
 
+CFLAGS+=		-I${.CURDIR}
+
+SRCS.fmtcheck2_test=	fmtcheck_test.c
+SRCS.fnmatch2_test=	fnmatch_test.c
+
 TESTS_SUBDIRS=	execve
 TESTS_SUBDIRS+=	posix_spawn
 
+# The old testcase name
+TEST_FNMATCH=	test-fnmatch
+CLEANFILES+=		${GEN_SH_CASE_TESTCASES}
+sh-tests: .PHONY
+.for target in clean obj depend all
+	@cd ${.CURDIR} && ${MAKE} PROG=${TEST_FNMATCH} \
+	    -DNO_SUBDIR ${target}
+.endfor
+	@cd ${.OBJDIR} && ./${TEST_FNMATCH} -s 1 > \
+	    ${SRCTOP}/bin/sh/tests/builtins/case2.0
+	@cd ${.OBJDIR} && ./${TEST_FNMATCH} -s 2 > \
+	    ${SRCTOP}/bin/sh/tests/builtins/case3.0
+
 .include <bsd.test.mk>
diff --git a/lib/libc/tests/gen/fmtcheck_test.c b/lib/libc/tests/gen/fmtcheck_test.c
new file mode 100644
index 0000000..3e180c9
--- /dev/null
+++ b/lib/libc/tests/gen/fmtcheck_test.c
@@ -0,0 +1,105 @@
+/*	$NetBSD: tfmtcheck.c,v 1.3 2008/04/28 20:23:04 martin Exp $	*/
+
+/*-
+ * Copyright (c) 2000 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code was contributed to The NetBSD Foundation by Allen Briggs.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <err.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include <atf-c.h>
+
+struct test_fmt {
+	char	*fmt1;
+	char	*fmt2;
+	int	correct;
+} test_fmts[] = {
+	{ "%d", "%d", 1 },
+	{ "%2d", "%2.2d", 1 },
+	{ "%x", "%d", 1 },
+	{ "%u", "%d", 1 },
+	{ "%03d", "%d", 1 },
+	{ "%-2d", "%d", 1 },
+	{ "%d", "%-12.1d", 1 },
+	{ "%d", "%-01.3d", 1 },
+	{ "%X", "%-01.3d", 1 },
+	{ "%D", "%ld", 1 },
+	{ "%s", "%s", 1 },
+	{ "%s", "This is a %s test", 1 },
+	{ "Hi, there.  This is a %s test", "%s", 1 },
+	{ "%d", "%s", 2 },
+	{ "%e", "%s", 2 },
+	{ "%r", "%d", 2 },
+	{ "%*.2d", "%*d", 1 },
+	{ "%2.*d", "%*d", 2 },
+	{ "%*d", "%*d", 1 },
+	{ "%-3", "%d", 2 },
+	{ "%d %s", "%d", 2 },
+	{ "%*.*.*d", "%*.*.*d", 2 },
+	{ "%d", "%d %s", 1 },
+	{ "%40s", "%20s", 1 },
+	{ "%x %x %x", "%o %u %d", 1 },
+	{ "%o %u %d", "%x %x %X", 1 },
+	{ "%#o %u %#-d", "%x %#x %X", 1 },
+	{ "%qd", "%llx", 1 },
+	{ "%%", "%llx", 1 },
+	{ "%p %30s %#llx %-10.*e", "This number %lu%% and string %s has %qd numbers and %.*g floats", 1 },
+};
+
+ATF_TC_WITHOUT_HEAD(fmtcheck_test);
+ATF_TC_BODY(fmtcheck_test, tc)
+{
+	int i;
+	const char *f, *cf, *f1, *f2;
+
+	for (i = 0; i < nitems(test_fmts); i++) {
+		f1 = test_fmts[i].fmt1;
+		f2 = test_fmts[i].fmt2;
+		f = fmtcheck(f1, f2);
+		if (test_fmts[i].correct == 1)
+			cf = f1;
+		else
+			cf = f2;
+		ATF_CHECK_MSG(f == cf,
+		    "Test %d: (%s) vs. (%s) failed "
+		    "(should have returned %s)", i + 1, f1, f2,
+		    (test_fmts[i].correct == 1) ? "1st" : "2nd");
+	}
+}
+
+ATF_TP_ADD_TCS(tp)
+{
+
+	ATF_TP_ADD_TC(tp, fmtcheck_test);
+
+	return (atf_no_error());
+}
diff --git a/lib/libc/tests/gen/fmtmsg_test.c b/lib/libc/tests/gen/fmtmsg_test.c
new file mode 100644
index 0000000..aa3ca19
--- /dev/null
+++ b/lib/libc/tests/gen/fmtmsg_test.c
@@ -0,0 +1,252 @@
+/*-
+ * Copyright (c) 2012 Jilles Tjoelker
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/wait.h>
+#include <err.h>
+#include <errno.h>
+#include <fmtmsg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#include <atf-c.h>
+
+static char *run_test(long classification, const char *label, int severity,
+    const char *text, const char *action, const char *tag);
+
+struct testcase {
+	long classification;
+	const char *label;
+	int severity;
+	const char *text;
+	const char *action;
+	const char *tag;
+	const char *msgverb;
+	const char *result;
+} testcases[] = {
+	{
+		MM_UTIL | MM_PRINT, "BSD:ls", MM_ERROR,
+		"illegal option -- z", "refer to manual", "BSD:ls:001",
+		NULL,
+		"BSD:ls: ERROR: illegal option -- z\n"
+		    "TO FIX: refer to manual BSD:ls:001\n"
+	},
+	{
+		MM_UTIL | MM_PRINT, "BSD:ls", MM_ERROR,
+		"illegal option -- z", "refer to manual", "BSD:ls:001",
+		"text:severity:action:tag",
+		"illegal option -- z: ERROR\n"
+		    "TO FIX: refer to manual BSD:ls:001\n"
+	},
+	{
+		MM_UTIL | MM_PRINT, "BSD:ls", MM_ERROR,
+		"illegal option -- z", "refer to manual", "BSD:ls:001",
+		"text",
+		"illegal option -- z\n"
+	},
+	{
+		MM_UTIL | MM_PRINT, "BSD:ls", MM_ERROR,
+		"illegal option -- z", "refer to manual", "BSD:ls:001",
+		"severity:text",
+		"ERROR: illegal option -- z\n"
+	},
+	{
+		MM_UTIL | MM_PRINT, "BSD:ls", MM_ERROR,
+		"illegal option -- z", "refer to manual", "BSD:ls:001",
+		"ignore me",
+		"BSD:ls: ERROR: illegal option -- z\n"
+		    "TO FIX: refer to manual BSD:ls:001\n"
+	},
+	{
+		MM_UTIL | MM_PRINT, "BSD:ls", MM_ERROR,
+		"illegal option -- z", "refer to manual", "BSD:ls:001",
+		"tag:severity:text:nothing:action",
+		"BSD:ls: ERROR: illegal option -- z\n"
+		    "TO FIX: refer to manual BSD:ls:001\n"
+	},
+	{
+		MM_UTIL | MM_PRINT, "BSD:ls", MM_ERROR,
+		"illegal option -- z", "refer to manual", "BSD:ls:001",
+		"",
+		"BSD:ls: ERROR: illegal option -- z\n"
+		    "TO FIX: refer to manual BSD:ls:001\n"
+	},
+	{
+		MM_UTIL | MM_PRINT, MM_NULLLBL, MM_ERROR,
+		"illegal option -- z", "refer to manual", "BSD:ls:001",
+		NULL,
+		"ERROR: illegal option -- z\n"
+		    "TO FIX: refer to manual BSD:ls:001\n"
+	},
+	{
+		MM_UTIL | MM_PRINT, "BSD:ls", MM_ERROR,
+		"illegal option -- z", MM_NULLACT, MM_NULLTAG,
+		NULL,
+		"BSD:ls: ERROR: illegal option -- z\n"
+	},
+	{
+		MM_UTIL | MM_NULLMC, "BSD:ls", MM_ERROR,
+		"illegal option -- z", "refer to manual", "BSD:ls:001",
+		NULL,
+		""
+	},
+	{
+		MM_APPL | MM_PRINT, "ABCDEFGHIJ:abcdefghijklmn", MM_INFO,
+		"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+		    "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+		    "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+		    "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
+		"refer to manual", "ABCDEFGHIJ:abcdefghijklmn:001",
+		NULL,
+		"ABCDEFGHIJ:abcdefghijklmn: INFO: "
+		    "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+		    "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+		    "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+		    "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\n"
+		    "TO FIX: refer to manual ABCDEFGHIJ:abcdefghijklmn:001\n"
+	},
+	{
+		MM_OPSYS | MM_PRINT, "TEST:test", MM_HALT,
+		"failed", "nothing can help me", "NOTHING",
+		NULL,
+		"TEST:test: HALT: failed\n"
+		    "TO FIX: nothing can help me NOTHING\n"
+	},
+	{
+		MM_OPSYS | MM_PRINT, "TEST:test", MM_WARNING,
+		"failed", "nothing can help me", "NOTHING",
+		NULL,
+		"TEST:test: WARNING: failed\n"
+		    "TO FIX: nothing can help me NOTHING\n"
+	},
+	{
+		MM_OPSYS | MM_PRINT, "TEST:test", MM_NOSEV,
+		"failed", "nothing can help me", "NOTHING",
+		NULL,
+		"TEST:test: failed\n"
+		    "TO FIX: nothing can help me NOTHING\n"
+	}
+};
+
+static char *
+run_test(long classification, const char *label, int severity,
+    const char *text, const char *action, const char *tag)
+{
+	int pip[2];
+	pid_t pid, wpid;
+	char *result, *p;
+	size_t resultsize;
+	ssize_t n;
+	int status;
+
+	if (pipe(pip) == -1)
+		err(2, "pipe");
+	pid = fork();
+	if (pid == -1)
+		err(2, "fork");
+	if (pid == 0) {
+		close(pip[0]);
+		if (pip[1] != STDERR_FILENO &&
+		    dup2(pip[1], STDERR_FILENO) == -1)
+			_exit(2);
+		if (fmtmsg(classification, label, severity, text, action, tag)
+		    != MM_OK)
+			_exit(1);
+		else
+			_exit(0);
+	}
+	close(pip[1]);
+	resultsize = 1024;
+	result = malloc(resultsize);
+	p = result;
+	while ((n = read(pip[0], p, result + resultsize - p - 1)) != 0) {
+		if (n == -1) {
+			if (errno == EINTR)
+				continue;
+			else
+				err(2, "read");
+		}
+		p += n;
+		if (result + resultsize == p - 1) {
+			resultsize *= 2;
+			result = realloc(result, resultsize);
+			if (result == NULL)
+				err(2, "realloc");
+		}
+	}
+	if (memchr(result, '\0', p - result) != NULL) {
+		free(result);
+		return (NULL);
+	}
+	*p = '\0';
+	close(pip[0]);
+	while ((wpid = waitpid(pid, &status, 0)) == -1 && errno == EINTR)
+		;
+	if (wpid == -1)
+		err(2, "waitpid");
+	if (status != 0) {
+		free(result);
+		return (NULL);
+	}
+	return (result);
+}
+
+ATF_TC_WITHOUT_HEAD(fmtmsg_test);
+ATF_TC_BODY(fmtmsg_test, tc)
+{
+	char *result;
+	struct testcase *t;
+	int i;
+
+	for (i = 0; i < nitems(testcases); i++) {
+		t = &testcases[i];
+		if (t->msgverb != NULL)
+			setenv("MSGVERB", t->msgverb, 1);
+		else
+			unsetenv("MSGVERB");
+		result = run_test(t->classification, t->label, t->severity,
+		    t->text, t->action, t->tag);
+		ATF_CHECK_MSG(result != NULL, "testcase %d failed", i + 1);
+		if (result != NULL)
+			ATF_CHECK_MSG(strcmp(result, t->result) == 0,
+			    "results for testcase %d didn't match; "
+			    "`%s` != `%s`", i + 1, result, t->result);
+		free(result);
+	}
+}
+
+ATF_TP_ADD_TCS(tp)
+{
+
+	ATF_TP_ADD_TC(tp, fmtmsg_test);
+
+	return (atf_no_error());
+}
diff --git a/lib/libc/tests/gen/fnmatch_test.c b/lib/libc/tests/gen/fnmatch_test.c
new file mode 100644
index 0000000..8d9ead2
--- /dev/null
+++ b/lib/libc/tests/gen/fnmatch_test.c
@@ -0,0 +1,188 @@
+/*-
+ * Copyright (c) 2010 Jilles Tjoelker
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#include <atf-c.h>
+
+#include "fnmatch_testcases.h"
+
+static const char *
+flags_to_string(int flags)
+{
+	static const int flagvalues[] = { FNM_NOESCAPE, FNM_PATHNAME,
+		FNM_PERIOD, FNM_LEADING_DIR, FNM_CASEFOLD, 0 };
+	static const char flagnames[] = "FNM_NOESCAPE\0FNM_PATHNAME\0FNM_PERIOD\0FNM_LEADING_DIR\0FNM_CASEFOLD\0";
+	static char result[sizeof(flagnames) + 3 * sizeof(int) + 2];
+	char *p;
+	size_t i, len;
+	const char *fp;
+
+	p = result;
+	fp = flagnames;
+	for (i = 0; flagvalues[i] != 0; i++) {
+		len = strlen(fp);
+		if (flags & flagvalues[i]) {
+			if (p != result)
+				*p++ = '|';
+			memcpy(p, fp, len);
+			p += len;
+			flags &= ~flagvalues[i];
+		}
+		fp += len + 1;
+	}
+	if (p == result)
+		memcpy(p, "0", 2);
+	else if (flags != 0)
+		sprintf(p, "%d", flags);
+	else
+		*p = '\0';
+	return result;
+}
+
+ATF_TC_WITHOUT_HEAD(fnmatch_test);
+ATF_TC_BODY(fnmatch_test, tc)
+{
+	size_t i;
+	int flags, result;
+	struct testcase *t;
+
+	for (i = 0; i < nitems(testcases); i++) {
+		t = &testcases[i];
+		flags = t->flags;
+		do {
+			result = fnmatch(t->pattern, t->string, flags);
+			if (result != t->result)
+				break;
+			if (strchr(t->pattern, '\\') == NULL &&
+			    !(flags & FNM_NOESCAPE)) {
+				flags |= FNM_NOESCAPE;
+				result = fnmatch(t->pattern, t->string, flags);
+				if (result != t->result)
+					break;
+				flags = t->flags;
+			}
+			if (strchr(t->pattern, '\\') != NULL &&
+			    strchr(t->string, '\\') == NULL &&
+			    t->result == FNM_NOMATCH &&
+			    !(flags & (FNM_NOESCAPE | FNM_LEADING_DIR))) {
+				flags |= FNM_NOESCAPE;
+				result = fnmatch(t->pattern, t->string, flags);
+				if (result != t->result)
+					break;
+				flags = t->flags;
+			}
+			if ((t->string[0] != '.' || t->pattern[0] == '.' ||
+			    t->result == FNM_NOMATCH) &&
+			    !(flags & (FNM_PATHNAME | FNM_PERIOD))) {
+				flags |= FNM_PERIOD;
+				result = fnmatch(t->pattern, t->string, flags);
+				if (result != t->result)
+					break;
+				flags = t->flags;
+			}
+			if ((strchr(t->string, '/') == NULL ||
+			    t->result == FNM_NOMATCH) &&
+			    !(flags & FNM_PATHNAME)) {
+				flags |= FNM_PATHNAME;
+				result = fnmatch(t->pattern, t->string, flags);
+				if (result != t->result)
+					break;
+				flags = t->flags;
+			}
+			if ((((t->string[0] != '.' || t->pattern[0] == '.') &&
+			    strstr(t->string, "/.") == NULL) ||
+			    t->result == FNM_NOMATCH) &&
+			    flags & FNM_PATHNAME && !(flags & FNM_PERIOD)) {
+				flags |= FNM_PERIOD;
+				result = fnmatch(t->pattern, t->string, flags);
+				if (result != t->result)
+					break;
+				flags = t->flags;
+			}
+			if ((((t->string[0] != '.' || t->pattern[0] == '.') &&
+			    strchr(t->string, '/') == NULL) ||
+			    t->result == FNM_NOMATCH) &&
+			    !(flags & (FNM_PATHNAME | FNM_PERIOD))) {
+				flags |= FNM_PATHNAME | FNM_PERIOD;
+				result = fnmatch(t->pattern, t->string, flags);
+				if (result != t->result)
+					break;
+				flags = t->flags;
+			}
+			if ((strchr(t->string, '/') == NULL || t->result == 0)
+			    && !(flags & FNM_LEADING_DIR)) {
+				flags |= FNM_LEADING_DIR;
+				result = fnmatch(t->pattern, t->string, flags);
+				if (result != t->result)
+					break;
+				flags = t->flags;
+			}
+			if (t->result == 0 && !(flags & FNM_CASEFOLD)) {
+				flags |= FNM_CASEFOLD;
+				result = fnmatch(t->pattern, t->string, flags);
+				if (result != t->result)
+					break;
+				flags = t->flags;
+			}
+			if (strchr(t->pattern, '\\') == NULL &&
+			    t->result == 0 &&
+			    !(flags & (FNM_NOESCAPE | FNM_CASEFOLD))) {
+				flags |= FNM_NOESCAPE | FNM_CASEFOLD;
+				result = fnmatch(t->pattern, t->string, flags);
+				if (result != t->result)
+					break;
+				flags = t->flags;
+			}
+		} while (0);
+
+		ATF_CHECK(result == t->result);
+		if (result == t->result)
+			printf("fnmatch(\"%s\", \"%s\", %s) == %d\n",
+			    t->pattern, t->string, flags_to_string(flags), result);
+		else
+			printf("fnmatch(\"%s\", \"%s\", %s) != %d (was %d)\n",
+			    t->pattern, t->string, flags_to_string(flags),
+			    t->result, result);
+	}
+
+}
+
+ATF_TP_ADD_TCS(tp)
+{
+
+	ATF_TP_ADD_TC(tp, fnmatch_test);
+
+	return (atf_no_error());
+}
diff --git a/lib/libc/tests/gen/fnmatch_testcases.h b/lib/libc/tests/gen/fnmatch_testcases.h
new file mode 100644
index 0000000..537011f
--- /dev/null
+++ b/lib/libc/tests/gen/fnmatch_testcases.h
@@ -0,0 +1,176 @@
+/*-
+ * Copyright (c) 2010 Jilles Tjoelker
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <fnmatch.h>
+
+struct testcase {
+	const char *pattern;
+	const char *string;
+	int flags;
+	int result;
+} testcases[] = {
+	{ "", "", 0, 0 },
+	{ "a", "a", 0, 0 },
+	{ "a", "b", 0, FNM_NOMATCH },
+	{ "a", "A", 0, FNM_NOMATCH },
+	{ "*", "a", 0, 0 },
+	{ "*", "aa", 0, 0 },
+	{ "*a", "a", 0, 0 },
+	{ "*a", "b", 0, FNM_NOMATCH },
+	{ "*a*", "b", 0, FNM_NOMATCH },
+	{ "*a*b*", "ab", 0, 0 },
+	{ "*a*b*", "qaqbq", 0, 0 },
+	{ "*a*bb*", "qaqbqbbq", 0, 0 },
+	{ "*a*bc*", "qaqbqbcq", 0, 0 },
+	{ "*a*bb*", "qaqbqbb", 0, 0 },
+	{ "*a*bc*", "qaqbqbc", 0, 0 },
+	{ "*a*bb", "qaqbqbb", 0, 0 },
+	{ "*a*bc", "qaqbqbc", 0, 0 },
+	{ "*a*bb", "qaqbqbbq", 0, FNM_NOMATCH },
+	{ "*a*bc", "qaqbqbcq", 0, FNM_NOMATCH },
+	{ "*a*a*a*a*a*a*a*a*a*a*", "aaaaaaaaa", 0, FNM_NOMATCH },
+	{ "*a*a*a*a*a*a*a*a*a*a*", "aaaaaaaaaa", 0, 0 },
+	{ "*a*a*a*a*a*a*a*a*a*a*", "aaaaaaaaaaa", 0, 0 },
+	{ ".*.*.*.*.*.*.*.*.*.*", ".........", 0, FNM_NOMATCH },
+	{ ".*.*.*.*.*.*.*.*.*.*", "..........", 0, 0 },
+	{ ".*.*.*.*.*.*.*.*.*.*", "...........", 0, 0 },
+	{ "*?*?*?*?*?*?*?*?*?*?*", "123456789", 0, FNM_NOMATCH },
+	{ "??????????*", "123456789", 0, FNM_NOMATCH },
+	{ "*??????????", "123456789", 0, FNM_NOMATCH },
+	{ "*?*?*?*?*?*?*?*?*?*?*", "1234567890", 0, 0 },
+	{ "??????????*", "1234567890", 0, 0 },
+	{ "*??????????", "1234567890", 0, 0 },
+	{ "*?*?*?*?*?*?*?*?*?*?*", "12345678901", 0, 0 },
+	{ "??????????*", "12345678901", 0, 0 },
+	{ "*??????????", "12345678901", 0, 0 },
+	{ "[x]", "x", 0, 0 },
+	{ "[*]", "*", 0, 0 },
+	{ "[?]", "?", 0, 0 },
+	{ "[", "[", 0, 0 },
+	{ "[[]", "[", 0, 0 },
+	{ "[[]", "x", 0, FNM_NOMATCH },
+	{ "[*]", "", 0, FNM_NOMATCH },
+	{ "[*]", "x", 0, FNM_NOMATCH },
+	{ "[?]", "x", 0, FNM_NOMATCH },
+	{ "*[*]*", "foo*foo", 0, 0 },
+	{ "*[*]*", "foo", 0, FNM_NOMATCH },
+	{ "[0-9]", "0", 0, 0 },
+	{ "[0-9]", "5", 0, 0 },
+	{ "[0-9]", "9", 0, 0 },
+	{ "[0-9]", "/", 0, FNM_NOMATCH },
+	{ "[0-9]", ":", 0, FNM_NOMATCH },
+	{ "[0-9]", "*", 0, FNM_NOMATCH },
+	{ "[!0-9]", "0", 0, FNM_NOMATCH },
+	{ "[!0-9]", "5", 0, FNM_NOMATCH },
+	{ "[!0-9]", "9", 0, FNM_NOMATCH },
+	{ "[!0-9]", "/", 0, 0 },
+	{ "[!0-9]", ":", 0, 0 },
+	{ "[!0-9]", "*", 0, 0 },
+	{ "*[0-9]", "a0", 0, 0 },
+	{ "*[0-9]", "a5", 0, 0 },
+	{ "*[0-9]", "a9", 0, 0 },
+	{ "*[0-9]", "a/", 0, FNM_NOMATCH },
+	{ "*[0-9]", "a:", 0, FNM_NOMATCH },
+	{ "*[0-9]", "a*", 0, FNM_NOMATCH },
+	{ "*[!0-9]", "a0", 0, FNM_NOMATCH },
+	{ "*[!0-9]", "a5", 0, FNM_NOMATCH },
+	{ "*[!0-9]", "a9", 0, FNM_NOMATCH },
+	{ "*[!0-9]", "a/", 0, 0 },
+	{ "*[!0-9]", "a:", 0, 0 },
+	{ "*[!0-9]", "a*", 0, 0 },
+	{ "*[0-9]", "a00", 0, 0 },
+	{ "*[0-9]", "a55", 0, 0 },
+	{ "*[0-9]", "a99", 0, 0 },
+	{ "*[0-9]", "a0a0", 0, 0 },
+	{ "*[0-9]", "a5a5", 0, 0 },
+	{ "*[0-9]", "a9a9", 0, 0 },
+	{ "\\*", "*", 0, 0 },
+	{ "\\?", "?", 0, 0 },
+	{ "\\[x]", "[x]", 0, 0 },
+	{ "\\[", "[", 0, 0 },
+	{ "\\\\", "\\", 0, 0 },
+	{ "*\\**", "foo*foo", 0, 0 },
+	{ "*\\**", "foo", 0, FNM_NOMATCH },
+	{ "*\\\\*", "foo\\foo", 0, 0 },
+	{ "*\\\\*", "foo", 0, FNM_NOMATCH },
+	{ "\\(", "(", 0, 0 },
+	{ "\\a", "a", 0, 0 },
+	{ "\\*", "a", 0, FNM_NOMATCH },
+	{ "\\?", "a", 0, FNM_NOMATCH },
+	{ "\\*", "\\*", 0, FNM_NOMATCH },
+	{ "\\?", "\\?", 0, FNM_NOMATCH },
+	{ "\\[x]", "\\[x]", 0, FNM_NOMATCH },
+	{ "\\[x]", "\\x", 0, FNM_NOMATCH },
+	{ "\\[", "\\[", 0, FNM_NOMATCH },
+	{ "\\(", "\\(", 0, FNM_NOMATCH },
+	{ "\\a", "\\a", 0, FNM_NOMATCH },
+	{ "\\", "\\", 0, FNM_NOMATCH },
+	{ "\\", "", 0, 0 },
+	{ "\\*", "\\*", FNM_NOESCAPE, 0 },
+	{ "\\?", "\\?", FNM_NOESCAPE, 0 },
+	{ "\\", "\\", FNM_NOESCAPE, 0 },
+	{ "\\\\", "\\", FNM_NOESCAPE, FNM_NOMATCH },
+	{ "\\\\", "\\\\", FNM_NOESCAPE, 0 },
+	{ "*\\*", "foo\\foo", FNM_NOESCAPE, 0 },
+	{ "*\\*", "foo", FNM_NOESCAPE, FNM_NOMATCH },
+	{ "*", ".", FNM_PERIOD, FNM_NOMATCH },
+	{ "?", ".", FNM_PERIOD, FNM_NOMATCH },
+	{ ".*", ".", 0, 0 },
+	{ ".*", "..", 0, 0 },
+	{ ".*", ".a", 0, 0 },
+	{ "[0-9]", ".", FNM_PERIOD, FNM_NOMATCH },
+	{ "a*", "a.", 0, 0 },
+	{ "a/a", "a/a", FNM_PATHNAME, 0 },
+	{ "a/*", "a/a", FNM_PATHNAME, 0 },
+	{ "*/a", "a/a", FNM_PATHNAME, 0 },
+	{ "*/*", "a/a", FNM_PATHNAME, 0 },
+	{ "a*b/*", "abbb/x", FNM_PATHNAME, 0 },
+	{ "a*b/*", "abbb/.x", FNM_PATHNAME, 0 },
+	{ "*", "a/a", FNM_PATHNAME, FNM_NOMATCH },
+	{ "*/*", "a/a/a", FNM_PATHNAME, FNM_NOMATCH },
+	{ "b/*", "b/.x", FNM_PATHNAME | FNM_PERIOD, FNM_NOMATCH },
+	{ "b*/*", "a/.x", FNM_PATHNAME | FNM_PERIOD, FNM_NOMATCH },
+	{ "b/.*", "b/.x", FNM_PATHNAME | FNM_PERIOD, 0 },
+	{ "b*/.*", "b/.x", FNM_PATHNAME | FNM_PERIOD, 0 },
+	{ "a", "A", FNM_CASEFOLD, 0 },
+	{ "A", "a", FNM_CASEFOLD, 0 },
+	{ "[a]", "A", FNM_CASEFOLD, 0 },
+	{ "[A]", "a", FNM_CASEFOLD, 0 },
+	{ "a", "b", FNM_CASEFOLD, FNM_NOMATCH },
+	{ "a", "a/b", FNM_PATHNAME, FNM_NOMATCH },
+	{ "*", "a/b", FNM_PATHNAME, FNM_NOMATCH },
+	{ "*b", "a/b", FNM_PATHNAME, FNM_NOMATCH },
+	{ "a", "a/b", FNM_PATHNAME | FNM_LEADING_DIR, 0 },
+	{ "*", "a/b", FNM_PATHNAME | FNM_LEADING_DIR, 0 },
+	{ "*", ".a/b", FNM_PATHNAME | FNM_LEADING_DIR, 0 },
+	{ "*a", ".a/b", FNM_PATHNAME | FNM_LEADING_DIR, 0 },
+	{ "*", ".a/b", FNM_PATHNAME | FNM_PERIOD | FNM_LEADING_DIR, FNM_NOMATCH },
+	{ "*a", ".a/b", FNM_PATHNAME | FNM_PERIOD | FNM_LEADING_DIR, FNM_NOMATCH },
+	{ "a*b/*", "abbb/.x", FNM_PATHNAME | FNM_PERIOD, FNM_NOMATCH },
+};
diff --git a/lib/libc/tests/gen/ftw_test.c b/lib/libc/tests/gen/ftw_test.c
new file mode 100644
index 0000000..b120f01
--- /dev/null
+++ b/lib/libc/tests/gen/ftw_test.c
@@ -0,0 +1,127 @@
+/*-
+ * Copyright (c) 2012 Jilles Tjoelker
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * Limited test program for nftw() as specified by IEEE Std. 1003.1-2008.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/wait.h>
+#include <err.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <ftw.h>
+#include <limits.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <spawn.h>
+#include <unistd.h>
+
+#include <atf-c.h>
+
+extern char **environ;
+
+static char template[] = "testftw.XXXXXXXXXX";
+static char dir[PATH_MAX];
+static int ftwflags;
+
+static int
+cb(const char *path, const struct stat *st, int type, struct FTW *f)
+{
+
+	switch (type) {
+	case FTW_D:
+		if ((ftwflags & FTW_DEPTH) == 0)
+			return (0);
+		break;
+	case FTW_DP:
+		if ((ftwflags & FTW_DEPTH) != 0)
+			return (0);
+		break;
+	case FTW_SL:
+		if ((ftwflags & FTW_PHYS) != 0)
+			return (0);
+		break;
+	}
+	ATF_CHECK_MSG(false,
+	    "unexpected path=%s type=%d f.level=%d\n",
+	    path, type, f->level);
+	return (0);
+}
+
+ATF_TC_WITHOUT_HEAD(ftw_test);
+ATF_TC_BODY(ftw_test, tc)
+{
+	int fd;
+
+	ATF_REQUIRE_MSG(mkdtemp(template) != NULL, "mkdtemp failed");
+
+	/* XXX: the path needs to be absolute for the 0/FTW_DEPTH testcases */
+	ATF_REQUIRE_MSG(realpath(template, dir) != NULL,
+	    "realpath failed; errno=%d", errno);
+
+	fd = open(dir, O_DIRECTORY|O_RDONLY);
+	ATF_REQUIRE_MSG(fd != -1, "open failed; errno=%d", errno);
+
+	ATF_REQUIRE_MSG(mkdirat(fd, "d1", 0777) == 0,
+	    "mkdirat failed; errno=%d", errno);
+
+	ATF_REQUIRE_MSG(symlinkat(dir, fd, "d1/looper") == 0,
+	    "symlinkat failed; errno=%d", errno);
+
+	printf("ftwflags=FTW_PHYS\n");
+	ftwflags = FTW_PHYS;
+	ATF_REQUIRE_MSG(nftw(dir, cb, 10, ftwflags) != -1,
+	    "nftw FTW_PHYS failed; errno=%d", errno);
+
+	printf("ftwflags=FTW_PHYS|FTW_DEPTH\n");
+	ftwflags = FTW_PHYS|FTW_DEPTH;
+	ATF_REQUIRE_MSG(nftw(dir, cb, 10, ftwflags) != -1,
+	    "nftw FTW_PHYS|FTW_DEPTH failed; errno=%d", errno);
+
+	printf("ftwflags=0\n");
+	ftwflags = 0;
+	ATF_REQUIRE_MSG(nftw(dir, cb, 10, ftwflags) != -1,
+	    "nftw 0 failed; errno=%d", errno);
+
+	printf("ftwflags=FTW_DEPTH\n");
+	ftwflags = FTW_DEPTH;
+	ATF_REQUIRE_MSG(nftw(dir, cb, 10, ftwflags) != -1,
+	    "nftw FTW_DEPTH failed; errno=%d", errno);
+
+	close(fd);
+}
+
+ATF_TP_ADD_TCS(tp)
+{
+
+	ATF_TP_ADD_TC(tp, ftw_test);
+
+	return (atf_no_error());
+}
diff --git a/lib/libc/tests/gen/popen_test.c b/lib/libc/tests/gen/popen_test.c
new file mode 100644
index 0000000..5c615b7
--- /dev/null
+++ b/lib/libc/tests/gen/popen_test.c
@@ -0,0 +1,251 @@
+/*-
+ * Copyright (c) 2013 Jilles Tjoelker
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * Limited test program for popen() as specified by IEEE Std. 1003.1-2008,
+ * with BSD extensions.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/wait.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <atf-c.h>
+
+static volatile sig_atomic_t got_sigpipe;
+
+static void
+sigpipe_handler(int sig __unused)
+{
+	got_sigpipe = 1;
+}
+
+static void
+check_cloexec(FILE *fp, const char *mode)
+{
+	int exp_flags, flags;
+
+	flags = fcntl(fileno(fp), F_GETFD);
+	ATF_CHECK_MSG(flags != -1, "fcntl(F_GETFD) failed; errno=%d", errno);
+	if (flags == -1)
+		return;
+	if (strchr(mode, 'e') != NULL)
+		exp_flags = FD_CLOEXEC;
+	else
+		exp_flags = 0;
+	ATF_CHECK_MSG((flags & FD_CLOEXEC) == exp_flags,
+	    "bad cloexec flag; %d != %d", flags, exp_flags);
+}
+
+ATF_TC_WITHOUT_HEAD(popen_all_modes_test);
+ATF_TC_BODY(popen_all_modes_test, tc)
+{
+	FILE *fp;
+	int i, status;
+	const char *mode;
+	const char *allmodes[] = { "r", "w", "r+", "re", "we", "r+e", "re+" };
+
+	for (i = 0; i < nitems(allmodes); i++) {
+		mode = allmodes[i];
+		fp = popen("exit 7", mode);
+		ATF_CHECK_MSG(fp != NULL, "popen(, \"%s\") failed", mode);
+		if (fp == NULL)
+			continue;
+		check_cloexec(fp, mode);
+		status = pclose(fp);
+		ATF_CHECK_MSG(WIFEXITED(status) && WEXITSTATUS(status) == 7,
+		    "bad exit status (no I/O)");
+	}
+}
+
+ATF_TC_WITHOUT_HEAD(popen_rmodes_test);
+ATF_TC_BODY(popen_rmodes_test, tc)
+{
+	FILE *fp;
+	const char *rmodes[] = { "r", "r+", "re", "r+e", "re+" };
+	const char *mode;
+	char buf[80];
+	int i, status;
+
+	for (i = 0; i < nitems(rmodes); i++) {
+		mode = rmodes[i];
+		fp = popen("exit 9", mode);
+		ATF_CHECK_MSG(fp != NULL, "popen(, \"%s\") failed", mode);
+		if (fp == NULL)
+			continue;
+		check_cloexec(fp, mode);
+		bool input_error_1 = !(fgetc(fp) != EOF || !feof(fp) || !ferror(fp));
+		ATF_CHECK_MSG(!input_error_1, "input error 1");
+		if (input_error_1)
+			continue;
+		status = pclose(fp);
+		ATF_CHECK_MSG(WIFEXITED(status) && WEXITSTATUS(status) == 9,
+		    "bad exit status (input)");
+	}
+
+	for (i = 0; i < nitems(rmodes); i++) {
+		char *sres;
+		mode = rmodes[i];
+		fp = popen("echo hi there", mode);
+		ATF_CHECK_MSG(fp != NULL, "popen(, \"%s\") failed", mode);
+		if (fp == NULL)
+			continue;
+		check_cloexec(fp, mode);
+		ATF_CHECK_MSG((sres = fgets(buf, sizeof(buf), fp)) != NULL,
+		    "Input error 2");
+		if (sres != NULL)
+			ATF_CHECK_MSG(strcmp(buf, "hi there\n") == 0,
+			    "Bad input 1");
+		status = pclose(fp);
+		ATF_CHECK_MSG(WIFEXITED(status) && WEXITSTATUS(status) == 0,
+		    "Bad exit status (input)");
+	}
+}
+
+ATF_TC_WITHOUT_HEAD(popen_wmodes_test);
+ATF_TC_BODY(popen_wmodes_test, tc)
+{
+	FILE *fp, *fp2;
+	const char *wmodes[] = { "w", "r+", "we", "r+e", "re+" };
+	const char *mode;
+	struct sigaction act, oact;
+	int i, j, status;
+
+	for (i = 0; i < nitems(wmodes); i++) {
+		mode = wmodes[i];
+		fp = popen("read x && [ \"$x\" = abcd ]", mode);
+		ATF_CHECK_MSG(fp != NULL, "popen(, \"%s\") failed", mode);
+		if (fp == NULL)
+			continue;
+		check_cloexec(fp, mode);
+		ATF_CHECK_MSG(fputs("abcd\n", fp) != EOF,
+		    "Output error 1");
+		status = pclose(fp);
+		ATF_CHECK_MSG(WIFEXITED(status) && WEXITSTATUS(status) == 0,
+		    "Bad exit status (output)");
+	}
+
+	act.sa_handler = sigpipe_handler;
+	act.sa_flags = SA_RESTART;
+	sigemptyset(&act.sa_mask);
+	ATF_CHECK_MSG(sigaction(SIGPIPE, &act, &oact) != -1,
+	    "sigaction() failed");
+	for (i = 0; i < nitems(wmodes); i++) {
+		mode = wmodes[i];
+		fp = popen("exit 88", mode);
+		ATF_CHECK_MSG(fp != NULL, "popen(, \"%s\") failed", mode);
+		if (fp == NULL)
+			continue;
+		check_cloexec(fp, mode);
+		got_sigpipe = 0;
+		while (fputs("abcd\n", fp) != EOF)
+			;
+		ATF_CHECK_MSG(ferror(fp) && errno == EPIPE, "Expected EPIPE");
+		ATF_CHECK_MSG(got_sigpipe, "Expected SIGPIPE");
+		status = pclose(fp);
+		ATF_CHECK_MSG(WIFEXITED(status) && WEXITSTATUS(status) == 88,
+		    "Bad exit status (EPIPE)");
+	}
+	ATF_CHECK_MSG(sigaction(SIGPIPE, &oact, NULL) != -1,
+	    "sigaction() failed");
+
+	for (i = 0; i < nitems(wmodes); i++) {
+		for (j = 0; j < nitems(wmodes); j++) {
+			mode = wmodes[i];
+			fp = popen("read x", mode);
+			ATF_CHECK_MSG(fp != NULL,
+			    "popen(, \"%s\") failed", mode);
+			if (fp == NULL)
+				continue;
+			mode = wmodes[j];
+			fp2 = popen("read x", mode);
+			ATF_CHECK_MSG(fp2 != NULL,
+			    "popen(, \"%s\") failed", mode);
+			if (fp2 == NULL) {
+				pclose(fp);
+				continue;
+			}
+			/* If fp2 inherits fp's pipe, we will deadlock here. */
+			status = pclose(fp);
+			ATF_CHECK_MSG(WIFEXITED(status) && WEXITSTATUS(status) == 1,
+			    "bad exit status (2 pipes)");
+			status = pclose(fp2);
+			ATF_CHECK_MSG(WIFEXITED(status) && WEXITSTATUS(status) == 1,
+			    "bad exit status (2 pipes)");
+		}
+	}
+}
+
+ATF_TC_WITHOUT_HEAD(popen_rwmodes_test);
+ATF_TC_BODY(popen_rwmodes_test, tc)
+{
+	const char *rwmodes[] = { "r+", "r+e", "re+" };
+	FILE *fp;
+	const char *mode;
+	char *sres;
+	char buf[80];
+	int i, ires, status;
+
+	for (i = 0; i < nitems(rwmodes); i++) {
+		mode = rwmodes[i];
+		fp = popen("read x && printf '%s\\n' \"Q${x#a}\"", mode);
+		ATF_CHECK_MSG(fp != NULL, "popen(, \"%s\") failed", mode);
+		if (fp == NULL)
+			continue;
+		check_cloexec(fp, mode);
+		ATF_CHECK_MSG((ires = fputs("abcd\n", fp)) != EOF,
+		    "Output error 2");
+		if (ires != EOF) {
+			sres = fgets(buf, sizeof(buf), fp);
+			ATF_CHECK_MSG(sres != NULL, "Input error 3");
+			if (sres != NULL)
+				ATF_CHECK_MSG(strcmp(buf, "Qbcd\n") == 0,
+				    "Bad input 2");
+		}
+		status = pclose(fp);
+		ATF_CHECK_MSG(WIFEXITED(status) && WEXITSTATUS(status) == 0,
+		    "bad exit status (I/O)");
+	}
+}
+
+ATF_TP_ADD_TCS(tp)
+{
+
+	ATF_TP_ADD_TC(tp, popen_all_modes_test);
+	ATF_TP_ADD_TC(tp, popen_rmodes_test);
+	ATF_TP_ADD_TC(tp, popen_wmodes_test);
+	ATF_TP_ADD_TC(tp, popen_rwmodes_test);
+
+	return (atf_no_error());
+}
diff --git a/lib/libc/tests/gen/posix_spawn_test.c b/lib/libc/tests/gen/posix_spawn_test.c
new file mode 100644
index 0000000..e04d103
--- /dev/null
+++ b/lib/libc/tests/gen/posix_spawn_test.c
@@ -0,0 +1,103 @@
+/*-
+ * Copyright (c) 2011 Jilles Tjoelker
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * Test program for posix_spawn() and posix_spawnp() as specified by
+ * IEEE Std. 1003.1-2008.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/wait.h>
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <spawn.h>
+
+#include <atf-c.h>
+
+char *myenv[2] = { "answer=42", NULL };
+
+ATF_TC_WITHOUT_HEAD(posix_spawn_simple_test);
+ATF_TC_BODY(posix_spawn_simple_test, tc)
+{
+	char *myargs[4];
+	int error, status;
+	pid_t pid, waitres;
+
+	/* Make sure we have no child processes. */
+	while (waitpid(-1, NULL, 0) != -1)
+		;
+	ATF_REQUIRE_MSG(errno == ECHILD, "errno was not ECHILD: %d", errno);
+
+	/* Simple test. */
+	myargs[0] = "sh";
+	myargs[1] = "-c";
+	myargs[2] = "exit $answer";
+	myargs[3] = NULL;
+	error = posix_spawnp(&pid, myargs[0], NULL, NULL, myargs, myenv);
+	ATF_REQUIRE(error == 0);
+	waitres = waitpid(pid, &status, 0);
+	ATF_REQUIRE(waitres == pid);
+	ATF_REQUIRE(WIFEXITED(status) && WEXITSTATUS(status) == 42);
+}
+
+ATF_TC_WITHOUT_HEAD(posix_spawn_no_such_command_negative_test);
+ATF_TC_BODY(posix_spawn_no_such_command_negative_test, tc)
+{
+	char *myargs[4];
+	int error, status;
+	pid_t pid, waitres;
+
+	/*
+	 * If the executable does not exist, the function shall either fail
+	 * and not create a child process or succeed and create a child
+	 * process that exits with status 127.
+	 */
+	myargs[0] = "/var/empty/nonexistent";
+	myargs[1] = NULL;
+	error = posix_spawn(&pid, myargs[0], NULL, NULL, myargs, myenv);
+	if (error == 0) {
+		waitres = waitpid(pid, &status, 0);
+		ATF_REQUIRE(waitres == pid);
+		ATF_REQUIRE(WIFEXITED(status) && WEXITSTATUS(status) == 127);
+	} else {
+		ATF_REQUIRE(error == ENOENT);
+		waitres = waitpid(-1, NULL, 0);
+		ATF_REQUIRE(waitres == -1 && errno == ECHILD);
+	}
+}
+
+ATF_TP_ADD_TCS(tp)
+{
+
+	ATF_TP_ADD_TC(tp, posix_spawn_simple_test);
+	ATF_TP_ADD_TC(tp, posix_spawn_no_such_command_negative_test);
+
+	return (atf_no_error());
+}
diff --git a/lib/libc/tests/gen/test-fnmatch.c b/lib/libc/tests/gen/test-fnmatch.c
new file mode 100644
index 0000000..7771ab3
--- /dev/null
+++ b/lib/libc/tests/gen/test-fnmatch.c
@@ -0,0 +1,102 @@
+/*-
+ * Copyright (c) 2010 Jilles Tjoelker
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "fnmatch_testcases.h"
+
+static int
+write_sh_tests(const char *progname, int num)
+{
+	size_t i;
+	struct testcase *t;
+
+	printf("# Generated by %s -s %d, do not edit.\n", progname, num);
+	printf("# $" "FreeBSD$\n");
+	printf("failures=\n");
+	printf("failed() { printf '%%s\\n' \"Failed: $1 '$2' '$3'\"; failures=x$failures; }\n");
+	if (num == 1) {
+		printf("testmatch() { eval \"case \\$2 in ''$1) ;; *) failed testmatch \\\"\\$@\\\";; esac\"; }\n");
+		printf("testnomatch() { eval \"case \\$2 in ''$1) failed testnomatch \\\"\\$@\\\";; esac\"; }\n");
+	} else if (num == 2) {
+		printf("# We do not treat a backslash specially in this case,\n");
+		printf("# but this is not the case in all shells.\n");
+		printf("netestmatch() { case $2 in $1) ;; *) failed netestmatch \"$@\";; esac; }\n");
+		printf("netestnomatch() { case $2 in $1) failed netestnomatch \"$@\";; esac; }\n");
+	}
+
+	for (i = 0; i < nitems(testcases); i++) {
+		t = &testcases[i];
+		if (strchr(t->pattern, '\'') != NULL ||
+		    strchr(t->string, '\'') != NULL)
+			continue;
+		if (t->flags == 0 && strcmp(t->pattern, "\\") == 0)
+			continue;
+		if (num == 1 && t->flags == 0)
+			printf("test%smatch '%s' '%s'\n",
+			    t->result == FNM_NOMATCH ? "no" : "",
+			    t->pattern, t->string);
+		if (num == 2 && (t->flags == FNM_NOESCAPE ||
+		    (t->flags == 0 && strchr(t->pattern, '\\') == NULL)))
+			printf("netest%smatch '%s' '%s'\n",
+			    t->result == FNM_NOMATCH ? "no" : "",
+			    t->pattern, t->string);
+	}
+	printf("[ -z \"$failures\" ]\n");
+	return 0;
+}
+
+static void
+usage(char *progname)
+{
+	fprintf(stderr, "usage: %s [-s num]\n", progname);
+	fprintf(stderr, "-s option writes tests for sh(1), num is 1 or 2\n");
+}
+
+int
+main(int argc, char *argv[])
+{
+	int opt;
+
+	while ((opt = getopt(argc, argv, "s:")) != -1) {
+		switch (opt) {
+		case 's':
+			return (write_sh_tests(argv[0], atoi(optarg)));
+		default:
+			usage(argv[0]);
+			exit(1);
+		}
+	}
+	usage(argv[0]);
+	exit(1);
+}
diff --git a/lib/libc/tests/gen/wordexp_test.c b/lib/libc/tests/gen/wordexp_test.c
new file mode 100644
index 0000000..3ccc677
--- /dev/null
+++ b/lib/libc/tests/gen/wordexp_test.c
@@ -0,0 +1,360 @@
+/*-
+ * Copyright (c) 2003 Tim J. Robbins
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * Test program for wordexp() and wordfree() as specified by
+ * IEEE Std. 1003.1-2001.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/wait.h>
+#include <errno.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <wordexp.h>
+
+#include <atf-c.h>
+
+static void
+chld_handler(int x)
+{
+	int status, serrno;
+
+	(void)x;
+	serrno = errno;
+	while (waitpid(-1, &status, WNOHANG) > 0)
+		;
+	errno = serrno;
+}
+
+ATF_TC_WITHOUT_HEAD(simple_test);
+ATF_TC_BODY(simple_test, tc)
+{
+	wordexp_t we;
+	int r;
+
+	/* Test that the macros are there. */
+	(void)(WRDE_APPEND + WRDE_DOOFFS + WRDE_NOCMD + WRDE_REUSE +
+	    WRDE_SHOWERR + WRDE_UNDEF);
+	(void)(WRDE_BADCHAR + WRDE_BADVAL + WRDE_CMDSUB + WRDE_NOSPACE +
+	    WRDE_SYNTAX);
+
+	/* Simple test. */
+	r = wordexp("hello world", &we, 0);
+	ATF_REQUIRE(r == 0);
+	ATF_REQUIRE(we.we_wordc == 2);
+	ATF_REQUIRE(strcmp(we.we_wordv[0], "hello") == 0);
+	ATF_REQUIRE(strcmp(we.we_wordv[1], "world") == 0);
+	ATF_REQUIRE(we.we_wordv[2] == NULL);
+	wordfree(&we);
+}
+
+ATF_TC_WITHOUT_HEAD(long_output_test);
+ATF_TC_BODY(long_output_test, tc)
+{
+	char longdata[6 * 10000 + 1];
+	wordexp_t we;
+	int i, r;
+
+	/* Long output. */
+	for (i = 0; i < 10000; i++)
+		snprintf(longdata + 6 * i, 7, "%05d ", i);
+	r = wordexp(longdata, &we, 0);
+	ATF_REQUIRE(r == 0);
+	ATF_REQUIRE(we.we_wordc == 10000);
+	ATF_REQUIRE(we.we_wordv[10000] == NULL);
+	wordfree(&we);
+}
+
+ATF_TC_WITHOUT_HEAD(WRDE_DOOFFS_test);
+ATF_TC_BODY(WRDE_DOOFFS_test, tc)
+{
+	wordexp_t we;
+	int r;
+
+	we.we_offs = 3;
+	r = wordexp("hello world", &we, WRDE_DOOFFS);
+	ATF_REQUIRE(r == 0);
+	ATF_REQUIRE(we.we_wordc == 2);
+	ATF_REQUIRE(we.we_wordv[0] == NULL);
+	ATF_REQUIRE(we.we_wordv[1] == NULL);
+	ATF_REQUIRE(we.we_wordv[2] == NULL);
+	ATF_REQUIRE(strcmp(we.we_wordv[3], "hello") == 0);
+	ATF_REQUIRE(strcmp(we.we_wordv[4], "world") == 0);
+	ATF_REQUIRE(we.we_wordv[5] == NULL);
+	wordfree(&we);
+}
+
+ATF_TC_WITHOUT_HEAD(WRDE_REUSE_test);
+ATF_TC_BODY(WRDE_REUSE_test, tc)
+{
+	wordexp_t we;
+	int r;
+
+	r = wordexp("hello world", &we, 0);
+	r = wordexp("hello world", &we, WRDE_REUSE);
+	ATF_REQUIRE(r == 0);
+	ATF_REQUIRE(we.we_wordc == 2);
+	ATF_REQUIRE(strcmp(we.we_wordv[0], "hello") == 0);
+	ATF_REQUIRE(strcmp(we.we_wordv[1], "world") == 0);
+	ATF_REQUIRE(we.we_wordv[2] == NULL);
+	wordfree(&we);
+}
+
+ATF_TC_WITHOUT_HEAD(WRDE_APPEND_test);
+ATF_TC_BODY(WRDE_APPEND_test, tc)
+{
+	wordexp_t we;
+	int r;
+
+	r = wordexp("this is", &we, 0);
+	ATF_REQUIRE(r == 0);
+	r = wordexp("a test", &we, WRDE_APPEND);
+	ATF_REQUIRE(r == 0);
+	ATF_REQUIRE(we.we_wordc == 4);
+	ATF_REQUIRE(strcmp(we.we_wordv[0], "this") == 0);
+	ATF_REQUIRE(strcmp(we.we_wordv[1], "is") == 0);
+	ATF_REQUIRE(strcmp(we.we_wordv[2], "a") == 0);
+	ATF_REQUIRE(strcmp(we.we_wordv[3], "test") == 0);
+	ATF_REQUIRE(we.we_wordv[4] == NULL);
+	wordfree(&we);
+}
+
+ATF_TC_WITHOUT_HEAD(WRDE_DOOFFS__WRDE_APPEND_test);
+ATF_TC_BODY(WRDE_DOOFFS__WRDE_APPEND_test, tc)
+{
+	wordexp_t we;
+	int r;
+
+	we.we_offs = 2;
+	r = wordexp("this is", &we, WRDE_DOOFFS);
+	ATF_REQUIRE(r == 0);
+	r = wordexp("a test", &we, WRDE_APPEND|WRDE_DOOFFS);
+	ATF_REQUIRE(r == 0);
+	r = wordexp("of wordexp", &we, WRDE_APPEND|WRDE_DOOFFS);
+	ATF_REQUIRE(r == 0);
+	ATF_REQUIRE(we.we_wordc == 6);
+	ATF_REQUIRE(we.we_wordv[0] == NULL);
+	ATF_REQUIRE(we.we_wordv[1] == NULL);
+	ATF_REQUIRE(strcmp(we.we_wordv[2], "this") == 0);
+	ATF_REQUIRE(strcmp(we.we_wordv[3], "is") == 0);
+	ATF_REQUIRE(strcmp(we.we_wordv[4], "a") == 0);
+	ATF_REQUIRE(strcmp(we.we_wordv[5], "test") == 0);
+	ATF_REQUIRE(strcmp(we.we_wordv[6], "of") == 0);
+	ATF_REQUIRE(strcmp(we.we_wordv[7], "wordexp") == 0);
+	ATF_REQUIRE(we.we_wordv[8] == NULL);
+	wordfree(&we);
+}
+
+ATF_TC_WITHOUT_HEAD(WRDE_UNDEF_test);
+ATF_TC_BODY(WRDE_UNDEF_test, tc)
+{
+	wordexp_t we;
+	int r;
+
+	r = wordexp("${dont_set_me}", &we, WRDE_UNDEF);
+	ATF_REQUIRE(r == WRDE_BADVAL);
+}
+
+ATF_TC_WITHOUT_HEAD(WRDE_NOCMD_test);
+ATF_TC_BODY(WRDE_NOCMD_test, tc)
+{
+	wordexp_t we;
+	int r;
+
+	r = wordexp("`date`", &we, WRDE_NOCMD);
+	ATF_REQUIRE(r == WRDE_CMDSUB);
+	r = wordexp("\"`date`\"", &we, WRDE_NOCMD);
+	ATF_REQUIRE(r == WRDE_CMDSUB);
+	r = wordexp("$(date)", &we, WRDE_NOCMD);
+	ATF_REQUIRE(r == WRDE_CMDSUB);
+	r = wordexp("\"$(date)\"", &we, WRDE_NOCMD);
+	ATF_REQUIRE(r == WRDE_CMDSUB);
+	r = wordexp("$((3+5))", &we, WRDE_NOCMD);
+	ATF_REQUIRE(r == 0);
+	r = wordexp("\\$\\(date\\)", &we, WRDE_NOCMD|WRDE_REUSE);
+	ATF_REQUIRE(r == 0);
+	r = wordexp("'`date`'", &we, WRDE_NOCMD|WRDE_REUSE);
+	ATF_REQUIRE(r == 0);
+	r = wordexp("'$(date)'", &we, WRDE_NOCMD|WRDE_REUSE);
+	ATF_REQUIRE(r == 0);
+	wordfree(&we);
+}
+
+ATF_TC_WITHOUT_HEAD(WRDE_BADCHAR_test);
+ATF_TC_BODY(WRDE_BADCHAR_test, tc)
+{
+	wordexp_t we;
+	int r;
+
+	r = wordexp("'\n|&;<>(){}'", &we, 0);
+	ATF_REQUIRE(r == 0);
+	r = wordexp("\"\n|&;<>(){}\"", &we, WRDE_REUSE);
+	ATF_REQUIRE(r == 0);
+	r = wordexp("\\\n\\|\\&\\;\\<\\>\\(\\)\\{\\}", &we, WRDE_REUSE);
+	ATF_REQUIRE(r == 0);
+	wordfree(&we);
+	r = wordexp("test \n test", &we, 0);
+	ATF_REQUIRE(r == WRDE_BADCHAR);
+	r = wordexp("test | test", &we, 0);
+	ATF_REQUIRE(r == WRDE_BADCHAR);
+	r = wordexp("test & test", &we, 0);
+	ATF_REQUIRE(r == WRDE_BADCHAR);
+	r = wordexp("test ; test", &we, 0);
+	ATF_REQUIRE(r == WRDE_BADCHAR);
+	r = wordexp("test > test", &we, 0);
+	ATF_REQUIRE(r == WRDE_BADCHAR);
+	r = wordexp("test < test", &we, 0);
+	ATF_REQUIRE(r == WRDE_BADCHAR);
+	r = wordexp("test ( test", &we, 0);
+	ATF_REQUIRE(r == WRDE_BADCHAR);
+	r = wordexp("test ) test", &we, 0);
+	ATF_REQUIRE(r == WRDE_BADCHAR);
+	r = wordexp("test { test", &we, 0);
+	ATF_REQUIRE(r == WRDE_BADCHAR);
+	r = wordexp("test } test", &we, 0);
+	ATF_REQUIRE(r == WRDE_BADCHAR);
+}
+
+ATF_TC_WITHOUT_HEAD(WRDE_SYNTAX_test);
+ATF_TC_BODY(WRDE_SYNTAX_test, tc)
+{
+	wordexp_t we;
+	int r;
+
+	r = wordexp("'", &we, 0);
+	ATF_REQUIRE(r == WRDE_SYNTAX);
+	r = wordexp("'", &we, WRDE_UNDEF);
+	ATF_REQUIRE(r == WRDE_SYNTAX);
+	r = wordexp("'\\'", &we, 0);
+	ATF_REQUIRE(r == 0);
+	ATF_REQUIRE(we.we_wordc == 1);
+	ATF_REQUIRE(strcmp(we.we_wordv[0], "\\") == 0);
+	ATF_REQUIRE(we.we_wordv[1] == NULL);
+	wordfree(&we);
+	/* Two syntax errors that are not detected by the current we_check(). */
+	r = wordexp("${IFS:+'}", &we, 0);
+	ATF_REQUIRE(r == WRDE_SYNTAX);
+	r = wordexp("${IFS:+'}", &we, WRDE_UNDEF);
+	ATF_REQUIRE(r == WRDE_SYNTAX);
+	r = wordexp("$(case)", &we, 0);
+	ATF_REQUIRE(r == WRDE_SYNTAX);
+	r = wordexp("$(case)", &we, WRDE_UNDEF);
+	ATF_REQUIRE(r == WRDE_SYNTAX);
+}
+
+ATF_TC_WITHOUT_HEAD(with_SIGCHILD_handler_test);
+ATF_TC_BODY(with_SIGCHILD_handler_test, tc)
+{
+	struct sigaction sa;
+	wordexp_t we;
+	int r;
+
+	/* With a SIGCHLD handler that reaps all zombies. */
+	sa.sa_flags = 0;
+	sigemptyset(&sa.sa_mask);
+	sa.sa_handler = chld_handler;
+	r = sigaction(SIGCHLD, &sa, NULL);
+	ATF_REQUIRE(r == 0);
+	r = wordexp("hello world", &we, 0);
+	ATF_REQUIRE(r == 0);
+	ATF_REQUIRE(we.we_wordc == 2);
+	ATF_REQUIRE(strcmp(we.we_wordv[0], "hello") == 0);
+	ATF_REQUIRE(strcmp(we.we_wordv[1], "world") == 0);
+	ATF_REQUIRE(we.we_wordv[2] == NULL);
+	wordfree(&we);
+	sa.sa_handler = SIG_DFL;
+	r = sigaction(SIGCHLD, &sa, NULL);
+	ATF_REQUIRE(r == 0);
+}
+
+ATF_TC_WITHOUT_HEAD(with_unused_non_default_IFS_test);
+ATF_TC_BODY(with_unused_non_default_IFS_test, tc)
+{
+	wordexp_t we;
+	int r;
+
+	/*
+	 * With IFS set to a non-default value (without depending on whether
+	 * IFS is inherited or not).
+	 */
+	r = setenv("IFS", ":", 1);
+	ATF_REQUIRE(r == 0);
+	r = wordexp("hello world", &we, 0);
+	ATF_REQUIRE(r == 0);
+	ATF_REQUIRE(we.we_wordc == 2);
+	ATF_REQUIRE(strcmp(we.we_wordv[0], "hello") == 0);
+	ATF_REQUIRE(strcmp(we.we_wordv[1], "world") == 0);
+	ATF_REQUIRE(we.we_wordv[2] == NULL);
+	wordfree(&we);
+	r = unsetenv("IFS");
+	ATF_REQUIRE(r == 0);
+}
+
+ATF_TC_WITHOUT_HEAD(with_used_non_default_IFS_test);
+ATF_TC_BODY(with_used_non_default_IFS_test, tc)
+{
+	wordexp_t we;
+	int r;
+
+	/*
+	 * With IFS set to a non-default value, and using it.
+	 */
+	r = setenv("IFS", ":", 1);
+	ATF_REQUIRE(r == 0);
+	r = wordexp("${IFS+hello:world}", &we, 0);
+	ATF_REQUIRE(r == 0);
+	ATF_REQUIRE(we.we_wordc == 2);
+	ATF_REQUIRE(strcmp(we.we_wordv[0], "hello") == 0);
+	ATF_REQUIRE(strcmp(we.we_wordv[1], "world") == 0);
+	ATF_REQUIRE(we.we_wordv[2] == NULL);
+	wordfree(&we);
+	r = unsetenv("IFS");
+	ATF_REQUIRE(r == 0);
+}
+
+ATF_TP_ADD_TCS(tp)
+{
+	ATF_TP_ADD_TC(tp, simple_test);
+	ATF_TP_ADD_TC(tp, long_output_test);
+	ATF_TP_ADD_TC(tp, WRDE_DOOFFS_test);
+	ATF_TP_ADD_TC(tp, WRDE_REUSE_test);
+	ATF_TP_ADD_TC(tp, WRDE_APPEND_test);
+	ATF_TP_ADD_TC(tp, WRDE_DOOFFS__WRDE_APPEND_test);
+	ATF_TP_ADD_TC(tp, WRDE_UNDEF_test);
+	ATF_TP_ADD_TC(tp, WRDE_NOCMD_test);
+	ATF_TP_ADD_TC(tp, WRDE_BADCHAR_test);
+	ATF_TP_ADD_TC(tp, WRDE_SYNTAX_test);
+	ATF_TP_ADD_TC(tp, with_SIGCHILD_handler_test);
+	ATF_TP_ADD_TC(tp, with_unused_non_default_IFS_test);
+	ATF_TP_ADD_TC(tp, with_used_non_default_IFS_test);
+
+	return (atf_no_error());
+}
diff --git a/lib/libc/tests/locale/Makefile b/lib/libc/tests/locale/Makefile
index e4dc553..d15fd5e 100644
--- a/lib/libc/tests/locale/Makefile
+++ b/lib/libc/tests/locale/Makefile
@@ -4,6 +4,24 @@
 
 TESTSDIR=	${TESTSBASE}/lib/libc/locale
 
+ATF_TESTS_C+=		btowc_test
+ATF_TESTS_C+=		c16rtomb_test
+ATF_TESTS_C+=		iswctype_test
+ATF_TESTS_C+=		mblen_test
+ATF_TESTS_C+=		mbrlen_test
+ATF_TESTS_C+=		mbrtoc16_test
+ATF_TESTS_C+=		mbrtowc_2_test
+ATF_TESTS_C+=		mbsnrtowcs_2_test
+ATF_TESTS_C+=		mbsrtowcs_test
+ATF_TESTS_C+=		mbstowcs_2_test
+ATF_TESTS_C+=		mbtowc_2_test
+ATF_TESTS_C+=		towctrans_test
+ATF_TESTS_C+=		wcrtomb_test
+ATF_TESTS_C+=		wcsnrtombs_test
+ATF_TESTS_C+=		wcsrtombs_test
+ATF_TESTS_C+=		wcstombs_test
+ATF_TESTS_C+=		wctomb_2_test
+
 NETBSD_ATF_TESTS_C=	io_test
 NETBSD_ATF_TESTS_C+=	mbrtowc_test
 NETBSD_ATF_TESTS_C+=	mbstowcs_test
@@ -15,7 +33,13 @@ NETBSD_ATF_TESTS_C+=	wcsspn_test
 NETBSD_ATF_TESTS_C+=	wcstod_test
 NETBSD_ATF_TESTS_C+=	wctomb_test
 
-CFLAGS.t_wctomb.c+= -Wno-stack-protector
+SRCS.mbrtowc_2_test=	mbrtowc_test.c
+SRCS.mbsnrtowcs_2_test=	mbsnrtowcs_test.c
+SRCS.mbstowcs_2_test=	mbstowcs_test.c
+SRCS.mbtowc_2_test=	mbtowc_test.c
+SRCS.wctomb_2_test=	wctomb_test.c
+
+CFLAGS.t_wctomb.c+=	-Wno-stack-protector
 
 .include "../Makefile.netbsd-tests"
 
diff --git a/tools/regression/lib/libc/locale/test-btowc.c b/lib/libc/tests/locale/btowc_test.c
index b01c043..01a1133 100644
--- a/tools/regression/lib/libc/locale/test-btowc.c
+++ b/lib/libc/tests/locale/btowc_test.c
@@ -34,7 +34,6 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
-#include <assert.h>
 #include <limits.h>
 #include <locale.h>
 #include <stdio.h>
@@ -42,32 +41,31 @@ __FBSDID("$FreeBSD$");
 #include <string.h>
 #include <wchar.h>
 
-int
-main(int argc, char *argv[])
+#include <atf-c.h>
+
+ATF_TC_WITHOUT_HEAD(btowc_test);
+ATF_TC_BODY(btowc_test, tc)
 {
 	int i;
 
-	printf("1..2\n");
-
-	/*
-	 * C/POSIX locale.
-	 */
-	assert(btowc(EOF) == WEOF);
-	assert(wctob(WEOF) == EOF);
+	/* C/POSIX locale. */
+	ATF_REQUIRE(btowc(EOF) == WEOF);
+	ATF_REQUIRE(wctob(WEOF) == EOF);
 	for (i = 0; i < UCHAR_MAX; i++)
-		assert(btowc(i) == (wchar_t)i && i == (int)wctob(i));
+		ATF_REQUIRE(btowc(i) == (wchar_t)i && i == (int)wctob(i));
 
-	/*
-	 * Japanese (EUC) locale.
-	 */
+	/* Japanese (EUC) locale. */
+	ATF_REQUIRE(strcmp(setlocale(LC_CTYPE, "ja_JP.eucJP"), "ja_JP.eucJP") == 0);
+	ATF_REQUIRE(MB_CUR_MAX > 1);
+	ATF_REQUIRE(btowc('A') == L'A' && wctob(L'A') == 'A');
+	ATF_REQUIRE(btowc(0xa3) == WEOF && wctob(0xa3c1) == EOF);
 
-	assert(strcmp(setlocale(LC_CTYPE, "ja_JP.eucJP"), "ja_JP.eucJP") == 0);
-	assert(MB_CUR_MAX > 1);
-	assert(btowc('A') == L'A' && wctob(L'A') == 'A');
-	assert(btowc(0xa3) == WEOF && wctob(0xa3c1) == EOF);
+}
+
+ATF_TP_ADD_TCS(tp)
+{
 
-	printf("ok 1 - btowc()\n");
-	printf("ok 2 - wctob()\n");
+	ATF_TP_ADD_TC(tp, btowc_test);
 
-	return (0);
+	return (atf_no_error());
 }
diff --git a/tools/regression/lib/libc/locale/test-c16rtomb.c b/lib/libc/tests/locale/c16rtomb_test.c
index 2c188fa..ea1f64f 100644
--- a/tools/regression/lib/libc/locale/test-c16rtomb.c
+++ b/lib/libc/tests/locale/c16rtomb_test.c
@@ -33,7 +33,6 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
-#include <assert.h>
 #include <errno.h>
 #include <limits.h>
 #include <locale.h>
@@ -41,105 +40,127 @@ __FBSDID("$FreeBSD$");
 #include <string.h>
 #include <uchar.h>
 
-int
-main(int argc, char *argv[])
+#include <atf-c.h>
+
+static void
+require_lc_ctype(const char *locale_name)
 {
-	mbstate_t s;
-	char buf[MB_LEN_MAX + 1];
+	char *lc_ctype_set;
 
-	/*
-	 * C/POSIX locale.
-	 */
+	lc_ctype_set = setlocale(LC_CTYPE, locale_name);
+	if (lc_ctype_set == NULL)
+		atf_tc_fail("setlocale(LC_CTYPE, \"%s\") failed; errno=%d",
+		    locale_name, errno);
+
+	ATF_REQUIRE(strcmp(lc_ctype_set, locale_name) == 0);
+}
+
+static mbstate_t s;
+static char buf[MB_LEN_MAX + 1];
 
-	printf("1..1\n");
+ATF_TC_WITHOUT_HEAD(c16rtomb_c_locale_test);
+ATF_TC_BODY(c16rtomb_c_locale_test, tc)
+{
+
+	require_lc_ctype("C");
 
 	/*
 	 * If the buffer argument is NULL, c16 is implicitly 0,
 	 * c16rtomb() resets its internal state.
 	 */
-	assert(c16rtomb(NULL, L'\0', NULL) == 1);
-	assert(c16rtomb(NULL, 0xdc00, NULL) == 1);
+	ATF_REQUIRE(c16rtomb(NULL, L'\0', NULL) == 1);
+	ATF_REQUIRE(c16rtomb(NULL, 0xdc00, NULL) == 1);
 
 	/* Null wide character. */
 	memset(&s, 0, sizeof(s));
 	memset(buf, 0xcc, sizeof(buf));
-	assert(c16rtomb(buf, 0, &s) == 1);
-	assert((unsigned char)buf[0] == 0 && (unsigned char)buf[1] == 0xcc);
+	ATF_REQUIRE(c16rtomb(buf, 0, &s) == 1);
+	ATF_REQUIRE((unsigned char)buf[0] == 0 && (unsigned char)buf[1] == 0xcc);
 
 	/* Latin letter A, internal state. */
-	assert(c16rtomb(NULL, L'\0', NULL) == 1);
-	assert(c16rtomb(NULL, L'A', NULL) == 1);
+	ATF_REQUIRE(c16rtomb(NULL, L'\0', NULL) == 1);
+	ATF_REQUIRE(c16rtomb(NULL, L'A', NULL) == 1);
 
 	/* Latin letter A. */
 	memset(&s, 0, sizeof(s));
 	memset(buf, 0xcc, sizeof(buf));
-	assert(c16rtomb(buf, L'A', &s) == 1);
-	assert((unsigned char)buf[0] == 'A' && (unsigned char)buf[1] == 0xcc);
+	ATF_REQUIRE(c16rtomb(buf, L'A', &s) == 1);
+	ATF_REQUIRE((unsigned char)buf[0] == 'A' && (unsigned char)buf[1] == 0xcc);
 
 	/* Unicode character 'Pile of poo'. */
 	memset(&s, 0, sizeof(s));
 	memset(buf, 0xcc, sizeof(buf));
-	assert(c16rtomb(buf, 0xd83d, &s) == 0);
-	assert(c16rtomb(buf, 0xdca9, &s) == (size_t)-1);
-	assert(errno == EILSEQ);
-	assert((unsigned char)buf[0] == 0xcc);
+	ATF_REQUIRE(c16rtomb(buf, 0xd83d, &s) == 0);
+	ATF_REQUIRE(c16rtomb(buf, 0xdca9, &s) == (size_t)-1);
+	ATF_REQUIRE(errno == EILSEQ);
+	ATF_REQUIRE((unsigned char)buf[0] == 0xcc);
+}
 
-	/*
-	 * ISO8859-1.
-	 */
+ATF_TC_WITHOUT_HEAD(c16rtomb_iso_8859_1_test);
+ATF_TC_BODY(c16rtomb_iso_8859_1_test, tc)
+{
 
-	assert(strcmp(setlocale(LC_CTYPE, "en_US.ISO8859-1"),
-	    "en_US.ISO8859-1") == 0);
+	require_lc_ctype("en_US.ISO8859-1");
 
 	/* Unicode character 'Euro sign'. */
 	memset(&s, 0, sizeof(s));
 	memset(buf, 0xcc, sizeof(buf));
-	assert(c16rtomb(buf, 0x20ac, &s) == (size_t)-1);
-	assert(errno == EILSEQ);
-	assert((unsigned char)buf[0] == 0xcc);
+	ATF_REQUIRE(c16rtomb(buf, 0x20ac, &s) == (size_t)-1);
+	ATF_REQUIRE(errno == EILSEQ);
+	ATF_REQUIRE((unsigned char)buf[0] == 0xcc);
+}
 
-	/*
-	 * ISO8859-15.
-	 */
+ATF_TC_WITHOUT_HEAD(c16rtomb_iso_8859_15_test);
+ATF_TC_BODY(c16rtomb_iso_8859_15_test, tc)
+{
 
-	assert(strcmp(setlocale(LC_CTYPE, "en_US.ISO8859-15"),
-	    "en_US.ISO8859-15") == 0);
+	require_lc_ctype("en_US.ISO8859-15");
 
 	/* Unicode character 'Euro sign'. */
 	memset(&s, 0, sizeof(s));
 	memset(buf, 0xcc, sizeof(buf));
-	assert(c16rtomb(buf, 0x20ac, &s) == 1);
-	assert((unsigned char)buf[0] == 0xa4 && (unsigned char)buf[1] == 0xcc);
+	ATF_REQUIRE(c16rtomb(buf, 0x20ac, &s) == 1);
+	ATF_REQUIRE((unsigned char)buf[0] == 0xa4 && (unsigned char)buf[1] == 0xcc);
+}
 
-	/*
-	 * UTF-8.
-	 */
+ATF_TC_WITHOUT_HEAD(c16rtomb_utf_8_test);
+ATF_TC_BODY(c16rtomb_utf_8_test, tc)
+{
 
-	assert(strcmp(setlocale(LC_CTYPE, "en_US.UTF-8"), "en_US.UTF-8") == 0);
+	require_lc_ctype("en_US.UTF-8");
 
 	/* Unicode character 'Pile of poo'. */
 	memset(&s, 0, sizeof(s));
 	memset(buf, 0xcc, sizeof(buf));
-	assert(c16rtomb(buf, 0xd83d, &s) == 0);
-	assert(c16rtomb(buf, 0xdca9, &s) == 4);
-	assert((unsigned char)buf[0] == 0xf0 && (unsigned char)buf[1] == 0x9f &&
+	ATF_REQUIRE(c16rtomb(buf, 0xd83d, &s) == 0);
+	ATF_REQUIRE(c16rtomb(buf, 0xdca9, &s) == 4);
+	ATF_REQUIRE((unsigned char)buf[0] == 0xf0 && (unsigned char)buf[1] == 0x9f &&
 	    (unsigned char)buf[2] == 0x92 && (unsigned char)buf[3] == 0xa9 &&
 	    (unsigned char)buf[4] == 0xcc);
 
 	/* Invalid code; 'Pile of poo' without the trail surrogate. */
 	memset(&s, 0, sizeof(s));
 	memset(buf, 0xcc, sizeof(buf));
-	assert(c16rtomb(buf, 0xd83d, &s) == 0);
-	assert(c16rtomb(buf, L'A', &s) == (size_t)-1);
-	assert(errno == EILSEQ);
-	assert((unsigned char)buf[0] == 0xcc);
+	ATF_REQUIRE(c16rtomb(buf, 0xd83d, &s) == 0);
+	ATF_REQUIRE(c16rtomb(buf, L'A', &s) == (size_t)-1);
+	ATF_REQUIRE(errno == EILSEQ);
+	ATF_REQUIRE((unsigned char)buf[0] == 0xcc);
 
 	/* Invalid code; 'Pile of poo' without the lead surrogate. */
 	memset(&s, 0, sizeof(s));
 	memset(buf, 0xcc, sizeof(buf));
-	assert(c16rtomb(buf, 0xdca9, &s) == (size_t)-1);
-	assert(errno == EILSEQ);
-	assert((unsigned char)buf[0] == 0xcc);
+	ATF_REQUIRE(c16rtomb(buf, 0xdca9, &s) == (size_t)-1);
+	ATF_REQUIRE(errno == EILSEQ);
+	ATF_REQUIRE((unsigned char)buf[0] == 0xcc);
+}
+
+ATF_TP_ADD_TCS(tp)
+{
+
+	ATF_TP_ADD_TC(tp, c16rtomb_c_locale_test);
+	ATF_TP_ADD_TC(tp, c16rtomb_iso_8859_1_test);
+	ATF_TP_ADD_TC(tp, c16rtomb_iso_8859_15_test);
+	ATF_TP_ADD_TC(tp, c16rtomb_utf_8_test);
 
-	printf("ok 1 - c16rtomb()\n");
+	return (atf_no_error());
 }
diff --git a/tools/regression/lib/libc/locale/test-iswctype.c b/lib/libc/tests/locale/iswctype_test.c
index 1c56b88..14e57b4 100644
--- a/tools/regression/lib/libc/locale/test-iswctype.c
+++ b/lib/libc/tests/locale/iswctype_test.c
@@ -34,69 +34,89 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
-#include <assert.h>
+#include <sys/param.h>
+#include <errno.h>
 #include <locale.h>
 #include <stdio.h>
 #include <string.h>
 #include <wchar.h>
 #include <wctype.h>
 
-int
-main(int argc, char *argv[])
+#include <atf-c.h>
+
+static void
+require_lc_ctype(const char *locale_name)
+{
+	char *lc_ctype_set;
+
+	lc_ctype_set = setlocale(LC_CTYPE, locale_name);
+	if (lc_ctype_set == NULL)
+		atf_tc_fail("setlocale(LC_CTYPE, \"%s\") failed; errno=%d",
+		    locale_name, errno);
+
+	ATF_REQUIRE(strcmp(lc_ctype_set, locale_name) == 0);
+}
+
+static wctype_t t;
+static int i, j;
+static struct {
+	const char *name;
+	int (*func)(wint_t);
+} cls[] = {
+	{ "alnum", iswalnum },
+	{ "alpha", iswalpha },
+	{ "blank", iswblank },
+	{ "cntrl", iswcntrl },
+	{ "digit", iswdigit },
+	{ "graph", iswgraph },
+	{ "lower", iswlower },
+	{ "print", iswprint },
+	{ "punct", iswpunct },
+	{ "space", iswspace },
+	{ "upper", iswupper },
+	{ "xdigit", iswxdigit }
+};
+
+ATF_TC_WITHOUT_HEAD(iswctype_c_locale_test);
+ATF_TC_BODY(iswctype_c_locale_test, tc)
 {
-	wctype_t t;
-	int i, j;
-	struct {
-		const char *name;
-		int (*func)(wint_t);
-	} cls[] = {
-		{ "alnum", iswalnum },
-		{ "alpha", iswalpha },
-		{ "blank", iswblank },
-		{ "cntrl", iswcntrl },
-		{ "digit", iswdigit },
-		{ "graph", iswgraph },
-		{ "lower", iswlower },
-		{ "print", iswprint },
-		{ "punct", iswpunct },
-		{ "space", iswspace },
-		{ "upper", iswupper },
-		{ "xdigit", iswxdigit }
-	};
-
-	printf("1..2\n");
-
-	/*
-	 * C/POSIX locale.
-	 */
-	for (i = 0; i < sizeof(cls) / sizeof(*cls); i++) {
+
+	require_lc_ctype("C");
+	for (i = 0; i < nitems(cls); i++) {
 		t = wctype(cls[i].name);
-		assert(t != 0);
+		ATF_REQUIRE(t != 0);
 		for (j = 0; j < 256; j++)
-			assert(cls[i].func(j) == iswctype(j, t));
+			ATF_REQUIRE(cls[i].func(j) == iswctype(j, t));
 	}
 	t = wctype("elephant");
-	assert(t == 0);
+	ATF_REQUIRE(t == 0);
 	for (i = 0; i < 256; i++)
-		assert(iswctype(i, t) == 0);
+		ATF_REQUIRE(iswctype(i, t) == 0);
+}
+
+ATF_TC_WITHOUT_HEAD(iswctype_euc_jp_test);
+ATF_TC_BODY(iswctype_euc_jp_test, tc)
+{
+
+	require_lc_ctype("ja_JP.eucJP");
 
-	/*
-	 * Japanese (EUC) locale.
-	 */
-	assert(strcmp(setlocale(LC_CTYPE, "ja_JP.eucJP"), "ja_JP.eucJP") == 0);
-	for (i = 0; i < sizeof(cls) / sizeof(*cls); i++) {
+	for (i = 0; i < nitems(cls); i++) {
 		t = wctype(cls[i].name);
-		assert(t != 0);
+		ATF_REQUIRE(t != 0);
 		for (j = 0; j < 65536; j++)
-			assert(cls[i].func(j) == iswctype(j, t));
+			ATF_REQUIRE(cls[i].func(j) == iswctype(j, t));
 	}
 	t = wctype("elephant");
-	assert(t == 0);
+	ATF_REQUIRE(t == 0);
 	for (i = 0; i < 65536; i++)
-		assert(iswctype(i, t) == 0);
+		ATF_REQUIRE(iswctype(i, t) == 0);
+}
+
+ATF_TP_ADD_TCS(tp)
+{
 
-	printf("ok 1 - iswctype()\n");
-	printf("ok 2 - wctype()\n");
+	ATF_TP_ADD_TC(tp, iswctype_c_locale_test);
+	ATF_TP_ADD_TC(tp, iswctype_euc_jp_test);
 
-	return (0);
+	return (atf_no_error());
 }
diff --git a/tools/regression/lib/libc/locale/test-mblen.c b/lib/libc/tests/locale/mblen_test.c
index dd07359..fbb938d 100644
--- a/tools/regression/lib/libc/locale/test-mblen.c
+++ b/lib/libc/tests/locale/mblen_test.c
@@ -35,80 +35,82 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
-#include <assert.h>
 #include <limits.h>
 #include <locale.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 
-int
-main(int argc, char *argv[])
+#include <atf-c.h>
+
+ATF_TC_WITHOUT_HEAD(mblen_test);
+ATF_TC_BODY(mblen_test, tc)
 {
-	size_t len;
 	char buf[MB_LEN_MAX + 1];
 
 	/*
 	 * C/POSIX locale.
 	 */
 
-	printf("1..1\n");
-
-	assert(MB_CUR_MAX == 1);
+	ATF_REQUIRE(MB_CUR_MAX == 1);
 
 	/* No shift states in C locale. */
-	assert(mblen(NULL, 0) == 0);
+	ATF_REQUIRE(mblen(NULL, 0) == 0);
 
 	/* Null wide character. */
 	memset(buf, 0xcc, sizeof(buf));
 	buf[0] = '\0';
-	assert(mblen(buf, 1) == 0);
+	ATF_REQUIRE(mblen(buf, 1) == 0);
 
 	/* Latin letter A. */
 	buf[0] = 'A';
-	assert(mblen(buf, 1) == 1);
+	ATF_REQUIRE(mblen(buf, 1) == 1);
 
 	/* Incomplete character sequence. */
 	buf[0] = '\0';
-	assert(mblen(buf, 0) == -1);
-	assert(mblen(NULL, 0) == 0);
+	ATF_REQUIRE(mblen(buf, 0) == -1);
+	ATF_REQUIRE(mblen(NULL, 0) == 0);
 
 	/*
 	 * Japanese (EUC) locale.
 	 */
 
-	assert(strcmp(setlocale(LC_CTYPE, "ja_JP.eucJP"), "ja_JP.eucJP") == 0);
-	assert(MB_CUR_MAX > 1);
+	ATF_REQUIRE(strcmp(setlocale(LC_CTYPE, "ja_JP.eucJP"), "ja_JP.eucJP") == 0);
+	ATF_REQUIRE(MB_CUR_MAX > 1);
 
 	/* No shift states in EUC. */
-	assert(mblen(NULL, 0) == 0);
+	ATF_REQUIRE(mblen(NULL, 0) == 0);
 
 	/* Null wide character. */
 	memset(buf, 0xcc, sizeof(buf));
 	buf[0] = '\0';
-	assert(mblen(buf, 1) == 0);
+	ATF_REQUIRE(mblen(buf, 1) == 0);
 
 	/* Latin letter A. */
 	buf[0] = 'A';
-	assert(mblen(buf, 1) == 1);
+	ATF_REQUIRE(mblen(buf, 1) == 1);
 
 	/* Incomplete character sequence. */
 	buf[0] = '\0';
-	assert(mblen(buf, 0) == -1);
-	assert(mblen(NULL, 0) == 0);
+	ATF_REQUIRE(mblen(buf, 0) == -1);
+	ATF_REQUIRE(mblen(NULL, 0) == 0);
 
 	/* Incomplete character sequence (truncated double-byte). */
 	memset(buf, 0xcc, sizeof(buf));
 	buf[0] = 0xa3;
 	buf[1] = 0x00;
-	assert(mblen(buf, 1) == -1);
-	assert(mblen(NULL, 0) == 0);
+	ATF_REQUIRE(mblen(buf, 1) == -1);
+	ATF_REQUIRE(mblen(NULL, 0) == 0);
 
 	/* Same as above, but complete. */
 	buf[1] = 0xc1;
-	assert(mblen(buf, 2) == 2);
+	ATF_REQUIRE(mblen(buf, 2) == 2);
+}
+
+ATF_TP_ADD_TCS(tp)
+{
 
-	printf("ok 1 - mblen()\n");
+	ATF_TP_ADD_TC(tp, mblen_test);
 
-	return (0);
+	return (atf_no_error());
 }
diff --git a/tools/regression/lib/libc/locale/test-mbrlen.c b/lib/libc/tests/locale/mbrlen_test.c
index 3198a77..c2bc548 100644
--- a/tools/regression/lib/libc/locale/test-mbrlen.c
+++ b/lib/libc/tests/locale/mbrlen_test.c
@@ -35,7 +35,6 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
-#include <assert.h>
 #include <errno.h>
 #include <limits.h>
 #include <locale.h>
@@ -44,86 +43,84 @@ __FBSDID("$FreeBSD$");
 #include <string.h>
 #include <wchar.h>
 
-int
-main(int argc, char *argv[])
+#include <atf-c.h>
+
+ATF_TC_WITHOUT_HEAD(mbrlen_test);
+ATF_TC_BODY(mbrlen_test, tc)
 {
 	mbstate_t s;
-	size_t len;
 	char buf[MB_LEN_MAX + 1];
 
-	/*
-	 * C/POSIX locale.
-	 */
-	
-	printf("1..1\n");
-
-	assert(MB_CUR_MAX == 1);
+	/* C/POSIX locale. */
+	ATF_REQUIRE(MB_CUR_MAX == 1);
 
 	/* Null wide character, internal state. */
 	memset(buf, 0xcc, sizeof(buf));
 	buf[0] = 0;
-	assert(mbrlen(buf, 1, NULL) == 0);
+	ATF_REQUIRE(mbrlen(buf, 1, NULL) == 0);
 
 	/* Null wide character. */
 	memset(&s, 0, sizeof(s));
-	assert(mbrlen(buf, 1, &s) == 0);
+	ATF_REQUIRE(mbrlen(buf, 1, &s) == 0);
 
 	/* Latin letter A, internal state. */
-	assert(mbrlen(NULL, 0, NULL) == 0);
+	ATF_REQUIRE(mbrlen(NULL, 0, NULL) == 0);
 	buf[0] = 'A';
-	assert(mbrlen(buf, 1, NULL) == 1);
+	ATF_REQUIRE(mbrlen(buf, 1, NULL) == 1);
 
 	/* Latin letter A. */
 	memset(&s, 0, sizeof(s));
-	assert(mbrlen(buf, 1, &s) == 1);
+	ATF_REQUIRE(mbrlen(buf, 1, &s) == 1);
 
 	/* Incomplete character sequence. */
 	memset(&s, 0, sizeof(s));
-	assert(mbrlen(buf, 0, &s) == (size_t)-2);
+	ATF_REQUIRE(mbrlen(buf, 0, &s) == (size_t)-2);
 
-	/*
-	 * Japanese (EUC) locale.
-	 */
+	/* Japanese (EUC) locale. */
 
-	assert(strcmp(setlocale(LC_CTYPE, "ja_JP.eucJP"), "ja_JP.eucJP") == 0);
-	assert(MB_CUR_MAX > 1);
+	ATF_REQUIRE(strcmp(setlocale(LC_CTYPE, "ja_JP.eucJP"), "ja_JP.eucJP") == 0);
+	ATF_REQUIRE(MB_CUR_MAX > 1);
 
 	/* Null wide character, internal state. */
-	assert(mbrlen(NULL, 0, NULL) == 0);
+	ATF_REQUIRE(mbrlen(NULL, 0, NULL) == 0);
 	memset(buf, 0xcc, sizeof(buf));
 	buf[0] = 0;
-	assert(mbrlen(buf, 1, NULL) == 0);
+	ATF_REQUIRE(mbrlen(buf, 1, NULL) == 0);
 
 	/* Null wide character. */
 	memset(&s, 0, sizeof(s));
-	assert(mbrlen(buf, 1, &s) == 0);
+	ATF_REQUIRE(mbrlen(buf, 1, &s) == 0);
 
 	/* Latin letter A, internal state. */
-	assert(mbrlen(NULL, 0, NULL) == 0);
+	ATF_REQUIRE(mbrlen(NULL, 0, NULL) == 0);
 	buf[0] = 'A';
-	assert(mbrlen(buf, 1, NULL) == 1);
+	ATF_REQUIRE(mbrlen(buf, 1, NULL) == 1);
 
 	/* Latin letter A. */
 	memset(&s, 0, sizeof(s));
-	assert(mbrlen(buf, 1, &s) == 1);
+	ATF_REQUIRE(mbrlen(buf, 1, &s) == 1);
 
 	/* Incomplete character sequence (zero length). */
 	memset(&s, 0, sizeof(s));
-	assert(mbrlen(buf, 0, &s) == (size_t)-2);
+	ATF_REQUIRE(mbrlen(buf, 0, &s) == (size_t)-2);
 
 	/* Incomplete character sequence (truncated double-byte). */
 	memset(buf, 0xcc, sizeof(buf));
 	buf[0] = 0xa3;
 	buf[1] = 0x00;
 	memset(&s, 0, sizeof(s));
-	assert(mbrlen(buf, 1, &s) == (size_t)-2);
+	ATF_REQUIRE(mbrlen(buf, 1, &s) == (size_t)-2);
 
 	/* Same as above, but complete. */
 	buf[1] = 0xc1;
 	memset(&s, 0, sizeof(s));
-	assert(mbrlen(buf, 2, &s) == 2);
+	ATF_REQUIRE(mbrlen(buf, 2, &s) == 2);
+}
+
+ATF_TP_ADD_TCS(tp)
+{
 
-	printf("ok 1 - mbrlen()\n");
+	ATF_TP_ADD_TC(tp, mbrlen_test);
 
-	return (0);
+	return (atf_no_error());
 }
diff --git a/lib/libc/tests/locale/mbrtoc16_test.c b/lib/libc/tests/locale/mbrtoc16_test.c
new file mode 100644
index 0000000..a6289e1
--- /dev/null
+++ b/lib/libc/tests/locale/mbrtoc16_test.c
@@ -0,0 +1,214 @@
+/*-
+ * Copyright (c) 2002 Tim J. Robbins
+ * All rights reserved.
+ *
+ * Copyright (c) 2013 Ed Schouten <ed@FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+/*
+ * Test program for mbrtoc16() as specified by ISO/IEC 9899:2011.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <errno.h>
+#include <limits.h>
+#include <locale.h>
+#include <stdio.h>
+#include <string.h>
+#include <uchar.h>
+
+#include <atf-c.h>
+
+static void
+require_lc_ctype(const char *locale_name)
+{
+	char *lc_ctype_set;
+
+	lc_ctype_set = setlocale(LC_CTYPE, locale_name);
+	if (lc_ctype_set == NULL)
+		atf_tc_fail("setlocale(LC_CTYPE, \"%s\") failed; errno=%d",
+		    locale_name, errno);
+
+	ATF_REQUIRE(strcmp(lc_ctype_set, locale_name) == 0);
+}
+
+static mbstate_t s;
+static char16_t c16;
+
+ATF_TC_WITHOUT_HEAD(mbrtoc16_c_locale_test);
+ATF_TC_BODY(mbrtoc16_c_locale_test, tc)
+{
+
+	require_lc_ctype("C");
+
+	/* Null wide character, internal state. */
+	ATF_REQUIRE(mbrtoc16(&c16, "", 1, NULL) == 0);
+	ATF_REQUIRE(c16 == 0);
+
+	/* Null wide character. */
+	memset(&s, 0, sizeof(s));
+	ATF_REQUIRE(mbrtoc16(&c16, "", 1, &s) == 0);
+	ATF_REQUIRE(c16 == 0);
+
+	/* Latin letter A, internal state. */
+	ATF_REQUIRE(mbrtoc16(NULL, 0, 0, NULL) == 0);
+	ATF_REQUIRE(mbrtoc16(&c16, "A", 1, NULL) == 1);
+	ATF_REQUIRE(c16 == L'A');
+
+	/* Latin letter A. */
+	memset(&s, 0, sizeof(s));
+	ATF_REQUIRE(mbrtoc16(&c16, "A", 1, &s) == 1);
+	ATF_REQUIRE(c16 == L'A');
+
+	/* Incomplete character sequence. */
+	c16 = L'z';
+	memset(&s, 0, sizeof(s));
+	ATF_REQUIRE(mbrtoc16(&c16, "", 0, &s) == (size_t)-2);
+	ATF_REQUIRE(c16 == L'z');
+
+	/* Check that mbrtoc16() doesn't access the buffer when n == 0. */
+	c16 = L'z';
+	memset(&s, 0, sizeof(s));
+	ATF_REQUIRE(mbrtoc16(&c16, "", 0, &s) == (size_t)-2);
+	ATF_REQUIRE(c16 == L'z');
+
+	/* Check that mbrtoc16() doesn't read ahead too aggressively. */
+	memset(&s, 0, sizeof(s));
+	ATF_REQUIRE(mbrtoc16(&c16, "AB", 2, &s) == 1);
+	ATF_REQUIRE(c16 == L'A');
+	ATF_REQUIRE(mbrtoc16(&c16, "C", 1, &s) == 1);
+	ATF_REQUIRE(c16 == L'C');
+
+}
+
+ATF_TC_WITHOUT_HEAD(mbrtoc16_iso_8859_1_test);
+ATF_TC_BODY(mbrtoc16_iso_8859_1_test, tc)
+{
+
+	require_lc_ctype("en_US.ISO8859-1");
+
+	/* Currency sign. */
+	memset(&s, 0, sizeof(s));
+	ATF_REQUIRE(mbrtoc16(&c16, "\xa4", 1, &s) == 1);
+	ATF_REQUIRE(c16 == 0xa4);
+}
+
+ATF_TC_WITHOUT_HEAD(mbrtoc16_iso_8859_15_test);
+ATF_TC_BODY(mbrtoc16_iso_8859_15_test, tc)
+{
+
+	require_lc_ctype("en_US.ISO8859-15");
+
+	/* Euro sign. */
+	memset(&s, 0, sizeof(s));
+	ATF_REQUIRE(mbrtoc16(&c16, "\xa4", 1, &s) == 1);
+	ATF_REQUIRE(c16 == 0x20ac);
+}
+
+ATF_TC_WITHOUT_HEAD(mbrtoc16_utf_8_test);
+ATF_TC_BODY(mbrtoc16_utf_8_test, tc)
+{
+
+	require_lc_ctype("en_US.UTF-8");
+
+	/* Null wide character, internal state. */
+	ATF_REQUIRE(mbrtoc16(NULL, 0, 0, NULL) == 0);
+	ATF_REQUIRE(mbrtoc16(&c16, "", 1, NULL) == 0);
+	ATF_REQUIRE(c16 == 0);
+
+	/* Null wide character. */
+	memset(&s, 0, sizeof(s));
+	ATF_REQUIRE(mbrtoc16(&c16, "", 1, &s) == 0);
+	ATF_REQUIRE(c16 == 0);
+
+	/* Latin letter A, internal state. */
+	ATF_REQUIRE(mbrtoc16(NULL, 0, 0, NULL) == 0);
+	ATF_REQUIRE(mbrtoc16(&c16, "A", 1, NULL) == 1);
+	ATF_REQUIRE(c16 == L'A');
+
+	/* Latin letter A. */
+	memset(&s, 0, sizeof(s));
+	ATF_REQUIRE(mbrtoc16(&c16, "A", 1, &s) == 1);
+	ATF_REQUIRE(c16 == L'A');
+
+	/* Incomplete character sequence (zero length). */
+	c16 = L'z';
+	memset(&s, 0, sizeof(s));
+	ATF_REQUIRE(mbrtoc16(&c16, "", 0, &s) == (size_t)-2);
+	ATF_REQUIRE(c16 == L'z');
+
+	/* Incomplete character sequence (truncated double-byte). */
+	memset(&s, 0, sizeof(s));
+	c16 = 0;
+	ATF_REQUIRE(mbrtoc16(&c16, "\xc3", 1, &s) == (size_t)-2);
+
+	/* Same as above, but complete. */
+	memset(&s, 0, sizeof(s));
+	c16 = 0;
+	ATF_REQUIRE(mbrtoc16(&c16, "\xc3\x84", 2, &s) == 2);
+	ATF_REQUIRE(c16 == 0xc4);
+
+	/* Test restarting behaviour. */
+	memset(&s, 0, sizeof(s));
+	c16 = 0;
+	ATF_REQUIRE(mbrtoc16(&c16, "\xc3", 1, &s) == (size_t)-2);
+	ATF_REQUIRE(c16 == 0);
+	ATF_REQUIRE(mbrtoc16(&c16, "\xb7", 1, &s) == 1);
+	ATF_REQUIRE(c16 == 0xf7);
+
+	/* Surrogate pair. */
+	memset(&s, 0, sizeof(s));
+	c16 = 0;
+	ATF_REQUIRE(mbrtoc16(&c16, "\xf0\x9f\x92\xa9", 4, &s) == 4);
+	ATF_REQUIRE(c16 == 0xd83d);
+	ATF_REQUIRE(mbrtoc16(&c16, "", 0, &s) == (size_t)-3);
+	ATF_REQUIRE(c16 == 0xdca9);
+
+	/* Letter e with acute, precomposed. */
+	memset(&s, 0, sizeof(s));
+	c16 = 0;
+	ATF_REQUIRE(mbrtoc16(&c16, "\xc3\xa9", 2, &s) == 2);
+	ATF_REQUIRE(c16 == 0xe9);
+
+	/* Letter e with acute, combined. */
+	memset(&s, 0, sizeof(s));
+	c16 = 0;
+	ATF_REQUIRE(mbrtoc16(&c16, "\x65\xcc\x81", 3, &s) == 1);
+	ATF_REQUIRE(c16 == 0x65);
+	ATF_REQUIRE(mbrtoc16(&c16, "\xcc\x81", 2, &s) == 2);
+	ATF_REQUIRE(c16 == 0x301);
+}
+
+ATF_TP_ADD_TCS(tp)
+{
+
+	ATF_TP_ADD_TC(tp, mbrtoc16_c_locale_test);
+	ATF_TP_ADD_TC(tp, mbrtoc16_iso_8859_1_test);
+	ATF_TP_ADD_TC(tp, mbrtoc16_iso_8859_15_test);
+	ATF_TP_ADD_TC(tp, mbrtoc16_utf_8_test);
+
+	return (atf_no_error());
+}
diff --git a/tools/regression/lib/libc/locale/test-mbrtowc.c b/lib/libc/tests/locale/mbrtowc_test.c
index e9a2ea0..37e0bfd 100644
--- a/tools/regression/lib/libc/locale/test-mbrtowc.c
+++ b/lib/libc/tests/locale/mbrtowc_test.c
@@ -35,7 +35,6 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
-#include <assert.h>
 #include <errno.h>
 #include <limits.h>
 #include <locale.h>
@@ -44,11 +43,12 @@ __FBSDID("$FreeBSD$");
 #include <string.h>
 #include <wchar.h>
 
-int
-main(int argc, char *argv[])
+#include <atf-c.h>
+
+ATF_TC_WITHOUT_HEAD(mbrtowc_test);
+ATF_TC_BODY(mbrtowc_test, tc)
 {
 	mbstate_t s;
-	size_t len;
 	wchar_t wc;
 	char buf[MB_LEN_MAX + 1];
 
@@ -56,80 +56,78 @@ main(int argc, char *argv[])
 	 * C/POSIX locale.
 	 */
 
-	printf("1..1\n");
-
-	assert(MB_CUR_MAX == 1);
+	ATF_REQUIRE(MB_CUR_MAX == 1);
 
 	/* Null wide character, internal state. */
 	memset(buf, 0xcc, sizeof(buf));
 	buf[0] = 0;
-	assert(mbrtowc(&wc, buf, 1, NULL) == 0);
-	assert(wc == 0);
+	ATF_REQUIRE(mbrtowc(&wc, buf, 1, NULL) == 0);
+	ATF_REQUIRE(wc == 0);
 
 	/* Null wide character. */
 	memset(&s, 0, sizeof(s));
-	assert(mbrtowc(&wc, buf, 1, &s) == 0);
-	assert(wc == 0);
+	ATF_REQUIRE(mbrtowc(&wc, buf, 1, &s) == 0);
+	ATF_REQUIRE(wc == 0);
 
 	/* Latin letter A, internal state. */
-	assert(mbrtowc(NULL, 0, 0, NULL) == 0);
+	ATF_REQUIRE(mbrtowc(NULL, 0, 0, NULL) == 0);
 	buf[0] = 'A';
-	assert(mbrtowc(&wc, buf, 1, NULL) == 1);
-	assert(wc == L'A');
+	ATF_REQUIRE(mbrtowc(&wc, buf, 1, NULL) == 1);
+	ATF_REQUIRE(wc == L'A');
 
 	/* Latin letter A. */
 	memset(&s, 0, sizeof(s));
-	assert(mbrtowc(&wc, buf, 1, &s) == 1);
-	assert(wc == L'A');
+	ATF_REQUIRE(mbrtowc(&wc, buf, 1, &s) == 1);
+	ATF_REQUIRE(wc == L'A');
 
 	/* Incomplete character sequence. */
 	wc = L'z';
 	memset(&s, 0, sizeof(s));
-	assert(mbrtowc(&wc, buf, 0, &s) == (size_t)-2);
-	assert(wc == L'z');
+	ATF_REQUIRE(mbrtowc(&wc, buf, 0, &s) == (size_t)-2);
+	ATF_REQUIRE(wc == L'z');
 
 	/* Check that mbrtowc() doesn't access the buffer when n == 0. */
 	wc = L'z';
 	memset(&s, 0, sizeof(s));
 	buf[0] = '\0';
-	assert(mbrtowc(&wc, buf, 0, &s) == (size_t)-2);
-	assert(wc == L'z');
+	ATF_REQUIRE(mbrtowc(&wc, buf, 0, &s) == (size_t)-2);
+	ATF_REQUIRE(wc == L'z');
 
 	/*
 	 * Japanese (EUC) locale.
 	 */
 
-	assert(strcmp(setlocale(LC_CTYPE, "ja_JP.eucJP"), "ja_JP.eucJP") == 0);
-	assert(MB_CUR_MAX > 1);
+	ATF_REQUIRE(strcmp(setlocale(LC_CTYPE, "ja_JP.eucJP"), "ja_JP.eucJP") == 0);
+	ATF_REQUIRE(MB_CUR_MAX > 1);
 
 	/* Null wide character, internal state. */
-	assert(mbrtowc(NULL, 0, 0, NULL) == 0);
+	ATF_REQUIRE(mbrtowc(NULL, 0, 0, NULL) == 0);
 	memset(buf, 0xcc, sizeof(buf));
 	buf[0] = 0;
-	assert(mbrtowc(&wc, buf, 1, NULL) == 0);
-	assert(wc == 0);
+	ATF_REQUIRE(mbrtowc(&wc, buf, 1, NULL) == 0);
+	ATF_REQUIRE(wc == 0);
 
 	/* Null wide character. */
 	memset(&s, 0, sizeof(s));
-	assert(mbrtowc(&wc, buf, 1, &s) == 0);
-	assert(wc == 0);
+	ATF_REQUIRE(mbrtowc(&wc, buf, 1, &s) == 0);
+	ATF_REQUIRE(wc == 0);
 
 	/* Latin letter A, internal state. */
-	assert(mbrtowc(NULL, 0, 0, NULL) == 0);
+	ATF_REQUIRE(mbrtowc(NULL, 0, 0, NULL) == 0);
 	buf[0] = 'A';
-	assert(mbrtowc(&wc, buf, 1, NULL) == 1);
-	assert(wc == L'A');
+	ATF_REQUIRE(mbrtowc(&wc, buf, 1, NULL) == 1);
+	ATF_REQUIRE(wc == L'A');
 
 	/* Latin letter A. */
 	memset(&s, 0, sizeof(s));
-	assert(mbrtowc(&wc, buf, 1, &s) == 1);
-	assert(wc == L'A');
+	ATF_REQUIRE(mbrtowc(&wc, buf, 1, &s) == 1);
+	ATF_REQUIRE(wc == L'A');
 
 	/* Incomplete character sequence (zero length). */
 	wc = L'z';
 	memset(&s, 0, sizeof(s));
-	assert(mbrtowc(&wc, buf, 0, &s) == (size_t)-2);
-	assert(wc == L'z');
+	ATF_REQUIRE(mbrtowc(&wc, buf, 0, &s) == (size_t)-2);
+	ATF_REQUIRE(wc == L'z');
 
 	/* Incomplete character sequence (truncated double-byte). */
 	memset(buf, 0xcc, sizeof(buf));
@@ -137,27 +135,31 @@ main(int argc, char *argv[])
 	buf[1] = 0x00;
 	memset(&s, 0, sizeof(s));
 	wc = 0;
-	assert(mbrtowc(&wc, buf, 1, &s) == (size_t)-2);
+	ATF_REQUIRE(mbrtowc(&wc, buf, 1, &s) == (size_t)-2);
 
 	/* Same as above, but complete. */
 	buf[1] = 0xc1;
 	memset(&s, 0, sizeof(s));
 	wc = 0;
-	assert(mbrtowc(&wc, buf, 2, &s) == 2);
-	assert(wc == 0xa3c1);
+	ATF_REQUIRE(mbrtowc(&wc, buf, 2, &s) == 2);
+	ATF_REQUIRE(wc == 0xa3c1);
 
 	/* Test restarting behaviour. */
 	memset(buf, 0xcc, sizeof(buf));
 	buf[0] = 0xa3;
 	memset(&s, 0, sizeof(s));
 	wc = 0;
-	assert(mbrtowc(&wc, buf, 1, &s) == (size_t)-2);
-	assert(wc == 0);
+	ATF_REQUIRE(mbrtowc(&wc, buf, 1, &s) == (size_t)-2);
+	ATF_REQUIRE(wc == 0);
 	buf[0] = 0xc1;
-	assert(mbrtowc(&wc, buf, 1, &s) == 1);
-	assert(wc == 0xa3c1);
+	ATF_REQUIRE(mbrtowc(&wc, buf, 1, &s) == 1);
+	ATF_REQUIRE(wc == 0xa3c1);
+}
+
+ATF_TP_ADD_TCS(tp)
+{
 
-	printf("ok 1 - mbrtowc()\n");
+	ATF_TP_ADD_TC(tp, mbrtowc_test);
 
-	return (0);
+	return (atf_no_error());
 }
diff --git a/tools/regression/lib/libc/locale/test-mbsnrtowcs.c b/lib/libc/tests/locale/mbsnrtowcs_test.c
index 585dd4e..4012b7e 100644
--- a/tools/regression/lib/libc/locale/test-mbsnrtowcs.c
+++ b/lib/libc/tests/locale/mbsnrtowcs_test.c
@@ -34,7 +34,6 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
-#include <assert.h>
 #include <errno.h>
 #include <limits.h>
 #include <locale.h>
@@ -43,19 +42,17 @@ __FBSDID("$FreeBSD$");
 #include <string.h>
 #include <wchar.h>
 
-int
-main(int argc, char *argv[])
+#include <atf-c.h>
+
+ATF_TC_WITHOUT_HEAD(mbsnrtowcs_test);
+ATF_TC_BODY(mbsnrtowcs_test, tc)
 {
 	char srcbuf[128];
 	wchar_t dstbuf[128];
 	char *src;
 	mbstate_t s;
 
-	/*
-	 * C/POSIX locale.
-	 */
-
-	printf("1..1\n");
+	/* C/POSIX locale. */
 
 	/* Simple null terminated string. */
 	memset(srcbuf, 0xcc, sizeof(srcbuf));
@@ -63,11 +60,11 @@ main(int argc, char *argv[])
 	wmemset(dstbuf, 0xcccc, sizeof(dstbuf) / sizeof(*dstbuf));
 	src = srcbuf;
 	memset(&s, 0, sizeof(s));
-	assert(mbsnrtowcs(dstbuf, (const char **)&src, 6, sizeof(dstbuf) /
+	ATF_REQUIRE(mbsnrtowcs(dstbuf, (const char **)&src, 6, sizeof(dstbuf) /
 	    sizeof(*dstbuf), &s) == 5);
-	assert(wcscmp(dstbuf, L"hello") == 0);
-	assert(dstbuf[6] == 0xcccc);
-	assert(src == NULL);
+	ATF_REQUIRE(wcscmp(dstbuf, L"hello") == 0);
+	ATF_REQUIRE(dstbuf[6] == 0xcccc);
+	ATF_REQUIRE(src == NULL);
 
 	/* Simple null terminated string, stopping early. */
 	memset(srcbuf, 0xcc, sizeof(srcbuf));
@@ -75,11 +72,11 @@ main(int argc, char *argv[])
 	wmemset(dstbuf, 0xcccc, sizeof(dstbuf) / sizeof(*dstbuf));
 	src = srcbuf;
 	memset(&s, 0, sizeof(s));
-	assert(mbsnrtowcs(dstbuf, (const char **)&src, 4, sizeof(dstbuf) /
+	ATF_REQUIRE(mbsnrtowcs(dstbuf, (const char **)&src, 4, sizeof(dstbuf) /
 	    sizeof(*dstbuf), &s) == 4);
-	assert(wmemcmp(dstbuf, L"hell", 4) == 0);
-	assert(dstbuf[5] == 0xcccc);
-	assert(src == srcbuf + 4);
+	ATF_REQUIRE(wmemcmp(dstbuf, L"hell", 4) == 0);
+	ATF_REQUIRE(dstbuf[5] == 0xcccc);
+	ATF_REQUIRE(src == srcbuf + 4);
 
 	/* Not enough space in destination buffer. */
 	memset(srcbuf, 0xcc, sizeof(srcbuf));
@@ -87,41 +84,41 @@ main(int argc, char *argv[])
 	wmemset(dstbuf, 0xcccc, sizeof(dstbuf) / sizeof(*dstbuf));
 	src = srcbuf;
 	memset(&s, 0, sizeof(s));
-	assert(mbsnrtowcs(dstbuf, (const char **)&src, 6, 4, &s) == 4);
-	assert(wmemcmp(dstbuf, L"hell", 4) == 0);
-	assert(dstbuf[5] == 0xcccc);
-	assert(src == srcbuf + 4);
+	ATF_REQUIRE(mbsnrtowcs(dstbuf, (const char **)&src, 6, 4, &s) == 4);
+	ATF_REQUIRE(wmemcmp(dstbuf, L"hell", 4) == 0);
+	ATF_REQUIRE(dstbuf[5] == 0xcccc);
+	ATF_REQUIRE(src == srcbuf + 4);
 
 	/* Null terminated string, internal dest. buffer */
 	memset(srcbuf, 0xcc, sizeof(srcbuf));
 	strcpy(srcbuf, "hello");
 	src = srcbuf;
 	memset(&s, 0, sizeof(s));
-	assert(mbsnrtowcs(NULL, (const char **)&src, 6, 0, &s) == 5);
+	ATF_REQUIRE(mbsnrtowcs(NULL, (const char **)&src, 6, 0, &s) == 5);
 
 	/* Null terminated string, internal dest. buffer, stopping early */
 	memset(srcbuf, 0xcc, sizeof(srcbuf));
 	strcpy(srcbuf, "hello");
 	src = srcbuf;
 	memset(&s, 0, sizeof(s));
-	assert(mbsnrtowcs(NULL, (const char **)&src, 4, 0, &s) == 4);
+	ATF_REQUIRE(mbsnrtowcs(NULL, (const char **)&src, 4, 0, &s) == 4);
 
 	/* Null terminated string, internal state. */
 	memset(srcbuf, 0xcc, sizeof(srcbuf));
 	strcpy(srcbuf, "hello");
 	wmemset(dstbuf, 0xcccc, sizeof(dstbuf) / sizeof(*dstbuf));
 	src = srcbuf;
-	assert(mbsnrtowcs(dstbuf, (const char **)&src, 6, sizeof(dstbuf) /
+	ATF_REQUIRE(mbsnrtowcs(dstbuf, (const char **)&src, 6, sizeof(dstbuf) /
 	    sizeof(*dstbuf), NULL) == 5);
-	assert(wcscmp(dstbuf, L"hello") == 0);
-	assert(dstbuf[6] == 0xcccc);
-	assert(src == NULL);
+	ATF_REQUIRE(wcscmp(dstbuf, L"hello") == 0);
+	ATF_REQUIRE(dstbuf[6] == 0xcccc);
+	ATF_REQUIRE(src == NULL);
 
 	/* Null terminated string, internal state, internal dest. buffer. */
 	memset(srcbuf, 0xcc, sizeof(srcbuf));
 	strcpy(srcbuf, "hello");
 	src = srcbuf;
-	assert(mbsnrtowcs(NULL, (const char **)&src, 6, 0, NULL) == 5);
+	ATF_REQUIRE(mbsnrtowcs(NULL, (const char **)&src, 6, 0, NULL) == 5);
 
 	/* Empty source buffer. */
 	memset(srcbuf, 0xcc, sizeof(srcbuf));
@@ -129,10 +126,10 @@ main(int argc, char *argv[])
 	src = srcbuf;
 	memset(&s, 0, sizeof(s));
 	wmemset(dstbuf, 0xcccc, sizeof(dstbuf) / sizeof(*dstbuf));
-	assert(mbsnrtowcs(dstbuf, (const char **)&src, 1, 1, &s) == 0);
-	assert(dstbuf[0] == 0);
-	assert(dstbuf[1] == 0xcccc);
-	assert(src == NULL);
+	ATF_REQUIRE(mbsnrtowcs(dstbuf, (const char **)&src, 1, 1, &s) == 0);
+	ATF_REQUIRE(dstbuf[0] == 0);
+	ATF_REQUIRE(dstbuf[1] == 0xcccc);
+	ATF_REQUIRE(src == NULL);
 
 	/* Zero length destination buffer. */
 	memset(srcbuf, 0xcc, sizeof(srcbuf));
@@ -140,36 +137,36 @@ main(int argc, char *argv[])
 	src = srcbuf;
 	memset(&s, 0, sizeof(s));
 	wmemset(dstbuf, 0xcccc, sizeof(dstbuf) / sizeof(*dstbuf));
-	assert(mbsnrtowcs(dstbuf, (const char **)&src, 1, 0, &s) == 0);
-	assert(dstbuf[0] == 0xcccc);
-	assert(src == srcbuf);
+	ATF_REQUIRE(mbsnrtowcs(dstbuf, (const char **)&src, 1, 0, &s) == 0);
+	ATF_REQUIRE(dstbuf[0] == 0xcccc);
+	ATF_REQUIRE(src == srcbuf);
 
 	/* Zero length source buffer. */
 	memset(srcbuf, 0xcc, sizeof(srcbuf));
 	src = srcbuf;
 	memset(&s, 0, sizeof(s));
 	wmemset(dstbuf, 0xcccc, sizeof(dstbuf) / sizeof(*dstbuf));
-	assert(mbsnrtowcs(dstbuf, (const char **)&src, 0, 1, &s) == 0);
-	assert(dstbuf[0] == 0xcccc);
-	assert(src == srcbuf);
+	ATF_REQUIRE(mbsnrtowcs(dstbuf, (const char **)&src, 0, 1, &s) == 0);
+	ATF_REQUIRE(dstbuf[0] == 0xcccc);
+	ATF_REQUIRE(src == srcbuf);
 
 	/*
 	 * Japanese (EUC) locale.
 	 */
 
-	assert(strcmp(setlocale(LC_CTYPE, "ja_JP.eucJP"), "ja_JP.eucJP") == 0);
-	assert(MB_CUR_MAX > 1);
+	ATF_REQUIRE(strcmp(setlocale(LC_CTYPE, "ja_JP.eucJP"), "ja_JP.eucJP") == 0);
+	ATF_REQUIRE(MB_CUR_MAX > 1);
 
 	memset(srcbuf, 0xcc, sizeof(srcbuf));
 	strcpy(srcbuf, "\xA3\xC1 B \xA3\xC3");
 	src = srcbuf;
 	memset(&s, 0, sizeof(s));
 	wmemset(dstbuf, 0xcccc, sizeof(dstbuf) / sizeof(*dstbuf));
-	assert(mbsnrtowcs(dstbuf, (const char **)&src, 8, sizeof(dstbuf) /
+	ATF_REQUIRE(mbsnrtowcs(dstbuf, (const char **)&src, 8, sizeof(dstbuf) /
 	    sizeof(*dstbuf), &s) == 5);
-	assert(dstbuf[0] == 0xA3C1 && dstbuf[1] == 0x20 && dstbuf[2] == 0x42 &&
+	ATF_REQUIRE(dstbuf[0] == 0xA3C1 && dstbuf[1] == 0x20 && dstbuf[2] == 0x42 &&
 	    dstbuf[3] == 0x20 && dstbuf[4] == 0xA3C3 && dstbuf[5] == 0);
-	assert(src == NULL);
+	ATF_REQUIRE(src == NULL);
 
 	/* Partial character. */
 	memset(srcbuf, 0xcc, sizeof(srcbuf));
@@ -177,18 +174,22 @@ main(int argc, char *argv[])
 	src = srcbuf;
 	memset(&s, 0, sizeof(s));
 	wmemset(dstbuf, 0xcccc, sizeof(dstbuf) / sizeof(*dstbuf));
-	assert(mbsnrtowcs(dstbuf, (const char **)&src, 6, sizeof(dstbuf) /
+	ATF_REQUIRE(mbsnrtowcs(dstbuf, (const char **)&src, 6, sizeof(dstbuf) /
 	    sizeof(*dstbuf), &s) == 4);
-	assert(src == srcbuf + 6);
-	assert(!mbsinit(&s));
-	assert(mbsnrtowcs(dstbuf, (const char **)&src, 1, sizeof(dstbuf) /
+	ATF_REQUIRE(src == srcbuf + 6);
+	ATF_REQUIRE(!mbsinit(&s));
+	ATF_REQUIRE(mbsnrtowcs(dstbuf, (const char **)&src, 1, sizeof(dstbuf) /
 	    sizeof(*dstbuf), &s) == 1);
-	assert(src == srcbuf + 7);
-	assert(mbsnrtowcs(dstbuf, (const char **)&src, 1, sizeof(dstbuf) /
+	ATF_REQUIRE(src == srcbuf + 7);
+	ATF_REQUIRE(mbsnrtowcs(dstbuf, (const char **)&src, 1, sizeof(dstbuf) /
 	    sizeof(*dstbuf), &s) == 0);
-	assert(src == NULL);
+	ATF_REQUIRE(src == NULL);
+}
+
+ATF_TP_ADD_TCS(tp)
+{
 
-	printf("ok 1 - mbsnrtowcs()\n");
+	ATF_TP_ADD_TC(tp, mbsnrtowcs_test);
 
-	return (0);
+	return (atf_no_error());
 }
diff --git a/tools/regression/lib/libc/locale/test-mbsrtowcs.c b/lib/libc/tests/locale/mbsrtowcs_test.c
index dd6a7e1..1b9d5c9 100644
--- a/tools/regression/lib/libc/locale/test-mbsrtowcs.c
+++ b/lib/libc/tests/locale/mbsrtowcs_test.c
@@ -35,7 +35,6 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
-#include <assert.h>
 #include <errno.h>
 #include <limits.h>
 #include <locale.h>
@@ -44,8 +43,10 @@ __FBSDID("$FreeBSD$");
 #include <string.h>
 #include <wchar.h>
 
-int
-main(int argc, char *argv[])
+#include <atf-c.h>
+
+ATF_TC_WITHOUT_HEAD(mbsrtowcs_test);
+ATF_TC_BODY(mbsrtowcs_test, tc)
 {
 	char srcbuf[128];
 	wchar_t dstbuf[128];
@@ -56,19 +57,17 @@ main(int argc, char *argv[])
 	 * C/POSIX locale.
 	 */
 
-	printf("1..1\n");
-
 	/* Simple null terminated string. */
 	memset(srcbuf, 0xcc, sizeof(srcbuf));
 	strcpy(srcbuf, "hello");
 	wmemset(dstbuf, 0xcccc, sizeof(dstbuf) / sizeof(*dstbuf));
 	src = srcbuf;
 	memset(&s, 0, sizeof(s));
-	assert(mbsrtowcs(dstbuf, (const char **)&src, sizeof(dstbuf) /
+	ATF_REQUIRE(mbsrtowcs(dstbuf, (const char **)&src, sizeof(dstbuf) /
 	    sizeof(*dstbuf), &s) == 5);
-	assert(wcscmp(dstbuf, L"hello") == 0);
-	assert(dstbuf[6] == 0xcccc);
-	assert(src == NULL);
+	ATF_REQUIRE(wcscmp(dstbuf, L"hello") == 0);
+	ATF_REQUIRE(dstbuf[6] == 0xcccc);
+	ATF_REQUIRE(src == NULL);
 
 	/* Not enough space in destination buffer. */
 	memset(srcbuf, 0xcc, sizeof(srcbuf));
@@ -76,34 +75,34 @@ main(int argc, char *argv[])
 	wmemset(dstbuf, 0xcccc, sizeof(dstbuf) / sizeof(*dstbuf));
 	src = srcbuf;
 	memset(&s, 0, sizeof(s));
-	assert(mbsrtowcs(dstbuf, (const char **)&src, 4, &s) == 4);
-	assert(wmemcmp(dstbuf, L"hell", 4) == 0);
-	assert(dstbuf[5] == 0xcccc);
-	assert(src == srcbuf + 4);
+	ATF_REQUIRE(mbsrtowcs(dstbuf, (const char **)&src, 4, &s) == 4);
+	ATF_REQUIRE(wmemcmp(dstbuf, L"hell", 4) == 0);
+	ATF_REQUIRE(dstbuf[5] == 0xcccc);
+	ATF_REQUIRE(src == srcbuf + 4);
 
 	/* Null terminated string, internal dest. buffer */
 	memset(srcbuf, 0xcc, sizeof(srcbuf));
 	strcpy(srcbuf, "hello");
 	src = srcbuf;
 	memset(&s, 0, sizeof(s));
-	assert(mbsrtowcs(NULL, (const char **)&src, 0, &s) == 5);
+	ATF_REQUIRE(mbsrtowcs(NULL, (const char **)&src, 0, &s) == 5);
 
 	/* Null terminated string, internal state. */
 	memset(srcbuf, 0xcc, sizeof(srcbuf));
 	strcpy(srcbuf, "hello");
 	wmemset(dstbuf, 0xcccc, sizeof(dstbuf) / sizeof(*dstbuf));
 	src = srcbuf;
-	assert(mbsrtowcs(dstbuf, (const char **)&src, sizeof(dstbuf) /
+	ATF_REQUIRE(mbsrtowcs(dstbuf, (const char **)&src, sizeof(dstbuf) /
 	    sizeof(*dstbuf), NULL) == 5);
-	assert(wcscmp(dstbuf, L"hello") == 0);
-	assert(dstbuf[6] == 0xcccc);
-	assert(src == NULL);
+	ATF_REQUIRE(wcscmp(dstbuf, L"hello") == 0);
+	ATF_REQUIRE(dstbuf[6] == 0xcccc);
+	ATF_REQUIRE(src == NULL);
 
 	/* Null terminated string, internal state, internal dest. buffer. */
 	memset(srcbuf, 0xcc, sizeof(srcbuf));
 	strcpy(srcbuf, "hello");
 	src = srcbuf;
-	assert(mbsrtowcs(NULL, (const char **)&src, 0, NULL) == 5);
+	ATF_REQUIRE(mbsrtowcs(NULL, (const char **)&src, 0, NULL) == 5);
 
 	/* Empty source buffer. */
 	memset(srcbuf, 0xcc, sizeof(srcbuf));
@@ -111,10 +110,10 @@ main(int argc, char *argv[])
 	src = srcbuf;
 	memset(&s, 0, sizeof(s));
 	wmemset(dstbuf, 0xcccc, sizeof(dstbuf) / sizeof(*dstbuf));
-	assert(mbsrtowcs(dstbuf, (const char **)&src, 1, &s) == 0);
-	assert(dstbuf[0] == 0);
-	assert(dstbuf[1] == 0xcccc);
-	assert(src == NULL);
+	ATF_REQUIRE(mbsrtowcs(dstbuf, (const char **)&src, 1, &s) == 0);
+	ATF_REQUIRE(dstbuf[0] == 0);
+	ATF_REQUIRE(dstbuf[1] == 0xcccc);
+	ATF_REQUIRE(src == NULL);
 
 	/* Zero length destination buffer. */
 	memset(srcbuf, 0xcc, sizeof(srcbuf));
@@ -122,29 +121,33 @@ main(int argc, char *argv[])
 	src = srcbuf;
 	memset(&s, 0, sizeof(s));
 	wmemset(dstbuf, 0xcccc, sizeof(dstbuf) / sizeof(*dstbuf));
-	assert(mbsrtowcs(dstbuf, (const char **)&src, 0, &s) == 0);
-	assert(dstbuf[0] == 0xcccc);
-	assert(src == srcbuf);
+	ATF_REQUIRE(mbsrtowcs(dstbuf, (const char **)&src, 0, &s) == 0);
+	ATF_REQUIRE(dstbuf[0] == 0xcccc);
+	ATF_REQUIRE(src == srcbuf);
 
 	/*
 	 * Japanese (EUC) locale.
 	 */
 
-	assert(strcmp(setlocale(LC_CTYPE, "ja_JP.eucJP"), "ja_JP.eucJP") == 0);
-	assert(MB_CUR_MAX > 1);
+	ATF_REQUIRE(strcmp(setlocale(LC_CTYPE, "ja_JP.eucJP"), "ja_JP.eucJP") == 0);
+	ATF_REQUIRE(MB_CUR_MAX > 1);
 
 	memset(srcbuf, 0xcc, sizeof(srcbuf));
 	strcpy(srcbuf, "\xA3\xC1 B \xA3\xC3");
 	src = srcbuf;
 	memset(&s, 0, sizeof(s));
 	wmemset(dstbuf, 0xcccc, sizeof(dstbuf) / sizeof(*dstbuf));
-	assert(mbsrtowcs(dstbuf, (const char **)&src, sizeof(dstbuf) /
+	ATF_REQUIRE(mbsrtowcs(dstbuf, (const char **)&src, sizeof(dstbuf) /
 	    sizeof(*dstbuf), &s) == 5);
-	assert(dstbuf[0] == 0xA3C1 && dstbuf[1] == 0x20 && dstbuf[2] == 0x42 &&
+	ATF_REQUIRE(dstbuf[0] == 0xA3C1 && dstbuf[1] == 0x20 && dstbuf[2] == 0x42 &&
 	    dstbuf[3] == 0x20 && dstbuf[4] == 0xA3C3 && dstbuf[5] == 0);
-	assert(src == NULL);
+	ATF_REQUIRE(src == NULL);
+}
+
+ATF_TP_ADD_TCS(tp)
+{
 
-	printf("ok 1 - mbsrtowcs()\n");
+	ATF_TP_ADD_TC(tp, mbsrtowcs_test);
 
-	return (0);
+	return (atf_no_error());
 }
diff --git a/tools/regression/lib/libc/locale/test-mbstowcs.c b/lib/libc/tests/locale/mbstowcs_test.c
index 18039dd..77b0e9e 100644
--- a/tools/regression/lib/libc/locale/test-mbstowcs.c
+++ b/lib/libc/tests/locale/mbstowcs_test.c
@@ -35,7 +35,6 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
-#include <assert.h>
 #include <errno.h>
 #include <limits.h>
 #include <locale.h>
@@ -44,69 +43,69 @@ __FBSDID("$FreeBSD$");
 #include <string.h>
 #include <wchar.h>
 
-int
-main(int argc, char *argv[])
+#include <atf-c.h>
+
+ATF_TC_WITHOUT_HEAD(mbstowcs_test);
+ATF_TC_BODY(mbstowcs_test, tc)
 {
 	char srcbuf[128];
 	wchar_t dstbuf[128];
 
-	/*
-	 * C/POSIX locale.
-	 */
-
-	printf("1..1\n");
+	/* C/POSIX locale. */
 
 	/* Simple null terminated string. */
 	memset(srcbuf, 0xcc, sizeof(srcbuf));
 	strcpy(srcbuf, "hello");
 	wmemset(dstbuf, 0xcccc, sizeof(dstbuf) / sizeof(*dstbuf));
-	assert(mbstowcs(dstbuf, srcbuf, sizeof(dstbuf) / sizeof(*dstbuf)) == 5);
-	assert(wcscmp(dstbuf, L"hello") == 0);
-	assert(dstbuf[6] == 0xcccc);
+	ATF_REQUIRE(mbstowcs(dstbuf, srcbuf, sizeof(dstbuf) / sizeof(*dstbuf)) == 5);
+	ATF_REQUIRE(wcscmp(dstbuf, L"hello") == 0);
+	ATF_REQUIRE(dstbuf[6] == 0xcccc);
 
 	/* Not enough space in destination buffer. */
 	memset(srcbuf, 0xcc, sizeof(srcbuf));
 	strcpy(srcbuf, "hello");
 	wmemset(dstbuf, 0xcccc, sizeof(dstbuf) / sizeof(*dstbuf));
-	assert(mbstowcs(dstbuf, srcbuf, 4) == 4);
-	assert(wmemcmp(dstbuf, L"hell", 4) == 0);
-	assert(dstbuf[5] == 0xcccc);
+	ATF_REQUIRE(mbstowcs(dstbuf, srcbuf, 4) == 4);
+	ATF_REQUIRE(wmemcmp(dstbuf, L"hell", 4) == 0);
+	ATF_REQUIRE(dstbuf[5] == 0xcccc);
 
 	/* Null terminated string, internal dest. buffer (XSI extension) */
 	memset(srcbuf, 0xcc, sizeof(srcbuf));
 	strcpy(srcbuf, "hello");
-	assert(mbstowcs(NULL, srcbuf, 0) == 5);
+	ATF_REQUIRE(mbstowcs(NULL, srcbuf, 0) == 5);
 
 	/* Empty source buffer. */
 	memset(srcbuf, 0xcc, sizeof(srcbuf));
 	srcbuf[0] = '\0';
 	wmemset(dstbuf, 0xcccc, sizeof(dstbuf) / sizeof(*dstbuf));
-	assert(mbstowcs(dstbuf, srcbuf, 1) == 0);
-	assert(dstbuf[0] == 0);
-	assert(dstbuf[1] == 0xcccc);
+	ATF_REQUIRE(mbstowcs(dstbuf, srcbuf, 1) == 0);
+	ATF_REQUIRE(dstbuf[0] == 0);
+	ATF_REQUIRE(dstbuf[1] == 0xcccc);
 
 	/* Zero length destination buffer. */
 	memset(srcbuf, 0xcc, sizeof(srcbuf));
 	strcpy(srcbuf, "hello");
 	wmemset(dstbuf, 0xcccc, sizeof(dstbuf) / sizeof(*dstbuf));
-	assert(mbstowcs(dstbuf, srcbuf, 0) == 0);
-	assert(dstbuf[0] == 0xcccc);
+	ATF_REQUIRE(mbstowcs(dstbuf, srcbuf, 0) == 0);
+	ATF_REQUIRE(dstbuf[0] == 0xcccc);
 
-	/*
-	 * Japanese (EUC) locale.
-	 */
+	/* Japanese (EUC) locale. */
 
-	assert(strcmp(setlocale(LC_CTYPE, "ja_JP.eucJP"), "ja_JP.eucJP") == 0);
-	assert(MB_CUR_MAX > 1);
+	ATF_REQUIRE(strcmp(setlocale(LC_CTYPE, "ja_JP.eucJP"), "ja_JP.eucJP") == 0);
+	ATF_REQUIRE(MB_CUR_MAX > 1);
 
 	memset(srcbuf, 0xcc, sizeof(srcbuf));
 	strcpy(srcbuf, "\xA3\xC1 B \xA3\xC3");
 	wmemset(dstbuf, 0xcccc, sizeof(dstbuf) / sizeof(*dstbuf));
-	assert(mbstowcs(dstbuf, srcbuf, sizeof(dstbuf) / sizeof(*dstbuf)) == 5);
-	assert(dstbuf[0] == 0xA3C1 && dstbuf[1] == 0x20 && dstbuf[2] == 0x42 &&
+	ATF_REQUIRE(mbstowcs(dstbuf, srcbuf, sizeof(dstbuf) / sizeof(*dstbuf)) == 5);
+	ATF_REQUIRE(dstbuf[0] == 0xA3C1 && dstbuf[1] == 0x20 && dstbuf[2] == 0x42 &&
 	    dstbuf[3] == 0x20 && dstbuf[4] == 0xA3C3 && dstbuf[5] == 0);
+}
+
+ATF_TP_ADD_TCS(tp)
+{
 
-	printf("ok 1 - mbstowcs()\n");
+	ATF_TP_ADD_TC(tp, mbstowcs_test);
 
-	return (0);
+	return (atf_no_error());
 }
diff --git a/tools/regression/lib/libc/locale/test-mbtowc.c b/lib/libc/tests/locale/mbtowc_test.c
index 5ee070c..7c7f80e 100644
--- a/tools/regression/lib/libc/locale/test-mbtowc.c
+++ b/lib/libc/tests/locale/mbtowc_test.c
@@ -35,90 +35,88 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
-#include <assert.h>
 #include <limits.h>
 #include <locale.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 
-int
-main(int argc, char *argv[])
+#include <atf-c.h>
+
+ATF_TC_WITHOUT_HEAD(mbtowc_test);
+ATF_TC_BODY(mbtowc_test, tc)
 {
-	size_t len;
-	wchar_t wc;
 	char buf[MB_LEN_MAX + 1];
+	wchar_t wc;
 
-	/*
-	 * C/POSIX locale.
-	 */
-
-	printf("1..1\n");
+	/* C/POSIX locale. */
 
-	assert(MB_CUR_MAX == 1);
+	ATF_REQUIRE(MB_CUR_MAX == 1);
 
 	/* No shift states in C locale. */
-	assert(mbtowc(NULL, NULL, 0) == 0);
+	ATF_REQUIRE(mbtowc(NULL, NULL, 0) == 0);
 
 	/* Null wide character. */
 	wc = 0xcccc;
 	memset(buf, 0, sizeof(buf));
-	assert(mbtowc(&wc, buf, 1) == 0);
-	assert(wc == 0);
+	ATF_REQUIRE(mbtowc(&wc, buf, 1) == 0);
+	ATF_REQUIRE(wc == 0);
 
 	/* Latin letter A. */
 	buf[0] = 'A';
-	assert(mbtowc(&wc, buf, 1) == 1);
-	assert(wc == L'A');
+	ATF_REQUIRE(mbtowc(&wc, buf, 1) == 1);
+	ATF_REQUIRE(wc == L'A');
 
 	/* Incomplete character sequence. */
 	wc = L'z';
 	buf[0] = '\0';
-	assert(mbtowc(&wc, buf, 0) == -1);
-	assert(wc == L'z');
-	assert(mbtowc(NULL, NULL, 0) == 0);
+	ATF_REQUIRE(mbtowc(&wc, buf, 0) == -1);
+	ATF_REQUIRE(wc == L'z');
+	ATF_REQUIRE(mbtowc(NULL, NULL, 0) == 0);
 
-	/*
-	 * Japanese (EUC) locale.
-	 */
+	/* Japanese (EUC) locale. */
 
-	assert(strcmp(setlocale(LC_CTYPE, "ja_JP.eucJP"), "ja_JP.eucJP") == 0);
-	assert(MB_CUR_MAX > 1);
+	ATF_REQUIRE(strcmp(setlocale(LC_CTYPE, "ja_JP.eucJP"), "ja_JP.eucJP") == 0);
+	ATF_REQUIRE(MB_CUR_MAX > 1);
 
 	/* Null wide character */
 	memset(buf, 0xcc, sizeof(buf));
 	buf[0] = 0;
 	wc = 0xcccc;
-	assert(mbtowc(&wc, buf, 1) == 0);
-	assert(wc == 0);
+	ATF_REQUIRE(mbtowc(&wc, buf, 1) == 0);
+	ATF_REQUIRE(wc == 0);
 
 	/* Latin letter A. */
 	buf[0] = 'A';
-	assert(mbtowc(&wc, buf, 1) == 1);
-	assert(wc == L'A');
+	ATF_REQUIRE(mbtowc(&wc, buf, 1) == 1);
+	ATF_REQUIRE(wc == L'A');
 
 	/* Incomplete character sequence (zero length). */
 	wc = L'z';
 	buf[0] = '\0';
-	assert(mbtowc(&wc, buf, 0) == -1);
-	assert(wc == L'z');
-	assert(mbtowc(NULL, NULL, 0) == 0);
+	ATF_REQUIRE(mbtowc(&wc, buf, 0) == -1);
+	ATF_REQUIRE(wc == L'z');
+	ATF_REQUIRE(mbtowc(NULL, NULL, 0) == 0);
 
 	/* Incomplete character sequence (truncated double-byte). */
 	memset(buf, 0xcc, sizeof(buf));
 	buf[0] = 0xa3;
 	buf[1] = 0x00;
 	wc = L'z';
-	assert(mbtowc(&wc, buf, 1) == -1);
-	assert(wc == L'z');
-	assert(mbtowc(NULL, NULL, 0) == 0);
+	ATF_REQUIRE(mbtowc(&wc, buf, 1) == -1);
+	ATF_REQUIRE(wc == L'z');
+	ATF_REQUIRE(mbtowc(NULL, NULL, 0) == 0);
 
 	/* Same as above, but complete. */
 	buf[1] = 0xc1;
-	assert(mbtowc(&wc, buf, 2) == 2);
-	assert(wc == 0xa3c1);
+	ATF_REQUIRE(mbtowc(&wc, buf, 2) == 2);
+	ATF_REQUIRE(wc == 0xa3c1);
+}
+
+ATF_TP_ADD_TCS(tp)
+{
 
-	printf("ok 1 - mbtowc()\n");
+	ATF_TP_ADD_TC(tp, mbtowc_test);
 
-	return (0);
+	return (atf_no_error());
 }
diff --git a/tools/regression/lib/libc/locale/test-towctrans.c b/lib/libc/tests/locale/towctrans_test.c
index fabb343..bb9d97c 100644
--- a/tools/regression/lib/libc/locale/test-towctrans.c
+++ b/lib/libc/tests/locale/towctrans_test.c
@@ -34,15 +34,16 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
-#include <assert.h>
 #include <locale.h>
 #include <stdio.h>
 #include <string.h>
 #include <wchar.h>
 #include <wctype.h>
 
-int
-main(int argc, char *argv[])
+#include <atf-c.h>
+
+ATF_TC_WITHOUT_HEAD(towctrans_test);
+ATF_TC_BODY(towctrans_test, tc)
 {
 	wctype_t t;
 	int i, j;
@@ -54,39 +55,36 @@ main(int argc, char *argv[])
 		{ "toupper", towupper },
 	};
 
-	printf("1..2\n");
-
-	/*
-	 * C/POSIX locale.
-	 */
+	/* C/POSIX locale. */
 	for (i = 0; i < sizeof(tran) / sizeof(*tran); i++) {
 		t = wctrans(tran[i].name);
-		assert(t != 0);
+		ATF_REQUIRE(t != 0);
 		for (j = 0; j < 256; j++)
-			assert(tran[i].func(j) == towctrans(j, t));
+			ATF_REQUIRE(tran[i].func(j) == towctrans(j, t));
 	}
 	t = wctrans("elephant");
-	assert(t == 0);
+	ATF_REQUIRE(t == 0);
 	for (i = 0; i < 256; i++)
-		assert(towctrans(i, t) == i);
+		ATF_REQUIRE(towctrans(i, t) == i);
 
-	/*
-	 * Japanese (EUC) locale.
-	 */
-	assert(strcmp(setlocale(LC_CTYPE, "ja_JP.eucJP"), "ja_JP.eucJP") == 0);
+	/* Japanese (EUC) locale. */
+	ATF_REQUIRE(strcmp(setlocale(LC_CTYPE, "ja_JP.eucJP"), "ja_JP.eucJP") == 0);
 	for (i = 0; i < sizeof(tran) / sizeof(*tran); i++) {
 		t = wctrans(tran[i].name);
-		assert(t != 0);
+		ATF_REQUIRE(t != 0);
 		for (j = 0; j < 65536; j++)
-			assert(tran[i].func(j) == towctrans(j, t));
+			ATF_REQUIRE(tran[i].func(j) == towctrans(j, t));
 	}
 	t = wctrans("elephant");
-	assert(t == 0);
+	ATF_REQUIRE(t == 0);
 	for (i = 0; i < 65536; i++)
-		assert(towctrans(i, t) == i);
+		ATF_REQUIRE(towctrans(i, t) == i);
+}
+
+ATF_TP_ADD_TCS(tp)
+{
 
-	printf("ok 1 - towctrans()\n");
-	printf("ok 2 - wctrans()\n");
+	ATF_TP_ADD_TC(tp, towctrans_test);
 
-	return (0);
+	return (atf_no_error());
 }
diff --git a/tools/regression/lib/libc/locale/test-wcrtomb.c b/lib/libc/tests/locale/wcrtomb_test.c
index d7ec029..09199ba 100644
--- a/tools/regression/lib/libc/locale/test-wcrtomb.c
+++ b/lib/libc/tests/locale/wcrtomb_test.c
@@ -35,7 +35,6 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
-#include <assert.h>
 #include <errno.h>
 #include <limits.h>
 #include <locale.h>
@@ -44,91 +43,93 @@ __FBSDID("$FreeBSD$");
 #include <string.h>
 #include <wchar.h>
 
-int
-main(int argc, char *argv[])
+#include <atf-c.h>
+
+ATF_TC_WITHOUT_HEAD(wcrtomb_test);
+ATF_TC_BODY(wcrtomb_test, tc)
 {
 	mbstate_t s;
 	size_t len;
 	char buf[MB_LEN_MAX + 1];
 
-	/*
-	 * C/POSIX locale.
-	 */
-
-	printf("1..1\n");
+	/* C/POSIX locale. */
 
-	assert(MB_CUR_MAX == 1);
+	ATF_REQUIRE(MB_CUR_MAX == 1);
 
 	/*
 	 * If the buffer argument is NULL, wc is implicitly L'\0',
 	 * wcrtomb() resets its internal state.
 	 */
-	assert(wcrtomb(NULL, L'\0', NULL) == 1);
-	assert(wcrtomb(NULL, UCHAR_MAX + 1, NULL) == 1);
+	ATF_REQUIRE(wcrtomb(NULL, L'\0', NULL) == 1);
+	ATF_REQUIRE(wcrtomb(NULL, UCHAR_MAX + 1, NULL) == 1);
 
 	/* Null wide character. */
 	memset(&s, 0, sizeof(s));
 	memset(buf, 0xcc, sizeof(buf));
 	len = wcrtomb(buf, L'\0', &s);
-	assert(len == 1);
-	assert((unsigned char)buf[0] == 0 && (unsigned char)buf[1] == 0xcc);
+	ATF_REQUIRE(len == 1);
+	ATF_REQUIRE((unsigned char)buf[0] == 0 && (unsigned char)buf[1] == 0xcc);
 
 	/* Latin letter A, internal state. */
-	assert(wcrtomb(NULL, L'\0', NULL) == 1);
-	assert(wcrtomb(NULL, L'A', NULL) == 1);
+	ATF_REQUIRE(wcrtomb(NULL, L'\0', NULL) == 1);
+	ATF_REQUIRE(wcrtomb(NULL, L'A', NULL) == 1);
 
 	/* Latin letter A. */
 	memset(&s, 0, sizeof(s));
 	memset(buf, 0xcc, sizeof(buf));
 	len = wcrtomb(buf, L'A', &s);
-	assert(len == 1);
-	assert((unsigned char)buf[0] == 'A' && (unsigned char)buf[1] == 0xcc);
+	ATF_REQUIRE(len == 1);
+	ATF_REQUIRE((unsigned char)buf[0] == 'A' && (unsigned char)buf[1] == 0xcc);
 
 	/* Invalid code. */
-	assert(wcrtomb(buf, UCHAR_MAX + 1, NULL) == (size_t)-1);
-	assert(errno == EILSEQ);
+	ATF_REQUIRE(wcrtomb(buf, UCHAR_MAX + 1, NULL) == (size_t)-1);
+	ATF_REQUIRE(errno == EILSEQ);
 
 	/*
 	 * Japanese (EUC) locale.
 	 */
 
-	assert(strcmp(setlocale(LC_CTYPE, "ja_JP.eucJP"), "ja_JP.eucJP") == 0);
-	assert(MB_CUR_MAX == 3);
+	ATF_REQUIRE(strcmp(setlocale(LC_CTYPE, "ja_JP.eucJP"), "ja_JP.eucJP") == 0);
+	ATF_REQUIRE(MB_CUR_MAX == 3);
 
 	/*
 	 * If the buffer argument is NULL, wc is implicitly L'\0',
 	 * wcrtomb() resets its internal state.
 	 */
-	assert(wcrtomb(NULL, L'\0', NULL) == 1);
+	ATF_REQUIRE(wcrtomb(NULL, L'\0', NULL) == 1);
 
 	/* Null wide character. */
 	memset(&s, 0, sizeof(s));
 	memset(buf, 0xcc, sizeof(buf));
 	len = wcrtomb(buf, L'\0', &s);
-	assert(len == 1);
-	assert((unsigned char)buf[0] == 0 && (unsigned char)buf[1] == 0xcc);
+	ATF_REQUIRE(len == 1);
+	ATF_REQUIRE((unsigned char)buf[0] == 0 && (unsigned char)buf[1] == 0xcc);
 
 	/* Latin letter A, internal state. */
-	assert(wcrtomb(NULL, L'\0', NULL) == 1);
-	assert(wcrtomb(NULL, L'A', NULL) == 1);
+	ATF_REQUIRE(wcrtomb(NULL, L'\0', NULL) == 1);
+	ATF_REQUIRE(wcrtomb(NULL, L'A', NULL) == 1);
 
 	/* Latin letter A. */
 	memset(&s, 0, sizeof(s));
 	memset(buf, 0xcc, sizeof(buf));
 	len = wcrtomb(buf, L'A', &s);
-	assert(len == 1);
-	assert((unsigned char)buf[0] == 'A' && (unsigned char)buf[1] == 0xcc);
+	ATF_REQUIRE(len == 1);
+	ATF_REQUIRE((unsigned char)buf[0] == 'A' && (unsigned char)buf[1] == 0xcc);
 
 	/* Full width letter A. */
 	memset(&s, 0, sizeof(s));
 	memset(buf, 0xcc, sizeof(buf));
 	len = wcrtomb(buf, 0xa3c1, &s);
-	assert(len == 2);
-	assert((unsigned char)buf[0] == 0xa3 &&
+	ATF_REQUIRE(len == 2);
+	ATF_REQUIRE((unsigned char)buf[0] == 0xa3 &&
 		(unsigned char)buf[1] == 0xc1 &&
 		(unsigned char)buf[2] == 0xcc);
+}
+
+ATF_TP_ADD_TCS(tp)
+{
 
-	printf("ok 1 - wcrtomb()\n");
+	ATF_TP_ADD_TC(tp, wcrtomb_test);
 
-	return (0);
+	return (atf_no_error());
 }
diff --git a/tools/regression/lib/libc/locale/test-wcsnrtombs.c b/lib/libc/tests/locale/wcsnrtombs_test.c
index 2290cac..2e330a1 100644
--- a/tools/regression/lib/libc/locale/test-wcsnrtombs.c
+++ b/lib/libc/tests/locale/wcsnrtombs_test.c
@@ -34,7 +34,6 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
-#include <assert.h>
 #include <errno.h>
 #include <limits.h>
 #include <locale.h>
@@ -43,19 +42,17 @@ __FBSDID("$FreeBSD$");
 #include <string.h>
 #include <wchar.h>
 
-int
-main(int argc, char *argv[])
+#include <atf-c.h>
+
+ATF_TC_WITHOUT_HEAD(wcsnrtombs_test);
+ATF_TC_BODY(wcsnrtombs_test, tc)
 {
 	wchar_t srcbuf[128];
 	char dstbuf[128];
 	wchar_t *src;
 	mbstate_t s;
 
-	/*
-	 * C/POSIX locale.
-	 */
-
-	printf("1..1\n");
+	/* C/POSIX locale. */
 
 	/* Simple null terminated string. */
 	wmemset(srcbuf, 0xcc, sizeof(srcbuf) / sizeof(*srcbuf));
@@ -63,11 +60,11 @@ main(int argc, char *argv[])
 	memset(dstbuf, 0xcc, sizeof(dstbuf));
 	src = srcbuf;
 	memset(&s, 0, sizeof(s));
-	assert(wcsnrtombs(dstbuf, (const wchar_t **)&src, 6, sizeof(dstbuf),
+	ATF_REQUIRE(wcsnrtombs(dstbuf, (const wchar_t **)&src, 6, sizeof(dstbuf),
 	    &s) == 5);
-	assert(strcmp(dstbuf, "hello") == 0);
-	assert((unsigned char)dstbuf[6] == 0xcc);
-	assert(src == NULL);
+	ATF_REQUIRE(strcmp(dstbuf, "hello") == 0);
+	ATF_REQUIRE((unsigned char)dstbuf[6] == 0xcc);
+	ATF_REQUIRE(src == NULL);
 
 	/* Simple null terminated string, stopping early. */
 	wmemset(srcbuf, 0xcc, sizeof(srcbuf) / sizeof(*srcbuf));
@@ -75,11 +72,11 @@ main(int argc, char *argv[])
 	memset(dstbuf, 0xcc, sizeof(dstbuf));
 	src = srcbuf;
 	memset(&s, 0, sizeof(s));
-	assert(wcsnrtombs(dstbuf, (const wchar_t **)&src, 4, sizeof(dstbuf),
+	ATF_REQUIRE(wcsnrtombs(dstbuf, (const wchar_t **)&src, 4, sizeof(dstbuf),
 	    &s) == 4);
-	assert(memcmp(dstbuf, "hell", 4) == 0);
-	assert((unsigned char)dstbuf[5] == 0xcc);
-	assert(src == srcbuf + 4);
+	ATF_REQUIRE(memcmp(dstbuf, "hell", 4) == 0);
+	ATF_REQUIRE((unsigned char)dstbuf[5] == 0xcc);
+	ATF_REQUIRE(src == srcbuf + 4);
 
 	/* Not enough space in destination buffer. */
 	wmemset(srcbuf, 0xcc, sizeof(srcbuf) / sizeof(*srcbuf));
@@ -87,18 +84,18 @@ main(int argc, char *argv[])
 	memset(dstbuf, 0xcc, sizeof(dstbuf));
 	src = srcbuf;
 	memset(&s, 0, sizeof(s));
-	assert(wcsnrtombs(dstbuf, (const wchar_t **)&src, 6, 4,
+	ATF_REQUIRE(wcsnrtombs(dstbuf, (const wchar_t **)&src, 6, 4,
 	    &s) == 4);
-	assert(memcmp(dstbuf, "hell", 4) == 0);
-	assert((unsigned char)dstbuf[5] == 0xcc);
-	assert(src == srcbuf + 4);
+	ATF_REQUIRE(memcmp(dstbuf, "hell", 4) == 0);
+	ATF_REQUIRE((unsigned char)dstbuf[5] == 0xcc);
+	ATF_REQUIRE(src == srcbuf + 4);
 
 	/* Null terminated string, internal dest. buffer */
 	wmemset(srcbuf, 0xcc, sizeof(srcbuf) / sizeof(*srcbuf));
 	wcscpy(srcbuf, L"hello");
 	src = srcbuf;
 	memset(&s, 0, sizeof(s));
-	assert(wcsnrtombs(NULL, (const wchar_t **)&src, 6, sizeof(dstbuf),
+	ATF_REQUIRE(wcsnrtombs(NULL, (const wchar_t **)&src, 6, sizeof(dstbuf),
 	    &s) == 5);
 
 	/* Null terminated string, internal dest. buffer, stopping early. */
@@ -106,7 +103,7 @@ main(int argc, char *argv[])
 	wcscpy(srcbuf, L"hello");
 	src = srcbuf;
 	memset(&s, 0, sizeof(s));
-	assert(wcsnrtombs(NULL, (const wchar_t **)&src, 4, sizeof(dstbuf),
+	ATF_REQUIRE(wcsnrtombs(NULL, (const wchar_t **)&src, 4, sizeof(dstbuf),
 	    &s) == 4);
 
 	/* Null terminated string, internal state. */
@@ -114,17 +111,17 @@ main(int argc, char *argv[])
 	wcscpy(srcbuf, L"hello");
 	memset(dstbuf, 0xcc, sizeof(dstbuf));
 	src = srcbuf;
-	assert(wcsnrtombs(dstbuf, (const wchar_t **)&src, 6, sizeof(dstbuf),
+	ATF_REQUIRE(wcsnrtombs(dstbuf, (const wchar_t **)&src, 6, sizeof(dstbuf),
 	    NULL) == 5);
-	assert(strcmp(dstbuf, "hello") == 0);
-	assert((unsigned char)dstbuf[6] == 0xcc);
-	assert(src == NULL);
+	ATF_REQUIRE(strcmp(dstbuf, "hello") == 0);
+	ATF_REQUIRE((unsigned char)dstbuf[6] == 0xcc);
+	ATF_REQUIRE(src == NULL);
 
 	/* Null terminated string, internal state, internal dest. buffer. */
 	wmemset(srcbuf, 0xcc, sizeof(srcbuf) / sizeof(*srcbuf));
 	wcscpy(srcbuf, L"hello");
 	src = srcbuf;
-	assert(wcsnrtombs(NULL, (const wchar_t **)&src, 6, 0, NULL) == 5);
+	ATF_REQUIRE(wcsnrtombs(NULL, (const wchar_t **)&src, 6, 0, NULL) == 5);
 
 	/* Empty source buffer. */
 	wmemset(srcbuf, 0xcc, sizeof(srcbuf) / sizeof(*srcbuf));
@@ -132,9 +129,9 @@ main(int argc, char *argv[])
 	memset(dstbuf, 0xcc, sizeof(dstbuf));
 	src = srcbuf;
 	memset(&s, 0, sizeof(s));
-	assert(wcsnrtombs(dstbuf, (const wchar_t **)&src, 1, sizeof(dstbuf),
+	ATF_REQUIRE(wcsnrtombs(dstbuf, (const wchar_t **)&src, 1, sizeof(dstbuf),
 	    &s) == 0);
-	assert(dstbuf[0] == L'\0');
+	ATF_REQUIRE(dstbuf[0] == L'\0');
 
 	/* Zero length destination buffer. */
 	wmemset(srcbuf, 0xcc, sizeof(srcbuf) / sizeof(*srcbuf));
@@ -142,25 +139,25 @@ main(int argc, char *argv[])
 	memset(dstbuf, 0xcc, sizeof(dstbuf));
 	src = srcbuf;
 	memset(&s, 0, sizeof(s));
-	assert(wcsnrtombs(dstbuf, (const wchar_t **)&src, 6, 0, &s) == 0);
-	assert((unsigned char)dstbuf[0] == 0xcc);
+	ATF_REQUIRE(wcsnrtombs(dstbuf, (const wchar_t **)&src, 6, 0, &s) == 0);
+	ATF_REQUIRE((unsigned char)dstbuf[0] == 0xcc);
 
 	/* Zero length source buffer. */
 	wmemset(srcbuf, 0xcc, sizeof(srcbuf) / sizeof(*srcbuf));
 	memset(dstbuf, 0xcc, sizeof(dstbuf));
 	src = srcbuf;
 	memset(&s, 0, sizeof(s));
-	assert(wcsnrtombs(dstbuf, (const wchar_t **)&src, 0, sizeof(dstbuf),
+	ATF_REQUIRE(wcsnrtombs(dstbuf, (const wchar_t **)&src, 0, sizeof(dstbuf),
 	    &s) == 0);
-	assert((unsigned char)dstbuf[0] == 0xcc);
-	assert(src == srcbuf);
+	ATF_REQUIRE((unsigned char)dstbuf[0] == 0xcc);
+	ATF_REQUIRE(src == srcbuf);
 
 	/*
 	 * Japanese (EUC) locale.
 	 */
 
-	assert(strcmp(setlocale(LC_CTYPE, "ja_JP.eucJP"), "ja_JP.eucJP") == 0);
-	assert(MB_CUR_MAX > 1);
+	ATF_REQUIRE(strcmp(setlocale(LC_CTYPE, "ja_JP.eucJP"), "ja_JP.eucJP") == 0);
+	ATF_REQUIRE(MB_CUR_MAX > 1);
 
 	wmemset(srcbuf, 0xcc, sizeof(srcbuf) / sizeof(*srcbuf));
 	srcbuf[0] = 0xA3C1;
@@ -172,23 +169,27 @@ main(int argc, char *argv[])
 	memset(dstbuf, 0xcc, sizeof(dstbuf));
 	src = srcbuf;
 	memset(&s, 0, sizeof(s));
-	assert(wcsnrtombs(dstbuf, (const wchar_t **)&src, 6, sizeof(dstbuf),
+	ATF_REQUIRE(wcsnrtombs(dstbuf, (const wchar_t **)&src, 6, sizeof(dstbuf),
 	    &s) == 7);
-	assert(strcmp(dstbuf, "\xA3\xC1 B \xA3\xC3") == 0);
-	assert((unsigned char)dstbuf[8] == 0xcc);
-	assert(src == NULL);
+	ATF_REQUIRE(strcmp(dstbuf, "\xA3\xC1 B \xA3\xC3") == 0);
+	ATF_REQUIRE((unsigned char)dstbuf[8] == 0xcc);
+	ATF_REQUIRE(src == NULL);
 
 	/* Stopping early. */
 	memset(dstbuf, 0xcc, sizeof(dstbuf));
 	src = srcbuf;
 	memset(&s, 0, sizeof(s));
-	assert(wcsnrtombs(dstbuf, (const wchar_t **)&src, 6, 6,
+	ATF_REQUIRE(wcsnrtombs(dstbuf, (const wchar_t **)&src, 6, 6,
 	    &s) == 5);
-	assert(memcmp(dstbuf, "\xA3\xC1 B ", 5) == 0);
-	assert((unsigned char)dstbuf[5] == 0xcc);
-	assert(src == srcbuf + 4);
+	ATF_REQUIRE(memcmp(dstbuf, "\xA3\xC1 B ", 5) == 0);
+	ATF_REQUIRE((unsigned char)dstbuf[5] == 0xcc);
+	ATF_REQUIRE(src == srcbuf + 4);
+}
+
+ATF_TP_ADD_TCS(tp)
+{
 
-	printf("ok 1 - wcsnrtombs()\n");
+	ATF_TP_ADD_TC(tp, wcsnrtombs_test);
 
-	return (0);
+	return (atf_no_error());
 }
diff --git a/tools/regression/lib/libc/locale/test-wcsrtombs.c b/lib/libc/tests/locale/wcsrtombs_test.c
index 48fe366..4c3455a8 100644
--- a/tools/regression/lib/libc/locale/test-wcsrtombs.c
+++ b/lib/libc/tests/locale/wcsrtombs_test.c
@@ -35,7 +35,6 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
-#include <assert.h>
 #include <errno.h>
 #include <limits.h>
 #include <locale.h>
@@ -44,19 +43,17 @@ __FBSDID("$FreeBSD$");
 #include <string.h>
 #include <wchar.h>
 
-int
-main(int argc, char *argv[])
+#include <atf-c.h>
+
+ATF_TC_WITHOUT_HEAD(wcsrtombs_test);
+ATF_TC_BODY(wcsrtombs_test, tc)
 {
 	wchar_t srcbuf[128];
 	char dstbuf[128];
 	wchar_t *src;
 	mbstate_t s;
 
-	/*
-	 * C/POSIX locale.
-	 */
-
-	printf("1..1\n");
+	/* C/POSIX locale. */
 
 	/* Simple null terminated string. */
 	wmemset(srcbuf, 0xcc, sizeof(srcbuf) / sizeof(*srcbuf));
@@ -64,11 +61,11 @@ main(int argc, char *argv[])
 	memset(dstbuf, 0xcc, sizeof(dstbuf));
 	src = srcbuf;
 	memset(&s, 0, sizeof(s));
-	assert(wcsrtombs(dstbuf, (const wchar_t **)&src, sizeof(dstbuf),
+	ATF_REQUIRE(wcsrtombs(dstbuf, (const wchar_t **)&src, sizeof(dstbuf),
 	    &s) == 5);
-	assert(strcmp(dstbuf, "hello") == 0);
-	assert((unsigned char)dstbuf[6] == 0xcc);
-	assert(src == NULL);
+	ATF_REQUIRE(strcmp(dstbuf, "hello") == 0);
+	ATF_REQUIRE((unsigned char)dstbuf[6] == 0xcc);
+	ATF_REQUIRE(src == NULL);
 
 	/* Not enough space in destination buffer. */
 	wmemset(srcbuf, 0xcc, sizeof(srcbuf) / sizeof(*srcbuf));
@@ -76,18 +73,18 @@ main(int argc, char *argv[])
 	memset(dstbuf, 0xcc, sizeof(dstbuf));
 	src = srcbuf;
 	memset(&s, 0, sizeof(s));
-	assert(wcsrtombs(dstbuf, (const wchar_t **)&src, 4,
+	ATF_REQUIRE(wcsrtombs(dstbuf, (const wchar_t **)&src, 4,
 	    &s) == 4);
-	assert(memcmp(dstbuf, "hell", 4) == 0);
-	assert((unsigned char)dstbuf[5] == 0xcc);
-	assert(src == srcbuf + 4);
+	ATF_REQUIRE(memcmp(dstbuf, "hell", 4) == 0);
+	ATF_REQUIRE((unsigned char)dstbuf[5] == 0xcc);
+	ATF_REQUIRE(src == srcbuf + 4);
 
 	/* Null terminated string, internal dest. buffer */
 	wmemset(srcbuf, 0xcc, sizeof(srcbuf) / sizeof(*srcbuf));
 	wcscpy(srcbuf, L"hello");
 	src = srcbuf;
 	memset(&s, 0, sizeof(s));
-	assert(wcsrtombs(NULL, (const wchar_t **)&src, sizeof(dstbuf),
+	ATF_REQUIRE(wcsrtombs(NULL, (const wchar_t **)&src, sizeof(dstbuf),
 	    &s) == 5);
 
 	/* Null terminated string, internal state. */
@@ -95,17 +92,17 @@ main(int argc, char *argv[])
 	wcscpy(srcbuf, L"hello");
 	memset(dstbuf, 0xcc, sizeof(dstbuf));
 	src = srcbuf;
-	assert(wcsrtombs(dstbuf, (const wchar_t **)&src, sizeof(dstbuf),
+	ATF_REQUIRE(wcsrtombs(dstbuf, (const wchar_t **)&src, sizeof(dstbuf),
 	    NULL) == 5);
-	assert(strcmp(dstbuf, "hello") == 0);
-	assert((unsigned char)dstbuf[6] == 0xcc);
-	assert(src == NULL);
+	ATF_REQUIRE(strcmp(dstbuf, "hello") == 0);
+	ATF_REQUIRE((unsigned char)dstbuf[6] == 0xcc);
+	ATF_REQUIRE(src == NULL);
 
 	/* Null terminated string, internal state, internal dest. buffer. */
 	wmemset(srcbuf, 0xcc, sizeof(srcbuf) / sizeof(*srcbuf));
 	wcscpy(srcbuf, L"hello");
 	src = srcbuf;
-	assert(wcsrtombs(NULL, (const wchar_t **)&src, 0, NULL) == 5);
+	ATF_REQUIRE(wcsrtombs(NULL, (const wchar_t **)&src, 0, NULL) == 5);
 
 	/* Empty source buffer. */
 	wmemset(srcbuf, 0xcc, sizeof(srcbuf) / sizeof(*srcbuf));
@@ -113,9 +110,9 @@ main(int argc, char *argv[])
 	memset(dstbuf, 0xcc, sizeof(dstbuf));
 	src = srcbuf;
 	memset(&s, 0, sizeof(s));
-	assert(wcsrtombs(dstbuf, (const wchar_t **)&src, sizeof(dstbuf),
+	ATF_REQUIRE(wcsrtombs(dstbuf, (const wchar_t **)&src, sizeof(dstbuf),
 	    &s) == 0);
-	assert(dstbuf[0] == L'\0');
+	ATF_REQUIRE(dstbuf[0] == L'\0');
 
 	/* Zero length destination buffer. */
 	wmemset(srcbuf, 0xcc, sizeof(srcbuf) / sizeof(*srcbuf));
@@ -123,15 +120,15 @@ main(int argc, char *argv[])
 	memset(dstbuf, 0xcc, sizeof(dstbuf));
 	src = srcbuf;
 	memset(&s, 0, sizeof(s));
-	assert(wcsrtombs(dstbuf, (const wchar_t **)&src, 0, &s) == 0);
-	assert((unsigned char)dstbuf[0] == 0xcc);
+	ATF_REQUIRE(wcsrtombs(dstbuf, (const wchar_t **)&src, 0, &s) == 0);
+	ATF_REQUIRE((unsigned char)dstbuf[0] == 0xcc);
 
 	/*
 	 * Japanese (EUC) locale.
 	 */
 
-	assert(strcmp(setlocale(LC_CTYPE, "ja_JP.eucJP"), "ja_JP.eucJP") == 0);
-	assert(MB_CUR_MAX > 1);
+	ATF_REQUIRE(strcmp(setlocale(LC_CTYPE, "ja_JP.eucJP"), "ja_JP.eucJP") == 0);
+	ATF_REQUIRE(MB_CUR_MAX > 1);
 
 	wmemset(srcbuf, 0xcc, sizeof(srcbuf) / sizeof(*srcbuf));
 	srcbuf[0] = 0xA3C1;
@@ -143,13 +140,17 @@ main(int argc, char *argv[])
 	memset(dstbuf, 0xcc, sizeof(dstbuf));
 	src = srcbuf;
 	memset(&s, 0, sizeof(s));
-	assert(wcsrtombs(dstbuf, (const wchar_t **)&src, sizeof(dstbuf),
+	ATF_REQUIRE(wcsrtombs(dstbuf, (const wchar_t **)&src, sizeof(dstbuf),
 	    &s) == 7);
-	assert(strcmp(dstbuf, "\xA3\xC1 B \xA3\xC3") == 0);
-	assert((unsigned char)dstbuf[8] == 0xcc);
-	assert(src == NULL);
+	ATF_REQUIRE(strcmp(dstbuf, "\xA3\xC1 B \xA3\xC3") == 0);
+	ATF_REQUIRE((unsigned char)dstbuf[8] == 0xcc);
+	ATF_REQUIRE(src == NULL);
+}
+
+ATF_TP_ADD_TCS(tp)
+{
 
-	printf("ok 1 - wcsrtombs()\n");
+	ATF_TP_ADD_TC(tp, wcsrtombs_test);
 
-	return (0);
+	return (atf_no_error());
 }
diff --git a/tools/regression/lib/libc/locale/test-wcstombs.c b/lib/libc/tests/locale/wcstombs_test.c
index f4a5770..f3fcd81 100644
--- a/tools/regression/lib/libc/locale/test-wcstombs.c
+++ b/lib/libc/tests/locale/wcstombs_test.c
@@ -35,7 +35,6 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
-#include <assert.h>
 #include <errno.h>
 #include <limits.h>
 #include <locale.h>
@@ -44,72 +43,70 @@ __FBSDID("$FreeBSD$");
 #include <string.h>
 #include <wchar.h>
 
-int
-main(int argc, char *argv[])
+#include <atf-c.h>
+
+ATF_TC_WITHOUT_HEAD(wcstombs_test);
+ATF_TC_BODY(wcstombs_test, tc)
 {
 	wchar_t srcbuf[128];
 	char dstbuf[128];
 
-	/*
-	 * C/POSIX locale.
-	 */
-
-	printf("1..1\n");
+	/* C/POSIX locale. */
 
 	/* Simple null terminated string. */
 	wmemset(srcbuf, 0xcc, sizeof(srcbuf) / sizeof(*srcbuf));
 	wcscpy(srcbuf, L"hello");
 	memset(dstbuf, 0xcc, sizeof(dstbuf));
-	assert(wcstombs(dstbuf, srcbuf, sizeof(dstbuf)) == 5);
-	assert(strcmp(dstbuf, "hello") == 0);
-	assert((unsigned char)dstbuf[6] == 0xcc);
+	ATF_REQUIRE(wcstombs(dstbuf, srcbuf, sizeof(dstbuf)) == 5);
+	ATF_REQUIRE(strcmp(dstbuf, "hello") == 0);
+	ATF_REQUIRE((unsigned char)dstbuf[6] == 0xcc);
 
 	/* Not enough space in destination buffer. */
 	wmemset(srcbuf, 0xcc, sizeof(srcbuf) / sizeof(*srcbuf));
 	wcscpy(srcbuf, L"hello");
 	memset(dstbuf, 0xcc, sizeof(dstbuf));
-	assert(wcstombs(dstbuf, srcbuf, 4) == 4);
-	assert(memcmp(dstbuf, "hell", 4) == 0);
-	assert((unsigned char)dstbuf[5] == 0xcc);
+	ATF_REQUIRE(wcstombs(dstbuf, srcbuf, 4) == 4);
+	ATF_REQUIRE(memcmp(dstbuf, "hell", 4) == 0);
+	ATF_REQUIRE((unsigned char)dstbuf[5] == 0xcc);
 
 	/* Null terminated string, internal dest. buffer */
 	wmemset(srcbuf, 0xcc, sizeof(srcbuf) / sizeof(*srcbuf));
 	wcscpy(srcbuf, L"hello");
-	assert(wcstombs(NULL, srcbuf, sizeof(dstbuf)) == 5);
+	ATF_REQUIRE(wcstombs(NULL, srcbuf, sizeof(dstbuf)) == 5);
 
 	/* Null terminated string, internal state. */
 	wmemset(srcbuf, 0xcc, sizeof(srcbuf) / sizeof(*srcbuf));
 	wcscpy(srcbuf, L"hello");
 	memset(dstbuf, 0xcc, sizeof(dstbuf));
-	assert(wcstombs(dstbuf, srcbuf, sizeof(dstbuf)) == 5);
-	assert(strcmp(dstbuf, "hello") == 0);
-	assert((unsigned char)dstbuf[6] == 0xcc);
+	ATF_REQUIRE(wcstombs(dstbuf, srcbuf, sizeof(dstbuf)) == 5);
+	ATF_REQUIRE(strcmp(dstbuf, "hello") == 0);
+	ATF_REQUIRE((unsigned char)dstbuf[6] == 0xcc);
 
 	/* Null terminated string, internal state, internal dest. buffer. */
 	wmemset(srcbuf, 0xcc, sizeof(srcbuf) / sizeof(*srcbuf));
 	wcscpy(srcbuf, L"hello");
-	assert(wcstombs(NULL, srcbuf, 0) == 5);
+	ATF_REQUIRE(wcstombs(NULL, srcbuf, 0) == 5);
 
 	/* Empty source buffer. */
 	wmemset(srcbuf, 0xcc, sizeof(srcbuf) / sizeof(*srcbuf));
 	srcbuf[0] = L'\0';
 	memset(dstbuf, 0xcc, sizeof(dstbuf));
-	assert(wcstombs(dstbuf, srcbuf, sizeof(dstbuf)) == 0);
-	assert(dstbuf[0] == L'\0');
+	ATF_REQUIRE(wcstombs(dstbuf, srcbuf, sizeof(dstbuf)) == 0);
+	ATF_REQUIRE(dstbuf[0] == L'\0');
 
 	/* Zero length destination buffer. */
 	wmemset(srcbuf, 0xcc, sizeof(srcbuf) / sizeof(*srcbuf));
 	wcscpy(srcbuf, L"hello");
 	memset(dstbuf, 0xcc, sizeof(dstbuf));
-	assert(wcstombs(dstbuf, srcbuf, 0) == 0);
-	assert((unsigned char)dstbuf[0] == 0xcc);
+	ATF_REQUIRE(wcstombs(dstbuf, srcbuf, 0) == 0);
+	ATF_REQUIRE((unsigned char)dstbuf[0] == 0xcc);
 
 	/*
 	 * Japanese (EUC) locale.
 	 */
 
-	assert(strcmp(setlocale(LC_CTYPE, "ja_JP.eucJP"), "ja_JP.eucJP") == 0);
-	assert(MB_CUR_MAX > 1);
+	ATF_REQUIRE(strcmp(setlocale(LC_CTYPE, "ja_JP.eucJP"), "ja_JP.eucJP") == 0);
+	ATF_REQUIRE(MB_CUR_MAX > 1);
 
 	wmemset(srcbuf, 0xcc, sizeof(srcbuf) / sizeof(*srcbuf));
 	srcbuf[0] = 0xA3C1;
@@ -119,11 +116,15 @@ main(int argc, char *argv[])
 	srcbuf[4] = 0xA3C3;
 	srcbuf[5] = 0x0000;
 	memset(dstbuf, 0xcc, sizeof(dstbuf));
-	assert(wcstombs(dstbuf, srcbuf, sizeof(dstbuf)) == 7);
-	assert(strcmp(dstbuf, "\xA3\xC1 B \xA3\xC3") == 0);
-	assert((unsigned char)dstbuf[8] == 0xcc);
+	ATF_REQUIRE(wcstombs(dstbuf, srcbuf, sizeof(dstbuf)) == 7);
+	ATF_REQUIRE(strcmp(dstbuf, "\xA3\xC1 B \xA3\xC3") == 0);
+	ATF_REQUIRE((unsigned char)dstbuf[8] == 0xcc);
+}
+
+ATF_TP_ADD_TCS(tp)
+{
 
-	printf("ok 1 - wcstombs()\n");
+	ATF_TP_ADD_TC(tp, wcstombs_test);
 
-	return (0);
+	return (atf_no_error());
 }
diff --git a/tools/regression/lib/libc/locale/test-wctomb.c b/lib/libc/tests/locale/wctomb_test.c
index d55b3db..255cda4 100644
--- a/tools/regression/lib/libc/locale/test-wctomb.c
+++ b/lib/libc/tests/locale/wctomb_test.c
@@ -35,7 +35,6 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
-#include <assert.h>
 #include <errno.h>
 #include <limits.h>
 #include <locale.h>
@@ -43,70 +42,72 @@ __FBSDID("$FreeBSD$");
 #include <stdlib.h>
 #include <string.h>
 
-int
-main(int argc, char *argv[])
+#include <atf-c.h>
+
+ATF_TC_WITHOUT_HEAD(wctomb_test);
+ATF_TC_BODY(wctomb_test, tc)
 {
 	size_t len;
 	char buf[MB_LEN_MAX + 1];
 
-	/*
-	 * C/POSIX locale.
-	 */
-
-	printf("1..1\n");
+	/* C/POSIX locale. */
 
-	assert(MB_CUR_MAX == 1);
+	ATF_REQUIRE(MB_CUR_MAX == 1);
 
 	/* No shift states in C locale. */
-	assert(wctomb(NULL, L'\0') == 0);
+	ATF_REQUIRE(wctomb(NULL, L'\0') == 0);
 
 	/* Null wide character. */
 	memset(buf, 0xcc, sizeof(buf));
 	len = wctomb(buf, L'\0');
-	assert(len == 1);
-	assert((unsigned char)buf[0] == 0 && (unsigned char)buf[1] == 0xcc);
+	ATF_REQUIRE(len == 1);
+	ATF_REQUIRE((unsigned char)buf[0] == 0 && (unsigned char)buf[1] == 0xcc);
 
 	/* Latin letter A. */
 	memset(buf, 0xcc, sizeof(buf));
 	len = wctomb(buf, L'A');
-	assert(len == 1);
-	assert((unsigned char)buf[0] == 'A' && (unsigned char)buf[1] == 0xcc);
+	ATF_REQUIRE(len == 1);
+	ATF_REQUIRE((unsigned char)buf[0] == 'A' && (unsigned char)buf[1] == 0xcc);
 
 	/* Invalid code. */
-	assert(wctomb(buf, UCHAR_MAX + 1) == -1);
-	assert(wctomb(NULL, 0) == 0);
+	ATF_REQUIRE(wctomb(buf, UCHAR_MAX + 1) == -1);
+	ATF_REQUIRE(wctomb(NULL, 0) == 0);
 
 	/*
 	 * Japanese (EUC) locale.
 	 */
 
-	assert(strcmp(setlocale(LC_CTYPE, "ja_JP.eucJP"), "ja_JP.eucJP") == 0);
-	assert(MB_CUR_MAX == 3);
+	ATF_REQUIRE(strcmp(setlocale(LC_CTYPE, "ja_JP.eucJP"), "ja_JP.eucJP") == 0);
+	ATF_REQUIRE(MB_CUR_MAX == 3);
 
 	/* No shift states in EUC encoding. */
-	assert(wctomb(NULL, L'\0') == 0);
+	ATF_REQUIRE(wctomb(NULL, L'\0') == 0);
 
 	/* Null wide character. */
 	memset(buf, 0xcc, sizeof(buf));
 	len = wctomb(buf, L'\0');
-	assert(len == 1);
-	assert((unsigned char)buf[0] == 0 && (unsigned char)buf[1] == 0xcc);
+	ATF_REQUIRE(len == 1);
+	ATF_REQUIRE((unsigned char)buf[0] == 0 && (unsigned char)buf[1] == 0xcc);
 
 	/* Latin letter A. */
 	memset(buf, 0xcc, sizeof(buf));
 	len = wctomb(buf, L'A');
-	assert(len == 1);
-	assert((unsigned char)buf[0] == 'A' && (unsigned char)buf[1] == 0xcc);
+	ATF_REQUIRE(len == 1);
+	ATF_REQUIRE((unsigned char)buf[0] == 'A' && (unsigned char)buf[1] == 0xcc);
 
 	/* Full width letter A. */
 	memset(buf, 0xcc, sizeof(buf));
 	len = wctomb(buf, 0xa3c1);
-	assert(len == 2);
-	assert((unsigned char)buf[0] == 0xa3 &&
+	ATF_REQUIRE(len == 2);
+	ATF_REQUIRE((unsigned char)buf[0] == 0xa3 &&
 		(unsigned char)buf[1] == 0xc1 &&
 		(unsigned char)buf[2] == 0xcc);
+}
+
+ATF_TP_ADD_TCS(tp)
+{
 
-	printf("ok 1 - wctomb()\n");
+	ATF_TP_ADD_TC(tp, wctomb_test);
 
-	return (0);
+	return (atf_no_error());
 }
diff --git a/lib/libc/tests/net/Makefile b/lib/libc/tests/net/Makefile
index eb199f0..b8f6d97 100644
--- a/lib/libc/tests/net/Makefile
+++ b/lib/libc/tests/net/Makefile
@@ -1,24 +1,25 @@
 # $FreeBSD$
 
-.include <bsd.own.mk>
-
 TESTSDIR=	${TESTSBASE}/lib/libc/net
 
-BINDIR=		${TESTSDIR}
+ATF_TESTS_C+=	ether_test
+ATF_TESTS_C+=	eui64_aton_test
+ATF_TESTS_C+=	eui64_ntoa_test
 
-NETBSD_ATF_TESTS_C=	getprotoent_test
-NETBSD_ATF_TESTS_C+=	ether_aton_test
+CFLAGS+=	-I${.CURDIR}
 
-SRCS.t_ether_aton=	aton_ether_subr.c t_ether_aton.c
+NETBSD_ATF_TESTS_C+=	getprotoent_test
+NETBSD_ATF_TESTS_C+=	ether_aton_test
 
-aton_ether_subr.c: gen_ether_subr ${.CURDIR:H:H:H:H}/sys/net/if_ethersubr.c
-	${HOST_SH} ${.ALLSRC} ${.TARGET}
+SRCS.ether_aton_test=	aton_ether_subr.c t_ether_aton.c
 
 # TODO: hostent_test
 NETBSD_ATF_TESTS_SH=	nsdispatch_test
 NETBSD_ATF_TESTS_SH+=	protoent_test
 NETBSD_ATF_TESTS_SH+=	servent_test
 
+BINDIR=		${TESTSDIR}
+
 PROGS=		h_nsd_recurse
 PROGS+=		h_protoent
 PROGS+=		h_servent
@@ -28,12 +29,14 @@ DPADD.h_nsd_recurse+=	${LIBPTHREAD}
 LDADD.h_nsd_recurse+=	-lpthread
 
 CLEANFILES+=	aton_ether_subr.c
+aton_ether_subr.c: gen_ether_subr ${SRCTOP}/sys/net/if_ethersubr.c
+	${HOST_SH} ${.ALLSRC} ${.TARGET}
 
 .include "../Makefile.netbsd-tests"
 
 # TODO: the testcases needs to be ported to FreeBSD
 #TESTS_SUBDIRS=	getaddrinfo
-FILES=	hosts
+FILES+=	hosts
 FILES+=	resolv.conf
 
 .include <bsd.test.mk>
diff --git a/lib/libc/tests/net/ether_test.c b/lib/libc/tests/net/ether_test.c
new file mode 100644
index 0000000..5b596f0
--- /dev/null
+++ b/lib/libc/tests/net/ether_test.c
@@ -0,0 +1,192 @@
+/*-
+ * Copyright (c) 2007 Robert N. M. Watson
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <sys/types.h>
+
+#include <net/ethernet.h>
+
+#include <errno.h>
+#include <stdio.h>
+#include <string.h>
+
+#include <atf-c.h>
+
+static const char *ether_line_string = "01:23:45:67:89:ab ether_line_hostname";
+static const char *ether_line_hostname = "ether_line_hostname";
+static const struct ether_addr	 ether_line_addr = {
+	{ 0x01, 0x23, 0x45, 0x67, 0x89, 0xab }
+};
+
+ATF_TC_WITHOUT_HEAD(ether_line);
+ATF_TC_BODY(ether_line, tc)
+{
+	struct ether_addr e;
+	char hostname[256];
+
+	ATF_REQUIRE_MSG(ether_line(ether_line_string, &e, hostname) == 0,
+	    "ether_line failed; errno=%d", errno);
+	ATF_REQUIRE_MSG(bcmp(&e, &ether_line_addr, ETHER_ADDR_LEN) == 0,
+	    "bad address");
+	ATF_REQUIRE_MSG(strcmp(hostname, ether_line_hostname) == 0,
+	    "bad hostname");
+}
+
+static const char *ether_line_bad_1_string = "x";
+
+ATF_TC_WITHOUT_HEAD(ether_line_bad_1);
+ATF_TC_BODY(ether_line_bad_1, tc)
+{
+	struct ether_addr e;
+	char hostname[256];
+
+	ATF_REQUIRE_MSG(ether_line(ether_line_bad_1_string, &e, hostname) != 0,
+	    "ether_line succeeded unexpectedly");
+}
+
+static const char *ether_line_bad_2_string = "x x";
+
+ATF_TC_WITHOUT_HEAD(ether_line_bad_2);
+ATF_TC_BODY(ether_line_bad_2, tc)
+{
+	struct ether_addr e;
+	char hostname[256];
+
+	ATF_REQUIRE_MSG(ether_line(ether_line_bad_2_string, &e, hostname) != 0,
+	    "ether_line succeeded unexpectedly");
+}
+
+static const char *ether_aton_string = "01:23:45:67:89:ab";
+static const struct ether_addr	 ether_aton_addr = {
+	{ 0x01, 0x23, 0x45, 0x67, 0x89, 0xab }
+};
+
+ATF_TC_WITHOUT_HEAD(ether_aton_r);
+ATF_TC_BODY(ether_aton_r, tc)
+{
+	struct ether_addr e, *ep;
+
+	ep = ether_aton_r(ether_aton_string, &e);
+
+	ATF_REQUIRE_MSG(ep != NULL, "ether_aton_r failed; errno=%d", errno);
+	ATF_REQUIRE_MSG(ep == &e,
+	    "ether_aton_r returned different pointers; %p != %p", ep, &e);
+}
+
+static const char		*ether_aton_bad_string = "x";
+
+ATF_TC_WITHOUT_HEAD(ether_aton_r_bad);
+ATF_TC_BODY(ether_aton_r_bad, tc)
+{
+	struct ether_addr e, *ep;
+
+	ep = ether_aton_r(ether_aton_bad_string, &e);
+	ATF_REQUIRE_MSG(ep == NULL, "ether_aton_r succeeded unexpectedly");
+}
+
+ATF_TC_WITHOUT_HEAD(ether_aton);
+ATF_TC_BODY(ether_aton, tc)
+{
+	struct ether_addr *ep;
+
+	ep = ether_aton(ether_aton_string);
+	ATF_REQUIRE_MSG(ep != NULL, "ether_aton failed");
+	ATF_REQUIRE_MSG(bcmp(ep, &ether_aton_addr, ETHER_ADDR_LEN) == 0,
+	    "bad address");
+}
+
+ATF_TC_WITHOUT_HEAD(ether_aton_bad);
+ATF_TC_BODY(ether_aton_bad, tc)
+{
+	struct ether_addr *ep;
+
+	ep = ether_aton(ether_aton_bad_string);
+	ATF_REQUIRE_MSG(ep == NULL, "ether_aton succeeded unexpectedly");
+}
+
+static const char *ether_ntoa_string = "01:23:45:67:89:ab";
+static const struct ether_addr	 ether_ntoa_addr = {
+	{ 0x01, 0x23, 0x45, 0x67, 0x89, 0xab }
+};
+
+ATF_TC_WITHOUT_HEAD(ether_ntoa_r);
+ATF_TC_BODY(ether_ntoa_r, tc)
+{
+	char buf[256], *cp;
+
+	cp = ether_ntoa_r(&ether_ntoa_addr, buf);
+	ATF_REQUIRE_MSG(cp != NULL, "ether_ntoa_r failed");
+	ATF_REQUIRE_MSG(cp == buf,
+	    "ether_ntoa_r returned a different pointer; %p != %p", cp, buf);
+	ATF_REQUIRE_MSG(strcmp(cp, ether_ntoa_string) == 0,
+	    "strings did not match (`%s` != `%s`)", cp, ether_ntoa_string);
+}
+
+ATF_TC_WITHOUT_HEAD(ether_ntoa);
+ATF_TC_BODY(ether_ntoa, tc)
+{
+	char *cp;
+
+	cp = ether_ntoa(&ether_ntoa_addr);
+	ATF_REQUIRE_MSG(cp != NULL, "ether_ntoa failed");
+	ATF_REQUIRE_MSG(strcmp(cp, ether_ntoa_string) == 0,
+	    "strings did not match (`%s` != `%s`)", cp, ether_ntoa_string);
+}
+
+#if 0
+ATF_TC_WITHOUT_HEAD(ether_ntohost);
+ATF_TC_BODY(ether_ntohost, tc)
+{
+
+}
+
+ATF_TC_WITHOUT_HEAD(ether_hostton);
+ATF_TC_BODY(ether_hostton, tc)
+{
+
+}
+#endif
+
+ATF_TP_ADD_TCS(tp)
+{
+
+	ATF_TP_ADD_TC(tp, ether_line);
+	ATF_TP_ADD_TC(tp, ether_line_bad_1);
+	ATF_TP_ADD_TC(tp, ether_line_bad_2);
+	ATF_TP_ADD_TC(tp, ether_aton_r);
+	ATF_TP_ADD_TC(tp, ether_aton_r_bad);
+	ATF_TP_ADD_TC(tp, ether_aton);
+	ATF_TP_ADD_TC(tp, ether_aton_bad);
+	ATF_TP_ADD_TC(tp, ether_ntoa_r);
+	ATF_TP_ADD_TC(tp, ether_ntoa);
+#if 0
+	ATF_TP_ADD_TC(tp, ether_ntohost);
+	ATF_TP_ADD_TC(tp, ether_hostton);
+#endif
+
+	return (atf_no_error());
+}
diff --git a/tools/regression/lib/libc/net/test-eui64_aton.c b/lib/libc/tests/net/eui64_aton_test.c
index e10f985..7b97d7f 100644
--- a/tools/regression/lib/libc/net/test-eui64_aton.c
+++ b/lib/libc/tests/net/eui64_aton_test.c
@@ -30,44 +30,75 @@
 
 #include <sys/types.h>
 #include <sys/eui64.h>
+#include <locale.h>
 #include <stdio.h>
 #include <string.h>
 
+#include <atf-c.h>
+
 #include "test-eui64.h"
 
-static int
-test_str( const char *str, const struct eui64 *eui)
+static void
+test_str(const char *str, const struct eui64 *eui)
 {
-	struct eui64	e;
-	char		buf[EUI64_SIZ];
-	static int	test = 0;
-
-	test++;
+	struct eui64 e;
+	char buf[EUI64_SIZ];
+	int rc;
 
-	if (eui64_aton(str, &e) != 0 &&
-	    memcmp(&e, &eui, sizeof(struct eui64)) != 0) {
-		printf("not ok %d - : eui64_aton(%s)\n", test, str);
+	ATF_REQUIRE_MSG(eui64_aton(str, &e) == 0, "eui64_aton failed");
+	rc = memcmp(&e, eui, sizeof(e));
+	if (rc != 0) {
 		eui64_ntoa(&e, buf, sizeof(buf));
-		printf("# got: %s\n", buf);
-		return (0);
-	} else {
-		printf("ok %d - eui64_aton(%s)\n", test, str);
-		return (1);
+		atf_tc_fail(
+		    "eui64_aton(\"%s\", ..) failed; memcmp returned %d. "
+		    "String obtained form eui64_ntoa was: `%s`",
+		    str, rc, buf);
 	}
-
 }
 
-int
-main(int argc, char **argv)
+ATF_TC_WITHOUT_HEAD(id_ascii);
+ATF_TC_BODY(id_ascii, tc)
 {
 
-	printf("1..5\n");
-
 	test_str(test_eui64_id_ascii, &test_eui64_id);
+}
+
+ATF_TC_WITHOUT_HEAD(id_colon_ascii);
+ATF_TC_BODY(id_colon_ascii, tc)
+{
+
 	test_str(test_eui64_id_colon_ascii, &test_eui64_id);
+}
+
+ATF_TC_WITHOUT_HEAD(mac_ascii);
+ATF_TC_BODY(mac_ascii, tc)
+{
+
 	test_str(test_eui64_mac_ascii, &test_eui64_eui48);
+}
+
+ATF_TC_WITHOUT_HEAD(mac_colon_ascii);
+ATF_TC_BODY(mac_colon_ascii, tc)
+{
+
 	test_str(test_eui64_mac_colon_ascii, &test_eui64_eui48);
+}
+
+ATF_TC_WITHOUT_HEAD(hex_ascii);
+ATF_TC_BODY(hex_ascii, tc)
+{
+
 	test_str(test_eui64_hex_ascii, &test_eui64_id);
+}
+
+ATF_TP_ADD_TCS(tp)
+{
+
+	ATF_TP_ADD_TC(tp, id_ascii);
+	ATF_TP_ADD_TC(tp, id_colon_ascii);
+	ATF_TP_ADD_TC(tp, mac_ascii);
+	ATF_TP_ADD_TC(tp, mac_colon_ascii);
+	ATF_TP_ADD_TC(tp, hex_ascii);
 
-	return (0);
+	return (atf_no_error());
 }
diff --git a/tools/regression/lib/libc/net/test-eui64_ntoa.c b/lib/libc/tests/net/eui64_ntoa_test.c
index f7582f0..4158e7a 100644
--- a/tools/regression/lib/libc/net/test-eui64_ntoa.c
+++ b/lib/libc/tests/net/eui64_ntoa_test.c
@@ -33,22 +33,32 @@
 #include <stdio.h>
 #include <string.h>
 
+#include <atf-c.h>
+
 #include "test-eui64.h"
 
-int
-main(int argc, char **argv)
+static void
+test_str(const char *str, const struct eui64 *eui)
 {
 	char a[EUI64_SIZ];
 
-	printf("1..1\n");
+	ATF_REQUIRE_MSG(eui64_ntoa(&test_eui64_id, a, sizeof(a)) == 0,
+	    "eui64_ntoa failed");
+	ATF_REQUIRE_MSG(strcmp(a, test_eui64_id_ascii) == 0,
+	    "the strings mismatched: `%s` != `%s`", a, test_eui64_id_ascii);
+}
+
+ATF_TC_WITHOUT_HEAD(id_ascii);
+ATF_TC_BODY(id_ascii, tc)
+{
+
+	test_str(test_eui64_id_ascii, &test_eui64_id);
+}
+
+ATF_TP_ADD_TCS(tp)
+{
 
-	if (eui64_ntoa(&test_eui64_id, a, sizeof(a)) == 0 &&
-	    strcmp(a, test_eui64_id_ascii) == 0) {
-		printf("ok 1 - eui64_ntoa\n");
-		return (0);
-	}
-	printf("# a = '%s'\n", a);
+	ATF_TP_ADD_TC(tp, id_ascii);
 
-	printf("not ok 1 - eui64_ntoa\n");
-	return (0);
+	return (atf_no_error());
 }
diff --git a/tools/regression/lib/libc/net/test-eui64.h b/lib/libc/tests/net/test-eui64.h
index ea82a10..ea82a10 100644
--- a/tools/regression/lib/libc/net/test-eui64.h
+++ b/lib/libc/tests/net/test-eui64.h
diff --git a/lib/libc/tests/sys/Makefile b/lib/libc/tests/sys/Makefile
index 89431bc..39aec1f 100644
--- a/lib/libc/tests/sys/Makefile
+++ b/lib/libc/tests/sys/Makefile
@@ -4,6 +4,8 @@
 
 TESTSDIR=		${TESTSBASE}/lib/libc/sys
 
+ATF_TESTS_C+=			queue_test
+
 # TODO: clone, lwp_create, lwp_ctl, posix_fadvise, recvmmsg,
 # swapcontext
 NETBSD_ATF_TESTS_C+=		access_test
diff --git a/lib/libc/tests/sys/queue_test.c b/lib/libc/tests/sys/queue_test.c
new file mode 100644
index 0000000..2405e3a
--- /dev/null
+++ b/lib/libc/tests/sys/queue_test.c
@@ -0,0 +1,237 @@
+/*-
+ * Copyright (c) 2015 EMC Corp.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/queue.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include <atf-c.h>
+
+ATF_TC(slist_test);
+ATF_TC_HEAD(slist_test, tc)
+{
+
+	atf_tc_set_md_var(tc, "descr", "SLIST macro feature tests");
+}
+
+ATF_TC_BODY(slist_test, tc)
+{
+	SLIST_HEAD(stailhead, entry) head = SLIST_HEAD_INITIALIZER(head);
+	struct entry {
+		SLIST_ENTRY(entry) entries;
+		int i;
+	} *n1, *n2, *n3, *np;
+	int i, j, length;
+
+	SLIST_INIT(&head);
+
+	printf("Ensuring SLIST_EMPTY works\n");
+
+	ATF_REQUIRE(SLIST_EMPTY(&head));
+
+	i = length = 0;
+
+	SLIST_FOREACH(np, &head, entries) {
+		length++;
+	}
+	ATF_REQUIRE_EQ(length, 0);
+
+	printf("Ensuring SLIST_INSERT_HEAD works\n");
+
+	n1 = malloc(sizeof(struct entry));
+	ATF_REQUIRE(n1 != NULL);
+	n1->i = i++;
+
+	SLIST_INSERT_HEAD(&head, n1, entries);
+
+	printf("Ensuring SLIST_FIRST returns element 1\n");
+	ATF_REQUIRE_EQ(SLIST_FIRST(&head), n1);
+
+	j = length = 0;
+	SLIST_FOREACH(np, &head, entries) {
+		ATF_REQUIRE_EQ_MSG(np->i, j,
+		    "%d (entry counter) != %d (counter)", np->i, j);
+		j++;
+		length++;
+	}
+	ATF_REQUIRE_EQ(length, 1);
+
+	printf("Ensuring SLIST_INSERT_AFTER works\n");
+
+	n2 = malloc(sizeof(struct entry));
+	ATF_REQUIRE(n2 != NULL);
+	n2->i = i++;
+
+	SLIST_INSERT_AFTER(n1, n2, entries);
+
+	n3 = malloc(sizeof(struct entry));
+	ATF_REQUIRE(n3 != NULL);
+	n3->i = i++;
+
+	SLIST_INSERT_AFTER(n2, n3, entries);
+
+	j = length = 0;
+	SLIST_FOREACH(np, &head, entries) {
+		ATF_REQUIRE_EQ_MSG(np->i, j,
+		    "%d (entry counter) != %d (counter)", np->i, j);
+		j++;
+		length++;
+	}
+	ATF_REQUIRE_EQ(length, 3);
+
+	printf("Ensuring SLIST_REMOVE_HEAD works\n");
+
+	printf("Ensuring SLIST_FIRST returns element 1\n");
+	ATF_REQUIRE_EQ(SLIST_FIRST(&head), n1);
+
+	SLIST_REMOVE_HEAD(&head, entries);
+
+	printf("Ensuring SLIST_FIRST now returns element 2\n");
+	ATF_REQUIRE_EQ(SLIST_FIRST(&head), n2);
+
+	j = 1; /* Starting point's 1 this time */
+	length = 0;
+	SLIST_FOREACH(np, &head, entries) {
+		ATF_REQUIRE_EQ_MSG(np->i, j,
+		    "%d (entry counter) != %d (counter)", np->i, j);
+		j++;
+		length++;
+	}
+	ATF_REQUIRE_EQ(length, 2);
+
+	printf("Ensuring SLIST_REMOVE_AFTER works by removing the tail\n");
+
+	SLIST_REMOVE_AFTER(n2, entries);
+
+	j = 1; /* Starting point's 1 this time */
+	length = 0;
+	SLIST_FOREACH(np, &head, entries) {
+		ATF_REQUIRE_EQ_MSG(np->i, j,
+		    "%d (entry counter) != %d (counter)", np->i, j);
+		j++;
+		length++;
+	}
+	ATF_REQUIRE_EQ(length, 1);
+
+	printf("Ensuring SLIST_FIRST returns element 2\n");
+	ATF_REQUIRE_EQ(SLIST_FIRST(&head), n2);
+
+}
+
+ATF_TC(stailq_test);
+ATF_TC_HEAD(stailq_test, tc)
+{
+
+	atf_tc_set_md_var(tc, "descr", "STAILQ macro feature tests");
+}
+
+ATF_TC_BODY(stailq_test, tc)
+{
+	STAILQ_HEAD(stailhead, entry) head = STAILQ_HEAD_INITIALIZER(head);
+	struct entry {
+		STAILQ_ENTRY(entry) entries;
+		int i;
+	} *n1, *n2, *n3, *np;
+	int i, j, length;
+
+	printf("Ensuring empty STAILQs are treated properly\n");
+	STAILQ_INIT(&head);
+	ATF_REQUIRE(STAILQ_EMPTY(&head));
+
+	i = length = 0;
+
+	STAILQ_FOREACH(np, &head, entries) {
+		length++;
+	}
+	ATF_REQUIRE_EQ(length, 0);
+
+	printf("Ensuring STAILQ_INSERT_HEAD works\n");
+
+	n1 = malloc(sizeof(struct entry));
+	ATF_REQUIRE(n1 != NULL);
+	n1->i = i++;
+
+	STAILQ_INSERT_HEAD(&head, n1, entries);
+
+	j = length = 0;
+	STAILQ_FOREACH(np, &head, entries) {
+		ATF_REQUIRE_EQ_MSG(np->i, j,
+		    "%d (entry counter) != %d (counter)", np->i, j);
+		j++;
+		length++;
+	}
+	ATF_REQUIRE_EQ(length, 1);
+
+	printf("Ensuring STAILQ_INSERT_TAIL works\n");
+
+	n2 = malloc(sizeof(struct entry));
+	ATF_REQUIRE(n2 != NULL);
+	n2->i = i++;
+
+	STAILQ_INSERT_TAIL(&head, n2, entries);
+
+	n3 = malloc(sizeof(struct entry));
+	ATF_REQUIRE(n3 != NULL);
+	n3->i = i++;
+
+	STAILQ_INSERT_TAIL(&head, n3, entries);
+
+	j = length = 0;
+	STAILQ_FOREACH(np, &head, entries) {
+		ATF_REQUIRE_EQ_MSG(np->i, j,
+		    "%d (entry counter) != %d (counter)", np->i, j);
+		j++;
+		length++;
+	}
+	ATF_REQUIRE_EQ(length, 3);
+
+	printf("Ensuring STAILQ_REMOVE_HEAD works\n");
+
+	STAILQ_REMOVE_HEAD(&head, entries);
+
+	j = 1; /* Starting point's 1 this time */
+	length = 0;
+	STAILQ_FOREACH(np, &head, entries) {
+		ATF_REQUIRE_EQ_MSG(np->i, j,
+		    "%d (entry counter) != %d (counter)", np->i, j);
+		j++;
+		length++;
+	}
+	ATF_REQUIRE_EQ(length, 2);
+
+}
+
+ATF_TP_ADD_TCS(tp)
+{
+
+	ATF_TP_ADD_TC(tp, slist_test);
+	ATF_TP_ADD_TC(tp, stailq_test);
+
+	return (atf_no_error());
+}
diff --git a/share/mk/bsd.progs.mk b/share/mk/bsd.progs.mk
index 8dcc66f..474dd31 100644
--- a/share/mk/bsd.progs.mk
+++ b/share/mk/bsd.progs.mk
@@ -42,8 +42,8 @@ PROG ?= $t
 
 .if defined(PROG)
 # just one of many
-PROG_OVERRIDE_VARS +=	BINDIR BINGRP BINOWN BINMODE DPSRCS MAN PROGNAME \
-			SRCS
+PROG_OVERRIDE_VARS +=	BINDIR BINGRP BINOWN BINMODE DPSRCS MAN NO_WERROR \
+			PROGNAME SRCS WARNS
 PROG_VARS +=	CFLAGS CPPFLAGS CXXFLAGS DPADD DPLIBS LDADD LINKS \
 		LDFLAGS MLINKS ${PROG_OVERRIDE_VARS}
 .for v in ${PROG_VARS:O:u}
diff --git a/sys/dev/mlx5/cq.h b/sys/dev/mlx5/cq.h
new file mode 100644
index 0000000..92fd8ba
--- /dev/null
+++ b/sys/dev/mlx5/cq.h
@@ -0,0 +1,169 @@
+/*-
+ * Copyright (c) 2013-2015, Mellanox Technologies, Ltd.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef MLX5_CORE_CQ_H
+#define MLX5_CORE_CQ_H
+
+#include <rdma/ib_verbs.h>
+#include <dev/mlx5/driver.h>
+#include <dev/mlx5/mlx5_ifc.h>
+
+
+struct mlx5_core_cq {
+	u32			cqn;
+	int			cqe_sz;
+	__be32		       *set_ci_db;
+	__be32		       *arm_db;
+	atomic_t		refcount;
+	struct completion	free;
+	unsigned		vector;
+	int			irqn;
+	void (*comp)		(struct mlx5_core_cq *);
+	void (*event)		(struct mlx5_core_cq *, int);
+	struct mlx5_uar	       *uar;
+	u32			cons_index;
+	unsigned		arm_sn;
+	struct mlx5_rsc_debug	*dbg;
+	int			pid;
+};
+
+
+enum {
+	MLX5_CQE_SYNDROME_LOCAL_LENGTH_ERR		= 0x01,
+	MLX5_CQE_SYNDROME_LOCAL_QP_OP_ERR		= 0x02,
+	MLX5_CQE_SYNDROME_LOCAL_PROT_ERR		= 0x04,
+	MLX5_CQE_SYNDROME_WR_FLUSH_ERR			= 0x05,
+	MLX5_CQE_SYNDROME_MW_BIND_ERR			= 0x06,
+	MLX5_CQE_SYNDROME_BAD_RESP_ERR			= 0x10,
+	MLX5_CQE_SYNDROME_LOCAL_ACCESS_ERR		= 0x11,
+	MLX5_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR		= 0x12,
+	MLX5_CQE_SYNDROME_REMOTE_ACCESS_ERR		= 0x13,
+	MLX5_CQE_SYNDROME_REMOTE_OP_ERR			= 0x14,
+	MLX5_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR	= 0x15,
+	MLX5_CQE_SYNDROME_RNR_RETRY_EXC_ERR		= 0x16,
+	MLX5_CQE_SYNDROME_REMOTE_ABORTED_ERR		= 0x22,
+};
+
+enum {
+	MLX5_CQE_OWNER_MASK	= 1,
+	MLX5_CQE_REQ		= 0,
+	MLX5_CQE_RESP_WR_IMM	= 1,
+	MLX5_CQE_RESP_SEND	= 2,
+	MLX5_CQE_RESP_SEND_IMM	= 3,
+	MLX5_CQE_RESP_SEND_INV	= 4,
+	MLX5_CQE_RESIZE_CQ	= 5,
+	MLX5_CQE_SIG_ERR	= 12,
+	MLX5_CQE_REQ_ERR	= 13,
+	MLX5_CQE_RESP_ERR	= 14,
+	MLX5_CQE_INVALID	= 15,
+};
+
+enum {
+	MLX5_CQ_MODIFY_PERIOD	= 1 << 0,
+	MLX5_CQ_MODIFY_COUNT	= 1 << 1,
+	MLX5_CQ_MODIFY_OVERRUN	= 1 << 2,
+};
+
+enum {
+	MLX5_CQ_OPMOD_RESIZE		= 1,
+	MLX5_MODIFY_CQ_MASK_LOG_SIZE	= 1 << 0,
+	MLX5_MODIFY_CQ_MASK_PG_OFFSET	= 1 << 1,
+	MLX5_MODIFY_CQ_MASK_PG_SIZE	= 1 << 2,
+};
+
+struct mlx5_cq_modify_params {
+	int	type;
+	union {
+		struct {
+			u32	page_offset;
+			u8	log_cq_size;
+		} resize;
+
+		struct {
+		} moder;
+
+		struct {
+		} mapping;
+	} params;
+};
+
+static inline int cqe_sz_to_mlx_sz(u8 size)
+{
+	return size == 64 ? CQE_SIZE_64 : CQE_SIZE_128;
+}
+
+static inline void mlx5_cq_set_ci(struct mlx5_core_cq *cq)
+{
+	*cq->set_ci_db = cpu_to_be32(cq->cons_index & 0xffffff);
+}
+
+enum {
+	MLX5_CQ_DB_REQ_NOT_SOL		= 1 << 24,
+	MLX5_CQ_DB_REQ_NOT		= 0 << 24
+};
+
+static inline void mlx5_cq_arm(struct mlx5_core_cq *cq, u32 cmd,
+			       void __iomem *uar_page,
+			       spinlock_t *doorbell_lock,
+			       u32 cons_index)
+{
+	__be32 doorbell[2];
+	u32 sn;
+	u32 ci;
+
+	sn = cq->arm_sn & 3;
+	ci = cons_index & 0xffffff;
+
+	*cq->arm_db = cpu_to_be32(sn << 28 | cmd | ci);
+
+	/* Make sure that the doorbell record in host memory is
+	 * written before ringing the doorbell via PCI MMIO.
+	 */
+	wmb();
+
+	doorbell[0] = cpu_to_be32(sn << 28 | cmd | ci);
+	doorbell[1] = cpu_to_be32(cq->cqn);
+
+	mlx5_write64(doorbell, uar_page + MLX5_CQ_DOORBELL, doorbell_lock);
+}
+
+int mlx5_init_cq_table(struct mlx5_core_dev *dev);
+void mlx5_cleanup_cq_table(struct mlx5_core_dev *dev);
+int mlx5_core_create_cq(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq,
+			struct mlx5_create_cq_mbox_in *in, int inlen);
+int mlx5_core_destroy_cq(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq);
+int mlx5_core_query_cq(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq,
+		       struct mlx5_query_cq_mbox_out *out);
+int mlx5_core_modify_cq(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq,
+			struct mlx5_modify_cq_mbox_in *in, int in_sz);
+int mlx5_core_modify_cq_moderation(struct mlx5_core_dev *dev,
+				   struct mlx5_core_cq *cq, u16 cq_period,
+				   u16 cq_max_count);
+int mlx5_debug_cq_add(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq);
+void mlx5_debug_cq_remove(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq);
+
+#endif /* MLX5_CORE_CQ_H */
diff --git a/sys/dev/mlx5/device.h b/sys/dev/mlx5/device.h
new file mode 100644
index 0000000..3e90859
--- /dev/null
+++ b/sys/dev/mlx5/device.h
@@ -0,0 +1,1187 @@
+/*-
+ * Copyright (c) 2013-2015, Mellanox Technologies, Ltd.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef MLX5_DEVICE_H
+#define MLX5_DEVICE_H
+
+#include <linux/types.h>
+#include <rdma/ib_verbs.h>
+#include <dev/mlx5/mlx5_ifc.h>
+
+#define FW_INIT_TIMEOUT_MILI 2000
+#define FW_INIT_WAIT_MS 2
+
+#if defined(__LITTLE_ENDIAN)
+#define MLX5_SET_HOST_ENDIANNESS	0
+#elif defined(__BIG_ENDIAN)
+#define MLX5_SET_HOST_ENDIANNESS	0x80
+#else
+#error Host endianness not defined
+#endif
+
+/* helper macros */
+#define __mlx5_nullp(typ) ((struct mlx5_ifc_##typ##_bits *)0)
+#define __mlx5_bit_sz(typ, fld) sizeof(__mlx5_nullp(typ)->fld)
+#define __mlx5_bit_off(typ, fld) __offsetof(struct mlx5_ifc_##typ##_bits, fld)
+#define __mlx5_dw_off(typ, fld) (__mlx5_bit_off(typ, fld) / 32)
+#define __mlx5_64_off(typ, fld) (__mlx5_bit_off(typ, fld) / 64)
+#define __mlx5_dw_bit_off(typ, fld) (32 - __mlx5_bit_sz(typ, fld) - (__mlx5_bit_off(typ, fld) & 0x1f))
+#define __mlx5_mask(typ, fld) ((u32)((1ull << __mlx5_bit_sz(typ, fld)) - 1))
+#define __mlx5_dw_mask(typ, fld) (__mlx5_mask(typ, fld) << __mlx5_dw_bit_off(typ, fld))
+#define __mlx5_st_sz_bits(typ) sizeof(struct mlx5_ifc_##typ##_bits)
+
+#define MLX5_FLD_SZ_BYTES(typ, fld) (__mlx5_bit_sz(typ, fld) / 8)
+#define MLX5_ST_SZ_BYTES(typ) (sizeof(struct mlx5_ifc_##typ##_bits) / 8)
+#define MLX5_ST_SZ_DW(typ) (sizeof(struct mlx5_ifc_##typ##_bits) / 32)
+#define MLX5_UN_SZ_BYTES(typ) (sizeof(union mlx5_ifc_##typ##_bits) / 8)
+#define MLX5_UN_SZ_DW(typ) (sizeof(union mlx5_ifc_##typ##_bits) / 32)
+#define MLX5_BYTE_OFF(typ, fld) (__mlx5_bit_off(typ, fld) / 8)
+#define MLX5_ADDR_OF(typ, p, fld) ((char *)(p) + MLX5_BYTE_OFF(typ, fld))
+
+/* insert a value to a struct */
+#define MLX5_SET(typ, p, fld, v) do { \
+	BUILD_BUG_ON(__mlx5_st_sz_bits(typ) % 32);             \
+	BUILD_BUG_ON(__mlx5_bit_sz(typ, fld) > 32); \
+	*((__be32 *)(p) + __mlx5_dw_off(typ, fld)) = \
+	cpu_to_be32((be32_to_cpu(*((__be32 *)(p) + __mlx5_dw_off(typ, fld))) & \
+		     (~__mlx5_dw_mask(typ, fld))) | (((v) & __mlx5_mask(typ, fld)) \
+		     << __mlx5_dw_bit_off(typ, fld))); \
+} while (0)
+
+#define MLX5_SET_TO_ONES(typ, p, fld) do { \
+	BUILD_BUG_ON(__mlx5_st_sz_bits(typ) % 32);             \
+	BUILD_BUG_ON(__mlx5_bit_sz(typ, fld) > 32); \
+	*((__be32 *)(p) + __mlx5_dw_off(typ, fld)) = \
+	cpu_to_be32((be32_to_cpu(*((__be32 *)(p) + __mlx5_dw_off(typ, fld))) & \
+		     (~__mlx5_dw_mask(typ, fld))) | ((__mlx5_mask(typ, fld)) \
+		     << __mlx5_dw_bit_off(typ, fld))); \
+} while (0)
+
+#define MLX5_GET(typ, p, fld) ((be32_to_cpu(*((__be32 *)(p) +\
+__mlx5_dw_off(typ, fld))) >> __mlx5_dw_bit_off(typ, fld)) & \
+__mlx5_mask(typ, fld))
+
+#define MLX5_GET_PR(typ, p, fld) ({ \
+	u32 ___t = MLX5_GET(typ, p, fld); \
+	pr_debug(#fld " = 0x%x\n", ___t); \
+	___t; \
+})
+
+#define MLX5_SET64(typ, p, fld, v) do { \
+	BUILD_BUG_ON(__mlx5_bit_sz(typ, fld) != 64); \
+	BUILD_BUG_ON(__mlx5_bit_off(typ, fld) % 64); \
+	*((__be64 *)(p) + __mlx5_64_off(typ, fld)) = cpu_to_be64(v); \
+} while (0)
+
+#define MLX5_GET64(typ, p, fld) be64_to_cpu(*((__be64 *)(p) + __mlx5_64_off(typ, fld)))
+
+enum {
+	MLX5_MAX_COMMANDS		= 32,
+	MLX5_CMD_DATA_BLOCK_SIZE	= 512,
+	MLX5_PCI_CMD_XPORT		= 7,
+	MLX5_MKEY_BSF_OCTO_SIZE		= 4,
+	MLX5_MAX_PSVS			= 4,
+};
+
+enum {
+	MLX5_EXTENDED_UD_AV		= 0x80000000,
+};
+
+enum {
+	MLX5_STAT_RATE_OFFSET	= 5,
+};
+
+enum {
+	MLX5_INLINE_SEG = 0x80000000,
+};
+
+enum {
+	MLX5_HW_START_PADDING = MLX5_INLINE_SEG,
+};
+
+enum {
+	MLX5_MIN_PKEY_TABLE_SIZE = 128,
+	MLX5_MAX_LOG_PKEY_TABLE  = 5,
+};
+
+enum {
+	MLX5_PERM_LOCAL_READ	= 1 << 2,
+	MLX5_PERM_LOCAL_WRITE	= 1 << 3,
+	MLX5_PERM_REMOTE_READ	= 1 << 4,
+	MLX5_PERM_REMOTE_WRITE	= 1 << 5,
+	MLX5_PERM_ATOMIC	= 1 << 6,
+	MLX5_PERM_UMR_EN	= 1 << 7,
+};
+
+enum {
+	MLX5_PCIE_CTRL_SMALL_FENCE	= 1 << 0,
+	MLX5_PCIE_CTRL_RELAXED_ORDERING	= 1 << 2,
+	MLX5_PCIE_CTRL_NO_SNOOP		= 1 << 3,
+	MLX5_PCIE_CTRL_TLP_PROCE_EN	= 1 << 6,
+	MLX5_PCIE_CTRL_TPH_MASK		= 3 << 4,
+};
+
+enum {
+	MLX5_MKEY_REMOTE_INVAL	= 1 << 24,
+	MLX5_MKEY_FLAG_SYNC_UMR = 1 << 29,
+	MLX5_MKEY_BSF_EN	= 1 << 30,
+	MLX5_MKEY_LEN64		= 1 << 31,
+};
+
+enum {
+	MLX5_EN_RD	= (u64)1,
+	MLX5_EN_WR	= (u64)2
+};
+
+enum {
+	MLX5_BF_REGS_PER_PAGE		= 4,
+	MLX5_MAX_UAR_PAGES		= 1 << 8,
+	MLX5_NON_FP_BF_REGS_PER_PAGE	= 2,
+	MLX5_MAX_UUARS	= MLX5_MAX_UAR_PAGES * MLX5_NON_FP_BF_REGS_PER_PAGE,
+};
+
+enum {
+	MLX5_MKEY_MASK_LEN		= 1ull << 0,
+	MLX5_MKEY_MASK_PAGE_SIZE	= 1ull << 1,
+	MLX5_MKEY_MASK_START_ADDR	= 1ull << 6,
+	MLX5_MKEY_MASK_PD		= 1ull << 7,
+	MLX5_MKEY_MASK_EN_RINVAL	= 1ull << 8,
+	MLX5_MKEY_MASK_EN_SIGERR	= 1ull << 9,
+	MLX5_MKEY_MASK_BSF_EN		= 1ull << 12,
+	MLX5_MKEY_MASK_KEY		= 1ull << 13,
+	MLX5_MKEY_MASK_QPN		= 1ull << 14,
+	MLX5_MKEY_MASK_LR		= 1ull << 17,
+	MLX5_MKEY_MASK_LW		= 1ull << 18,
+	MLX5_MKEY_MASK_RR		= 1ull << 19,
+	MLX5_MKEY_MASK_RW		= 1ull << 20,
+	MLX5_MKEY_MASK_A		= 1ull << 21,
+	MLX5_MKEY_MASK_SMALL_FENCE	= 1ull << 23,
+	MLX5_MKEY_MASK_FREE		= 1ull << 29,
+};
+
+enum {
+	MLX5_PORT_CHANGE_SUBTYPE_DOWN		= 1,
+	MLX5_PORT_CHANGE_SUBTYPE_ACTIVE		= 4,
+	MLX5_PORT_CHANGE_SUBTYPE_INITIALIZED	= 5,
+	MLX5_PORT_CHANGE_SUBTYPE_LID		= 6,
+	MLX5_PORT_CHANGE_SUBTYPE_PKEY		= 7,
+	MLX5_PORT_CHANGE_SUBTYPE_GUID		= 8,
+	MLX5_PORT_CHANGE_SUBTYPE_CLIENT_REREG	= 9,
+};
+
+enum {
+	MLX5_DEV_CAP_FLAG_XRC		= 1LL <<  3,
+	MLX5_DEV_CAP_FLAG_BAD_PKEY_CNTR	= 1LL <<  8,
+	MLX5_DEV_CAP_FLAG_BAD_QKEY_CNTR	= 1LL <<  9,
+	MLX5_DEV_CAP_FLAG_APM		= 1LL << 17,
+	MLX5_DEV_CAP_FLAG_ATOMIC	= 1LL << 18,
+	MLX5_DEV_CAP_FLAG_SCQE_BRK_MOD	= 1LL << 21,
+	MLX5_DEV_CAP_FLAG_BLOCK_MCAST	= 1LL << 23,
+	MLX5_DEV_CAP_FLAG_CQ_MODER	= 1LL << 29,
+	MLX5_DEV_CAP_FLAG_RESIZE_CQ	= 1LL << 30,
+	MLX5_DEV_CAP_FLAG_ROCE          = 1LL << 34,
+	MLX5_DEV_CAP_FLAG_DCT		= 1LL << 37,
+	MLX5_DEV_CAP_FLAG_SIG_HAND_OVER	= 1LL << 40,
+	MLX5_DEV_CAP_FLAG_CMDIF_CSUM	= 3LL << 46,
+};
+
+enum {
+	MLX5_ROCE_VERSION_1		= 0,
+	MLX5_ROCE_VERSION_1_5		= 1,
+	MLX5_ROCE_VERSION_2		= 2,
+};
+
+enum {
+	MLX5_ROCE_VERSION_1_CAP		= 1 << MLX5_ROCE_VERSION_1,
+	MLX5_ROCE_VERSION_1_5_CAP	= 1 << MLX5_ROCE_VERSION_1_5,
+	MLX5_ROCE_VERSION_2_CAP		= 1 << MLX5_ROCE_VERSION_2,
+};
+
+enum {
+	MLX5_ROCE_L3_TYPE_IPV4		= 0,
+	MLX5_ROCE_L3_TYPE_IPV6		= 1,
+};
+
+enum {
+	MLX5_ROCE_L3_TYPE_IPV4_CAP	= 1 << 1,
+	MLX5_ROCE_L3_TYPE_IPV6_CAP	= 1 << 2,
+};
+
+enum {
+	MLX5_OPCODE_NOP			= 0x00,
+	MLX5_OPCODE_SEND_INVAL		= 0x01,
+	MLX5_OPCODE_RDMA_WRITE		= 0x08,
+	MLX5_OPCODE_RDMA_WRITE_IMM	= 0x09,
+	MLX5_OPCODE_SEND		= 0x0a,
+	MLX5_OPCODE_SEND_IMM		= 0x0b,
+	MLX5_OPCODE_LSO			= 0x0e,
+	MLX5_OPCODE_RDMA_READ		= 0x10,
+	MLX5_OPCODE_ATOMIC_CS		= 0x11,
+	MLX5_OPCODE_ATOMIC_FA		= 0x12,
+	MLX5_OPCODE_ATOMIC_MASKED_CS	= 0x14,
+	MLX5_OPCODE_ATOMIC_MASKED_FA	= 0x15,
+	MLX5_OPCODE_BIND_MW		= 0x18,
+	MLX5_OPCODE_CONFIG_CMD		= 0x1f,
+
+	MLX5_RECV_OPCODE_RDMA_WRITE_IMM	= 0x00,
+	MLX5_RECV_OPCODE_SEND		= 0x01,
+	MLX5_RECV_OPCODE_SEND_IMM	= 0x02,
+	MLX5_RECV_OPCODE_SEND_INVAL	= 0x03,
+
+	MLX5_CQE_OPCODE_ERROR		= 0x1e,
+	MLX5_CQE_OPCODE_RESIZE		= 0x16,
+
+	MLX5_OPCODE_SET_PSV		= 0x20,
+	MLX5_OPCODE_GET_PSV		= 0x21,
+	MLX5_OPCODE_CHECK_PSV		= 0x22,
+	MLX5_OPCODE_RGET_PSV		= 0x26,
+	MLX5_OPCODE_RCHECK_PSV		= 0x27,
+
+	MLX5_OPCODE_UMR			= 0x25,
+
+};
+
+enum {
+	MLX5_SET_PORT_RESET_QKEY	= 0,
+	MLX5_SET_PORT_GUID0		= 16,
+	MLX5_SET_PORT_NODE_GUID		= 17,
+	MLX5_SET_PORT_SYS_GUID		= 18,
+	MLX5_SET_PORT_GID_TABLE		= 19,
+	MLX5_SET_PORT_PKEY_TABLE	= 20,
+};
+
+enum {
+	MLX5_MAX_PAGE_SHIFT		= 31
+};
+
+enum {
+	MLX5_ADAPTER_PAGE_SHIFT		= 12,
+	MLX5_ADAPTER_PAGE_SIZE		= 1 << MLX5_ADAPTER_PAGE_SHIFT,
+};
+
+enum {
+	MLX5_CAP_OFF_CMDIF_CSUM		= 46,
+};
+
+struct mlx5_inbox_hdr {
+	__be16		opcode;
+	u8		rsvd[4];
+	__be16		opmod;
+};
+
+struct mlx5_outbox_hdr {
+	u8		status;
+	u8		rsvd[3];
+	__be32		syndrome;
+};
+
+struct mlx5_cmd_layout {
+	u8		type;
+	u8		rsvd0[3];
+	__be32		inlen;
+	__be64		in_ptr;
+	__be32		in[4];
+	__be32		out[4];
+	__be64		out_ptr;
+	__be32		outlen;
+	u8		token;
+	u8		sig;
+	u8		rsvd1;
+	u8		status_own;
+};
+
+
+struct mlx5_health_buffer {
+	__be32		assert_var[5];
+	__be32		rsvd0[3];
+	__be32		assert_exit_ptr;
+	__be32		assert_callra;
+	__be32		rsvd1[2];
+	__be32		fw_ver;
+	__be32		hw_id;
+	__be32		rsvd2;
+	u8		irisc_index;
+	u8		synd;
+	__be16		ext_sync;
+};
+
+struct mlx5_init_seg {
+	__be32			fw_rev;
+	__be32			cmdif_rev_fw_sub;
+	__be32			rsvd0[2];
+	__be32			cmdq_addr_h;
+	__be32			cmdq_addr_l_sz;
+	__be32			cmd_dbell;
+	__be32			rsvd1[120];
+	__be32			initializing;
+	struct mlx5_health_buffer  health;
+	__be32			rsvd2[884];
+	__be32			health_counter;
+	__be32			rsvd3[1019];
+	__be64			ieee1588_clk;
+	__be32			ieee1588_clk_type;
+	__be32			clr_intx;
+};
+
+struct mlx5_eqe_comp {
+	__be32	reserved[6];
+	__be32	cqn;
+};
+
+struct mlx5_eqe_qp_srq {
+	__be32	reserved[6];
+	__be32	qp_srq_n;
+};
+
+struct mlx5_eqe_cq_err {
+	__be32	cqn;
+	u8	reserved1[7];
+	u8	syndrome;
+};
+
+struct mlx5_eqe_port_state {
+	u8	reserved0[8];
+	u8	port;
+};
+
+struct mlx5_eqe_gpio {
+	__be32	reserved0[2];
+	__be64	gpio_event;
+};
+
+struct mlx5_eqe_congestion {
+	u8	type;
+	u8	rsvd0;
+	u8	congestion_level;
+};
+
+struct mlx5_eqe_stall_vl {
+	u8	rsvd0[3];
+	u8	port_vl;
+};
+
+struct mlx5_eqe_cmd {
+	__be32	vector;
+	__be32	rsvd[6];
+};
+
+struct mlx5_eqe_page_req {
+	u8		rsvd0[2];
+	__be16		func_id;
+	__be32		num_pages;
+	__be32		rsvd1[5];
+};
+
+struct mlx5_eqe_vport_change {
+	u8		rsvd0[2];
+	__be16		vport_num;
+	__be32		rsvd1[6];
+};
+
+
+#define PORT_MODULE_EVENT_MODULE_STATUS_MASK  0xF
+#define PORT_MODULE_EVENT_ERROR_TYPE_MASK     0xF
+
+enum {
+	MLX5_MODULE_STATUS_PLUGGED    = 0x1,
+	MLX5_MODULE_STATUS_UNPLUGGED  = 0x2,
+	MLX5_MODULE_STATUS_ERROR      = 0x3,
+};
+
+enum {
+	MLX5_MODULE_EVENT_ERROR_POWER_BUDGET_EXCEEDED                 = 0x0,
+	MLX5_MODULE_EVENT_ERROR_LONG_RANGE_FOR_NON_MLNX_CABLE_MODULE  = 0x1,
+	MLX5_MODULE_EVENT_ERROR_BUS_STUCK                             = 0x2,
+	MLX5_MODULE_EVENT_ERROR_NO_EEPROM_RETRY_TIMEOUT               = 0x3,
+	MLX5_MODULE_EVENT_ERROR_ENFORCE_PART_NUMBER_LIST              = 0x4,
+	MLX5_MODULE_EVENT_ERROR_UNKNOWN_IDENTIFIER                    = 0x5,
+	MLX5_MODULE_EVENT_ERROR_HIGH_TEMPERATURE                      = 0x6,
+};
+
+struct mlx5_eqe_port_module_event {
+	u8        rsvd0;
+	u8        module;
+	u8        rsvd1;
+	u8        module_status;
+	u8        rsvd2[2];
+	u8        error_type;
+};
+
+union ev_data {
+	__be32				raw[7];
+	struct mlx5_eqe_cmd		cmd;
+	struct mlx5_eqe_comp		comp;
+	struct mlx5_eqe_qp_srq		qp_srq;
+	struct mlx5_eqe_cq_err		cq_err;
+	struct mlx5_eqe_port_state	port;
+	struct mlx5_eqe_gpio		gpio;
+	struct mlx5_eqe_congestion	cong;
+	struct mlx5_eqe_stall_vl	stall_vl;
+	struct mlx5_eqe_page_req	req_pages;
+	struct mlx5_eqe_port_module_event port_module_event;
+	struct mlx5_eqe_vport_change	vport_change;
+} __packed;
+
+struct mlx5_eqe {
+	u8		rsvd0;
+	u8		type;
+	u8		rsvd1;
+	u8		sub_type;
+	__be32		rsvd2[7];
+	union ev_data	data;
+	__be16		rsvd3;
+	u8		signature;
+	u8		owner;
+} __packed;
+
+struct mlx5_cmd_prot_block {
+	u8		data[MLX5_CMD_DATA_BLOCK_SIZE];
+	u8		rsvd0[48];
+	__be64		next;
+	__be32		block_num;
+	u8		rsvd1;
+	u8		token;
+	u8		ctrl_sig;
+	u8		sig;
+};
+
+enum {
+	MLX5_CQE_SYND_FLUSHED_IN_ERROR = 5,
+};
+
+struct mlx5_err_cqe {
+	u8	rsvd0[32];
+	__be32	srqn;
+	u8	rsvd1[18];
+	u8	vendor_err_synd;
+	u8	syndrome;
+	__be32	s_wqe_opcode_qpn;
+	__be16	wqe_counter;
+	u8	signature;
+	u8	op_own;
+};
+
+struct mlx5_cqe64 {
+	u8		tunneled_etc;
+	u8		rsvd0[3];
+	u8		lro_tcppsh_abort_dupack;
+	u8		lro_min_ttl;
+	__be16		lro_tcp_win;
+	__be32		lro_ack_seq_num;
+	__be32		rss_hash_result;
+	u8		rss_hash_type;
+	u8		ml_path;
+	u8		rsvd20[2];
+	__be16		check_sum;
+	__be16		slid;
+	__be32		flags_rqpn;
+	u8		hds_ip_ext;
+	u8		l4_hdr_type_etc;
+	__be16		vlan_info;
+	__be32		srqn; /* [31:24]: lro_num_seg, [23:0]: srqn */
+	__be32		imm_inval_pkey;
+	u8		rsvd40[4];
+	__be32		byte_cnt;
+	__be64		timestamp;
+	__be32		sop_drop_qpn;
+	__be16		wqe_counter;
+	u8		signature;
+	u8		op_own;
+};
+
+static inline bool get_cqe_lro_timestamp_valid(struct mlx5_cqe64 *cqe)
+{
+	return (cqe->lro_tcppsh_abort_dupack >> 7) & 1;
+}
+
+static inline bool get_cqe_lro_tcppsh(struct mlx5_cqe64 *cqe)
+{
+	return (cqe->lro_tcppsh_abort_dupack >> 6) & 1;
+}
+
+static inline u8 get_cqe_l4_hdr_type(struct mlx5_cqe64 *cqe)
+{
+	return (cqe->l4_hdr_type_etc >> 4) & 0x7;
+}
+
+static inline u16 get_cqe_vlan(struct mlx5_cqe64 *cqe)
+{
+	return be16_to_cpu(cqe->vlan_info) & 0xfff;
+}
+
+static inline void get_cqe_smac(struct mlx5_cqe64 *cqe, u8 *smac)
+{
+	memcpy(smac, &cqe->rss_hash_type , 4);
+	memcpy(smac + 4, &cqe->slid , 2);
+}
+
+static inline bool cqe_has_vlan(struct mlx5_cqe64 *cqe)
+{
+	return cqe->l4_hdr_type_etc & 0x1;
+}
+
+static inline bool cqe_is_tunneled(struct mlx5_cqe64 *cqe)
+{
+	return cqe->tunneled_etc & 0x1;
+}
+
+enum {
+	CQE_L4_HDR_TYPE_NONE			= 0x0,
+	CQE_L4_HDR_TYPE_TCP_NO_ACK		= 0x1,
+	CQE_L4_HDR_TYPE_UDP			= 0x2,
+	CQE_L4_HDR_TYPE_TCP_ACK_NO_DATA		= 0x3,
+	CQE_L4_HDR_TYPE_TCP_ACK_AND_DATA	= 0x4,
+};
+
+enum {
+	/* source L3 hash types */
+	CQE_RSS_SRC_HTYPE_IP	= 0x3 << 0,
+	CQE_RSS_SRC_HTYPE_IPV4	= 0x1 << 0,
+	CQE_RSS_SRC_HTYPE_IPV6	= 0x2 << 0,
+
+	/* destination L3 hash types */
+	CQE_RSS_DST_HTYPE_IP	= 0x3 << 2,
+	CQE_RSS_DST_HTYPE_IPV4	= 0x1 << 2,
+	CQE_RSS_DST_HTYPE_IPV6	= 0x2 << 2,
+
+	/* source L4 hash types */
+	CQE_RSS_SRC_HTYPE_L4	= 0x3 << 4,
+	CQE_RSS_SRC_HTYPE_TCP	= 0x1 << 4,
+	CQE_RSS_SRC_HTYPE_UDP	= 0x2 << 4,
+	CQE_RSS_SRC_HTYPE_IPSEC	= 0x3 << 4,
+
+	/* destination L4 hash types */
+	CQE_RSS_DST_HTYPE_L4	= 0x3 << 6,
+	CQE_RSS_DST_HTYPE_TCP	= 0x1 << 6,
+	CQE_RSS_DST_HTYPE_UDP	= 0x2 << 6,
+	CQE_RSS_DST_HTYPE_IPSEC	= 0x3 << 6,
+};
+
+enum {
+	CQE_ROCE_L3_HEADER_TYPE_GRH	= 0x0,
+	CQE_ROCE_L3_HEADER_TYPE_IPV6	= 0x1,
+	CQE_ROCE_L3_HEADER_TYPE_IPV4	= 0x2,
+};
+
+enum {
+	CQE_L2_OK	= 1 << 0,
+	CQE_L3_OK	= 1 << 1,
+	CQE_L4_OK	= 1 << 2,
+};
+
+struct mlx5_sig_err_cqe {
+	u8		rsvd0[16];
+	__be32		expected_trans_sig;
+	__be32		actual_trans_sig;
+	__be32		expected_reftag;
+	__be32		actual_reftag;
+	__be16		syndrome;
+	u8		rsvd22[2];
+	__be32		mkey;
+	__be64		err_offset;
+	u8		rsvd30[8];
+	__be32		qpn;
+	u8		rsvd38[2];
+	u8		signature;
+	u8		op_own;
+};
+
+struct mlx5_wqe_srq_next_seg {
+	u8			rsvd0[2];
+	__be16			next_wqe_index;
+	u8			signature;
+	u8			rsvd1[11];
+};
+
+union mlx5_ext_cqe {
+	struct ib_grh	grh;
+	u8		inl[64];
+};
+
+struct mlx5_cqe128 {
+	union mlx5_ext_cqe	inl_grh;
+	struct mlx5_cqe64	cqe64;
+};
+
+struct mlx5_srq_ctx {
+	u8			state_log_sz;
+	u8			rsvd0[3];
+	__be32			flags_xrcd;
+	__be32			pgoff_cqn;
+	u8			rsvd1[4];
+	u8			log_pg_sz;
+	u8			rsvd2[7];
+	__be32			pd;
+	__be16			lwm;
+	__be16			wqe_cnt;
+	u8			rsvd3[8];
+	__be64			db_record;
+};
+
+struct mlx5_create_srq_mbox_in {
+	struct mlx5_inbox_hdr	hdr;
+	__be32			input_srqn;
+	u8			rsvd0[4];
+	struct mlx5_srq_ctx	ctx;
+	u8			rsvd1[208];
+	__be64			pas[0];
+};
+
+struct mlx5_create_srq_mbox_out {
+	struct mlx5_outbox_hdr	hdr;
+	__be32			srqn;
+	u8			rsvd[4];
+};
+
+struct mlx5_destroy_srq_mbox_in {
+	struct mlx5_inbox_hdr	hdr;
+	__be32			srqn;
+	u8			rsvd[4];
+};
+
+struct mlx5_destroy_srq_mbox_out {
+	struct mlx5_outbox_hdr	hdr;
+	u8			rsvd[8];
+};
+
+struct mlx5_query_srq_mbox_in {
+	struct mlx5_inbox_hdr	hdr;
+	__be32			srqn;
+	u8			rsvd0[4];
+};
+
+struct mlx5_query_srq_mbox_out {
+	struct mlx5_outbox_hdr	hdr;
+	u8			rsvd0[8];
+	struct mlx5_srq_ctx	ctx;
+	u8			rsvd1[32];
+	__be64			pas[0];
+};
+
+struct mlx5_arm_srq_mbox_in {
+	struct mlx5_inbox_hdr	hdr;
+	__be32			srqn;
+	__be16			rsvd;
+	__be16			lwm;
+};
+
+struct mlx5_arm_srq_mbox_out {
+	struct mlx5_outbox_hdr	hdr;
+	u8			rsvd[8];
+};
+
+struct mlx5_cq_context {
+	u8			status;
+	u8			cqe_sz_flags;
+	u8			st;
+	u8			rsvd3;
+	u8			rsvd4[6];
+	__be16			page_offset;
+	__be32			log_sz_usr_page;
+	__be16			cq_period;
+	__be16			cq_max_count;
+	__be16			rsvd20;
+	__be16			c_eqn;
+	u8			log_pg_sz;
+	u8			rsvd25[7];
+	__be32			last_notified_index;
+	__be32			solicit_producer_index;
+	__be32			consumer_counter;
+	__be32			producer_counter;
+	u8			rsvd48[8];
+	__be64			db_record_addr;
+};
+
+struct mlx5_create_cq_mbox_in {
+	struct mlx5_inbox_hdr	hdr;
+	__be32			input_cqn;
+	u8			rsvdx[4];
+	struct mlx5_cq_context	ctx;
+	u8			rsvd6[192];
+	__be64			pas[0];
+};
+
+struct mlx5_create_cq_mbox_out {
+	struct mlx5_outbox_hdr	hdr;
+	__be32			cqn;
+	u8			rsvd0[4];
+};
+
+struct mlx5_destroy_cq_mbox_in {
+	struct mlx5_inbox_hdr	hdr;
+	__be32			cqn;
+	u8			rsvd0[4];
+};
+
+struct mlx5_destroy_cq_mbox_out {
+	struct mlx5_outbox_hdr	hdr;
+	u8			rsvd0[8];
+};
+
+struct mlx5_query_cq_mbox_in {
+	struct mlx5_inbox_hdr	hdr;
+	__be32			cqn;
+	u8			rsvd0[4];
+};
+
+struct mlx5_query_cq_mbox_out {
+	struct mlx5_outbox_hdr	hdr;
+	u8			rsvd0[8];
+	struct mlx5_cq_context	ctx;
+	u8			rsvd6[16];
+	__be64			pas[0];
+};
+
+struct mlx5_modify_cq_mbox_in {
+	struct mlx5_inbox_hdr	hdr;
+	__be32			cqn;
+	__be32			field_select;
+	struct mlx5_cq_context	ctx;
+	u8			rsvd[192];
+	__be64			pas[0];
+};
+
+struct mlx5_modify_cq_mbox_out {
+	struct mlx5_outbox_hdr	hdr;
+	u8			rsvd[8];
+};
+
+struct mlx5_eq_context {
+	u8			status;
+	u8			ec_oi;
+	u8			st;
+	u8			rsvd2[7];
+	__be16			page_pffset;
+	__be32			log_sz_usr_page;
+	u8			rsvd3[7];
+	u8			intr;
+	u8			log_page_size;
+	u8			rsvd4[15];
+	__be32			consumer_counter;
+	__be32			produser_counter;
+	u8			rsvd5[16];
+};
+
+struct mlx5_create_eq_mbox_in {
+	struct mlx5_inbox_hdr	hdr;
+	u8			rsvd0[3];
+	u8			input_eqn;
+	u8			rsvd1[4];
+	struct mlx5_eq_context	ctx;
+	u8			rsvd2[8];
+	__be64			events_mask;
+	u8			rsvd3[176];
+	__be64			pas[0];
+};
+
+struct mlx5_create_eq_mbox_out {
+	struct mlx5_outbox_hdr	hdr;
+	u8			rsvd0[3];
+	u8			eq_number;
+	u8			rsvd1[4];
+};
+
+struct mlx5_map_eq_mbox_in {
+	struct mlx5_inbox_hdr	hdr;
+	__be64			mask;
+	u8			mu;
+	u8			rsvd0[2];
+	u8			eqn;
+	u8			rsvd1[24];
+};
+
+struct mlx5_map_eq_mbox_out {
+	struct mlx5_outbox_hdr	hdr;
+	u8			rsvd[8];
+};
+
+struct mlx5_query_eq_mbox_in {
+	struct mlx5_inbox_hdr	hdr;
+	u8			rsvd0[3];
+	u8			eqn;
+	u8			rsvd1[4];
+};
+
+struct mlx5_query_eq_mbox_out {
+	struct mlx5_outbox_hdr	hdr;
+	u8			rsvd[8];
+	struct mlx5_eq_context	ctx;
+};
+
+struct mlx5_mkey_seg {
+	/* This is a two bit field occupying bits 31-30.
+	 * bit 31 is always 0,
+	 * bit 30 is zero for regular MRs and 1 (e.g free) for UMRs that do not have tanslation
+	 */
+	u8		status;
+	u8		pcie_control;
+	u8		flags;
+	u8		version;
+	__be32		qpn_mkey7_0;
+	u8		rsvd1[4];
+	__be32		flags_pd;
+	__be64		start_addr;
+	__be64		len;
+	__be32		bsfs_octo_size;
+	u8		rsvd2[16];
+	__be32		xlt_oct_size;
+	u8		rsvd3[3];
+	u8		log2_page_size;
+	u8		rsvd4[4];
+};
+
+struct mlx5_query_special_ctxs_mbox_in {
+	struct mlx5_inbox_hdr	hdr;
+	u8			rsvd[8];
+};
+
+struct mlx5_query_special_ctxs_mbox_out {
+	struct mlx5_outbox_hdr	hdr;
+	__be32			dump_fill_mkey;
+	__be32			reserved_lkey;
+};
+
+struct mlx5_create_mkey_mbox_in {
+	struct mlx5_inbox_hdr	hdr;
+	__be32			input_mkey_index;
+	u8			rsvd0[4];
+	struct mlx5_mkey_seg	seg;
+	u8			rsvd1[16];
+	__be32			xlat_oct_act_size;
+	__be32			rsvd2;
+	u8			rsvd3[168];
+	__be64			pas[0];
+};
+
+struct mlx5_create_mkey_mbox_out {
+	struct mlx5_outbox_hdr	hdr;
+	__be32			mkey;
+	u8			rsvd[4];
+};
+
+struct mlx5_query_mkey_mbox_in {
+	struct mlx5_inbox_hdr	hdr;
+	__be32			mkey;
+};
+
+struct mlx5_query_mkey_mbox_out {
+	struct mlx5_outbox_hdr	hdr;
+	__be64			pas[0];
+};
+
+struct mlx5_modify_mkey_mbox_in {
+	struct mlx5_inbox_hdr	hdr;
+	__be32			mkey;
+	__be64			pas[0];
+};
+
+struct mlx5_modify_mkey_mbox_out {
+	struct mlx5_outbox_hdr	hdr;
+	u8			rsvd[8];
+};
+
+struct mlx5_dump_mkey_mbox_in {
+	struct mlx5_inbox_hdr	hdr;
+};
+
+struct mlx5_dump_mkey_mbox_out {
+	struct mlx5_outbox_hdr	hdr;
+	__be32			mkey;
+};
+
+struct mlx5_mad_ifc_mbox_in {
+	struct mlx5_inbox_hdr	hdr;
+	__be16			remote_lid;
+	u8			rsvd0;
+	u8			port;
+	u8			rsvd1[4];
+	u8			data[256];
+};
+
+struct mlx5_mad_ifc_mbox_out {
+	struct mlx5_outbox_hdr	hdr;
+	u8			rsvd[8];
+	u8			data[256];
+};
+
+struct mlx5_access_reg_mbox_in {
+	struct mlx5_inbox_hdr		hdr;
+	u8				rsvd0[2];
+	__be16				register_id;
+	__be32				arg;
+	__be32				data[0];
+};
+
+struct mlx5_access_reg_mbox_out {
+	struct mlx5_outbox_hdr		hdr;
+	u8				rsvd[8];
+	__be32				data[0];
+};
+
+#define MLX5_ATTR_EXTENDED_PORT_INFO	cpu_to_be16(0xff90)
+
+enum {
+	MLX_EXT_PORT_CAP_FLAG_EXTENDED_PORT_INFO	= 1 <<  0
+};
+
+struct mlx5_allocate_psv_in {
+	struct mlx5_inbox_hdr   hdr;
+	__be32			npsv_pd;
+	__be32			rsvd_psv0;
+};
+
+struct mlx5_allocate_psv_out {
+	struct mlx5_outbox_hdr  hdr;
+	u8			rsvd[8];
+	__be32			psv_idx[4];
+};
+
+struct mlx5_destroy_psv_in {
+	struct mlx5_inbox_hdr	hdr;
+	__be32                  psv_number;
+	u8                      rsvd[4];
+};
+
+struct mlx5_destroy_psv_out {
+	struct mlx5_outbox_hdr  hdr;
+	u8                      rsvd[8];
+};
+
+#define MLX5_CMD_OP_MAX 0x939
+
+enum {
+	VPORT_STATE_DOWN		= 0x0,
+	VPORT_STATE_UP			= 0x1,
+};
+
+enum {
+	MLX5_L3_PROT_TYPE_IPV4		= 0,
+	MLX5_L3_PROT_TYPE_IPV6		= 1,
+};
+
+enum {
+	MLX5_L4_PROT_TYPE_TCP		= 0,
+	MLX5_L4_PROT_TYPE_UDP		= 1,
+};
+
+enum {
+	MLX5_HASH_FIELD_SEL_SRC_IP	= 1 << 0,
+	MLX5_HASH_FIELD_SEL_DST_IP	= 1 << 1,
+	MLX5_HASH_FIELD_SEL_L4_SPORT	= 1 << 2,
+	MLX5_HASH_FIELD_SEL_L4_DPORT	= 1 << 3,
+	MLX5_HASH_FIELD_SEL_IPSEC_SPI	= 1 << 4,
+};
+
+enum {
+	MLX5_MATCH_OUTER_HEADERS	= 1 << 0,
+	MLX5_MATCH_MISC_PARAMETERS	= 1 << 1,
+	MLX5_MATCH_INNER_HEADERS	= 1 << 2,
+
+};
+
+enum {
+	MLX5_FLOW_TABLE_TYPE_NIC_RCV	 = 0,
+	MLX5_FLOW_TABLE_TYPE_EGRESS_ACL  = 2,
+	MLX5_FLOW_TABLE_TYPE_INGRESS_ACL = 3,
+	MLX5_FLOW_TABLE_TYPE_ESWITCH	 = 4,
+};
+
+enum {
+	MLX5_MODIFY_ESW_VPORT_CONTEXT_CVLAN_INSERT_NONE	      = 0,
+	MLX5_MODIFY_ESW_VPORT_CONTEXT_CVLAN_INSERT_IF_NO_VLAN = 1,
+	MLX5_MODIFY_ESW_VPORT_CONTEXT_CVLAN_INSERT_OVERWRITE  = 2
+};
+
+enum {
+	MLX5_MODIFY_ESW_VPORT_CONTEXT_FIELD_SELECT_SVLAN_STRIP  = 1 << 0,
+	MLX5_MODIFY_ESW_VPORT_CONTEXT_FIELD_SELECT_CVLAN_STRIP  = 1 << 1,
+	MLX5_MODIFY_ESW_VPORT_CONTEXT_FIELD_SELECT_SVLAN_INSERT = 1 << 2,
+	MLX5_MODIFY_ESW_VPORT_CONTEXT_FIELD_SELECT_CVLAN_INSERT = 1 << 3
+};
+
+/* MLX5 DEV CAPs */
+
+/* TODO: EAT.ME */
+enum mlx5_cap_mode {
+	HCA_CAP_OPMOD_GET_MAX	= 0,
+	HCA_CAP_OPMOD_GET_CUR	= 1,
+};
+
+enum mlx5_cap_type {
+	MLX5_CAP_GENERAL = 0,
+	MLX5_CAP_ETHERNET_OFFLOADS,
+	MLX5_CAP_ODP,
+	MLX5_CAP_ATOMIC,
+	MLX5_CAP_ROCE,
+	MLX5_CAP_IPOIB_OFFLOADS,
+	MLX5_CAP_EOIB_OFFLOADS,
+	MLX5_CAP_FLOW_TABLE,
+	MLX5_CAP_ESWITCH_FLOW_TABLE,
+	MLX5_CAP_ESWITCH,
+	/* NUM OF CAP Types */
+	MLX5_CAP_NUM
+};
+
+/* GET Dev Caps macros */
+#define MLX5_CAP_GEN(mdev, cap) \
+	MLX5_GET(cmd_hca_cap, mdev->hca_caps_cur[MLX5_CAP_GENERAL], cap)
+
+#define MLX5_CAP_GEN_MAX(mdev, cap) \
+	MLX5_GET(cmd_hca_cap, mdev->hca_caps_max[MLX5_CAP_GENERAL], cap)
+
+#define MLX5_CAP_ETH(mdev, cap) \
+	MLX5_GET(per_protocol_networking_offload_caps,\
+		 mdev->hca_caps_cur[MLX5_CAP_ETHERNET_OFFLOADS], cap)
+
+#define MLX5_CAP_ETH_MAX(mdev, cap) \
+	MLX5_GET(per_protocol_networking_offload_caps,\
+		 mdev->hca_caps_max[MLX5_CAP_ETHERNET_OFFLOADS], cap)
+
+#define MLX5_CAP_ROCE(mdev, cap) \
+	MLX5_GET(roce_cap, mdev->hca_caps_cur[MLX5_CAP_ROCE], cap)
+
+#define MLX5_CAP_ROCE_MAX(mdev, cap) \
+	MLX5_GET(roce_cap, mdev->hca_caps_max[MLX5_CAP_ROCE], cap)
+
+#define MLX5_CAP_ATOMIC(mdev, cap) \
+	MLX5_GET(atomic_caps, mdev->hca_caps_cur[MLX5_CAP_ATOMIC], cap)
+
+#define MLX5_CAP_ATOMIC_MAX(mdev, cap) \
+	MLX5_GET(atomic_caps, mdev->hca_caps_max[MLX5_CAP_ATOMIC], cap)
+
+#define MLX5_CAP_FLOWTABLE(mdev, cap) \
+	MLX5_GET(flow_table_nic_cap, mdev->hca_caps_cur[MLX5_CAP_FLOW_TABLE], cap)
+
+#define MLX5_CAP_FLOWTABLE_MAX(mdev, cap) \
+	MLX5_GET(flow_table_nic_cap, mdev->hca_caps_max[MLX5_CAP_FLOW_TABLE], cap)
+
+#define MLX5_CAP_ESW_FLOWTABLE(mdev, cap) \
+	MLX5_GET(flow_table_eswitch_cap, \
+		 mdev->hca_caps_cur[MLX5_CAP_ESWITCH_FLOW_TABLE], cap)
+
+#define MLX5_CAP_ESW_FLOWTABLE_MAX(mdev, cap) \
+	MLX5_GET(flow_table_eswitch_cap, \
+		 mdev->hca_caps_max[MLX5_CAP_ESWITCH_FLOW_TABLE], cap)
+
+#define MLX5_CAP_ESW(mdev, cap) \
+	MLX5_GET(e_switch_cap, \
+		 mdev->hca_caps_cur[MLX5_CAP_ESWITCH], cap)
+
+#define MLX5_CAP_ESW_MAX(mdev, cap) \
+	MLX5_GET(e_switch_cap, \
+		 mdev->hca_caps_max[MLX5_CAP_ESWITCH], cap)
+
+#define MLX5_CAP_ODP(mdev, cap)\
+	MLX5_GET(odp_cap, mdev->hca_caps_cur[MLX5_CAP_ODP], cap)
+
+#define MLX5_CAP_ODP_MAX(mdev, cap)\
+	MLX5_GET(odp_cap, mdev->hca_caps_max[MLX5_CAP_ODP], cap)
+
+enum {
+	MLX5_CMD_STAT_OK			= 0x0,
+	MLX5_CMD_STAT_INT_ERR			= 0x1,
+	MLX5_CMD_STAT_BAD_OP_ERR		= 0x2,
+	MLX5_CMD_STAT_BAD_PARAM_ERR		= 0x3,
+	MLX5_CMD_STAT_BAD_SYS_STATE_ERR		= 0x4,
+	MLX5_CMD_STAT_BAD_RES_ERR		= 0x5,
+	MLX5_CMD_STAT_RES_BUSY			= 0x6,
+	MLX5_CMD_STAT_LIM_ERR			= 0x8,
+	MLX5_CMD_STAT_BAD_RES_STATE_ERR		= 0x9,
+	MLX5_CMD_STAT_IX_ERR			= 0xa,
+	MLX5_CMD_STAT_NO_RES_ERR		= 0xf,
+	MLX5_CMD_STAT_BAD_INP_LEN_ERR		= 0x50,
+	MLX5_CMD_STAT_BAD_OUTP_LEN_ERR		= 0x51,
+	MLX5_CMD_STAT_BAD_QP_STATE_ERR		= 0x10,
+	MLX5_CMD_STAT_BAD_PKT_ERR		= 0x30,
+	MLX5_CMD_STAT_BAD_SIZE_OUTS_CQES_ERR	= 0x40,
+};
+
+enum {
+	MLX5_IEEE_802_3_COUNTERS_GROUP	      = 0x0,
+	MLX5_RFC_2863_COUNTERS_GROUP	      = 0x1,
+	MLX5_RFC_2819_COUNTERS_GROUP	      = 0x2,
+	MLX5_RFC_3635_COUNTERS_GROUP	      = 0x3,
+	MLX5_ETHERNET_EXTENDED_COUNTERS_GROUP = 0x5,
+	MLX5_PER_PRIORITY_COUNTERS_GROUP      = 0x10,
+	MLX5_PER_TRAFFIC_CLASS_COUNTERS_GROUP = 0x11,
+	MLX5_PHYSICAL_LAYER_COUNTERS_GROUP    = 0x12,
+};
+
+enum {
+	MLX5_CAP_PORT_TYPE_IB  = 0x0,
+	MLX5_CAP_PORT_TYPE_ETH = 0x1,
+};
+
+enum {
+	MLX5_CMD_HCA_CAP_MIN_WQE_INLINE_MODE_L2           = 0x0,
+	MLX5_CMD_HCA_CAP_MIN_WQE_INLINE_MODE_VPORT_CONFIG = 0x1,
+	MLX5_CMD_HCA_CAP_MIN_WQE_INLINE_MODE_NOT_REQUIRED = 0x2
+};
+
+enum {
+	MLX5_QUERY_VPORT_STATE_OUT_STATE_FOLLOW = 0x2,
+};
+
+static inline u16 mlx5_to_sw_pkey_sz(int pkey_sz)
+{
+	if (pkey_sz > MLX5_MAX_LOG_PKEY_TABLE)
+		return 0;
+	return MLX5_MIN_PKEY_TABLE_SIZE << pkey_sz;
+}
+
+struct mlx5_ifc_mcia_reg_bits {
+	u8         l[0x1];
+	u8         reserved_0[0x7];
+	u8         module[0x8];
+	u8         reserved_1[0x8];
+	u8         status[0x8];
+
+	u8         i2c_device_address[0x8];
+	u8         page_number[0x8];
+	u8         device_address[0x10];
+
+	u8         reserved_2[0x10];
+	u8         size[0x10];
+
+	u8         reserved_3[0x20];
+
+	u8         dword_0[0x20];
+	u8         dword_1[0x20];
+	u8         dword_2[0x20];
+	u8         dword_3[0x20];
+	u8         dword_4[0x20];
+	u8         dword_5[0x20];
+	u8         dword_6[0x20];
+	u8         dword_7[0x20];
+	u8         dword_8[0x20];
+	u8         dword_9[0x20];
+	u8         dword_10[0x20];
+	u8         dword_11[0x20];
+};
+
+#define MLX5_CMD_OP_QUERY_EEPROM 0x93c
+#endif /* MLX5_DEVICE_H */
diff --git a/sys/dev/mlx5/doorbell.h b/sys/dev/mlx5/doorbell.h
new file mode 100644
index 0000000..4365f01
--- /dev/null
+++ b/sys/dev/mlx5/doorbell.h
@@ -0,0 +1,74 @@
+/*-
+ * Copyright (c) 2013-2015, Mellanox Technologies, Ltd.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef MLX5_DOORBELL_H
+#define MLX5_DOORBELL_H
+
+#define MLX5_BF_OFFSET	      0x800
+#define MLX5_CQ_DOORBELL      0x20
+
+#if BITS_PER_LONG == 64
+/* Assume that we can just write a 64-bit doorbell atomically.  s390
+ * actually doesn't have writeq() but S/390 systems don't even have
+ * PCI so we won't worry about it.
+ */
+
+#define MLX5_DECLARE_DOORBELL_LOCK(name)
+#define MLX5_INIT_DOORBELL_LOCK(ptr)    do { } while (0)
+#define MLX5_GET_DOORBELL_LOCK(ptr)      (NULL)
+
+static inline void mlx5_write64(__be32 val[2], void __iomem *dest,
+				spinlock_t *doorbell_lock)
+{
+	__raw_writeq(*(u64 *)val, dest);
+}
+
+#else
+
+/* Just fall back to a spinlock to protect the doorbell if
+ * BITS_PER_LONG is 32 -- there's no portable way to do atomic 64-bit
+ * MMIO writes.
+ */
+
+#define MLX5_DECLARE_DOORBELL_LOCK(name) spinlock_t name;
+#define MLX5_INIT_DOORBELL_LOCK(ptr)     spin_lock_init(ptr)
+#define MLX5_GET_DOORBELL_LOCK(ptr)      (ptr)
+
+static inline void mlx5_write64(__be32 val[2], void __iomem *dest,
+				spinlock_t *doorbell_lock)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(doorbell_lock, flags);
+	__raw_writel((__force u32) val[0], dest);
+	__raw_writel((__force u32) val[1], dest + 4);
+	spin_unlock_irqrestore(doorbell_lock, flags);
+}
+
+#endif
+
+#endif /* MLX5_DOORBELL_H */
diff --git a/sys/dev/mlx5/driver.h b/sys/dev/mlx5/driver.h
new file mode 100644
index 0000000..6f4031e
--- /dev/null
+++ b/sys/dev/mlx5/driver.h
@@ -0,0 +1,941 @@
+/*-
+ * Copyright (c) 2013-2015, Mellanox Technologies, Ltd.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef MLX5_DRIVER_H
+#define MLX5_DRIVER_H
+
+#include <linux/kernel.h>
+#include <linux/completion.h>
+#include <linux/pci.h>
+#include <linux/cache.h>
+#include <linux/rbtree.h>
+#include <linux/semaphore.h>
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+#include <linux/radix-tree.h>
+
+#include <dev/mlx5/device.h>
+#include <dev/mlx5/doorbell.h>
+
+enum {
+	MLX5_BOARD_ID_LEN = 64,
+	MLX5_MAX_NAME_LEN = 16,
+};
+
+enum {
+	/* one minute for the sake of bringup. Generally, commands must always
+	 * complete and we may need to increase this timeout value
+	 */
+	MLX5_CMD_TIMEOUT_MSEC	= 7200 * 1000,
+	MLX5_CMD_WQ_MAX_NAME	= 32,
+};
+
+enum {
+	CMD_OWNER_SW		= 0x0,
+	CMD_OWNER_HW		= 0x1,
+	CMD_STATUS_SUCCESS	= 0,
+};
+
+enum mlx5_sqp_t {
+	MLX5_SQP_SMI		= 0,
+	MLX5_SQP_GSI		= 1,
+	MLX5_SQP_IEEE_1588	= 2,
+	MLX5_SQP_SNIFFER	= 3,
+	MLX5_SQP_SYNC_UMR	= 4,
+};
+
+enum {
+	MLX5_MAX_PORTS	= 2,
+};
+
+enum {
+	MLX5_EQ_VEC_PAGES	 = 0,
+	MLX5_EQ_VEC_CMD		 = 1,
+	MLX5_EQ_VEC_ASYNC	 = 2,
+	MLX5_EQ_VEC_COMP_BASE,
+};
+
+enum {
+	MLX5_MAX_IRQ_NAME	= 32
+};
+
+enum {
+	MLX5_ATOMIC_MODE_IB_COMP	= 1 << 16,
+	MLX5_ATOMIC_MODE_CX		= 2 << 16,
+	MLX5_ATOMIC_MODE_8B		= 3 << 16,
+	MLX5_ATOMIC_MODE_16B		= 4 << 16,
+	MLX5_ATOMIC_MODE_32B		= 5 << 16,
+	MLX5_ATOMIC_MODE_64B		= 6 << 16,
+	MLX5_ATOMIC_MODE_128B		= 7 << 16,
+	MLX5_ATOMIC_MODE_256B		= 8 << 16,
+};
+
+enum {
+	MLX5_REG_QETCR		 = 0x4005,
+	MLX5_REG_QPDP		 = 0x4007,
+	MLX5_REG_QTCT		 = 0x400A,
+	MLX5_REG_PCAP		 = 0x5001,
+	MLX5_REG_PMTU		 = 0x5003,
+	MLX5_REG_PTYS		 = 0x5004,
+	MLX5_REG_PAOS		 = 0x5006,
+	MLX5_REG_PFCC		 = 0x5007,
+	MLX5_REG_PPCNT		 = 0x5008,
+	MLX5_REG_PMAOS		 = 0x5012,
+	MLX5_REG_PUDE		 = 0x5009,
+	MLX5_REG_PPTB		 = 0x500B,
+	MLX5_REG_PBMC		 = 0x500C,
+	MLX5_REG_PMPE		 = 0x5010,
+	MLX5_REG_PELC		 = 0x500e,
+	MLX5_REG_PVLC		 = 0x500f,
+	MLX5_REG_PMLP		 = 0x5002,
+	MLX5_REG_NODE_DESC	 = 0x6001,
+	MLX5_REG_HOST_ENDIANNESS = 0x7004,
+	MLX5_REG_MCIA		 = 0x9014,
+};
+
+enum dbg_rsc_type {
+	MLX5_DBG_RSC_QP,
+	MLX5_DBG_RSC_EQ,
+	MLX5_DBG_RSC_CQ,
+};
+
+struct mlx5_field_desc {
+	struct dentry	       *dent;
+	int			i;
+};
+
+struct mlx5_rsc_debug {
+	struct mlx5_core_dev   *dev;
+	void		       *object;
+	enum dbg_rsc_type	type;
+	struct dentry	       *root;
+	struct mlx5_field_desc	fields[0];
+};
+
+enum mlx5_dev_event {
+	MLX5_DEV_EVENT_SYS_ERROR,
+	MLX5_DEV_EVENT_PORT_UP,
+	MLX5_DEV_EVENT_PORT_DOWN,
+	MLX5_DEV_EVENT_PORT_INITIALIZED,
+	MLX5_DEV_EVENT_LID_CHANGE,
+	MLX5_DEV_EVENT_PKEY_CHANGE,
+	MLX5_DEV_EVENT_GUID_CHANGE,
+	MLX5_DEV_EVENT_CLIENT_REREG,
+	MLX5_DEV_EVENT_VPORT_CHANGE,
+};
+
+enum mlx5_port_status {
+	MLX5_PORT_UP        = 1 << 0,
+	MLX5_PORT_DOWN      = 1 << 1,
+};
+
+enum mlx5_link_mode {
+	MLX5_1000BASE_CX_SGMII	= 0,
+	MLX5_1000BASE_KX	= 1,
+	MLX5_10GBASE_CX4	= 2,
+	MLX5_10GBASE_KX4	= 3,
+	MLX5_10GBASE_KR		= 4,
+	MLX5_20GBASE_KR2	= 5,
+	MLX5_40GBASE_CR4	= 6,
+	MLX5_40GBASE_KR4	= 7,
+	MLX5_56GBASE_R4		= 8,
+	MLX5_10GBASE_CR		= 12,
+	MLX5_10GBASE_SR		= 13,
+	MLX5_10GBASE_ER		= 14,
+	MLX5_40GBASE_SR4	= 15,
+	MLX5_40GBASE_LR4	= 16,
+	MLX5_100GBASE_CR4	= 20,
+	MLX5_100GBASE_SR4	= 21,
+	MLX5_100GBASE_KR4	= 22,
+	MLX5_100GBASE_LR4	= 23,
+	MLX5_100BASE_TX		= 24,
+	MLX5_1000BASE_T		= 25,
+	MLX5_10GBASE_T		= 26,
+	MLX5_25GBASE_CR		= 27,
+	MLX5_25GBASE_KR		= 28,
+	MLX5_25GBASE_SR		= 29,
+	MLX5_50GBASE_CR2	= 30,
+	MLX5_50GBASE_KR2	= 31,
+	MLX5_LINK_MODES_NUMBER,
+};
+
+#define MLX5_PROT_MASK(link_mode) (1 << link_mode)
+
+struct mlx5_uuar_info {
+	struct mlx5_uar	       *uars;
+	int			num_uars;
+	int			num_low_latency_uuars;
+	unsigned long	       *bitmap;
+	unsigned int	       *count;
+	struct mlx5_bf	       *bfs;
+
+	/*
+	 * protect uuar allocation data structs
+	 */
+	struct mutex		lock;
+	u32			ver;
+};
+
+struct mlx5_bf {
+	void __iomem	       *reg;
+	void __iomem	       *regreg;
+	int			buf_size;
+	struct mlx5_uar	       *uar;
+	unsigned long		offset;
+	int			need_lock;
+	/* protect blue flame buffer selection when needed
+	 */
+	spinlock_t		lock;
+
+	/* serialize 64 bit writes when done as two 32 bit accesses
+	 */
+	spinlock_t		lock32;
+	int			uuarn;
+};
+
+struct mlx5_cmd_first {
+	__be32		data[4];
+};
+
+struct mlx5_cmd_msg {
+	struct list_head		list;
+	struct cache_ent	       *cache;
+	u32				len;
+	struct mlx5_cmd_first		first;
+	struct mlx5_cmd_mailbox	       *next;
+};
+
+struct mlx5_cmd_debug {
+	struct dentry	       *dbg_root;
+	struct dentry	       *dbg_in;
+	struct dentry	       *dbg_out;
+	struct dentry	       *dbg_outlen;
+	struct dentry	       *dbg_status;
+	struct dentry	       *dbg_run;
+	void		       *in_msg;
+	void		       *out_msg;
+	u8			status;
+	u16			inlen;
+	u16			outlen;
+};
+
+struct cache_ent {
+	/* protect block chain allocations
+	 */
+	spinlock_t		lock;
+	struct list_head	head;
+};
+
+struct cmd_msg_cache {
+	struct cache_ent	large;
+	struct cache_ent	med;
+
+};
+
+struct mlx5_cmd_stats {
+	u64		sum;
+	u64		n;
+	struct dentry  *root;
+	struct dentry  *avg;
+	struct dentry  *count;
+	/* protect command average calculations */
+	spinlock_t	lock;
+};
+
+struct mlx5_cmd {
+	void	       *cmd_alloc_buf;
+	dma_addr_t	alloc_dma;
+	int		alloc_size;
+	void	       *cmd_buf;
+	dma_addr_t	dma;
+	u16		cmdif_rev;
+	u8		log_sz;
+	u8		log_stride;
+	int		max_reg_cmds;
+	int		events;
+	u32 __iomem    *vector;
+
+	/* protect command queue allocations
+	 */
+	spinlock_t	alloc_lock;
+
+	/* protect token allocations
+	 */
+	spinlock_t	token_lock;
+	u8		token;
+	unsigned long	bitmask;
+	char		wq_name[MLX5_CMD_WQ_MAX_NAME];
+	struct workqueue_struct *wq;
+	struct semaphore sem;
+	struct semaphore pages_sem;
+	int	mode;
+	struct mlx5_cmd_work_ent *ent_arr[MLX5_MAX_COMMANDS];
+	struct pci_pool *pool;
+	struct mlx5_cmd_debug dbg;
+	struct cmd_msg_cache cache;
+	int checksum_disabled;
+	struct mlx5_cmd_stats stats[MLX5_CMD_OP_MAX];
+	int moving_to_polling;
+};
+
+struct mlx5_port_caps {
+	int	gid_table_len;
+	int	pkey_table_len;
+	u8	ext_port_cap;
+};
+
+struct mlx5_cmd_mailbox {
+	void	       *buf;
+	dma_addr_t	dma;
+	struct mlx5_cmd_mailbox *next;
+};
+
+struct mlx5_buf_list {
+	void		       *buf;
+	dma_addr_t		map;
+};
+
+struct mlx5_buf {
+	struct mlx5_buf_list	direct;
+	struct mlx5_buf_list   *page_list;
+	int			nbufs;
+	int			npages;
+	int			size;
+	u8			page_shift;
+};
+
+struct mlx5_eq {
+	struct mlx5_core_dev   *dev;
+	__be32 __iomem	       *doorbell;
+	u32			cons_index;
+	struct mlx5_buf		buf;
+	int			size;
+	u8			irqn;
+	u8			eqn;
+	int			nent;
+	u64			mask;
+	struct list_head	list;
+	int			index;
+	struct mlx5_rsc_debug	*dbg;
+};
+
+struct mlx5_core_psv {
+	u32	psv_idx;
+	struct psv_layout {
+		u32	pd;
+		u16	syndrome;
+		u16	reserved;
+		u16	bg;
+		u16	app_tag;
+		u32	ref_tag;
+	} psv;
+};
+
+struct mlx5_core_sig_ctx {
+	struct mlx5_core_psv	psv_memory;
+	struct mlx5_core_psv	psv_wire;
+#if (__FreeBSD_version >= 1100000)
+	struct ib_sig_err       err_item;
+#endif
+	bool			sig_status_checked;
+	bool			sig_err_exists;
+	u32			sigerr_count;
+};
+
+struct mlx5_core_mr {
+	u64			iova;
+	u64			size;
+	u32			key;
+	u32			pd;
+};
+
+enum mlx5_res_type {
+	MLX5_RES_QP,
+	MLX5_RES_SRQ,
+	MLX5_RES_XSRQ,
+};
+
+struct mlx5_core_rsc_common {
+	enum mlx5_res_type	res;
+	atomic_t		refcount;
+	struct completion	free;
+};
+
+struct mlx5_core_srq {
+	struct mlx5_core_rsc_common	common; /* must be first */
+	u32				srqn;
+	int				max;
+	int				max_gs;
+	int				max_avail_gather;
+	int				wqe_shift;
+	void				(*event)(struct mlx5_core_srq *, int);
+	atomic_t			refcount;
+	struct completion		free;
+};
+
+struct mlx5_eq_table {
+	void __iomem	       *update_ci;
+	void __iomem	       *update_arm_ci;
+	struct list_head	comp_eqs_list;
+	struct mlx5_eq		pages_eq;
+	struct mlx5_eq		async_eq;
+	struct mlx5_eq		cmd_eq;
+	int			num_comp_vectors;
+	/* protect EQs list
+	 */
+	spinlock_t		lock;
+};
+
+struct mlx5_uar {
+	u32			index;
+	struct list_head	bf_list;
+	unsigned		free_bf_bmap;
+	void __iomem	       *bf_map;
+	void __iomem	       *map;
+};
+
+
+struct mlx5_core_health {
+	struct mlx5_health_buffer __iomem	*health;
+	__be32 __iomem		       *health_counter;
+	struct timer_list		timer;
+	struct list_head		list;
+	u32				prev;
+	int				miss_counter;
+};
+
+#define	MLX5_CQ_LINEAR_ARRAY_SIZE	1024
+
+struct mlx5_cq_linear_array_entry {
+	spinlock_t	lock;
+	struct mlx5_core_cq * volatile cq;
+};
+
+struct mlx5_cq_table {
+	/* protect radix tree
+	 */
+	spinlock_t		lock;
+	struct radix_tree_root	tree;
+	struct mlx5_cq_linear_array_entry linear_array[MLX5_CQ_LINEAR_ARRAY_SIZE];
+};
+
+struct mlx5_qp_table {
+	/* protect radix tree
+	 */
+	spinlock_t		lock;
+	struct radix_tree_root	tree;
+};
+
+struct mlx5_srq_table {
+	/* protect radix tree
+	 */
+	spinlock_t		lock;
+	struct radix_tree_root	tree;
+};
+
+struct mlx5_mr_table {
+	/* protect radix tree
+	 */
+	rwlock_t		lock;
+	struct radix_tree_root	tree;
+};
+
+struct mlx5_irq_info {
+	char name[MLX5_MAX_IRQ_NAME];
+};
+
+struct mlx5_priv {
+	char			name[MLX5_MAX_NAME_LEN];
+	struct mlx5_eq_table	eq_table;
+	struct msix_entry	*msix_arr;
+	struct mlx5_irq_info	*irq_info;
+	struct mlx5_uuar_info	uuari;
+	MLX5_DECLARE_DOORBELL_LOCK(cq_uar_lock);
+
+	struct io_mapping	*bf_mapping;
+
+	/* pages stuff */
+	struct workqueue_struct *pg_wq;
+	struct rb_root		page_root;
+	int			fw_pages;
+	int			reg_pages;
+	struct list_head	free_list;
+
+	struct mlx5_core_health health;
+
+	struct mlx5_srq_table	srq_table;
+
+	/* start: qp staff */
+	struct mlx5_qp_table	qp_table;
+	struct dentry	       *qp_debugfs;
+	struct dentry	       *eq_debugfs;
+	struct dentry	       *cq_debugfs;
+	struct dentry	       *cmdif_debugfs;
+	/* end: qp staff */
+
+	/* start: cq staff */
+	struct mlx5_cq_table	cq_table;
+	/* end: cq staff */
+
+	/* start: mr staff */
+	struct mlx5_mr_table	mr_table;
+	/* end: mr staff */
+
+	/* start: alloc staff */
+	int			numa_node;
+
+	struct mutex   pgdir_mutex;
+	struct list_head        pgdir_list;
+	/* end: alloc staff */
+	struct dentry	       *dbg_root;
+
+	/* protect mkey key part */
+	spinlock_t		mkey_lock;
+	u8			mkey_key;
+
+	struct list_head        dev_list;
+	struct list_head        ctx_list;
+	spinlock_t              ctx_lock;
+};
+
+struct mlx5_special_contexts {
+	int resd_lkey;
+};
+
+struct mlx5_core_dev {
+	struct pci_dev	       *pdev;
+	char			board_id[MLX5_BOARD_ID_LEN];
+	struct mlx5_cmd		cmd;
+	struct mlx5_port_caps	port_caps[MLX5_MAX_PORTS];
+	u32 hca_caps_cur[MLX5_CAP_NUM][MLX5_UN_SZ_DW(hca_cap_union)];
+	u32 hca_caps_max[MLX5_CAP_NUM][MLX5_UN_SZ_DW(hca_cap_union)];
+	struct mlx5_init_seg __iomem *iseg;
+	void			(*event) (struct mlx5_core_dev *dev,
+					  enum mlx5_dev_event event,
+					  unsigned long param);
+	struct mlx5_priv	priv;
+	struct mlx5_profile	*profile;
+	atomic_t		num_qps;
+	u32			issi;
+	struct mlx5_special_contexts special_contexts;
+};
+
+enum {
+	MLX5_WOL_DISABLE       = 0,
+	MLX5_WOL_SECURED_MAGIC = 1 << 1,
+	MLX5_WOL_MAGIC         = 1 << 2,
+	MLX5_WOL_ARP           = 1 << 3,
+	MLX5_WOL_BROADCAST     = 1 << 4,
+	MLX5_WOL_MULTICAST     = 1 << 5,
+	MLX5_WOL_UNICAST       = 1 << 6,
+	MLX5_WOL_PHY_ACTIVITY  = 1 << 7,
+};
+
+struct mlx5_db {
+	__be32			*db;
+	union {
+		struct mlx5_db_pgdir		*pgdir;
+		struct mlx5_ib_user_db_page	*user_page;
+	}			u;
+	dma_addr_t		dma;
+	int			index;
+};
+
+struct mlx5_net_counters {
+	u64	packets;
+	u64	octets;
+};
+
+struct mlx5_ptys_reg {
+	u8	local_port;
+	u8	proto_mask;
+	u32	eth_proto_cap;
+	u16	ib_link_width_cap;
+	u16	ib_proto_cap;
+	u32	eth_proto_admin;
+	u16	ib_link_width_admin;
+	u16	ib_proto_admin;
+	u32	eth_proto_oper;
+	u16	ib_link_width_oper;
+	u16	ib_proto_oper;
+	u32	eth_proto_lp_advertise;
+};
+
+struct mlx5_pvlc_reg {
+	u8	local_port;
+	u8	vl_hw_cap;
+	u8	vl_admin;
+	u8	vl_operational;
+};
+
+struct mlx5_pmtu_reg {
+	u8	local_port;
+	u16	max_mtu;
+	u16	admin_mtu;
+	u16	oper_mtu;
+};
+
+struct mlx5_vport_counters {
+	struct mlx5_net_counters	received_errors;
+	struct mlx5_net_counters	transmit_errors;
+	struct mlx5_net_counters	received_ib_unicast;
+	struct mlx5_net_counters	transmitted_ib_unicast;
+	struct mlx5_net_counters	received_ib_multicast;
+	struct mlx5_net_counters	transmitted_ib_multicast;
+	struct mlx5_net_counters	received_eth_broadcast;
+	struct mlx5_net_counters	transmitted_eth_broadcast;
+	struct mlx5_net_counters	received_eth_unicast;
+	struct mlx5_net_counters	transmitted_eth_unicast;
+	struct mlx5_net_counters	received_eth_multicast;
+	struct mlx5_net_counters	transmitted_eth_multicast;
+};
+
+enum {
+	MLX5_DB_PER_PAGE = PAGE_SIZE / L1_CACHE_BYTES,
+};
+
+enum {
+	MLX5_COMP_EQ_SIZE = 1024,
+};
+
+enum {
+	MLX5_PTYS_IB = 1 << 0,
+	MLX5_PTYS_EN = 1 << 2,
+};
+
+struct mlx5_db_pgdir {
+	struct list_head	list;
+	DECLARE_BITMAP(bitmap, MLX5_DB_PER_PAGE);
+	__be32		       *db_page;
+	dma_addr_t		db_dma;
+};
+
+typedef void (*mlx5_cmd_cbk_t)(int status, void *context);
+
+struct mlx5_cmd_work_ent {
+	struct mlx5_cmd_msg    *in;
+	struct mlx5_cmd_msg    *out;
+	void		       *uout;
+	int			uout_size;
+	mlx5_cmd_cbk_t		callback;
+	void		       *context;
+	int			idx;
+	struct completion	done;
+	struct mlx5_cmd        *cmd;
+	struct work_struct	work;
+	struct mlx5_cmd_layout *lay;
+	int			ret;
+	int			page_queue;
+	u8			status;
+	u8			token;
+	u64			ts1;
+	u64			ts2;
+	u16			op;
+};
+
+struct mlx5_pas {
+	u64	pa;
+	u8	log_sz;
+};
+
+static inline void *mlx5_buf_offset(struct mlx5_buf *buf, int offset)
+{
+	if (likely(BITS_PER_LONG == 64 || buf->nbufs == 1))
+		return buf->direct.buf + offset;
+	else
+		return buf->page_list[offset >> PAGE_SHIFT].buf +
+			(offset & (PAGE_SIZE - 1));
+}
+
+
+extern struct workqueue_struct *mlx5_core_wq;
+
+#define STRUCT_FIELD(header, field) \
+	.struct_offset_bytes = offsetof(struct ib_unpacked_ ## header, field),      \
+	.struct_size_bytes   = sizeof((struct ib_unpacked_ ## header *)0)->field
+
+static inline struct mlx5_core_dev *pci2mlx5_core_dev(struct pci_dev *pdev)
+{
+	return pci_get_drvdata(pdev);
+}
+
+extern struct dentry *mlx5_debugfs_root;
+
+static inline u16 fw_rev_maj(struct mlx5_core_dev *dev)
+{
+	return ioread32be(&dev->iseg->fw_rev) & 0xffff;
+}
+
+static inline u16 fw_rev_min(struct mlx5_core_dev *dev)
+{
+	return ioread32be(&dev->iseg->fw_rev) >> 16;
+}
+
+static inline u16 fw_rev_sub(struct mlx5_core_dev *dev)
+{
+	return ioread32be(&dev->iseg->cmdif_rev_fw_sub) & 0xffff;
+}
+
+static inline u16 cmdif_rev_get(struct mlx5_core_dev *dev)
+{
+	return ioread32be(&dev->iseg->cmdif_rev_fw_sub) >> 16;
+}
+
+static inline int mlx5_get_gid_table_len(u16 param)
+{
+	if (param > 4) {
+		printf("M4_CORE_DRV_NAME: WARN: ""gid table length is zero\n");
+		return 0;
+	}
+
+	return 8 * (1 << param);
+}
+
+static inline void *mlx5_vzalloc(unsigned long size)
+{
+	void *rtn;
+
+	rtn = kzalloc(size, GFP_KERNEL | __GFP_NOWARN);
+	return rtn;
+}
+
+static inline u32 mlx5_base_mkey(const u32 key)
+{
+	return key & 0xffffff00u;
+}
+
+int mlx5_cmd_init(struct mlx5_core_dev *dev);
+void mlx5_cmd_cleanup(struct mlx5_core_dev *dev);
+void mlx5_cmd_use_events(struct mlx5_core_dev *dev);
+void mlx5_cmd_use_polling(struct mlx5_core_dev *dev);
+int mlx5_cmd_status_to_err(struct mlx5_outbox_hdr *hdr);
+int mlx5_cmd_status_to_err_v2(void *ptr);
+int mlx5_core_get_caps(struct mlx5_core_dev *dev, enum mlx5_cap_type cap_type,
+		       enum mlx5_cap_mode cap_mode);
+int mlx5_cmd_exec(struct mlx5_core_dev *dev, void *in, int in_size, void *out,
+		  int out_size);
+int mlx5_cmd_exec_cb(struct mlx5_core_dev *dev, void *in, int in_size,
+		     void *out, int out_size, mlx5_cmd_cbk_t callback,
+		     void *context);
+int mlx5_cmd_alloc_uar(struct mlx5_core_dev *dev, u32 *uarn);
+int mlx5_cmd_free_uar(struct mlx5_core_dev *dev, u32 uarn);
+int mlx5_alloc_uuars(struct mlx5_core_dev *dev, struct mlx5_uuar_info *uuari);
+int mlx5_free_uuars(struct mlx5_core_dev *dev, struct mlx5_uuar_info *uuari);
+int mlx5_alloc_map_uar(struct mlx5_core_dev *mdev, struct mlx5_uar *uar);
+void mlx5_unmap_free_uar(struct mlx5_core_dev *mdev, struct mlx5_uar *uar);
+void mlx5_health_cleanup(void);
+void  __init mlx5_health_init(void);
+void mlx5_start_health_poll(struct mlx5_core_dev *dev);
+void mlx5_stop_health_poll(struct mlx5_core_dev *dev);
+int mlx5_buf_alloc_node(struct mlx5_core_dev *dev, int size, int max_direct,
+			struct mlx5_buf *buf, int node);
+int mlx5_buf_alloc(struct mlx5_core_dev *dev, int size, int max_direct,
+		   struct mlx5_buf *buf);
+void mlx5_buf_free(struct mlx5_core_dev *dev, struct mlx5_buf *buf);
+int mlx5_core_create_srq(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,
+			 struct mlx5_create_srq_mbox_in *in, int inlen,
+			 int is_xrc);
+int mlx5_core_destroy_srq(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq);
+int mlx5_core_query_srq(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,
+			struct mlx5_query_srq_mbox_out *out);
+int mlx5_core_query_vendor_id(struct mlx5_core_dev *mdev, u32 *vendor_id);
+int mlx5_core_arm_srq(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,
+		      u16 lwm, int is_srq);
+void mlx5_init_mr_table(struct mlx5_core_dev *dev);
+void mlx5_cleanup_mr_table(struct mlx5_core_dev *dev);
+int mlx5_core_create_mkey(struct mlx5_core_dev *dev, struct mlx5_core_mr *mr,
+			  struct mlx5_create_mkey_mbox_in *in, int inlen,
+			  mlx5_cmd_cbk_t callback, void *context,
+			  struct mlx5_create_mkey_mbox_out *out);
+int mlx5_core_destroy_mkey(struct mlx5_core_dev *dev, struct mlx5_core_mr *mr);
+int mlx5_core_query_mkey(struct mlx5_core_dev *dev, struct mlx5_core_mr *mr,
+			 struct mlx5_query_mkey_mbox_out *out, int outlen);
+int mlx5_core_dump_fill_mkey(struct mlx5_core_dev *dev, struct mlx5_core_mr *mr,
+			     u32 *mkey);
+int mlx5_core_alloc_pd(struct mlx5_core_dev *dev, u32 *pdn);
+int mlx5_core_dealloc_pd(struct mlx5_core_dev *dev, u32 pdn);
+int mlx5_core_mad_ifc(struct mlx5_core_dev *dev, void *inb, void *outb,
+		      u16 opmod, u8 port);
+void mlx5_pagealloc_init(struct mlx5_core_dev *dev);
+void mlx5_pagealloc_cleanup(struct mlx5_core_dev *dev);
+int mlx5_pagealloc_start(struct mlx5_core_dev *dev);
+void mlx5_pagealloc_stop(struct mlx5_core_dev *dev);
+void mlx5_core_req_pages_handler(struct mlx5_core_dev *dev, u16 func_id,
+				 s32 npages);
+int mlx5_satisfy_startup_pages(struct mlx5_core_dev *dev, int boot);
+int mlx5_reclaim_startup_pages(struct mlx5_core_dev *dev);
+void mlx5_register_debugfs(void);
+void mlx5_unregister_debugfs(void);
+int mlx5_eq_init(struct mlx5_core_dev *dev);
+void mlx5_eq_cleanup(struct mlx5_core_dev *dev);
+void mlx5_fill_page_array(struct mlx5_buf *buf, __be64 *pas);
+void mlx5_cq_completion(struct mlx5_core_dev *dev, u32 cqn);
+void mlx5_rsc_event(struct mlx5_core_dev *dev, u32 rsn, int event_type);
+void mlx5_srq_event(struct mlx5_core_dev *dev, u32 srqn, int event_type);
+struct mlx5_core_srq *mlx5_core_get_srq(struct mlx5_core_dev *dev, u32 srqn);
+void mlx5_cmd_comp_handler(struct mlx5_core_dev *dev, unsigned long vector);
+void mlx5_cq_event(struct mlx5_core_dev *dev, u32 cqn, int event_type);
+int mlx5_create_map_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq, u8 vecidx,
+		       int nent, u64 mask, const char *name, struct mlx5_uar *uar);
+int mlx5_destroy_unmap_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq);
+int mlx5_start_eqs(struct mlx5_core_dev *dev);
+int mlx5_stop_eqs(struct mlx5_core_dev *dev);
+int mlx5_vector2eqn(struct mlx5_core_dev *dev, int vector, int *eqn, int *irqn);
+int mlx5_core_attach_mcg(struct mlx5_core_dev *dev, union ib_gid *mgid, u32 qpn);
+int mlx5_core_detach_mcg(struct mlx5_core_dev *dev, union ib_gid *mgid, u32 qpn);
+
+int mlx5_qp_debugfs_init(struct mlx5_core_dev *dev);
+void mlx5_qp_debugfs_cleanup(struct mlx5_core_dev *dev);
+int mlx5_core_access_reg(struct mlx5_core_dev *dev, void *data_in,
+			 int size_in, void *data_out, int size_out,
+			 u16 reg_num, int arg, int write);
+
+int mlx5_set_port_caps(struct mlx5_core_dev *dev, u8 port_num, u32 caps);
+int mlx5_query_port_ptys(struct mlx5_core_dev *dev, u32 *ptys,
+			 int ptys_size, int proto_mask);
+int mlx5_query_port_proto_cap(struct mlx5_core_dev *dev,
+			      u32 *proto_cap, int proto_mask);
+int mlx5_query_port_proto_admin(struct mlx5_core_dev *dev,
+				u32 *proto_admin, int proto_mask);
+int mlx5_set_port_proto(struct mlx5_core_dev *dev, u32 proto_admin,
+			int proto_mask);
+int mlx5_set_port_status(struct mlx5_core_dev *dev,
+			 enum mlx5_port_status status);
+int mlx5_query_port_status(struct mlx5_core_dev *dev, u8 *status);
+int mlx5_set_port_pause(struct mlx5_core_dev *dev, u32 port,
+			u32 rx_pause, u32 tx_pause);
+int mlx5_query_port_pause(struct mlx5_core_dev *dev, u32 port,
+			  u32 *rx_pause, u32 *tx_pause);
+
+int mlx5_set_port_mtu(struct mlx5_core_dev *dev, int mtu);
+int mlx5_query_port_max_mtu(struct mlx5_core_dev *dev, int *max_mtu);
+int mlx5_query_port_oper_mtu(struct mlx5_core_dev *dev, int *oper_mtu);
+
+int mlx5_query_module_num(struct mlx5_core_dev *dev, int *module_num);
+int mlx5_query_eeprom(struct mlx5_core_dev *dev, int i2c_addr, int page_num,
+		      int device_addr, int size, int module_num, u32 *data,
+		      int *size_read);
+
+int mlx5_debug_eq_add(struct mlx5_core_dev *dev, struct mlx5_eq *eq);
+void mlx5_debug_eq_remove(struct mlx5_core_dev *dev, struct mlx5_eq *eq);
+int mlx5_core_eq_query(struct mlx5_core_dev *dev, struct mlx5_eq *eq,
+		       struct mlx5_query_eq_mbox_out *out, int outlen);
+int mlx5_eq_debugfs_init(struct mlx5_core_dev *dev);
+void mlx5_eq_debugfs_cleanup(struct mlx5_core_dev *dev);
+int mlx5_cq_debugfs_init(struct mlx5_core_dev *dev);
+void mlx5_cq_debugfs_cleanup(struct mlx5_core_dev *dev);
+int mlx5_db_alloc(struct mlx5_core_dev *dev, struct mlx5_db *db);
+int mlx5_db_alloc_node(struct mlx5_core_dev *dev, struct mlx5_db *db,
+		       int node);
+void mlx5_db_free(struct mlx5_core_dev *dev, struct mlx5_db *db);
+
+const char *mlx5_command_str(int command);
+int mlx5_cmdif_debugfs_init(struct mlx5_core_dev *dev);
+void mlx5_cmdif_debugfs_cleanup(struct mlx5_core_dev *dev);
+int mlx5_core_create_psv(struct mlx5_core_dev *dev, u32 pdn,
+			 int npsvs, u32 *sig_index);
+int mlx5_core_destroy_psv(struct mlx5_core_dev *dev, int psv_num);
+void mlx5_core_put_rsc(struct mlx5_core_rsc_common *common);
+u8 mlx5_is_wol_supported(struct mlx5_core_dev *dev);
+int mlx5_set_wol(struct mlx5_core_dev *dev, u8 wol_mode);
+int mlx5_query_wol(struct mlx5_core_dev *dev, u8 *wol_mode);
+int mlx5_core_access_pvlc(struct mlx5_core_dev *dev,
+			  struct mlx5_pvlc_reg *pvlc, int write);
+int mlx5_core_access_ptys(struct mlx5_core_dev *dev,
+			  struct mlx5_ptys_reg *ptys, int write);
+int mlx5_core_access_pmtu(struct mlx5_core_dev *dev,
+			  struct mlx5_pmtu_reg *pmtu, int write);
+int mlx5_vxlan_udp_port_add(struct mlx5_core_dev *dev, u16 port);
+int mlx5_vxlan_udp_port_delete(struct mlx5_core_dev *dev, u16 port);
+int mlx5_query_port_cong_status(struct mlx5_core_dev *mdev, int protocol,
+				int priority, int *is_enable);
+int mlx5_modify_port_cong_status(struct mlx5_core_dev *mdev, int protocol,
+				 int priority, int enable);
+int mlx5_query_port_cong_params(struct mlx5_core_dev *mdev, int protocol,
+				void *out, int out_size);
+int mlx5_modify_port_cong_params(struct mlx5_core_dev *mdev,
+				 void *in, int in_size);
+int mlx5_query_port_cong_statistics(struct mlx5_core_dev *mdev, int clear,
+				    void *out, int out_size);
+static inline u32 mlx5_mkey_to_idx(u32 mkey)
+{
+	return mkey >> 8;
+}
+
+static inline u32 mlx5_idx_to_mkey(u32 mkey_idx)
+{
+	return mkey_idx << 8;
+}
+
+static inline u8 mlx5_mkey_variant(u32 mkey)
+{
+	return mkey & 0xff;
+}
+
+enum {
+	MLX5_PROF_MASK_QP_SIZE		= (u64)1 << 0,
+	MLX5_PROF_MASK_MR_CACHE		= (u64)1 << 1,
+};
+
+enum {
+	MAX_MR_CACHE_ENTRIES    = 16,
+};
+
+enum {
+	MLX5_INTERFACE_PROTOCOL_IB  = 0,
+	MLX5_INTERFACE_PROTOCOL_ETH = 1,
+};
+
+struct mlx5_interface {
+	void *			(*add)(struct mlx5_core_dev *dev);
+	void			(*remove)(struct mlx5_core_dev *dev, void *context);
+	void			(*event)(struct mlx5_core_dev *dev, void *context,
+					 enum mlx5_dev_event event, unsigned long param);
+	void *                  (*get_dev)(void *context);
+	int			protocol;
+	struct list_head	list;
+};
+
+void *mlx5_get_protocol_dev(struct mlx5_core_dev *mdev, int protocol);
+int mlx5_register_interface(struct mlx5_interface *intf);
+void mlx5_unregister_interface(struct mlx5_interface *intf);
+
+struct mlx5_profile {
+	u64	mask;
+	u8	log_max_qp;
+	struct {
+		int	size;
+		int	limit;
+	} mr_cache[MAX_MR_CACHE_ENTRIES];
+};
+
+
+#define MLX5_EEPROM_MAX_BYTES			48
+#define MLX5_EEPROM_IDENTIFIER_BYTE_MASK	0x000000ff
+#define MLX5_EEPROM_REVISION_ID_BYTE_MASK	0x0000ff00
+#define MLX5_EEPROM_PAGE_3_VALID_BIT_MASK	0x00040000
+#endif /* MLX5_DRIVER_H */
diff --git a/sys/dev/mlx5/flow_table.h b/sys/dev/mlx5/flow_table.h
new file mode 100644
index 0000000..61ce2baf
--- /dev/null
+++ b/sys/dev/mlx5/flow_table.h
@@ -0,0 +1,50 @@
+/*-
+ * Copyright (c) 2013-2015, Mellanox Technologies, Ltd.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef MLX5_FLOW_TABLE_H
+#define MLX5_FLOW_TABLE_H
+
+#include <dev/mlx5/driver.h>
+
+struct mlx5_flow_table_group {
+	u8	log_sz;
+	u8	match_criteria_enable;
+	u32	match_criteria[MLX5_ST_SZ_DW(fte_match_param)];
+};
+
+void *mlx5_create_flow_table(struct mlx5_core_dev *dev, u8 level, u8 table_type,
+			     u16 vport,
+			     u16 num_groups,
+			     struct mlx5_flow_table_group *group);
+void mlx5_destroy_flow_table(void *flow_table);
+int mlx5_add_flow_table_entry(void *flow_table, u8 match_criteria_enable,
+			      void *match_criteria, void *flow_context,
+			      u32 *flow_index);
+void mlx5_del_flow_table_entry(void *flow_table, u32 flow_index);
+u32 mlx5_get_flow_table_id(void *flow_table);
+
+#endif /* MLX5_FLOW_TABLE_H */
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_alloc.c b/sys/dev/mlx5/mlx5_core/mlx5_alloc.c
new file mode 100644
index 0000000..513fc6f
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_core/mlx5_alloc.c
@@ -0,0 +1,256 @@
+/*-
+ * Copyright (c) 2013-2015, Mellanox Technologies, Ltd.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <linux/errno.h>
+#include <linux/slab.h>
+#include <linux/mm.h>
+#include <linux/dma-mapping.h>
+#include <linux/vmalloc.h>
+#include <dev/mlx5/driver.h>
+
+#include "mlx5_core.h"
+
+/* Handling for queue buffers -- we allocate a bunch of memory and
+ * register it in a memory region at HCA virtual address 0.  If the
+ * requested size is > max_direct, we split the allocation into
+ * multiple pages, so we don't require too much contiguous memory.
+ */
+
+static void *mlx5_dma_zalloc_coherent_node(struct mlx5_core_dev *dev,
+					   size_t size, dma_addr_t *dma_handle,
+					   int node)
+{
+	void *cpu_handle;
+
+	cpu_handle = dma_zalloc_coherent(&dev->pdev->dev, size,
+					 dma_handle, GFP_KERNEL);
+	return cpu_handle;
+}
+
+int mlx5_buf_alloc_node(struct mlx5_core_dev *dev, int size, int max_direct,
+			struct mlx5_buf *buf, int node)
+{
+	dma_addr_t t;
+
+	buf->size = size;
+	if (size <= max_direct) {
+		buf->nbufs        = 1;
+		buf->npages       = 1;
+		buf->page_shift   = (u8)get_order(size) + PAGE_SHIFT;
+		buf->direct.buf   = mlx5_dma_zalloc_coherent_node(dev, size,
+								  &t, node);
+		if (!buf->direct.buf)
+			return -ENOMEM;
+
+		buf->direct.map = t;
+
+		while (t & ((1 << buf->page_shift) - 1)) {
+			--buf->page_shift;
+			buf->npages *= 2;
+		}
+	} else {
+		int i;
+
+		buf->direct.buf  = NULL;
+		buf->nbufs       = (size + PAGE_SIZE - 1) / PAGE_SIZE;
+		buf->npages      = buf->nbufs;
+		buf->page_shift  = PAGE_SHIFT;
+		buf->page_list   = kcalloc(buf->nbufs, sizeof(*buf->page_list),
+					   GFP_KERNEL);
+
+		for (i = 0; i < buf->nbufs; i++) {
+			buf->page_list[i].buf =
+				mlx5_dma_zalloc_coherent_node(dev, PAGE_SIZE,
+							      &t, node);
+
+			buf->page_list[i].map = t;
+		}
+
+		if (BITS_PER_LONG == 64) {
+			struct page **pages;
+
+			pages = kmalloc(sizeof(*pages) * (buf->nbufs + 1),
+					GFP_KERNEL);
+			for (i = 0; i < buf->nbufs; i++)
+				pages[i] = virt_to_page(buf->page_list[i].buf);
+			pages[buf->nbufs] = pages[0];
+			buf->direct.buf = vmap(pages, buf->nbufs + 1, VM_MAP,
+					       PAGE_KERNEL);
+			kfree(pages);
+			if (!buf->direct.buf)
+				goto err_free;
+		}
+	}
+
+	return 0;
+
+err_free:
+	mlx5_buf_free(dev, buf);
+
+	return -ENOMEM;
+}
+
+int mlx5_buf_alloc(struct mlx5_core_dev *dev, int size, int max_direct,
+		   struct mlx5_buf *buf)
+{
+	return mlx5_buf_alloc_node(dev, size, max_direct,
+				   buf, dev->priv.numa_node);
+}
+EXPORT_SYMBOL_GPL(mlx5_buf_alloc);
+
+
+void mlx5_buf_free(struct mlx5_core_dev *dev, struct mlx5_buf *buf)
+{
+	if (buf->nbufs == 1)
+		dma_free_coherent(&dev->pdev->dev, buf->size, buf->direct.buf,
+				  buf->direct.map);
+	else {
+		int i;
+		if (BITS_PER_LONG == 64 && buf->direct.buf)
+			vunmap(buf->direct.buf);
+
+		for (i = 0; i < buf->nbufs; i++)
+			if (buf->page_list[i].buf)
+				dma_free_coherent(&dev->pdev->dev, PAGE_SIZE,
+						  buf->page_list[i].buf,
+						  buf->page_list[i].map);
+		kfree(buf->page_list);
+	}
+}
+EXPORT_SYMBOL_GPL(mlx5_buf_free);
+
+static struct mlx5_db_pgdir *mlx5_alloc_db_pgdir(struct mlx5_core_dev *dev,
+						 int node)
+{
+	struct mlx5_db_pgdir *pgdir;
+
+	pgdir = kzalloc(sizeof(*pgdir), GFP_KERNEL);
+
+	bitmap_fill(pgdir->bitmap, MLX5_DB_PER_PAGE);
+
+	pgdir->db_page = mlx5_dma_zalloc_coherent_node(dev, PAGE_SIZE,
+						       &pgdir->db_dma, node);
+	if (!pgdir->db_page) {
+		kfree(pgdir);
+		return NULL;
+	}
+
+	return pgdir;
+}
+
+static int mlx5_alloc_db_from_pgdir(struct mlx5_db_pgdir *pgdir,
+				    struct mlx5_db *db)
+{
+	int offset;
+	int i;
+
+	i = find_first_bit(pgdir->bitmap, MLX5_DB_PER_PAGE);
+	if (i >= MLX5_DB_PER_PAGE)
+		return -ENOMEM;
+
+	__clear_bit(i, pgdir->bitmap);
+
+	db->u.pgdir = pgdir;
+	db->index   = i;
+	offset = db->index * L1_CACHE_BYTES;
+	db->db      = pgdir->db_page + offset / sizeof(*pgdir->db_page);
+	db->dma     = pgdir->db_dma  + offset;
+
+	db->db[0] = 0;
+	db->db[1] = 0;
+
+	return 0;
+}
+
+int mlx5_db_alloc_node(struct mlx5_core_dev *dev, struct mlx5_db *db, int node)
+{
+	struct mlx5_db_pgdir *pgdir;
+	int ret = 0;
+
+	mutex_lock(&dev->priv.pgdir_mutex);
+
+	list_for_each_entry(pgdir, &dev->priv.pgdir_list, list)
+		if (!mlx5_alloc_db_from_pgdir(pgdir, db))
+			goto out;
+
+	pgdir = mlx5_alloc_db_pgdir(dev, node);
+	if (!pgdir) {
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	list_add(&pgdir->list, &dev->priv.pgdir_list);
+
+	/* This should never fail -- we just allocated an empty page: */
+	WARN_ON(mlx5_alloc_db_from_pgdir(pgdir, db));
+
+out:
+	mutex_unlock(&dev->priv.pgdir_mutex);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(mlx5_db_alloc_node);
+
+int mlx5_db_alloc(struct mlx5_core_dev *dev, struct mlx5_db *db)
+{
+	return mlx5_db_alloc_node(dev, db, dev->priv.numa_node);
+}
+EXPORT_SYMBOL_GPL(mlx5_db_alloc);
+
+void mlx5_db_free(struct mlx5_core_dev *dev, struct mlx5_db *db)
+{
+	mutex_lock(&dev->priv.pgdir_mutex);
+
+	__set_bit(db->index, db->u.pgdir->bitmap);
+
+	if (bitmap_full(db->u.pgdir->bitmap, MLX5_DB_PER_PAGE)) {
+		dma_free_coherent(&(dev->pdev->dev), PAGE_SIZE,
+				  db->u.pgdir->db_page, db->u.pgdir->db_dma);
+		list_del(&db->u.pgdir->list);
+		kfree(db->u.pgdir);
+	}
+
+	mutex_unlock(&dev->priv.pgdir_mutex);
+}
+EXPORT_SYMBOL_GPL(mlx5_db_free);
+
+
+void mlx5_fill_page_array(struct mlx5_buf *buf, __be64 *pas)
+{
+	u64 addr;
+	int i;
+
+	for (i = 0; i < buf->npages; i++) {
+		if (buf->nbufs == 1)
+			addr = buf->direct.map + ((u64)i << buf->page_shift);
+		else
+			addr = buf->page_list[i].map;
+
+		pas[i] = cpu_to_be64(addr);
+	}
+}
+EXPORT_SYMBOL_GPL(mlx5_fill_page_array);
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_cmd.c b/sys/dev/mlx5/mlx5_core/mlx5_cmd.c
new file mode 100644
index 0000000..42087da
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_core/mlx5_cmd.c
@@ -0,0 +1,1571 @@
+/*-
+ * Copyright (c) 2013-2015, Mellanox Technologies, Ltd.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <linux/module.h>
+#include <linux/errno.h>
+#include <linux/pci.h>
+#include <linux/dma-mapping.h>
+#include <linux/slab.h>
+#include <linux/delay.h>
+#include <linux/random.h>
+#include <linux/io-mapping.h>
+#include <linux/hardirq.h>
+#include <linux/ktime.h>
+#include <dev/mlx5/driver.h>
+
+#include "mlx5_core.h"
+
+enum {
+	CMD_IF_REV = 5,
+};
+
+enum {
+	CMD_MODE_POLLING,
+	CMD_MODE_EVENTS
+};
+
+enum {
+	NUM_LONG_LISTS	  = 2,
+	NUM_MED_LISTS	  = 64,
+	LONG_LIST_SIZE	  = (2ULL * 1024 * 1024 * 1024 / PAGE_SIZE) * 8 + 16 +
+				MLX5_CMD_DATA_BLOCK_SIZE,
+	MED_LIST_SIZE	  = 16 + MLX5_CMD_DATA_BLOCK_SIZE,
+};
+
+enum {
+	MLX5_CMD_DELIVERY_STAT_OK			= 0x0,
+	MLX5_CMD_DELIVERY_STAT_SIGNAT_ERR		= 0x1,
+	MLX5_CMD_DELIVERY_STAT_TOK_ERR			= 0x2,
+	MLX5_CMD_DELIVERY_STAT_BAD_BLK_NUM_ERR		= 0x3,
+	MLX5_CMD_DELIVERY_STAT_OUT_PTR_ALIGN_ERR	= 0x4,
+	MLX5_CMD_DELIVERY_STAT_IN_PTR_ALIGN_ERR		= 0x5,
+	MLX5_CMD_DELIVERY_STAT_FW_ERR			= 0x6,
+	MLX5_CMD_DELIVERY_STAT_IN_LENGTH_ERR		= 0x7,
+	MLX5_CMD_DELIVERY_STAT_OUT_LENGTH_ERR		= 0x8,
+	MLX5_CMD_DELIVERY_STAT_RES_FLD_NOT_CLR_ERR	= 0x9,
+	MLX5_CMD_DELIVERY_STAT_CMD_DESCR_ERR		= 0x10,
+};
+
+static struct mlx5_cmd_work_ent *alloc_cmd(struct mlx5_cmd *cmd,
+					   struct mlx5_cmd_msg *in,
+					   struct mlx5_cmd_msg *out,
+					   void *uout, int uout_size,
+					   mlx5_cmd_cbk_t cbk,
+					   void *context, int page_queue)
+{
+	gfp_t alloc_flags = cbk ? GFP_ATOMIC : GFP_KERNEL;
+	struct mlx5_cmd_work_ent *ent;
+
+	ent = kzalloc(sizeof(*ent), alloc_flags);
+	if (!ent)
+		return ERR_PTR(-ENOMEM);
+
+	ent->in		= in;
+	ent->out	= out;
+	ent->uout	= uout;
+	ent->uout_size	= uout_size;
+	ent->callback	= cbk;
+	ent->context	= context;
+	ent->cmd	= cmd;
+	ent->page_queue = page_queue;
+
+	return ent;
+}
+
+static u8 alloc_token(struct mlx5_cmd *cmd)
+{
+	u8 token;
+
+	spin_lock(&cmd->token_lock);
+	cmd->token++;
+	if (cmd->token == 0)
+		cmd->token++;
+	token = cmd->token;
+	spin_unlock(&cmd->token_lock);
+
+	return token;
+}
+
+static int alloc_ent(struct mlx5_cmd *cmd)
+{
+	unsigned long flags;
+	int ret;
+
+	spin_lock_irqsave(&cmd->alloc_lock, flags);
+	ret = find_first_bit(&cmd->bitmask, cmd->max_reg_cmds);
+	if (ret < cmd->max_reg_cmds)
+		clear_bit(ret, &cmd->bitmask);
+	spin_unlock_irqrestore(&cmd->alloc_lock, flags);
+
+	return ret < cmd->max_reg_cmds ? ret : -ENOMEM;
+}
+
+static void free_ent(struct mlx5_cmd *cmd, int idx)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&cmd->alloc_lock, flags);
+	set_bit(idx, &cmd->bitmask);
+	spin_unlock_irqrestore(&cmd->alloc_lock, flags);
+}
+
+static struct mlx5_cmd_layout *get_inst(struct mlx5_cmd *cmd, int idx)
+{
+	return cmd->cmd_buf + (idx << cmd->log_stride);
+}
+
+static u8 xor8_buf(void *buf, int len)
+{
+	u8 *ptr = buf;
+	u8 sum = 0;
+	int i;
+
+	for (i = 0; i < len; i++)
+		sum ^= ptr[i];
+
+	return sum;
+}
+
+static int verify_block_sig(struct mlx5_cmd_prot_block *block)
+{
+	if (xor8_buf(block->rsvd0, sizeof(*block) - sizeof(block->data) - 1) != 0xff)
+		return -EINVAL;
+
+	if (xor8_buf(block, sizeof(*block)) != 0xff)
+		return -EINVAL;
+
+	return 0;
+}
+
+static void calc_block_sig(struct mlx5_cmd_prot_block *block, u8 token,
+			   int csum)
+{
+	block->token = token;
+	if (csum) {
+		block->ctrl_sig = ~xor8_buf(block->rsvd0, sizeof(*block) -
+					    sizeof(block->data) - 2);
+		block->sig = ~xor8_buf(block, sizeof(*block) - 1);
+	}
+}
+
+static void calc_chain_sig(struct mlx5_cmd_msg *msg, u8 token, int csum)
+{
+	struct mlx5_cmd_mailbox *next = msg->next;
+
+	while (next) {
+		calc_block_sig(next->buf, token, csum);
+		next = next->next;
+	}
+}
+
+static void set_signature(struct mlx5_cmd_work_ent *ent, int csum)
+{
+	ent->lay->sig = ~xor8_buf(ent->lay, sizeof(*ent->lay));
+	calc_chain_sig(ent->in, ent->token, csum);
+	calc_chain_sig(ent->out, ent->token, csum);
+}
+
+static void poll_timeout(struct mlx5_cmd_work_ent *ent)
+{
+	int poll_end = jiffies + msecs_to_jiffies(MLX5_CMD_TIMEOUT_MSEC + 1000);
+	u8 own;
+
+	do {
+		own = ent->lay->status_own;
+		if (!(own & CMD_OWNER_HW)) {
+			ent->ret = 0;
+			return;
+		}
+		usleep_range(5000, 10000);
+	} while (time_before(jiffies, poll_end));
+
+	ent->ret = -ETIMEDOUT;
+}
+
+static void free_cmd(struct mlx5_cmd_work_ent *ent)
+{
+	kfree(ent);
+}
+
+
+static int verify_signature(struct mlx5_cmd_work_ent *ent)
+{
+	struct mlx5_cmd_mailbox *next = ent->out->next;
+	int err;
+	u8 sig;
+
+	sig = xor8_buf(ent->lay, sizeof(*ent->lay));
+	if (sig != 0xff)
+		return -EINVAL;
+
+	while (next) {
+		err = verify_block_sig(next->buf);
+		if (err)
+			return err;
+
+		next = next->next;
+	}
+
+	return 0;
+}
+
+static void dump_buf(void *buf, int size, int data_only, int offset)
+{
+	__be32 *p = buf;
+	int i;
+
+	for (i = 0; i < size; i += 16) {
+		pr_debug("%03x: %08x %08x %08x %08x\n", offset, be32_to_cpu(p[0]),
+			 be32_to_cpu(p[1]), be32_to_cpu(p[2]),
+			 be32_to_cpu(p[3]));
+		p += 4;
+		offset += 16;
+	}
+	if (!data_only)
+		pr_debug("\n");
+}
+
+const char *mlx5_command_str(int command)
+{
+	switch (command) {
+	case MLX5_CMD_OP_QUERY_HCA_CAP:
+		return "QUERY_HCA_CAP";
+
+	case MLX5_CMD_OP_SET_HCA_CAP:
+		return "SET_HCA_CAP";
+
+	case MLX5_CMD_OP_QUERY_ADAPTER:
+		return "QUERY_ADAPTER";
+
+	case MLX5_CMD_OP_INIT_HCA:
+		return "INIT_HCA";
+
+	case MLX5_CMD_OP_TEARDOWN_HCA:
+		return "TEARDOWN_HCA";
+
+	case MLX5_CMD_OP_ENABLE_HCA:
+		return "MLX5_CMD_OP_ENABLE_HCA";
+
+	case MLX5_CMD_OP_DISABLE_HCA:
+		return "MLX5_CMD_OP_DISABLE_HCA";
+
+	case MLX5_CMD_OP_QUERY_PAGES:
+		return "QUERY_PAGES";
+
+	case MLX5_CMD_OP_MANAGE_PAGES:
+		return "MANAGE_PAGES";
+
+	case MLX5_CMD_OP_QUERY_ISSI:
+		return "QUERY_ISSI";
+
+	case MLX5_CMD_OP_SET_ISSI:
+		return "SET_ISSI";
+
+	case MLX5_CMD_OP_CREATE_MKEY:
+		return "CREATE_MKEY";
+
+	case MLX5_CMD_OP_QUERY_MKEY:
+		return "QUERY_MKEY";
+
+	case MLX5_CMD_OP_DESTROY_MKEY:
+		return "DESTROY_MKEY";
+
+	case MLX5_CMD_OP_QUERY_SPECIAL_CONTEXTS:
+		return "QUERY_SPECIAL_CONTEXTS";
+
+	case MLX5_CMD_OP_PAGE_FAULT_RESUME:
+		return "PAGE_FAULT_RESUME";
+
+	case MLX5_CMD_OP_CREATE_EQ:
+		return "CREATE_EQ";
+
+	case MLX5_CMD_OP_DESTROY_EQ:
+		return "DESTROY_EQ";
+
+	case MLX5_CMD_OP_QUERY_EQ:
+		return "QUERY_EQ";
+
+	case MLX5_CMD_OP_GEN_EQE:
+		return "GEN_EQE";
+
+	case MLX5_CMD_OP_CREATE_CQ:
+		return "CREATE_CQ";
+
+	case MLX5_CMD_OP_DESTROY_CQ:
+		return "DESTROY_CQ";
+
+	case MLX5_CMD_OP_QUERY_CQ:
+		return "QUERY_CQ";
+
+	case MLX5_CMD_OP_MODIFY_CQ:
+		return "MODIFY_CQ";
+
+	case MLX5_CMD_OP_CREATE_QP:
+		return "CREATE_QP";
+
+	case MLX5_CMD_OP_DESTROY_QP:
+		return "DESTROY_QP";
+
+	case MLX5_CMD_OP_RST2INIT_QP:
+		return "RST2INIT_QP";
+
+	case MLX5_CMD_OP_INIT2RTR_QP:
+		return "INIT2RTR_QP";
+
+	case MLX5_CMD_OP_RTR2RTS_QP:
+		return "RTR2RTS_QP";
+
+	case MLX5_CMD_OP_RTS2RTS_QP:
+		return "RTS2RTS_QP";
+
+	case MLX5_CMD_OP_SQERR2RTS_QP:
+		return "SQERR2RTS_QP";
+
+	case MLX5_CMD_OP_2ERR_QP:
+		return "2ERR_QP";
+
+	case MLX5_CMD_OP_2RST_QP:
+		return "2RST_QP";
+
+	case MLX5_CMD_OP_QUERY_QP:
+		return "QUERY_QP";
+
+	case MLX5_CMD_OP_SQD_RTS_QP:
+		return "SQD_RTS_QP";
+
+	case MLX5_CMD_OP_MAD_IFC:
+		return "MAD_IFC";
+
+	case MLX5_CMD_OP_INIT2INIT_QP:
+		return "INIT2INIT_QP";
+
+	case MLX5_CMD_OP_CREATE_PSV:
+		return "CREATE_PSV";
+
+	case MLX5_CMD_OP_DESTROY_PSV:
+		return "DESTROY_PSV";
+
+	case MLX5_CMD_OP_CREATE_SRQ:
+		return "CREATE_SRQ";
+
+	case MLX5_CMD_OP_DESTROY_SRQ:
+		return "DESTROY_SRQ";
+
+	case MLX5_CMD_OP_QUERY_SRQ:
+		return "QUERY_SRQ";
+
+	case MLX5_CMD_OP_ARM_RQ:
+		return "ARM_RQ";
+
+	case MLX5_CMD_OP_CREATE_XRC_SRQ:
+		return "CREATE_XRC_SRQ";
+
+	case MLX5_CMD_OP_DESTROY_XRC_SRQ:
+		return "DESTROY_XRC_SRQ";
+
+	case MLX5_CMD_OP_QUERY_XRC_SRQ:
+		return "QUERY_XRC_SRQ";
+
+	case MLX5_CMD_OP_ARM_XRC_SRQ:
+		return "ARM_XRC_SRQ";
+
+	case MLX5_CMD_OP_CREATE_DCT:
+		return "CREATE_DCT";
+
+	case MLX5_CMD_OP_DESTROY_DCT:
+		return "DESTROY_DCT";
+
+	case MLX5_CMD_OP_DRAIN_DCT:
+		return "DRAIN_DCT";
+
+	case MLX5_CMD_OP_QUERY_DCT:
+		return "QUERY_DCT";
+
+	case MLX5_CMD_OP_ARM_DCT_FOR_KEY_VIOLATION:
+		return "ARM_DCT_FOR_KEY_VIOLATION";
+
+	case MLX5_CMD_OP_QUERY_VPORT_STATE:
+		return "QUERY_VPORT_STATE";
+
+	case MLX5_CMD_OP_MODIFY_VPORT_STATE:
+		return "MODIFY_VPORT_STATE";
+
+	case MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT:
+		return "QUERY_ESW_VPORT_CONTEXT";
+
+	case MLX5_CMD_OP_MODIFY_ESW_VPORT_CONTEXT:
+		return "MODIFY_ESW_VPORT_CONTEXT";
+
+	case MLX5_CMD_OP_QUERY_NIC_VPORT_CONTEXT:
+		return "QUERY_NIC_VPORT_CONTEXT";
+
+	case MLX5_CMD_OP_MODIFY_NIC_VPORT_CONTEXT:
+		return "MODIFY_NIC_VPORT_CONTEXT";
+
+	case MLX5_CMD_OP_QUERY_ROCE_ADDRESS:
+		return "QUERY_ROCE_ADDRESS";
+
+	case MLX5_CMD_OP_SET_ROCE_ADDRESS:
+		return "SET_ROCE_ADDRESS";
+
+	case MLX5_CMD_OP_QUERY_HCA_VPORT_CONTEXT:
+		return "QUERY_HCA_VPORT_CONTEXT";
+
+	case MLX5_CMD_OP_MODIFY_HCA_VPORT_CONTEXT:
+		return "MODIFY_HCA_VPORT_CONTEXT";
+
+	case MLX5_CMD_OP_QUERY_HCA_VPORT_GID:
+		return "QUERY_HCA_VPORT_GID";
+
+	case MLX5_CMD_OP_QUERY_HCA_VPORT_PKEY:
+		return "QUERY_HCA_VPORT_PKEY";
+
+	case MLX5_CMD_OP_QUERY_VPORT_COUNTER:
+		return "QUERY_VPORT_COUNTER";
+
+	case MLX5_CMD_OP_SET_WOL_ROL:
+		return "SET_WOL_ROL";
+
+	case MLX5_CMD_OP_QUERY_WOL_ROL:
+		return "QUERY_WOL_ROL";
+
+	case MLX5_CMD_OP_ALLOC_Q_COUNTER:
+		return "ALLOC_Q_COUNTER";
+
+	case MLX5_CMD_OP_DEALLOC_Q_COUNTER:
+		return "DEALLOC_Q_COUNTER";
+
+	case MLX5_CMD_OP_QUERY_Q_COUNTER:
+		return "QUERY_Q_COUNTER";
+
+	case MLX5_CMD_OP_ALLOC_PD:
+		return "ALLOC_PD";
+
+	case MLX5_CMD_OP_DEALLOC_PD:
+		return "DEALLOC_PD";
+
+	case MLX5_CMD_OP_ALLOC_UAR:
+		return "ALLOC_UAR";
+
+	case MLX5_CMD_OP_DEALLOC_UAR:
+		return "DEALLOC_UAR";
+
+	case MLX5_CMD_OP_CONFIG_INT_MODERATION:
+		return "CONFIG_INT_MODERATION";
+
+	case MLX5_CMD_OP_ATTACH_TO_MCG:
+		return "ATTACH_TO_MCG";
+
+	case MLX5_CMD_OP_DETACH_FROM_MCG:
+		return "DETACH_FROM_MCG";
+
+	case MLX5_CMD_OP_GET_DROPPED_PACKET_LOG:
+		return "GET_DROPPED_PACKET_LOG";
+
+	case MLX5_CMD_OP_QUERY_MAD_DEMUX:
+		return "QUERY_MAD_DEMUX";
+
+	case MLX5_CMD_OP_SET_MAD_DEMUX:
+		return "SET_MAD_DEMUX";
+
+	case MLX5_CMD_OP_NOP:
+		return "NOP";
+
+	case MLX5_CMD_OP_ALLOC_XRCD:
+		return "ALLOC_XRCD";
+
+	case MLX5_CMD_OP_DEALLOC_XRCD:
+		return "DEALLOC_XRCD";
+
+	case MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN:
+		return "ALLOC_TRANSPORT_DOMAIN";
+
+	case MLX5_CMD_OP_DEALLOC_TRANSPORT_DOMAIN:
+		return "DEALLOC_TRANSPORT_DOMAIN";
+
+	case MLX5_CMD_OP_QUERY_CONG_STATUS:
+		return "QUERY_CONG_STATUS";
+
+	case MLX5_CMD_OP_MODIFY_CONG_STATUS:
+		return "MODIFY_CONG_STATUS";
+
+	case MLX5_CMD_OP_QUERY_CONG_PARAMS:
+		return "QUERY_CONG_PARAMS";
+
+	case MLX5_CMD_OP_MODIFY_CONG_PARAMS:
+		return "MODIFY_CONG_PARAMS";
+
+	case MLX5_CMD_OP_QUERY_CONG_STATISTICS:
+		return "QUERY_CONG_STATISTICS";
+
+	case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
+		return "ADD_VXLAN_UDP_DPORT";
+
+	case MLX5_CMD_OP_DELETE_VXLAN_UDP_DPORT:
+		return "DELETE_VXLAN_UDP_DPORT";
+
+	case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
+		return "SET_L2_TABLE_ENTRY";
+
+	case MLX5_CMD_OP_QUERY_L2_TABLE_ENTRY:
+		return "QUERY_L2_TABLE_ENTRY";
+
+	case MLX5_CMD_OP_DELETE_L2_TABLE_ENTRY:
+		return "DELETE_L2_TABLE_ENTRY";
+
+	case MLX5_CMD_OP_CREATE_RMP:
+		return "CREATE_RMP";
+
+	case MLX5_CMD_OP_MODIFY_RMP:
+		return "MODIFY_RMP";
+
+	case MLX5_CMD_OP_DESTROY_RMP:
+		return "DESTROY_RMP";
+
+	case MLX5_CMD_OP_QUERY_RMP:
+		return "QUERY_RMP";
+
+	case MLX5_CMD_OP_CREATE_RQT:
+		return "CREATE_RQT";
+
+	case MLX5_CMD_OP_MODIFY_RQT:
+		return "MODIFY_RQT";
+
+	case MLX5_CMD_OP_DESTROY_RQT:
+		return "DESTROY_RQT";
+
+	case MLX5_CMD_OP_QUERY_RQT:
+		return "QUERY_RQT";
+
+	case MLX5_CMD_OP_ACCESS_REG:
+		return "MLX5_CMD_OP_ACCESS_REG";
+
+	case MLX5_CMD_OP_CREATE_SQ:
+		return "CREATE_SQ";
+
+	case MLX5_CMD_OP_MODIFY_SQ:
+		return "MODIFY_SQ";
+
+	case MLX5_CMD_OP_DESTROY_SQ:
+		return "DESTROY_SQ";
+
+	case MLX5_CMD_OP_QUERY_SQ:
+		return "QUERY_SQ";
+
+	case MLX5_CMD_OP_CREATE_RQ:
+		return "CREATE_RQ";
+
+	case MLX5_CMD_OP_MODIFY_RQ:
+		return "MODIFY_RQ";
+
+	case MLX5_CMD_OP_DESTROY_RQ:
+		return "DESTROY_RQ";
+
+	case MLX5_CMD_OP_QUERY_RQ:
+		return "QUERY_RQ";
+
+	case MLX5_CMD_OP_CREATE_TIR:
+		return "CREATE_TIR";
+
+	case MLX5_CMD_OP_MODIFY_TIR:
+		return "MODIFY_TIR";
+
+	case MLX5_CMD_OP_DESTROY_TIR:
+		return "DESTROY_TIR";
+
+	case MLX5_CMD_OP_QUERY_TIR:
+		return "QUERY_TIR";
+
+	case MLX5_CMD_OP_CREATE_TIS:
+		return "CREATE_TIS";
+
+	case MLX5_CMD_OP_MODIFY_TIS:
+		return "MODIFY_TIS";
+
+	case MLX5_CMD_OP_DESTROY_TIS:
+		return "DESTROY_TIS";
+
+	case MLX5_CMD_OP_QUERY_TIS:
+		return "QUERY_TIS";
+
+	case MLX5_CMD_OP_CREATE_FLOW_TABLE:
+		return "CREATE_FLOW_TABLE";
+
+	case MLX5_CMD_OP_DESTROY_FLOW_TABLE:
+		return "DESTROY_FLOW_TABLE";
+
+	case MLX5_CMD_OP_QUERY_FLOW_TABLE:
+		return "QUERY_FLOW_TABLE";
+
+	case MLX5_CMD_OP_CREATE_FLOW_GROUP:
+		return "CREATE_FLOW_GROUP";
+
+	case MLX5_CMD_OP_DESTROY_FLOW_GROUP:
+		return "DESTROY_FLOW_GROUP";
+
+	case MLX5_CMD_OP_QUERY_FLOW_GROUP:
+		return "QUERY_FLOW_GROUP";
+
+	case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
+		return "SET_FLOW_TABLE_ENTRY";
+
+	case MLX5_CMD_OP_QUERY_FLOW_TABLE_ENTRY:
+		return "QUERY_FLOW_TABLE_ENTRY";
+
+	case MLX5_CMD_OP_DELETE_FLOW_TABLE_ENTRY:
+		return "DELETE_FLOW_TABLE_ENTRY";
+
+	default: return "unknown command opcode";
+	}
+}
+
+static void dump_command(struct mlx5_core_dev *dev,
+			 struct mlx5_cmd_work_ent *ent, int input)
+{
+	u16 op = be16_to_cpu(((struct mlx5_inbox_hdr *)(ent->lay->in))->opcode);
+	struct mlx5_cmd_msg *msg = input ? ent->in : ent->out;
+	struct mlx5_cmd_mailbox *next = msg->next;
+	int data_only;
+	u32 offset = 0;
+	int dump_len;
+
+	data_only = !!(mlx5_core_debug_mask & (1 << MLX5_CMD_DATA));
+
+	if (data_only)
+		mlx5_core_dbg_mask(dev, 1 << MLX5_CMD_DATA,
+				   "dump command data %s(0x%x) %s\n",
+				   mlx5_command_str(op), op,
+				   input ? "INPUT" : "OUTPUT");
+	else
+		mlx5_core_dbg(dev, "dump command %s(0x%x) %s\n",
+			      mlx5_command_str(op), op,
+			      input ? "INPUT" : "OUTPUT");
+
+	if (data_only) {
+		if (input) {
+			dump_buf(ent->lay->in, sizeof(ent->lay->in), 1, offset);
+			offset += sizeof(ent->lay->in);
+		} else {
+			dump_buf(ent->lay->out, sizeof(ent->lay->out), 1, offset);
+			offset += sizeof(ent->lay->out);
+		}
+	} else {
+		dump_buf(ent->lay, sizeof(*ent->lay), 0, offset);
+		offset += sizeof(*ent->lay);
+	}
+
+	while (next && offset < msg->len) {
+		if (data_only) {
+			dump_len = min_t(int, MLX5_CMD_DATA_BLOCK_SIZE, msg->len - offset);
+			dump_buf(next->buf, dump_len, 1, offset);
+			offset += MLX5_CMD_DATA_BLOCK_SIZE;
+		} else {
+			mlx5_core_dbg(dev, "command block:\n");
+			dump_buf(next->buf, sizeof(struct mlx5_cmd_prot_block), 0, offset);
+			offset += sizeof(struct mlx5_cmd_prot_block);
+		}
+		next = next->next;
+	}
+
+	if (data_only)
+		pr_debug("\n");
+}
+
+static void cmd_work_handler(struct work_struct *work)
+{
+	struct mlx5_cmd_work_ent *ent = container_of(work, struct mlx5_cmd_work_ent, work);
+	struct mlx5_cmd *cmd = ent->cmd;
+	struct mlx5_core_dev *dev = container_of(cmd, struct mlx5_core_dev, cmd);
+	struct mlx5_cmd_layout *lay;
+	struct semaphore *sem;
+
+	sem = ent->page_queue ? &cmd->pages_sem : &cmd->sem;
+	if (cmd->moving_to_polling) {
+		mlx5_core_warn(dev, "not expecting command execution, ignoring...\n");
+		return;
+	}
+
+	down(sem);
+	if (!ent->page_queue) {
+		ent->idx = alloc_ent(cmd);
+		if (ent->idx < 0) {
+			mlx5_core_err(dev, "failed to allocate command entry\n");
+			up(sem);
+			return;
+		}
+	} else {
+		ent->idx = cmd->max_reg_cmds;
+	}
+
+	ent->token = alloc_token(cmd);
+	cmd->ent_arr[ent->idx] = ent;
+	lay = get_inst(cmd, ent->idx);
+	ent->lay = lay;
+	memset(lay, 0, sizeof(*lay));
+	memcpy(lay->in, ent->in->first.data, sizeof(lay->in));
+	ent->op = be32_to_cpu(lay->in[0]) >> 16;
+	if (ent->in->next)
+		lay->in_ptr = cpu_to_be64(ent->in->next->dma);
+	lay->inlen = cpu_to_be32(ent->in->len);
+	if (ent->out->next)
+		lay->out_ptr = cpu_to_be64(ent->out->next->dma);
+	lay->outlen = cpu_to_be32(ent->out->len);
+	lay->type = MLX5_PCI_CMD_XPORT;
+	lay->token = ent->token;
+	lay->status_own = CMD_OWNER_HW;
+	set_signature(ent, !cmd->checksum_disabled);
+	dump_command(dev, ent, 1);
+	ent->ts1 = ktime_get_ns();
+
+	/* ring doorbell after the descriptor is valid */
+	mlx5_core_dbg(dev, "writing 0x%x to command doorbell\n", 1 << ent->idx);
+	wmb();
+	iowrite32be(1 << ent->idx, &dev->iseg->cmd_dbell);
+	mmiowb();
+	/* if not in polling don't use ent after this point*/
+	if (cmd->mode == CMD_MODE_POLLING) {
+		poll_timeout(ent);
+		/* make sure we read the descriptor after ownership is SW */
+		rmb();
+		mlx5_cmd_comp_handler(dev, 1UL << ent->idx);
+	}
+}
+
+static const char *deliv_status_to_str(u8 status)
+{
+	switch (status) {
+	case MLX5_CMD_DELIVERY_STAT_OK:
+		return "no errors";
+	case MLX5_CMD_DELIVERY_STAT_SIGNAT_ERR:
+		return "signature error";
+	case MLX5_CMD_DELIVERY_STAT_TOK_ERR:
+		return "token error";
+	case MLX5_CMD_DELIVERY_STAT_BAD_BLK_NUM_ERR:
+		return "bad block number";
+	case MLX5_CMD_DELIVERY_STAT_OUT_PTR_ALIGN_ERR:
+		return "output pointer not aligned to block size";
+	case MLX5_CMD_DELIVERY_STAT_IN_PTR_ALIGN_ERR:
+		return "input pointer not aligned to block size";
+	case MLX5_CMD_DELIVERY_STAT_FW_ERR:
+		return "firmware internal error";
+	case MLX5_CMD_DELIVERY_STAT_IN_LENGTH_ERR:
+		return "command input length error";
+	case MLX5_CMD_DELIVERY_STAT_OUT_LENGTH_ERR:
+		return "command ouput length error";
+	case MLX5_CMD_DELIVERY_STAT_RES_FLD_NOT_CLR_ERR:
+		return "reserved fields not cleared";
+	case MLX5_CMD_DELIVERY_STAT_CMD_DESCR_ERR:
+		return "bad command descriptor type";
+	default:
+		return "unknown status code";
+	}
+}
+
+static u16 msg_to_opcode(struct mlx5_cmd_msg *in)
+{
+	struct mlx5_inbox_hdr *hdr = (struct mlx5_inbox_hdr *)(in->first.data);
+
+	return be16_to_cpu(hdr->opcode);
+}
+
+static int wait_func(struct mlx5_core_dev *dev, struct mlx5_cmd_work_ent *ent)
+{
+	int timeout = msecs_to_jiffies(MLX5_CMD_TIMEOUT_MSEC);
+	struct mlx5_cmd *cmd = &dev->cmd;
+	int err;
+
+	if (cmd->mode == CMD_MODE_POLLING) {
+		wait_for_completion(&ent->done);
+		err = ent->ret;
+	} else {
+		if (!wait_for_completion_timeout(&ent->done, timeout))
+			err = -ETIMEDOUT;
+		else
+			err = 0;
+	}
+	if (err == -ETIMEDOUT) {
+		mlx5_core_warn(dev, "%s(0x%x) timeout. Will cause a leak of a command resource\n",
+			       mlx5_command_str(msg_to_opcode(ent->in)),
+			       msg_to_opcode(ent->in));
+	}
+	mlx5_core_dbg(dev, "err %d, delivery status %s(%d)\n",
+		      err, deliv_status_to_str(ent->status), ent->status);
+
+	return err;
+}
+
+/*  Notes:
+ *    1. Callback functions may not sleep
+ *    2. page queue commands do not support asynchrous completion
+ */
+static int mlx5_cmd_invoke(struct mlx5_core_dev *dev, struct mlx5_cmd_msg *in,
+			   struct mlx5_cmd_msg *out, void *uout, int uout_size,
+			   mlx5_cmd_cbk_t callback,
+			   void *context, int page_queue, u8 *status)
+{
+	struct mlx5_cmd *cmd = &dev->cmd;
+	struct mlx5_cmd_work_ent *ent;
+	struct mlx5_cmd_stats *stats;
+	int err = 0;
+	s64 ds;
+	u16 op;
+
+	if (callback && page_queue)
+		return -EINVAL;
+
+	ent = alloc_cmd(cmd, in, out, uout, uout_size, callback, context,
+			page_queue);
+	if (IS_ERR(ent))
+		return PTR_ERR(ent);
+
+	if (!callback)
+		init_completion(&ent->done);
+
+	INIT_WORK(&ent->work, cmd_work_handler);
+	if (page_queue) {
+		cmd_work_handler(&ent->work);
+	} else if (!queue_work(cmd->wq, &ent->work)) {
+		mlx5_core_warn(dev, "failed to queue work\n");
+		err = -ENOMEM;
+		goto out_free;
+	}
+
+	if (!callback) {
+		err = wait_func(dev, ent);
+		if (err == -ETIMEDOUT)
+			goto out;
+
+		ds = ent->ts2 - ent->ts1;
+		op = be16_to_cpu(((struct mlx5_inbox_hdr *)in->first.data)->opcode);
+		if (op < ARRAY_SIZE(cmd->stats)) {
+			stats = &cmd->stats[op];
+			spin_lock_irq(&stats->lock);
+			stats->sum += ds;
+			++stats->n;
+			spin_unlock_irq(&stats->lock);
+		}
+		mlx5_core_dbg_mask(dev, 1 << MLX5_CMD_TIME,
+				   "fw exec time for %s is %lld nsec\n",
+				   mlx5_command_str(op), (long long)ds);
+		*status = ent->status;
+		free_cmd(ent);
+	}
+
+	return err;
+
+out_free:
+	free_cmd(ent);
+out:
+	return err;
+}
+
+static int mlx5_copy_to_msg(struct mlx5_cmd_msg *to, void *from, int size)
+{
+	struct mlx5_cmd_prot_block *block;
+	struct mlx5_cmd_mailbox *next;
+	int copy;
+
+	if (!to || !from)
+		return -ENOMEM;
+
+	copy = min_t(int, size, sizeof(to->first.data));
+	memcpy(to->first.data, from, copy);
+	size -= copy;
+	from += copy;
+
+	next = to->next;
+	while (size) {
+		if (!next) {
+			/* this is a BUG */
+			return -ENOMEM;
+		}
+
+		copy = min_t(int, size, MLX5_CMD_DATA_BLOCK_SIZE);
+		block = next->buf;
+		memcpy(block->data, from, copy);
+		from += copy;
+		size -= copy;
+		next = next->next;
+	}
+
+	return 0;
+}
+
+static int mlx5_copy_from_msg(void *to, struct mlx5_cmd_msg *from, int size)
+{
+	struct mlx5_cmd_prot_block *block;
+	struct mlx5_cmd_mailbox *next;
+	int copy;
+
+	if (!to || !from)
+		return -ENOMEM;
+
+	copy = min_t(int, size, sizeof(from->first.data));
+	memcpy(to, from->first.data, copy);
+	size -= copy;
+	to += copy;
+
+	next = from->next;
+	while (size) {
+		if (!next) {
+			/* this is a BUG */
+			return -ENOMEM;
+		}
+
+		copy = min_t(int, size, MLX5_CMD_DATA_BLOCK_SIZE);
+		block = next->buf;
+
+		memcpy(to, block->data, copy);
+		to += copy;
+		size -= copy;
+		next = next->next;
+	}
+
+	return 0;
+}
+
+static struct mlx5_cmd_mailbox *alloc_cmd_box(struct mlx5_core_dev *dev,
+					      gfp_t flags)
+{
+	struct mlx5_cmd_mailbox *mailbox;
+
+	mailbox = kmalloc(sizeof(*mailbox), flags);
+	if (!mailbox)
+		return ERR_PTR(-ENOMEM);
+
+	mailbox->buf = pci_pool_alloc(dev->cmd.pool, flags,
+				      &mailbox->dma);
+	if (!mailbox->buf) {
+		mlx5_core_dbg(dev, "failed allocation\n");
+		kfree(mailbox);
+		return ERR_PTR(-ENOMEM);
+	}
+	memset(mailbox->buf, 0, sizeof(struct mlx5_cmd_prot_block));
+	mailbox->next = NULL;
+
+	return mailbox;
+}
+
+static void free_cmd_box(struct mlx5_core_dev *dev,
+			 struct mlx5_cmd_mailbox *mailbox)
+{
+	pci_pool_free(dev->cmd.pool, mailbox->buf, mailbox->dma);
+	kfree(mailbox);
+}
+
+static struct mlx5_cmd_msg *mlx5_alloc_cmd_msg(struct mlx5_core_dev *dev,
+					       gfp_t flags, int size)
+{
+	struct mlx5_cmd_mailbox *tmp, *head = NULL;
+	struct mlx5_cmd_prot_block *block;
+	struct mlx5_cmd_msg *msg;
+	int blen;
+	int err;
+	int n;
+	int i;
+
+	msg = kzalloc(sizeof(*msg), flags);
+	if (!msg)
+		return ERR_PTR(-ENOMEM);
+
+	blen = size - min_t(int, sizeof(msg->first.data), size);
+	n = (blen + MLX5_CMD_DATA_BLOCK_SIZE - 1) / MLX5_CMD_DATA_BLOCK_SIZE;
+
+	for (i = 0; i < n; i++) {
+		tmp = alloc_cmd_box(dev, flags);
+		if (IS_ERR(tmp)) {
+			mlx5_core_warn(dev, "failed allocating block\n");
+			err = PTR_ERR(tmp);
+			goto err_alloc;
+		}
+
+		block = tmp->buf;
+		tmp->next = head;
+		block->next = cpu_to_be64(tmp->next ? tmp->next->dma : 0);
+		block->block_num = cpu_to_be32(n - i - 1);
+		head = tmp;
+	}
+	msg->next = head;
+	msg->len = size;
+	return msg;
+
+err_alloc:
+	while (head) {
+		tmp = head->next;
+		free_cmd_box(dev, head);
+		head = tmp;
+	}
+	kfree(msg);
+
+	return ERR_PTR(err);
+}
+
+static void mlx5_free_cmd_msg(struct mlx5_core_dev *dev,
+				  struct mlx5_cmd_msg *msg)
+{
+	struct mlx5_cmd_mailbox *head = msg->next;
+	struct mlx5_cmd_mailbox *next;
+
+	while (head) {
+		next = head->next;
+		free_cmd_box(dev, head);
+		head = next;
+	}
+	kfree(msg);
+}
+
+static void set_wqname(struct mlx5_core_dev *dev)
+{
+	struct mlx5_cmd *cmd = &dev->cmd;
+
+	snprintf(cmd->wq_name, sizeof(cmd->wq_name), "mlx5_cmd_%s",
+		 dev_name(&dev->pdev->dev));
+}
+
+static void clean_debug_files(struct mlx5_core_dev *dev)
+{
+}
+
+
+void mlx5_cmd_use_events(struct mlx5_core_dev *dev)
+{
+	struct mlx5_cmd *cmd = &dev->cmd;
+	int i;
+
+	for (i = 0; i < cmd->max_reg_cmds; i++)
+		down(&cmd->sem);
+
+	down(&cmd->pages_sem);
+
+	flush_workqueue(cmd->wq);
+
+	cmd->mode = CMD_MODE_EVENTS;
+
+	up(&cmd->pages_sem);
+	for (i = 0; i < cmd->max_reg_cmds; i++)
+		up(&cmd->sem);
+}
+
+void mlx5_cmd_use_polling(struct mlx5_core_dev *dev)
+{
+	struct mlx5_cmd *cmd = &dev->cmd;
+
+	synchronize_irq(dev->priv.eq_table.pages_eq.irqn);
+	flush_workqueue(dev->priv.pg_wq);
+	cmd->moving_to_polling = 1;
+	flush_workqueue(cmd->wq);
+	cmd->mode = CMD_MODE_POLLING;
+	cmd->moving_to_polling = 0;
+}
+
+static void free_msg(struct mlx5_core_dev *dev, struct mlx5_cmd_msg *msg)
+{
+	unsigned long flags;
+
+	if (msg->cache) {
+		spin_lock_irqsave(&msg->cache->lock, flags);
+		list_add_tail(&msg->list, &msg->cache->head);
+		spin_unlock_irqrestore(&msg->cache->lock, flags);
+	} else {
+		mlx5_free_cmd_msg(dev, msg);
+	}
+}
+
+void mlx5_cmd_comp_handler(struct mlx5_core_dev *dev, unsigned long vector)
+{
+	struct mlx5_cmd *cmd = &dev->cmd;
+	struct mlx5_cmd_work_ent *ent;
+	mlx5_cmd_cbk_t callback;
+	void *context;
+	int err;
+	int i;
+	s64 ds;
+	struct mlx5_cmd_stats *stats;
+	unsigned long flags;
+
+	for (i = 0; i < (1 << cmd->log_sz); i++) {
+		if (test_bit(i, &vector)) {
+			struct semaphore *sem;
+
+			ent = cmd->ent_arr[i];
+			if (ent->page_queue)
+				sem = &cmd->pages_sem;
+			else
+				sem = &cmd->sem;
+			ent->ts2 = ktime_get_ns();
+			memcpy(ent->out->first.data, ent->lay->out, sizeof(ent->lay->out));
+			dump_command(dev, ent, 0);
+			if (!ent->ret) {
+				if (!cmd->checksum_disabled)
+					ent->ret = verify_signature(ent);
+				else
+					ent->ret = 0;
+				ent->status = ent->lay->status_own >> 1;
+				mlx5_core_dbg(dev, "command completed. ret 0x%x, delivery status %s(0x%x)\n",
+					      ent->ret, deliv_status_to_str(ent->status), ent->status);
+			}
+			free_ent(cmd, ent->idx);
+			if (ent->callback) {
+				ds = ent->ts2 - ent->ts1;
+				if (ent->op < ARRAY_SIZE(cmd->stats)) {
+					stats = &cmd->stats[ent->op];
+					spin_lock_irqsave(&stats->lock, flags);
+					stats->sum += ds;
+					++stats->n;
+					spin_unlock_irqrestore(&stats->lock, flags);
+				}
+
+				callback = ent->callback;
+				context = ent->context;
+				err = ent->ret;
+				if (!err)
+					err = mlx5_copy_from_msg(ent->uout,
+								 ent->out,
+								 ent->uout_size);
+
+				mlx5_free_cmd_msg(dev, ent->out);
+				free_msg(dev, ent->in);
+
+				free_cmd(ent);
+				callback(err, context);
+			} else {
+				complete(&ent->done);
+			}
+			up(sem);
+		}
+	}
+}
+EXPORT_SYMBOL(mlx5_cmd_comp_handler);
+
+static int status_to_err(u8 status)
+{
+	return status ? -1 : 0; /* TBD more meaningful codes */
+}
+
+static struct mlx5_cmd_msg *alloc_msg(struct mlx5_core_dev *dev, int in_size,
+				      gfp_t gfp)
+{
+	struct mlx5_cmd_msg *msg = ERR_PTR(-ENOMEM);
+	struct mlx5_cmd *cmd = &dev->cmd;
+	struct cache_ent *ent = NULL;
+
+	if (in_size > MED_LIST_SIZE && in_size <= LONG_LIST_SIZE)
+		ent = &cmd->cache.large;
+	else if (in_size > 16 && in_size <= MED_LIST_SIZE)
+		ent = &cmd->cache.med;
+
+	if (ent) {
+		spin_lock_irq(&ent->lock);
+		if (!list_empty(&ent->head)) {
+			msg = list_entry(ent->head.next, struct mlx5_cmd_msg,
+					 list);
+			/* For cached lists, we must explicitly state what is
+			 * the real size
+			 */
+			msg->len = in_size;
+			list_del(&msg->list);
+		}
+		spin_unlock_irq(&ent->lock);
+	}
+
+	if (IS_ERR(msg))
+		msg = mlx5_alloc_cmd_msg(dev, gfp, in_size);
+
+	return msg;
+}
+
+static int is_manage_pages(struct mlx5_inbox_hdr *in)
+{
+	return be16_to_cpu(in->opcode) == MLX5_CMD_OP_MANAGE_PAGES;
+}
+
+static int cmd_exec_helper(struct mlx5_core_dev *dev, void *in, int in_size, void *out,
+		    int out_size, mlx5_cmd_cbk_t callback, void *context)
+{
+	struct mlx5_cmd_msg *inb;
+	struct mlx5_cmd_msg *outb;
+	int pages_queue;
+	gfp_t gfp;
+	int err;
+	u8 status = 0;
+
+	pages_queue = is_manage_pages(in);
+	gfp = callback ? GFP_ATOMIC : GFP_KERNEL;
+
+	inb = alloc_msg(dev, in_size, gfp);
+	if (IS_ERR(inb)) {
+		err = PTR_ERR(inb);
+		return err;
+	}
+
+	err = mlx5_copy_to_msg(inb, in, in_size);
+	if (err) {
+		mlx5_core_warn(dev, "err %d\n", err);
+		goto out_in;
+	}
+
+	outb = mlx5_alloc_cmd_msg(dev, gfp, out_size);
+	if (IS_ERR(outb)) {
+		err = PTR_ERR(outb);
+		goto out_in;
+	}
+
+	err = mlx5_cmd_invoke(dev, inb, outb, out, out_size, callback, context,
+			      pages_queue, &status);
+	if (err)
+		goto out_out;
+
+	mlx5_core_dbg(dev, "err %d, status %d\n", err, status);
+	if (status) {
+		err = status_to_err(status);
+		goto out_out;
+	}
+
+	if (callback)
+		return err;
+
+	err = mlx5_copy_from_msg(out, outb, out_size);
+
+out_out:
+	mlx5_free_cmd_msg(dev, outb);
+
+out_in:
+	free_msg(dev, inb);
+	return err;
+}
+
+int mlx5_cmd_exec(struct mlx5_core_dev *dev, void *in, int in_size, void *out,
+		  int out_size)
+{
+	return cmd_exec_helper(dev, in, in_size, out, out_size, NULL, NULL);
+}
+EXPORT_SYMBOL(mlx5_cmd_exec);
+
+int mlx5_cmd_exec_cb(struct mlx5_core_dev *dev, void *in, int in_size,
+		     void *out, int out_size, mlx5_cmd_cbk_t callback,
+		     void *context)
+{
+	return cmd_exec_helper(dev, in, in_size, out, out_size, callback, context);
+}
+EXPORT_SYMBOL(mlx5_cmd_exec_cb);
+
+static void destroy_msg_cache(struct mlx5_core_dev *dev)
+{
+	struct mlx5_cmd *cmd = &dev->cmd;
+	struct mlx5_cmd_msg *msg;
+	struct mlx5_cmd_msg *n;
+
+	list_for_each_entry_safe(msg, n, &cmd->cache.large.head, list) {
+		list_del(&msg->list);
+		mlx5_free_cmd_msg(dev, msg);
+	}
+
+	list_for_each_entry_safe(msg, n, &cmd->cache.med.head, list) {
+		list_del(&msg->list);
+		mlx5_free_cmd_msg(dev, msg);
+	}
+}
+
+static int create_msg_cache(struct mlx5_core_dev *dev)
+{
+	struct mlx5_cmd *cmd = &dev->cmd;
+	struct mlx5_cmd_msg *msg;
+	int err;
+	int i;
+
+	spin_lock_init(&cmd->cache.large.lock);
+	INIT_LIST_HEAD(&cmd->cache.large.head);
+	spin_lock_init(&cmd->cache.med.lock);
+	INIT_LIST_HEAD(&cmd->cache.med.head);
+
+	for (i = 0; i < NUM_LONG_LISTS; i++) {
+		msg = mlx5_alloc_cmd_msg(dev, GFP_KERNEL, LONG_LIST_SIZE);
+		if (IS_ERR(msg)) {
+			err = PTR_ERR(msg);
+			goto ex_err;
+		}
+		msg->cache = &cmd->cache.large;
+		list_add_tail(&msg->list, &cmd->cache.large.head);
+	}
+
+	for (i = 0; i < NUM_MED_LISTS; i++) {
+		msg = mlx5_alloc_cmd_msg(dev, GFP_KERNEL, MED_LIST_SIZE);
+		if (IS_ERR(msg)) {
+			err = PTR_ERR(msg);
+			goto ex_err;
+		}
+		msg->cache = &cmd->cache.med;
+		list_add_tail(&msg->list, &cmd->cache.med.head);
+	}
+
+	return 0;
+
+ex_err:
+	destroy_msg_cache(dev);
+	return err;
+}
+
+static int alloc_cmd_page(struct mlx5_core_dev *dev, struct mlx5_cmd *cmd)
+{
+	struct device *ddev = &dev->pdev->dev;
+	cmd->cmd_alloc_buf = dma_zalloc_coherent(ddev, MLX5_ADAPTER_PAGE_SIZE,
+						 &cmd->alloc_dma, GFP_KERNEL);
+	if (!cmd->cmd_alloc_buf)
+		return -ENOMEM;
+
+	/* make sure it is aligned to 4K */
+	if (!((uintptr_t)cmd->cmd_alloc_buf & (MLX5_ADAPTER_PAGE_SIZE - 1))) {
+		cmd->cmd_buf = cmd->cmd_alloc_buf;
+		cmd->dma = cmd->alloc_dma;
+		cmd->alloc_size = MLX5_ADAPTER_PAGE_SIZE;
+		return 0;
+	}
+
+	dma_free_coherent(ddev, MLX5_ADAPTER_PAGE_SIZE, cmd->cmd_alloc_buf, cmd->alloc_dma);
+	cmd->cmd_alloc_buf = dma_zalloc_coherent(ddev, 2 * MLX5_ADAPTER_PAGE_SIZE - 1,
+						 &cmd->alloc_dma, GFP_KERNEL);
+	if (!cmd->cmd_alloc_buf)
+		return -ENOMEM;
+
+	cmd->cmd_buf = PTR_ALIGN(cmd->cmd_alloc_buf, MLX5_ADAPTER_PAGE_SIZE);
+	cmd->dma = ALIGN(cmd->alloc_dma, MLX5_ADAPTER_PAGE_SIZE);
+	cmd->alloc_size = 2 * MLX5_ADAPTER_PAGE_SIZE - 1;
+	return 0;
+}
+
+static void free_cmd_page(struct mlx5_core_dev *dev, struct mlx5_cmd *cmd)
+{
+	struct device *ddev = &dev->pdev->dev;
+	dma_free_coherent(ddev, cmd->alloc_size, cmd->cmd_alloc_buf, cmd->alloc_dma);
+}
+
+int mlx5_cmd_init(struct mlx5_core_dev *dev)
+{
+	int size = sizeof(struct mlx5_cmd_prot_block);
+	int align = roundup_pow_of_two(size);
+	struct mlx5_cmd *cmd = &dev->cmd;
+	u32 cmd_h, cmd_l;
+	u16 cmd_if_rev;
+	int err;
+	int i;
+
+	cmd_if_rev = cmdif_rev_get(dev);
+	if (cmd_if_rev != CMD_IF_REV) {
+		device_printf((&dev->pdev->dev)->bsddev, "ERR: ""Driver cmdif rev(%d) differs from firmware's(%d)\n", CMD_IF_REV, cmd_if_rev);
+		return -EINVAL;
+	}
+
+	cmd->pool = pci_pool_create("mlx5_cmd", dev->pdev, size, align, 0);
+	if (!cmd->pool)
+		return -ENOMEM;
+
+	err = alloc_cmd_page(dev, cmd);
+	if (err)
+		goto err_free_pool;
+
+	cmd_l = ioread32be(&dev->iseg->cmdq_addr_l_sz) & 0xff;
+	cmd->log_sz = cmd_l >> 4 & 0xf;
+	cmd->log_stride = cmd_l & 0xf;
+	if (1 << cmd->log_sz > MLX5_MAX_COMMANDS) {
+		device_printf((&dev->pdev->dev)->bsddev, "ERR: ""firmware reports too many outstanding commands %d\n", 1 << cmd->log_sz);
+		err = -EINVAL;
+		goto err_free_page;
+	}
+
+	if (cmd->log_sz + cmd->log_stride > MLX5_ADAPTER_PAGE_SHIFT) {
+		device_printf((&dev->pdev->dev)->bsddev, "ERR: ""command queue size overflow\n");
+		err = -EINVAL;
+		goto err_free_page;
+	}
+
+	cmd->checksum_disabled = 1;
+	cmd->max_reg_cmds = (1 << cmd->log_sz) - 1;
+	cmd->bitmask = (1 << cmd->max_reg_cmds) - 1;
+
+	cmd->cmdif_rev = ioread32be(&dev->iseg->cmdif_rev_fw_sub) >> 16;
+	if (cmd->cmdif_rev > CMD_IF_REV) {
+		device_printf((&dev->pdev->dev)->bsddev, "ERR: ""driver does not support command interface version. driver %d, firmware %d\n", CMD_IF_REV, cmd->cmdif_rev);
+		err = -ENOTSUPP;
+		goto err_free_page;
+	}
+
+	spin_lock_init(&cmd->alloc_lock);
+	spin_lock_init(&cmd->token_lock);
+	for (i = 0; i < ARRAY_SIZE(cmd->stats); i++)
+		spin_lock_init(&cmd->stats[i].lock);
+
+	sema_init(&cmd->sem, cmd->max_reg_cmds);
+	sema_init(&cmd->pages_sem, 1);
+
+	cmd_h = (u32)((u64)(cmd->dma) >> 32);
+	cmd_l = (u32)(cmd->dma);
+	if (cmd_l & 0xfff) {
+		device_printf((&dev->pdev->dev)->bsddev, "ERR: ""invalid command queue address\n");
+		err = -ENOMEM;
+		goto err_free_page;
+	}
+
+	iowrite32be(cmd_h, &dev->iseg->cmdq_addr_h);
+	iowrite32be(cmd_l, &dev->iseg->cmdq_addr_l_sz);
+
+	/* Make sure firmware sees the complete address before we proceed */
+	wmb();
+
+	mlx5_core_dbg(dev, "descriptor at dma 0x%llx\n", (unsigned long long)(cmd->dma));
+
+	cmd->mode = CMD_MODE_POLLING;
+
+	err = create_msg_cache(dev);
+	if (err) {
+		device_printf((&dev->pdev->dev)->bsddev, "ERR: ""failed to create command cache\n");
+		goto err_free_page;
+	}
+
+	set_wqname(dev);
+	cmd->wq = create_singlethread_workqueue(cmd->wq_name);
+	if (!cmd->wq) {
+		device_printf((&dev->pdev->dev)->bsddev, "ERR: ""failed to create command workqueue\n");
+		err = -ENOMEM;
+		goto err_cache;
+	}
+
+	return 0;
+
+err_cache:
+	destroy_msg_cache(dev);
+
+err_free_page:
+	free_cmd_page(dev, cmd);
+
+err_free_pool:
+	pci_pool_destroy(cmd->pool);
+
+	return err;
+}
+EXPORT_SYMBOL(mlx5_cmd_init);
+
+void mlx5_cmd_cleanup(struct mlx5_core_dev *dev)
+{
+	struct mlx5_cmd *cmd = &dev->cmd;
+
+	clean_debug_files(dev);
+	destroy_workqueue(cmd->wq);
+	destroy_msg_cache(dev);
+	free_cmd_page(dev, cmd);
+	pci_pool_destroy(cmd->pool);
+}
+EXPORT_SYMBOL(mlx5_cmd_cleanup);
+
+static const char *cmd_status_str(u8 status)
+{
+	switch (status) {
+	case MLX5_CMD_STAT_OK:
+		return "OK";
+	case MLX5_CMD_STAT_INT_ERR:
+		return "internal error";
+	case MLX5_CMD_STAT_BAD_OP_ERR:
+		return "bad operation";
+	case MLX5_CMD_STAT_BAD_PARAM_ERR:
+		return "bad parameter";
+	case MLX5_CMD_STAT_BAD_SYS_STATE_ERR:
+		return "bad system state";
+	case MLX5_CMD_STAT_BAD_RES_ERR:
+		return "bad resource";
+	case MLX5_CMD_STAT_RES_BUSY:
+		return "resource busy";
+	case MLX5_CMD_STAT_LIM_ERR:
+		return "limits exceeded";
+	case MLX5_CMD_STAT_BAD_RES_STATE_ERR:
+		return "bad resource state";
+	case MLX5_CMD_STAT_IX_ERR:
+		return "bad index";
+	case MLX5_CMD_STAT_NO_RES_ERR:
+		return "no resources";
+	case MLX5_CMD_STAT_BAD_INP_LEN_ERR:
+		return "bad input length";
+	case MLX5_CMD_STAT_BAD_OUTP_LEN_ERR:
+		return "bad output length";
+	case MLX5_CMD_STAT_BAD_QP_STATE_ERR:
+		return "bad QP state";
+	case MLX5_CMD_STAT_BAD_PKT_ERR:
+		return "bad packet (discarded)";
+	case MLX5_CMD_STAT_BAD_SIZE_OUTS_CQES_ERR:
+		return "bad size too many outstanding CQEs";
+	default:
+		return "unknown status";
+	}
+}
+
+static int cmd_status_to_err_helper(u8 status)
+{
+	switch (status) {
+	case MLX5_CMD_STAT_OK:				return 0;
+	case MLX5_CMD_STAT_INT_ERR:			return -EIO;
+	case MLX5_CMD_STAT_BAD_OP_ERR:			return -EINVAL;
+	case MLX5_CMD_STAT_BAD_PARAM_ERR:		return -EINVAL;
+	case MLX5_CMD_STAT_BAD_SYS_STATE_ERR:		return -EIO;
+	case MLX5_CMD_STAT_BAD_RES_ERR:			return -EINVAL;
+	case MLX5_CMD_STAT_RES_BUSY:			return -EBUSY;
+	case MLX5_CMD_STAT_LIM_ERR:			return -ENOMEM;
+	case MLX5_CMD_STAT_BAD_RES_STATE_ERR:		return -EINVAL;
+	case MLX5_CMD_STAT_IX_ERR:			return -EINVAL;
+	case MLX5_CMD_STAT_NO_RES_ERR:			return -EAGAIN;
+	case MLX5_CMD_STAT_BAD_INP_LEN_ERR:		return -EIO;
+	case MLX5_CMD_STAT_BAD_OUTP_LEN_ERR:		return -EIO;
+	case MLX5_CMD_STAT_BAD_QP_STATE_ERR:		return -EINVAL;
+	case MLX5_CMD_STAT_BAD_PKT_ERR:			return -EINVAL;
+	case MLX5_CMD_STAT_BAD_SIZE_OUTS_CQES_ERR:	return -EINVAL;
+	default:					return -EIO;
+	}
+}
+
+/* this will be available till all the commands use set/get macros */
+int mlx5_cmd_status_to_err(struct mlx5_outbox_hdr *hdr)
+{
+	if (!hdr->status)
+		return 0;
+
+	printf("mlx5_core: WARN: ""command failed, status %s(0x%x), syndrome 0x%x\n", cmd_status_str(hdr->status), hdr->status, be32_to_cpu(hdr->syndrome));
+
+	return cmd_status_to_err_helper(hdr->status);
+}
+
+int mlx5_cmd_status_to_err_v2(void *ptr)
+{
+	u32	syndrome;
+	u8	status;
+
+	status = be32_to_cpu(*(__be32 *)ptr) >> 24;
+	if (!status)
+		return 0;
+
+	syndrome = be32_to_cpu(*(__be32 *)(ptr + 4));
+
+	printf("mlx5_core: WARN: ""command failed, status %s(0x%x), syndrome 0x%x\n", cmd_status_str(status), status, syndrome);
+
+	return cmd_status_to_err_helper(status);
+}
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_core.h b/sys/dev/mlx5/mlx5_core/mlx5_core.h
new file mode 100644
index 0000000..bac7fc0
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_core/mlx5_core.h
@@ -0,0 +1,92 @@
+/*-
+ * Copyright (c) 2013-2015, Mellanox Technologies, Ltd.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef __MLX5_CORE_H__
+#define __MLX5_CORE_H__
+
+#include <linux/types.h>
+#include <linux/kernel.h>
+#include <linux/sched.h>
+
+#define DRIVER_NAME "mlx5_core"
+#define DRIVER_VERSION "1.23.0 (03 Mar 2015)"
+#define DRIVER_RELDATE "03 Mar 2015"
+
+extern int mlx5_core_debug_mask;
+
+#define mlx5_core_dbg(dev, format, ...)					\
+	pr_debug("%s:%s:%d:(pid %d): " format,				\
+		 (dev)->priv.name, __func__, __LINE__, curthread->td_proc->p_pid,	\
+		 ##__VA_ARGS__)
+
+#define mlx5_core_dbg_mask(dev, mask, format, ...)			\
+do {									\
+	if ((mask) & mlx5_core_debug_mask)				\
+		mlx5_core_dbg(dev, format, ##__VA_ARGS__);		\
+} while (0)
+
+#define mlx5_core_err(dev, format, ...)					\
+	printf("mlx5_core: ERR: ""%s:%s:%d:(pid %d): " format, \
+	       (dev)->priv.name, __func__, __LINE__, curthread->td_proc->p_pid, \
+	       ##__VA_ARGS__)
+
+#define mlx5_core_warn(dev, format, ...)				\
+	printf("mlx5_core: WARN: ""%s:%s:%d:(pid %d): " format, \
+		(dev)->priv.name, __func__, __LINE__, curthread->td_proc->p_pid, \
+		##__VA_ARGS__)
+
+enum {
+	MLX5_CMD_DATA, /* print command payload only */
+	MLX5_CMD_TIME, /* print command execution time */
+};
+
+int mlx5_query_hca_caps(struct mlx5_core_dev *dev);
+int mlx5_query_board_id(struct mlx5_core_dev *dev);
+int mlx5_cmd_init_hca(struct mlx5_core_dev *dev);
+int mlx5_cmd_teardown_hca(struct mlx5_core_dev *dev);
+
+void mlx5e_init(void);
+void mlx5e_cleanup(void);
+
+static inline int mlx5_cmd_exec_check_status(struct mlx5_core_dev *dev, u32 *in,
+						int in_size, u32 *out,
+						int out_size)
+{
+	int err;
+	err = mlx5_cmd_exec(dev, in, in_size, out, out_size);
+
+	if (err) {
+		return err;
+	}
+
+	err =  mlx5_cmd_status_to_err((struct mlx5_outbox_hdr *)out);
+	return err;
+}
+
+int mlx5_rename_eq(struct mlx5_core_dev *dev, int eq_ix, char *name);
+
+#endif /* __MLX5_CORE_H__ */
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_cq.c b/sys/dev/mlx5/mlx5_core/mlx5_cq.c
new file mode 100644
index 0000000..47d3780
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_core/mlx5_cq.c
@@ -0,0 +1,282 @@
+/*-
+ * Copyright (c) 2013-2015, Mellanox Technologies, Ltd.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/hardirq.h>
+#include <dev/mlx5/driver.h>
+#include <rdma/ib_verbs.h>
+#include <dev/mlx5/cq.h>
+#include "mlx5_core.h"
+
+void mlx5_cq_completion(struct mlx5_core_dev *dev, u32 cqn)
+{
+	struct mlx5_core_cq *cq;
+	struct mlx5_cq_table *table = &dev->priv.cq_table;
+
+	if (cqn < MLX5_CQ_LINEAR_ARRAY_SIZE) {
+		struct mlx5_cq_linear_array_entry *entry;
+
+		entry = &table->linear_array[cqn];
+		spin_lock(&entry->lock);
+		cq = entry->cq;
+		if (cq == NULL) {
+			mlx5_core_warn(dev,
+			    "Completion event for bogus CQ 0x%x\n", cqn);
+		} else {
+			++cq->arm_sn;
+			cq->comp(cq);
+		}
+		spin_unlock(&entry->lock);
+		return;
+	}
+
+	spin_lock(&table->lock);
+	cq = radix_tree_lookup(&table->tree, cqn);
+	if (likely(cq))
+		atomic_inc(&cq->refcount);
+	spin_unlock(&table->lock);
+
+	if (!cq) {
+		mlx5_core_warn(dev, "Completion event for bogus CQ 0x%x\n", cqn);
+		return;
+	}
+
+	++cq->arm_sn;
+
+	cq->comp(cq);
+
+	if (atomic_dec_and_test(&cq->refcount))
+		complete(&cq->free);
+}
+
+void mlx5_cq_event(struct mlx5_core_dev *dev, u32 cqn, int event_type)
+{
+	struct mlx5_cq_table *table = &dev->priv.cq_table;
+	struct mlx5_core_cq *cq;
+
+	spin_lock(&table->lock);
+
+	cq = radix_tree_lookup(&table->tree, cqn);
+	if (cq)
+		atomic_inc(&cq->refcount);
+
+	spin_unlock(&table->lock);
+
+	if (!cq) {
+		mlx5_core_warn(dev, "Async event for bogus CQ 0x%x\n", cqn);
+		return;
+	}
+
+	cq->event(cq, event_type);
+
+	if (atomic_dec_and_test(&cq->refcount))
+		complete(&cq->free);
+}
+
+
+int mlx5_core_create_cq(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq,
+			struct mlx5_create_cq_mbox_in *in, int inlen)
+{
+	int err;
+	struct mlx5_cq_table *table = &dev->priv.cq_table;
+	struct mlx5_create_cq_mbox_out out;
+	struct mlx5_destroy_cq_mbox_in din;
+	struct mlx5_destroy_cq_mbox_out dout;
+
+	in->hdr.opcode = cpu_to_be16(MLX5_CMD_OP_CREATE_CQ);
+	memset(&out, 0, sizeof(out));
+	err = mlx5_cmd_exec(dev, in, inlen, &out, sizeof(out));
+	if (err)
+		return err;
+
+	if (out.hdr.status)
+		return mlx5_cmd_status_to_err(&out.hdr);
+
+	cq->cqn = be32_to_cpu(out.cqn) & 0xffffff;
+	cq->cons_index = 0;
+	cq->arm_sn     = 0;
+	atomic_set(&cq->refcount, 1);
+	init_completion(&cq->free);
+
+	spin_lock_irq(&table->lock);
+	err = radix_tree_insert(&table->tree, cq->cqn, cq);
+	spin_unlock_irq(&table->lock);
+	if (err)
+		goto err_cmd;
+
+	if (cq->cqn < MLX5_CQ_LINEAR_ARRAY_SIZE) {
+		struct mlx5_cq_linear_array_entry *entry;
+
+		entry = &table->linear_array[cq->cqn];
+		spin_lock_irq(&entry->lock);
+		entry->cq = cq;
+		spin_unlock_irq(&entry->lock);
+	}
+
+	cq->pid = curthread->td_proc->p_pid;
+
+	return 0;
+
+err_cmd:
+	memset(&din, 0, sizeof(din));
+	memset(&dout, 0, sizeof(dout));
+	din.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_DESTROY_CQ);
+	din.cqn = cpu_to_be32(cq->cqn);
+	mlx5_cmd_exec(dev, &din, sizeof(din), &dout, sizeof(dout));
+	return err;
+}
+EXPORT_SYMBOL(mlx5_core_create_cq);
+
+int mlx5_core_destroy_cq(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq)
+{
+	struct mlx5_cq_table *table = &dev->priv.cq_table;
+	struct mlx5_destroy_cq_mbox_in in;
+	struct mlx5_destroy_cq_mbox_out out;
+	struct mlx5_core_cq *tmp;
+	int err;
+
+	if (cq->cqn < MLX5_CQ_LINEAR_ARRAY_SIZE) {
+		struct mlx5_cq_linear_array_entry *entry;
+
+		entry = &table->linear_array[cq->cqn];
+		spin_lock_irq(&entry->lock);
+		entry->cq = NULL;
+		spin_unlock_irq(&entry->lock);
+	}
+
+	spin_lock_irq(&table->lock);
+	tmp = radix_tree_delete(&table->tree, cq->cqn);
+	spin_unlock_irq(&table->lock);
+	if (!tmp) {
+		mlx5_core_warn(dev, "cq 0x%x not found in tree\n", cq->cqn);
+		return -EINVAL;
+	}
+	if (tmp != cq) {
+		mlx5_core_warn(dev, "corruption on srqn 0x%x\n", cq->cqn);
+		return -EINVAL;
+	}
+
+	memset(&in, 0, sizeof(in));
+	memset(&out, 0, sizeof(out));
+	in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_DESTROY_CQ);
+	in.cqn = cpu_to_be32(cq->cqn);
+	err = mlx5_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out));
+	if (err)
+		return err;
+
+	if (out.hdr.status)
+		return mlx5_cmd_status_to_err(&out.hdr);
+
+	synchronize_irq(cq->irqn);
+
+	if (atomic_dec_and_test(&cq->refcount))
+		complete(&cq->free);
+	wait_for_completion(&cq->free);
+
+	return 0;
+}
+EXPORT_SYMBOL(mlx5_core_destroy_cq);
+
+int mlx5_core_query_cq(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq,
+		       struct mlx5_query_cq_mbox_out *out)
+{
+	struct mlx5_query_cq_mbox_in in;
+	int err;
+
+	memset(&in, 0, sizeof(in));
+	memset(out, 0, sizeof(*out));
+
+	in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_QUERY_CQ);
+	in.cqn = cpu_to_be32(cq->cqn);
+	err = mlx5_cmd_exec(dev, &in, sizeof(in), out, sizeof(*out));
+	if (err)
+		return err;
+
+	if (out->hdr.status)
+		return mlx5_cmd_status_to_err(&out->hdr);
+
+	return err;
+}
+EXPORT_SYMBOL(mlx5_core_query_cq);
+
+
+int mlx5_core_modify_cq(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq,
+			struct mlx5_modify_cq_mbox_in *in, int in_sz)
+{
+	struct mlx5_modify_cq_mbox_out out;
+	int err;
+
+	memset(&out, 0, sizeof(out));
+	in->hdr.opcode = cpu_to_be16(MLX5_CMD_OP_MODIFY_CQ);
+	err = mlx5_cmd_exec(dev, in, in_sz, &out, sizeof(out));
+	if (err)
+		return err;
+
+	if (out.hdr.status)
+		return mlx5_cmd_status_to_err(&out.hdr);
+
+	return 0;
+}
+EXPORT_SYMBOL(mlx5_core_modify_cq);
+
+int mlx5_core_modify_cq_moderation(struct mlx5_core_dev *dev,
+				   struct mlx5_core_cq *cq,
+				   u16 cq_period,
+				   u16 cq_max_count)
+{
+	struct mlx5_modify_cq_mbox_in in;
+
+	memset(&in, 0, sizeof(in));
+
+	in.cqn              = cpu_to_be32(cq->cqn);
+	in.ctx.cq_period    = cpu_to_be16(cq_period);
+	in.ctx.cq_max_count = cpu_to_be16(cq_max_count);
+	in.field_select     = cpu_to_be32(MLX5_CQ_MODIFY_PERIOD |
+					  MLX5_CQ_MODIFY_COUNT);
+
+	return mlx5_core_modify_cq(dev, cq, &in, sizeof(in));
+}
+
+int mlx5_init_cq_table(struct mlx5_core_dev *dev)
+{
+	struct mlx5_cq_table *table = &dev->priv.cq_table;
+	int err;
+	int x;
+
+	spin_lock_init(&table->lock);
+	for (x = 0; x != MLX5_CQ_LINEAR_ARRAY_SIZE; x++)
+		spin_lock_init(&table->linear_array[x].lock);
+	INIT_RADIX_TREE(&table->tree, GFP_ATOMIC);
+	err = 0;
+
+	return err;
+}
+
+void mlx5_cleanup_cq_table(struct mlx5_core_dev *dev)
+{
+}
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_eq.c b/sys/dev/mlx5/mlx5_core/mlx5_eq.c
new file mode 100644
index 0000000..11b560a
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_core/mlx5_eq.c
@@ -0,0 +1,592 @@
+/*-
+ * Copyright (c) 2013-2015, Mellanox Technologies, Ltd.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <dev/mlx5/driver.h>
+#include <dev/mlx5/mlx5_ifc.h>
+#include "mlx5_core.h"
+
+enum {
+	MLX5_EQE_SIZE		= sizeof(struct mlx5_eqe),
+	MLX5_EQE_OWNER_INIT_VAL	= 0x1,
+};
+
+enum {
+	MLX5_NUM_SPARE_EQE	= 0x80,
+	MLX5_NUM_ASYNC_EQE	= 0x100,
+	MLX5_NUM_CMD_EQE	= 32,
+};
+
+enum {
+	MLX5_EQ_DOORBEL_OFFSET	= 0x40,
+};
+
+#define MLX5_ASYNC_EVENT_MASK ((1ull << MLX5_EVENT_TYPE_PATH_MIG)	    | \
+			       (1ull << MLX5_EVENT_TYPE_COMM_EST)	    | \
+			       (1ull << MLX5_EVENT_TYPE_SQ_DRAINED)	    | \
+			       (1ull << MLX5_EVENT_TYPE_CQ_ERROR)	    | \
+			       (1ull << MLX5_EVENT_TYPE_WQ_CATAS_ERROR)	    | \
+			       (1ull << MLX5_EVENT_TYPE_PATH_MIG_FAILED)    | \
+			       (1ull << MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR) | \
+			       (1ull << MLX5_EVENT_TYPE_WQ_ACCESS_ERROR)    | \
+			       (1ull << MLX5_EVENT_TYPE_PORT_CHANGE)	    | \
+			       (1ull << MLX5_EVENT_TYPE_NIC_VPORT_CHANGE)   | \
+			       (1ull << MLX5_EVENT_TYPE_SRQ_CATAS_ERROR)    | \
+			       (1ull << MLX5_EVENT_TYPE_SRQ_LAST_WQE)	    | \
+			       (1ull << MLX5_EVENT_TYPE_SRQ_RQ_LIMIT))
+
+struct map_eq_in {
+	u64	mask;
+	u32	reserved;
+	u32	unmap_eqn;
+};
+
+struct cre_des_eq {
+	u8	reserved[15];
+	u8	eqn;
+};
+
+/*Function prototype*/
+static void mlx5_port_module_event(struct mlx5_core_dev *dev,
+				   struct mlx5_eqe *eqe);
+
+static int mlx5_cmd_destroy_eq(struct mlx5_core_dev *dev, u8 eqn)
+{
+	u32 in[MLX5_ST_SZ_DW(destroy_eq_in)];
+	u32 out[MLX5_ST_SZ_DW(destroy_eq_out)];
+
+	memset(in, 0, sizeof(in));
+
+	MLX5_SET(destroy_eq_in, in, opcode, MLX5_CMD_OP_DESTROY_EQ);
+	MLX5_SET(destroy_eq_in, in, eq_number, eqn);
+
+	memset(out, 0, sizeof(out));
+	return mlx5_cmd_exec_check_status(dev, in,  sizeof(in),
+					       out, sizeof(out));
+}
+
+static struct mlx5_eqe *get_eqe(struct mlx5_eq *eq, u32 entry)
+{
+	return mlx5_buf_offset(&eq->buf, entry * MLX5_EQE_SIZE);
+}
+
+static struct mlx5_eqe *next_eqe_sw(struct mlx5_eq *eq)
+{
+	struct mlx5_eqe *eqe = get_eqe(eq, eq->cons_index & (eq->nent - 1));
+
+	return ((eqe->owner & 1) ^ !!(eq->cons_index & eq->nent)) ? NULL : eqe;
+}
+
+static const char *eqe_type_str(u8 type)
+{
+	switch (type) {
+	case MLX5_EVENT_TYPE_COMP:
+		return "MLX5_EVENT_TYPE_COMP";
+	case MLX5_EVENT_TYPE_PATH_MIG:
+		return "MLX5_EVENT_TYPE_PATH_MIG";
+	case MLX5_EVENT_TYPE_COMM_EST:
+		return "MLX5_EVENT_TYPE_COMM_EST";
+	case MLX5_EVENT_TYPE_SQ_DRAINED:
+		return "MLX5_EVENT_TYPE_SQ_DRAINED";
+	case MLX5_EVENT_TYPE_SRQ_LAST_WQE:
+		return "MLX5_EVENT_TYPE_SRQ_LAST_WQE";
+	case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT:
+		return "MLX5_EVENT_TYPE_SRQ_RQ_LIMIT";
+	case MLX5_EVENT_TYPE_CQ_ERROR:
+		return "MLX5_EVENT_TYPE_CQ_ERROR";
+	case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
+		return "MLX5_EVENT_TYPE_WQ_CATAS_ERROR";
+	case MLX5_EVENT_TYPE_PATH_MIG_FAILED:
+		return "MLX5_EVENT_TYPE_PATH_MIG_FAILED";
+	case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
+		return "MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR";
+	case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
+		return "MLX5_EVENT_TYPE_WQ_ACCESS_ERROR";
+	case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR:
+		return "MLX5_EVENT_TYPE_SRQ_CATAS_ERROR";
+	case MLX5_EVENT_TYPE_INTERNAL_ERROR:
+		return "MLX5_EVENT_TYPE_INTERNAL_ERROR";
+	case MLX5_EVENT_TYPE_PORT_CHANGE:
+		return "MLX5_EVENT_TYPE_PORT_CHANGE";
+	case MLX5_EVENT_TYPE_GPIO_EVENT:
+		return "MLX5_EVENT_TYPE_GPIO_EVENT";
+	case MLX5_EVENT_TYPE_CODING_PORT_MODULE_EVENT:
+		return "MLX5_EVENT_TYPE_PORT_MODULE_EVENT";
+	case MLX5_EVENT_TYPE_REMOTE_CONFIG:
+		return "MLX5_EVENT_TYPE_REMOTE_CONFIG";
+	case MLX5_EVENT_TYPE_DB_BF_CONGESTION:
+		return "MLX5_EVENT_TYPE_DB_BF_CONGESTION";
+	case MLX5_EVENT_TYPE_STALL_EVENT:
+		return "MLX5_EVENT_TYPE_STALL_EVENT";
+	case MLX5_EVENT_TYPE_CMD:
+		return "MLX5_EVENT_TYPE_CMD";
+	case MLX5_EVENT_TYPE_PAGE_REQUEST:
+		return "MLX5_EVENT_TYPE_PAGE_REQUEST";
+	case MLX5_EVENT_TYPE_NIC_VPORT_CHANGE:
+		return "MLX5_EVENT_TYPE_NIC_VPORT_CHANGE";
+	default:
+		return "Unrecognized event";
+	}
+}
+
+static enum mlx5_dev_event port_subtype_event(u8 subtype)
+{
+	switch (subtype) {
+	case MLX5_PORT_CHANGE_SUBTYPE_DOWN:
+		return MLX5_DEV_EVENT_PORT_DOWN;
+	case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE:
+		return MLX5_DEV_EVENT_PORT_UP;
+	case MLX5_PORT_CHANGE_SUBTYPE_INITIALIZED:
+		return MLX5_DEV_EVENT_PORT_INITIALIZED;
+	case MLX5_PORT_CHANGE_SUBTYPE_LID:
+		return MLX5_DEV_EVENT_LID_CHANGE;
+	case MLX5_PORT_CHANGE_SUBTYPE_PKEY:
+		return MLX5_DEV_EVENT_PKEY_CHANGE;
+	case MLX5_PORT_CHANGE_SUBTYPE_GUID:
+		return MLX5_DEV_EVENT_GUID_CHANGE;
+	case MLX5_PORT_CHANGE_SUBTYPE_CLIENT_REREG:
+		return MLX5_DEV_EVENT_CLIENT_REREG;
+	}
+	return -1;
+}
+
+static void eq_update_ci(struct mlx5_eq *eq, int arm)
+{
+	__be32 __iomem *addr = eq->doorbell + (arm ? 0 : 2);
+	u32 val = (eq->cons_index & 0xffffff) | (eq->eqn << 24);
+	__raw_writel((__force u32) cpu_to_be32(val), addr);
+	/* We still want ordering, just not swabbing, so add a barrier */
+	mb();
+}
+
+static int mlx5_eq_int(struct mlx5_core_dev *dev, struct mlx5_eq *eq)
+{
+	struct mlx5_eqe *eqe;
+	int eqes_found = 0;
+	int set_ci = 0;
+	u32 cqn;
+	u32 rsn;
+	u8 port;
+
+	while ((eqe = next_eqe_sw(eq))) {
+		/*
+		 * Make sure we read EQ entry contents after we've
+		 * checked the ownership bit.
+		 */
+		rmb();
+
+		mlx5_core_dbg(eq->dev, "eqn %d, eqe type %s\n",
+			      eq->eqn, eqe_type_str(eqe->type));
+		switch (eqe->type) {
+		case MLX5_EVENT_TYPE_COMP:
+			cqn = be32_to_cpu(eqe->data.comp.cqn) & 0xffffff;
+			mlx5_cq_completion(dev, cqn);
+			break;
+
+		case MLX5_EVENT_TYPE_PATH_MIG:
+		case MLX5_EVENT_TYPE_COMM_EST:
+		case MLX5_EVENT_TYPE_SQ_DRAINED:
+		case MLX5_EVENT_TYPE_SRQ_LAST_WQE:
+		case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
+		case MLX5_EVENT_TYPE_PATH_MIG_FAILED:
+		case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
+		case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
+			rsn = be32_to_cpu(eqe->data.qp_srq.qp_srq_n) & 0xffffff;
+			mlx5_core_dbg(dev, "event %s(%d) arrived on resource 0x%x\n",
+				      eqe_type_str(eqe->type), eqe->type, rsn);
+			mlx5_rsc_event(dev, rsn, eqe->type);
+			break;
+
+		case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT:
+		case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR:
+			rsn = be32_to_cpu(eqe->data.qp_srq.qp_srq_n) & 0xffffff;
+			mlx5_core_dbg(dev, "SRQ event %s(%d): srqn 0x%x\n",
+				      eqe_type_str(eqe->type), eqe->type, rsn);
+			mlx5_srq_event(dev, rsn, eqe->type);
+			break;
+
+		case MLX5_EVENT_TYPE_CMD:
+			mlx5_cmd_comp_handler(dev, be32_to_cpu(eqe->data.cmd.vector));
+			break;
+
+		case MLX5_EVENT_TYPE_PORT_CHANGE:
+			port = (eqe->data.port.port >> 4) & 0xf;
+			switch (eqe->sub_type) {
+			case MLX5_PORT_CHANGE_SUBTYPE_DOWN:
+			case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE:
+			case MLX5_PORT_CHANGE_SUBTYPE_LID:
+			case MLX5_PORT_CHANGE_SUBTYPE_PKEY:
+			case MLX5_PORT_CHANGE_SUBTYPE_GUID:
+			case MLX5_PORT_CHANGE_SUBTYPE_CLIENT_REREG:
+			case MLX5_PORT_CHANGE_SUBTYPE_INITIALIZED:
+				if (dev->event)
+					dev->event(dev, port_subtype_event(eqe->sub_type),
+						   (unsigned long)port);
+				break;
+			default:
+				mlx5_core_warn(dev, "Port event with unrecognized subtype: port %d, sub_type %d\n",
+					       port, eqe->sub_type);
+			}
+			break;
+		case MLX5_EVENT_TYPE_CQ_ERROR:
+			cqn = be32_to_cpu(eqe->data.cq_err.cqn) & 0xffffff;
+			mlx5_core_warn(dev, "CQ error on CQN 0x%x, syndrom 0x%x\n",
+				       cqn, eqe->data.cq_err.syndrome);
+			mlx5_cq_event(dev, cqn, eqe->type);
+			break;
+
+		case MLX5_EVENT_TYPE_PAGE_REQUEST:
+			{
+				u16 func_id = be16_to_cpu(eqe->data.req_pages.func_id);
+				s32 npages = be32_to_cpu(eqe->data.req_pages.num_pages);
+
+				mlx5_core_dbg(dev, "page request for func 0x%x, npages %d\n",
+					      func_id, npages);
+				mlx5_core_req_pages_handler(dev, func_id, npages);
+			}
+			break;
+
+		case MLX5_EVENT_TYPE_CODING_PORT_MODULE_EVENT:
+			mlx5_port_module_event(dev, eqe);
+			break;
+
+		case MLX5_EVENT_TYPE_NIC_VPORT_CHANGE:
+			{
+				struct mlx5_eqe_vport_change *vc_eqe =
+						&eqe->data.vport_change;
+				u16 vport_num = be16_to_cpu(vc_eqe->vport_num);
+
+				if (dev->event)
+					dev->event(dev,
+					     MLX5_DEV_EVENT_VPORT_CHANGE,
+					     (unsigned long)vport_num);
+			}
+			break;
+
+		default:
+			mlx5_core_warn(dev, "Unhandled event 0x%x on EQ 0x%x\n",
+				       eqe->type, eq->eqn);
+			break;
+		}
+
+		++eq->cons_index;
+		eqes_found = 1;
+		++set_ci;
+
+		/* The HCA will think the queue has overflowed if we
+		 * don't tell it we've been processing events.  We
+		 * create our EQs with MLX5_NUM_SPARE_EQE extra
+		 * entries, so we must update our consumer index at
+		 * least that often.
+		 */
+		if (unlikely(set_ci >= MLX5_NUM_SPARE_EQE)) {
+			eq_update_ci(eq, 0);
+			set_ci = 0;
+		}
+	}
+
+	eq_update_ci(eq, 1);
+
+	return eqes_found;
+}
+
+static irqreturn_t mlx5_msix_handler(int irq, void *eq_ptr)
+{
+	struct mlx5_eq *eq = eq_ptr;
+	struct mlx5_core_dev *dev = eq->dev;
+
+	mlx5_eq_int(dev, eq);
+
+	/* MSI-X vectors always belong to us */
+	return IRQ_HANDLED;
+}
+
+static void init_eq_buf(struct mlx5_eq *eq)
+{
+	struct mlx5_eqe *eqe;
+	int i;
+
+	for (i = 0; i < eq->nent; i++) {
+		eqe = get_eqe(eq, i);
+		eqe->owner = MLX5_EQE_OWNER_INIT_VAL;
+	}
+}
+
+int mlx5_create_map_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq, u8 vecidx,
+		       int nent, u64 mask, const char *name, struct mlx5_uar *uar)
+{
+	struct mlx5_priv *priv = &dev->priv;
+	struct mlx5_create_eq_mbox_in *in;
+	struct mlx5_create_eq_mbox_out out;
+	int err;
+	int inlen;
+
+	eq->nent = roundup_pow_of_two(nent + MLX5_NUM_SPARE_EQE);
+	err = mlx5_buf_alloc(dev, eq->nent * MLX5_EQE_SIZE, 2 * PAGE_SIZE,
+			     &eq->buf);
+	if (err)
+		return err;
+
+	init_eq_buf(eq);
+
+	inlen = sizeof(*in) + sizeof(in->pas[0]) * eq->buf.npages;
+	in = mlx5_vzalloc(inlen);
+	if (!in) {
+		err = -ENOMEM;
+		goto err_buf;
+	}
+	memset(&out, 0, sizeof(out));
+
+	mlx5_fill_page_array(&eq->buf, in->pas);
+
+	in->hdr.opcode = cpu_to_be16(MLX5_CMD_OP_CREATE_EQ);
+	in->ctx.log_sz_usr_page = cpu_to_be32(ilog2(eq->nent) << 24 | uar->index);
+	in->ctx.intr = vecidx;
+	in->ctx.log_page_size = eq->buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT;
+	in->events_mask = cpu_to_be64(mask);
+
+	err = mlx5_cmd_exec(dev, in, inlen, &out, sizeof(out));
+	if (err)
+		goto err_in;
+
+	if (out.hdr.status) {
+		err = mlx5_cmd_status_to_err(&out.hdr);
+		goto err_in;
+	}
+
+	eq->eqn = out.eq_number;
+	eq->irqn = vecidx;
+	eq->dev = dev;
+	eq->doorbell = uar->map + MLX5_EQ_DOORBEL_OFFSET;
+	snprintf(priv->irq_info[vecidx].name, MLX5_MAX_IRQ_NAME, "%s@pci:%s",
+		 name, pci_name(dev->pdev));
+	err = request_irq(priv->msix_arr[vecidx].vector, mlx5_msix_handler, 0,
+			  priv->irq_info[vecidx].name, eq);
+	if (err)
+		goto err_eq;
+
+
+	/* EQs are created in ARMED state
+	 */
+	eq_update_ci(eq, 1);
+
+	kvfree(in);
+	return 0;
+
+
+err_eq:
+	mlx5_cmd_destroy_eq(dev, eq->eqn);
+
+err_in:
+	kvfree(in);
+
+err_buf:
+	mlx5_buf_free(dev, &eq->buf);
+	return err;
+}
+EXPORT_SYMBOL_GPL(mlx5_create_map_eq);
+
+int mlx5_destroy_unmap_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq)
+{
+	int err;
+
+	free_irq(dev->priv.msix_arr[eq->irqn].vector, eq);
+	err = mlx5_cmd_destroy_eq(dev, eq->eqn);
+	if (err)
+		mlx5_core_warn(dev, "failed to destroy a previously created eq: eqn %d\n",
+			       eq->eqn);
+	mlx5_buf_free(dev, &eq->buf);
+
+	return err;
+}
+EXPORT_SYMBOL_GPL(mlx5_destroy_unmap_eq);
+
+int mlx5_eq_init(struct mlx5_core_dev *dev)
+{
+	int err;
+
+	spin_lock_init(&dev->priv.eq_table.lock);
+
+	err = 0;
+
+	return err;
+}
+
+
+void mlx5_eq_cleanup(struct mlx5_core_dev *dev)
+{
+}
+
+int mlx5_start_eqs(struct mlx5_core_dev *dev)
+{
+	struct mlx5_eq_table *table = &dev->priv.eq_table;
+	u32 async_event_mask = MLX5_ASYNC_EVENT_MASK;
+	int err;
+
+	if (MLX5_CAP_GEN(dev, port_module_event))
+		async_event_mask |= (1ull <<
+				     MLX5_EVENT_TYPE_CODING_PORT_MODULE_EVENT);
+
+	err = mlx5_create_map_eq(dev, &table->cmd_eq, MLX5_EQ_VEC_CMD,
+				 MLX5_NUM_CMD_EQE, 1ull << MLX5_EVENT_TYPE_CMD,
+				 "mlx5_cmd_eq", &dev->priv.uuari.uars[0]);
+	if (err) {
+		mlx5_core_warn(dev, "failed to create cmd EQ %d\n", err);
+		return err;
+	}
+
+	mlx5_cmd_use_events(dev);
+
+	err = mlx5_create_map_eq(dev, &table->async_eq, MLX5_EQ_VEC_ASYNC,
+				 MLX5_NUM_ASYNC_EQE, async_event_mask,
+				 "mlx5_async_eq", &dev->priv.uuari.uars[0]);
+	if (err) {
+		mlx5_core_warn(dev, "failed to create async EQ %d\n", err);
+		goto err1;
+	}
+
+	err = mlx5_create_map_eq(dev, &table->pages_eq,
+				 MLX5_EQ_VEC_PAGES,
+				 /* TODO: sriov max_vf + */ 1,
+				 1 << MLX5_EVENT_TYPE_PAGE_REQUEST, "mlx5_pages_eq",
+				 &dev->priv.uuari.uars[0]);
+	if (err) {
+		mlx5_core_warn(dev, "failed to create pages EQ %d\n", err);
+		goto err2;
+	}
+
+	return err;
+
+err2:
+	mlx5_destroy_unmap_eq(dev, &table->async_eq);
+
+err1:
+	mlx5_cmd_use_polling(dev);
+	mlx5_destroy_unmap_eq(dev, &table->cmd_eq);
+	return err;
+}
+
+int mlx5_stop_eqs(struct mlx5_core_dev *dev)
+{
+	struct mlx5_eq_table *table = &dev->priv.eq_table;
+	int err;
+
+	err = mlx5_destroy_unmap_eq(dev, &table->pages_eq);
+	if (err)
+		return err;
+
+	mlx5_destroy_unmap_eq(dev, &table->async_eq);
+	mlx5_cmd_use_polling(dev);
+
+	err = mlx5_destroy_unmap_eq(dev, &table->cmd_eq);
+	if (err)
+		mlx5_cmd_use_events(dev);
+
+	return err;
+}
+
+int mlx5_core_eq_query(struct mlx5_core_dev *dev, struct mlx5_eq *eq,
+		       struct mlx5_query_eq_mbox_out *out, int outlen)
+{
+	struct mlx5_query_eq_mbox_in in;
+	int err;
+
+	memset(&in, 0, sizeof(in));
+	memset(out, 0, outlen);
+	in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_QUERY_EQ);
+	in.eqn = eq->eqn;
+	err = mlx5_cmd_exec(dev, &in, sizeof(in), out, outlen);
+	if (err)
+		return err;
+
+	if (out->hdr.status)
+		err = mlx5_cmd_status_to_err(&out->hdr);
+
+	return err;
+}
+
+EXPORT_SYMBOL_GPL(mlx5_core_eq_query);
+
+static const char *mlx5_port_module_event_error_type_to_string(u8 error_type)
+{
+	switch (error_type) {
+	case MLX5_MODULE_EVENT_ERROR_POWER_BUDGET_EXCEEDED:
+		return "Power Budget Exceeded";
+	case MLX5_MODULE_EVENT_ERROR_LONG_RANGE_FOR_NON_MLNX_CABLE_MODULE:
+		return "Long Range for non MLNX cable/module";
+	case MLX5_MODULE_EVENT_ERROR_BUS_STUCK:
+		return "Bus stuck(I2C or data shorted)";
+	case MLX5_MODULE_EVENT_ERROR_NO_EEPROM_RETRY_TIMEOUT:
+		return "No EEPROM/retry timeout";
+	case MLX5_MODULE_EVENT_ERROR_ENFORCE_PART_NUMBER_LIST:
+		return "Enforce part number list";
+	case MLX5_MODULE_EVENT_ERROR_UNKNOWN_IDENTIFIER:
+		return "Unknown identifier";
+	case MLX5_MODULE_EVENT_ERROR_HIGH_TEMPERATURE:
+		return "High Temperature";
+
+	default:
+		return "Unknown error type";
+	}
+}
+
+static void mlx5_port_module_event(struct mlx5_core_dev *dev,
+				   struct mlx5_eqe *eqe)
+{
+	unsigned int module_num;
+	unsigned int module_status;
+	unsigned int error_type;
+	struct mlx5_eqe_port_module_event *module_event_eqe;
+	struct pci_dev *pdev = dev->pdev;
+
+	module_event_eqe = &eqe->data.port_module_event;
+
+	module_num = (unsigned int)module_event_eqe->module;
+	module_status = (unsigned int)module_event_eqe->module_status &
+			PORT_MODULE_EVENT_MODULE_STATUS_MASK;
+	error_type = (unsigned int)module_event_eqe->error_type &
+		     PORT_MODULE_EVENT_ERROR_TYPE_MASK;
+
+	switch (module_status) {
+	case MLX5_MODULE_STATUS_PLUGGED:
+		device_printf((&pdev->dev)->bsddev, "INFO: ""Module %u, status: plugged", module_num);
+		break;
+
+	case MLX5_MODULE_STATUS_UNPLUGGED:
+		device_printf((&pdev->dev)->bsddev, "INFO: ""Module %u, status: unplugged", module_num);
+		break;
+
+	case MLX5_MODULE_STATUS_ERROR:
+		device_printf((&pdev->dev)->bsddev, "INFO: ""Module %u, status: error, %s", module_num, mlx5_port_module_event_error_type_to_string(error_type));
+		break;
+
+	default:
+		device_printf((&pdev->dev)->bsddev, "INFO: ""Module %u, unknown status", module_num);
+	}
+}
+
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_flow_table.c b/sys/dev/mlx5/mlx5_core/mlx5_flow_table.c
new file mode 100644
index 0000000..4eac6f3
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_core/mlx5_flow_table.c
@@ -0,0 +1,432 @@
+/*-
+ * Copyright (c) 2013-2015, Mellanox Technologies, Ltd.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <dev/mlx5/driver.h>
+#include <dev/mlx5/flow_table.h>
+#include "mlx5_core.h"
+
+struct mlx5_ftg {
+	struct mlx5_flow_table_group    g;
+	u32				id;
+	u32				start_ix;
+};
+
+struct mlx5_flow_table {
+	struct mlx5_core_dev	*dev;
+	u8			level;
+	u8			type;
+	u32			id;
+	u16			vport;
+	struct mutex		mutex; /* sync bitmap alloc */
+	u16			num_groups;
+	struct mlx5_ftg		*group;
+	unsigned long		*bitmap;
+	u32			size;
+};
+
+static int mlx5_set_flow_entry_cmd(struct mlx5_flow_table *ft, u32 group_ix,
+				   u32 flow_index, void *flow_context)
+{
+	u32 out[MLX5_ST_SZ_DW(set_fte_out)];
+	u32 *in;
+	void *in_flow_context;
+	int fcdls =
+		MLX5_GET(flow_context, flow_context, destination_list_size) *
+		MLX5_ST_SZ_BYTES(dest_format_struct);
+	int inlen = MLX5_ST_SZ_BYTES(set_fte_in) + fcdls;
+	int err;
+
+	in = mlx5_vzalloc(inlen);
+	if (!in) {
+		mlx5_core_warn(ft->dev, "failed to allocate inbox\n");
+		return -ENOMEM;
+	}
+
+	MLX5_SET(set_fte_in, in, vport_number, ft->vport);
+	MLX5_SET(set_fte_in, in, other_vport,  !!ft->vport);
+	MLX5_SET(set_fte_in, in, table_type,   ft->type);
+	MLX5_SET(set_fte_in, in, table_id,     ft->id);
+	MLX5_SET(set_fte_in, in, flow_index,   flow_index);
+	MLX5_SET(set_fte_in, in, opcode, MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY);
+
+	in_flow_context = MLX5_ADDR_OF(set_fte_in, in, flow_context);
+	memcpy(in_flow_context, flow_context,
+	       MLX5_ST_SZ_BYTES(flow_context) + fcdls);
+
+	MLX5_SET(flow_context, in_flow_context, group_id, ft->group[group_ix].id);
+
+	memset(out, 0, sizeof(out));
+	err = mlx5_cmd_exec_check_status(ft->dev, in, inlen, out,
+					 sizeof(out));
+	kvfree(in);
+
+	return err;
+}
+
+static void mlx5_del_flow_entry_cmd(struct mlx5_flow_table *ft, u32 flow_index)
+{
+	u32 in[MLX5_ST_SZ_DW(delete_fte_in)];
+	u32 out[MLX5_ST_SZ_DW(delete_fte_out)];
+
+	memset(in, 0, sizeof(in));
+	memset(out, 0, sizeof(out));
+
+#define MLX5_SET_DFTEI(p, x, v) MLX5_SET(delete_fte_in, p, x, v)
+	MLX5_SET_DFTEI(in, vport_number, ft->vport);
+	MLX5_SET_DFTEI(in, other_vport,  !!ft->vport);
+	MLX5_SET_DFTEI(in, table_type,   ft->type);
+	MLX5_SET_DFTEI(in, table_id,     ft->id);
+	MLX5_SET_DFTEI(in, flow_index,   flow_index);
+	MLX5_SET_DFTEI(in, opcode,     MLX5_CMD_OP_DELETE_FLOW_TABLE_ENTRY);
+
+	mlx5_cmd_exec_check_status(ft->dev, in, sizeof(in), out, sizeof(out));
+}
+
+static void mlx5_destroy_flow_group_cmd(struct mlx5_flow_table *ft, int i)
+{
+	u32 in[MLX5_ST_SZ_DW(destroy_flow_group_in)];
+	u32 out[MLX5_ST_SZ_DW(destroy_flow_group_out)];
+
+	memset(in, 0, sizeof(in));
+	memset(out, 0, sizeof(out));
+
+#define MLX5_SET_DFGI(p, x, v) MLX5_SET(destroy_flow_group_in, p, x, v)
+	MLX5_SET_DFGI(in, vport_number, ft->vport);
+	MLX5_SET_DFGI(in, other_vport,  !!ft->vport);
+	MLX5_SET_DFGI(in, table_type,   ft->type);
+	MLX5_SET_DFGI(in, table_id,     ft->id);
+	MLX5_SET_DFGI(in, opcode, MLX5_CMD_OP_DESTROY_FLOW_GROUP);
+	MLX5_SET_DFGI(in, group_id, ft->group[i].id);
+	mlx5_cmd_exec_check_status(ft->dev, in, sizeof(in), out, sizeof(out));
+}
+
+static int mlx5_create_flow_group_cmd(struct mlx5_flow_table *ft, int i)
+{
+	u32 out[MLX5_ST_SZ_DW(create_flow_group_out)];
+	u32 *in;
+	void *in_match_criteria;
+	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
+	struct mlx5_flow_table_group *g = &ft->group[i].g;
+	u32 start_ix = ft->group[i].start_ix;
+	u32 end_ix = start_ix + (1 << g->log_sz) - 1;
+	int err;
+
+	in = mlx5_vzalloc(inlen);
+	if (!in) {
+		mlx5_core_warn(ft->dev, "failed to allocate inbox\n");
+		return -ENOMEM;
+	}
+	in_match_criteria = MLX5_ADDR_OF(create_flow_group_in, in,
+					 match_criteria);
+
+	memset(out, 0, sizeof(out));
+
+#define MLX5_SET_CFGI(p, x, v) MLX5_SET(create_flow_group_in, p, x, v)
+	MLX5_SET_CFGI(in, vport_number,          ft->vport);
+	MLX5_SET_CFGI(in, other_vport,           !!ft->vport);
+	MLX5_SET_CFGI(in, table_type,            ft->type);
+	MLX5_SET_CFGI(in, table_id,              ft->id);
+	MLX5_SET_CFGI(in, opcode,                MLX5_CMD_OP_CREATE_FLOW_GROUP);
+	MLX5_SET_CFGI(in, start_flow_index,      start_ix);
+	MLX5_SET_CFGI(in, end_flow_index,        end_ix);
+	MLX5_SET_CFGI(in, match_criteria_enable, g->match_criteria_enable);
+
+	memcpy(in_match_criteria, g->match_criteria,
+	       MLX5_ST_SZ_BYTES(fte_match_param));
+
+	err = mlx5_cmd_exec_check_status(ft->dev, in, inlen, out,
+					 sizeof(out));
+	if (!err)
+		ft->group[i].id = MLX5_GET(create_flow_group_out, out,
+					   group_id);
+
+	kvfree(in);
+
+	return err;
+}
+
+static void mlx5_destroy_flow_table_groups(struct mlx5_flow_table *ft)
+{
+	int i;
+
+	for (i = 0; i < ft->num_groups; i++)
+		mlx5_destroy_flow_group_cmd(ft, i);
+}
+
+static int mlx5_create_flow_table_groups(struct mlx5_flow_table *ft)
+{
+	int err;
+	int i;
+
+	for (i = 0; i < ft->num_groups; i++) {
+		err = mlx5_create_flow_group_cmd(ft, i);
+		if (err)
+			goto err_destroy_flow_table_groups;
+	}
+
+	return 0;
+
+err_destroy_flow_table_groups:
+	for (i--; i >= 0; i--)
+		mlx5_destroy_flow_group_cmd(ft, i);
+
+	return err;
+}
+
+static int mlx5_create_flow_table_cmd(struct mlx5_flow_table *ft)
+{
+	u32 in[MLX5_ST_SZ_DW(create_flow_table_in)];
+	u32 out[MLX5_ST_SZ_DW(create_flow_table_out)];
+	int err;
+
+	memset(in, 0, sizeof(in));
+
+	MLX5_SET(create_flow_table_in, in, vport_number, ft->vport);
+	MLX5_SET(create_flow_table_in, in, other_vport,  !!ft->vport);
+	MLX5_SET(create_flow_table_in, in, table_type,   ft->type);
+	MLX5_SET(create_flow_table_in, in, level,        ft->level);
+	MLX5_SET(create_flow_table_in, in, log_size, order_base_2(ft->size));
+
+	MLX5_SET(create_flow_table_in, in, opcode,
+		 MLX5_CMD_OP_CREATE_FLOW_TABLE);
+
+	memset(out, 0, sizeof(out));
+	err = mlx5_cmd_exec_check_status(ft->dev, in, sizeof(in), out,
+					 sizeof(out));
+	if (err)
+		return err;
+
+	ft->id = MLX5_GET(create_flow_table_out, out, table_id);
+
+	return 0;
+}
+
+static void mlx5_destroy_flow_table_cmd(struct mlx5_flow_table *ft)
+{
+	u32 in[MLX5_ST_SZ_DW(destroy_flow_table_in)];
+	u32 out[MLX5_ST_SZ_DW(destroy_flow_table_out)];
+
+	memset(in, 0, sizeof(in));
+	memset(out, 0, sizeof(out));
+
+#define MLX5_SET_DFTI(p, x, v) MLX5_SET(destroy_flow_table_in, p, x, v)
+	MLX5_SET_DFTI(in, vport_number, ft->vport);
+	MLX5_SET_DFTI(in, other_vport,  !!ft->vport);
+	MLX5_SET_DFTI(in, table_type,   ft->type);
+	MLX5_SET_DFTI(in, table_id,     ft->id);
+	MLX5_SET_DFTI(in, opcode, MLX5_CMD_OP_DESTROY_FLOW_TABLE);
+
+	mlx5_cmd_exec_check_status(ft->dev, in, sizeof(in), out, sizeof(out));
+}
+
+static int mlx5_find_group(struct mlx5_flow_table *ft, u8 match_criteria_enable,
+			   u32 *match_criteria, int *group_ix)
+{
+	void *mc_outer = MLX5_ADDR_OF(fte_match_param, match_criteria,
+				      outer_headers);
+	void *mc_misc  = MLX5_ADDR_OF(fte_match_param, match_criteria,
+				      misc_parameters);
+	void *mc_inner = MLX5_ADDR_OF(fte_match_param, match_criteria,
+				      inner_headers);
+	int mc_outer_sz = MLX5_ST_SZ_BYTES(fte_match_set_lyr_2_4);
+	int mc_misc_sz  = MLX5_ST_SZ_BYTES(fte_match_set_misc);
+	int mc_inner_sz = MLX5_ST_SZ_BYTES(fte_match_set_lyr_2_4);
+	int i;
+
+	for (i = 0; i < ft->num_groups; i++) {
+		struct mlx5_flow_table_group *g = &ft->group[i].g;
+		void *gmc_outer = MLX5_ADDR_OF(fte_match_param,
+					       g->match_criteria,
+					       outer_headers);
+		void *gmc_misc  = MLX5_ADDR_OF(fte_match_param,
+					       g->match_criteria,
+					       misc_parameters);
+		void *gmc_inner = MLX5_ADDR_OF(fte_match_param,
+					       g->match_criteria,
+					       inner_headers);
+
+		if (g->match_criteria_enable != match_criteria_enable)
+			continue;
+
+		if (match_criteria_enable & MLX5_MATCH_OUTER_HEADERS)
+			if (memcmp(mc_outer, gmc_outer, mc_outer_sz))
+				continue;
+
+		if (match_criteria_enable & MLX5_MATCH_MISC_PARAMETERS)
+			if (memcmp(mc_misc, gmc_misc, mc_misc_sz))
+				continue;
+
+		if (match_criteria_enable & MLX5_MATCH_INNER_HEADERS)
+			if (memcmp(mc_inner, gmc_inner, mc_inner_sz))
+				continue;
+
+		*group_ix = i;
+		return 0;
+	}
+
+	return -EINVAL;
+}
+
+static int alloc_flow_index(struct mlx5_flow_table *ft, int group_ix, u32 *ix)
+{
+	struct mlx5_ftg *g = &ft->group[group_ix];
+	int err = 0;
+
+	mutex_lock(&ft->mutex);
+
+	*ix = find_next_zero_bit(ft->bitmap, ft->size, g->start_ix);
+	if (*ix >= (g->start_ix + (1 << g->g.log_sz)))
+		err = -ENOSPC;
+	else
+		__set_bit(*ix, ft->bitmap);
+
+	mutex_unlock(&ft->mutex);
+
+	return err;
+}
+
+static void mlx5_free_flow_index(struct mlx5_flow_table *ft, u32 ix)
+{
+	__clear_bit(ix, ft->bitmap);
+}
+
+int mlx5_add_flow_table_entry(void *flow_table, u8 match_criteria_enable,
+			      void *match_criteria, void *flow_context,
+			      u32 *flow_index)
+{
+	struct mlx5_flow_table *ft = flow_table;
+	int group_ix;
+	int err;
+
+	err = mlx5_find_group(ft, match_criteria_enable, match_criteria,
+			      &group_ix);
+	if (err) {
+		mlx5_core_warn(ft->dev, "mlx5_find_group failed\n");
+		return err;
+	}
+
+	err = alloc_flow_index(ft, group_ix, flow_index);
+	if (err) {
+		mlx5_core_warn(ft->dev, "alloc_flow_index failed\n");
+		return err;
+	}
+
+	err = mlx5_set_flow_entry_cmd(ft, group_ix, *flow_index, flow_context);
+	if (err)
+		mlx5_free_flow_index(ft, *flow_index);
+
+	return err;
+}
+EXPORT_SYMBOL(mlx5_add_flow_table_entry);
+
+void mlx5_del_flow_table_entry(void *flow_table, u32 flow_index)
+{
+	struct mlx5_flow_table *ft = flow_table;
+
+	mlx5_del_flow_entry_cmd(ft, flow_index);
+	mlx5_free_flow_index(ft, flow_index);
+}
+EXPORT_SYMBOL(mlx5_del_flow_table_entry);
+
+void *mlx5_create_flow_table(struct mlx5_core_dev *dev, u8 level, u8 table_type,
+			     u16 vport,
+			     u16 num_groups,
+			     struct mlx5_flow_table_group *group)
+{
+	struct mlx5_flow_table *ft;
+	u32 start_ix = 0;
+	u32 ft_size = 0;
+	void *gr;
+	void *bm;
+	int err;
+	int i;
+
+	for (i = 0; i < num_groups; i++)
+		ft_size += (1 << group[i].log_sz);
+
+	ft = kzalloc(sizeof(*ft), GFP_KERNEL);
+	gr = kcalloc(num_groups, sizeof(struct mlx5_ftg), GFP_KERNEL);
+	bm = kcalloc(BITS_TO_LONGS(ft_size), sizeof(uintptr_t), GFP_KERNEL);
+
+	ft->group	= gr;
+	ft->bitmap	= bm;
+	ft->num_groups	= num_groups;
+	ft->level	= level;
+	ft->vport	= vport;
+	ft->type	= table_type;
+	ft->size	= ft_size;
+	ft->dev		= dev;
+	mutex_init(&ft->mutex);
+
+	for (i = 0; i < ft->num_groups; i++) {
+		memcpy(&ft->group[i].g, &group[i], sizeof(*group));
+		ft->group[i].start_ix = start_ix;
+		start_ix += 1 << group[i].log_sz;
+	}
+
+	err = mlx5_create_flow_table_cmd(ft);
+	if (err)
+		goto err_free_ft;
+
+	err = mlx5_create_flow_table_groups(ft);
+	if (err)
+		goto err_destroy_flow_table_cmd;
+
+	return ft;
+
+err_destroy_flow_table_cmd:
+	mlx5_destroy_flow_table_cmd(ft);
+
+err_free_ft:
+	mlx5_core_warn(dev, "failed to alloc flow table\n");
+	kfree(bm);
+	kfree(gr);
+	kfree(ft);
+
+	return NULL;
+}
+EXPORT_SYMBOL(mlx5_create_flow_table);
+
+void mlx5_destroy_flow_table(void *flow_table)
+{
+	struct mlx5_flow_table *ft = flow_table;
+
+	mlx5_destroy_flow_table_groups(ft);
+	mlx5_destroy_flow_table_cmd(ft);
+	kfree(ft->bitmap);
+	kfree(ft->group);
+	kfree(ft);
+}
+EXPORT_SYMBOL(mlx5_destroy_flow_table);
+
+u32 mlx5_get_flow_table_id(void *flow_table)
+{
+	struct mlx5_flow_table *ft = flow_table;
+
+	return ft->id;
+}
+EXPORT_SYMBOL(mlx5_get_flow_table_id);
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_fw.c b/sys/dev/mlx5/mlx5_core/mlx5_fw.c
new file mode 100644
index 0000000..06fafdc
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_core/mlx5_fw.c
@@ -0,0 +1,237 @@
+/*-
+ * Copyright (c) 2013-2015, Mellanox Technologies, Ltd.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <dev/mlx5/driver.h>
+#include <linux/module.h>
+#include "mlx5_core.h"
+
+static int mlx5_cmd_query_adapter(struct mlx5_core_dev *dev, u32 *out,
+				  int outlen)
+{
+	u32 in[MLX5_ST_SZ_DW(query_adapter_in)];
+	int err;
+
+	memset(in, 0, sizeof(in));
+
+	MLX5_SET(query_adapter_in, in, opcode, MLX5_CMD_OP_QUERY_ADAPTER);
+
+	err = mlx5_cmd_exec_check_status(dev, in, sizeof(in), out, outlen);
+	return err;
+}
+
+int mlx5_query_board_id(struct mlx5_core_dev *dev)
+{
+	u32 *out;
+	int outlen = MLX5_ST_SZ_BYTES(query_adapter_out);
+	int err;
+
+	out = kzalloc(outlen, GFP_KERNEL);
+
+	err = mlx5_cmd_query_adapter(dev, out, outlen);
+	if (err)
+		goto out_out;
+
+	memcpy(dev->board_id,
+	       MLX5_ADDR_OF(query_adapter_out, out,
+			    query_adapter_struct.vsd_contd_psid),
+	       MLX5_FLD_SZ_BYTES(query_adapter_out,
+				 query_adapter_struct.vsd_contd_psid));
+
+out_out:
+	kfree(out);
+
+	return err;
+}
+
+int mlx5_core_query_vendor_id(struct mlx5_core_dev *mdev, u32 *vendor_id)
+{
+	u32 *out;
+	int outlen = MLX5_ST_SZ_BYTES(query_adapter_out);
+	int err;
+
+	out = kzalloc(outlen, GFP_KERNEL);
+
+	err = mlx5_cmd_query_adapter(mdev, out, outlen);
+	if (err)
+		goto out_out;
+
+	*vendor_id = MLX5_GET(query_adapter_out, out,
+			      query_adapter_struct.ieee_vendor_id);
+
+out_out:
+	kfree(out);
+
+	return err;
+}
+EXPORT_SYMBOL(mlx5_core_query_vendor_id);
+
+static int mlx5_core_query_special_contexts(struct mlx5_core_dev *dev)
+{
+	u32 in[MLX5_ST_SZ_DW(query_special_contexts_in)];
+	u32 out[MLX5_ST_SZ_DW(query_special_contexts_out)];
+	int err;
+
+	memset(in, 0, sizeof(in));
+	memset(out, 0, sizeof(out));
+
+	MLX5_SET(query_special_contexts_in, in, opcode,
+		 MLX5_CMD_OP_QUERY_SPECIAL_CONTEXTS);
+	err = mlx5_cmd_exec_check_status(dev, in, sizeof(in), out,
+					 sizeof(out));
+	if (err)
+		return err;
+
+	dev->special_contexts.resd_lkey = MLX5_GET(query_special_contexts_out,
+						   out, resd_lkey);
+
+	return err;
+}
+
+int mlx5_query_hca_caps(struct mlx5_core_dev *dev)
+{
+	int err;
+
+	err = mlx5_core_get_caps(dev, MLX5_CAP_GENERAL, HCA_CAP_OPMOD_GET_CUR);
+	if (err)
+		return err;
+
+	err = mlx5_core_get_caps(dev, MLX5_CAP_GENERAL, HCA_CAP_OPMOD_GET_MAX);
+	if (err)
+		return err;
+
+	if (MLX5_CAP_GEN(dev, eth_net_offloads)) {
+		err = mlx5_core_get_caps(dev, MLX5_CAP_ETHERNET_OFFLOADS,
+					 HCA_CAP_OPMOD_GET_CUR);
+		if (err)
+			return err;
+		err = mlx5_core_get_caps(dev, MLX5_CAP_ETHERNET_OFFLOADS,
+					 HCA_CAP_OPMOD_GET_MAX);
+		if (err)
+			return err;
+	}
+
+	if (MLX5_CAP_GEN(dev, pg)) {
+		err = mlx5_core_get_caps(dev, MLX5_CAP_ODP,
+					 HCA_CAP_OPMOD_GET_CUR);
+		if (err)
+			return err;
+		err = mlx5_core_get_caps(dev, MLX5_CAP_ODP,
+					 HCA_CAP_OPMOD_GET_MAX);
+		if (err)
+			return err;
+	}
+
+	if (MLX5_CAP_GEN(dev, atomic)) {
+		err = mlx5_core_get_caps(dev, MLX5_CAP_ATOMIC,
+					 HCA_CAP_OPMOD_GET_CUR);
+		if (err)
+			return err;
+		err = mlx5_core_get_caps(dev, MLX5_CAP_ATOMIC,
+					 HCA_CAP_OPMOD_GET_MAX);
+		if (err)
+			return err;
+	}
+
+	if (MLX5_CAP_GEN(dev, roce)) {
+		err = mlx5_core_get_caps(dev, MLX5_CAP_ROCE,
+					 HCA_CAP_OPMOD_GET_CUR);
+		if (err)
+			return err;
+		err = mlx5_core_get_caps(dev, MLX5_CAP_ROCE,
+					 HCA_CAP_OPMOD_GET_MAX);
+		if (err)
+			return err;
+	}
+
+	if (MLX5_CAP_GEN(dev, nic_flow_table)) {
+		err = mlx5_core_get_caps(dev, MLX5_CAP_FLOW_TABLE,
+					 HCA_CAP_OPMOD_GET_CUR);
+		if (err)
+			return err;
+		err = mlx5_core_get_caps(dev, MLX5_CAP_FLOW_TABLE,
+					 HCA_CAP_OPMOD_GET_MAX);
+		if (err)
+			return err;
+	}
+
+	if (
+	    MLX5_CAP_GEN(dev, eswitch_flow_table)) {
+		err = mlx5_core_get_caps(dev, MLX5_CAP_ESWITCH_FLOW_TABLE,
+					 HCA_CAP_OPMOD_GET_CUR);
+		if (err)
+			return err;
+		err = mlx5_core_get_caps(dev, MLX5_CAP_ESWITCH_FLOW_TABLE,
+					 HCA_CAP_OPMOD_GET_MAX);
+		if (err)
+			return err;
+	}
+
+	if (MLX5_CAP_GEN(dev, vport_group_manager)) {
+		err = mlx5_core_get_caps(dev, MLX5_CAP_ESWITCH,
+					 HCA_CAP_OPMOD_GET_CUR);
+		if (err)
+			return err;
+		err = mlx5_core_get_caps(dev, MLX5_CAP_ESWITCH,
+					 HCA_CAP_OPMOD_GET_MAX);
+		if (err)
+			return err;
+	}
+
+	err = mlx5_core_query_special_contexts(dev);
+	if (err)
+		return err;
+
+	return 0;
+}
+
+int mlx5_cmd_init_hca(struct mlx5_core_dev *dev)
+{
+	u32 in[MLX5_ST_SZ_DW(init_hca_in)];
+	u32 out[MLX5_ST_SZ_DW(init_hca_out)];
+
+	memset(in, 0, sizeof(in));
+
+	MLX5_SET(init_hca_in, in, opcode, MLX5_CMD_OP_INIT_HCA);
+
+	memset(out, 0, sizeof(out));
+	return mlx5_cmd_exec_check_status(dev, in,  sizeof(in),
+					       out, sizeof(out));
+}
+
+int mlx5_cmd_teardown_hca(struct mlx5_core_dev *dev)
+{
+	u32 in[MLX5_ST_SZ_DW(teardown_hca_in)];
+	u32 out[MLX5_ST_SZ_DW(teardown_hca_out)];
+
+	memset(in, 0, sizeof(in));
+
+	MLX5_SET(teardown_hca_in, in, opcode, MLX5_CMD_OP_TEARDOWN_HCA);
+
+	memset(out, 0, sizeof(out));
+	return mlx5_cmd_exec_check_status(dev, in,  sizeof(in),
+					       out, sizeof(out));
+}
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_health.c b/sys/dev/mlx5/mlx5_core/mlx5_health.c
new file mode 100644
index 0000000..e2c6afb
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_core/mlx5_health.c
@@ -0,0 +1,183 @@
+/*-
+ * Copyright (c) 2013-2015, Mellanox Technologies, Ltd.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/random.h>
+#include <linux/vmalloc.h>
+#include <dev/mlx5/driver.h>
+#include <dev/mlx5/mlx5_ifc.h>
+#include "mlx5_core.h"
+
+#define	MLX5_HEALTH_POLL_INTERVAL	(2 * HZ)
+#define	MAX_MISSES			3
+
+static DEFINE_SPINLOCK(health_lock);
+static LIST_HEAD(health_list);
+static struct work_struct health_work;
+
+static void health_care(struct work_struct *work)
+{
+	struct mlx5_core_health *health, *n;
+	struct mlx5_core_dev *dev;
+	struct mlx5_priv *priv;
+	LIST_HEAD(tlist);
+
+	spin_lock_irq(&health_lock);
+	list_splice_init(&health_list, &tlist);
+
+	spin_unlock_irq(&health_lock);
+
+	list_for_each_entry_safe(health, n, &tlist, list) {
+		priv = container_of(health, struct mlx5_priv, health);
+		dev = container_of(priv, struct mlx5_core_dev, priv);
+		mlx5_core_warn(dev, "handling bad device here\n");
+		/* nothing yet */
+		spin_lock_irq(&health_lock);
+		list_del_init(&health->list);
+		spin_unlock_irq(&health_lock);
+	}
+}
+
+static const char *hsynd_str(u8 synd)
+{
+	switch (synd) {
+	case MLX5_HEALTH_SYNDR_FW_ERR:
+		return "firmware internal error";
+	case MLX5_HEALTH_SYNDR_IRISC_ERR:
+		return "irisc not responding";
+	case MLX5_HEALTH_SYNDR_CRC_ERR:
+		return "firmware CRC error";
+	case MLX5_HEALTH_SYNDR_FETCH_PCI_ERR:
+		return "ICM fetch PCI error";
+	case MLX5_HEALTH_SYNDR_HW_FTL_ERR:
+		return "HW fatal error\n";
+	case MLX5_HEALTH_SYNDR_ASYNC_EQ_OVERRUN_ERR:
+		return "async EQ buffer overrun";
+	case MLX5_HEALTH_SYNDR_EQ_ERR:
+		return "EQ error";
+	case MLX5_HEALTH_SYNDR_FFSER_ERR:
+		return "FFSER error";
+	default:
+		return "unrecognized error";
+	}
+}
+
+static u16 read_be16(__be16 __iomem *p)
+{
+	return swab16(readl((__force u16 __iomem *) p));
+}
+
+static u32 read_be32(__be32 __iomem *p)
+{
+	return swab32(readl((__force u32 __iomem *) p));
+}
+
+static void print_health_info(struct mlx5_core_dev *dev)
+{
+	struct mlx5_core_health *health = &dev->priv.health;
+	struct mlx5_health_buffer __iomem *h = health->health;
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(h->assert_var); i++)
+		printf("mlx5_core: INFO: ""assert_var[%d] 0x%08x\n", i, read_be32(h->assert_var + i));
+
+	printf("mlx5_core: INFO: ""assert_exit_ptr 0x%08x\n", read_be32(&h->assert_exit_ptr));
+	printf("mlx5_core: INFO: ""assert_callra 0x%08x\n", read_be32(&h->assert_callra));
+	printf("mlx5_core: INFO: ""fw_ver 0x%08x\n", read_be32(&h->fw_ver));
+	printf("mlx5_core: INFO: ""hw_id 0x%08x\n", read_be32(&h->hw_id));
+	printf("mlx5_core: INFO: ""irisc_index %d\n", readb(&h->irisc_index));
+	printf("mlx5_core: INFO: ""synd 0x%x: %s\n", readb(&h->synd), hsynd_str(readb(&h->synd)));
+	printf("mlx5_core: INFO: ""ext_sync 0x%04x\n", read_be16(&h->ext_sync));
+}
+
+static void poll_health(unsigned long data)
+{
+	struct mlx5_core_dev *dev = (struct mlx5_core_dev *)data;
+	struct mlx5_core_health *health = &dev->priv.health;
+	int next;
+	u32 count;
+
+	count = ioread32be(health->health_counter);
+	if (count == health->prev)
+		++health->miss_counter;
+	else
+		health->miss_counter = 0;
+
+	health->prev = count;
+	if (health->miss_counter == MAX_MISSES) {
+		mlx5_core_err(dev, "device's health compromised\n");
+		print_health_info(dev);
+		spin_lock_irq(&health_lock);
+		list_add_tail(&health->list, &health_list);
+		spin_unlock_irq(&health_lock);
+
+		if (!queue_work(mlx5_core_wq, &health_work))
+			mlx5_core_warn(dev, "failed to queue health work\n");
+	} else {
+		get_random_bytes(&next, sizeof(next));
+		next %= HZ;
+		next += jiffies + MLX5_HEALTH_POLL_INTERVAL;
+		mod_timer(&health->timer, next);
+	}
+}
+
+void mlx5_start_health_poll(struct mlx5_core_dev *dev)
+{
+	struct mlx5_core_health *health = &dev->priv.health;
+
+	INIT_LIST_HEAD(&health->list);
+	init_timer(&health->timer);
+	health->health = &dev->iseg->health;
+	health->health_counter = &dev->iseg->health_counter;
+
+	setup_timer(&health->timer, poll_health, (unsigned long)dev);
+	mod_timer(&health->timer,
+		  round_jiffies(jiffies + MLX5_HEALTH_POLL_INTERVAL));
+}
+
+void mlx5_stop_health_poll(struct mlx5_core_dev *dev)
+{
+	struct mlx5_core_health *health = &dev->priv.health;
+
+	del_timer_sync(&health->timer);
+
+	spin_lock_irq(&health_lock);
+	if (!list_empty(&health->list))
+		list_del_init(&health->list);
+	spin_unlock_irq(&health_lock);
+}
+
+void mlx5_health_cleanup(void)
+{
+}
+
+void  __init mlx5_health_init(void)
+{
+
+	INIT_WORK(&health_work, health_care);
+}
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_mad.c b/sys/dev/mlx5/mlx5_core/mlx5_mad.c
new file mode 100644
index 0000000..166930e
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_core/mlx5_mad.c
@@ -0,0 +1,66 @@
+/*-
+ * Copyright (c) 2013-2015, Mellanox Technologies, Ltd.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <dev/mlx5/driver.h>
+#include "mlx5_core.h"
+
+int mlx5_core_mad_ifc(struct mlx5_core_dev *dev, void *inb, void *outb,
+		      u16 opmod, u8 port)
+{
+	struct mlx5_mad_ifc_mbox_in *in = NULL;
+	struct mlx5_mad_ifc_mbox_out *out = NULL;
+	int err;
+
+	in = kzalloc(sizeof(*in), GFP_KERNEL);
+
+	out = kzalloc(sizeof(*out), GFP_KERNEL);
+
+	in->hdr.opcode = cpu_to_be16(MLX5_CMD_OP_MAD_IFC);
+	in->hdr.opmod = cpu_to_be16(opmod);
+	in->port = port;
+
+	memcpy(in->data, inb, sizeof(in->data));
+
+	err = mlx5_cmd_exec(dev, in, sizeof(*in), out, sizeof(*out));
+	if (err)
+		goto out;
+
+	if (out->hdr.status) {
+		err = mlx5_cmd_status_to_err(&out->hdr);
+		goto out;
+	}
+
+	memcpy(outb, out->data, sizeof(out->data));
+
+out:
+	kfree(out);
+	kfree(in);
+	return err;
+}
+EXPORT_SYMBOL_GPL(mlx5_core_mad_ifc);
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_main.c b/sys/dev/mlx5/mlx5_core/mlx5_main.c
new file mode 100644
index 0000000..5e5a9ee
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_core/mlx5_main.c
@@ -0,0 +1,1126 @@
+/*-
+ * Copyright (c) 2013-2015, Mellanox Technologies, Ltd.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <linux/kmod.h>
+#include <linux/module.h>
+#include <linux/errno.h>
+#include <linux/pci.h>
+#include <linux/dma-mapping.h>
+#include <linux/slab.h>
+#include <linux/io-mapping.h>
+#include <linux/interrupt.h>
+#include <dev/mlx5/driver.h>
+#include <dev/mlx5/cq.h>
+#include <dev/mlx5/qp.h>
+#include <dev/mlx5/srq.h>
+#include <linux/delay.h>
+#include <dev/mlx5/mlx5_ifc.h>
+#include "mlx5_core.h"
+
+MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>");
+MODULE_DESCRIPTION("Mellanox Connect-IB, ConnectX-4 core driver");
+MODULE_LICENSE("Dual BSD/GPL");
+#if (__FreeBSD_version >= 1100000)
+MODULE_DEPEND(mlx5, linuxkpi, 1, 1, 1);
+#endif
+MODULE_VERSION(mlx5, 1);
+
+int mlx5_core_debug_mask;
+module_param_named(debug_mask, mlx5_core_debug_mask, int, 0644);
+MODULE_PARM_DESC(debug_mask, "debug mask: 1 = dump cmd data, 2 = dump cmd exec time, 3 = both. Default=0");
+
+#define MLX5_DEFAULT_PROF	2
+static int prof_sel = MLX5_DEFAULT_PROF;
+module_param_named(prof_sel, prof_sel, int, 0444);
+MODULE_PARM_DESC(prof_sel, "profile selector. Valid range 0 - 2");
+
+#define NUMA_NO_NODE       -1
+
+struct workqueue_struct *mlx5_core_wq;
+static LIST_HEAD(intf_list);
+static LIST_HEAD(dev_list);
+static DEFINE_MUTEX(intf_mutex);
+
+struct mlx5_device_context {
+	struct list_head	list;
+	struct mlx5_interface  *intf;
+	void		       *context;
+};
+
+static struct mlx5_profile profiles[] = {
+	[0] = {
+		.mask           = 0,
+	},
+	[1] = {
+		.mask		= MLX5_PROF_MASK_QP_SIZE,
+		.log_max_qp	= 12,
+	},
+	[2] = {
+		.mask		= MLX5_PROF_MASK_QP_SIZE |
+				  MLX5_PROF_MASK_MR_CACHE,
+		.log_max_qp	= 17,
+		.mr_cache[0]	= {
+			.size	= 500,
+			.limit	= 250
+		},
+		.mr_cache[1]	= {
+			.size	= 500,
+			.limit	= 250
+		},
+		.mr_cache[2]	= {
+			.size	= 500,
+			.limit	= 250
+		},
+		.mr_cache[3]	= {
+			.size	= 500,
+			.limit	= 250
+		},
+		.mr_cache[4]	= {
+			.size	= 500,
+			.limit	= 250
+		},
+		.mr_cache[5]	= {
+			.size	= 500,
+			.limit	= 250
+		},
+		.mr_cache[6]	= {
+			.size	= 500,
+			.limit	= 250
+		},
+		.mr_cache[7]	= {
+			.size	= 500,
+			.limit	= 250
+		},
+		.mr_cache[8]	= {
+			.size	= 500,
+			.limit	= 250
+		},
+		.mr_cache[9]	= {
+			.size	= 500,
+			.limit	= 250
+		},
+		.mr_cache[10]	= {
+			.size	= 500,
+			.limit	= 250
+		},
+		.mr_cache[11]	= {
+			.size	= 500,
+			.limit	= 250
+		},
+		.mr_cache[12]	= {
+			.size	= 64,
+			.limit	= 32
+		},
+		.mr_cache[13]	= {
+			.size	= 32,
+			.limit	= 16
+		},
+		.mr_cache[14]	= {
+			.size	= 16,
+			.limit	= 8
+		},
+		.mr_cache[15]	= {
+			.size	= 8,
+			.limit	= 4
+		},
+	},
+	[3] = {
+		.mask		= MLX5_PROF_MASK_QP_SIZE,
+		.log_max_qp	= 17,
+	},
+};
+
+static int set_dma_caps(struct pci_dev *pdev)
+{
+	int err;
+
+	err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
+	if (err) {
+		device_printf((&pdev->dev)->bsddev, "WARN: ""Warning: couldn't set 64-bit PCI DMA mask\n");
+		err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
+		if (err) {
+			device_printf((&pdev->dev)->bsddev, "ERR: ""Can't set PCI DMA mask, aborting\n");
+			return err;
+		}
+	}
+
+	err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
+	if (err) {
+		device_printf((&pdev->dev)->bsddev, "WARN: ""Warning: couldn't set 64-bit consistent PCI DMA mask\n");
+		err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
+		if (err) {
+			device_printf((&pdev->dev)->bsddev, "ERR: ""Can't set consistent PCI DMA mask, aborting\n");
+			return err;
+		}
+	}
+
+	dma_set_max_seg_size(&pdev->dev, 2u * 1024 * 1024 * 1024);
+	return err;
+}
+
+static int request_bar(struct pci_dev *pdev)
+{
+	int err = 0;
+
+	if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) {
+		device_printf((&pdev->dev)->bsddev, "ERR: ""Missing registers BAR, aborting\n");
+		return -ENODEV;
+	}
+
+	err = pci_request_regions(pdev, DRIVER_NAME);
+	if (err)
+		device_printf((&pdev->dev)->bsddev, "ERR: ""Couldn't get PCI resources, aborting\n");
+
+	return err;
+}
+
+static void release_bar(struct pci_dev *pdev)
+{
+	pci_release_regions(pdev);
+}
+
+static int mlx5_enable_msix(struct mlx5_core_dev *dev)
+{
+	struct mlx5_priv *priv = &dev->priv;
+	struct mlx5_eq_table *table = &priv->eq_table;
+	int num_eqs = 1 << MLX5_CAP_GEN(dev, log_max_eq);
+	int nvec;
+	int i;
+
+	nvec = MLX5_CAP_GEN(dev, num_ports) * num_online_cpus() +
+	       MLX5_EQ_VEC_COMP_BASE;
+	nvec = min_t(int, nvec, num_eqs);
+	if (nvec <= MLX5_EQ_VEC_COMP_BASE)
+		return -ENOMEM;
+
+	priv->msix_arr = kzalloc(nvec * sizeof(*priv->msix_arr), GFP_KERNEL);
+
+	priv->irq_info = kzalloc(nvec * sizeof(*priv->irq_info), GFP_KERNEL);
+
+	for (i = 0; i < nvec; i++)
+		priv->msix_arr[i].entry = i;
+
+	nvec = pci_enable_msix_range(dev->pdev, priv->msix_arr,
+				     MLX5_EQ_VEC_COMP_BASE + 1, nvec);
+	if (nvec < 0)
+		return nvec;
+
+	table->num_comp_vectors = nvec - MLX5_EQ_VEC_COMP_BASE;
+
+	return 0;
+
+}
+
+static void mlx5_disable_msix(struct mlx5_core_dev *dev)
+{
+	struct mlx5_priv *priv = &dev->priv;
+
+	pci_disable_msix(dev->pdev);
+	kfree(priv->irq_info);
+	kfree(priv->msix_arr);
+}
+
+struct mlx5_reg_host_endianess {
+	u8	he;
+	u8      rsvd[15];
+};
+
+
+#define CAP_MASK(pos, size) ((u64)((1 << (size)) - 1) << (pos))
+
+enum {
+	MLX5_CAP_BITS_RW_MASK = CAP_MASK(MLX5_CAP_OFF_CMDIF_CSUM, 2) |
+				MLX5_DEV_CAP_FLAG_DCT,
+};
+
+static u16 to_fw_pkey_sz(u32 size)
+{
+	switch (size) {
+	case 128:
+		return 0;
+	case 256:
+		return 1;
+	case 512:
+		return 2;
+	case 1024:
+		return 3;
+	case 2048:
+		return 4;
+	case 4096:
+		return 5;
+	default:
+		printf("mlx5_core: WARN: ""invalid pkey table size %d\n", size);
+		return 0;
+	}
+}
+
+int mlx5_core_get_caps(struct mlx5_core_dev *dev, enum mlx5_cap_type cap_type,
+		       enum mlx5_cap_mode cap_mode)
+{
+	u8 in[MLX5_ST_SZ_BYTES(query_hca_cap_in)];
+	int out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out);
+	void *out, *hca_caps;
+	u16 opmod = (cap_type << 1) | (cap_mode & 0x01);
+	int err;
+
+	memset(in, 0, sizeof(in));
+	out = kzalloc(out_sz, GFP_KERNEL);
+
+	MLX5_SET(query_hca_cap_in, in, opcode, MLX5_CMD_OP_QUERY_HCA_CAP);
+	MLX5_SET(query_hca_cap_in, in, op_mod, opmod);
+	err = mlx5_cmd_exec(dev, in, sizeof(in), out, out_sz);
+	if (err)
+		goto query_ex;
+
+	err = mlx5_cmd_status_to_err_v2(out);
+	if (err) {
+		mlx5_core_warn(dev,
+			       "QUERY_HCA_CAP : type(%x) opmode(%x) Failed(%d)\n",
+			       cap_type, cap_mode, err);
+		goto query_ex;
+	}
+
+	hca_caps =  MLX5_ADDR_OF(query_hca_cap_out, out, capability);
+
+	switch (cap_mode) {
+	case HCA_CAP_OPMOD_GET_MAX:
+		memcpy(dev->hca_caps_max[cap_type], hca_caps,
+		       MLX5_UN_SZ_BYTES(hca_cap_union));
+		break;
+	case HCA_CAP_OPMOD_GET_CUR:
+		memcpy(dev->hca_caps_cur[cap_type], hca_caps,
+		       MLX5_UN_SZ_BYTES(hca_cap_union));
+		break;
+	default:
+		mlx5_core_warn(dev,
+			       "Tried to query dev cap type(%x) with wrong opmode(%x)\n",
+			       cap_type, cap_mode);
+		err = -EINVAL;
+		break;
+	}
+query_ex:
+	kfree(out);
+	return err;
+}
+
+static int set_caps(struct mlx5_core_dev *dev, void *in, int in_sz)
+{
+	u32 out[MLX5_ST_SZ_DW(set_hca_cap_out)];
+	int err;
+
+	memset(out, 0, sizeof(out));
+
+	MLX5_SET(set_hca_cap_in, in, opcode, MLX5_CMD_OP_SET_HCA_CAP);
+	err = mlx5_cmd_exec(dev, in, in_sz, out, sizeof(out));
+	if (err)
+		return err;
+
+	err = mlx5_cmd_status_to_err_v2(out);
+
+	return err;
+}
+
+static int handle_hca_cap(struct mlx5_core_dev *dev)
+{
+	void *set_ctx = NULL;
+	struct mlx5_profile *prof = dev->profile;
+	int err = -ENOMEM;
+	int set_sz = MLX5_ST_SZ_BYTES(set_hca_cap_in);
+	void *set_hca_cap;
+
+	set_ctx = kzalloc(set_sz, GFP_KERNEL);
+
+	err = mlx5_core_get_caps(dev, MLX5_CAP_GENERAL, HCA_CAP_OPMOD_GET_MAX);
+	if (err)
+		goto query_ex;
+
+	err = mlx5_core_get_caps(dev, MLX5_CAP_GENERAL, HCA_CAP_OPMOD_GET_CUR);
+	if (err)
+		goto query_ex;
+
+	set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, set_ctx,
+				   capability);
+	memcpy(set_hca_cap, dev->hca_caps_cur[MLX5_CAP_GENERAL],
+	       MLX5_ST_SZ_BYTES(cmd_hca_cap));
+
+	mlx5_core_dbg(dev, "Current Pkey table size %d Setting new size %d\n",
+		      mlx5_to_sw_pkey_sz(MLX5_CAP_GEN(dev, pkey_table_size)),
+		      128);
+	/* we limit the size of the pkey table to 128 entries for now */
+	MLX5_SET(cmd_hca_cap, set_hca_cap, pkey_table_size,
+		 to_fw_pkey_sz(128));
+
+	if (prof->mask & MLX5_PROF_MASK_QP_SIZE)
+		MLX5_SET(cmd_hca_cap, set_hca_cap, log_max_qp,
+			 prof->log_max_qp);
+
+	/* disable cmdif checksum */
+	MLX5_SET(cmd_hca_cap, set_hca_cap, cmdif_checksum, 0);
+
+	MLX5_SET(cmd_hca_cap, set_hca_cap, log_uar_page_sz, PAGE_SHIFT - 12);
+
+	err = set_caps(dev, set_ctx, set_sz);
+
+query_ex:
+	kfree(set_ctx);
+	return err;
+}
+
+static int set_hca_ctrl(struct mlx5_core_dev *dev)
+{
+	struct mlx5_reg_host_endianess he_in;
+	struct mlx5_reg_host_endianess he_out;
+	int err;
+
+	memset(&he_in, 0, sizeof(he_in));
+	he_in.he = MLX5_SET_HOST_ENDIANNESS;
+	err = mlx5_core_access_reg(dev, &he_in,  sizeof(he_in),
+					&he_out, sizeof(he_out),
+					MLX5_REG_HOST_ENDIANNESS, 0, 1);
+	return err;
+}
+
+static int mlx5_core_enable_hca(struct mlx5_core_dev *dev)
+{
+	u32 in[MLX5_ST_SZ_DW(enable_hca_in)];
+	u32 out[MLX5_ST_SZ_DW(enable_hca_out)];
+
+	memset(in, 0, sizeof(in));
+	MLX5_SET(enable_hca_in, in, opcode, MLX5_CMD_OP_ENABLE_HCA);
+	memset(out, 0, sizeof(out));
+	return mlx5_cmd_exec_check_status(dev, in,  sizeof(in),
+					       out, sizeof(out));
+}
+
+static int mlx5_core_disable_hca(struct mlx5_core_dev *dev)
+{
+	u32 in[MLX5_ST_SZ_DW(disable_hca_in)];
+	u32 out[MLX5_ST_SZ_DW(disable_hca_out)];
+
+	memset(in, 0, sizeof(in));
+
+	MLX5_SET(disable_hca_in, in, opcode, MLX5_CMD_OP_DISABLE_HCA);
+	memset(out, 0, sizeof(out));
+	return mlx5_cmd_exec_check_status(dev, in,  sizeof(in),
+					       out, sizeof(out));
+}
+
+static int mlx5_core_set_issi(struct mlx5_core_dev *dev)
+{
+	u32 query_in[MLX5_ST_SZ_DW(query_issi_in)];
+	u32 query_out[MLX5_ST_SZ_DW(query_issi_out)];
+	u32 set_in[MLX5_ST_SZ_DW(set_issi_in)];
+	u32 set_out[MLX5_ST_SZ_DW(set_issi_out)];
+	int err;
+	u32 sup_issi;
+
+	memset(query_in, 0, sizeof(query_in));
+	memset(query_out, 0, sizeof(query_out));
+
+	MLX5_SET(query_issi_in, query_in, opcode, MLX5_CMD_OP_QUERY_ISSI);
+
+	err = mlx5_cmd_exec_check_status(dev, query_in, sizeof(query_in),
+					 query_out, sizeof(query_out));
+	if (err) {
+		if (((struct mlx5_outbox_hdr *)query_out)->status ==
+		    MLX5_CMD_STAT_BAD_OP_ERR) {
+			pr_debug("Only ISSI 0 is supported\n");
+			return 0;
+		}
+
+		printf("mlx5_core: ERR: ""failed to query ISSI\n");
+		return err;
+	}
+
+	sup_issi = MLX5_GET(query_issi_out, query_out, supported_issi_dw0);
+
+	if (sup_issi & (1 << 1)) {
+		memset(set_in, 0, sizeof(set_in));
+		memset(set_out, 0, sizeof(set_out));
+
+		MLX5_SET(set_issi_in, set_in, opcode, MLX5_CMD_OP_SET_ISSI);
+		MLX5_SET(set_issi_in, set_in, current_issi, 1);
+
+		err = mlx5_cmd_exec_check_status(dev, set_in, sizeof(set_in),
+						 set_out, sizeof(set_out));
+		if (err) {
+			printf("mlx5_core: ERR: ""failed to set ISSI=1\n");
+			return err;
+		}
+
+		dev->issi = 1;
+
+		return 0;
+	} else if (sup_issi & (1 << 0)) {
+		return 0;
+	}
+
+	return -ENOTSUPP;
+}
+
+
+int mlx5_vector2eqn(struct mlx5_core_dev *dev, int vector, int *eqn, int *irqn)
+{
+	struct mlx5_eq_table *table = &dev->priv.eq_table;
+	struct mlx5_eq *eq;
+	int err = -ENOENT;
+
+	spin_lock(&table->lock);
+	list_for_each_entry(eq, &table->comp_eqs_list, list) {
+		if (eq->index == vector) {
+			*eqn = eq->eqn;
+			*irqn = eq->irqn;
+			err = 0;
+			break;
+		}
+	}
+	spin_unlock(&table->lock);
+
+	return err;
+}
+EXPORT_SYMBOL(mlx5_vector2eqn);
+
+int mlx5_rename_eq(struct mlx5_core_dev *dev, int eq_ix, char *name)
+{
+	struct mlx5_priv *priv = &dev->priv;
+	struct mlx5_eq_table *table = &priv->eq_table;
+	struct mlx5_eq *eq;
+	int err = -ENOENT;
+
+	spin_lock(&table->lock);
+	list_for_each_entry(eq, &table->comp_eqs_list, list) {
+		if (eq->index == eq_ix) {
+			int irq_ix = eq_ix + MLX5_EQ_VEC_COMP_BASE;
+
+			snprintf(priv->irq_info[irq_ix].name, MLX5_MAX_IRQ_NAME,
+				 "%s-%d", name, eq_ix);
+
+			err = 0;
+			break;
+		}
+	}
+	spin_unlock(&table->lock);
+
+	return err;
+}
+
+static void free_comp_eqs(struct mlx5_core_dev *dev)
+{
+	struct mlx5_eq_table *table = &dev->priv.eq_table;
+	struct mlx5_eq *eq, *n;
+
+	spin_lock(&table->lock);
+	list_for_each_entry_safe(eq, n, &table->comp_eqs_list, list) {
+		list_del(&eq->list);
+		spin_unlock(&table->lock);
+		if (mlx5_destroy_unmap_eq(dev, eq))
+			mlx5_core_warn(dev, "failed to destroy EQ 0x%x\n",
+				       eq->eqn);
+		kfree(eq);
+		spin_lock(&table->lock);
+	}
+	spin_unlock(&table->lock);
+}
+
+static int alloc_comp_eqs(struct mlx5_core_dev *dev)
+{
+	struct mlx5_eq_table *table = &dev->priv.eq_table;
+	char name[MLX5_MAX_IRQ_NAME];
+	struct mlx5_eq *eq;
+	int ncomp_vec;
+	int nent;
+	int err;
+	int i;
+
+	INIT_LIST_HEAD(&table->comp_eqs_list);
+	ncomp_vec = table->num_comp_vectors;
+	nent = MLX5_COMP_EQ_SIZE;
+	for (i = 0; i < ncomp_vec; i++) {
+		eq = kzalloc(sizeof(*eq), GFP_KERNEL);
+
+		snprintf(name, MLX5_MAX_IRQ_NAME, "mlx5_comp%d", i);
+		err = mlx5_create_map_eq(dev, eq,
+					 i + MLX5_EQ_VEC_COMP_BASE, nent, 0,
+					 name, &dev->priv.uuari.uars[0]);
+		if (err) {
+			kfree(eq);
+			goto clean;
+		}
+		mlx5_core_dbg(dev, "allocated completion EQN %d\n", eq->eqn);
+		eq->index = i;
+		spin_lock(&table->lock);
+		list_add_tail(&eq->list, &table->comp_eqs_list);
+		spin_unlock(&table->lock);
+	}
+
+	return 0;
+
+clean:
+	free_comp_eqs(dev);
+	return err;
+}
+
+static int map_bf_area(struct mlx5_core_dev *dev)
+{
+	resource_size_t bf_start = pci_resource_start(dev->pdev, 0);
+	resource_size_t bf_len = pci_resource_len(dev->pdev, 0);
+
+	dev->priv.bf_mapping = io_mapping_create_wc(bf_start, bf_len);
+
+	return dev->priv.bf_mapping ? 0 : -ENOMEM;
+}
+
+static void unmap_bf_area(struct mlx5_core_dev *dev)
+{
+	if (dev->priv.bf_mapping)
+		io_mapping_free(dev->priv.bf_mapping);
+}
+
+static inline int fw_initializing(struct mlx5_core_dev *dev)
+{
+	return ioread32be(&dev->iseg->initializing) >> 31;
+}
+
+static int wait_fw_init(struct mlx5_core_dev *dev, u32 max_wait_mili)
+{
+	u64 end = jiffies + msecs_to_jiffies(max_wait_mili);
+	int err = 0;
+
+	while (fw_initializing(dev)) {
+		if (time_after(jiffies, end)) {
+			err = -EBUSY;
+			break;
+		}
+		msleep(FW_INIT_WAIT_MS);
+	}
+
+	return err;
+}
+
+static int mlx5_dev_init(struct mlx5_core_dev *dev, struct pci_dev *pdev)
+{
+	struct mlx5_priv *priv = &dev->priv;
+	int err;
+
+	dev->pdev = pdev;
+	pci_set_drvdata(dev->pdev, dev);
+	strncpy(priv->name, dev_name(&pdev->dev), MLX5_MAX_NAME_LEN);
+	priv->name[MLX5_MAX_NAME_LEN - 1] = 0;
+
+	mutex_init(&priv->pgdir_mutex);
+	INIT_LIST_HEAD(&priv->pgdir_list);
+	spin_lock_init(&priv->mkey_lock);
+
+	priv->numa_node = NUMA_NO_NODE;
+
+	err = pci_enable_device(pdev);
+	if (err) {
+		device_printf((&pdev->dev)->bsddev, "ERR: ""Cannot enable PCI device, aborting\n");
+		goto err_dbg;
+	}
+
+	err = request_bar(pdev);
+	if (err) {
+		device_printf((&pdev->dev)->bsddev, "ERR: ""error requesting BARs, aborting\n");
+		goto err_disable;
+	}
+
+	pci_set_master(pdev);
+
+	err = set_dma_caps(pdev);
+	if (err) {
+		device_printf((&pdev->dev)->bsddev, "ERR: ""Failed setting DMA capabilities mask, aborting\n");
+		goto err_clr_master;
+	}
+
+	dev->iseg = ioremap(pci_resource_start(dev->pdev, 0),
+			    sizeof(*dev->iseg));
+	if (!dev->iseg) {
+		err = -ENOMEM;
+		device_printf((&pdev->dev)->bsddev, "ERR: ""Failed mapping initialization segment, aborting\n");
+		goto err_clr_master;
+	}
+	device_printf((&pdev->dev)->bsddev, "INFO: ""firmware version: %d.%d.%d\n", fw_rev_maj(dev), fw_rev_min(dev), fw_rev_sub(dev));
+
+	err = mlx5_cmd_init(dev);
+	if (err) {
+		device_printf((&pdev->dev)->bsddev, "ERR: ""Failed initializing command interface, aborting\n");
+		goto err_unmap;
+	}
+
+	err = wait_fw_init(dev, FW_INIT_TIMEOUT_MILI);
+	if (err) {
+		device_printf((&dev->pdev->dev)->bsddev, "ERR: ""Firmware over %d MS in initializing state, aborting\n", FW_INIT_TIMEOUT_MILI);
+		goto err_cmd_cleanup;
+	}
+
+	mlx5_pagealloc_init(dev);
+
+	err = mlx5_core_enable_hca(dev);
+	if (err) {
+		device_printf((&pdev->dev)->bsddev, "ERR: ""enable hca failed\n");
+		goto err_pagealloc_cleanup;
+	}
+
+	err = mlx5_core_set_issi(dev);
+	if (err) {
+		device_printf((&pdev->dev)->bsddev, "ERR: ""failed to set issi\n");
+		goto err_disable_hca;
+	}
+
+	err = mlx5_pagealloc_start(dev);
+	if (err) {
+		device_printf((&pdev->dev)->bsddev, "ERR: ""mlx5_pagealloc_start failed\n");
+		goto err_disable_hca;
+	}
+
+	err = mlx5_satisfy_startup_pages(dev, 1);
+	if (err) {
+		device_printf((&pdev->dev)->bsddev, "ERR: ""failed to allocate boot pages\n");
+		goto err_pagealloc_stop;
+	}
+
+	err = set_hca_ctrl(dev);
+	if (err) {
+		device_printf((&pdev->dev)->bsddev, "ERR: ""set_hca_ctrl failed\n");
+		goto reclaim_boot_pages;
+	}
+
+	err = handle_hca_cap(dev);
+	if (err) {
+		device_printf((&pdev->dev)->bsddev, "ERR: ""handle_hca_cap failed\n");
+		goto reclaim_boot_pages;
+	}
+
+	err = mlx5_satisfy_startup_pages(dev, 0);
+	if (err) {
+		device_printf((&pdev->dev)->bsddev, "ERR: ""failed to allocate init pages\n");
+		goto reclaim_boot_pages;
+	}
+
+	err = mlx5_cmd_init_hca(dev);
+	if (err) {
+		device_printf((&pdev->dev)->bsddev, "ERR: ""init hca failed\n");
+		goto reclaim_boot_pages;
+	}
+
+	mlx5_start_health_poll(dev);
+
+	err = mlx5_query_hca_caps(dev);
+	if (err) {
+		device_printf((&pdev->dev)->bsddev, "ERR: ""query hca failed\n");
+		goto err_stop_poll;
+	}
+
+	err = mlx5_query_board_id(dev);
+	if (err) {
+		device_printf((&pdev->dev)->bsddev, "ERR: ""query board id failed\n");
+		goto err_stop_poll;
+	}
+
+	err = mlx5_enable_msix(dev);
+	if (err) {
+		device_printf((&pdev->dev)->bsddev, "ERR: ""enable msix failed\n");
+		goto err_stop_poll;
+	}
+
+	err = mlx5_eq_init(dev);
+	if (err) {
+		device_printf((&pdev->dev)->bsddev, "ERR: ""failed to initialize eq\n");
+		goto disable_msix;
+	}
+
+	err = mlx5_alloc_uuars(dev, &priv->uuari);
+	if (err) {
+		device_printf((&pdev->dev)->bsddev, "ERR: ""Failed allocating uar, aborting\n");
+		goto err_eq_cleanup;
+	}
+
+	err = mlx5_start_eqs(dev);
+	if (err) {
+		device_printf((&pdev->dev)->bsddev, "ERR: ""Failed to start pages and async EQs\n");
+		goto err_free_uar;
+	}
+
+	err = alloc_comp_eqs(dev);
+	if (err) {
+		device_printf((&pdev->dev)->bsddev, "ERR: ""Failed to alloc completion EQs\n");
+		goto err_stop_eqs;
+	}
+
+	if (map_bf_area(dev))
+		device_printf((&pdev->dev)->bsddev, "ERR: ""Failed to map blue flame area\n");
+
+	MLX5_INIT_DOORBELL_LOCK(&priv->cq_uar_lock);
+
+	mlx5_init_cq_table(dev);
+	mlx5_init_qp_table(dev);
+	mlx5_init_srq_table(dev);
+	mlx5_init_mr_table(dev);
+
+	return 0;
+
+err_stop_eqs:
+	mlx5_stop_eqs(dev);
+
+err_free_uar:
+	mlx5_free_uuars(dev, &priv->uuari);
+
+err_eq_cleanup:
+	mlx5_eq_cleanup(dev);
+
+disable_msix:
+	mlx5_disable_msix(dev);
+
+err_stop_poll:
+	mlx5_stop_health_poll(dev);
+	if (mlx5_cmd_teardown_hca(dev)) {
+		device_printf((&dev->pdev->dev)->bsddev, "ERR: ""tear_down_hca failed, skip cleanup\n");
+		return err;
+	}
+
+reclaim_boot_pages:
+	mlx5_reclaim_startup_pages(dev);
+
+err_pagealloc_stop:
+	mlx5_pagealloc_stop(dev);
+
+err_disable_hca:
+	mlx5_core_disable_hca(dev);
+
+err_pagealloc_cleanup:
+	mlx5_pagealloc_cleanup(dev);
+err_cmd_cleanup:
+	mlx5_cmd_cleanup(dev);
+
+err_unmap:
+	iounmap(dev->iseg);
+
+err_clr_master:
+	pci_clear_master(dev->pdev);
+	release_bar(dev->pdev);
+
+err_disable:
+	pci_disable_device(dev->pdev);
+
+err_dbg:
+	return err;
+}
+
+static void mlx5_dev_cleanup(struct mlx5_core_dev *dev)
+{
+	struct mlx5_priv *priv = &dev->priv;
+
+	mlx5_cleanup_mr_table(dev);
+	mlx5_cleanup_srq_table(dev);
+	mlx5_cleanup_qp_table(dev);
+	mlx5_cleanup_cq_table(dev);
+	unmap_bf_area(dev);
+	free_comp_eqs(dev);
+	mlx5_stop_eqs(dev);
+	mlx5_free_uuars(dev, &priv->uuari);
+	mlx5_eq_cleanup(dev);
+	mlx5_disable_msix(dev);
+	mlx5_stop_health_poll(dev);
+	if (mlx5_cmd_teardown_hca(dev)) {
+		device_printf((&dev->pdev->dev)->bsddev, "ERR: ""tear_down_hca failed, skip cleanup\n");
+		return;
+	}
+	mlx5_pagealloc_stop(dev);
+	mlx5_reclaim_startup_pages(dev);
+	mlx5_core_disable_hca(dev);
+	mlx5_pagealloc_cleanup(dev);
+	mlx5_cmd_cleanup(dev);
+	iounmap(dev->iseg);
+	pci_clear_master(dev->pdev);
+	release_bar(dev->pdev);
+	pci_disable_device(dev->pdev);
+}
+
+static void mlx5_add_device(struct mlx5_interface *intf, struct mlx5_priv *priv)
+{
+	struct mlx5_device_context *dev_ctx;
+	struct mlx5_core_dev *dev = container_of(priv, struct mlx5_core_dev, priv);
+
+	dev_ctx = kmalloc(sizeof(*dev_ctx), GFP_KERNEL);
+
+	dev_ctx->intf    = intf;
+	dev_ctx->context = intf->add(dev);
+
+	if (dev_ctx->context) {
+		spin_lock_irq(&priv->ctx_lock);
+		list_add_tail(&dev_ctx->list, &priv->ctx_list);
+		spin_unlock_irq(&priv->ctx_lock);
+	} else {
+		kfree(dev_ctx);
+	}
+}
+
+static void mlx5_remove_device(struct mlx5_interface *intf, struct mlx5_priv *priv)
+{
+	struct mlx5_device_context *dev_ctx;
+	struct mlx5_core_dev *dev = container_of(priv, struct mlx5_core_dev, priv);
+
+	list_for_each_entry(dev_ctx, &priv->ctx_list, list)
+		if (dev_ctx->intf == intf) {
+			spin_lock_irq(&priv->ctx_lock);
+			list_del(&dev_ctx->list);
+			spin_unlock_irq(&priv->ctx_lock);
+
+			intf->remove(dev, dev_ctx->context);
+			kfree(dev_ctx);
+			return;
+		}
+}
+static int mlx5_register_device(struct mlx5_core_dev *dev)
+{
+	struct mlx5_priv *priv = &dev->priv;
+	struct mlx5_interface *intf;
+
+	mutex_lock(&intf_mutex);
+	list_add_tail(&priv->dev_list, &dev_list);
+	list_for_each_entry(intf, &intf_list, list)
+		mlx5_add_device(intf, priv);
+	mutex_unlock(&intf_mutex);
+
+	return 0;
+}
+static void mlx5_unregister_device(struct mlx5_core_dev *dev)
+{
+	struct mlx5_priv *priv = &dev->priv;
+	struct mlx5_interface *intf;
+
+	mutex_lock(&intf_mutex);
+	list_for_each_entry(intf, &intf_list, list)
+		mlx5_remove_device(intf, priv);
+	list_del(&priv->dev_list);
+	mutex_unlock(&intf_mutex);
+}
+
+int mlx5_register_interface(struct mlx5_interface *intf)
+{
+	struct mlx5_priv *priv;
+
+	if (!intf->add || !intf->remove)
+		return -EINVAL;
+
+	mutex_lock(&intf_mutex);
+	list_add_tail(&intf->list, &intf_list);
+	list_for_each_entry(priv, &dev_list, dev_list)
+		mlx5_add_device(intf, priv);
+	mutex_unlock(&intf_mutex);
+
+	return 0;
+}
+EXPORT_SYMBOL(mlx5_register_interface);
+
+void mlx5_unregister_interface(struct mlx5_interface *intf)
+{
+	struct mlx5_priv *priv;
+
+	mutex_lock(&intf_mutex);
+	list_for_each_entry(priv, &dev_list, dev_list)
+	       mlx5_remove_device(intf, priv);
+	list_del(&intf->list);
+	mutex_unlock(&intf_mutex);
+}
+EXPORT_SYMBOL(mlx5_unregister_interface);
+
+void *mlx5_get_protocol_dev(struct mlx5_core_dev *mdev, int protocol)
+{
+	struct mlx5_priv *priv = &mdev->priv;
+	struct mlx5_device_context *dev_ctx;
+	unsigned long flags;
+	void *result = NULL;
+
+	spin_lock_irqsave(&priv->ctx_lock, flags);
+
+	list_for_each_entry(dev_ctx, &mdev->priv.ctx_list, list)
+		if ((dev_ctx->intf->protocol == protocol) &&
+		    dev_ctx->intf->get_dev) {
+			result = dev_ctx->intf->get_dev(dev_ctx->context);
+			break;
+		}
+
+	spin_unlock_irqrestore(&priv->ctx_lock, flags);
+
+	return result;
+}
+EXPORT_SYMBOL(mlx5_get_protocol_dev);
+
+static void mlx5_core_event(struct mlx5_core_dev *dev, enum mlx5_dev_event event,
+			    unsigned long param)
+{
+	struct mlx5_priv *priv = &dev->priv;
+	struct mlx5_device_context *dev_ctx;
+	unsigned long flags;
+
+	spin_lock_irqsave(&priv->ctx_lock, flags);
+
+	list_for_each_entry(dev_ctx, &priv->ctx_list, list)
+		if (dev_ctx->intf->event)
+			dev_ctx->intf->event(dev, dev_ctx->context, event, param);
+
+	spin_unlock_irqrestore(&priv->ctx_lock, flags);
+}
+
+struct mlx5_core_event_handler {
+	void (*event)(struct mlx5_core_dev *dev,
+		      enum mlx5_dev_event event,
+		      void *data);
+};
+
+
+static int init_one(struct pci_dev *pdev,
+		    const struct pci_device_id *id)
+{
+	struct mlx5_core_dev *dev;
+	struct mlx5_priv *priv;
+	int err;
+
+	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
+	priv = &dev->priv;
+
+	if (prof_sel < 0 || prof_sel >= ARRAY_SIZE(profiles)) {
+		printf("mlx5_core: WARN: ""selected profile out of range, selecting default (%d)\n", MLX5_DEFAULT_PROF);
+		prof_sel = MLX5_DEFAULT_PROF;
+	}
+	dev->profile = &profiles[prof_sel];
+	dev->event = mlx5_core_event;
+
+	INIT_LIST_HEAD(&priv->ctx_list);
+	spin_lock_init(&priv->ctx_lock);
+	err = mlx5_dev_init(dev, pdev);
+	if (err) {
+		device_printf((&pdev->dev)->bsddev, "ERR: ""mlx5_dev_init failed %d\n", err);
+		goto out;
+	}
+
+	err = mlx5_register_device(dev);
+	if (err) {
+		device_printf((&pdev->dev)->bsddev, "ERR: ""mlx5_register_device failed %d\n", err);
+		goto out_init;
+	}
+
+
+	return 0;
+
+out_init:
+	mlx5_dev_cleanup(dev);
+out:
+	kfree(dev);
+	return err;
+}
+
+static void remove_one(struct pci_dev *pdev)
+{
+	struct mlx5_core_dev *dev  = pci_get_drvdata(pdev);
+
+	mlx5_unregister_device(dev);
+	mlx5_dev_cleanup(dev);
+	kfree(dev);
+}
+
+static const struct pci_device_id mlx5_core_pci_table[] = {
+	{ PCI_VDEVICE(MELLANOX, 4113) }, /* Connect-IB */
+	{ PCI_VDEVICE(MELLANOX, 4114) }, /* Connect-IB VF */
+	{ PCI_VDEVICE(MELLANOX, 4115) }, /* ConnectX-4 */
+	{ PCI_VDEVICE(MELLANOX, 4116) }, /* ConnectX-4 VF */
+	{ PCI_VDEVICE(MELLANOX, 4117) }, /* ConnectX-4LX */
+	{ PCI_VDEVICE(MELLANOX, 4118) }, /* ConnectX-4LX VF */
+	{ PCI_VDEVICE(MELLANOX, 4119) },
+	{ PCI_VDEVICE(MELLANOX, 4120) },
+	{ PCI_VDEVICE(MELLANOX, 4121) },
+	{ PCI_VDEVICE(MELLANOX, 4122) },
+	{ PCI_VDEVICE(MELLANOX, 4123) },
+	{ PCI_VDEVICE(MELLANOX, 4124) },
+	{ PCI_VDEVICE(MELLANOX, 4125) },
+	{ PCI_VDEVICE(MELLANOX, 4126) },
+	{ PCI_VDEVICE(MELLANOX, 4127) },
+	{ PCI_VDEVICE(MELLANOX, 4128) },
+	{ PCI_VDEVICE(MELLANOX, 4129) },
+	{ PCI_VDEVICE(MELLANOX, 4130) },
+	{ PCI_VDEVICE(MELLANOX, 4131) },
+	{ PCI_VDEVICE(MELLANOX, 4132) },
+	{ PCI_VDEVICE(MELLANOX, 4133) },
+	{ PCI_VDEVICE(MELLANOX, 4134) },
+	{ PCI_VDEVICE(MELLANOX, 4135) },
+	{ PCI_VDEVICE(MELLANOX, 4136) },
+	{ PCI_VDEVICE(MELLANOX, 4137) },
+	{ PCI_VDEVICE(MELLANOX, 4138) },
+	{ PCI_VDEVICE(MELLANOX, 4139) },
+	{ PCI_VDEVICE(MELLANOX, 4140) },
+	{ PCI_VDEVICE(MELLANOX, 4141) },
+	{ PCI_VDEVICE(MELLANOX, 4142) },
+	{ PCI_VDEVICE(MELLANOX, 4143) },
+	{ PCI_VDEVICE(MELLANOX, 4144) },
+	{ 0, }
+};
+
+MODULE_DEVICE_TABLE(pci, mlx5_core_pci_table);
+
+static struct pci_driver mlx5_core_driver = {
+	.name           = DRIVER_NAME,
+	.id_table       = mlx5_core_pci_table,
+	.probe          = init_one,
+	.remove         = remove_one
+};
+
+static int __init init(void)
+{
+	int err;
+
+	mlx5_core_wq = create_singlethread_workqueue("mlx5_core_wq");
+	if (!mlx5_core_wq) {
+		err = -ENOMEM;
+		goto err_debug;
+	}
+	mlx5_health_init();
+
+	err = pci_register_driver(&mlx5_core_driver);
+	if (err)
+		goto err_health;
+
+
+	return 0;
+
+err_health:
+	mlx5_health_cleanup();
+	destroy_workqueue(mlx5_core_wq);
+err_debug:
+	return err;
+}
+
+static void __exit cleanup(void)
+{
+	pci_unregister_driver(&mlx5_core_driver);
+	mlx5_health_cleanup();
+	destroy_workqueue(mlx5_core_wq);
+}
+
+module_init(init);
+module_exit(cleanup);
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_mcg.c b/sys/dev/mlx5/mlx5_core/mlx5_mcg.c
new file mode 100644
index 0000000..362fa16
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_core/mlx5_mcg.c
@@ -0,0 +1,68 @@
+/*-
+ * Copyright (c) 2013-2015, Mellanox Technologies, Ltd.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <dev/mlx5/driver.h>
+#include <rdma/ib_verbs.h>
+#include "mlx5_core.h"
+
+int mlx5_core_attach_mcg(struct mlx5_core_dev *dev, union ib_gid *mgid, u32 qpn)
+{
+	u32 in[MLX5_ST_SZ_DW(attach_to_mcg_in)];
+	u32 out[MLX5_ST_SZ_DW(attach_to_mcg_out)];
+
+	memset(in, 0, sizeof(in));
+
+	MLX5_SET(attach_to_mcg_in, in, opcode, MLX5_CMD_OP_ATTACH_TO_MCG);
+	MLX5_SET(attach_to_mcg_in, in, qpn, qpn);
+	memcpy(MLX5_ADDR_OF(attach_to_mcg_in, in, multicast_gid), mgid,
+	       sizeof(*mgid));
+
+	memset(out, 0, sizeof(out));
+	return mlx5_cmd_exec_check_status(dev, in,  sizeof(in),
+					       out, sizeof(out));
+}
+EXPORT_SYMBOL(mlx5_core_attach_mcg);
+
+int mlx5_core_detach_mcg(struct mlx5_core_dev *dev, union ib_gid *mgid, u32 qpn)
+{
+	u32 in[MLX5_ST_SZ_DW(detach_from_mcg_in)];
+	u32 out[MLX5_ST_SZ_DW(detach_from_mcg_out)];
+
+	memset(in, 0, sizeof(in));
+
+	MLX5_SET(detach_from_mcg_in, in, opcode, MLX5_CMD_OP_DETACH_FROM_MCG);
+	MLX5_SET(detach_from_mcg_in, in, qpn, qpn);
+	memcpy(MLX5_ADDR_OF(detach_from_mcg_in, in, multicast_gid), mgid,
+	       sizeof(*mgid));
+
+	memset(out, 0, sizeof(out));
+	return mlx5_cmd_exec_check_status(dev, in,  sizeof(in),
+					       out, sizeof(out));
+}
+EXPORT_SYMBOL(mlx5_core_detach_mcg);
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_mr.c b/sys/dev/mlx5/mlx5_core/mlx5_mr.c
new file mode 100644
index 0000000..0c7fa09
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_core/mlx5_mr.c
@@ -0,0 +1,237 @@
+/*-
+ * Copyright (c) 2013-2015, Mellanox Technologies, Ltd.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <dev/mlx5/driver.h>
+#include "mlx5_core.h"
+
+void mlx5_init_mr_table(struct mlx5_core_dev *dev)
+{
+	struct mlx5_mr_table *table = &dev->priv.mr_table;
+
+	rwlock_init(&table->lock);
+	INIT_RADIX_TREE(&table->tree, GFP_ATOMIC);
+}
+
+void mlx5_cleanup_mr_table(struct mlx5_core_dev *dev)
+{
+}
+
+int mlx5_core_create_mkey(struct mlx5_core_dev *dev, struct mlx5_core_mr *mr,
+			  struct mlx5_create_mkey_mbox_in *in, int inlen,
+			  mlx5_cmd_cbk_t callback, void *context,
+			  struct mlx5_create_mkey_mbox_out *out)
+{
+	struct mlx5_mr_table *table = &dev->priv.mr_table;
+	struct mlx5_create_mkey_mbox_out lout;
+	int err;
+	u8 key;
+	unsigned long irql;
+
+	memset(&lout, 0, sizeof(lout));
+	spin_lock_irq(&dev->priv.mkey_lock);
+	key = dev->priv.mkey_key++;
+	spin_unlock_irq(&dev->priv.mkey_lock);
+	in->seg.qpn_mkey7_0 |= cpu_to_be32(key);
+	in->hdr.opcode = cpu_to_be16(MLX5_CMD_OP_CREATE_MKEY);
+	if (callback) {
+		err = mlx5_cmd_exec_cb(dev, in, inlen, out, sizeof(*out),
+				       callback, context);
+		return err;
+	} else {
+		err = mlx5_cmd_exec(dev, in, inlen, &lout, sizeof(lout));
+	}
+
+	if (err) {
+		mlx5_core_dbg(dev, "cmd exec failed %d\n", err);
+		return err;
+	}
+
+	if (lout.hdr.status) {
+		mlx5_core_dbg(dev, "status %d\n", lout.hdr.status);
+		return mlx5_cmd_status_to_err(&lout.hdr);
+	}
+
+	mr->iova = be64_to_cpu(in->seg.start_addr);
+	mr->size = be64_to_cpu(in->seg.len);
+	mr->key = mlx5_idx_to_mkey(be32_to_cpu(lout.mkey) & 0xffffff) | key;
+	mr->pd = be32_to_cpu(in->seg.flags_pd) & 0xffffff;
+
+	mlx5_core_dbg(dev, "out 0x%x, key 0x%x, mkey 0x%x\n",
+		      be32_to_cpu(lout.mkey), key, mr->key);
+
+	/* connect to MR tree */
+	write_lock_irqsave(&table->lock, irql);
+	err = radix_tree_insert(&table->tree, mlx5_base_mkey(mr->key), mr);
+	write_unlock_irqrestore(&table->lock, irql);
+	if (err) {
+		mlx5_core_warn(dev, "failed radix tree insert of mr 0x%x, %d\n",
+			       mlx5_base_mkey(mr->key), err);
+		mlx5_core_destroy_mkey(dev, mr);
+	}
+
+	return err;
+}
+EXPORT_SYMBOL(mlx5_core_create_mkey);
+
+int mlx5_core_destroy_mkey(struct mlx5_core_dev *dev, struct mlx5_core_mr *mr)
+{
+	struct mlx5_mr_table *table = &dev->priv.mr_table;
+	u32 in[MLX5_ST_SZ_DW(destroy_mkey_in)];
+	u32 out[MLX5_ST_SZ_DW(destroy_mkey_out)];
+	struct mlx5_core_mr *deleted_mr;
+	unsigned long flags;
+
+	memset(in, 0, sizeof(in));
+
+	write_lock_irqsave(&table->lock, flags);
+	deleted_mr = radix_tree_delete(&table->tree, mlx5_base_mkey(mr->key));
+	write_unlock_irqrestore(&table->lock, flags);
+	if (!deleted_mr) {
+		mlx5_core_warn(dev, "failed radix tree delete of mr 0x%x\n",
+			       mlx5_base_mkey(mr->key));
+		return -ENOENT;
+	}
+
+	MLX5_SET(destroy_mkey_in, in, opcode, MLX5_CMD_OP_DESTROY_MKEY);
+	MLX5_SET(destroy_mkey_in, in, mkey_index, mlx5_mkey_to_idx(mr->key));
+
+	memset(out, 0, sizeof(out));
+	return mlx5_cmd_exec_check_status(dev, in,  sizeof(in),
+					       out, sizeof(out));
+}
+EXPORT_SYMBOL(mlx5_core_destroy_mkey);
+
+int mlx5_core_query_mkey(struct mlx5_core_dev *dev, struct mlx5_core_mr *mr,
+			 struct mlx5_query_mkey_mbox_out *out, int outlen)
+{
+	struct mlx5_query_mkey_mbox_in in;
+	int err;
+
+	memset(&in, 0, sizeof(in));
+	memset(out, 0, outlen);
+
+	in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_QUERY_MKEY);
+	in.mkey = cpu_to_be32(mlx5_mkey_to_idx(mr->key));
+	err = mlx5_cmd_exec(dev, &in, sizeof(in), out, outlen);
+	if (err)
+		return err;
+
+	if (out->hdr.status)
+		return mlx5_cmd_status_to_err(&out->hdr);
+
+	return err;
+}
+EXPORT_SYMBOL(mlx5_core_query_mkey);
+
+int mlx5_core_dump_fill_mkey(struct mlx5_core_dev *dev, struct mlx5_core_mr *mr,
+			     u32 *mkey)
+{
+	struct mlx5_query_special_ctxs_mbox_in in;
+	struct mlx5_query_special_ctxs_mbox_out out;
+	int err;
+
+	memset(&in, 0, sizeof(in));
+	memset(&out, 0, sizeof(out));
+
+	in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_QUERY_SPECIAL_CONTEXTS);
+	err = mlx5_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out));
+	if (err)
+		return err;
+
+	if (out.hdr.status)
+		return mlx5_cmd_status_to_err(&out.hdr);
+
+	*mkey = be32_to_cpu(out.dump_fill_mkey);
+
+	return err;
+}
+EXPORT_SYMBOL(mlx5_core_dump_fill_mkey);
+
+int mlx5_core_create_psv(struct mlx5_core_dev *dev, u32 pdn,
+			 int npsvs, u32 *sig_index)
+{
+	struct mlx5_allocate_psv_in in;
+	struct mlx5_allocate_psv_out out;
+	int i, err;
+
+	if (npsvs > MLX5_MAX_PSVS)
+		return -EINVAL;
+
+	memset(&in, 0, sizeof(in));
+	memset(&out, 0, sizeof(out));
+
+	in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_CREATE_PSV);
+	in.npsv_pd = cpu_to_be32((npsvs << 28) | pdn);
+	err = mlx5_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out));
+	if (err) {
+		mlx5_core_err(dev, "cmd exec failed %d\n", err);
+		return err;
+	}
+
+	if (out.hdr.status) {
+		mlx5_core_err(dev, "create_psv bad status %d\n",
+			      out.hdr.status);
+		return mlx5_cmd_status_to_err(&out.hdr);
+	}
+
+	for (i = 0; i < npsvs; i++)
+		sig_index[i] = be32_to_cpu(out.psv_idx[i]) & 0xffffff;
+
+	return err;
+}
+EXPORT_SYMBOL(mlx5_core_create_psv);
+
+int mlx5_core_destroy_psv(struct mlx5_core_dev *dev, int psv_num)
+{
+	struct mlx5_destroy_psv_in in;
+	struct mlx5_destroy_psv_out out;
+	int err;
+
+	memset(&in, 0, sizeof(in));
+	memset(&out, 0, sizeof(out));
+
+	in.psv_number = cpu_to_be32(psv_num);
+	in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_DESTROY_PSV);
+	err = mlx5_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out));
+	if (err) {
+		mlx5_core_err(dev, "destroy_psv cmd exec failed %d\n", err);
+		goto out;
+	}
+
+	if (out.hdr.status) {
+		mlx5_core_err(dev, "destroy_psv bad status %d\n",
+			      out.hdr.status);
+		err = mlx5_cmd_status_to_err(&out.hdr);
+		goto out;
+	}
+
+out:
+	return err;
+}
+EXPORT_SYMBOL(mlx5_core_destroy_psv);
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_pagealloc.c b/sys/dev/mlx5/mlx5_core/mlx5_pagealloc.c
new file mode 100644
index 0000000..91904d7
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_core/mlx5_pagealloc.c
@@ -0,0 +1,494 @@
+/*-
+ * Copyright (c) 2013-2015, Mellanox Technologies, Ltd.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <dev/mlx5/driver.h>
+#include "mlx5_core.h"
+
+struct mlx5_pages_req {
+	struct mlx5_core_dev *dev;
+	u16	func_id;
+	s32	npages;
+	struct work_struct work;
+};
+
+struct fw_page {
+	struct rb_node		rb_node;
+	u64			addr;
+	struct page	       *page;
+	u16			func_id;
+	unsigned long		bitmask;
+	struct list_head	list;
+	unsigned		free_count;
+};
+
+struct mlx5_manage_pages_inbox {
+	struct mlx5_inbox_hdr	hdr;
+	__be16			rsvd;
+	__be16			func_id;
+	__be32			num_entries;
+	__be64			pas[0];
+};
+
+struct mlx5_manage_pages_outbox {
+	struct mlx5_outbox_hdr	hdr;
+	__be32			num_entries;
+	u8			rsvd[4];
+	__be64			pas[0];
+};
+
+enum {
+	MAX_RECLAIM_TIME_MSECS	= 5000,
+};
+
+enum {
+	MLX5_MAX_RECLAIM_TIME_MILI	= 5000,
+	MLX5_NUM_4K_IN_PAGE		= PAGE_SIZE / MLX5_ADAPTER_PAGE_SIZE,
+};
+
+static int insert_page(struct mlx5_core_dev *dev, u64 addr, struct page *page, u16 func_id)
+{
+	struct rb_root *root = &dev->priv.page_root;
+	struct rb_node **new = &root->rb_node;
+	struct rb_node *parent = NULL;
+	struct fw_page *nfp;
+	struct fw_page *tfp;
+	int i;
+
+	while (*new) {
+		parent = *new;
+		tfp = rb_entry(parent, struct fw_page, rb_node);
+		if (tfp->addr < addr)
+			new = &parent->rb_left;
+		else if (tfp->addr > addr)
+			new = &parent->rb_right;
+		else
+			return -EEXIST;
+	}
+
+	nfp = kzalloc(sizeof(*nfp), GFP_KERNEL);
+
+	nfp->addr = addr;
+	nfp->page = page;
+	nfp->func_id = func_id;
+	nfp->free_count = MLX5_NUM_4K_IN_PAGE;
+	for (i = 0; i < MLX5_NUM_4K_IN_PAGE; i++)
+		set_bit(i, &nfp->bitmask);
+
+	rb_link_node(&nfp->rb_node, parent, new);
+	rb_insert_color(&nfp->rb_node, root);
+	list_add(&nfp->list, &dev->priv.free_list);
+
+	return 0;
+}
+
+static struct fw_page *find_fw_page(struct mlx5_core_dev *dev, u64 addr)
+{
+	struct rb_root *root = &dev->priv.page_root;
+	struct rb_node *tmp = root->rb_node;
+	struct fw_page *result = NULL;
+	struct fw_page *tfp;
+
+	while (tmp) {
+		tfp = rb_entry(tmp, struct fw_page, rb_node);
+		if (tfp->addr < addr) {
+			tmp = tmp->rb_left;
+		} else if (tfp->addr > addr) {
+			tmp = tmp->rb_right;
+		} else {
+			result = tfp;
+			break;
+		}
+	}
+
+	return result;
+}
+
+static int mlx5_cmd_query_pages(struct mlx5_core_dev *dev, u16 *func_id,
+				s32 *npages, int boot)
+{
+	u32 in[MLX5_ST_SZ_DW(query_pages_in)];
+	u32 out[MLX5_ST_SZ_DW(query_pages_out)];
+	int err;
+
+	memset(in, 0, sizeof(in));
+
+	MLX5_SET(query_pages_in, in, opcode, MLX5_CMD_OP_QUERY_PAGES);
+	MLX5_SET(query_pages_in, in, op_mod,
+		 boot ? MLX5_BOOT_PAGES : MLX5_INIT_PAGES);
+
+	memset(out, 0, sizeof(out));
+	err = mlx5_cmd_exec_check_status(dev, in, sizeof(in), out, sizeof(out));
+	if (err)
+		return err;
+
+	*npages = MLX5_GET(query_pages_out, out, num_pages);
+	*func_id = MLX5_GET(query_pages_out, out, function_id);
+
+	return 0;
+}
+
+static int alloc_4k(struct mlx5_core_dev *dev, u64 *addr)
+{
+	struct fw_page *fp;
+	unsigned n;
+
+	if (list_empty(&dev->priv.free_list))
+		return -ENOMEM;
+
+	fp = list_entry(dev->priv.free_list.next, struct fw_page, list);
+	n = find_first_bit(&fp->bitmask, 8 * sizeof(fp->bitmask));
+	if (n >= MLX5_NUM_4K_IN_PAGE) {
+		mlx5_core_warn(dev, "alloc 4k bug\n");
+		return -ENOENT;
+	}
+	clear_bit(n, &fp->bitmask);
+	fp->free_count--;
+	if (!fp->free_count)
+		list_del(&fp->list);
+
+	*addr = fp->addr + n * MLX5_ADAPTER_PAGE_SIZE;
+
+	return 0;
+}
+
+static void free_4k(struct mlx5_core_dev *dev, u64 addr)
+{
+	struct fw_page *fwp;
+	int n;
+
+	fwp = find_fw_page(dev, addr & PAGE_MASK);
+	if (!fwp) {
+		mlx5_core_warn(dev, "page not found\n");
+		return;
+	}
+
+	n = (addr & ~PAGE_MASK) >> MLX5_ADAPTER_PAGE_SHIFT;
+	fwp->free_count++;
+	set_bit(n, &fwp->bitmask);
+	if (fwp->free_count == MLX5_NUM_4K_IN_PAGE) {
+		rb_erase(&fwp->rb_node, &dev->priv.page_root);
+		if (fwp->free_count != 1)
+			list_del(&fwp->list);
+		dma_unmap_page(&dev->pdev->dev, addr & PAGE_MASK, PAGE_SIZE,
+			       DMA_BIDIRECTIONAL);
+		__free_page(fwp->page);
+		kfree(fwp);
+	} else if (fwp->free_count == 1) {
+		list_add(&fwp->list, &dev->priv.free_list);
+	}
+}
+
+static int alloc_system_page(struct mlx5_core_dev *dev, u16 func_id)
+{
+	struct page *page;
+	u64 addr;
+	int err;
+
+	page = alloc_page(GFP_HIGHUSER);
+	if (!page) {
+		mlx5_core_warn(dev, "failed to allocate page\n");
+		return -ENOMEM;
+	}
+	addr = dma_map_page(&dev->pdev->dev, page, 0,
+			    PAGE_SIZE, DMA_BIDIRECTIONAL);
+	if (dma_mapping_error(&dev->pdev->dev, addr)) {
+		mlx5_core_warn(dev, "failed dma mapping page\n");
+		err = -ENOMEM;
+		goto out_alloc;
+	}
+	err = insert_page(dev, addr, page, func_id);
+	if (err) {
+		mlx5_core_err(dev, "failed to track allocated page\n");
+		goto out_mapping;
+	}
+
+	return 0;
+
+out_mapping:
+	dma_unmap_page(&dev->pdev->dev, addr, PAGE_SIZE, DMA_BIDIRECTIONAL);
+
+out_alloc:
+	__free_page(page);
+	return err;
+}
+static int give_pages(struct mlx5_core_dev *dev, u16 func_id, int npages,
+		      int notify_fail)
+{
+	struct mlx5_manage_pages_inbox *in;
+	struct mlx5_manage_pages_outbox out;
+	struct mlx5_manage_pages_inbox *nin;
+	int inlen;
+	u64 addr;
+	int err;
+	int i;
+
+	inlen = sizeof(*in) + npages * sizeof(in->pas[0]);
+	in = mlx5_vzalloc(inlen);
+	if (!in) {
+		mlx5_core_warn(dev, "vzalloc failed %d\n", inlen);
+		return -ENOMEM;
+	}
+	memset(&out, 0, sizeof(out));
+
+	for (i = 0; i < npages; i++) {
+retry:
+		err = alloc_4k(dev, &addr);
+		if (err) {
+			if (err == -ENOMEM)
+				err = alloc_system_page(dev, func_id);
+			if (err)
+				goto out_4k;
+
+			goto retry;
+		}
+		in->pas[i] = cpu_to_be64(addr);
+	}
+
+	in->hdr.opcode = cpu_to_be16(MLX5_CMD_OP_MANAGE_PAGES);
+	in->hdr.opmod = cpu_to_be16(MLX5_PAGES_GIVE);
+	in->func_id = cpu_to_be16(func_id);
+	in->num_entries = cpu_to_be32(npages);
+	err = mlx5_cmd_exec(dev, in, inlen, &out, sizeof(out));
+	if (err) {
+		mlx5_core_warn(dev, "func_id 0x%x, npages %d, err %d\n",
+			       func_id, npages, err);
+		goto out_alloc;
+	}
+	dev->priv.fw_pages += npages;
+
+	if (out.hdr.status) {
+		err = mlx5_cmd_status_to_err(&out.hdr);
+		if (err) {
+			mlx5_core_warn(dev, "func_id 0x%x, npages %d, status %d\n",
+				       func_id, npages, out.hdr.status);
+			goto out_alloc;
+		}
+	}
+
+	mlx5_core_dbg(dev, "err %d\n", err);
+
+	goto out_free;
+
+out_alloc:
+	if (notify_fail) {
+		nin = kzalloc(sizeof(*nin), GFP_KERNEL);
+		memset(&out, 0, sizeof(out));
+		nin->hdr.opcode = cpu_to_be16(MLX5_CMD_OP_MANAGE_PAGES);
+		nin->hdr.opmod = cpu_to_be16(MLX5_PAGES_CANT_GIVE);
+		if (mlx5_cmd_exec(dev, nin, sizeof(*nin), &out, sizeof(out)))
+			mlx5_core_warn(dev, "page notify failed\n");
+		kfree(nin);
+	}
+
+out_4k:
+	for (i--; i >= 0; i--)
+		free_4k(dev, be64_to_cpu(in->pas[i]));
+out_free:
+	kvfree(in);
+	return err;
+}
+
+static int reclaim_pages(struct mlx5_core_dev *dev, u32 func_id, int npages,
+			 int *nclaimed)
+{
+	struct mlx5_manage_pages_inbox   in;
+	struct mlx5_manage_pages_outbox *out;
+	int num_claimed;
+	int outlen;
+	u64 addr;
+	int err;
+	int i;
+
+	if (nclaimed)
+		*nclaimed = 0;
+
+	memset(&in, 0, sizeof(in));
+	outlen = sizeof(*out) + npages * sizeof(out->pas[0]);
+	out = mlx5_vzalloc(outlen);
+	if (!out)
+		return -ENOMEM;
+
+	in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_MANAGE_PAGES);
+	in.hdr.opmod = cpu_to_be16(MLX5_PAGES_TAKE);
+	in.func_id = cpu_to_be16(func_id);
+	in.num_entries = cpu_to_be32(npages);
+	mlx5_core_dbg(dev, "npages %d, outlen %d\n", npages, outlen);
+	err = mlx5_cmd_exec(dev, &in, sizeof(in), out, outlen);
+	if (err) {
+		mlx5_core_err(dev, "failed reclaiming pages\n");
+		goto out_free;
+	}
+
+	if (out->hdr.status) {
+		err = mlx5_cmd_status_to_err(&out->hdr);
+		goto out_free;
+	}
+
+	num_claimed = be32_to_cpu(out->num_entries);
+	if (nclaimed)
+		*nclaimed = num_claimed;
+
+	dev->priv.fw_pages -= num_claimed;
+
+	for (i = 0; i < num_claimed; i++) {
+		addr = be64_to_cpu(out->pas[i]);
+		free_4k(dev, addr);
+	}
+
+out_free:
+	kvfree(out);
+	return err;
+}
+
+static void pages_work_handler(struct work_struct *work)
+{
+	struct mlx5_pages_req *req = container_of(work, struct mlx5_pages_req, work);
+	struct mlx5_core_dev *dev = req->dev;
+	int err = 0;
+
+	if (req->npages < 0)
+		err = reclaim_pages(dev, req->func_id, -1 * req->npages, NULL);
+	else if (req->npages > 0)
+		err = give_pages(dev, req->func_id, req->npages, 1);
+
+	if (err)
+		mlx5_core_warn(dev, "%s fail %d\n",
+			       req->npages < 0 ? "reclaim" : "give", err);
+
+	kfree(req);
+}
+
+void mlx5_core_req_pages_handler(struct mlx5_core_dev *dev, u16 func_id,
+				 s32 npages)
+{
+	struct mlx5_pages_req *req;
+
+	req = kzalloc(sizeof(*req), GFP_ATOMIC);
+	if (!req) {
+		mlx5_core_warn(dev, "failed to allocate pages request\n");
+		return;
+	}
+
+	req->dev = dev;
+	req->func_id = func_id;
+	req->npages = npages;
+	INIT_WORK(&req->work, pages_work_handler);
+	if (!queue_work(dev->priv.pg_wq, &req->work))
+		mlx5_core_warn(dev, "failed to queue pages handler work\n");
+}
+
+int mlx5_satisfy_startup_pages(struct mlx5_core_dev *dev, int boot)
+{
+	u16 uninitialized_var(func_id);
+	s32 uninitialized_var(npages);
+	int err;
+
+	err = mlx5_cmd_query_pages(dev, &func_id, &npages, boot);
+	if (err)
+		return err;
+
+	mlx5_core_dbg(dev, "requested %d %s pages for func_id 0x%x\n",
+		      npages, boot ? "boot" : "init", func_id);
+
+	return give_pages(dev, func_id, npages, 0);
+}
+
+enum {
+	MLX5_BLKS_FOR_RECLAIM_PAGES = 12
+};
+
+static int optimal_reclaimed_pages(void)
+{
+	struct mlx5_cmd_prot_block *block;
+	struct mlx5_cmd_layout *lay;
+	int ret;
+
+	ret = (sizeof(lay->out) + MLX5_BLKS_FOR_RECLAIM_PAGES * sizeof(block->data) -
+	       sizeof(struct mlx5_manage_pages_outbox)) /
+	       FIELD_SIZEOF(struct mlx5_manage_pages_outbox, pas[0]);
+
+	return ret;
+}
+
+int mlx5_reclaim_startup_pages(struct mlx5_core_dev *dev)
+{
+	int end = jiffies + msecs_to_jiffies(MAX_RECLAIM_TIME_MSECS);
+	struct fw_page *fwp;
+	struct rb_node *p;
+	int nclaimed = 0;
+	int err;
+
+	do {
+		p = rb_first(&dev->priv.page_root);
+		if (p) {
+			fwp = rb_entry(p, struct fw_page, rb_node);
+			err = reclaim_pages(dev, fwp->func_id,
+					    optimal_reclaimed_pages(),
+					    &nclaimed);
+			if (err) {
+				mlx5_core_warn(dev, "failed reclaiming pages (%d)\n",
+					       err);
+				return err;
+			}
+			if (nclaimed)
+				end = jiffies + msecs_to_jiffies(MAX_RECLAIM_TIME_MSECS);
+		}
+		if (time_after(jiffies, end)) {
+			mlx5_core_warn(dev, "FW did not return all pages. giving up...\n");
+			break;
+		}
+	} while (p);
+
+	return 0;
+}
+
+void mlx5_pagealloc_init(struct mlx5_core_dev *dev)
+{
+	dev->priv.page_root = RB_ROOT;
+	INIT_LIST_HEAD(&dev->priv.free_list);
+}
+
+void mlx5_pagealloc_cleanup(struct mlx5_core_dev *dev)
+{
+	/* nothing */
+}
+
+int mlx5_pagealloc_start(struct mlx5_core_dev *dev)
+{
+	dev->priv.pg_wq = create_singlethread_workqueue("mlx5_page_allocator");
+	if (!dev->priv.pg_wq)
+		return -ENOMEM;
+
+	return 0;
+}
+
+void mlx5_pagealloc_stop(struct mlx5_core_dev *dev)
+{
+	destroy_workqueue(dev->priv.pg_wq);
+}
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_pd.c b/sys/dev/mlx5/mlx5_core/mlx5_pd.c
new file mode 100644
index 0000000..17b7adb
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_core/mlx5_pd.c
@@ -0,0 +1,67 @@
+/*-
+ * Copyright (c) 2013-2015, Mellanox Technologies, Ltd.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <dev/mlx5/driver.h>
+#include "mlx5_core.h"
+
+int mlx5_core_alloc_pd(struct mlx5_core_dev *dev, u32 *pdn)
+{
+	u32 in[MLX5_ST_SZ_DW(alloc_pd_in)];
+	u32 out[MLX5_ST_SZ_DW(alloc_pd_out)];
+	int err;
+
+	memset(in, 0, sizeof(in));
+
+	MLX5_SET(alloc_pd_in, in, opcode, MLX5_CMD_OP_ALLOC_PD);
+
+	memset(out, 0, sizeof(out));
+	err = mlx5_cmd_exec_check_status(dev, in, sizeof(in), out, sizeof(out));
+	if (err)
+		return err;
+
+	*pdn = MLX5_GET(alloc_pd_out, out, pd);
+	return 0;
+}
+EXPORT_SYMBOL(mlx5_core_alloc_pd);
+
+int mlx5_core_dealloc_pd(struct mlx5_core_dev *dev, u32 pdn)
+{
+	u32 in[MLX5_ST_SZ_DW(dealloc_pd_in)];
+	u32 out[MLX5_ST_SZ_DW(dealloc_pd_out)];
+
+	memset(in, 0, sizeof(in));
+
+	MLX5_SET(dealloc_pd_in, in, opcode, MLX5_CMD_OP_DEALLOC_PD);
+	MLX5_SET(dealloc_pd_in, in, pd, pdn);
+
+	memset(out, 0, sizeof(out));
+	return mlx5_cmd_exec_check_status(dev, in,  sizeof(in),
+					       out, sizeof(out));
+}
+EXPORT_SYMBOL(mlx5_core_dealloc_pd);
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_port.c b/sys/dev/mlx5/mlx5_core/mlx5_port.c
new file mode 100644
index 0000000..1e938a6
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_core/mlx5_port.c
@@ -0,0 +1,718 @@
+/*-
+ * Copyright (c) 2013-2015, Mellanox Technologies, Ltd.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <linux/module.h>
+#include <dev/mlx5/driver.h>
+#include "mlx5_core.h"
+
+int mlx5_core_access_reg(struct mlx5_core_dev *dev, void *data_in,
+			 int size_in, void *data_out, int size_out,
+			 u16 reg_num, int arg, int write)
+{
+	struct mlx5_access_reg_mbox_in *in = NULL;
+	struct mlx5_access_reg_mbox_out *out = NULL;
+	int err = -ENOMEM;
+
+	in = mlx5_vzalloc(sizeof(*in) + size_in);
+	if (!in)
+		return -ENOMEM;
+
+	out = mlx5_vzalloc(sizeof(*out) + size_out);
+	if (!out)
+		goto ex1;
+
+	memcpy(in->data, data_in, size_in);
+	in->hdr.opcode = cpu_to_be16(MLX5_CMD_OP_ACCESS_REG);
+	in->hdr.opmod = cpu_to_be16(!write);
+	in->arg = cpu_to_be32(arg);
+	in->register_id = cpu_to_be16(reg_num);
+	err = mlx5_cmd_exec(dev, in, sizeof(*in) + size_in, out,
+			    sizeof(*out) + size_out);
+	if (err)
+		goto ex2;
+
+	if (out->hdr.status)
+		err = mlx5_cmd_status_to_err(&out->hdr);
+
+	if (!err)
+		memcpy(data_out, out->data, size_out);
+
+ex2:
+	kvfree(out);
+ex1:
+	kvfree(in);
+	return err;
+}
+EXPORT_SYMBOL_GPL(mlx5_core_access_reg);
+
+
+struct mlx5_reg_pcap {
+	u8			rsvd0;
+	u8			port_num;
+	u8			rsvd1[2];
+	__be32			caps_127_96;
+	__be32			caps_95_64;
+	__be32			caps_63_32;
+	__be32			caps_31_0;
+};
+
+int mlx5_set_port_caps(struct mlx5_core_dev *dev, u8 port_num, u32 caps)
+{
+	struct mlx5_reg_pcap in;
+	struct mlx5_reg_pcap out;
+	int err;
+
+	memset(&in, 0, sizeof(in));
+	in.caps_127_96 = cpu_to_be32(caps);
+	in.port_num = port_num;
+
+	err = mlx5_core_access_reg(dev, &in, sizeof(in), &out,
+				   sizeof(out), MLX5_REG_PCAP, 0, 1);
+
+	return err;
+}
+EXPORT_SYMBOL_GPL(mlx5_set_port_caps);
+
+int mlx5_query_port_ptys(struct mlx5_core_dev *dev, u32 *ptys,
+			 int ptys_size, int proto_mask)
+{
+	u32 in[MLX5_ST_SZ_DW(ptys_reg)];
+	int err;
+
+	memset(in, 0, sizeof(in));
+	MLX5_SET(ptys_reg, in, local_port, 1);
+	MLX5_SET(ptys_reg, in, proto_mask, proto_mask);
+
+	err = mlx5_core_access_reg(dev, in, sizeof(in), ptys,
+				   ptys_size, MLX5_REG_PTYS, 0, 0);
+
+	return err;
+}
+EXPORT_SYMBOL_GPL(mlx5_query_port_ptys);
+
+int mlx5_query_port_proto_cap(struct mlx5_core_dev *dev,
+			      u32 *proto_cap, int proto_mask)
+{
+	u32 out[MLX5_ST_SZ_DW(ptys_reg)];
+	int err;
+
+	err = mlx5_query_port_ptys(dev, out, sizeof(out), proto_mask);
+	if (err)
+		return err;
+
+	if (proto_mask == MLX5_PTYS_EN)
+		*proto_cap = MLX5_GET(ptys_reg, out, eth_proto_capability);
+	else
+		*proto_cap = MLX5_GET(ptys_reg, out, ib_proto_capability);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(mlx5_query_port_proto_cap);
+
+int mlx5_query_port_proto_admin(struct mlx5_core_dev *dev,
+				u32 *proto_admin, int proto_mask)
+{
+	u32 out[MLX5_ST_SZ_DW(ptys_reg)];
+	int err;
+
+	err = mlx5_query_port_ptys(dev, out, sizeof(out), proto_mask);
+	if (err)
+		return err;
+
+	if (proto_mask == MLX5_PTYS_EN)
+		*proto_admin = MLX5_GET(ptys_reg, out, eth_proto_admin);
+	else
+		*proto_admin = MLX5_GET(ptys_reg, out, ib_proto_admin);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(mlx5_query_port_proto_admin);
+
+int mlx5_set_port_proto(struct mlx5_core_dev *dev, u32 proto_admin,
+			int proto_mask)
+{
+	u32 in[MLX5_ST_SZ_DW(ptys_reg)];
+	u32 out[MLX5_ST_SZ_DW(ptys_reg)];
+	int err;
+
+	memset(in, 0, sizeof(in));
+
+	MLX5_SET(ptys_reg, in, local_port, 1);
+	MLX5_SET(ptys_reg, in, proto_mask, proto_mask);
+	if (proto_mask == MLX5_PTYS_EN)
+		MLX5_SET(ptys_reg, in, eth_proto_admin, proto_admin);
+	else
+		MLX5_SET(ptys_reg, in, ib_proto_admin, proto_admin);
+
+	err = mlx5_core_access_reg(dev, in, sizeof(in), out,
+				   sizeof(out), MLX5_REG_PTYS, 0, 1);
+	return err;
+}
+EXPORT_SYMBOL_GPL(mlx5_set_port_proto);
+
+int mlx5_set_port_status(struct mlx5_core_dev *dev,
+			 enum mlx5_port_status status)
+{
+	u32 in[MLX5_ST_SZ_DW(paos_reg)];
+	u32 out[MLX5_ST_SZ_DW(paos_reg)];
+	int err;
+
+	memset(in, 0, sizeof(in));
+
+	MLX5_SET(paos_reg, in, local_port, 1);
+
+	MLX5_SET(paos_reg, in, admin_status, status);
+	MLX5_SET(paos_reg, in, ase, 1);
+
+	err = mlx5_core_access_reg(dev, in, sizeof(in), out,
+				   sizeof(out), MLX5_REG_PAOS, 0, 1);
+	return err;
+}
+
+int mlx5_query_port_status(struct mlx5_core_dev *dev, u8 *status)
+{
+	u32 in[MLX5_ST_SZ_DW(paos_reg)];
+	u32 out[MLX5_ST_SZ_DW(paos_reg)];
+	int err;
+
+	memset(in, 0, sizeof(in));
+
+	MLX5_SET(paos_reg, in, local_port, 1);
+
+	err = mlx5_core_access_reg(dev, in, sizeof(in), out,
+				   sizeof(out), MLX5_REG_PAOS, 0, 0);
+	if (err)
+		return err;
+
+	*status = MLX5_GET(paos_reg, out, oper_status);
+	return err;
+}
+
+static int mlx5_query_port_mtu(struct mlx5_core_dev *dev,
+			       int *admin_mtu, int *max_mtu, int *oper_mtu)
+{
+	u32 in[MLX5_ST_SZ_DW(pmtu_reg)];
+	u32 out[MLX5_ST_SZ_DW(pmtu_reg)];
+	int err;
+
+	memset(in, 0, sizeof(in));
+
+	MLX5_SET(pmtu_reg, in, local_port, 1);
+
+	err = mlx5_core_access_reg(dev, in, sizeof(in), out,
+				   sizeof(out), MLX5_REG_PMTU, 0, 0);
+	if (err)
+		return err;
+
+	if (max_mtu)
+		*max_mtu  = MLX5_GET(pmtu_reg, out, max_mtu);
+	if (oper_mtu)
+		*oper_mtu = MLX5_GET(pmtu_reg, out, oper_mtu);
+	if (admin_mtu)
+		*admin_mtu = MLX5_GET(pmtu_reg, out, admin_mtu);
+
+	return err;
+}
+
+int mlx5_set_port_mtu(struct mlx5_core_dev *dev, int mtu)
+{
+	u32 in[MLX5_ST_SZ_DW(pmtu_reg)];
+	u32 out[MLX5_ST_SZ_DW(pmtu_reg)];
+
+	memset(in, 0, sizeof(in));
+
+	MLX5_SET(pmtu_reg, in, admin_mtu, mtu);
+	MLX5_SET(pmtu_reg, in, local_port, 1);
+
+	return mlx5_core_access_reg(dev, in, sizeof(in), out,
+				   sizeof(out), MLX5_REG_PMTU, 0, 1);
+}
+EXPORT_SYMBOL_GPL(mlx5_set_port_mtu);
+
+int mlx5_query_port_max_mtu(struct mlx5_core_dev *dev, int *max_mtu)
+{
+	return mlx5_query_port_mtu(dev, NULL, max_mtu, NULL);
+}
+EXPORT_SYMBOL_GPL(mlx5_query_port_max_mtu);
+
+int mlx5_set_port_pause(struct mlx5_core_dev *dev, u32 port,
+			u32 rx_pause, u32 tx_pause)
+{
+	u32 in[MLX5_ST_SZ_DW(pfcc_reg)];
+	u32 out[MLX5_ST_SZ_DW(pfcc_reg)];
+
+	memset(in, 0, sizeof(in));
+	memset(out, 0, sizeof(out));
+
+	MLX5_SET(pfcc_reg, in, local_port, port);
+	MLX5_SET(pfcc_reg, in, pptx, tx_pause);
+	MLX5_SET(pfcc_reg, in, pprx, rx_pause);
+
+	return mlx5_core_access_reg(dev, in, sizeof(in), out,
+				   sizeof(out), MLX5_REG_PFCC, 0, 1);
+}
+
+int mlx5_query_port_pause(struct mlx5_core_dev *dev, u32 port,
+			  u32 *rx_pause, u32 *tx_pause)
+{
+	u32 in[MLX5_ST_SZ_DW(pfcc_reg)];
+	u32 out[MLX5_ST_SZ_DW(pfcc_reg)];
+	int err;
+
+	memset(in, 0, sizeof(in));
+	memset(out, 0, sizeof(out));
+
+	MLX5_SET(pfcc_reg, in, local_port, port);
+
+	err = mlx5_core_access_reg(dev, in, sizeof(in), out,
+				   sizeof(out), MLX5_REG_PFCC, 0, 0);
+	if (err)
+		return err;
+
+	*rx_pause = MLX5_GET(pfcc_reg, out, pprx);
+	*tx_pause = MLX5_GET(pfcc_reg, out, pptx);
+
+	return 0;
+}
+
+int mlx5_query_port_oper_mtu(struct mlx5_core_dev *dev, int *oper_mtu)
+{
+	return mlx5_query_port_mtu(dev, NULL, NULL, oper_mtu);
+}
+EXPORT_SYMBOL_GPL(mlx5_query_port_oper_mtu);
+
+u8 mlx5_is_wol_supported(struct mlx5_core_dev *dev)
+{
+	u8 wol_supported = 0;
+
+	if (MLX5_CAP_GEN(dev, wol_s))
+		wol_supported |= MLX5_WOL_SECURED_MAGIC;
+	if (MLX5_CAP_GEN(dev, wol_g))
+		wol_supported |= MLX5_WOL_MAGIC;
+	if (MLX5_CAP_GEN(dev, wol_a))
+		wol_supported |= MLX5_WOL_ARP;
+	if (MLX5_CAP_GEN(dev, wol_b))
+		wol_supported |= MLX5_WOL_BROADCAST;
+	if (MLX5_CAP_GEN(dev, wol_m))
+		wol_supported |= MLX5_WOL_MULTICAST;
+	if (MLX5_CAP_GEN(dev, wol_u))
+		wol_supported |= MLX5_WOL_UNICAST;
+	if (MLX5_CAP_GEN(dev, wol_p))
+		wol_supported |= MLX5_WOL_PHY_ACTIVITY;
+
+	return wol_supported;
+}
+EXPORT_SYMBOL_GPL(mlx5_is_wol_supported);
+
+int mlx5_set_wol(struct mlx5_core_dev *dev, u8 wol_mode)
+{
+	u32 in[MLX5_ST_SZ_DW(set_wol_rol_in)];
+	u32 out[MLX5_ST_SZ_DW(set_wol_rol_out)];
+
+	memset(in, 0, sizeof(in));
+	memset(out, 0, sizeof(out));
+
+	MLX5_SET(set_wol_rol_in, in, opcode, MLX5_CMD_OP_SET_WOL_ROL);
+	MLX5_SET(set_wol_rol_in, in, wol_mode_valid, 1);
+	MLX5_SET(set_wol_rol_in, in, wol_mode, wol_mode);
+
+	return mlx5_cmd_exec_check_status(dev, in, sizeof(in),
+					  out, sizeof(out));
+}
+EXPORT_SYMBOL_GPL(mlx5_set_wol);
+
+int mlx5_core_access_pvlc(struct mlx5_core_dev *dev,
+			  struct mlx5_pvlc_reg *pvlc, int write)
+{
+	int sz = MLX5_ST_SZ_BYTES(pvlc_reg);
+	u8 in[MLX5_ST_SZ_BYTES(pvlc_reg)];
+	u8 out[MLX5_ST_SZ_BYTES(pvlc_reg)];
+	int err;
+
+	memset(out, 0, sizeof(out));
+	memset(in, 0, sizeof(in));
+
+	MLX5_SET(pvlc_reg, in, local_port, pvlc->local_port);
+	if (write)
+		MLX5_SET(pvlc_reg, in, vl_admin, pvlc->vl_admin);
+
+	err = mlx5_core_access_reg(dev, in, sz, out, sz, MLX5_REG_PVLC, 0,
+				   !!write);
+	if (err)
+		return err;
+
+	if (!write) {
+		pvlc->local_port = MLX5_GET(pvlc_reg, out, local_port);
+		pvlc->vl_hw_cap = MLX5_GET(pvlc_reg, out, vl_hw_cap);
+		pvlc->vl_admin = MLX5_GET(pvlc_reg, out, vl_admin);
+		pvlc->vl_operational = MLX5_GET(pvlc_reg, out, vl_operational);
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(mlx5_core_access_pvlc);
+
+int mlx5_core_access_ptys(struct mlx5_core_dev *dev,
+			  struct mlx5_ptys_reg *ptys, int write)
+{
+	int sz = MLX5_ST_SZ_BYTES(ptys_reg);
+	void *out = NULL;
+	void *in = NULL;
+	int err;
+
+	in = mlx5_vzalloc(sz);
+	if (!in)
+		return -ENOMEM;
+
+	out = mlx5_vzalloc(sz);
+	if (!out) {
+		kfree(in);
+		return -ENOMEM;
+	}
+
+	MLX5_SET(ptys_reg, in, local_port, ptys->local_port);
+	MLX5_SET(ptys_reg, in, proto_mask, ptys->proto_mask);
+	if (write) {
+		MLX5_SET(ptys_reg, in, eth_proto_capability,
+			 ptys->eth_proto_cap);
+		MLX5_SET(ptys_reg, in, ib_link_width_capability,
+			 ptys->ib_link_width_cap);
+		MLX5_SET(ptys_reg, in, ib_proto_capability,
+			 ptys->ib_proto_cap);
+		MLX5_SET(ptys_reg, in, eth_proto_admin, ptys->eth_proto_admin);
+		MLX5_SET(ptys_reg, in, ib_link_width_admin,
+			 ptys->ib_link_width_admin);
+		MLX5_SET(ptys_reg, in, ib_proto_admin, ptys->ib_proto_admin);
+		MLX5_SET(ptys_reg, in, eth_proto_oper, ptys->eth_proto_oper);
+		MLX5_SET(ptys_reg, in, ib_link_width_oper,
+			 ptys->ib_link_width_oper);
+		MLX5_SET(ptys_reg, in, ib_proto_oper, ptys->ib_proto_oper);
+		MLX5_SET(ptys_reg, in, eth_proto_lp_advertise,
+			 ptys->eth_proto_lp_advertise);
+	}
+
+	err = mlx5_core_access_reg(dev, in, sz, out, sz, MLX5_REG_PTYS, 0,
+				   !!write);
+	if (err)
+		goto out;
+
+	if (!write) {
+		ptys->local_port = MLX5_GET(ptys_reg, out, local_port);
+		ptys->proto_mask = MLX5_GET(ptys_reg, out, proto_mask);
+		ptys->eth_proto_cap = MLX5_GET(ptys_reg, out,
+					       eth_proto_capability);
+		ptys->ib_link_width_cap = MLX5_GET(ptys_reg, out,
+					   ib_link_width_capability);
+		ptys->ib_proto_cap = MLX5_GET(ptys_reg, out,
+					      ib_proto_capability);
+		ptys->eth_proto_admin = MLX5_GET(ptys_reg, out,
+						 eth_proto_admin);
+		ptys->ib_link_width_admin = MLX5_GET(ptys_reg, out,
+						     ib_link_width_admin);
+		ptys->ib_proto_admin = MLX5_GET(ptys_reg, out, ib_proto_admin);
+		ptys->eth_proto_oper = MLX5_GET(ptys_reg, out, eth_proto_oper);
+		ptys->ib_link_width_oper = MLX5_GET(ptys_reg, out,
+						    ib_link_width_oper);
+		ptys->ib_proto_oper = MLX5_GET(ptys_reg, out, ib_proto_oper);
+		ptys->eth_proto_lp_advertise = MLX5_GET(ptys_reg, out,
+							eth_proto_lp_advertise);
+	}
+
+out:
+	kvfree(in);
+	kvfree(out);
+	return err;
+}
+EXPORT_SYMBOL_GPL(mlx5_core_access_ptys);
+
+static int mtu_to_ib_mtu(int mtu)
+{
+	switch (mtu) {
+	case 256: return 1;
+	case 512: return 2;
+	case 1024: return 3;
+	case 2048: return 4;
+	case 4096: return 5;
+	default:
+		printf("mlx5_core: WARN: ""invalid mtu\n");
+		return -1;
+	}
+}
+
+int mlx5_core_access_pmtu(struct mlx5_core_dev *dev,
+			  struct mlx5_pmtu_reg *pmtu, int write)
+{
+	int sz = MLX5_ST_SZ_BYTES(pmtu_reg);
+	void *out = NULL;
+	void *in = NULL;
+	int err;
+
+	in = mlx5_vzalloc(sz);
+	if (!in)
+		return -ENOMEM;
+
+	out = mlx5_vzalloc(sz);
+	if (!out) {
+		kfree(in);
+		return -ENOMEM;
+	}
+
+	MLX5_SET(pmtu_reg, in, local_port, pmtu->local_port);
+	if (write)
+		MLX5_SET(pmtu_reg, in, admin_mtu, pmtu->admin_mtu);
+
+	err = mlx5_core_access_reg(dev, in, sz, out, sz, MLX5_REG_PMTU, 0,
+				   !!write);
+	if (err)
+		goto out;
+
+	if (!write) {
+		pmtu->local_port = MLX5_GET(pmtu_reg, out, local_port);
+		pmtu->max_mtu = mtu_to_ib_mtu(MLX5_GET(pmtu_reg, out,
+						       max_mtu));
+		pmtu->admin_mtu = mtu_to_ib_mtu(MLX5_GET(pmtu_reg, out,
+							 admin_mtu));
+		pmtu->oper_mtu = mtu_to_ib_mtu(MLX5_GET(pmtu_reg, out,
+							oper_mtu));
+	}
+
+out:
+	kvfree(in);
+	kvfree(out);
+	return err;
+}
+EXPORT_SYMBOL_GPL(mlx5_core_access_pmtu);
+
+int mlx5_query_module_num(struct mlx5_core_dev *dev, int *module_num)
+{
+	u32 in[MLX5_ST_SZ_DW(pmlp_reg)];
+	u32 out[MLX5_ST_SZ_DW(pmlp_reg)];
+	int lane = 0;
+	int err;
+
+	memset(in, 0, sizeof(in));
+
+	MLX5_SET(pmlp_reg, in, local_port, 1);
+
+	err = mlx5_core_access_reg(dev, in, sizeof(in), out,
+				   sizeof(out), MLX5_REG_PMLP, 0, 0);
+	if (err)
+		return err;
+
+	lane = MLX5_GET(pmlp_reg, out, lane0_module_mapping);
+	*module_num = lane & MLX5_EEPROM_IDENTIFIER_BYTE_MASK;
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(mlx5_query_module_num);
+
+int mlx5_query_eeprom(struct mlx5_core_dev *dev,
+		      int i2c_addr, int page_num, int device_addr,
+		      int size, int module_num, u32 *data, int *size_read)
+{
+	u32 in[MLX5_ST_SZ_DW(mcia_reg)];
+	u32 out[MLX5_ST_SZ_DW(mcia_reg)];
+	u32 *ptr = (u32 *)MLX5_ADDR_OF(mcia_reg, out, dword_0);
+	int status;
+	int err;
+
+	memset(in, 0, sizeof(in));
+	size = min_t(int, size, MLX5_EEPROM_MAX_BYTES);
+
+	MLX5_SET(mcia_reg, in, l, 0);
+	MLX5_SET(mcia_reg, in, module, module_num);
+	MLX5_SET(mcia_reg, in, i2c_device_address, i2c_addr);
+	MLX5_SET(mcia_reg, in, page_number, page_num);
+	MLX5_SET(mcia_reg, in, device_address, device_addr);
+	MLX5_SET(mcia_reg, in, size, size);
+
+	err = mlx5_core_access_reg(dev, in, sizeof(in), out,
+				   sizeof(out), MLX5_REG_MCIA, 0, 0);
+	if (err)
+		return err;
+
+	status = MLX5_GET(mcia_reg, out, status);
+	if (status)
+		return status;
+
+	memcpy(data, ptr, size);
+	*size_read = size;
+	return 0;
+}
+EXPORT_SYMBOL_GPL(mlx5_query_eeprom);
+
+int mlx5_vxlan_udp_port_add(struct mlx5_core_dev *dev, u16 port)
+{
+	u32 in[MLX5_ST_SZ_DW(add_vxlan_udp_dport_in)];
+	u32 out[MLX5_ST_SZ_DW(add_vxlan_udp_dport_out)];
+	int err;
+
+	memset(in, 0, sizeof(in));
+	memset(out, 0, sizeof(out));
+
+	MLX5_SET(add_vxlan_udp_dport_in, in, opcode,
+		 MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT);
+	MLX5_SET(add_vxlan_udp_dport_in, in, vxlan_udp_port, port);
+
+	err = mlx5_cmd_exec_check_status(dev, in, sizeof(in), out, sizeof(out));
+	if (err) {
+		mlx5_core_err(dev, "Failed %s, port %u, err - %d",
+			      mlx5_command_str(MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT),
+			      port, err);
+	}
+
+	return err;
+}
+
+int mlx5_vxlan_udp_port_delete(struct mlx5_core_dev *dev, u16 port)
+{
+	u32 in[MLX5_ST_SZ_DW(delete_vxlan_udp_dport_in)];
+	u32 out[MLX5_ST_SZ_DW(delete_vxlan_udp_dport_out)];
+	int err;
+
+	memset(in, 0, sizeof(in));
+	memset(out, 0, sizeof(out));
+
+	MLX5_SET(delete_vxlan_udp_dport_in, in, opcode,
+		 MLX5_CMD_OP_DELETE_VXLAN_UDP_DPORT);
+	MLX5_SET(delete_vxlan_udp_dport_in, in, vxlan_udp_port, port);
+
+	err = mlx5_cmd_exec_check_status(dev, in, sizeof(in), out, sizeof(out));
+	if (err) {
+		mlx5_core_err(dev, "Failed %s, port %u, err - %d",
+			      mlx5_command_str(MLX5_CMD_OP_DELETE_VXLAN_UDP_DPORT),
+			      port, err);
+	}
+
+	return err;
+}
+
+int mlx5_query_wol(struct mlx5_core_dev *dev, u8 *wol_mode)
+{
+	u32 in[MLX5_ST_SZ_DW(query_wol_rol_in)];
+	u32 out[MLX5_ST_SZ_DW(query_wol_rol_out)];
+	int err;
+
+	memset(in, 0, sizeof(in));
+	memset(out, 0, sizeof(out));
+
+	MLX5_SET(query_wol_rol_in, in, opcode, MLX5_CMD_OP_QUERY_WOL_ROL);
+
+	err = mlx5_cmd_exec_check_status(dev, in, sizeof(in), out, sizeof(out));
+
+	if (!err)
+		*wol_mode = MLX5_GET(query_wol_rol_out, out, wol_mode);
+
+	return err;
+}
+EXPORT_SYMBOL_GPL(mlx5_query_wol);
+
+int mlx5_query_port_cong_status(struct mlx5_core_dev *mdev, int protocol,
+				int priority, int *is_enable)
+{
+	u32 in[MLX5_ST_SZ_DW(query_cong_status_in)];
+	u32 out[MLX5_ST_SZ_DW(query_cong_status_out)];
+	int err;
+
+	memset(in, 0, sizeof(in));
+	memset(out, 0, sizeof(out));
+
+	*is_enable = 0;
+
+	MLX5_SET(query_cong_status_in, in, opcode,
+		 MLX5_CMD_OP_QUERY_CONG_STATUS);
+	MLX5_SET(query_cong_status_in, in, cong_protocol, protocol);
+	MLX5_SET(query_cong_status_in, in, priority, priority);
+
+	err = mlx5_cmd_exec_check_status(mdev, in, sizeof(in),
+					 out, sizeof(out));
+	if (!err)
+		*is_enable = MLX5_GET(query_cong_status_out, out, enable);
+	return err;
+}
+
+int mlx5_modify_port_cong_status(struct mlx5_core_dev *mdev, int protocol,
+				 int priority, int enable)
+{
+	u32 in[MLX5_ST_SZ_DW(modify_cong_status_in)];
+	u32 out[MLX5_ST_SZ_DW(modify_cong_status_out)];
+
+	memset(in, 0, sizeof(in));
+	memset(out, 0, sizeof(out));
+
+	MLX5_SET(modify_cong_status_in, in, opcode,
+		 MLX5_CMD_OP_MODIFY_CONG_STATUS);
+	MLX5_SET(modify_cong_status_in, in, cong_protocol, protocol);
+	MLX5_SET(modify_cong_status_in, in, priority, priority);
+	MLX5_SET(modify_cong_status_in, in, enable, enable);
+
+	return mlx5_cmd_exec_check_status(mdev, in, sizeof(in),
+					  out, sizeof(out));
+}
+
+int mlx5_query_port_cong_params(struct mlx5_core_dev *mdev, int protocol,
+				void *out, int out_size)
+{
+	u32 in[MLX5_ST_SZ_DW(query_cong_params_in)];
+
+	memset(in, 0, sizeof(in));
+
+	MLX5_SET(query_cong_params_in, in, opcode,
+		 MLX5_CMD_OP_QUERY_CONG_PARAMS);
+	MLX5_SET(query_cong_params_in, in, cong_protocol, protocol);
+
+	return mlx5_cmd_exec_check_status(mdev, in, sizeof(in),
+					  out, out_size);
+}
+
+int mlx5_modify_port_cong_params(struct mlx5_core_dev *mdev,
+				 void *in, int in_size)
+{
+	u32 out[MLX5_ST_SZ_DW(modify_cong_params_out)];
+
+	memset(out, 0, sizeof(out));
+
+	MLX5_SET(modify_cong_params_in, in, opcode,
+		 MLX5_CMD_OP_MODIFY_CONG_PARAMS);
+
+	return mlx5_cmd_exec_check_status(mdev, in, in_size, out, sizeof(out));
+}
+
+int mlx5_query_port_cong_statistics(struct mlx5_core_dev *mdev, int clear,
+				    void *out, int out_size)
+{
+	u32 in[MLX5_ST_SZ_DW(query_cong_statistics_in)];
+
+	memset(in, 0, sizeof(in));
+
+	MLX5_SET(query_cong_statistics_in, in, opcode,
+		 MLX5_CMD_OP_QUERY_CONG_STATISTICS);
+	MLX5_SET(query_cong_statistics_in, in, clear, clear);
+
+	return mlx5_cmd_exec_check_status(mdev, in, sizeof(in),
+					  out, out_size);
+}
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_qp.c b/sys/dev/mlx5/mlx5_core/mlx5_qp.c
new file mode 100644
index 0000000..c106abe
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_core/mlx5_qp.c
@@ -0,0 +1,308 @@
+/*-
+ * Copyright (c) 2013-2015, Mellanox Technologies, Ltd.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+
+#include <linux/gfp.h>
+#include <dev/mlx5/qp.h>
+#include <dev/mlx5/driver.h>
+
+#include "mlx5_core.h"
+
+static struct mlx5_core_rsc_common *mlx5_get_rsc(struct mlx5_core_dev *dev,
+						 u32 rsn)
+{
+	struct mlx5_qp_table *table = &dev->priv.qp_table;
+	struct mlx5_core_rsc_common *common;
+
+	spin_lock(&table->lock);
+
+	common = radix_tree_lookup(&table->tree, rsn);
+	if (common)
+		atomic_inc(&common->refcount);
+
+	spin_unlock(&table->lock);
+
+	if (!common) {
+		mlx5_core_warn(dev, "Async event for bogus resource 0x%x\n",
+			       rsn);
+		return NULL;
+	}
+	return common;
+}
+
+void mlx5_core_put_rsc(struct mlx5_core_rsc_common *common)
+{
+	if (atomic_dec_and_test(&common->refcount))
+		complete(&common->free);
+}
+
+void mlx5_rsc_event(struct mlx5_core_dev *dev, u32 rsn, int event_type)
+{
+	struct mlx5_core_rsc_common *common = mlx5_get_rsc(dev, rsn);
+	struct mlx5_core_qp *qp;
+
+	if (!common)
+		return;
+
+	switch (common->res) {
+	case MLX5_RES_QP:
+		qp = (struct mlx5_core_qp *)common;
+		qp->event(qp, event_type);
+		break;
+
+	default:
+		mlx5_core_warn(dev, "invalid resource type for 0x%x\n", rsn);
+	}
+
+	mlx5_core_put_rsc(common);
+}
+
+int mlx5_core_create_qp(struct mlx5_core_dev *dev,
+			struct mlx5_core_qp *qp,
+			struct mlx5_create_qp_mbox_in *in,
+			int inlen)
+{
+	struct mlx5_qp_table *table = &dev->priv.qp_table;
+	struct mlx5_create_qp_mbox_out out;
+	struct mlx5_destroy_qp_mbox_in din;
+	struct mlx5_destroy_qp_mbox_out dout;
+	int err;
+	void *qpc;
+
+	memset(&out, 0, sizeof(out));
+	in->hdr.opcode = cpu_to_be16(MLX5_CMD_OP_CREATE_QP);
+	if (dev->issi) {
+		qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
+		/* 0xffffff means we ask to work with cqe version 0 */
+		MLX5_SET(qpc, qpc, user_index, 0xffffff);
+	}
+
+	err = mlx5_cmd_exec(dev, in, inlen, &out, sizeof(out));
+	if (err) {
+		mlx5_core_warn(dev, "ret %d\n", err);
+		return err;
+	}
+
+	if (out.hdr.status) {
+		mlx5_core_warn(dev, "current num of QPs 0x%x\n",
+			       atomic_read(&dev->num_qps));
+		return mlx5_cmd_status_to_err(&out.hdr);
+	}
+
+	qp->qpn = be32_to_cpu(out.qpn) & 0xffffff;
+	mlx5_core_dbg(dev, "qpn = 0x%x\n", qp->qpn);
+
+	qp->common.res = MLX5_RES_QP;
+	spin_lock_irq(&table->lock);
+	err = radix_tree_insert(&table->tree, qp->qpn, qp);
+	spin_unlock_irq(&table->lock);
+	if (err) {
+		mlx5_core_warn(dev, "err %d\n", err);
+		goto err_cmd;
+	}
+
+	qp->pid = curthread->td_proc->p_pid;
+	atomic_set(&qp->common.refcount, 1);
+	atomic_inc(&dev->num_qps);
+	init_completion(&qp->common.free);
+
+	return 0;
+
+err_cmd:
+	memset(&din, 0, sizeof(din));
+	memset(&dout, 0, sizeof(dout));
+	din.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_DESTROY_QP);
+	din.qpn = cpu_to_be32(qp->qpn);
+	mlx5_cmd_exec(dev, &din, sizeof(din), &out, sizeof(dout));
+
+	return err;
+}
+EXPORT_SYMBOL_GPL(mlx5_core_create_qp);
+
+int mlx5_core_destroy_qp(struct mlx5_core_dev *dev,
+			 struct mlx5_core_qp *qp)
+{
+	struct mlx5_destroy_qp_mbox_in in;
+	struct mlx5_destroy_qp_mbox_out out;
+	struct mlx5_qp_table *table = &dev->priv.qp_table;
+	unsigned long flags;
+	int err;
+
+
+	spin_lock_irqsave(&table->lock, flags);
+	radix_tree_delete(&table->tree, qp->qpn);
+	spin_unlock_irqrestore(&table->lock, flags);
+
+	mlx5_core_put_rsc((struct mlx5_core_rsc_common *)qp);
+	wait_for_completion(&qp->common.free);
+
+	memset(&in, 0, sizeof(in));
+	memset(&out, 0, sizeof(out));
+	in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_DESTROY_QP);
+	in.qpn = cpu_to_be32(qp->qpn);
+	err = mlx5_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out));
+	if (err)
+		return err;
+
+	if (out.hdr.status)
+		return mlx5_cmd_status_to_err(&out.hdr);
+
+	atomic_dec(&dev->num_qps);
+	return 0;
+}
+EXPORT_SYMBOL_GPL(mlx5_core_destroy_qp);
+
+int mlx5_core_qp_modify(struct mlx5_core_dev *dev, enum mlx5_qp_state cur_state,
+			enum mlx5_qp_state new_state,
+			struct mlx5_modify_qp_mbox_in *in, int sqd_event,
+			struct mlx5_core_qp *qp)
+{
+	static const u16 optab[MLX5_QP_NUM_STATE][MLX5_QP_NUM_STATE] = {
+		[MLX5_QP_STATE_RST] = {
+			[MLX5_QP_STATE_RST]	= MLX5_CMD_OP_2RST_QP,
+			[MLX5_QP_STATE_ERR]	= MLX5_CMD_OP_2ERR_QP,
+			[MLX5_QP_STATE_INIT]	= MLX5_CMD_OP_RST2INIT_QP,
+		},
+		[MLX5_QP_STATE_INIT]  = {
+			[MLX5_QP_STATE_RST]	= MLX5_CMD_OP_2RST_QP,
+			[MLX5_QP_STATE_ERR]	= MLX5_CMD_OP_2ERR_QP,
+			[MLX5_QP_STATE_INIT]	= MLX5_CMD_OP_INIT2INIT_QP,
+			[MLX5_QP_STATE_RTR]	= MLX5_CMD_OP_INIT2RTR_QP,
+		},
+		[MLX5_QP_STATE_RTR]   = {
+			[MLX5_QP_STATE_RST]	= MLX5_CMD_OP_2RST_QP,
+			[MLX5_QP_STATE_ERR]	= MLX5_CMD_OP_2ERR_QP,
+			[MLX5_QP_STATE_RTS]	= MLX5_CMD_OP_RTR2RTS_QP,
+		},
+		[MLX5_QP_STATE_RTS]   = {
+			[MLX5_QP_STATE_RST]	= MLX5_CMD_OP_2RST_QP,
+			[MLX5_QP_STATE_ERR]	= MLX5_CMD_OP_2ERR_QP,
+			[MLX5_QP_STATE_RTS]	= MLX5_CMD_OP_RTS2RTS_QP,
+		},
+		[MLX5_QP_STATE_SQD] = {
+			[MLX5_QP_STATE_RST]	= MLX5_CMD_OP_2RST_QP,
+			[MLX5_QP_STATE_ERR]	= MLX5_CMD_OP_2ERR_QP,
+		},
+		[MLX5_QP_STATE_SQER] = {
+			[MLX5_QP_STATE_RST]	= MLX5_CMD_OP_2RST_QP,
+			[MLX5_QP_STATE_ERR]	= MLX5_CMD_OP_2ERR_QP,
+			[MLX5_QP_STATE_RTS]	= MLX5_CMD_OP_SQERR2RTS_QP,
+		},
+		[MLX5_QP_STATE_ERR] = {
+			[MLX5_QP_STATE_RST]	= MLX5_CMD_OP_2RST_QP,
+			[MLX5_QP_STATE_ERR]	= MLX5_CMD_OP_2ERR_QP,
+		}
+	};
+
+	struct mlx5_modify_qp_mbox_out out;
+	int err = 0;
+	u16 op;
+
+	if (cur_state >= MLX5_QP_NUM_STATE || new_state >= MLX5_QP_NUM_STATE ||
+	    !optab[cur_state][new_state])
+		return -EINVAL;
+
+	memset(&out, 0, sizeof(out));
+	op = optab[cur_state][new_state];
+	in->hdr.opcode = cpu_to_be16(op);
+	in->qpn = cpu_to_be32(qp->qpn);
+	err = mlx5_cmd_exec(dev, in, sizeof(*in), &out, sizeof(out));
+	if (err)
+		return err;
+
+	return mlx5_cmd_status_to_err(&out.hdr);
+}
+EXPORT_SYMBOL_GPL(mlx5_core_qp_modify);
+
+void mlx5_init_qp_table(struct mlx5_core_dev *dev)
+{
+	struct mlx5_qp_table *table = &dev->priv.qp_table;
+
+	spin_lock_init(&table->lock);
+	INIT_RADIX_TREE(&table->tree, GFP_ATOMIC);
+}
+
+void mlx5_cleanup_qp_table(struct mlx5_core_dev *dev)
+{
+}
+
+int mlx5_core_qp_query(struct mlx5_core_dev *dev, struct mlx5_core_qp *qp,
+		       struct mlx5_query_qp_mbox_out *out, int outlen)
+{
+	struct mlx5_query_qp_mbox_in in;
+	int err;
+
+	memset(&in, 0, sizeof(in));
+	memset(out, 0, outlen);
+	in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_QUERY_QP);
+	in.qpn = cpu_to_be32(qp->qpn);
+	err = mlx5_cmd_exec(dev, &in, sizeof(in), out, outlen);
+	if (err)
+		return err;
+
+	if (out->hdr.status)
+		return mlx5_cmd_status_to_err(&out->hdr);
+
+	return err;
+}
+EXPORT_SYMBOL_GPL(mlx5_core_qp_query);
+
+int mlx5_core_xrcd_alloc(struct mlx5_core_dev *dev, u32 *xrcdn)
+{
+	u32 in[MLX5_ST_SZ_DW(alloc_xrcd_in)];
+	u32 out[MLX5_ST_SZ_DW(alloc_xrcd_out)];
+	int err;
+
+	memset(in, 0, sizeof(in));
+
+	MLX5_SET(alloc_xrcd_in, in, opcode, MLX5_CMD_OP_ALLOC_XRCD);
+
+	memset(out, 0, sizeof(out));
+	err = mlx5_cmd_exec_check_status(dev, in, sizeof(in), out, sizeof(out));
+	if (err)
+		return err;
+
+	*xrcdn = MLX5_GET(alloc_xrcd_out, out, xrcd);
+	return 0;
+}
+EXPORT_SYMBOL_GPL(mlx5_core_xrcd_alloc);
+
+int mlx5_core_xrcd_dealloc(struct mlx5_core_dev *dev, u32 xrcdn)
+{
+	u32 in[MLX5_ST_SZ_DW(dealloc_xrcd_in)];
+	u32 out[MLX5_ST_SZ_DW(dealloc_xrcd_out)];
+
+	memset(in, 0, sizeof(in));
+
+	MLX5_SET(dealloc_xrcd_in, in, opcode, MLX5_CMD_OP_DEALLOC_XRCD);
+	MLX5_SET(dealloc_xrcd_in, in, xrcd, xrcdn);
+
+	memset(out, 0, sizeof(out));
+	return mlx5_cmd_exec_check_status(dev, in,  sizeof(in),
+					       out, sizeof(out));
+}
+EXPORT_SYMBOL_GPL(mlx5_core_xrcd_dealloc);
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_srq.c b/sys/dev/mlx5/mlx5_core/mlx5_srq.c
new file mode 100644
index 0000000..3146111
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_core/mlx5_srq.c
@@ -0,0 +1,457 @@
+/*-
+ * Copyright (c) 2013-2015, Mellanox Technologies, Ltd.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <dev/mlx5/driver.h>
+#include <dev/mlx5/srq.h>
+#include <rdma/ib_verbs.h>
+#include "mlx5_core.h"
+#include "transobj.h"
+
+void mlx5_srq_event(struct mlx5_core_dev *dev, u32 srqn, int event_type)
+{
+	struct mlx5_srq_table *table = &dev->priv.srq_table;
+	struct mlx5_core_srq *srq;
+
+	spin_lock(&table->lock);
+
+	srq = radix_tree_lookup(&table->tree, srqn);
+	if (srq)
+		atomic_inc(&srq->refcount);
+
+	spin_unlock(&table->lock);
+
+	if (!srq) {
+		mlx5_core_warn(dev, "Async event for bogus SRQ 0x%08x\n", srqn);
+		return;
+	}
+
+	srq->event(srq, event_type);
+
+	if (atomic_dec_and_test(&srq->refcount))
+		complete(&srq->free);
+}
+
+static void rmpc_srqc_reformat(void *srqc, void *rmpc, bool srqc_to_rmpc)
+{
+	void *wq = MLX5_ADDR_OF(rmpc, rmpc, wq);
+
+	if (srqc_to_rmpc) {
+		switch (MLX5_GET(srqc, srqc, state)) {
+		case MLX5_SRQC_STATE_GOOD:
+			MLX5_SET(rmpc, rmpc, state, MLX5_RMPC_STATE_RDY);
+			break;
+		case MLX5_SRQC_STATE_ERROR:
+			MLX5_SET(rmpc, rmpc, state, MLX5_RMPC_STATE_ERR);
+			break;
+		default:
+			printf("mlx5_core: WARN: ""%s: %d: Unknown srq state = 0x%x\n", __func__, __LINE__, MLX5_GET(srqc, srqc, state));
+		}
+
+		MLX5_SET(wq,   wq, wq_signature,  MLX5_GET(srqc, srqc, wq_signature));
+		MLX5_SET(wq,   wq, log_wq_pg_sz,  MLX5_GET(srqc, srqc, log_page_size));
+		MLX5_SET(wq,   wq, log_wq_stride, MLX5_GET(srqc, srqc, log_rq_stride) + 4);
+		MLX5_SET(wq,   wq, log_wq_sz,     MLX5_GET(srqc, srqc, log_srq_size));
+		MLX5_SET(wq,   wq, page_offset,   MLX5_GET(srqc, srqc, page_offset));
+		MLX5_SET(wq,   wq, lwm,           MLX5_GET(srqc, srqc, lwm));
+		MLX5_SET(wq,   wq, pd,            MLX5_GET(srqc, srqc, pd));
+		MLX5_SET64(wq, wq, dbr_addr,
+			   ((u64)MLX5_GET(srqc, srqc, db_record_addr_h)) << 32 |
+			   ((u64)MLX5_GET(srqc, srqc, db_record_addr_l)) << 2);
+	} else {
+		switch (MLX5_GET(rmpc, rmpc, state)) {
+		case MLX5_RMPC_STATE_RDY:
+			MLX5_SET(srqc, srqc, state, MLX5_SRQC_STATE_GOOD);
+			break;
+		case MLX5_RMPC_STATE_ERR:
+			MLX5_SET(srqc, srqc, state, MLX5_SRQC_STATE_ERROR);
+			break;
+		default:
+			printf("mlx5_core: WARN: ""%s: %d: Unknown rmp state = 0x%x\n", __func__, __LINE__, MLX5_GET(rmpc, rmpc, state));
+		}
+
+		MLX5_SET(srqc, srqc, wq_signature,     MLX5_GET(wq,   wq, wq_signature));
+		MLX5_SET(srqc, srqc, log_page_size,    MLX5_GET(wq,   wq, log_wq_pg_sz));
+		MLX5_SET(srqc, srqc, log_rq_stride,    MLX5_GET(wq,   wq, log_wq_stride) - 4);
+		MLX5_SET(srqc, srqc, log_srq_size,     MLX5_GET(wq,   wq, log_wq_sz));
+		MLX5_SET(srqc, srqc, page_offset,      MLX5_GET(wq,   wq, page_offset));
+		MLX5_SET(srqc, srqc, lwm,	       MLX5_GET(wq,   wq, lwm));
+		MLX5_SET(srqc, srqc, pd,	       MLX5_GET(wq,   wq, pd));
+		MLX5_SET(srqc, srqc, db_record_addr_h, MLX5_GET64(wq, wq, dbr_addr) >> 32);
+		MLX5_SET(srqc, srqc, db_record_addr_l, (MLX5_GET64(wq, wq, dbr_addr) >> 2) & 0x3fffffff);
+	}
+}
+
+struct mlx5_core_srq *mlx5_core_get_srq(struct mlx5_core_dev *dev, u32 srqn)
+{
+	struct mlx5_srq_table *table = &dev->priv.srq_table;
+	struct mlx5_core_srq *srq;
+
+	spin_lock(&table->lock);
+
+	srq = radix_tree_lookup(&table->tree, srqn);
+	if (srq)
+		atomic_inc(&srq->refcount);
+
+	spin_unlock(&table->lock);
+
+	return srq;
+}
+EXPORT_SYMBOL(mlx5_core_get_srq);
+
+static int get_pas_size(void *srqc)
+{
+	u32 log_page_size = MLX5_GET(srqc, srqc, log_page_size) + 12;
+	u32 log_srq_size  = MLX5_GET(srqc, srqc, log_srq_size);
+	u32 log_rq_stride = MLX5_GET(srqc, srqc, log_rq_stride);
+	u32 page_offset   = MLX5_GET(srqc, srqc, page_offset);
+	u32 po_quanta	  = 1 << (log_page_size - 6);
+	u32 rq_sz	  = 1 << (log_srq_size + 4 + log_rq_stride);
+	u32 page_size	  = 1 << log_page_size;
+	u32 rq_sz_po      = rq_sz + (page_offset * po_quanta);
+	u32 rq_num_pas	  = (rq_sz_po + page_size - 1) / page_size;
+
+	return rq_num_pas * sizeof(u64);
+
+}
+
+static int create_rmp_cmd(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,
+			  struct mlx5_create_srq_mbox_in *in, int srq_inlen)
+{
+	void *create_in;
+	void *rmpc;
+	void *srqc;
+	int pas_size;
+	int inlen;
+	int err;
+
+	srqc = MLX5_ADDR_OF(create_srq_in, in, srq_context_entry);
+	pas_size = get_pas_size(srqc);
+	inlen = MLX5_ST_SZ_BYTES(create_rmp_in) + pas_size;
+	create_in = mlx5_vzalloc(inlen);
+	if (!create_in)
+		return -ENOMEM;
+
+	rmpc = MLX5_ADDR_OF(create_rmp_in, create_in, ctx);
+
+	memcpy(MLX5_ADDR_OF(rmpc, rmpc, wq.pas), in->pas, pas_size);
+	rmpc_srqc_reformat(srqc, rmpc, true);
+
+	err = mlx5_core_create_rmp(dev, create_in, inlen, &srq->srqn);
+
+	kvfree(create_in);
+	return err;
+}
+
+static int destroy_rmp_cmd(struct mlx5_core_dev *dev,
+			    struct mlx5_core_srq *srq)
+{
+	return mlx5_core_destroy_rmp(dev, srq->srqn);
+}
+
+static int query_rmp_cmd(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,
+			 struct mlx5_query_srq_mbox_out *out)
+{
+	u32 *rmp_out;
+	void *rmpc;
+	void *srqc;
+	int err;
+
+	rmp_out =  mlx5_vzalloc(MLX5_ST_SZ_BYTES(query_rmp_out));
+	if (!rmp_out)
+		return -ENOMEM;
+
+	err = mlx5_core_query_rmp(dev, srq->srqn, rmp_out);
+	if (err)
+		goto out;
+
+	srqc = MLX5_ADDR_OF(query_srq_out, out,	    srq_context_entry);
+	rmpc = MLX5_ADDR_OF(query_rmp_out, rmp_out, rmp_context);
+	rmpc_srqc_reformat(srqc, rmpc, false);
+
+out:
+	kvfree(rmp_out);
+	return 0;
+}
+
+static int arm_rmp_cmd(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq, u16 lwm)
+{
+	return mlx5_core_arm_rmp(dev, srq->srqn, lwm);
+}
+
+static int create_xrc_srq_cmd(struct mlx5_core_dev *dev,
+			      struct mlx5_core_srq *srq,
+			      struct mlx5_create_srq_mbox_in *in,
+			      int srq_inlen)
+{
+	void *create_in;
+	void *srqc;
+	void *xrc_srqc;
+	void *pas;
+	int pas_size;
+	int inlen;
+	int err;
+
+	srqc	  = MLX5_ADDR_OF(create_srq_in, in, srq_context_entry);
+	pas_size  = get_pas_size(srqc);
+	inlen	  = MLX5_ST_SZ_BYTES(create_xrc_srq_in) + pas_size;
+	create_in = mlx5_vzalloc(inlen);
+	if (!create_in)
+		return -ENOMEM;
+
+	xrc_srqc = MLX5_ADDR_OF(create_xrc_srq_in, create_in, xrc_srq_context_entry);
+	pas	 = MLX5_ADDR_OF(create_xrc_srq_in, create_in, pas);
+
+	memcpy(xrc_srqc, srqc, MLX5_ST_SZ_BYTES(srqc));
+	memcpy(pas, in->pas, pas_size);
+	/* 0xffffff means we ask to work with cqe version 0 */
+	MLX5_SET(xrc_srqc, xrc_srqc,  user_index, 0xffffff);
+
+	err = mlx5_core_create_xsrq(dev, create_in, inlen, &srq->srqn);
+	if (err)
+		goto out;
+
+out:
+	kvfree(create_in);
+	return err;
+}
+
+static int destroy_xrc_srq_cmd(struct mlx5_core_dev *dev,
+			       struct mlx5_core_srq *srq)
+{
+	return mlx5_core_destroy_xsrq(dev, srq->srqn);
+}
+
+static int query_xrc_srq_cmd(struct mlx5_core_dev *dev,
+			     struct mlx5_core_srq *srq,
+			     struct mlx5_query_srq_mbox_out *out)
+{
+	u32 *xrcsrq_out;
+	int err;
+
+	xrcsrq_out = mlx5_vzalloc(MLX5_ST_SZ_BYTES(query_xrc_srq_out));
+	if (!xrcsrq_out)
+		return -ENOMEM;
+
+	err = mlx5_core_query_xsrq(dev, srq->srqn, xrcsrq_out);
+	if (err)
+		goto out;
+
+out:
+	kvfree(xrcsrq_out);
+	return err;
+}
+
+static int arm_xrc_srq_cmd(struct mlx5_core_dev *dev,
+			   struct mlx5_core_srq *srq, u16 lwm)
+{
+	return mlx5_core_arm_xsrq(dev, srq->srqn, lwm);
+}
+
+static int create_srq_cmd(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,
+			  struct mlx5_create_srq_mbox_in *in, int inlen)
+{
+	struct mlx5_create_srq_mbox_out out;
+	int err;
+
+	memset(&out, 0, sizeof(out));
+
+	in->hdr.opcode = cpu_to_be16(MLX5_CMD_OP_CREATE_SRQ);
+
+	err = mlx5_cmd_exec_check_status(dev, (u32 *)in, inlen, (u32 *)(&out), sizeof(out));
+
+	srq->srqn = be32_to_cpu(out.srqn) & 0xffffff;
+
+	return err;
+}
+
+static int destroy_srq_cmd(struct mlx5_core_dev *dev,
+			   struct mlx5_core_srq *srq)
+{
+	struct mlx5_destroy_srq_mbox_in in;
+	struct mlx5_destroy_srq_mbox_out out;
+
+	memset(&in, 0, sizeof(in));
+	memset(&out, 0, sizeof(out));
+	in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_DESTROY_SRQ);
+	in.srqn = cpu_to_be32(srq->srqn);
+
+	return mlx5_cmd_exec_check_status(dev, (u32 *)(&in), sizeof(in), (u32 *)(&out), sizeof(out));
+}
+
+static int query_srq_cmd(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,
+			 struct mlx5_query_srq_mbox_out *out)
+{
+	struct mlx5_query_srq_mbox_in in;
+
+	memset(&in, 0, sizeof(in));
+
+	in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_QUERY_SRQ);
+	in.srqn = cpu_to_be32(srq->srqn);
+
+	return mlx5_cmd_exec_check_status(dev, (u32 *)(&in), sizeof(in), (u32 *)out, sizeof(*out));
+}
+
+static int arm_srq_cmd(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,
+		       u16 lwm, int is_srq)
+{
+	struct mlx5_arm_srq_mbox_in	in;
+	struct mlx5_arm_srq_mbox_out	out;
+
+	memset(&in, 0, sizeof(in));
+	memset(&out, 0, sizeof(out));
+
+	in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_ARM_RQ);
+	in.hdr.opmod = cpu_to_be16(!!is_srq);
+	in.srqn = cpu_to_be32(srq->srqn);
+	in.lwm = cpu_to_be16(lwm);
+
+	return mlx5_cmd_exec_check_status(dev, (u32 *)(&in), sizeof(in), (u32 *)(&out), sizeof(out));
+}
+
+static int create_srq_split(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,
+			    struct mlx5_create_srq_mbox_in *in, int inlen,
+			    int is_xrc)
+{
+	if (!dev->issi)
+		return create_srq_cmd(dev, srq, in, inlen);
+	else if (srq->common.res == MLX5_RES_XSRQ)
+		return create_xrc_srq_cmd(dev, srq, in, inlen);
+	else
+		return create_rmp_cmd(dev, srq, in, inlen);
+}
+
+static int destroy_srq_split(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq)
+{
+	if (!dev->issi)
+		return destroy_srq_cmd(dev, srq);
+	else if (srq->common.res == MLX5_RES_XSRQ)
+		return destroy_xrc_srq_cmd(dev, srq);
+	else
+		return destroy_rmp_cmd(dev, srq);
+}
+
+int mlx5_core_create_srq(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,
+			 struct mlx5_create_srq_mbox_in *in, int inlen,
+			 int is_xrc)
+{
+	int err;
+	struct mlx5_srq_table *table = &dev->priv.srq_table;
+
+	srq->common.res = is_xrc ? MLX5_RES_XSRQ : MLX5_RES_SRQ;
+
+	err = create_srq_split(dev, srq, in, inlen, is_xrc);
+	if (err)
+		return err;
+
+	atomic_set(&srq->refcount, 1);
+	init_completion(&srq->free);
+
+	spin_lock_irq(&table->lock);
+	err = radix_tree_insert(&table->tree, srq->srqn, srq);
+	spin_unlock_irq(&table->lock);
+	if (err) {
+		mlx5_core_warn(dev, "err %d, srqn 0x%x\n", err, srq->srqn);
+		goto err_destroy_srq_split;
+	}
+
+	return 0;
+
+err_destroy_srq_split:
+	destroy_srq_split(dev, srq);
+
+	return err;
+}
+EXPORT_SYMBOL(mlx5_core_create_srq);
+
+int mlx5_core_destroy_srq(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq)
+{
+	struct mlx5_srq_table *table = &dev->priv.srq_table;
+	struct mlx5_core_srq *tmp;
+	int err;
+
+	spin_lock_irq(&table->lock);
+	tmp = radix_tree_delete(&table->tree, srq->srqn);
+	spin_unlock_irq(&table->lock);
+	if (!tmp) {
+		mlx5_core_warn(dev, "srq 0x%x not found in tree\n", srq->srqn);
+		return -EINVAL;
+	}
+	if (tmp != srq) {
+		mlx5_core_warn(dev, "corruption on srqn 0x%x\n", srq->srqn);
+		return -EINVAL;
+	}
+
+	err = destroy_srq_split(dev, srq);
+	if (err)
+		return err;
+
+	if (atomic_dec_and_test(&srq->refcount))
+		complete(&srq->free);
+	wait_for_completion(&srq->free);
+
+	return 0;
+}
+EXPORT_SYMBOL(mlx5_core_destroy_srq);
+
+int mlx5_core_query_srq(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,
+			struct mlx5_query_srq_mbox_out *out)
+{
+	if (!dev->issi)
+		return query_srq_cmd(dev, srq, out);
+	else if (srq->common.res == MLX5_RES_XSRQ)
+		return query_xrc_srq_cmd(dev, srq, out);
+	else
+		return query_rmp_cmd(dev, srq, out);
+}
+EXPORT_SYMBOL(mlx5_core_query_srq);
+
+int mlx5_core_arm_srq(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,
+		      u16 lwm, int is_srq)
+{
+	if (!dev->issi)
+		return arm_srq_cmd(dev, srq, lwm, is_srq);
+	else if (srq->common.res == MLX5_RES_XSRQ)
+		return arm_xrc_srq_cmd(dev, srq, lwm);
+	else
+		return arm_rmp_cmd(dev, srq, lwm);
+}
+EXPORT_SYMBOL(mlx5_core_arm_srq);
+
+void mlx5_init_srq_table(struct mlx5_core_dev *dev)
+{
+	struct mlx5_srq_table *table = &dev->priv.srq_table;
+
+	spin_lock_init(&table->lock);
+	INIT_RADIX_TREE(&table->tree, GFP_ATOMIC);
+}
+
+void mlx5_cleanup_srq_table(struct mlx5_core_dev *dev)
+{
+	/* nothing */
+}
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_transobj.c b/sys/dev/mlx5/mlx5_core/mlx5_transobj.c
new file mode 100644
index 0000000..0039b89
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_core/mlx5_transobj.c
@@ -0,0 +1,393 @@
+/*-
+ * Copyright (c) 2013-2015, Mellanox Technologies, Ltd.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <dev/mlx5/driver.h>
+
+#include "mlx5_core.h"
+#include "transobj.h"
+
+int mlx5_alloc_transport_domain(struct mlx5_core_dev *dev, u32 *tdn)
+{
+	u32 in[MLX5_ST_SZ_DW(alloc_transport_domain_in)];
+	u32 out[MLX5_ST_SZ_DW(alloc_transport_domain_out)];
+	int err;
+
+	memset(in, 0, sizeof(in));
+	memset(out, 0, sizeof(out));
+
+	MLX5_SET(alloc_transport_domain_in, in, opcode,
+		 MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN);
+
+	err = mlx5_cmd_exec_check_status(dev, in, sizeof(in), out, sizeof(out));
+	if (!err)
+		*tdn = MLX5_GET(alloc_transport_domain_out, out,
+				transport_domain);
+
+	return err;
+}
+
+void mlx5_dealloc_transport_domain(struct mlx5_core_dev *dev, u32 tdn)
+{
+	u32 in[MLX5_ST_SZ_DW(dealloc_transport_domain_in)];
+	u32 out[MLX5_ST_SZ_DW(dealloc_transport_domain_out)];
+
+	memset(in, 0, sizeof(in));
+	memset(out, 0, sizeof(out));
+
+	MLX5_SET(dealloc_transport_domain_in, in, opcode,
+		 MLX5_CMD_OP_DEALLOC_TRANSPORT_DOMAIN);
+	MLX5_SET(dealloc_transport_domain_in, in, transport_domain, tdn);
+
+	mlx5_cmd_exec_check_status(dev, in, sizeof(in), out, sizeof(out));
+}
+
+int mlx5_core_create_rq(struct mlx5_core_dev *dev, u32 *in, int inlen, u32 *rqn)
+{
+	u32 out[MLX5_ST_SZ_DW(create_rq_out)];
+	int err;
+
+	MLX5_SET(create_rq_in, in, opcode, MLX5_CMD_OP_CREATE_RQ);
+
+	memset(out, 0, sizeof(out));
+	err = mlx5_cmd_exec_check_status(dev, in, inlen, out, sizeof(out));
+	if (!err)
+		*rqn = MLX5_GET(create_rq_out, out, rqn);
+
+	return err;
+}
+
+int mlx5_core_modify_rq(struct mlx5_core_dev *dev, u32 *in, int inlen)
+{
+	u32 out[MLX5_ST_SZ_DW(modify_rq_out)];
+
+	MLX5_SET(modify_rq_in, in, opcode, MLX5_CMD_OP_MODIFY_RQ);
+
+	memset(out, 0, sizeof(out));
+	return mlx5_cmd_exec_check_status(dev, in, inlen, out, sizeof(out));
+}
+
+void mlx5_core_destroy_rq(struct mlx5_core_dev *dev, u32 rqn)
+{
+	u32 in[MLX5_ST_SZ_DW(destroy_rq_in)];
+	u32 out[MLX5_ST_SZ_DW(destroy_rq_out)];
+
+	memset(in, 0, sizeof(in));
+
+	MLX5_SET(destroy_rq_in, in, opcode, MLX5_CMD_OP_DESTROY_RQ);
+	MLX5_SET(destroy_rq_in, in, rqn, rqn);
+
+	mlx5_cmd_exec_check_status(dev, in, sizeof(in), out, sizeof(out));
+}
+
+int mlx5_core_create_sq(struct mlx5_core_dev *dev, u32 *in, int inlen, u32 *sqn)
+{
+	u32 out[MLX5_ST_SZ_DW(create_sq_out)];
+	int err;
+
+	MLX5_SET(create_sq_in, in, opcode, MLX5_CMD_OP_CREATE_SQ);
+
+	memset(out, 0, sizeof(out));
+	err = mlx5_cmd_exec_check_status(dev, in, inlen, out, sizeof(out));
+	if (!err)
+		*sqn = MLX5_GET(create_sq_out, out, sqn);
+
+	return err;
+}
+
+int mlx5_core_modify_sq(struct mlx5_core_dev *dev, u32 *in, int inlen)
+{
+	u32 out[MLX5_ST_SZ_DW(modify_sq_out)];
+
+	MLX5_SET(modify_sq_in, in, opcode, MLX5_CMD_OP_MODIFY_SQ);
+
+	memset(out, 0, sizeof(out));
+	return mlx5_cmd_exec_check_status(dev, in, inlen, out, sizeof(out));
+}
+
+void mlx5_core_destroy_sq(struct mlx5_core_dev *dev, u32 sqn)
+{
+	u32 in[MLX5_ST_SZ_DW(destroy_sq_in)];
+	u32 out[MLX5_ST_SZ_DW(destroy_sq_out)];
+
+	memset(in, 0, sizeof(in));
+
+	MLX5_SET(destroy_sq_in, in, opcode, MLX5_CMD_OP_DESTROY_SQ);
+	MLX5_SET(destroy_sq_in, in, sqn, sqn);
+
+	mlx5_cmd_exec_check_status(dev, in, sizeof(in), out, sizeof(out));
+}
+
+int mlx5_core_create_tir(struct mlx5_core_dev *dev, u32 *in, int inlen,
+			 u32 *tirn)
+{
+	u32 out[MLX5_ST_SZ_DW(create_tir_out)];
+	int err;
+
+	MLX5_SET(create_tir_in, in, opcode, MLX5_CMD_OP_CREATE_TIR);
+
+	memset(out, 0, sizeof(out));
+	err = mlx5_cmd_exec_check_status(dev, in, inlen, out, sizeof(out));
+	if (!err)
+		*tirn = MLX5_GET(create_tir_out, out, tirn);
+
+	return err;
+}
+
+void mlx5_core_destroy_tir(struct mlx5_core_dev *dev, u32 tirn)
+{
+	u32 in[MLX5_ST_SZ_DW(destroy_tir_in)];
+	u32 out[MLX5_ST_SZ_DW(destroy_tir_out)];
+
+	memset(in, 0, sizeof(in));
+
+	MLX5_SET(destroy_tir_in, in, opcode, MLX5_CMD_OP_DESTROY_TIR);
+	MLX5_SET(destroy_tir_in, in, tirn, tirn);
+
+	mlx5_cmd_exec_check_status(dev, in, sizeof(in), out, sizeof(out));
+}
+
+int mlx5_core_create_tis(struct mlx5_core_dev *dev, u32 *in, int inlen,
+			 u32 *tisn)
+{
+	u32 out[MLX5_ST_SZ_DW(create_tis_out)];
+	int err;
+
+	MLX5_SET(create_tis_in, in, opcode, MLX5_CMD_OP_CREATE_TIS);
+
+	memset(out, 0, sizeof(out));
+	err = mlx5_cmd_exec_check_status(dev, in, inlen, out, sizeof(out));
+	if (!err)
+		*tisn = MLX5_GET(create_tis_out, out, tisn);
+
+	return err;
+}
+
+void mlx5_core_destroy_tis(struct mlx5_core_dev *dev, u32 tisn)
+{
+	u32 in[MLX5_ST_SZ_DW(destroy_tis_in)];
+	u32 out[MLX5_ST_SZ_DW(destroy_tis_out)];
+
+	memset(in, 0, sizeof(in));
+
+	MLX5_SET(destroy_tis_in, in, opcode, MLX5_CMD_OP_DESTROY_TIS);
+	MLX5_SET(destroy_tis_in, in, tisn, tisn);
+
+	mlx5_cmd_exec_check_status(dev, in, sizeof(in), out, sizeof(out));
+}
+
+int mlx5_core_create_rmp(struct mlx5_core_dev *dev, u32 *in, int inlen, u32 *rmpn)
+{
+	u32 out[MLX5_ST_SZ_DW(create_rmp_out)];
+	int err;
+
+	MLX5_SET(create_rmp_in, in, opcode, MLX5_CMD_OP_CREATE_RMP);
+
+	memset(out, 0, sizeof(out));
+	err = mlx5_cmd_exec_check_status(dev, in, inlen, out, sizeof(out));
+	if (!err)
+		*rmpn = MLX5_GET(create_rmp_out, out, rmpn);
+
+	return err;
+}
+
+int mlx5_core_modify_rmp(struct mlx5_core_dev *dev, u32 *in, int inlen)
+{
+	u32 out[MLX5_ST_SZ_DW(modify_rmp_out)];
+
+	MLX5_SET(modify_rmp_in, in, opcode, MLX5_CMD_OP_MODIFY_RMP);
+
+	memset(out, 0, sizeof(out));
+	return mlx5_cmd_exec_check_status(dev, in, inlen, out, sizeof(out));
+}
+
+int mlx5_core_destroy_rmp(struct mlx5_core_dev *dev, u32 rmpn)
+{
+	u32 in[MLX5_ST_SZ_DW(destroy_rmp_in)];
+	u32 out[MLX5_ST_SZ_DW(destroy_rmp_out)];
+
+	memset(in, 0, sizeof(in));
+
+	MLX5_SET(destroy_rmp_in, in, opcode, MLX5_CMD_OP_DESTROY_RMP);
+	MLX5_SET(destroy_rmp_in, in, rmpn, rmpn);
+
+	return mlx5_cmd_exec_check_status(dev, in, sizeof(in), out, sizeof(out));
+}
+
+int mlx5_core_query_rmp(struct mlx5_core_dev *dev, u32 rmpn, u32 *out)
+{
+	u32 in[MLX5_ST_SZ_DW(query_rmp_in)];
+	int outlen = MLX5_ST_SZ_BYTES(query_rmp_out);
+
+	memset(in, 0, sizeof(in));
+	MLX5_SET(query_rmp_in, in, opcode, MLX5_CMD_OP_QUERY_RMP);
+	MLX5_SET(query_rmp_in, in, rmpn,   rmpn);
+
+	return mlx5_cmd_exec_check_status(dev, in, sizeof(in), out, outlen);
+}
+
+int mlx5_core_arm_rmp(struct mlx5_core_dev *dev, u32 rmpn, u16 lwm)
+{
+	void *in;
+	void *rmpc;
+	void *wq;
+	void *bitmask;
+	int  err;
+
+	in = mlx5_vzalloc(MLX5_ST_SZ_BYTES(modify_rmp_in));
+	if (!in)
+		return -ENOMEM;
+
+	rmpc    = MLX5_ADDR_OF(modify_rmp_in,   in,   ctx);
+	bitmask = MLX5_ADDR_OF(modify_rmp_in,   in,   bitmask);
+	wq      = MLX5_ADDR_OF(rmpc,	        rmpc, wq);
+
+	MLX5_SET(modify_rmp_in, in,	 rmp_state, MLX5_RMPC_STATE_RDY);
+	MLX5_SET(modify_rmp_in, in,	 rmpn,      rmpn);
+	MLX5_SET(wq,		wq,	 lwm,	    lwm);
+	MLX5_SET(rmp_bitmask,	bitmask, lwm,	    1);
+	MLX5_SET(rmpc,		rmpc,	 state,	    MLX5_RMPC_STATE_RDY);
+
+	err =  mlx5_core_modify_rmp(dev, in, MLX5_ST_SZ_BYTES(modify_rmp_in));
+
+	kvfree(in);
+
+	return err;
+}
+
+int mlx5_core_create_xsrq(struct mlx5_core_dev *dev, u32 *in, int inlen, u32 *xsrqn)
+{
+	u32 out[MLX5_ST_SZ_DW(create_xrc_srq_out)];
+	int err;
+
+	MLX5_SET(create_xrc_srq_in, in, opcode,     MLX5_CMD_OP_CREATE_XRC_SRQ);
+
+	memset(out, 0, sizeof(out));
+	err = mlx5_cmd_exec_check_status(dev, in, inlen, out, sizeof(out));
+	if (!err)
+		*xsrqn = MLX5_GET(create_xrc_srq_out, out, xrc_srqn);
+
+	return err;
+}
+
+int mlx5_core_destroy_xsrq(struct mlx5_core_dev *dev, u32 xsrqn)
+{
+	u32 in[MLX5_ST_SZ_DW(destroy_xrc_srq_in)];
+	u32 out[MLX5_ST_SZ_DW(destroy_xrc_srq_out)];
+
+	memset(in, 0, sizeof(in));
+	memset(out, 0, sizeof(out));
+
+	MLX5_SET(destroy_xrc_srq_in, in, opcode,   MLX5_CMD_OP_DESTROY_XRC_SRQ);
+	MLX5_SET(destroy_xrc_srq_in, in, xrc_srqn, xsrqn);
+
+	return mlx5_cmd_exec_check_status(dev, in, sizeof(in), out,
+					  sizeof(out));
+}
+
+int mlx5_core_query_xsrq(struct mlx5_core_dev *dev, u32 xsrqn, u32 *out)
+{
+	u32 in[MLX5_ST_SZ_DW(query_xrc_srq_in)];
+	void *srqc;
+	void *xrc_srqc;
+	int err;
+
+	memset(in, 0, sizeof(in));
+	MLX5_SET(query_xrc_srq_in, in, opcode,   MLX5_CMD_OP_QUERY_XRC_SRQ);
+	MLX5_SET(query_xrc_srq_in, in, xrc_srqn, xsrqn);
+
+	err =  mlx5_cmd_exec_check_status(dev, in, sizeof(in),
+					  out,
+					  MLX5_ST_SZ_BYTES(query_xrc_srq_out));
+	if (!err) {
+		xrc_srqc = MLX5_ADDR_OF(query_xrc_srq_out, out,
+					xrc_srq_context_entry);
+		srqc = MLX5_ADDR_OF(query_srq_out, out, srq_context_entry);
+		memcpy(srqc, xrc_srqc, MLX5_ST_SZ_BYTES(srqc));
+	}
+
+	return err;
+}
+
+int mlx5_core_arm_xsrq(struct mlx5_core_dev *dev, u32 xsrqn, u16 lwm)
+{
+	u32 in[MLX5_ST_SZ_DW(arm_xrc_srq_in)];
+	u32 out[MLX5_ST_SZ_DW(arm_xrc_srq_out)];
+
+	memset(in, 0, sizeof(in));
+	memset(out, 0, sizeof(out));
+
+	MLX5_SET(arm_xrc_srq_in, in, opcode,   MLX5_CMD_OP_ARM_XRC_SRQ);
+	MLX5_SET(arm_xrc_srq_in, in, xrc_srqn, xsrqn);
+	MLX5_SET(arm_xrc_srq_in, in, lwm,      lwm);
+	MLX5_SET(arm_xrc_srq_in, in, op_mod,
+		 MLX5_ARM_XRC_SRQ_IN_OP_MOD_XRC_SRQ);
+
+	return  mlx5_cmd_exec_check_status(dev, in, sizeof(in), out,
+					   sizeof(out));
+
+}
+
+int mlx5_core_create_rqt(struct mlx5_core_dev *dev, u32 *in, int inlen,
+			 u32 *rqtn)
+{
+	u32 out[MLX5_ST_SZ_DW(create_rqt_out)];
+	int err;
+
+	MLX5_SET(create_rqt_in, in, opcode, MLX5_CMD_OP_CREATE_RQT);
+
+	memset(out, 0, sizeof(out));
+	err = mlx5_cmd_exec_check_status(dev, in, inlen, out, sizeof(out));
+	if (!err)
+		*rqtn = MLX5_GET(create_rqt_out, out, rqtn);
+
+	return err;
+}
+
+int mlx5_core_modify_rqt(struct mlx5_core_dev *dev, u32 rqtn, u32 *in,
+			 int inlen)
+{
+	u32 out[MLX5_ST_SZ_DW(modify_rqt_out)];
+
+	MLX5_SET(modify_rqt_in, in, rqtn, rqtn);
+	MLX5_SET(modify_rqt_in, in, opcode, MLX5_CMD_OP_MODIFY_RQT);
+
+	memset(out, 0, sizeof(out));
+	return mlx5_cmd_exec_check_status(dev, in, inlen, out, sizeof(out));
+}
+
+void mlx5_core_destroy_rqt(struct mlx5_core_dev *dev, u32 rqtn)
+{
+	u32 in[MLX5_ST_SZ_DW(destroy_rqt_in)];
+	u32 out[MLX5_ST_SZ_DW(destroy_rqt_out)];
+
+	memset(in, 0, sizeof(in));
+
+	MLX5_SET(destroy_rqt_in, in, opcode, MLX5_CMD_OP_DESTROY_RQT);
+	MLX5_SET(destroy_rqt_in, in, rqtn, rqtn);
+
+	mlx5_cmd_exec_check_status(dev, in, sizeof(in), out, sizeof(out));
+}
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_uar.c b/sys/dev/mlx5/mlx5_core/mlx5_uar.c
new file mode 100644
index 0000000..30725e7
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_core/mlx5_uar.c
@@ -0,0 +1,214 @@
+/*-
+ * Copyright (c) 2013-2015, Mellanox Technologies, Ltd.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/io-mapping.h>
+#include <dev/mlx5/driver.h>
+#include "mlx5_core.h"
+
+enum {
+	NUM_DRIVER_UARS		= 4,
+	NUM_LOW_LAT_UUARS	= 4,
+};
+
+int mlx5_cmd_alloc_uar(struct mlx5_core_dev *dev, u32 *uarn)
+{
+	u32 in[MLX5_ST_SZ_DW(alloc_uar_in)];
+	u32 out[MLX5_ST_SZ_DW(alloc_uar_out)];
+	int err;
+
+	memset(in, 0, sizeof(in));
+
+	MLX5_SET(alloc_uar_in, in, opcode, MLX5_CMD_OP_ALLOC_UAR);
+
+	memset(out, 0, sizeof(out));
+	err = mlx5_cmd_exec_check_status(dev, in, sizeof(in), out, sizeof(out));
+	if (err)
+		return err;
+
+	*uarn = MLX5_GET(alloc_uar_out, out, uar);
+
+	return 0;
+}
+EXPORT_SYMBOL(mlx5_cmd_alloc_uar);
+
+int mlx5_cmd_free_uar(struct mlx5_core_dev *dev, u32 uarn)
+{
+	u32 in[MLX5_ST_SZ_DW(dealloc_uar_in)];
+	u32 out[MLX5_ST_SZ_DW(dealloc_uar_out)];
+
+	memset(in, 0, sizeof(in));
+
+	MLX5_SET(dealloc_uar_in, in, opcode, MLX5_CMD_OP_DEALLOC_UAR);
+	MLX5_SET(dealloc_uar_in, in, uar, uarn);
+
+	memset(out, 0, sizeof(out));
+	return mlx5_cmd_exec_check_status(dev, in,  sizeof(in),
+					       out, sizeof(out));
+}
+EXPORT_SYMBOL(mlx5_cmd_free_uar);
+
+static int need_uuar_lock(int uuarn)
+{
+	int tot_uuars = NUM_DRIVER_UARS * MLX5_BF_REGS_PER_PAGE;
+
+	if (uuarn == 0 || tot_uuars - NUM_LOW_LAT_UUARS)
+		return 0;
+
+	return 1;
+}
+
+int mlx5_alloc_uuars(struct mlx5_core_dev *dev, struct mlx5_uuar_info *uuari)
+{
+	int tot_uuars = NUM_DRIVER_UARS * MLX5_BF_REGS_PER_PAGE;
+	struct mlx5_bf *bf;
+	phys_addr_t addr;
+	int err;
+	int i;
+
+	uuari->num_uars = NUM_DRIVER_UARS;
+	uuari->num_low_latency_uuars = NUM_LOW_LAT_UUARS;
+
+	mutex_init(&uuari->lock);
+	uuari->uars = kcalloc(uuari->num_uars, sizeof(*uuari->uars), GFP_KERNEL);
+
+	uuari->bfs = kcalloc(tot_uuars, sizeof(*uuari->bfs), GFP_KERNEL);
+
+	uuari->bitmap = kcalloc(BITS_TO_LONGS(tot_uuars), sizeof(*uuari->bitmap),
+				GFP_KERNEL);
+
+	uuari->count = kcalloc(tot_uuars, sizeof(*uuari->count), GFP_KERNEL);
+
+	for (i = 0; i < uuari->num_uars; i++) {
+		err = mlx5_cmd_alloc_uar(dev, &uuari->uars[i].index);
+		if (err)
+			goto out_count;
+
+		addr = pci_resource_start(dev->pdev, 0) +
+		       ((phys_addr_t)(uuari->uars[i].index) << PAGE_SHIFT);
+		uuari->uars[i].map = ioremap(addr, PAGE_SIZE);
+		if (!uuari->uars[i].map) {
+			mlx5_cmd_free_uar(dev, uuari->uars[i].index);
+			err = -ENOMEM;
+			goto out_count;
+		}
+		mlx5_core_dbg(dev, "allocated uar index 0x%x, mmaped at %p\n",
+			      uuari->uars[i].index, uuari->uars[i].map);
+	}
+
+	for (i = 0; i < tot_uuars; i++) {
+		bf = &uuari->bfs[i];
+
+		bf->buf_size = (1 << MLX5_CAP_GEN(dev, log_bf_reg_size)) / 2;
+		bf->uar = &uuari->uars[i / MLX5_BF_REGS_PER_PAGE];
+		bf->regreg = uuari->uars[i / MLX5_BF_REGS_PER_PAGE].map;
+		bf->reg = NULL; /* Add WC support */
+		bf->offset = (i % MLX5_BF_REGS_PER_PAGE) *
+			     (1 << MLX5_CAP_GEN(dev, log_bf_reg_size)) +
+			     MLX5_BF_OFFSET;
+		bf->need_lock = need_uuar_lock(i);
+		spin_lock_init(&bf->lock);
+		spin_lock_init(&bf->lock32);
+		bf->uuarn = i;
+	}
+
+	return 0;
+
+out_count:
+	for (i--; i >= 0; i--) {
+		iounmap(uuari->uars[i].map);
+		mlx5_cmd_free_uar(dev, uuari->uars[i].index);
+	}
+	kfree(uuari->count);
+
+	kfree(uuari->bitmap);
+
+	kfree(uuari->bfs);
+
+	kfree(uuari->uars);
+	return err;
+}
+
+int mlx5_free_uuars(struct mlx5_core_dev *dev, struct mlx5_uuar_info *uuari)
+{
+	int i = uuari->num_uars;
+
+	for (i--; i >= 0; i--) {
+		iounmap(uuari->uars[i].map);
+		mlx5_cmd_free_uar(dev, uuari->uars[i].index);
+	}
+
+	kfree(uuari->count);
+	kfree(uuari->bitmap);
+	kfree(uuari->bfs);
+	kfree(uuari->uars);
+
+	return 0;
+}
+
+int mlx5_alloc_map_uar(struct mlx5_core_dev *mdev, struct mlx5_uar *uar)
+{
+	phys_addr_t pfn;
+	phys_addr_t uar_bar_start;
+	int err;
+
+	err = mlx5_cmd_alloc_uar(mdev, &uar->index);
+	if (err) {
+		mlx5_core_warn(mdev, "mlx5_cmd_alloc_uar() failed, %d\n", err);
+		return err;
+	}
+
+	uar_bar_start = pci_resource_start(mdev->pdev, 0);
+	pfn           = (uar_bar_start >> PAGE_SHIFT) + uar->index;
+	uar->map      = ioremap(pfn << PAGE_SHIFT, PAGE_SIZE);
+	if (!uar->map) {
+		mlx5_core_warn(mdev, "ioremap() failed, %d\n", err);
+		err = -ENOMEM;
+		goto err_free_uar;
+	}
+
+	if (mdev->priv.bf_mapping)
+		uar->bf_map = io_mapping_map_wc(mdev->priv.bf_mapping,
+						uar->index << PAGE_SHIFT);
+
+	return 0;
+
+err_free_uar:
+	mlx5_cmd_free_uar(mdev, uar->index);
+
+	return err;
+}
+EXPORT_SYMBOL(mlx5_alloc_map_uar);
+
+void mlx5_unmap_free_uar(struct mlx5_core_dev *mdev, struct mlx5_uar *uar)
+{
+	io_mapping_unmap(uar->bf_map);
+	iounmap(uar->map);
+	mlx5_cmd_free_uar(mdev, uar->index);
+}
+EXPORT_SYMBOL(mlx5_unmap_free_uar);
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_vport.c b/sys/dev/mlx5/mlx5_core/mlx5_vport.c
new file mode 100644
index 0000000..803f119
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_core/mlx5_vport.c
@@ -0,0 +1,796 @@
+/*-
+ * Copyright (c) 2013-2015, Mellanox Technologies, Ltd.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <linux/etherdevice.h>
+#include <dev/mlx5/driver.h>
+#include <dev/mlx5/vport.h>
+#include "mlx5_core.h"
+
+u8 mlx5_query_vport_state(struct mlx5_core_dev *mdev, u8 opmod)
+{
+	u32 in[MLX5_ST_SZ_DW(query_vport_state_in)];
+	u32 out[MLX5_ST_SZ_DW(query_vport_state_out)];
+	int err;
+
+	memset(in, 0, sizeof(in));
+
+	MLX5_SET(query_vport_state_in, in, opcode,
+		 MLX5_CMD_OP_QUERY_VPORT_STATE);
+	MLX5_SET(query_vport_state_in, in, op_mod, opmod);
+
+	err = mlx5_cmd_exec_check_status(mdev, in, sizeof(in), out,
+					 sizeof(out));
+	if (err)
+		mlx5_core_warn(mdev, "MLX5_CMD_OP_QUERY_VPORT_STATE failed\n");
+
+	return MLX5_GET(query_vport_state_out, out, state);
+}
+EXPORT_SYMBOL_GPL(mlx5_query_vport_state);
+
+static int mlx5_query_nic_vport_context(struct mlx5_core_dev *mdev, u32 vport,
+					u32 *out, int outlen)
+{
+	u32 in[MLX5_ST_SZ_DW(query_nic_vport_context_in)];
+
+	memset(in, 0, sizeof(in));
+
+	MLX5_SET(query_nic_vport_context_in, in, opcode,
+		 MLX5_CMD_OP_QUERY_NIC_VPORT_CONTEXT);
+
+	MLX5_SET(query_nic_vport_context_in, in, vport_number, vport);
+	if (vport)
+		MLX5_SET(query_nic_vport_context_in, in, other_vport, 1);
+
+	return mlx5_cmd_exec_check_status(mdev, in, sizeof(in), out, outlen);
+}
+
+int mlx5_vport_alloc_q_counter(struct mlx5_core_dev *mdev, int *counter_set_id)
+{
+	u32 in[MLX5_ST_SZ_DW(alloc_q_counter_in)];
+	u32 out[MLX5_ST_SZ_DW(alloc_q_counter_in)];
+	int err;
+
+	memset(in, 0, sizeof(in));
+	memset(out, 0, sizeof(out));
+
+	MLX5_SET(alloc_q_counter_in, in, opcode,
+		 MLX5_CMD_OP_ALLOC_Q_COUNTER);
+
+	err = mlx5_cmd_exec_check_status(mdev, in, sizeof(in),
+					 out, sizeof(out));
+
+	if (err)
+		return err;
+
+	*counter_set_id = MLX5_GET(alloc_q_counter_out, out,
+				   counter_set_id);
+	return err;
+}
+
+int mlx5_vport_dealloc_q_counter(struct mlx5_core_dev *mdev,
+				 int counter_set_id)
+{
+	u32 in[MLX5_ST_SZ_DW(dealloc_q_counter_in)];
+	u32 out[MLX5_ST_SZ_DW(dealloc_q_counter_out)];
+
+	memset(in, 0, sizeof(in));
+	memset(out, 0, sizeof(out));
+
+	MLX5_SET(dealloc_q_counter_in, in, opcode,
+		 MLX5_CMD_OP_DEALLOC_Q_COUNTER);
+	MLX5_SET(dealloc_q_counter_in, in, counter_set_id,
+		 counter_set_id);
+
+	return mlx5_cmd_exec_check_status(mdev, in, sizeof(in),
+					  out, sizeof(out));
+}
+
+static int mlx5_vport_query_q_counter(struct mlx5_core_dev *mdev,
+				      int counter_set_id,
+				      int reset,
+				      void *out,
+				      int out_size)
+{
+	u32 in[MLX5_ST_SZ_DW(query_q_counter_in)];
+
+	memset(in, 0, sizeof(in));
+
+	MLX5_SET(query_q_counter_in, in, opcode, MLX5_CMD_OP_QUERY_Q_COUNTER);
+	MLX5_SET(query_q_counter_in, in, clear, reset);
+	MLX5_SET(query_q_counter_in, in, counter_set_id, counter_set_id);
+
+	return mlx5_cmd_exec_check_status(mdev, in, sizeof(in),
+					  out, out_size);
+}
+
+int mlx5_vport_query_out_of_rx_buffer(struct mlx5_core_dev *mdev,
+				      int counter_set_id,
+				      u32 *out_of_rx_buffer)
+{
+	u32 out[MLX5_ST_SZ_DW(query_q_counter_out)];
+	int err;
+
+	memset(out, 0, sizeof(out));
+
+	err = mlx5_vport_query_q_counter(mdev, counter_set_id, 0, out,
+					 sizeof(out));
+
+	if (err)
+		return err;
+
+	*out_of_rx_buffer = MLX5_GET(query_q_counter_out, out,
+				     out_of_buffer);
+	return err;
+}
+
+int mlx5_query_nic_vport_mac_address(struct mlx5_core_dev *mdev,
+				     u32 vport, u8 *addr)
+{
+	u32 *out;
+	int outlen = MLX5_ST_SZ_BYTES(query_nic_vport_context_out);
+	u8 *out_addr;
+	int err;
+
+	out = mlx5_vzalloc(outlen);
+	if (!out)
+		return -ENOMEM;
+
+	out_addr = MLX5_ADDR_OF(query_nic_vport_context_out, out,
+				nic_vport_context.permanent_address);
+
+	err = mlx5_query_nic_vport_context(mdev, vport, out, outlen);
+	if (err)
+		goto out;
+
+	ether_addr_copy(addr, &out_addr[2]);
+
+out:
+	kvfree(out);
+	return err;
+}
+EXPORT_SYMBOL_GPL(mlx5_query_nic_vport_mac_address);
+
+int mlx5_query_nic_vport_system_image_guid(struct mlx5_core_dev *mdev,
+					   u64 *system_image_guid)
+{
+	u32 *out;
+	int outlen = MLX5_ST_SZ_BYTES(query_nic_vport_context_out);
+	int err;
+
+	out = mlx5_vzalloc(outlen);
+	if (!out)
+		return -ENOMEM;
+
+	err = mlx5_query_nic_vport_context(mdev, 0, out, outlen);
+	if (err)
+		goto out;
+
+	*system_image_guid = MLX5_GET64(query_nic_vport_context_out, out,
+					nic_vport_context.system_image_guid);
+out:
+	kvfree(out);
+	return err;
+}
+EXPORT_SYMBOL_GPL(mlx5_query_nic_vport_system_image_guid);
+
+int mlx5_query_nic_vport_node_guid(struct mlx5_core_dev *mdev, u64 *node_guid)
+{
+	u32 *out;
+	int outlen = MLX5_ST_SZ_BYTES(query_nic_vport_context_out);
+	int err;
+
+	out = mlx5_vzalloc(outlen);
+	if (!out)
+		return -ENOMEM;
+
+	err = mlx5_query_nic_vport_context(mdev, 0, out, outlen);
+	if (err)
+		goto out;
+
+	*node_guid = MLX5_GET64(query_nic_vport_context_out, out,
+				nic_vport_context.node_guid);
+
+out:
+	kvfree(out);
+	return err;
+}
+EXPORT_SYMBOL_GPL(mlx5_query_nic_vport_node_guid);
+
+int mlx5_query_nic_vport_port_guid(struct mlx5_core_dev *mdev, u64 *port_guid)
+{
+	u32 *out;
+	int outlen = MLX5_ST_SZ_BYTES(query_nic_vport_context_out);
+	int err;
+
+	out = mlx5_vzalloc(outlen);
+	if (!out)
+		return -ENOMEM;
+
+	err = mlx5_query_nic_vport_context(mdev, 0, out, outlen);
+	if (err)
+		goto out;
+
+	*port_guid = MLX5_GET64(query_nic_vport_context_out, out,
+				nic_vport_context.port_guid);
+
+out:
+	kvfree(out);
+	return err;
+}
+EXPORT_SYMBOL_GPL(mlx5_query_nic_vport_port_guid);
+
+int mlx5_query_nic_vport_qkey_viol_cntr(struct mlx5_core_dev *mdev,
+					u16 *qkey_viol_cntr)
+{
+	u32 *out;
+	int outlen = MLX5_ST_SZ_BYTES(query_nic_vport_context_out);
+	int err;
+
+	out = mlx5_vzalloc(outlen);
+	if (!out)
+		return -ENOMEM;
+
+	err = mlx5_query_nic_vport_context(mdev, 0, out, outlen);
+	if (err)
+		goto out;
+
+	*qkey_viol_cntr = MLX5_GET(query_nic_vport_context_out, out,
+				nic_vport_context.qkey_violation_counter);
+
+out:
+	kvfree(out);
+	return err;
+}
+EXPORT_SYMBOL_GPL(mlx5_query_nic_vport_qkey_viol_cntr);
+
+static int mlx5_modify_nic_vport_context(struct mlx5_core_dev *mdev, void *in,
+					 int inlen)
+{
+	u32 out[MLX5_ST_SZ_DW(modify_nic_vport_context_out)];
+
+	MLX5_SET(modify_nic_vport_context_in, in, opcode,
+		 MLX5_CMD_OP_MODIFY_NIC_VPORT_CONTEXT);
+
+	memset(out, 0, sizeof(out));
+	return mlx5_cmd_exec_check_status(mdev, in, inlen, out, sizeof(out));
+}
+
+static int mlx5_nic_vport_enable_disable_roce(struct mlx5_core_dev *mdev,
+					      int enable_disable)
+{
+	void *in;
+	int inlen = MLX5_ST_SZ_BYTES(modify_nic_vport_context_in);
+	int err;
+
+	in = mlx5_vzalloc(inlen);
+	if (!in) {
+		mlx5_core_warn(mdev, "failed to allocate inbox\n");
+		return -ENOMEM;
+	}
+
+	MLX5_SET(modify_nic_vport_context_in, in, field_select.roce_en, 1);
+	MLX5_SET(modify_nic_vport_context_in, in, nic_vport_context.roce_en,
+		 enable_disable);
+
+	err = mlx5_modify_nic_vport_context(mdev, in, inlen);
+
+	kvfree(in);
+
+	return err;
+}
+
+int mlx5_set_nic_vport_current_mac(struct mlx5_core_dev *mdev, int vport,
+				   bool other_vport, u8 *addr)
+{
+	void *in;
+	int inlen = MLX5_ST_SZ_BYTES(modify_nic_vport_context_in)
+		  + MLX5_ST_SZ_BYTES(mac_address_layout);
+	u8  *mac_layout;
+	u8  *mac_ptr;
+	int err;
+
+	in = mlx5_vzalloc(inlen);
+	if (!in) {
+		mlx5_core_warn(mdev, "failed to allocate inbox\n");
+		return -ENOMEM;
+	}
+
+	MLX5_SET(modify_nic_vport_context_in, in,
+		 opcode, MLX5_CMD_OP_MODIFY_NIC_VPORT_CONTEXT);
+	MLX5_SET(modify_nic_vport_context_in, in,
+		 vport_number, vport);
+	MLX5_SET(modify_nic_vport_context_in, in,
+		 other_vport, other_vport);
+	MLX5_SET(modify_nic_vport_context_in, in,
+		 field_select.addresses_list, 1);
+	MLX5_SET(modify_nic_vport_context_in, in,
+		 nic_vport_context.allowed_list_type, 0);
+	MLX5_SET(modify_nic_vport_context_in, in,
+		 nic_vport_context.allowed_list_size, 1);
+
+	mac_layout = (u8 *)MLX5_ADDR_OF(modify_nic_vport_context_in, in,
+		nic_vport_context.current_uc_mac_address);
+	mac_ptr = (u8 *)MLX5_ADDR_OF(mac_address_layout, mac_layout,
+		mac_addr_47_32);
+	ether_addr_copy(mac_ptr, addr);
+
+	err = mlx5_modify_nic_vport_context(mdev, in, inlen);
+
+	kvfree(in);
+
+	return err;
+}
+EXPORT_SYMBOL_GPL(mlx5_set_nic_vport_current_mac);
+int mlx5_set_nic_vport_permanent_mac(struct mlx5_core_dev *mdev, int vport,
+				     u8 *addr)
+{
+	void *in;
+	int inlen = MLX5_ST_SZ_BYTES(modify_nic_vport_context_in);
+	u8  *mac_ptr;
+	int err;
+
+	in = mlx5_vzalloc(inlen);
+	if (!in) {
+		mlx5_core_warn(mdev, "failed to allocate inbox\n");
+		return -ENOMEM;
+	}
+
+	MLX5_SET(modify_nic_vport_context_in, in,
+		 opcode, MLX5_CMD_OP_MODIFY_NIC_VPORT_CONTEXT);
+	MLX5_SET(modify_nic_vport_context_in, in, vport_number, vport);
+	MLX5_SET(modify_nic_vport_context_in, in, other_vport, 1);
+	MLX5_SET(modify_nic_vport_context_in, in,
+		 field_select.permanent_address, 1);
+	mac_ptr = (u8 *)MLX5_ADDR_OF(modify_nic_vport_context_in, in,
+		nic_vport_context.permanent_address.mac_addr_47_32);
+	ether_addr_copy(mac_ptr, addr);
+
+	err = mlx5_modify_nic_vport_context(mdev, in, inlen);
+
+	kvfree(in);
+
+	return err;
+}
+EXPORT_SYMBOL_GPL(mlx5_set_nic_vport_permanent_mac);
+
+int mlx5_nic_vport_enable_roce(struct mlx5_core_dev *mdev)
+{
+	return mlx5_nic_vport_enable_disable_roce(mdev, 1);
+}
+EXPORT_SYMBOL_GPL(mlx5_nic_vport_enable_roce);
+
+int mlx5_nic_vport_disable_roce(struct mlx5_core_dev *mdev)
+{
+	return mlx5_nic_vport_enable_disable_roce(mdev, 0);
+}
+EXPORT_SYMBOL_GPL(mlx5_nic_vport_disable_roce);
+
+int mlx5_query_hca_vport_context(struct mlx5_core_dev *mdev,
+				 u8 port_num, u8 vport_num, u32 *out,
+				 int outlen)
+{
+	u32 in[MLX5_ST_SZ_DW(query_hca_vport_context_in)];
+	int is_group_manager;
+
+	is_group_manager = MLX5_CAP_GEN(mdev, vport_group_manager);
+
+	memset(in, 0, sizeof(in));
+
+	MLX5_SET(query_hca_vport_context_in, in, opcode,
+		 MLX5_CMD_OP_QUERY_HCA_VPORT_CONTEXT);
+
+	if (vport_num) {
+		if (is_group_manager) {
+			MLX5_SET(query_hca_vport_context_in, in, other_vport,
+				 1);
+			MLX5_SET(query_hca_vport_context_in, in, vport_number,
+				 vport_num);
+		} else {
+			return -EPERM;
+		}
+	}
+
+	if (MLX5_CAP_GEN(mdev, num_ports) == 2)
+		MLX5_SET(query_hca_vport_context_in, in, port_num, port_num);
+
+	return mlx5_cmd_exec_check_status(mdev, in, sizeof(in), out, outlen);
+}
+
+int mlx5_query_hca_vport_system_image_guid(struct mlx5_core_dev *mdev,
+					   u64 *system_image_guid)
+{
+	u32 *out;
+	int outlen = MLX5_ST_SZ_BYTES(query_hca_vport_context_out);
+	int err;
+
+	out = mlx5_vzalloc(outlen);
+	if (!out)
+		return -ENOMEM;
+
+	err = mlx5_query_hca_vport_context(mdev, 1, 0, out, outlen);
+	if (err)
+		goto out;
+
+	*system_image_guid = MLX5_GET64(query_hca_vport_context_out, out,
+					hca_vport_context.system_image_guid);
+
+out:
+	kvfree(out);
+	return err;
+}
+EXPORT_SYMBOL_GPL(mlx5_query_hca_vport_system_image_guid);
+
+int mlx5_query_hca_vport_node_guid(struct mlx5_core_dev *mdev, u64 *node_guid)
+{
+	u32 *out;
+	int outlen = MLX5_ST_SZ_BYTES(query_hca_vport_context_out);
+	int err;
+
+	out = mlx5_vzalloc(outlen);
+	if (!out)
+		return -ENOMEM;
+
+	err = mlx5_query_hca_vport_context(mdev, 1, 0, out, outlen);
+	if (err)
+		goto out;
+
+	*node_guid = MLX5_GET64(query_hca_vport_context_out, out,
+				hca_vport_context.node_guid);
+
+out:
+	kvfree(out);
+	return err;
+}
+EXPORT_SYMBOL_GPL(mlx5_query_hca_vport_node_guid);
+
+int mlx5_query_hca_vport_gid(struct mlx5_core_dev *dev, u8 port_num,
+			     u16 vport_num, u16 gid_index, union ib_gid *gid)
+{
+	int in_sz = MLX5_ST_SZ_BYTES(query_hca_vport_gid_in);
+	int out_sz = MLX5_ST_SZ_BYTES(query_hca_vport_gid_out);
+	int is_group_manager;
+	void *out = NULL;
+	void *in = NULL;
+	union ib_gid *tmp;
+	int tbsz;
+	int nout;
+	int err;
+
+	is_group_manager = MLX5_CAP_GEN(dev, vport_group_manager);
+	tbsz = mlx5_get_gid_table_len(MLX5_CAP_GEN(dev, gid_table_size));
+
+	if (gid_index > tbsz && gid_index != 0xffff)
+		return -EINVAL;
+
+	if (gid_index == 0xffff)
+		nout = tbsz;
+	else
+		nout = 1;
+
+	out_sz += nout * sizeof(*gid);
+
+	in = mlx5_vzalloc(in_sz);
+	out = mlx5_vzalloc(out_sz);
+	if (!in || !out) {
+		err = -ENOMEM;
+		goto out;
+	}
+
+	MLX5_SET(query_hca_vport_gid_in, in, opcode,
+		 MLX5_CMD_OP_QUERY_HCA_VPORT_GID);
+	if (vport_num) {
+		if (is_group_manager) {
+			MLX5_SET(query_hca_vport_gid_in, in, vport_number,
+				 vport_num);
+			MLX5_SET(query_hca_vport_gid_in, in, other_vport, 1);
+		} else {
+			err = -EPERM;
+			goto out;
+		}
+	}
+
+	MLX5_SET(query_hca_vport_gid_in, in, gid_index, gid_index);
+
+	if (MLX5_CAP_GEN(dev, num_ports) == 2)
+		MLX5_SET(query_hca_vport_gid_in, in, port_num, port_num);
+
+	err = mlx5_cmd_exec(dev, in, in_sz, out, out_sz);
+	if (err)
+		goto out;
+
+	err = mlx5_cmd_status_to_err_v2(out);
+	if (err)
+		goto out;
+
+	tmp = (union ib_gid *)MLX5_ADDR_OF(query_hca_vport_gid_out, out, gid);
+	gid->global.subnet_prefix = tmp->global.subnet_prefix;
+	gid->global.interface_id = tmp->global.interface_id;
+
+out:
+	kvfree(in);
+	kvfree(out);
+	return err;
+}
+EXPORT_SYMBOL_GPL(mlx5_query_hca_vport_gid);
+
+int mlx5_query_hca_vport_pkey(struct mlx5_core_dev *dev, u8 other_vport,
+			      u8 port_num, u16 vf_num, u16 pkey_index,
+			      u16 *pkey)
+{
+	int in_sz = MLX5_ST_SZ_BYTES(query_hca_vport_pkey_in);
+	int out_sz = MLX5_ST_SZ_BYTES(query_hca_vport_pkey_out);
+	int is_group_manager;
+	void *out = NULL;
+	void *in = NULL;
+	void *pkarr;
+	int nout;
+	int tbsz;
+	int err;
+	int i;
+
+	is_group_manager = MLX5_CAP_GEN(dev, vport_group_manager);
+
+	tbsz = mlx5_to_sw_pkey_sz(MLX5_CAP_GEN(dev, pkey_table_size));
+	if (pkey_index > tbsz && pkey_index != 0xffff)
+		return -EINVAL;
+
+	if (pkey_index == 0xffff)
+		nout = tbsz;
+	else
+		nout = 1;
+
+	out_sz += nout * MLX5_ST_SZ_BYTES(pkey);
+
+	in = kzalloc(in_sz, GFP_KERNEL);
+	out = kzalloc(out_sz, GFP_KERNEL);
+
+	MLX5_SET(query_hca_vport_pkey_in, in, opcode,
+		 MLX5_CMD_OP_QUERY_HCA_VPORT_PKEY);
+	if (other_vport) {
+		if (is_group_manager) {
+			MLX5_SET(query_hca_vport_pkey_in, in, vport_number,
+				 vf_num);
+			MLX5_SET(query_hca_vport_pkey_in, in, other_vport, 1);
+		} else {
+			err = -EPERM;
+			goto out;
+		}
+	}
+	MLX5_SET(query_hca_vport_pkey_in, in, pkey_index, pkey_index);
+
+	if (MLX5_CAP_GEN(dev, num_ports) == 2)
+		MLX5_SET(query_hca_vport_pkey_in, in, port_num, port_num);
+
+	err = mlx5_cmd_exec(dev, in, in_sz, out, out_sz);
+	if (err)
+		goto out;
+
+	err = mlx5_cmd_status_to_err_v2(out);
+	if (err)
+		goto out;
+
+	pkarr = MLX5_ADDR_OF(query_hca_vport_pkey_out, out, pkey);
+	for (i = 0; i < nout; i++, pkey++,
+	     pkarr += MLX5_ST_SZ_BYTES(pkey))
+		*pkey = MLX5_GET_PR(pkey, pkarr, pkey);
+
+out:
+	kfree(in);
+	kfree(out);
+	return err;
+}
+EXPORT_SYMBOL_GPL(mlx5_query_hca_vport_pkey);
+
+static int mlx5_modify_eswitch_vport_context(struct mlx5_core_dev *mdev,
+					     u16 vport, void *in, int inlen)
+{
+	u32 out[MLX5_ST_SZ_DW(modify_esw_vport_context_out)];
+	int err;
+
+	memset(out, 0, sizeof(out));
+
+	MLX5_SET(modify_esw_vport_context_in, in, vport_number, vport);
+	if (vport)
+		MLX5_SET(modify_esw_vport_context_in, in, other_vport, 1);
+
+	MLX5_SET(modify_esw_vport_context_in, in, opcode,
+		 MLX5_CMD_OP_MODIFY_ESW_VPORT_CONTEXT);
+
+	err = mlx5_cmd_exec_check_status(mdev, in, inlen,
+					 out, sizeof(out));
+	if (err)
+		mlx5_core_warn(mdev, "MLX5_CMD_OP_MODIFY_ESW_VPORT_CONTEXT failed\n");
+
+	return err;
+}
+
+int mlx5_set_eswitch_cvlan_info(struct mlx5_core_dev *mdev, u8 vport,
+				u8 insert_mode, u8 strip_mode,
+				u16 vlan, u8 cfi, u8 pcp)
+{
+	u32 in[MLX5_ST_SZ_DW(modify_esw_vport_context_in)];
+
+	memset(in, 0, sizeof(in));
+
+	if (insert_mode != MLX5_MODIFY_ESW_VPORT_CONTEXT_CVLAN_INSERT_NONE) {
+		MLX5_SET(modify_esw_vport_context_in, in,
+			 esw_vport_context.cvlan_cfi, cfi);
+		MLX5_SET(modify_esw_vport_context_in, in,
+			 esw_vport_context.cvlan_pcp, pcp);
+		MLX5_SET(modify_esw_vport_context_in, in,
+			 esw_vport_context.cvlan_id, vlan);
+	}
+
+	MLX5_SET(modify_esw_vport_context_in, in,
+		 esw_vport_context.vport_cvlan_insert, insert_mode);
+
+	MLX5_SET(modify_esw_vport_context_in, in,
+		 esw_vport_context.vport_cvlan_strip, strip_mode);
+
+	MLX5_SET(modify_esw_vport_context_in, in, field_select,
+		 MLX5_MODIFY_ESW_VPORT_CONTEXT_FIELD_SELECT_CVLAN_STRIP |
+		 MLX5_MODIFY_ESW_VPORT_CONTEXT_FIELD_SELECT_CVLAN_INSERT);
+
+	return mlx5_modify_eswitch_vport_context(mdev, vport, in, sizeof(in));
+}
+EXPORT_SYMBOL_GPL(mlx5_set_eswitch_cvlan_info);
+
+int mlx5_query_vport_counter(struct mlx5_core_dev *dev,
+			     u8 port_num, u16 vport_num,
+			     void *out, int out_size)
+{
+	int in_sz = MLX5_ST_SZ_BYTES(query_vport_counter_in);
+	int is_group_manager;
+	void *in;
+	int err;
+
+	is_group_manager = MLX5_CAP_GEN(dev, vport_group_manager);
+
+	in = mlx5_vzalloc(in_sz);
+	if (!in)
+		return -ENOMEM;
+
+	MLX5_SET(query_vport_counter_in, in, opcode,
+		 MLX5_CMD_OP_QUERY_VPORT_COUNTER);
+	if (vport_num) {
+		if (is_group_manager) {
+			MLX5_SET(query_vport_counter_in, in, other_vport, 1);
+			MLX5_SET(query_vport_counter_in, in, vport_number,
+				 vport_num);
+		} else {
+			err = -EPERM;
+			goto ex;
+		}
+	}
+	if (MLX5_CAP_GEN(dev, num_ports) == 2)
+		MLX5_SET(query_vport_counter_in, in, port_num, port_num);
+
+	err = mlx5_cmd_exec(dev, in, in_sz, out,  out_size);
+	if (err)
+		goto ex;
+	err = mlx5_cmd_status_to_err_v2(out);
+	if (err)
+		goto ex;
+
+ex:
+	kvfree(in);
+	return err;
+}
+EXPORT_SYMBOL_GPL(mlx5_query_vport_counter);
+
+int mlx5_get_vport_counters(struct mlx5_core_dev *dev, u8 port_num,
+			    struct mlx5_vport_counters *vc)
+{
+	int out_sz = MLX5_ST_SZ_BYTES(query_vport_counter_out);
+	void *out;
+	int err;
+
+	out = mlx5_vzalloc(out_sz);
+	if (!out)
+		return -ENOMEM;
+
+	err = mlx5_query_vport_counter(dev, port_num, 0, out, out_sz);
+	if (err)
+		goto ex;
+
+	vc->received_errors.packets =
+		MLX5_GET64(query_vport_counter_out,
+			   out, received_errors.packets);
+	vc->received_errors.octets =
+		MLX5_GET64(query_vport_counter_out,
+			   out, received_errors.octets);
+	vc->transmit_errors.packets =
+		MLX5_GET64(query_vport_counter_out,
+			   out, transmit_errors.packets);
+	vc->transmit_errors.octets =
+		MLX5_GET64(query_vport_counter_out,
+			   out, transmit_errors.octets);
+	vc->received_ib_unicast.packets =
+		MLX5_GET64(query_vport_counter_out,
+			   out, received_ib_unicast.packets);
+	vc->received_ib_unicast.octets =
+		MLX5_GET64(query_vport_counter_out,
+			   out, received_ib_unicast.octets);
+	vc->transmitted_ib_unicast.packets =
+		MLX5_GET64(query_vport_counter_out,
+			   out, transmitted_ib_unicast.packets);
+	vc->transmitted_ib_unicast.octets =
+		MLX5_GET64(query_vport_counter_out,
+			   out, transmitted_ib_unicast.octets);
+	vc->received_ib_multicast.packets =
+		MLX5_GET64(query_vport_counter_out,
+			   out, received_ib_multicast.packets);
+	vc->received_ib_multicast.octets =
+		MLX5_GET64(query_vport_counter_out,
+			   out, received_ib_multicast.octets);
+	vc->transmitted_ib_multicast.packets =
+		MLX5_GET64(query_vport_counter_out,
+			   out, transmitted_ib_multicast.packets);
+	vc->transmitted_ib_multicast.octets =
+		MLX5_GET64(query_vport_counter_out,
+			   out, transmitted_ib_multicast.octets);
+	vc->received_eth_broadcast.packets =
+		MLX5_GET64(query_vport_counter_out,
+			   out, received_eth_broadcast.packets);
+	vc->received_eth_broadcast.octets =
+		MLX5_GET64(query_vport_counter_out,
+			   out, received_eth_broadcast.octets);
+	vc->transmitted_eth_broadcast.packets =
+		MLX5_GET64(query_vport_counter_out,
+			   out, transmitted_eth_broadcast.packets);
+	vc->transmitted_eth_broadcast.octets =
+		MLX5_GET64(query_vport_counter_out,
+			   out, transmitted_eth_broadcast.octets);
+	vc->received_eth_unicast.octets =
+		MLX5_GET64(query_vport_counter_out,
+			   out, received_eth_unicast.octets);
+	vc->received_eth_unicast.packets =
+		MLX5_GET64(query_vport_counter_out,
+			   out, received_eth_unicast.packets);
+	vc->transmitted_eth_unicast.octets =
+		MLX5_GET64(query_vport_counter_out,
+			   out, transmitted_eth_unicast.octets);
+	vc->transmitted_eth_unicast.packets =
+		MLX5_GET64(query_vport_counter_out,
+			   out, transmitted_eth_unicast.packets);
+	vc->received_eth_multicast.octets =
+		MLX5_GET64(query_vport_counter_out,
+			   out, received_eth_multicast.octets);
+	vc->received_eth_multicast.packets =
+		MLX5_GET64(query_vport_counter_out,
+			   out, received_eth_multicast.packets);
+	vc->transmitted_eth_multicast.octets =
+		MLX5_GET64(query_vport_counter_out,
+			   out, transmitted_eth_multicast.octets);
+	vc->transmitted_eth_multicast.packets =
+		MLX5_GET64(query_vport_counter_out,
+			   out, transmitted_eth_multicast.packets);
+
+ex:
+	kvfree(out);
+	return err;
+}
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_wq.c b/sys/dev/mlx5/mlx5_core/mlx5_wq.c
new file mode 100644
index 0000000..31318b5
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_core/mlx5_wq.c
@@ -0,0 +1,187 @@
+/*-
+ * Copyright (c) 2013-2015, Mellanox Technologies, Ltd.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <dev/mlx5/driver.h>
+#include "wq.h"
+#include "mlx5_core.h"
+
+u32 mlx5_wq_cyc_get_size(struct mlx5_wq_cyc *wq)
+{
+	return (u32)wq->sz_m1 + 1;
+}
+
+u32 mlx5_cqwq_get_size(struct mlx5_cqwq *wq)
+{
+	return wq->sz_m1 + 1;
+}
+
+u32 mlx5_wq_ll_get_size(struct mlx5_wq_ll *wq)
+{
+	return (u32)wq->sz_m1 + 1;
+}
+
+static u32 mlx5_wq_cyc_get_byte_size(struct mlx5_wq_cyc *wq)
+{
+	return mlx5_wq_cyc_get_size(wq) << wq->log_stride;
+}
+
+static u32 mlx5_cqwq_get_byte_size(struct mlx5_cqwq *wq)
+{
+	return mlx5_cqwq_get_size(wq) << wq->log_stride;
+}
+
+static u32 mlx5_wq_ll_get_byte_size(struct mlx5_wq_ll *wq)
+{
+	return mlx5_wq_ll_get_size(wq) << wq->log_stride;
+}
+
+int mlx5_wq_cyc_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param,
+		       void *wqc, struct mlx5_wq_cyc *wq,
+		       struct mlx5_wq_ctrl *wq_ctrl)
+{
+	int max_direct = param->linear ? INT_MAX : 0;
+	int err;
+
+	wq->log_stride = MLX5_GET(wq, wqc, log_wq_stride);
+	wq->sz_m1 = (1 << MLX5_GET(wq, wqc, log_wq_sz)) - 1;
+
+	err = mlx5_db_alloc_node(mdev, &wq_ctrl->db, param->db_numa_node);
+	if (err) {
+		mlx5_core_warn(mdev, "mlx5_db_alloc() failed, %d\n", err);
+		return err;
+	}
+
+	err = mlx5_buf_alloc_node(mdev, mlx5_wq_cyc_get_byte_size(wq),
+				  max_direct, &wq_ctrl->buf,
+				  param->buf_numa_node);
+	if (err) {
+		mlx5_core_warn(mdev, "mlx5_buf_alloc() failed, %d\n", err);
+		goto err_db_free;
+	}
+
+	wq->buf = wq_ctrl->buf.direct.buf;
+	wq->db  = wq_ctrl->db.db;
+
+	wq_ctrl->mdev = mdev;
+
+	return 0;
+
+err_db_free:
+	mlx5_db_free(mdev, &wq_ctrl->db);
+
+	return err;
+}
+
+int mlx5_cqwq_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param,
+		     void *cqc, struct mlx5_cqwq *wq,
+		     struct mlx5_wq_ctrl *wq_ctrl)
+{
+	int max_direct = param->linear ? INT_MAX : 0;
+	int err;
+
+	wq->log_stride = 6 + MLX5_GET(cqc, cqc, cqe_sz);
+	wq->log_sz = MLX5_GET(cqc, cqc, log_cq_size);
+	wq->sz_m1 = (1 << wq->log_sz) - 1;
+
+	err = mlx5_db_alloc_node(mdev, &wq_ctrl->db, param->db_numa_node);
+	if (err) {
+		mlx5_core_warn(mdev, "mlx5_db_alloc() failed, %d\n", err);
+		return err;
+	}
+
+	err = mlx5_buf_alloc_node(mdev, mlx5_cqwq_get_byte_size(wq),
+				  max_direct, &wq_ctrl->buf,
+				  param->buf_numa_node);
+	if (err) {
+		mlx5_core_warn(mdev, "mlx5_buf_alloc() failed, %d\n", err);
+		goto err_db_free;
+	}
+
+	wq->buf = wq_ctrl->buf.direct.buf;
+	wq->db  = wq_ctrl->db.db;
+
+	wq_ctrl->mdev = mdev;
+
+	return 0;
+
+err_db_free:
+	mlx5_db_free(mdev, &wq_ctrl->db);
+
+	return err;
+}
+
+int mlx5_wq_ll_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param,
+		      void *wqc, struct mlx5_wq_ll *wq,
+		      struct mlx5_wq_ctrl *wq_ctrl)
+{
+	struct mlx5_wqe_srq_next_seg *next_seg;
+	int max_direct = param->linear ? INT_MAX : 0;
+	int err;
+	int i;
+
+	wq->log_stride = MLX5_GET(wq, wqc, log_wq_stride);
+	wq->sz_m1 = (1 << MLX5_GET(wq, wqc, log_wq_sz)) - 1;
+
+	err = mlx5_db_alloc_node(mdev, &wq_ctrl->db, param->db_numa_node);
+	if (err) {
+		mlx5_core_warn(mdev, "mlx5_db_alloc() failed, %d\n", err);
+		return err;
+	}
+
+	err = mlx5_buf_alloc_node(mdev, mlx5_wq_ll_get_byte_size(wq),
+				  max_direct, &wq_ctrl->buf,
+				  param->buf_numa_node);
+	if (err) {
+		mlx5_core_warn(mdev, "mlx5_buf_alloc() failed, %d\n", err);
+		goto err_db_free;
+	}
+
+	wq->buf = wq_ctrl->buf.direct.buf;
+	wq->db  = wq_ctrl->db.db;
+
+	for (i = 0; i < wq->sz_m1; i++) {
+		next_seg = mlx5_wq_ll_get_wqe(wq, i);
+		next_seg->next_wqe_index = cpu_to_be16(i + 1);
+	}
+	next_seg = mlx5_wq_ll_get_wqe(wq, i);
+	wq->tail_next = &next_seg->next_wqe_index;
+
+	wq_ctrl->mdev = mdev;
+
+	return 0;
+
+err_db_free:
+	mlx5_db_free(mdev, &wq_ctrl->db);
+
+	return err;
+}
+
+void mlx5_wq_destroy(struct mlx5_wq_ctrl *wq_ctrl)
+{
+	mlx5_buf_free(wq_ctrl->mdev, &wq_ctrl->buf);
+	mlx5_db_free(wq_ctrl->mdev, &wq_ctrl->db);
+}
diff --git a/sys/dev/mlx5/mlx5_core/transobj.h b/sys/dev/mlx5/mlx5_core/transobj.h
new file mode 100644
index 0000000..655e625
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_core/transobj.h
@@ -0,0 +1,63 @@
+/*-
+ * Copyright (c) 2013-2015, Mellanox Technologies, Ltd.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef __TRANSOBJ_H__
+#define __TRANSOBJ_H__
+
+int mlx5_alloc_transport_domain(struct mlx5_core_dev *dev, u32 *tdn);
+void mlx5_dealloc_transport_domain(struct mlx5_core_dev *dev, u32 tdn);
+int mlx5_core_create_rq(struct mlx5_core_dev *dev, u32 *in, int inlen,
+			u32 *rqn);
+int mlx5_core_modify_rq(struct mlx5_core_dev *dev, u32 *in, int inlen);
+void mlx5_core_destroy_rq(struct mlx5_core_dev *dev, u32 rqn);
+int mlx5_core_create_sq(struct mlx5_core_dev *dev, u32 *in, int inlen,
+			u32 *sqn);
+int mlx5_core_modify_sq(struct mlx5_core_dev *dev, u32 *in, int inlen);
+void mlx5_core_destroy_sq(struct mlx5_core_dev *dev, u32 sqn);
+int mlx5_core_create_tir(struct mlx5_core_dev *dev, u32 *in, int inlen,
+			 u32 *tirn);
+void mlx5_core_destroy_tir(struct mlx5_core_dev *dev, u32 tirn);
+int mlx5_core_create_tis(struct mlx5_core_dev *dev, u32 *in, int inlen,
+			 u32 *tisn);
+void mlx5_core_destroy_tis(struct mlx5_core_dev *dev, u32 tisn);
+int mlx5_core_create_rmp(struct mlx5_core_dev *dev, u32 *in, int inlen, u32 *rmpn);
+int mlx5_core_modify_rmp(struct mlx5_core_dev *dev, u32 *in, int inlen);
+int mlx5_core_destroy_rmp(struct mlx5_core_dev *dev, u32 rmpn);
+int mlx5_core_query_rmp(struct mlx5_core_dev *dev, u32 rmpn, u32 *out);
+int mlx5_core_arm_rmp(struct mlx5_core_dev *dev, u32 rmpn, u16 lwm);
+int mlx5_core_create_xsrq(struct mlx5_core_dev *dev, u32 *in, int inlen, u32 *rmpn);
+int mlx5_core_destroy_xsrq(struct mlx5_core_dev *dev, u32 rmpn);
+int mlx5_core_query_xsrq(struct mlx5_core_dev *dev, u32 rmpn, u32 *out);
+int mlx5_core_arm_xsrq(struct mlx5_core_dev *dev, u32 rmpn, u16 lwm);
+
+int mlx5_core_create_rqt(struct mlx5_core_dev *dev, u32 *in, int inlen,
+			 u32 *rqtn);
+int mlx5_core_modify_rqt(struct mlx5_core_dev *dev, u32 rqtn, u32 *in,
+			 int inlen);
+void mlx5_core_destroy_rqt(struct mlx5_core_dev *dev, u32 rqtn);
+
+#endif /* __TRANSOBJ_H__ */
diff --git a/sys/dev/mlx5/mlx5_core/wq.h b/sys/dev/mlx5/mlx5_core/wq.h
new file mode 100644
index 0000000..39e0aa9
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_core/wq.h
@@ -0,0 +1,166 @@
+/*-
+ * Copyright (c) 2013-2015, Mellanox Technologies, Ltd.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef __MLX5_WQ_H__
+#define __MLX5_WQ_H__
+
+#include <dev/mlx5/mlx5_ifc.h>
+
+struct mlx5_wq_param {
+	int		linear;
+	int		buf_numa_node;
+	int		db_numa_node;
+};
+
+struct mlx5_wq_ctrl {
+	struct mlx5_core_dev	*mdev;
+	struct mlx5_buf		buf;
+	struct mlx5_db		db;
+};
+
+struct mlx5_wq_cyc {
+	void			*buf;
+	__be32			*db;
+	u16			sz_m1;
+	u8			log_stride;
+};
+
+struct mlx5_cqwq {
+	void			*buf;
+	__be32			*db;
+	u32			sz_m1;
+	u32			cc; /* consumer counter */
+	u8			log_sz;
+	u8			log_stride;
+};
+
+struct mlx5_wq_ll {
+	void			*buf;
+	__be32			*db;
+	__be16			*tail_next;
+	u16			sz_m1;
+	u16			head;
+	u16			wqe_ctr;
+	u16			cur_sz;
+	u8			log_stride;
+};
+
+int mlx5_wq_cyc_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param,
+		       void *wqc, struct mlx5_wq_cyc *wq,
+		       struct mlx5_wq_ctrl *wq_ctrl);
+u32 mlx5_wq_cyc_get_size(struct mlx5_wq_cyc *wq);
+
+int mlx5_cqwq_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param,
+		     void *cqc, struct mlx5_cqwq *wq,
+		     struct mlx5_wq_ctrl *wq_ctrl);
+u32 mlx5_cqwq_get_size(struct mlx5_cqwq *wq);
+
+int mlx5_wq_ll_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param,
+		      void *wqc, struct mlx5_wq_ll *wq,
+		      struct mlx5_wq_ctrl *wq_ctrl);
+u32 mlx5_wq_ll_get_size(struct mlx5_wq_ll *wq);
+
+void mlx5_wq_destroy(struct mlx5_wq_ctrl *wq_ctrl);
+
+static inline u16 mlx5_wq_cyc_ctr2ix(struct mlx5_wq_cyc *wq, u16 ctr)
+{
+	return ctr & wq->sz_m1;
+}
+
+static inline void *mlx5_wq_cyc_get_wqe(struct mlx5_wq_cyc *wq, u16 ix)
+{
+	return wq->buf + (ix << wq->log_stride);
+}
+
+static inline int mlx5_wq_cyc_cc_bigger(u16 cc1, u16 cc2)
+{
+	int equal   = (cc1 == cc2);
+	int smaller = 0x8000 & (cc1 - cc2);
+
+	return !equal && !smaller;
+}
+
+static inline u32 mlx5_cqwq_get_ci(struct mlx5_cqwq *wq)
+{
+	return wq->cc & wq->sz_m1;
+}
+
+static inline void *mlx5_cqwq_get_wqe(struct mlx5_cqwq *wq, u32 ix)
+{
+	return wq->buf + (ix << wq->log_stride);
+}
+
+static inline u32 mlx5_cqwq_get_wrap_cnt(struct mlx5_cqwq *wq)
+{
+	return wq->cc >> wq->log_sz;
+}
+
+static inline void mlx5_cqwq_pop(struct mlx5_cqwq *wq)
+{
+	wq->cc++;
+}
+
+static inline void mlx5_cqwq_update_db_record(struct mlx5_cqwq *wq)
+{
+	*wq->db = cpu_to_be32(wq->cc & 0xffffff);
+}
+
+static inline int mlx5_wq_ll_is_full(struct mlx5_wq_ll *wq)
+{
+	return wq->cur_sz == wq->sz_m1;
+}
+
+static inline int mlx5_wq_ll_is_empty(struct mlx5_wq_ll *wq)
+{
+	return !wq->cur_sz;
+}
+
+static inline void mlx5_wq_ll_push(struct mlx5_wq_ll *wq, u16 head_next)
+{
+	wq->head = head_next;
+	wq->wqe_ctr++;
+	wq->cur_sz++;
+}
+
+static inline void mlx5_wq_ll_pop(struct mlx5_wq_ll *wq, __be16 ix,
+				  __be16 *next_tail_next)
+{
+	*wq->tail_next = ix;
+	wq->tail_next = next_tail_next;
+	wq->cur_sz--;
+}
+static inline void mlx5_wq_ll_update_db_record(struct mlx5_wq_ll *wq)
+{
+	*wq->db = cpu_to_be32(wq->wqe_ctr);
+}
+
+static inline void *mlx5_wq_ll_get_wqe(struct mlx5_wq_ll *wq, u16 ix)
+{
+	return wq->buf + (ix << wq->log_stride);
+}
+
+#endif /* __MLX5_WQ_H__ */
diff --git a/sys/dev/mlx5/mlx5_en/en.h b/sys/dev/mlx5/mlx5_en/en.h
new file mode 100644
index 0000000..5339006
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_en/en.h
@@ -0,0 +1,780 @@
+/*-
+ * Copyright (c) 2015 Mellanox Technologies. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _MLX5_EN_H_
+#define	_MLX5_EN_H_
+
+#include <linux/kmod.h>
+#include <linux/page.h>
+#include <linux/slab.h>
+#include <linux/if_vlan.h>
+#include <linux/if_ether.h>
+#include <linux/vmalloc.h>
+#include <linux/moduleparam.h>
+#include <linux/delay.h>
+#include <linux/netdevice.h>
+#include <linux/etherdevice.h>
+
+#include <netinet/in_systm.h>
+#include <netinet/in.h>
+#include <netinet/if_ether.h>
+#include <netinet/ip.h>
+#include <netinet/ip6.h>
+#include <netinet/tcp.h>
+#include <netinet/tcp_lro.h>
+#include <netinet/udp.h>
+#include <net/ethernet.h>
+#include <sys/buf_ring.h>
+
+#include <machine/bus.h>
+
+#ifdef HAVE_TURBO_LRO
+#include "tcp_tlro.h"
+#endif
+
+#include <dev/mlx5/driver.h>
+#include <dev/mlx5/qp.h>
+#include <dev/mlx5/cq.h>
+#include <dev/mlx5/vport.h>
+
+#include <dev/mlx5/mlx5_core/wq.h>
+#include <dev/mlx5/mlx5_core/transobj.h>
+#include <dev/mlx5/mlx5_core/mlx5_core.h>
+
+#define	MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE                0x7
+#define	MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE                0xa
+#define	MLX5E_PARAMS_MAXIMUM_LOG_SQ_SIZE                0xd
+
+#define	MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE                0x7
+#define	MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE                0xa
+#define	MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE                0xd
+
+/* freeBSD HW LRO is limited by 16KB - the size of max mbuf */
+#define	MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ                 MJUM16BYTES
+#define	MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC      0x10
+#define	MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC_FROM_CQE	0x3
+#define	MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_PKTS      0x20
+#define	MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC      0x10
+#define	MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_PKTS      0x20
+#define	MLX5E_PARAMS_DEFAULT_MIN_RX_WQES                0x80
+#define	MLX5E_PARAMS_DEFAULT_RX_HASH_LOG_TBL_SZ         0x7
+#define	MLX5E_CACHELINE_SIZE CACHE_LINE_SIZE
+#define	MLX5E_HW2SW_MTU(hwmtu) \
+    ((hwmtu) - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN + ETHER_CRC_LEN))
+#define	MLX5E_SW2HW_MTU(swmtu) \
+    ((swmtu) + (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN + ETHER_CRC_LEN))
+#define	MLX5E_SW2MB_MTU(swmtu) \
+    (MLX5E_SW2HW_MTU(swmtu) + MLX5E_NET_IP_ALIGN)
+#define	MLX5E_MTU_MIN		72	/* Min MTU allowed by the kernel */
+#define	MLX5E_MTU_MAX		MIN(ETHERMTU_JUMBO, MJUM16BYTES)	/* Max MTU of Ethernet
+									 * jumbo frames */
+
+#define	MLX5E_BUDGET_MAX	8192	/* RX and TX */
+#define	MLX5E_RX_BUDGET_MAX	256
+#define	MLX5E_SQ_BF_BUDGET	16
+#define	MLX5E_SQ_TX_QUEUE_SIZE	4096	/* SQ drbr queue size */
+
+#define	MLX5E_MAX_TX_NUM_TC	8	/* units */
+#define	MLX5E_MAX_TX_HEADER	128	/* bytes */
+#define	MLX5E_MAX_TX_PAYLOAD_SIZE	65536	/* bytes */
+#define	MLX5E_MAX_TX_MBUF_SIZE	65536	/* bytes */
+#define	MLX5E_MAX_TX_MBUF_FRAGS	\
+    ((MLX5_SEND_WQE_MAX_WQEBBS * MLX5_SEND_WQEBB_NUM_DS) - \
+    (MLX5E_MAX_TX_HEADER / MLX5_SEND_WQE_DS))	/* units */
+#define	MLX5E_MAX_TX_INLINE \
+  (MLX5E_MAX_TX_HEADER - sizeof(struct mlx5e_tx_wqe) + \
+  sizeof(((struct mlx5e_tx_wqe *)0)->eth.inline_hdr_start))	/* bytes */
+
+MALLOC_DECLARE(M_MLX5EN);
+
+struct mlx5_core_dev;
+struct mlx5e_cq;
+
+typedef void (mlx5e_cq_comp_t)(struct mlx5_core_cq *);
+
+#define	MLX5E_STATS_COUNT(a,b,c,d) a
+#define	MLX5E_STATS_VAR(a,b,c,d) b;
+#define	MLX5E_STATS_DESC(a,b,c,d) c, d,
+
+#define	MLX5E_VPORT_STATS(m)						\
+  /* HW counters */							\
+  m(+1, u64 rx_packets, "rx_packets", "Received packets")		\
+  m(+1, u64 rx_bytes, "rx_bytes", "Received bytes")			\
+  m(+1, u64 tx_packets, "tx_packets", "Transmitted packets")		\
+  m(+1, u64 tx_bytes, "tx_bytes", "Transmitted bytes")			\
+  m(+1, u64 rx_error_packets, "rx_error_packets", "Received error packets") \
+  m(+1, u64 rx_error_bytes, "rx_error_bytes", "Received error bytes")	\
+  m(+1, u64 tx_error_packets, "tx_error_packets", "Transmitted error packets") \
+  m(+1, u64 tx_error_bytes, "tx_error_bytes", "Transmitted error bytes") \
+  m(+1, u64 rx_unicast_packets, "rx_unicast_packets", "Received unicast packets") \
+  m(+1, u64 rx_unicast_bytes, "rx_unicast_bytes", "Received unicast bytes") \
+  m(+1, u64 tx_unicast_packets, "tx_unicast_packets", "Transmitted unicast packets") \
+  m(+1, u64 tx_unicast_bytes, "tx_unicast_bytes", "Transmitted unicast bytes") \
+  m(+1, u64 rx_multicast_packets, "rx_multicast_packets", "Received multicast packets") \
+  m(+1, u64 rx_multicast_bytes, "rx_multicast_bytes", "Received multicast bytes") \
+  m(+1, u64 tx_multicast_packets, "tx_multicast_packets", "Transmitted multicast packets") \
+  m(+1, u64 tx_multicast_bytes, "tx_multicast_bytes", "Transmitted multicast bytes") \
+  m(+1, u64 rx_broadcast_packets, "rx_broadcast_packets", "Received broadcast packets") \
+  m(+1, u64 rx_broadcast_bytes, "rx_broadcast_bytes", "Received broadcast bytes") \
+  m(+1, u64 tx_broadcast_packets, "tx_broadcast_packets", "Transmitted broadcast packets") \
+  m(+1, u64 tx_broadcast_bytes, "tx_broadcast_bytes", "Transmitted broadcast bytes") \
+  m(+1, u64 rx_out_of_buffer, "rx_out_of_buffer", "Receive out of buffer, no recv wqes events") \
+  /* SW counters */							\
+  m(+1, u64 tso_packets, "tso_packets", "Transmitted TSO packets")	\
+  m(+1, u64 tso_bytes, "tso_bytes", "Transmitted TSO bytes")		\
+  m(+1, u64 lro_packets, "lro_packets", "Received LRO packets")		\
+  m(+1, u64 lro_bytes, "lro_bytes", "Received LRO bytes")		\
+  m(+1, u64 sw_lro_queued, "sw_lro_queued", "Packets queued for SW LRO")	\
+  m(+1, u64 sw_lro_flushed, "sw_lro_flushed", "Packets flushed from SW LRO")	\
+  m(+1, u64 rx_csum_good, "rx_csum_good", "Received checksum valid packets") \
+  m(+1, u64 rx_csum_none, "rx_csum_none", "Received no checksum packets") \
+  m(+1, u64 tx_csum_offload, "tx_csum_offload", "Transmit checksum offload packets") \
+  m(+1, u64 tx_queue_dropped, "tx_queue_dropped", "Transmit queue dropped") \
+  m(+1, u64 tx_defragged, "tx_defragged", "Transmit queue defragged") \
+  m(+1, u64 rx_wqe_err, "rx_wqe_err", "Receive WQE errors")
+
+#define	MLX5E_VPORT_STATS_NUM (0 MLX5E_VPORT_STATS(MLX5E_STATS_COUNT))
+
+struct mlx5e_vport_stats {
+	struct	sysctl_ctx_list ctx;
+	u64	arg [0];
+	MLX5E_VPORT_STATS(MLX5E_STATS_VAR)
+	u32	rx_out_of_buffer_prev;
+};
+
+#define	MLX5E_PPORT_IEEE802_3_STATS(m)					\
+  m(+1, u64 frames_tx, "frames_tx", "Frames transmitted")		\
+  m(+1, u64 frames_rx, "frames_rx", "Frames received")			\
+  m(+1, u64 check_seq_err, "check_seq_err", "Sequence errors")		\
+  m(+1, u64 alignment_err, "alignment_err", "Alignment errors")	\
+  m(+1, u64 octets_tx, "octets_tx", "Bytes transmitted")		\
+  m(+1, u64 octets_received, "octets_received", "Bytes received")	\
+  m(+1, u64 multicast_xmitted, "multicast_xmitted", "Multicast transmitted") \
+  m(+1, u64 broadcast_xmitted, "broadcast_xmitted", "Broadcast transmitted") \
+  m(+1, u64 multicast_rx, "multicast_rx", "Multicast received")	\
+  m(+1, u64 broadcast_rx, "broadcast_rx", "Broadcast received")	\
+  m(+1, u64 in_range_len_errors, "in_range_len_errors", "In range length errors") \
+  m(+1, u64 out_of_range_len, "out_of_range_len", "Out of range length errors") \
+  m(+1, u64 too_long_errors, "too_long_errors", "Too long errors")	\
+  m(+1, u64 symbol_err, "symbol_err", "Symbol errors")			\
+  m(+1, u64 mac_control_tx, "mac_control_tx", "MAC control transmitted") \
+  m(+1, u64 mac_control_rx, "mac_control_rx", "MAC control received")	\
+  m(+1, u64 unsupported_op_rx, "unsupported_op_rx", "Unsupported operation received") \
+  m(+1, u64 pause_ctrl_rx, "pause_ctrl_rx", "Pause control received")	\
+  m(+1, u64 pause_ctrl_tx, "pause_ctrl_tx", "Pause control transmitted")
+
+#define	MLX5E_PPORT_RFC2819_STATS(m)					\
+  m(+1, u64 drop_events, "drop_events", "Dropped events")		\
+  m(+1, u64 octets, "octets", "Octets")					\
+  m(+1, u64 pkts, "pkts", "Packets")					\
+  m(+1, u64 broadcast_pkts, "broadcast_pkts", "Broadcast packets")	\
+  m(+1, u64 multicast_pkts, "multicast_pkts", "Multicast packets")	\
+  m(+1, u64 crc_align_errors, "crc_align_errors", "CRC alignment errors") \
+  m(+1, u64 undersize_pkts, "undersize_pkts", "Undersized packets")	\
+  m(+1, u64 oversize_pkts, "oversize_pkts", "Oversized packets")	\
+  m(+1, u64 fragments, "fragments", "Fragments")			\
+  m(+1, u64 jabbers, "jabbers", "Jabbers")				\
+  m(+1, u64 collisions, "collisions", "Collisions")
+
+#define	MLX5E_PPORT_RFC2819_STATS_DEBUG(m)				\
+  m(+1, u64 p64octets, "p64octets", "Bytes")				\
+  m(+1, u64 p65to127octets, "p65to127octets", "Bytes")			\
+  m(+1, u64 p128to255octets, "p128to255octets", "Bytes")		\
+  m(+1, u64 p256to511octets, "p256to511octets", "Bytes")		\
+  m(+1, u64 p512to1023octets, "p512to1023octets", "Bytes")		\
+  m(+1, u64 p1024to1518octets, "p1024to1518octets", "Bytes")		\
+  m(+1, u64 p1519to2047octets, "p1519to2047octets", "Bytes")		\
+  m(+1, u64 p2048to4095octets, "p2048to4095octets", "Bytes")		\
+  m(+1, u64 p4096to8191octets, "p4096to8191octets", "Bytes")		\
+  m(+1, u64 p8192to10239octets, "p8192to10239octets", "Bytes")
+
+#define	MLX5E_PPORT_RFC2863_STATS_DEBUG(m)				\
+  m(+1, u64 in_octets, "in_octets", "In octets")			\
+  m(+1, u64 in_ucast_pkts, "in_ucast_pkts", "In unicast packets")	\
+  m(+1, u64 in_discards, "in_discards", "In discards")			\
+  m(+1, u64 in_errors, "in_errors", "In errors")			\
+  m(+1, u64 in_unknown_protos, "in_unknown_protos", "In unknown protocols") \
+  m(+1, u64 out_octets, "out_octets", "Out octets")			\
+  m(+1, u64 out_ucast_pkts, "out_ucast_pkts", "Out unicast packets")	\
+  m(+1, u64 out_discards, "out_discards", "Out discards")		\
+  m(+1, u64 out_errors, "out_errors", "Out errors")			\
+  m(+1, u64 in_multicast_pkts, "in_multicast_pkts", "In multicast packets") \
+  m(+1, u64 in_broadcast_pkts, "in_broadcast_pkts", "In broadcast packets") \
+  m(+1, u64 out_multicast_pkts, "out_multicast_pkts", "Out multicast packets") \
+  m(+1, u64 out_broadcast_pkts, "out_broadcast_pkts", "Out broadcast packets")
+
+#define	MLX5E_PPORT_PHYSICAL_LAYER_STATS_DEBUG(m)                                    		\
+  m(+1, u64 time_since_last_clear, "time_since_last_clear",				\
+			"Time since the last counters clear event (msec)")		\
+  m(+1, u64 symbol_errors, "symbol_errors", "Symbol errors")				\
+  m(+1, u64 sync_headers_errors, "sync_headers_errors", "Sync header error counter")	\
+  m(+1, u64 bip_errors_lane0, "edpl_bip_errors_lane0",					\
+			"Indicates the number of PRBS errors on lane 0")		\
+  m(+1, u64 bip_errors_lane1, "edpl_bip_errors_lane1",					\
+			"Indicates the number of PRBS errors on lane 1")		\
+  m(+1, u64 bip_errors_lane2, "edpl_bip_errors_lane2",					\
+			"Indicates the number of PRBS errors on lane 2")		\
+  m(+1, u64 bip_errors_lane3, "edpl_bip_errors_lane3",					\
+			"Indicates the number of PRBS errors on lane 3")		\
+  m(+1, u64 fc_corrected_blocks_lane0, "fc_corrected_blocks_lane0",			\
+			"FEC correctable block counter lane 0")				\
+  m(+1, u64 fc_corrected_blocks_lane1, "fc_corrected_blocks_lane1",			\
+			"FEC correctable block counter lane 1")				\
+  m(+1, u64 fc_corrected_blocks_lane2, "fc_corrected_blocks_lane2",			\
+			"FEC correctable block counter lane 2")				\
+  m(+1, u64 fc_corrected_blocks_lane3, "fc_corrected_blocks_lane3",			\
+			"FEC correctable block counter lane 3")				\
+  m(+1, u64 rs_corrected_blocks, "rs_corrected_blocks",					\
+			"FEC correcable block counter")					\
+  m(+1, u64 rs_uncorrectable_blocks, "rs_uncorrectable_blocks",				\
+			"FEC uncorrecable block counter")				\
+  m(+1, u64 rs_no_errors_blocks, "rs_no_errors_blocks",					\
+			"The number of RS-FEC blocks received that had no errors")	\
+  m(+1, u64 rs_single_error_blocks, "rs_single_error_blocks",				\
+			"The number of corrected RS-FEC blocks received that had"	\
+			"exactly 1 error symbol")					\
+  m(+1, u64 rs_corrected_symbols_total, "rs_corrected_symbols_total",			\
+			"Port FEC corrected symbol counter")				\
+  m(+1, u64 rs_corrected_symbols_lane0, "rs_corrected_symbols_lane0",			\
+			"FEC corrected symbol counter lane 0")				\
+  m(+1, u64 rs_corrected_symbols_lane1, "rs_corrected_symbols_lane1",			\
+			"FEC corrected symbol counter lane 1")				\
+  m(+1, u64 rs_corrected_symbols_lane2, "rs_corrected_symbols_lane2",			\
+			"FEC corrected symbol counter lane 2")				\
+  m(+1, u64 rs_corrected_symbols_lane3, "rs_corrected_symbols_lane3",			\
+			"FEC corrected symbol counter lane 3")				\
+
+/*
+ * Make sure to update mlx5e_update_pport_counters()
+ * when adding a new MLX5E_PPORT_STATS block
+ */
+#define	MLX5E_PPORT_STATS(m)			\
+  MLX5E_PPORT_IEEE802_3_STATS(m)		\
+  MLX5E_PPORT_RFC2819_STATS(m)
+
+#define	MLX5E_PORT_STATS_DEBUG(m)		\
+  MLX5E_PPORT_RFC2819_STATS_DEBUG(m)		\
+  MLX5E_PPORT_RFC2863_STATS_DEBUG(m)		\
+  MLX5E_PPORT_PHYSICAL_LAYER_STATS_DEBUG(m)
+
+#define	MLX5E_PPORT_IEEE802_3_STATS_NUM \
+  (0 MLX5E_PPORT_IEEE802_3_STATS(MLX5E_STATS_COUNT))
+#define	MLX5E_PPORT_RFC2819_STATS_NUM \
+  (0 MLX5E_PPORT_RFC2819_STATS(MLX5E_STATS_COUNT))
+#define	MLX5E_PPORT_STATS_NUM \
+  (0 MLX5E_PPORT_STATS(MLX5E_STATS_COUNT))
+
+#define	MLX5E_PPORT_RFC2819_STATS_DEBUG_NUM \
+  (0 MLX5E_PPORT_RFC2819_STATS_DEBUG(MLX5E_STATS_COUNT))
+#define	MLX5E_PPORT_RFC2863_STATS_DEBUG_NUM \
+  (0 MLX5E_PPORT_RFC2863_STATS_DEBUG(MLX5E_STATS_COUNT))
+#define	MLX5E_PPORT_PHYSICAL_LAYER_STATS_DEBUG_NUM \
+  (0 MLX5E_PPORT_PHYSICAL_LAYER_STATS_DEBUG(MLX5E_STATS_COUNT))
+#define	MLX5E_PORT_STATS_DEBUG_NUM \
+  (0 MLX5E_PORT_STATS_DEBUG(MLX5E_STATS_COUNT))
+
+struct mlx5e_pport_stats {
+	struct	sysctl_ctx_list ctx;
+	u64	arg [0];
+	MLX5E_PPORT_STATS(MLX5E_STATS_VAR)
+};
+
+struct mlx5e_port_stats_debug {
+	struct	sysctl_ctx_list ctx;
+	u64	arg [0];
+	MLX5E_PORT_STATS_DEBUG(MLX5E_STATS_VAR)
+};
+
+#define	MLX5E_RQ_STATS(m)					\
+  m(+1, u64 packets, "packets", "Received packets")		\
+  m(+1, u64 csum_none, "csum_none", "Received packets")		\
+  m(+1, u64 lro_packets, "lro_packets", "Received packets")	\
+  m(+1, u64 lro_bytes, "lro_bytes", "Received packets")		\
+  m(+1, u64 sw_lro_queued, "sw_lro_queued", "Packets queued for SW LRO")	\
+  m(+1, u64 sw_lro_flushed, "sw_lro_flushed", "Packets flushed from SW LRO")	\
+  m(+1, u64 wqe_err, "wqe_err", "Received packets")
+
+#define	MLX5E_RQ_STATS_NUM (0 MLX5E_RQ_STATS(MLX5E_STATS_COUNT))
+
+struct mlx5e_rq_stats {
+	struct	sysctl_ctx_list ctx;
+	u64	arg [0];
+	MLX5E_RQ_STATS(MLX5E_STATS_VAR)
+};
+
+#define	MLX5E_SQ_STATS(m)						\
+  m(+1, u64 packets, "packets", "Transmitted packets")			\
+  m(+1, u64 tso_packets, "tso_packets", "Transmitted packets")		\
+  m(+1, u64 tso_bytes, "tso_bytes", "Transmitted bytes")		\
+  m(+1, u64 csum_offload_none, "csum_offload_none", "Transmitted packets")	\
+  m(+1, u64 defragged, "defragged", "Transmitted packets")		\
+  m(+1, u64 dropped, "dropped", "Transmitted packets")			\
+  m(+1, u64 nop, "nop", "Transmitted packets")
+
+#define	MLX5E_SQ_STATS_NUM (0 MLX5E_SQ_STATS(MLX5E_STATS_COUNT))
+
+struct mlx5e_sq_stats {
+	struct	sysctl_ctx_list ctx;
+	u64	arg [0];
+	MLX5E_SQ_STATS(MLX5E_STATS_VAR)
+};
+
+struct mlx5e_stats {
+	struct mlx5e_vport_stats vport;
+	struct mlx5e_pport_stats pport;
+	struct mlx5e_port_stats_debug port_stats_debug;
+};
+
+struct mlx5e_params {
+	u8	log_sq_size;
+	u8	log_rq_size;
+	u16	num_channels;
+	u8	default_vlan_prio;
+	u8	num_tc;
+	u8	rx_cq_moderation_mode;
+	u16	rx_cq_moderation_usec;
+	u16	rx_cq_moderation_pkts;
+	u16	tx_cq_moderation_usec;
+	u16	tx_cq_moderation_pkts;
+	u16	min_rx_wqes;
+	bool	hw_lro_en;
+	u32	lro_wqe_sz;
+	u16	rx_hash_log_tbl_sz;
+};
+
+#define	MLX5E_PARAMS(m)							\
+  m(+1, u64 tx_pauseframe_control, "tx_pauseframe_control", "Set to enable TX pause frames. Clear to disable.") \
+  m(+1, u64 rx_pauseframe_control, "rx_pauseframe_control", "Set to enable RX pause frames. Clear to disable.") \
+  m(+1, u64 tx_queue_size_max, "tx_queue_size_max", "Max send queue size") \
+  m(+1, u64 rx_queue_size_max, "rx_queue_size_max", "Max receive queue size") \
+  m(+1, u64 tx_queue_size, "tx_queue_size", "Default send queue size")	\
+  m(+1, u64 rx_queue_size, "rx_queue_size", "Default receive queue size") \
+  m(+1, u64 channels, "channels", "Default number of channels")		\
+  m(+1, u64 coalesce_usecs_max, "coalesce_usecs_max", "Maximum usecs for joining packets") \
+  m(+1, u64 coalesce_pkts_max, "coalesce_pkts_max", "Maximum packets to join") \
+  m(+1, u64 rx_coalesce_usecs, "rx_coalesce_usecs", "Limit in usec for joining rx packets") \
+  m(+1, u64 rx_coalesce_pkts, "rx_coalesce_pkts", "Maximum number of rx packets to join") \
+  m(+1, u64 rx_coalesce_mode, "rx_coalesce_mode", "0: EQE mode 1: CQE mode") \
+  m(+1, u64 tx_coalesce_usecs, "tx_coalesce_usecs", "Limit in usec for joining tx packets") \
+  m(+1, u64 tx_coalesce_pkts, "tx_coalesce_pkts", "Maximum number of tx packets to join") \
+  m(+1, u64 hw_lro, "hw_lro", "set to enable hw_lro")
+
+#define	MLX5E_PARAMS_NUM (0 MLX5E_PARAMS(MLX5E_STATS_COUNT))
+
+struct mlx5e_params_ethtool {
+	u64	arg [0];
+	MLX5E_PARAMS(MLX5E_STATS_VAR)
+};
+
+/* EEPROM Standards for plug in modules */
+#ifndef MLX5E_ETH_MODULE_SFF_8472
+#define	MLX5E_ETH_MODULE_SFF_8472	0x1
+#define	MLX5E_ETH_MODULE_SFF_8472_LEN	128
+#endif
+
+#ifndef MLX5E_ETH_MODULE_SFF_8636
+#define	MLX5E_ETH_MODULE_SFF_8636	0x2
+#define	MLX5E_ETH_MODULE_SFF_8636_LEN	256
+#endif
+
+#ifndef MLX5E_ETH_MODULE_SFF_8436
+#define	MLX5E_ETH_MODULE_SFF_8436	0x3
+#define	MLX5E_ETH_MODULE_SFF_8436_LEN	256
+#endif
+
+/* EEPROM I2C Addresses */
+#define	MLX5E_I2C_ADDR_LOW		0x50
+#define	MLX5E_I2C_ADDR_HIGH		0x51
+
+#define	MLX5E_EEPROM_LOW_PAGE		0x0
+#define	MLX5E_EEPROM_HIGH_PAGE		0x3
+
+#define	MLX5E_EEPROM_HIGH_PAGE_OFFSET	128
+#define	MLX5E_EEPROM_PAGE_LENGTH	256
+
+#define	MLX5E_EEPROM_INFO_BYTES		0x3
+
+struct mlx5e_cq {
+	/* data path - accessed per cqe */
+	struct mlx5_cqwq wq;
+
+	/* data path - accessed per HW polling */
+	struct mlx5_core_cq mcq;
+	struct mlx5e_channel *channel;
+
+	/* control */
+	struct mlx5_wq_ctrl wq_ctrl;
+} __aligned(MLX5E_CACHELINE_SIZE);
+
+struct mlx5e_rq_mbuf {
+	bus_dmamap_t	dma_map;
+	caddr_t		data;
+	struct mbuf	*mbuf;
+};
+
+struct mlx5e_rq {
+	/* data path */
+	struct mlx5_wq_ll wq;
+	struct mtx mtx;
+	bus_dma_tag_t dma_tag;
+	u32	wqe_sz;
+	struct mlx5e_rq_mbuf *mbuf;
+	struct device *pdev;
+	struct ifnet *ifp;
+	struct mlx5e_rq_stats stats;
+	struct mlx5e_cq cq;
+#ifdef HAVE_TURBO_LRO
+	struct tlro_ctrl lro;
+#else
+	struct lro_ctrl lro;
+#endif
+	volatile int enabled;
+	int	ix;
+
+	/* control */
+	struct mlx5_wq_ctrl wq_ctrl;
+	u32	rqn;
+	struct mlx5e_channel *channel;
+} __aligned(MLX5E_CACHELINE_SIZE);
+
+struct mlx5e_sq_mbuf {
+	bus_dmamap_t dma_map;
+	struct mbuf *mbuf;
+	u32	num_bytes;
+	u32	num_wqebbs;
+};
+
+enum {
+	MLX5E_SQ_READY,
+	MLX5E_SQ_FULL
+};
+
+struct mlx5e_sq {
+	/* data path */
+	struct	mtx lock;
+	bus_dma_tag_t dma_tag;
+	struct	mtx comp_lock;
+
+	/* dirtied @completion */
+	u16	cc;
+
+	/* dirtied @xmit */
+	u16	pc __aligned(MLX5E_CACHELINE_SIZE);
+	u16	bf_offset;
+	struct	mlx5e_sq_stats stats;
+
+	struct	mlx5e_cq cq;
+	struct	task sq_task;
+	struct	taskqueue *sq_tq;
+
+	/* pointers to per packet info: write@xmit, read@completion */
+	struct	mlx5e_sq_mbuf *mbuf;
+	struct	buf_ring *br;
+
+	/* read only */
+	struct	mlx5_wq_cyc wq;
+	void	__iomem *uar_map;
+	void	__iomem *uar_bf_map;
+	u32	sqn;
+	u32	bf_buf_size;
+	struct  device *pdev;
+	u32	mkey_be;
+
+	/* control path */
+	struct	mlx5_wq_ctrl wq_ctrl;
+	struct	mlx5_uar uar;
+	struct	mlx5e_channel *channel;
+	int	tc;
+	unsigned int queue_state;
+} __aligned(MLX5E_CACHELINE_SIZE);
+
+static inline bool
+mlx5e_sq_has_room_for(struct mlx5e_sq *sq, u16 n)
+{
+	return ((sq->wq.sz_m1 & (sq->cc - sq->pc)) >= n ||
+	    sq->cc == sq->pc);
+}
+
+struct mlx5e_channel {
+	/* data path */
+	struct mlx5e_rq rq;
+	struct mlx5e_sq sq[MLX5E_MAX_TX_NUM_TC];
+	struct device *pdev;
+	struct ifnet *ifp;
+	u32	mkey_be;
+	u8	num_tc;
+
+	/* control */
+	struct mlx5e_priv *priv;
+	int	ix;
+	int	cpu;
+} __aligned(MLX5E_CACHELINE_SIZE);
+
+enum mlx5e_traffic_types {
+	MLX5E_TT_IPV4_TCP,
+	MLX5E_TT_IPV6_TCP,
+	MLX5E_TT_IPV4_UDP,
+	MLX5E_TT_IPV6_UDP,
+	MLX5E_TT_IPV4_IPSEC_AH,
+	MLX5E_TT_IPV6_IPSEC_AH,
+	MLX5E_TT_IPV4_IPSEC_ESP,
+	MLX5E_TT_IPV6_IPSEC_ESP,
+	MLX5E_TT_IPV4,
+	MLX5E_TT_IPV6,
+	MLX5E_TT_ANY,
+	MLX5E_NUM_TT,
+};
+
+enum {
+	MLX5E_RQT_SPREADING = 0,
+	MLX5E_RQT_DEFAULT_RQ = 1,
+	MLX5E_NUM_RQT = 2,
+};
+
+struct mlx5e_eth_addr_info {
+	u8	addr [ETH_ALEN + 2];
+	u32	tt_vec;
+	u32	ft_ix[MLX5E_NUM_TT];	/* flow table index per traffic type */
+};
+
+#define	MLX5E_ETH_ADDR_HASH_SIZE (1 << BITS_PER_BYTE)
+
+struct mlx5e_eth_addr_hash_node;
+
+struct mlx5e_eth_addr_hash_head {
+	struct mlx5e_eth_addr_hash_node *lh_first;
+};
+
+struct mlx5e_eth_addr_db {
+	struct mlx5e_eth_addr_hash_head if_uc[MLX5E_ETH_ADDR_HASH_SIZE];
+	struct mlx5e_eth_addr_hash_head if_mc[MLX5E_ETH_ADDR_HASH_SIZE];
+	struct mlx5e_eth_addr_info broadcast;
+	struct mlx5e_eth_addr_info allmulti;
+	struct mlx5e_eth_addr_info promisc;
+	bool	broadcast_enabled;
+	bool	allmulti_enabled;
+	bool	promisc_enabled;
+};
+
+enum {
+	MLX5E_STATE_ASYNC_EVENTS_ENABLE,
+	MLX5E_STATE_OPENED,
+};
+
+struct mlx5e_vlan_db {
+	unsigned long active_vlans[BITS_TO_LONGS(VLAN_N_VID)];
+	u32	active_vlans_ft_ix[VLAN_N_VID];
+	u32	untagged_rule_ft_ix;
+	u32	any_vlan_rule_ft_ix;
+	bool	filter_disabled;
+};
+
+struct mlx5e_flow_table {
+	void   *vlan;
+	void   *main;
+};
+
+struct mlx5e_priv {
+	/* priv data path fields - start */
+	int	order_base_2_num_channels;
+	int	queue_mapping_channel_mask;
+	int	num_tc;
+	int	default_vlan_prio;
+	/* priv data path fields - end */
+
+	unsigned long state;
+	int	gone;
+#define	PRIV_LOCK(priv) sx_xlock(&(priv)->state_lock)
+#define	PRIV_UNLOCK(priv) sx_xunlock(&(priv)->state_lock)
+#define	PRIV_LOCKED(priv) sx_xlocked(&(priv)->state_lock)
+	struct sx state_lock;		/* Protects Interface state */
+	struct mlx5_uar cq_uar;
+	u32	pdn;
+	u32	tdn;
+	struct mlx5_core_mr mr;
+
+	struct mlx5e_channel *volatile *channel;
+	u32	tisn[MLX5E_MAX_TX_NUM_TC];
+	u32	rqtn;
+	u32	tirn[MLX5E_NUM_TT];
+
+	struct mlx5e_flow_table ft;
+	struct mlx5e_eth_addr_db eth_addr;
+	struct mlx5e_vlan_db vlan;
+
+	struct mlx5e_params params;
+	struct mlx5e_params_ethtool params_ethtool;
+	struct mtx async_events_mtx;	/* sync hw events */
+	struct work_struct update_stats_work;
+	struct work_struct update_carrier_work;
+	struct work_struct set_rx_mode_work;
+
+	struct mlx5_core_dev *mdev;
+	struct ifnet *ifp;
+	struct sysctl_ctx_list sysctl_ctx;
+	struct sysctl_oid *sysctl_ifnet;
+	struct sysctl_oid *sysctl_hw;
+	int	sysctl_debug;
+	struct mlx5e_stats stats;
+	int	counter_set_id;
+
+	eventhandler_tag vlan_detach;
+	eventhandler_tag vlan_attach;
+	struct ifmedia media;
+	int	media_status_last;
+	int	media_active_last;
+
+	struct callout watchdog;
+};
+
+#define	MLX5E_NET_IP_ALIGN 2
+
+struct mlx5e_tx_wqe {
+	struct mlx5_wqe_ctrl_seg ctrl;
+	struct mlx5_wqe_eth_seg eth;
+};
+
+struct mlx5e_rx_wqe {
+	struct mlx5_wqe_srq_next_seg next;
+	struct mlx5_wqe_data_seg data;
+};
+
+struct mlx5e_eeprom {
+	int	lock_bit;
+	int	i2c_addr;
+	int	page_num;
+	int	device_addr;
+	int	module_num;
+	int	len;
+	int	type;
+	int	page_valid;
+	u32	*data;
+};
+
+enum mlx5e_link_mode {
+	MLX5E_1000BASE_CX_SGMII = 0,
+	MLX5E_1000BASE_KX = 1,
+	MLX5E_10GBASE_CX4 = 2,
+	MLX5E_10GBASE_KX4 = 3,
+	MLX5E_10GBASE_KR = 4,
+	MLX5E_20GBASE_KR2 = 5,
+	MLX5E_40GBASE_CR4 = 6,
+	MLX5E_40GBASE_KR4 = 7,
+	MLX5E_56GBASE_R4 = 8,
+	MLX5E_10GBASE_CR = 12,
+	MLX5E_10GBASE_SR = 13,
+	MLX5E_10GBASE_ER = 14,
+	MLX5E_40GBASE_SR4 = 15,
+	MLX5E_40GBASE_LR4 = 16,
+	MLX5E_100GBASE_CR4 = 20,
+	MLX5E_100GBASE_SR4 = 21,
+	MLX5E_100GBASE_KR4 = 22,
+	MLX5E_100GBASE_LR4 = 23,
+	MLX5E_100BASE_TX = 24,
+	MLX5E_100BASE_T = 25,
+	MLX5E_10GBASE_T = 26,
+	MLX5E_25GBASE_CR = 27,
+	MLX5E_25GBASE_KR = 28,
+	MLX5E_25GBASE_SR = 29,
+	MLX5E_50GBASE_CR2 = 30,
+	MLX5E_50GBASE_KR2 = 31,
+	MLX5E_LINK_MODES_NUMBER,
+};
+
+#define	MLX5E_PROT_MASK(link_mode) (1 << (link_mode))
+#define	MLX5E_FLD_MAX(typ, fld) ((1ULL << __mlx5_bit_sz(typ, fld)) - 1ULL)
+
+int	mlx5e_xmit(struct ifnet *, struct mbuf *);
+
+int	mlx5e_open_locked(struct ifnet *);
+int	mlx5e_close_locked(struct ifnet *);
+
+void	mlx5e_cq_error_event(struct mlx5_core_cq *mcq, int event);
+void	mlx5e_rx_cq_comp(struct mlx5_core_cq *);
+void	mlx5e_tx_cq_comp(struct mlx5_core_cq *);
+struct mlx5_cqe64 *mlx5e_get_cqe(struct mlx5e_cq *cq);
+void	mlx5e_tx_que(void *context, int pending);
+
+int	mlx5e_open_flow_table(struct mlx5e_priv *priv);
+void	mlx5e_close_flow_table(struct mlx5e_priv *priv);
+void	mlx5e_set_rx_mode_core(struct mlx5e_priv *priv);
+void	mlx5e_set_rx_mode_work(struct work_struct *work);
+
+void	mlx5e_vlan_rx_add_vid(void *, struct ifnet *, u16);
+void	mlx5e_vlan_rx_kill_vid(void *, struct ifnet *, u16);
+void	mlx5e_enable_vlan_filter(struct mlx5e_priv *priv);
+void	mlx5e_disable_vlan_filter(struct mlx5e_priv *priv);
+int	mlx5e_add_all_vlan_rules(struct mlx5e_priv *priv);
+void	mlx5e_del_all_vlan_rules(struct mlx5e_priv *priv);
+
+static inline void
+mlx5e_tx_notify_hw(struct mlx5e_sq *sq,
+    struct mlx5e_tx_wqe *wqe, int bf_sz)
+{
+	u16 ofst = MLX5_BF_OFFSET + sq->bf_offset;
+
+	/* ensure wqe is visible to device before updating doorbell record */
+	wmb();
+
+	*sq->wq.db = cpu_to_be32(sq->pc);
+
+	/*
+	 * Ensure the doorbell record is visible to device before ringing
+	 * the doorbell:
+	 */
+	wmb();
+
+	if (bf_sz) {
+		__iowrite64_copy(sq->uar_bf_map + ofst, &wqe->ctrl, bf_sz);
+
+		/* flush the write-combining mapped buffer */
+		wmb();
+
+	} else {
+		mlx5_write64((__be32 *)&wqe->ctrl, sq->uar_map + ofst, NULL);
+	}
+
+	sq->bf_offset ^= sq->bf_buf_size;
+}
+
+static inline void
+mlx5e_cq_arm(struct mlx5e_cq *cq)
+{
+	struct mlx5_core_cq *mcq;
+
+	mcq = &cq->mcq;
+	mlx5_cq_arm(mcq, MLX5_CQ_DB_REQ_NOT, mcq->uar->map, NULL, cq->wq.cc);
+}
+
+extern const struct ethtool_ops mlx5e_ethtool_ops;
+void	mlx5e_create_ethtool(struct mlx5e_priv *);
+void	mlx5e_create_stats(struct sysctl_ctx_list *,
+    struct sysctl_oid_list *, const char *,
+    const char **, unsigned, u64 *);
+void	mlx5e_send_nop(struct mlx5e_sq *, u32, bool);
+
+#endif					/* _MLX5_EN_H_ */
diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_ethtool.c b/sys/dev/mlx5/mlx5_en/mlx5_en_ethtool.c
new file mode 100644
index 0000000..6a7b911
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_en/mlx5_en_ethtool.c
@@ -0,0 +1,503 @@
+/*-
+ * Copyright (c) 2015 Mellanox Technologies. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include "en.h"
+#include <net/sff8472.h>
+
+void
+mlx5e_create_stats(struct sysctl_ctx_list *ctx,
+    struct sysctl_oid_list *parent, const char *buffer,
+    const char **desc, unsigned num, u64 * arg)
+{
+	struct sysctl_oid *node;
+	unsigned x;
+
+	sysctl_ctx_init(ctx);
+
+	node = SYSCTL_ADD_NODE(ctx, parent, OID_AUTO,
+	    buffer, CTLFLAG_RD, NULL, "Statistics");
+	if (node == NULL)
+		return;
+	for (x = 0; x != num; x++) {
+		SYSCTL_ADD_UQUAD(ctx, SYSCTL_CHILDREN(node), OID_AUTO,
+		    desc[2 * x], CTLFLAG_RD, arg + x, desc[2 * x + 1]);
+	}
+}
+
+static int
+mlx5e_ethtool_handler(SYSCTL_HANDLER_ARGS)
+{
+	struct mlx5e_priv *priv = arg1;
+	uint64_t value;
+	int was_opened;
+	int error;
+
+	PRIV_LOCK(priv);
+	value = priv->params_ethtool.arg[arg2];
+	error = sysctl_handle_64(oidp, &value, 0, req);
+	if (error || req->newptr == NULL ||
+	    value == priv->params_ethtool.arg[arg2])
+		goto done;
+
+	/* assign new value */
+	priv->params_ethtool.arg[arg2] = value;
+
+	/* check if device is gone */
+	if (priv->gone) {
+		error = ENXIO;
+		goto done;
+	}
+
+	if (&priv->params_ethtool.arg[arg2] == &priv->params_ethtool.rx_pauseframe_control ||
+	    &priv->params_ethtool.arg[arg2] == &priv->params_ethtool.tx_pauseframe_control) {
+		/* range check parameters */
+		priv->params_ethtool.rx_pauseframe_control =
+		    priv->params_ethtool.rx_pauseframe_control ? 1 : 0;
+		priv->params_ethtool.tx_pauseframe_control =
+		    priv->params_ethtool.tx_pauseframe_control ? 1 : 0;
+
+		/* update firmware */
+		error = -mlx5_set_port_pause(priv->mdev, 1,
+		    priv->params_ethtool.rx_pauseframe_control,
+		    priv->params_ethtool.tx_pauseframe_control);
+		goto done;
+	}
+
+	was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
+	if (was_opened)
+		mlx5e_close_locked(priv->ifp);
+
+	/* import TX queue size */
+	if (priv->params_ethtool.tx_queue_size <
+	    (1 << MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE)) {
+		priv->params_ethtool.tx_queue_size =
+		    (1 << MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE);
+	} else if (priv->params_ethtool.tx_queue_size >
+	    priv->params_ethtool.tx_queue_size_max) {
+		priv->params_ethtool.tx_queue_size =
+		    priv->params_ethtool.tx_queue_size_max;
+	}
+	priv->params.log_sq_size =
+	    order_base_2(priv->params_ethtool.tx_queue_size);
+
+	/* import RX queue size */
+	if (priv->params_ethtool.rx_queue_size <
+	    (1 << MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE)) {
+		priv->params_ethtool.rx_queue_size =
+		    (1 << MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE);
+	} else if (priv->params_ethtool.rx_queue_size >
+	    priv->params_ethtool.rx_queue_size_max) {
+		priv->params_ethtool.rx_queue_size =
+		    priv->params_ethtool.rx_queue_size_max;
+	}
+	priv->params.log_rq_size =
+	    order_base_2(priv->params_ethtool.rx_queue_size);
+
+	priv->params.min_rx_wqes = min_t (u16,
+	          priv->params_ethtool.rx_queue_size - 1,
+	          MLX5E_PARAMS_DEFAULT_MIN_RX_WQES);
+
+	/* import number of channels */
+	if (priv->params_ethtool.channels < 1)
+		priv->params_ethtool.channels = 1;
+	else if (priv->params_ethtool.channels >
+	    (u64) priv->mdev->priv.eq_table.num_comp_vectors) {
+		priv->params_ethtool.channels =
+		    (u64) priv->mdev->priv.eq_table.num_comp_vectors;
+	}
+	priv->params.num_channels = priv->params_ethtool.channels;
+
+	/* import RX mode */
+	if (priv->params_ethtool.rx_coalesce_mode != 0)
+		priv->params_ethtool.rx_coalesce_mode = 1;
+	priv->params.rx_cq_moderation_mode = priv->params_ethtool.rx_coalesce_mode;
+
+	/* import RX coal time */
+	if (priv->params_ethtool.rx_coalesce_usecs < 1)
+		priv->params_ethtool.rx_coalesce_usecs = 0;
+	else if (priv->params_ethtool.rx_coalesce_usecs >
+	    MLX5E_FLD_MAX(cqc, cq_period)) {
+		priv->params_ethtool.rx_coalesce_usecs =
+		    MLX5E_FLD_MAX(cqc, cq_period);
+	}
+	priv->params.rx_cq_moderation_usec = priv->params_ethtool.rx_coalesce_usecs;
+
+	/* import RX coal pkts */
+	if (priv->params_ethtool.rx_coalesce_pkts < 1)
+		priv->params_ethtool.rx_coalesce_pkts = 0;
+	else if (priv->params_ethtool.rx_coalesce_pkts >
+	    MLX5E_FLD_MAX(cqc, cq_max_count)) {
+		priv->params_ethtool.rx_coalesce_pkts =
+		    MLX5E_FLD_MAX(cqc, cq_max_count);
+	}
+	priv->params.rx_cq_moderation_pkts = priv->params_ethtool.rx_coalesce_pkts;
+
+	/* import TX coal time */
+	if (priv->params_ethtool.tx_coalesce_usecs < 1)
+		priv->params_ethtool.tx_coalesce_usecs = 0;
+	else if (priv->params_ethtool.tx_coalesce_usecs >
+	    MLX5E_FLD_MAX(cqc, cq_period)) {
+		priv->params_ethtool.tx_coalesce_usecs =
+		    MLX5E_FLD_MAX(cqc, cq_period);
+	}
+	priv->params.tx_cq_moderation_usec = priv->params_ethtool.tx_coalesce_usecs;
+
+	/* import TX coal pkts */
+	if (priv->params_ethtool.tx_coalesce_pkts < 1)
+		priv->params_ethtool.tx_coalesce_pkts = 0;
+	else if (priv->params_ethtool.tx_coalesce_pkts >
+	    MLX5E_FLD_MAX(cqc, cq_max_count)) {
+		priv->params_ethtool.tx_coalesce_pkts = MLX5E_FLD_MAX(cqc, cq_max_count);
+	}
+	priv->params.tx_cq_moderation_pkts = priv->params_ethtool.tx_coalesce_pkts;
+
+	/* we always agree to turn off HW LRO - but not always to turn on */
+	if (priv->params_ethtool.hw_lro) {
+		if (priv->params_ethtool.hw_lro != 1) {
+			priv->params_ethtool.hw_lro = priv->params.hw_lro_en;
+			error = EINVAL;
+			goto done;
+		}
+		if (priv->ifp->if_capenable & IFCAP_LRO)
+			priv->params.hw_lro_en = !!MLX5_CAP_ETH(priv->mdev, lro_cap);
+		else {
+			/* set the correct (0) value to params_ethtool.hw_lro, issue a warning and return error */
+			priv->params_ethtool.hw_lro = 0;
+			error = EINVAL;
+			if_printf(priv->ifp, "Can't set HW_LRO to a device with LRO turned off");
+			goto done;
+		}
+	} else {
+		priv->params.hw_lro_en = false;
+	}
+
+	if (was_opened)
+		mlx5e_open_locked(priv->ifp);
+done:
+	PRIV_UNLOCK(priv);
+	return (error);
+}
+
+/*
+ * Read the first three bytes of the eeprom in order to get the needed info
+ * for the whole reading.
+ * Byte 0 - Identifier byte
+ * Byte 1 - Revision byte
+ * Byte 2 - Status byte
+ */
+static int
+mlx5e_get_eeprom_info(struct mlx5e_priv *priv, struct mlx5e_eeprom *eeprom)
+{
+	struct mlx5_core_dev *dev = priv->mdev;
+	u32 data = 0;
+	int size_read = 0;
+	int ret;
+
+	ret = mlx5_query_module_num(dev, &eeprom->module_num);
+	if (ret) {
+		if_printf(priv->ifp, "%s:%d: Failed query module error=%d\n",
+		    __func__, __LINE__, ret);
+		return (ret);
+	}
+
+	/* Read the first three bytes to get Identifier, Revision and Status */
+	ret = mlx5_query_eeprom(dev, eeprom->i2c_addr, eeprom->page_num,
+	    eeprom->device_addr, MLX5E_EEPROM_INFO_BYTES, eeprom->module_num, &data,
+	    &size_read);
+	if (ret) {
+		if_printf(priv->ifp, "%s:%d: Failed query eeprom module error=0x%x\n",
+		    __func__, __LINE__, ret);
+		return (ret);
+	}
+
+	switch (data & MLX5_EEPROM_IDENTIFIER_BYTE_MASK) {
+	case SFF_8024_ID_QSFP:
+		eeprom->type = MLX5E_ETH_MODULE_SFF_8436;
+		eeprom->len = MLX5E_ETH_MODULE_SFF_8436_LEN;
+		break;
+	case SFF_8024_ID_QSFPPLUS:
+	case SFF_8024_ID_QSFP28:
+		if ((data & MLX5_EEPROM_IDENTIFIER_BYTE_MASK) == SFF_8024_ID_QSFP28 ||
+		    ((data & MLX5_EEPROM_REVISION_ID_BYTE_MASK) >> 8) >= 0x3) {
+			eeprom->type = MLX5E_ETH_MODULE_SFF_8636;
+			eeprom->len = MLX5E_ETH_MODULE_SFF_8636_LEN;
+		} else {
+			eeprom->type = MLX5E_ETH_MODULE_SFF_8436;
+			eeprom->len = MLX5E_ETH_MODULE_SFF_8436_LEN;
+		}
+		if ((data & MLX5_EEPROM_PAGE_3_VALID_BIT_MASK) == 0)
+			eeprom->page_valid = 1;
+		break;
+	case SFF_8024_ID_SFP:
+		eeprom->type = MLX5E_ETH_MODULE_SFF_8472;
+		eeprom->len = MLX5E_ETH_MODULE_SFF_8472_LEN;
+		break;
+	default:
+		if_printf(priv->ifp, "%s:%d: Not recognized cable type = 0x%x(%s)\n",
+		    __func__, __LINE__, data & MLX5_EEPROM_IDENTIFIER_BYTE_MASK,
+		    sff_8024_id[data & MLX5_EEPROM_IDENTIFIER_BYTE_MASK]);
+		return (EINVAL);
+	}
+	return (0);
+}
+
+/* Read both low and high pages of the eeprom */
+static int
+mlx5e_get_eeprom(struct mlx5e_priv *priv, struct mlx5e_eeprom *ee)
+{
+	struct mlx5_core_dev *dev = priv->mdev;
+	int size_read = 0;
+	int ret;
+
+	if (ee->len == 0)
+		return (EINVAL);
+
+	/* Read low page of the eeprom */
+	while (ee->device_addr < ee->len) {
+		ret = mlx5_query_eeprom(dev, ee->i2c_addr, ee->page_num, ee->device_addr,
+		    ee->len - ee->device_addr, ee->module_num,
+		    ee->data + (ee->device_addr / 4), &size_read);
+		if (ret) {
+			if_printf(priv->ifp, "%s:%d: Failed reading eeprom, "
+			    "error = 0x%02x\n", __func__, __LINE__, ret);
+			return (ret);
+		}
+		ee->device_addr += size_read;
+	}
+
+	/* Read high page of the eeprom */
+	if (ee->page_valid) {
+		ee->device_addr = MLX5E_EEPROM_HIGH_PAGE_OFFSET;
+		ee->page_num = MLX5E_EEPROM_HIGH_PAGE;
+		size_read = 0;
+		while (ee->device_addr < MLX5E_EEPROM_PAGE_LENGTH) {
+			ret = mlx5_query_eeprom(dev, ee->i2c_addr, ee->page_num,
+			    ee->device_addr, MLX5E_EEPROM_PAGE_LENGTH - ee->device_addr,
+			    ee->module_num, ee->data + (ee->len / 4) +
+			    ((ee->device_addr - MLX5E_EEPROM_HIGH_PAGE_OFFSET) / 4),
+			    &size_read);
+			if (ret) {
+				if_printf(priv->ifp, "%s:%d: Failed reading eeprom, "
+				    "error = 0x%02x\n", __func__, __LINE__, ret);
+				return (ret);
+			}
+			ee->device_addr += size_read;
+		}
+	}
+	return (0);
+}
+
+static void
+mlx5e_print_eeprom(struct mlx5e_eeprom *eeprom)
+{
+	int i, j = 0;
+	int row = 0;
+
+	printf("\nOffset\t\tValues\n");
+	printf("------\t\t------\n");
+	while (row < eeprom->len) {
+		printf("0x%04x\t\t", row);
+		for (i = 0; i < 16; i++) {
+			printf("%02x ", ((u8 *)eeprom->data)[j]);
+			j++;
+			row++;
+		}
+		printf("\n");
+	}
+
+	if (eeprom->page_valid) {
+		row = MLX5E_EEPROM_HIGH_PAGE_OFFSET;
+		printf("\nUpper Page 0x03\n");
+		printf("\nOffset\t\tValues\n");
+		printf("------\t\t------\n");
+		while (row < MLX5E_EEPROM_PAGE_LENGTH) {
+			printf("0x%04x\t\t", row);
+			for (i = 0; i < 16; i++) {
+				printf("%02x ", ((u8 *)eeprom->data)[j]);
+				j++;
+				row++;
+			}
+			printf("\n");
+		}
+	}
+}
+
+/*
+ * Read cable EEPROM module information by first inspecting the first
+ * three bytes to get the initial information for a whole reading.
+ * Information will be printed to dmesg.
+ */
+static int
+mlx5e_read_eeprom(SYSCTL_HANDLER_ARGS)
+{
+	struct mlx5e_priv *priv = arg1;
+	struct mlx5e_eeprom eeprom;
+	int error;
+	int result = 0;
+
+	PRIV_LOCK(priv);
+	error = sysctl_handle_int(oidp, &result, 0, req);
+	if (error || !req->newptr)
+		goto done;
+
+	/* Check if device is gone */
+	if (priv->gone) {
+		error = ENXIO;
+		goto done;
+	}
+
+	if (result == 1) {
+		eeprom.i2c_addr = MLX5E_I2C_ADDR_LOW;
+		eeprom.device_addr = 0;
+		eeprom.page_num = MLX5E_EEPROM_LOW_PAGE;
+		eeprom.page_valid = 0;
+
+		/* Read three first bytes to get important info */
+		error = mlx5e_get_eeprom_info(priv, &eeprom);
+		if (error) {
+			if_printf(priv->ifp, "%s:%d: Failed reading eeprom's "
+			    "initial information\n", __func__, __LINE__);
+			error = 0;
+			goto done;
+		}
+		/*
+		 * Allocate needed length buffer and additional space for
+		 * page 0x03
+		 */
+		eeprom.data = malloc(eeprom.len + MLX5E_EEPROM_PAGE_LENGTH,
+		    M_MLX5EN, M_WAITOK | M_ZERO);
+
+		/* Read the whole eeprom information */
+		error = mlx5e_get_eeprom(priv, &eeprom);
+		if (error) {
+			if_printf(priv->ifp, "%s:%d: Failed reading eeprom\n",
+			    __func__, __LINE__);
+			error = 0;
+			/*
+			 * Continue printing partial information in case of
+			 * an error
+			 */
+		}
+		mlx5e_print_eeprom(&eeprom);
+		free(eeprom.data, M_MLX5EN);
+	}
+done:
+	PRIV_UNLOCK(priv);
+	return (error);
+}
+
+static const char *mlx5e_params_desc[] = {
+	MLX5E_PARAMS(MLX5E_STATS_DESC)
+};
+
+static const char *mlx5e_port_stats_debug_desc[] = {
+	MLX5E_PORT_STATS_DEBUG(MLX5E_STATS_DESC)
+};
+
+static int
+mlx5e_ethtool_debug_stats(SYSCTL_HANDLER_ARGS)
+{
+	struct mlx5e_priv *priv = arg1;
+	int error;
+	int sys_debug;
+
+	sys_debug = priv->sysctl_debug;
+	error = sysctl_handle_int(oidp, &priv->sysctl_debug, 0, req);
+	if (error || !req->newptr)
+		return (error);
+	priv->sysctl_debug = !!priv->sysctl_debug;
+	if (sys_debug == priv->sysctl_debug)
+		return (error);
+	if (priv->sysctl_debug)
+		mlx5e_create_stats(&priv->stats.port_stats_debug.ctx,
+		    SYSCTL_CHILDREN(priv->sysctl_ifnet), "debug_stats",
+		    mlx5e_port_stats_debug_desc, MLX5E_PORT_STATS_DEBUG_NUM,
+		    priv->stats.port_stats_debug.arg);
+	else
+		sysctl_ctx_free(&priv->stats.port_stats_debug.ctx);
+	return (error);
+}
+
+void
+mlx5e_create_ethtool(struct mlx5e_priv *priv)
+{
+	struct sysctl_oid *node;
+	const char *pnameunit;
+	unsigned x;
+
+	/* set some defaults */
+	priv->params_ethtool.tx_queue_size_max = 1 << MLX5E_PARAMS_MAXIMUM_LOG_SQ_SIZE;
+	priv->params_ethtool.rx_queue_size_max = 1 << MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE;
+	priv->params_ethtool.tx_queue_size = 1 << priv->params.log_sq_size;
+	priv->params_ethtool.rx_queue_size = 1 << priv->params.log_rq_size;
+	priv->params_ethtool.channels = priv->params.num_channels;
+	priv->params_ethtool.coalesce_pkts_max = MLX5E_FLD_MAX(cqc, cq_max_count);
+	priv->params_ethtool.coalesce_usecs_max = MLX5E_FLD_MAX(cqc, cq_period);
+	priv->params_ethtool.rx_coalesce_mode = priv->params.rx_cq_moderation_mode;
+	priv->params_ethtool.rx_coalesce_usecs = priv->params.rx_cq_moderation_usec;
+	priv->params_ethtool.rx_coalesce_pkts = priv->params.rx_cq_moderation_pkts;
+	priv->params_ethtool.tx_coalesce_usecs = priv->params.tx_cq_moderation_usec;
+	priv->params_ethtool.tx_coalesce_pkts = priv->params.tx_cq_moderation_pkts;
+	priv->params_ethtool.hw_lro = priv->params.hw_lro_en;
+
+	/* create root node */
+	node = SYSCTL_ADD_NODE(&priv->sysctl_ctx,
+	    SYSCTL_CHILDREN(priv->sysctl_ifnet), OID_AUTO,
+	    "conf", CTLFLAG_RW, NULL, "Configuration");
+	if (node == NULL)
+		return;
+	for (x = 0; x != MLX5E_PARAMS_NUM; x++) {
+		/* check for read-only parameter */
+		if (strstr(mlx5e_params_desc[2 * x], "_max") != NULL) {
+			SYSCTL_ADD_PROC(&priv->sysctl_ctx, SYSCTL_CHILDREN(node), OID_AUTO,
+			    mlx5e_params_desc[2 * x], CTLTYPE_U64 | CTLFLAG_RD |
+			    CTLFLAG_MPSAFE, priv, x, &mlx5e_ethtool_handler, "QU",
+			    mlx5e_params_desc[2 * x + 1]);
+		} else {
+			SYSCTL_ADD_PROC(&priv->sysctl_ctx, SYSCTL_CHILDREN(node), OID_AUTO,
+			    mlx5e_params_desc[2 * x], CTLTYPE_U64 | CTLFLAG_RWTUN |
+			    CTLFLAG_MPSAFE, priv, x, &mlx5e_ethtool_handler, "QU",
+			    mlx5e_params_desc[2 * x + 1]);
+		}
+	}
+
+	SYSCTL_ADD_PROC(&priv->sysctl_ctx, SYSCTL_CHILDREN(node), OID_AUTO,
+	    "debug_stats", CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, priv,
+	    0, &mlx5e_ethtool_debug_stats, "I", "Extended debug statistics");
+
+	pnameunit = device_get_nameunit(priv->mdev->pdev->dev.bsddev);
+
+	SYSCTL_ADD_STRING(&priv->sysctl_ctx, SYSCTL_CHILDREN(node),
+	    OID_AUTO, "device_name", CTLFLAG_RD,
+	    __DECONST(void *, pnameunit), 0,
+	    "PCI device name");
+
+	/* EEPROM support */
+	SYSCTL_ADD_PROC(&priv->sysctl_ctx, SYSCTL_CHILDREN(node), OID_AUTO, "eeprom_info",
+	    CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, priv, 0,
+	    mlx5e_read_eeprom, "I", "EEPROM information");
+}
diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_flow_table.c b/sys/dev/mlx5/mlx5_en/mlx5_en_flow_table.c
new file mode 100644
index 0000000..c618c92
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_en/mlx5_en_flow_table.c
@@ -0,0 +1,867 @@
+/*-
+ * Copyright (c) 2015 Mellanox Technologies. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include "en.h"
+
+#include <linux/list.h>
+#include <dev/mlx5/flow_table.h>
+
+enum {
+	MLX5E_FULLMATCH = 0,
+	MLX5E_ALLMULTI = 1,
+	MLX5E_PROMISC = 2,
+};
+
+enum {
+	MLX5E_UC = 0,
+	MLX5E_MC_IPV4 = 1,
+	MLX5E_MC_IPV6 = 2,
+	MLX5E_MC_OTHER = 3,
+};
+
+enum {
+	MLX5E_ACTION_NONE = 0,
+	MLX5E_ACTION_ADD = 1,
+	MLX5E_ACTION_DEL = 2,
+};
+
+struct mlx5e_eth_addr_hash_node {
+	LIST_ENTRY(mlx5e_eth_addr_hash_node) hlist;
+	u8	action;
+	struct mlx5e_eth_addr_info ai;
+};
+
+static inline int
+mlx5e_hash_eth_addr(const u8 * addr)
+{
+	return (addr[5]);
+}
+
+static void
+mlx5e_add_eth_addr_to_hash(struct mlx5e_eth_addr_hash_head *hash,
+    const u8 * addr)
+{
+	struct mlx5e_eth_addr_hash_node *hn;
+	int ix = mlx5e_hash_eth_addr(addr);
+
+	LIST_FOREACH(hn, &hash[ix], hlist) {
+		if (bcmp(hn->ai.addr, addr, ETHER_ADDR_LEN) == 0) {
+			if (hn->action == MLX5E_ACTION_DEL)
+				hn->action = MLX5E_ACTION_NONE;
+			return;
+		}
+	}
+
+	hn = malloc(sizeof(*hn), M_MLX5EN, M_NOWAIT | M_ZERO);
+	if (hn == NULL)
+		return;
+
+	ether_addr_copy(hn->ai.addr, addr);
+	hn->action = MLX5E_ACTION_ADD;
+
+	LIST_INSERT_HEAD(&hash[ix], hn, hlist);
+}
+
+static void
+mlx5e_del_eth_addr_from_hash(struct mlx5e_eth_addr_hash_node *hn)
+{
+	LIST_REMOVE(hn, hlist);
+	free(hn, M_MLX5EN);
+}
+
+static void
+mlx5e_del_eth_addr_from_flow_table(struct mlx5e_priv *priv,
+    struct mlx5e_eth_addr_info *ai)
+{
+	void *ft = priv->ft.main;
+
+	if (ai->tt_vec & (1 << MLX5E_TT_IPV6_TCP))
+		mlx5_del_flow_table_entry(ft, ai->ft_ix[MLX5E_TT_IPV6_TCP]);
+
+	if (ai->tt_vec & (1 << MLX5E_TT_IPV4_TCP))
+		mlx5_del_flow_table_entry(ft, ai->ft_ix[MLX5E_TT_IPV4_TCP]);
+
+	if (ai->tt_vec & (1 << MLX5E_TT_IPV6_UDP))
+		mlx5_del_flow_table_entry(ft, ai->ft_ix[MLX5E_TT_IPV6_UDP]);
+
+	if (ai->tt_vec & (1 << MLX5E_TT_IPV4_UDP))
+		mlx5_del_flow_table_entry(ft, ai->ft_ix[MLX5E_TT_IPV4_UDP]);
+
+	if (ai->tt_vec & (1 << MLX5E_TT_IPV6))
+		mlx5_del_flow_table_entry(ft, ai->ft_ix[MLX5E_TT_IPV6]);
+
+	if (ai->tt_vec & (1 << MLX5E_TT_IPV4))
+		mlx5_del_flow_table_entry(ft, ai->ft_ix[MLX5E_TT_IPV4]);
+
+	if (ai->tt_vec & (1 << MLX5E_TT_ANY))
+		mlx5_del_flow_table_entry(ft, ai->ft_ix[MLX5E_TT_ANY]);
+}
+
+static int
+mlx5e_get_eth_addr_type(const u8 * addr)
+{
+	if (ETHER_IS_MULTICAST(addr) == 0)
+		return (MLX5E_UC);
+
+	if ((addr[0] == 0x01) &&
+	    (addr[1] == 0x00) &&
+	    (addr[2] == 0x5e) &&
+	    !(addr[3] & 0x80))
+		return (MLX5E_MC_IPV4);
+
+	if ((addr[0] == 0x33) &&
+	    (addr[1] == 0x33))
+		return (MLX5E_MC_IPV6);
+
+	return (MLX5E_MC_OTHER);
+}
+
+static	u32
+mlx5e_get_tt_vec(struct mlx5e_eth_addr_info *ai, int type)
+{
+	int eth_addr_type;
+	u32 ret;
+
+	switch (type) {
+	case MLX5E_FULLMATCH:
+		eth_addr_type = mlx5e_get_eth_addr_type(ai->addr);
+		switch (eth_addr_type) {
+		case MLX5E_UC:
+			ret =
+			    (1 << MLX5E_TT_IPV4_TCP) |
+			    (1 << MLX5E_TT_IPV6_TCP) |
+			    (1 << MLX5E_TT_IPV4_UDP) |
+			    (1 << MLX5E_TT_IPV6_UDP) |
+			    (1 << MLX5E_TT_IPV4) |
+			    (1 << MLX5E_TT_IPV6) |
+			    (1 << MLX5E_TT_ANY) |
+			    0;
+			break;
+
+		case MLX5E_MC_IPV4:
+			ret =
+			    (1 << MLX5E_TT_IPV4_UDP) |
+			    (1 << MLX5E_TT_IPV4) |
+			    0;
+			break;
+
+		case MLX5E_MC_IPV6:
+			ret =
+			    (1 << MLX5E_TT_IPV6_UDP) |
+			    (1 << MLX5E_TT_IPV6) |
+			    0;
+			break;
+
+		default:
+			ret =
+			    (1 << MLX5E_TT_ANY) |
+			    0;
+			break;
+		}
+		break;
+
+	case MLX5E_ALLMULTI:
+		ret =
+		    (1 << MLX5E_TT_IPV4_UDP) |
+		    (1 << MLX5E_TT_IPV6_UDP) |
+		    (1 << MLX5E_TT_IPV4) |
+		    (1 << MLX5E_TT_IPV6) |
+		    (1 << MLX5E_TT_ANY) |
+		    0;
+		break;
+
+	default:			/* MLX5E_PROMISC */
+		ret =
+		    (1 << MLX5E_TT_IPV4_TCP) |
+		    (1 << MLX5E_TT_IPV6_TCP) |
+		    (1 << MLX5E_TT_IPV4_UDP) |
+		    (1 << MLX5E_TT_IPV6_UDP) |
+		    (1 << MLX5E_TT_IPV4) |
+		    (1 << MLX5E_TT_IPV6) |
+		    (1 << MLX5E_TT_ANY) |
+		    0;
+		break;
+	}
+
+	return (ret);
+}
+
+static int
+mlx5e_add_eth_addr_rule_sub(struct mlx5e_priv *priv,
+    struct mlx5e_eth_addr_info *ai, int type,
+    void *flow_context, void *match_criteria)
+{
+	u8 match_criteria_enable = 0;
+	void *match_value;
+	void *dest;
+	u8 *dmac;
+	u8 *match_criteria_dmac;
+	void *ft = priv->ft.main;
+	u32 *tirn = priv->tirn;
+	u32 tt_vec;
+	int err;
+
+	match_value = MLX5_ADDR_OF(flow_context, flow_context, match_value);
+	dmac = MLX5_ADDR_OF(fte_match_param, match_value,
+	    outer_headers.dmac_47_16);
+	match_criteria_dmac = MLX5_ADDR_OF(fte_match_param, match_criteria,
+	    outer_headers.dmac_47_16);
+	dest = MLX5_ADDR_OF(flow_context, flow_context, destination);
+
+	MLX5_SET(flow_context, flow_context, action,
+	    MLX5_FLOW_CONTEXT_ACTION_FWD_DEST);
+	MLX5_SET(flow_context, flow_context, destination_list_size, 1);
+	MLX5_SET(dest_format_struct, dest, destination_type,
+	    MLX5_FLOW_CONTEXT_DEST_TYPE_TIR);
+
+	switch (type) {
+	case MLX5E_FULLMATCH:
+		match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
+		memset(match_criteria_dmac, 0xff, ETH_ALEN);
+		ether_addr_copy(dmac, ai->addr);
+		break;
+
+	case MLX5E_ALLMULTI:
+		match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
+		match_criteria_dmac[0] = 0x01;
+		dmac[0] = 0x01;
+		break;
+
+	case MLX5E_PROMISC:
+		break;
+	default:
+		break;
+	}
+
+	tt_vec = mlx5e_get_tt_vec(ai, type);
+
+	if (tt_vec & (1 << MLX5E_TT_ANY)) {
+		MLX5_SET(dest_format_struct, dest, destination_id,
+		    tirn[MLX5E_TT_ANY]);
+		err = mlx5_add_flow_table_entry(ft, match_criteria_enable,
+		    match_criteria, flow_context, &ai->ft_ix[MLX5E_TT_ANY]);
+		if (err) {
+			mlx5e_del_eth_addr_from_flow_table(priv, ai);
+			return (err);
+		}
+		ai->tt_vec |= (1 << MLX5E_TT_ANY);
+	}
+	match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
+	MLX5_SET_TO_ONES(fte_match_param, match_criteria,
+	    outer_headers.ethertype);
+
+	if (tt_vec & (1 << MLX5E_TT_IPV4)) {
+		MLX5_SET(fte_match_param, match_value, outer_headers.ethertype,
+		    ETHERTYPE_IP);
+		MLX5_SET(dest_format_struct, dest, destination_id,
+		    tirn[MLX5E_TT_IPV4]);
+		err = mlx5_add_flow_table_entry(ft, match_criteria_enable,
+		    match_criteria, flow_context, &ai->ft_ix[MLX5E_TT_IPV4]);
+		if (err) {
+			mlx5e_del_eth_addr_from_flow_table(priv, ai);
+			return (err);
+		}
+		ai->tt_vec |= (1 << MLX5E_TT_IPV4);
+	}
+	if (tt_vec & (1 << MLX5E_TT_IPV6)) {
+		MLX5_SET(fte_match_param, match_value, outer_headers.ethertype,
+		    ETHERTYPE_IPV6);
+		MLX5_SET(dest_format_struct, dest, destination_id,
+		    tirn[MLX5E_TT_IPV6]);
+		err = mlx5_add_flow_table_entry(ft, match_criteria_enable,
+		    match_criteria, flow_context, &ai->ft_ix[MLX5E_TT_IPV6]);
+		if (err) {
+			mlx5e_del_eth_addr_from_flow_table(priv, ai);
+			return (err);
+		}
+		ai->tt_vec |= (1 << MLX5E_TT_IPV6);
+	}
+	MLX5_SET_TO_ONES(fte_match_param, match_criteria,
+	    outer_headers.ip_protocol);
+	MLX5_SET(fte_match_param, match_value, outer_headers.ip_protocol,
+	    IPPROTO_UDP);
+
+	if (tt_vec & (1 << MLX5E_TT_IPV4_UDP)) {
+		MLX5_SET(fte_match_param, match_value, outer_headers.ethertype,
+		    ETHERTYPE_IP);
+		MLX5_SET(dest_format_struct, dest, destination_id,
+		    tirn[MLX5E_TT_IPV4_UDP]);
+		err = mlx5_add_flow_table_entry(ft, match_criteria_enable,
+		    match_criteria, flow_context, &ai->ft_ix[MLX5E_TT_IPV4_UDP]);
+		if (err) {
+			mlx5e_del_eth_addr_from_flow_table(priv, ai);
+			return (err);
+		}
+		ai->tt_vec |= (1 << MLX5E_TT_IPV4_UDP);
+	}
+	if (tt_vec & (1 << MLX5E_TT_IPV6_UDP)) {
+		MLX5_SET(fte_match_param, match_value, outer_headers.ethertype,
+		    ETHERTYPE_IPV6);
+		MLX5_SET(dest_format_struct, dest, destination_id,
+		    tirn[MLX5E_TT_IPV6_UDP]);
+		err = mlx5_add_flow_table_entry(ft, match_criteria_enable,
+		    match_criteria, flow_context, &ai->ft_ix[MLX5E_TT_IPV6_UDP]);
+		if (err) {
+			mlx5e_del_eth_addr_from_flow_table(priv, ai);
+			return (err);
+		}
+		ai->tt_vec |= (1 << MLX5E_TT_IPV6_UDP);
+	}
+	MLX5_SET(fte_match_param, match_value, outer_headers.ip_protocol,
+	    IPPROTO_TCP);
+
+	if (tt_vec & (1 << MLX5E_TT_IPV4_TCP)) {
+		MLX5_SET(fte_match_param, match_value, outer_headers.ethertype,
+		    ETHERTYPE_IP);
+		MLX5_SET(dest_format_struct, dest, destination_id,
+		    tirn[MLX5E_TT_IPV4_TCP]);
+		err = mlx5_add_flow_table_entry(ft, match_criteria_enable,
+		    match_criteria, flow_context, &ai->ft_ix[MLX5E_TT_IPV4_TCP]);
+		if (err) {
+			mlx5e_del_eth_addr_from_flow_table(priv, ai);
+			return (err);
+		}
+		ai->tt_vec |= (1 << MLX5E_TT_IPV4_TCP);
+	}
+	if (tt_vec & (1 << MLX5E_TT_IPV6_TCP)) {
+		MLX5_SET(fte_match_param, match_value, outer_headers.ethertype,
+		    ETHERTYPE_IPV6);
+		MLX5_SET(dest_format_struct, dest, destination_id,
+		    tirn[MLX5E_TT_IPV6_TCP]);
+		err = mlx5_add_flow_table_entry(ft, match_criteria_enable,
+		    match_criteria, flow_context, &ai->ft_ix[MLX5E_TT_IPV6_TCP]);
+		if (err) {
+			mlx5e_del_eth_addr_from_flow_table(priv, ai);
+			return (err);
+		}
+		ai->tt_vec |= (1 << MLX5E_TT_IPV6_TCP);
+	}
+	return (0);
+}
+
+static int
+mlx5e_add_eth_addr_rule(struct mlx5e_priv *priv,
+    struct mlx5e_eth_addr_info *ai, int type)
+{
+	u32 *flow_context;
+	u32 *match_criteria;
+	int err;
+
+	flow_context = mlx5_vzalloc(MLX5_ST_SZ_BYTES(flow_context) +
+	    MLX5_ST_SZ_BYTES(dest_format_struct));
+	match_criteria = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param));
+	if (!flow_context || !match_criteria) {
+		if_printf(priv->ifp, "%s: alloc failed\n", __func__);
+		err = -ENOMEM;
+		goto add_eth_addr_rule_out;
+	}
+	err = mlx5e_add_eth_addr_rule_sub(priv, ai, type, flow_context,
+	    match_criteria);
+	if (err)
+		if_printf(priv->ifp, "%s: failed\n", __func__);
+
+add_eth_addr_rule_out:
+	kvfree(match_criteria);
+	kvfree(flow_context);
+	return (err);
+}
+
+enum mlx5e_vlan_rule_type {
+	MLX5E_VLAN_RULE_TYPE_UNTAGGED,
+	MLX5E_VLAN_RULE_TYPE_ANY_VID,
+	MLX5E_VLAN_RULE_TYPE_MATCH_VID,
+};
+
+static int
+mlx5e_add_vlan_rule(struct mlx5e_priv *priv,
+    enum mlx5e_vlan_rule_type rule_type, u16 vid)
+{
+	u8 match_criteria_enable = 0;
+	u32 *flow_context;
+	void *match_value;
+	void *dest;
+	u32 *match_criteria;
+	u32 *ft_ix;
+	int err;
+
+	flow_context = mlx5_vzalloc(MLX5_ST_SZ_BYTES(flow_context) +
+	    MLX5_ST_SZ_BYTES(dest_format_struct));
+	match_criteria = mlx5_vzalloc(MLX5_ST_SZ_BYTES(fte_match_param));
+	if (!flow_context || !match_criteria) {
+		if_printf(priv->ifp, "%s: alloc failed\n", __func__);
+		err = -ENOMEM;
+		goto add_vlan_rule_out;
+	}
+	match_value = MLX5_ADDR_OF(flow_context, flow_context, match_value);
+	dest = MLX5_ADDR_OF(flow_context, flow_context, destination);
+
+	MLX5_SET(flow_context, flow_context, action,
+	    MLX5_FLOW_CONTEXT_ACTION_FWD_DEST);
+	MLX5_SET(flow_context, flow_context, destination_list_size, 1);
+	MLX5_SET(dest_format_struct, dest, destination_type,
+	    MLX5_FLOW_CONTEXT_DEST_TYPE_FLOW_TABLE);
+	MLX5_SET(dest_format_struct, dest, destination_id,
+	    mlx5_get_flow_table_id(priv->ft.main));
+
+	match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
+	MLX5_SET_TO_ONES(fte_match_param, match_criteria,
+	    outer_headers.vlan_tag);
+
+	switch (rule_type) {
+	case MLX5E_VLAN_RULE_TYPE_UNTAGGED:
+		ft_ix = &priv->vlan.untagged_rule_ft_ix;
+		break;
+	case MLX5E_VLAN_RULE_TYPE_ANY_VID:
+		ft_ix = &priv->vlan.any_vlan_rule_ft_ix;
+		MLX5_SET(fte_match_param, match_value, outer_headers.vlan_tag,
+		    1);
+		break;
+	default:			/* MLX5E_VLAN_RULE_TYPE_MATCH_VID */
+		ft_ix = &priv->vlan.active_vlans_ft_ix[vid];
+		MLX5_SET(fte_match_param, match_value, outer_headers.vlan_tag,
+		    1);
+		MLX5_SET_TO_ONES(fte_match_param, match_criteria,
+		    outer_headers.first_vid);
+		MLX5_SET(fte_match_param, match_value, outer_headers.first_vid,
+		    vid);
+		break;
+	}
+
+	err = mlx5_add_flow_table_entry(priv->ft.vlan, match_criteria_enable,
+	    match_criteria, flow_context, ft_ix);
+	if (err)
+		if_printf(priv->ifp, "%s: failed\n", __func__);
+
+add_vlan_rule_out:
+	kvfree(match_criteria);
+	kvfree(flow_context);
+	return (err);
+}
+
+static void
+mlx5e_del_vlan_rule(struct mlx5e_priv *priv,
+    enum mlx5e_vlan_rule_type rule_type, u16 vid)
+{
+	switch (rule_type) {
+	case MLX5E_VLAN_RULE_TYPE_UNTAGGED:
+		mlx5_del_flow_table_entry(priv->ft.vlan,
+		    priv->vlan.untagged_rule_ft_ix);
+		break;
+	case MLX5E_VLAN_RULE_TYPE_ANY_VID:
+		mlx5_del_flow_table_entry(priv->ft.vlan,
+		    priv->vlan.any_vlan_rule_ft_ix);
+		break;
+	case MLX5E_VLAN_RULE_TYPE_MATCH_VID:
+		mlx5_del_flow_table_entry(priv->ft.vlan,
+		    priv->vlan.active_vlans_ft_ix[vid]);
+		break;
+	}
+}
+
+void
+mlx5e_enable_vlan_filter(struct mlx5e_priv *priv)
+{
+	if (priv->vlan.filter_disabled) {
+		priv->vlan.filter_disabled = false;
+		if (test_bit(MLX5E_STATE_OPENED, &priv->state))
+			mlx5e_del_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_ANY_VID,
+			    0);
+	}
+}
+
+void
+mlx5e_disable_vlan_filter(struct mlx5e_priv *priv)
+{
+	if (!priv->vlan.filter_disabled) {
+		priv->vlan.filter_disabled = true;
+		if (test_bit(MLX5E_STATE_OPENED, &priv->state))
+			mlx5e_add_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_ANY_VID,
+			    0);
+	}
+}
+
+void
+mlx5e_vlan_rx_add_vid(void *arg, struct ifnet *ifp, u16 vid)
+{
+	struct mlx5e_priv *priv = arg;
+
+	if (ifp != priv->ifp)
+		return;
+
+	PRIV_LOCK(priv);
+	set_bit(vid, priv->vlan.active_vlans);
+	if (test_bit(MLX5E_STATE_OPENED, &priv->state))
+		mlx5e_add_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_MATCH_VID, vid);
+	PRIV_UNLOCK(priv);
+}
+
+void
+mlx5e_vlan_rx_kill_vid(void *arg, struct ifnet *ifp, u16 vid)
+{
+	struct mlx5e_priv *priv = arg;
+
+	if (ifp != priv->ifp)
+		return;
+
+	PRIV_LOCK(priv);
+	clear_bit(vid, priv->vlan.active_vlans);
+	if (test_bit(MLX5E_STATE_OPENED, &priv->state))
+		mlx5e_del_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_MATCH_VID, vid);
+	PRIV_UNLOCK(priv);
+}
+
+int
+mlx5e_add_all_vlan_rules(struct mlx5e_priv *priv)
+{
+	u16 vid;
+	int err;
+
+	for_each_set_bit(vid, priv->vlan.active_vlans, VLAN_N_VID) {
+		err = mlx5e_add_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_MATCH_VID,
+		    vid);
+		if (err)
+			return (err);
+	}
+
+	err = mlx5e_add_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_UNTAGGED, 0);
+	if (err)
+		return (err);
+
+	if (priv->vlan.filter_disabled) {
+		err = mlx5e_add_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_ANY_VID,
+		    0);
+		if (err)
+			return (err);
+	}
+	return (0);
+}
+
+void
+mlx5e_del_all_vlan_rules(struct mlx5e_priv *priv)
+{
+	u16 vid;
+
+	if (priv->vlan.filter_disabled)
+		mlx5e_del_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_ANY_VID, 0);
+
+	mlx5e_del_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_UNTAGGED, 0);
+
+	for_each_set_bit(vid, priv->vlan.active_vlans, VLAN_N_VID)
+	    mlx5e_del_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_MATCH_VID, vid);
+}
+
+#define	mlx5e_for_each_hash_node(hn, tmp, hash, i) \
+	for (i = 0; i < MLX5E_ETH_ADDR_HASH_SIZE; i++) \
+		LIST_FOREACH_SAFE(hn, &(hash)[i], hlist, tmp)
+
+static void
+mlx5e_execute_action(struct mlx5e_priv *priv,
+    struct mlx5e_eth_addr_hash_node *hn)
+{
+	switch (hn->action) {
+	case MLX5E_ACTION_ADD:
+		mlx5e_add_eth_addr_rule(priv, &hn->ai, MLX5E_FULLMATCH);
+		hn->action = MLX5E_ACTION_NONE;
+		break;
+
+	case MLX5E_ACTION_DEL:
+		mlx5e_del_eth_addr_from_flow_table(priv, &hn->ai);
+		mlx5e_del_eth_addr_from_hash(hn);
+		break;
+
+	default:
+		break;
+	}
+}
+
+static void
+mlx5e_sync_ifp_addr(struct mlx5e_priv *priv)
+{
+	struct ifnet *ifp = priv->ifp;
+	struct ifaddr *ifa;
+	struct ifmultiaddr *ifma;
+
+	/* XXX adding this entry might not be needed */
+	mlx5e_add_eth_addr_to_hash(priv->eth_addr.if_uc,
+	    LLADDR((struct sockaddr_dl *)(ifp->if_addr->ifa_addr)));
+
+	if_addr_rlock(ifp);
+	TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
+		if (ifa->ifa_addr->sa_family != AF_LINK)
+			continue;
+		mlx5e_add_eth_addr_to_hash(priv->eth_addr.if_uc,
+		    LLADDR((struct sockaddr_dl *)ifa->ifa_addr));
+	}
+	if_addr_runlock(ifp);
+
+	if_maddr_rlock(ifp);
+	TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
+		if (ifma->ifma_addr->sa_family != AF_LINK)
+			continue;
+		mlx5e_add_eth_addr_to_hash(priv->eth_addr.if_mc,
+		    LLADDR((struct sockaddr_dl *)ifma->ifma_addr));
+	}
+	if_maddr_runlock(ifp);
+}
+
+static void
+mlx5e_apply_ifp_addr(struct mlx5e_priv *priv)
+{
+	struct mlx5e_eth_addr_hash_node *hn;
+	struct mlx5e_eth_addr_hash_node *tmp;
+	int i;
+
+	mlx5e_for_each_hash_node(hn, tmp, priv->eth_addr.if_uc, i)
+	    mlx5e_execute_action(priv, hn);
+
+	mlx5e_for_each_hash_node(hn, tmp, priv->eth_addr.if_mc, i)
+	    mlx5e_execute_action(priv, hn);
+}
+
+static void
+mlx5e_handle_ifp_addr(struct mlx5e_priv *priv)
+{
+	struct mlx5e_eth_addr_hash_node *hn;
+	struct mlx5e_eth_addr_hash_node *tmp;
+	int i;
+
+	mlx5e_for_each_hash_node(hn, tmp, priv->eth_addr.if_uc, i)
+	    hn->action = MLX5E_ACTION_DEL;
+	mlx5e_for_each_hash_node(hn, tmp, priv->eth_addr.if_mc, i)
+	    hn->action = MLX5E_ACTION_DEL;
+
+	if (test_bit(MLX5E_STATE_OPENED, &priv->state))
+		mlx5e_sync_ifp_addr(priv);
+
+	mlx5e_apply_ifp_addr(priv);
+}
+
+void
+mlx5e_set_rx_mode_core(struct mlx5e_priv *priv)
+{
+	struct mlx5e_eth_addr_db *ea = &priv->eth_addr;
+	struct ifnet *ndev = priv->ifp;
+
+	bool rx_mode_enable = test_bit(MLX5E_STATE_OPENED, &priv->state);
+	bool promisc_enabled = rx_mode_enable && (ndev->if_flags & IFF_PROMISC);
+	bool allmulti_enabled = rx_mode_enable && (ndev->if_flags & IFF_ALLMULTI);
+	bool broadcast_enabled = rx_mode_enable;
+
+	bool enable_promisc = !ea->promisc_enabled && promisc_enabled;
+	bool disable_promisc = ea->promisc_enabled && !promisc_enabled;
+	bool enable_allmulti = !ea->allmulti_enabled && allmulti_enabled;
+	bool disable_allmulti = ea->allmulti_enabled && !allmulti_enabled;
+	bool enable_broadcast = !ea->broadcast_enabled && broadcast_enabled;
+	bool disable_broadcast = ea->broadcast_enabled && !broadcast_enabled;
+
+	/* update broadcast address */
+	ether_addr_copy(priv->eth_addr.broadcast.addr,
+	    priv->ifp->if_broadcastaddr);
+
+	if (enable_promisc)
+		mlx5e_add_eth_addr_rule(priv, &ea->promisc, MLX5E_PROMISC);
+	if (enable_allmulti)
+		mlx5e_add_eth_addr_rule(priv, &ea->allmulti, MLX5E_ALLMULTI);
+	if (enable_broadcast)
+		mlx5e_add_eth_addr_rule(priv, &ea->broadcast, MLX5E_FULLMATCH);
+
+	mlx5e_handle_ifp_addr(priv);
+
+	if (disable_broadcast)
+		mlx5e_del_eth_addr_from_flow_table(priv, &ea->broadcast);
+	if (disable_allmulti)
+		mlx5e_del_eth_addr_from_flow_table(priv, &ea->allmulti);
+	if (disable_promisc)
+		mlx5e_del_eth_addr_from_flow_table(priv, &ea->promisc);
+
+	ea->promisc_enabled = promisc_enabled;
+	ea->allmulti_enabled = allmulti_enabled;
+	ea->broadcast_enabled = broadcast_enabled;
+}
+
+void
+mlx5e_set_rx_mode_work(struct work_struct *work)
+{
+	struct mlx5e_priv *priv =
+	    container_of(work, struct mlx5e_priv, set_rx_mode_work);
+
+	PRIV_LOCK(priv);
+	if (test_bit(MLX5E_STATE_OPENED, &priv->state))
+		mlx5e_set_rx_mode_core(priv);
+	PRIV_UNLOCK(priv);
+}
+
+static int
+mlx5e_create_main_flow_table(struct mlx5e_priv *priv)
+{
+	struct mlx5_flow_table_group *g;
+	u8 *dmac;
+
+	g = malloc(9 * sizeof(*g), M_MLX5EN, M_WAITOK | M_ZERO);
+	if (g == NULL)
+		return (-ENOMEM);
+
+	g[0].log_sz = 2;
+	g[0].match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
+	MLX5_SET_TO_ONES(fte_match_param, g[0].match_criteria,
+	    outer_headers.ethertype);
+	MLX5_SET_TO_ONES(fte_match_param, g[0].match_criteria,
+	    outer_headers.ip_protocol);
+
+	g[1].log_sz = 1;
+	g[1].match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
+	MLX5_SET_TO_ONES(fte_match_param, g[1].match_criteria,
+	    outer_headers.ethertype);
+
+	g[2].log_sz = 0;
+
+	g[3].log_sz = 14;
+	g[3].match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
+	dmac = MLX5_ADDR_OF(fte_match_param, g[3].match_criteria,
+	    outer_headers.dmac_47_16);
+	memset(dmac, 0xff, ETH_ALEN);
+	MLX5_SET_TO_ONES(fte_match_param, g[3].match_criteria,
+	    outer_headers.ethertype);
+	MLX5_SET_TO_ONES(fte_match_param, g[3].match_criteria,
+	    outer_headers.ip_protocol);
+
+	g[4].log_sz = 13;
+	g[4].match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
+	dmac = MLX5_ADDR_OF(fte_match_param, g[4].match_criteria,
+	    outer_headers.dmac_47_16);
+	memset(dmac, 0xff, ETH_ALEN);
+	MLX5_SET_TO_ONES(fte_match_param, g[4].match_criteria,
+	    outer_headers.ethertype);
+
+	g[5].log_sz = 11;
+	g[5].match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
+	dmac = MLX5_ADDR_OF(fte_match_param, g[5].match_criteria,
+	    outer_headers.dmac_47_16);
+	memset(dmac, 0xff, ETH_ALEN);
+
+	g[6].log_sz = 2;
+	g[6].match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
+	dmac = MLX5_ADDR_OF(fte_match_param, g[6].match_criteria,
+	    outer_headers.dmac_47_16);
+	dmac[0] = 0x01;
+	MLX5_SET_TO_ONES(fte_match_param, g[6].match_criteria,
+	    outer_headers.ethertype);
+	MLX5_SET_TO_ONES(fte_match_param, g[6].match_criteria,
+	    outer_headers.ip_protocol);
+
+	g[7].log_sz = 1;
+	g[7].match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
+	dmac = MLX5_ADDR_OF(fte_match_param, g[7].match_criteria,
+	    outer_headers.dmac_47_16);
+	dmac[0] = 0x01;
+	MLX5_SET_TO_ONES(fte_match_param, g[7].match_criteria,
+	    outer_headers.ethertype);
+
+	g[8].log_sz = 0;
+	g[8].match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
+	dmac = MLX5_ADDR_OF(fte_match_param, g[8].match_criteria,
+	    outer_headers.dmac_47_16);
+	dmac[0] = 0x01;
+	priv->ft.main = mlx5_create_flow_table(priv->mdev, 1,
+	    MLX5_FLOW_TABLE_TYPE_NIC_RCV,
+	    0, 9, g);
+	free(g, M_MLX5EN);
+
+	return (priv->ft.main ? 0 : -ENOMEM);
+}
+
+static void
+mlx5e_destroy_main_flow_table(struct mlx5e_priv *priv)
+{
+	mlx5_destroy_flow_table(priv->ft.main);
+	priv->ft.main = NULL;
+}
+
+static int
+mlx5e_create_vlan_flow_table(struct mlx5e_priv *priv)
+{
+	struct mlx5_flow_table_group *g;
+
+	g = malloc(2 * sizeof(*g), M_MLX5EN, M_WAITOK | M_ZERO);
+	if (g == NULL)
+		return (-ENOMEM);
+
+	g[0].log_sz = 12;
+	g[0].match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
+	MLX5_SET_TO_ONES(fte_match_param, g[0].match_criteria,
+	    outer_headers.vlan_tag);
+	MLX5_SET_TO_ONES(fte_match_param, g[0].match_criteria,
+	    outer_headers.first_vid);
+
+	/* untagged + any vlan id */
+	g[1].log_sz = 1;
+	g[1].match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
+	MLX5_SET_TO_ONES(fte_match_param, g[1].match_criteria,
+	    outer_headers.vlan_tag);
+
+	priv->ft.vlan = mlx5_create_flow_table(priv->mdev, 0,
+	    MLX5_FLOW_TABLE_TYPE_NIC_RCV,
+	    0, 2, g);
+	free(g, M_MLX5EN);
+
+	return (priv->ft.vlan ? 0 : -ENOMEM);
+}
+
+static void
+mlx5e_destroy_vlan_flow_table(struct mlx5e_priv *priv)
+{
+	mlx5_destroy_flow_table(priv->ft.vlan);
+	priv->ft.vlan = NULL;
+}
+
+int
+mlx5e_open_flow_table(struct mlx5e_priv *priv)
+{
+	int err;
+
+	err = mlx5e_create_main_flow_table(priv);
+	if (err)
+		return (err);
+
+	err = mlx5e_create_vlan_flow_table(priv);
+	if (err)
+		goto err_destroy_main_flow_table;
+
+	return (0);
+
+err_destroy_main_flow_table:
+	mlx5e_destroy_main_flow_table(priv);
+
+	return (err);
+}
+
+void
+mlx5e_close_flow_table(struct mlx5e_priv *priv)
+{
+	mlx5e_destroy_vlan_flow_table(priv);
+	mlx5e_destroy_main_flow_table(priv);
+}
diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_main.c b/sys/dev/mlx5/mlx5_en/mlx5_en_main.c
new file mode 100644
index 0000000..a4a1ec7
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_en/mlx5_en_main.c
@@ -0,0 +1,2906 @@
+/*-
+ * Copyright (c) 2015 Mellanox Technologies. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include "en.h"
+
+#include <sys/sockio.h>
+#include <machine/atomic.h>
+
+#define	ETH_DRIVER_VERSION	"3.1.0-dev"
+char mlx5e_version[] = "Mellanox Ethernet driver"
+    " (" ETH_DRIVER_VERSION ")";
+
+struct mlx5e_rq_param {
+	u32	rqc [MLX5_ST_SZ_DW(rqc)];
+	struct mlx5_wq_param wq;
+};
+
+struct mlx5e_sq_param {
+	u32	sqc [MLX5_ST_SZ_DW(sqc)];
+	struct mlx5_wq_param wq;
+};
+
+struct mlx5e_cq_param {
+	u32	cqc [MLX5_ST_SZ_DW(cqc)];
+	struct mlx5_wq_param wq;
+	u16	eq_ix;
+};
+
+struct mlx5e_channel_param {
+	struct mlx5e_rq_param rq;
+	struct mlx5e_sq_param sq;
+	struct mlx5e_cq_param rx_cq;
+	struct mlx5e_cq_param tx_cq;
+};
+
+static const struct {
+	u32	subtype;
+	u64	baudrate;
+}	mlx5e_mode_table[MLX5E_LINK_MODES_NUMBER] = {
+
+	[MLX5E_1000BASE_CX_SGMII] = {
+		.subtype = IFM_1000_CX_SGMII,
+		.baudrate = IF_Mbps(1000ULL),
+	},
+	[MLX5E_1000BASE_KX] = {
+		.subtype = IFM_1000_KX,
+		.baudrate = IF_Mbps(1000ULL),
+	},
+	[MLX5E_10GBASE_CX4] = {
+		.subtype = IFM_10G_CX4,
+		.baudrate = IF_Gbps(10ULL),
+	},
+	[MLX5E_10GBASE_KX4] = {
+		.subtype = IFM_10G_KX4,
+		.baudrate = IF_Gbps(10ULL),
+	},
+	[MLX5E_10GBASE_KR] = {
+		.subtype = IFM_10G_KR,
+		.baudrate = IF_Gbps(10ULL),
+	},
+	[MLX5E_20GBASE_KR2] = {
+		.subtype = IFM_20G_KR2,
+		.baudrate = IF_Gbps(20ULL),
+	},
+	[MLX5E_40GBASE_CR4] = {
+		.subtype = IFM_40G_CR4,
+		.baudrate = IF_Gbps(40ULL),
+	},
+	[MLX5E_40GBASE_KR4] = {
+		.subtype = IFM_40G_KR4,
+		.baudrate = IF_Gbps(40ULL),
+	},
+	[MLX5E_56GBASE_R4] = {
+		.subtype = IFM_56G_R4,
+		.baudrate = IF_Gbps(56ULL),
+	},
+	[MLX5E_10GBASE_CR] = {
+		.subtype = IFM_10G_CR1,
+		.baudrate = IF_Gbps(10ULL),
+	},
+	[MLX5E_10GBASE_SR] = {
+		.subtype = IFM_10G_SR,
+		.baudrate = IF_Gbps(10ULL),
+	},
+	[MLX5E_10GBASE_ER] = {
+		.subtype = IFM_10G_ER,
+		.baudrate = IF_Gbps(10ULL),
+	},
+	[MLX5E_40GBASE_SR4] = {
+		.subtype = IFM_40G_SR4,
+		.baudrate = IF_Gbps(40ULL),
+	},
+	[MLX5E_40GBASE_LR4] = {
+		.subtype = IFM_40G_LR4,
+		.baudrate = IF_Gbps(40ULL),
+	},
+	[MLX5E_100GBASE_CR4] = {
+		.subtype = IFM_100G_CR4,
+		.baudrate = IF_Gbps(100ULL),
+	},
+	[MLX5E_100GBASE_SR4] = {
+		.subtype = IFM_100G_SR4,
+		.baudrate = IF_Gbps(100ULL),
+	},
+	[MLX5E_100GBASE_KR4] = {
+		.subtype = IFM_100G_KR4,
+		.baudrate = IF_Gbps(100ULL),
+	},
+	[MLX5E_100GBASE_LR4] = {
+		.subtype = IFM_100G_LR4,
+		.baudrate = IF_Gbps(100ULL),
+	},
+	[MLX5E_100BASE_TX] = {
+		.subtype = IFM_100_TX,
+		.baudrate = IF_Mbps(100ULL),
+	},
+	[MLX5E_100BASE_T] = {
+		.subtype = IFM_100_T,
+		.baudrate = IF_Mbps(100ULL),
+	},
+	[MLX5E_10GBASE_T] = {
+		.subtype = IFM_10G_T,
+		.baudrate = IF_Gbps(10ULL),
+	},
+	[MLX5E_25GBASE_CR] = {
+		.subtype = IFM_25G_CR,
+		.baudrate = IF_Gbps(25ULL),
+	},
+	[MLX5E_25GBASE_KR] = {
+		.subtype = IFM_25G_KR,
+		.baudrate = IF_Gbps(25ULL),
+	},
+	[MLX5E_25GBASE_SR] = {
+		.subtype = IFM_25G_SR,
+		.baudrate = IF_Gbps(25ULL),
+	},
+	[MLX5E_50GBASE_CR2] = {
+		.subtype = IFM_50G_CR2,
+		.baudrate = IF_Gbps(50ULL),
+	},
+	[MLX5E_50GBASE_KR2] = {
+		.subtype = IFM_50G_KR2,
+		.baudrate = IF_Gbps(50ULL),
+	},
+};
+
+MALLOC_DEFINE(M_MLX5EN, "MLX5EN", "MLX5 Ethernet");
+
+static void
+mlx5e_update_carrier(struct mlx5e_priv *priv)
+{
+	struct mlx5_core_dev *mdev = priv->mdev;
+	u32 out[MLX5_ST_SZ_DW(ptys_reg)];
+	u32 eth_proto_oper;
+	int error;
+	u8 port_state;
+	u8 i;
+
+	port_state = mlx5_query_vport_state(mdev,
+	    MLX5_QUERY_VPORT_STATE_IN_OP_MOD_VNIC_VPORT);
+
+	if (port_state == VPORT_STATE_UP) {
+		priv->media_status_last |= IFM_ACTIVE;
+	} else {
+		priv->media_status_last &= ~IFM_ACTIVE;
+		priv->media_active_last = IFM_ETHER;
+		if_link_state_change(priv->ifp, LINK_STATE_DOWN);
+		return;
+	}
+
+	error = mlx5_query_port_ptys(mdev, out, sizeof(out), MLX5_PTYS_EN);
+	if (error) {
+		priv->media_active_last = IFM_ETHER;
+		priv->ifp->if_baudrate = 1;
+		if_printf(priv->ifp, "%s: query port ptys failed: 0x%x\n",
+		    __func__, error);
+		return;
+	}
+	eth_proto_oper = MLX5_GET(ptys_reg, out, eth_proto_oper);
+
+	for (i = 0; i != MLX5E_LINK_MODES_NUMBER; i++) {
+		if (mlx5e_mode_table[i].baudrate == 0)
+			continue;
+		if (MLX5E_PROT_MASK(i) & eth_proto_oper) {
+			priv->ifp->if_baudrate =
+			    mlx5e_mode_table[i].baudrate;
+			priv->media_active_last =
+			    mlx5e_mode_table[i].subtype | IFM_ETHER | IFM_FDX;
+		}
+	}
+	if_link_state_change(priv->ifp, LINK_STATE_UP);
+}
+
+static void
+mlx5e_media_status(struct ifnet *dev, struct ifmediareq *ifmr)
+{
+	struct mlx5e_priv *priv = dev->if_softc;
+
+	ifmr->ifm_status = priv->media_status_last;
+	ifmr->ifm_active = priv->media_active_last |
+	    (priv->params_ethtool.rx_pauseframe_control ? IFM_ETH_RXPAUSE : 0) |
+	    (priv->params_ethtool.tx_pauseframe_control ? IFM_ETH_TXPAUSE : 0);
+
+}
+
+static u32
+mlx5e_find_link_mode(u32 subtype)
+{
+	u32 i;
+	u32 link_mode = 0;
+
+	for (i = 0; i < MLX5E_LINK_MODES_NUMBER; ++i) {
+		if (mlx5e_mode_table[i].baudrate == 0)
+			continue;
+		if (mlx5e_mode_table[i].subtype == subtype)
+			link_mode |= MLX5E_PROT_MASK(i);
+	}
+
+	return (link_mode);
+}
+
+static int
+mlx5e_media_change(struct ifnet *dev)
+{
+	struct mlx5e_priv *priv = dev->if_softc;
+	struct mlx5_core_dev *mdev = priv->mdev;
+	u32 eth_proto_cap;
+	u32 link_mode;
+	int locked;
+	int error;
+
+	locked = PRIV_LOCKED(priv);
+	if (!locked)
+		PRIV_LOCK(priv);
+
+	if (IFM_TYPE(priv->media.ifm_media) != IFM_ETHER) {
+		error = EINVAL;
+		goto done;
+	}
+	link_mode = mlx5e_find_link_mode(IFM_SUBTYPE(priv->media.ifm_media));
+
+	error = mlx5_query_port_proto_cap(mdev, &eth_proto_cap, MLX5_PTYS_EN);
+	if (error) {
+		if_printf(dev, "Query port media capability failed\n");
+		goto done;
+	}
+	if (IFM_SUBTYPE(priv->media.ifm_media) == IFM_AUTO)
+		link_mode = eth_proto_cap;
+	else
+		link_mode = link_mode & eth_proto_cap;
+
+	if (!link_mode) {
+		if_printf(dev, "Not supported link mode requested\n");
+		error = EINVAL;
+		goto done;
+	}
+	mlx5_set_port_status(mdev, MLX5_PORT_DOWN);
+	mlx5_set_port_proto(mdev, link_mode, MLX5_PTYS_EN);
+	mlx5_set_port_status(mdev, MLX5_PORT_UP);
+
+done:
+	if (!locked)
+		PRIV_UNLOCK(priv);
+	return (error);
+}
+
+static void
+mlx5e_update_carrier_work(struct work_struct *work)
+{
+	struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv,
+	    update_carrier_work);
+
+	PRIV_LOCK(priv);
+	if (test_bit(MLX5E_STATE_OPENED, &priv->state))
+		mlx5e_update_carrier(priv);
+	PRIV_UNLOCK(priv);
+}
+
+static void
+mlx5e_update_pport_counters(struct mlx5e_priv *priv)
+{
+	struct mlx5_core_dev *mdev = priv->mdev;
+	struct mlx5e_pport_stats *s = &priv->stats.pport;
+	struct mlx5e_port_stats_debug *s_debug = &priv->stats.port_stats_debug;
+	u32 *in;
+	u32 *out;
+	u64 *ptr;
+	unsigned sz = MLX5_ST_SZ_BYTES(ppcnt_reg);
+	unsigned x;
+	unsigned y;
+
+	in = mlx5_vzalloc(sz);
+	out = mlx5_vzalloc(sz);
+	if (in == NULL || out == NULL)
+		goto free_out;
+
+	ptr = (uint64_t *)MLX5_ADDR_OF(ppcnt_reg, out, counter_set);
+
+	MLX5_SET(ppcnt_reg, in, local_port, 1);
+
+	MLX5_SET(ppcnt_reg, in, grp, MLX5_IEEE_802_3_COUNTERS_GROUP);
+	mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
+	for (x = y = 0; x != MLX5E_PPORT_IEEE802_3_STATS_NUM; x++, y++)
+		s->arg[y] = be64toh(ptr[x]);
+
+	MLX5_SET(ppcnt_reg, in, grp, MLX5_RFC_2819_COUNTERS_GROUP);
+	mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
+	for (x = 0; x != MLX5E_PPORT_RFC2819_STATS_NUM; x++, y++)
+		s->arg[y] = be64toh(ptr[x]);
+	for (y = 0; x != MLX5E_PPORT_RFC2819_STATS_NUM +
+	    MLX5E_PPORT_RFC2819_STATS_DEBUG_NUM; x++, y++)
+		s_debug->arg[y] = be64toh(ptr[x]);
+
+	MLX5_SET(ppcnt_reg, in, grp, MLX5_RFC_2863_COUNTERS_GROUP);
+	mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
+	for (x = 0; x != MLX5E_PPORT_RFC2863_STATS_DEBUG_NUM; x++, y++)
+		s_debug->arg[y] = be64toh(ptr[x]);
+
+	MLX5_SET(ppcnt_reg, in, grp, MLX5_PHYSICAL_LAYER_COUNTERS_GROUP);
+	mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
+	for (x = 0; x != MLX5E_PPORT_PHYSICAL_LAYER_STATS_DEBUG_NUM; x++, y++)
+		s_debug->arg[y] = be64toh(ptr[x]);
+free_out:
+	kvfree(in);
+	kvfree(out);
+}
+
+static void
+mlx5e_update_stats_work(struct work_struct *work)
+{
+	struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv,
+	    update_stats_work);
+	struct mlx5_core_dev *mdev = priv->mdev;
+	struct mlx5e_vport_stats *s = &priv->stats.vport;
+	struct mlx5e_rq_stats *rq_stats;
+	struct mlx5e_sq_stats *sq_stats;
+	struct buf_ring *sq_br;
+#if (__FreeBSD_version < 1100000)
+	struct ifnet *ifp = priv->ifp;
+#endif
+
+	u32 in[MLX5_ST_SZ_DW(query_vport_counter_in)];
+	u32 *out;
+	int outlen = MLX5_ST_SZ_BYTES(query_vport_counter_out);
+	u64 tso_packets = 0;
+	u64 tso_bytes = 0;
+	u64 tx_queue_dropped = 0;
+	u64 tx_defragged = 0;
+	u64 tx_offload_none = 0;
+	u64 lro_packets = 0;
+	u64 lro_bytes = 0;
+	u64 sw_lro_queued = 0;
+	u64 sw_lro_flushed = 0;
+	u64 rx_csum_none = 0;
+	u64 rx_wqe_err = 0;
+	u32 rx_out_of_buffer = 0;
+	int i;
+	int j;
+
+	PRIV_LOCK(priv);
+	out = mlx5_vzalloc(outlen);
+	if (out == NULL)
+		goto free_out;
+	if (test_bit(MLX5E_STATE_OPENED, &priv->state) == 0)
+		goto free_out;
+
+	/* Collect firts the SW counters and then HW for consistency */
+	for (i = 0; i < priv->params.num_channels; i++) {
+		struct mlx5e_rq *rq = &priv->channel[i]->rq;
+
+		rq_stats = &priv->channel[i]->rq.stats;
+
+		/* collect stats from LRO */
+		rq_stats->sw_lro_queued = rq->lro.lro_queued;
+		rq_stats->sw_lro_flushed = rq->lro.lro_flushed;
+		sw_lro_queued += rq_stats->sw_lro_queued;
+		sw_lro_flushed += rq_stats->sw_lro_flushed;
+		lro_packets += rq_stats->lro_packets;
+		lro_bytes += rq_stats->lro_bytes;
+		rx_csum_none += rq_stats->csum_none;
+		rx_wqe_err += rq_stats->wqe_err;
+
+		for (j = 0; j < priv->num_tc; j++) {
+			sq_stats = &priv->channel[i]->sq[j].stats;
+			sq_br = priv->channel[i]->sq[j].br;
+
+			tso_packets += sq_stats->tso_packets;
+			tso_bytes += sq_stats->tso_bytes;
+			tx_queue_dropped += sq_stats->dropped;
+			tx_queue_dropped += sq_br->br_drops;
+			tx_defragged += sq_stats->defragged;
+			tx_offload_none += sq_stats->csum_offload_none;
+		}
+	}
+
+	/* update counters */
+	s->tso_packets = tso_packets;
+	s->tso_bytes = tso_bytes;
+	s->tx_queue_dropped = tx_queue_dropped;
+	s->tx_defragged = tx_defragged;
+	s->lro_packets = lro_packets;
+	s->lro_bytes = lro_bytes;
+	s->sw_lro_queued = sw_lro_queued;
+	s->sw_lro_flushed = sw_lro_flushed;
+	s->rx_csum_none = rx_csum_none;
+	s->rx_wqe_err = rx_wqe_err;
+
+	/* HW counters */
+	memset(in, 0, sizeof(in));
+
+	MLX5_SET(query_vport_counter_in, in, opcode,
+	    MLX5_CMD_OP_QUERY_VPORT_COUNTER);
+	MLX5_SET(query_vport_counter_in, in, op_mod, 0);
+	MLX5_SET(query_vport_counter_in, in, other_vport, 0);
+
+	memset(out, 0, outlen);
+
+	/* get number of out-of-buffer drops first */
+	if (mlx5_vport_query_out_of_rx_buffer(mdev, priv->counter_set_id,
+	    &rx_out_of_buffer))
+		goto free_out;
+
+	/* accumulate difference into a 64-bit counter */
+	s->rx_out_of_buffer += (u64)(u32)(rx_out_of_buffer - s->rx_out_of_buffer_prev);
+	s->rx_out_of_buffer_prev = rx_out_of_buffer;
+
+	/* get port statistics */
+	if (mlx5_cmd_exec(mdev, in, sizeof(in), out, outlen))
+		goto free_out;
+
+#define	MLX5_GET_CTR(out, x) \
+	MLX5_GET64(query_vport_counter_out, out, x)
+
+	s->rx_error_packets =
+	    MLX5_GET_CTR(out, received_errors.packets);
+	s->rx_error_bytes =
+	    MLX5_GET_CTR(out, received_errors.octets);
+	s->tx_error_packets =
+	    MLX5_GET_CTR(out, transmit_errors.packets);
+	s->tx_error_bytes =
+	    MLX5_GET_CTR(out, transmit_errors.octets);
+
+	s->rx_unicast_packets =
+	    MLX5_GET_CTR(out, received_eth_unicast.packets);
+	s->rx_unicast_bytes =
+	    MLX5_GET_CTR(out, received_eth_unicast.octets);
+	s->tx_unicast_packets =
+	    MLX5_GET_CTR(out, transmitted_eth_unicast.packets);
+	s->tx_unicast_bytes =
+	    MLX5_GET_CTR(out, transmitted_eth_unicast.octets);
+
+	s->rx_multicast_packets =
+	    MLX5_GET_CTR(out, received_eth_multicast.packets);
+	s->rx_multicast_bytes =
+	    MLX5_GET_CTR(out, received_eth_multicast.octets);
+	s->tx_multicast_packets =
+	    MLX5_GET_CTR(out, transmitted_eth_multicast.packets);
+	s->tx_multicast_bytes =
+	    MLX5_GET_CTR(out, transmitted_eth_multicast.octets);
+
+	s->rx_broadcast_packets =
+	    MLX5_GET_CTR(out, received_eth_broadcast.packets);
+	s->rx_broadcast_bytes =
+	    MLX5_GET_CTR(out, received_eth_broadcast.octets);
+	s->tx_broadcast_packets =
+	    MLX5_GET_CTR(out, transmitted_eth_broadcast.packets);
+	s->tx_broadcast_bytes =
+	    MLX5_GET_CTR(out, transmitted_eth_broadcast.octets);
+
+	s->rx_packets =
+	    s->rx_unicast_packets +
+	    s->rx_multicast_packets +
+	    s->rx_broadcast_packets -
+	    s->rx_out_of_buffer;
+	s->rx_bytes =
+	    s->rx_unicast_bytes +
+	    s->rx_multicast_bytes +
+	    s->rx_broadcast_bytes;
+	s->tx_packets =
+	    s->tx_unicast_packets +
+	    s->tx_multicast_packets +
+	    s->tx_broadcast_packets;
+	s->tx_bytes =
+	    s->tx_unicast_bytes +
+	    s->tx_multicast_bytes +
+	    s->tx_broadcast_bytes;
+
+	/* Update calculated offload counters */
+	s->tx_csum_offload = s->tx_packets - tx_offload_none;
+	s->rx_csum_good = s->rx_packets - s->rx_csum_none;
+
+	/* Update per port counters */
+	mlx5e_update_pport_counters(priv);
+
+#if (__FreeBSD_version < 1100000)
+	/* no get_counters interface in fbsd 10 */
+	ifp->if_ipackets = s->rx_packets;
+	ifp->if_ierrors = s->rx_error_packets;
+	ifp->if_iqdrops = s->rx_out_of_buffer;
+	ifp->if_opackets = s->tx_packets;
+	ifp->if_oerrors = s->tx_error_packets;
+	ifp->if_snd.ifq_drops = s->tx_queue_dropped;
+	ifp->if_ibytes = s->rx_bytes;
+	ifp->if_obytes = s->tx_bytes;
+#endif
+
+free_out:
+	kvfree(out);
+	PRIV_UNLOCK(priv);
+}
+
+static void
+mlx5e_update_stats(void *arg)
+{
+	struct mlx5e_priv *priv = arg;
+
+	schedule_work(&priv->update_stats_work);
+
+	callout_reset(&priv->watchdog, hz, &mlx5e_update_stats, priv);
+}
+
+static void
+mlx5e_async_event_sub(struct mlx5e_priv *priv,
+    enum mlx5_dev_event event)
+{
+	switch (event) {
+	case MLX5_DEV_EVENT_PORT_UP:
+	case MLX5_DEV_EVENT_PORT_DOWN:
+		schedule_work(&priv->update_carrier_work);
+		break;
+
+	default:
+		break;
+	}
+}
+
+static void
+mlx5e_async_event(struct mlx5_core_dev *mdev, void *vpriv,
+    enum mlx5_dev_event event, unsigned long param)
+{
+	struct mlx5e_priv *priv = vpriv;
+
+	mtx_lock(&priv->async_events_mtx);
+	if (test_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLE, &priv->state))
+		mlx5e_async_event_sub(priv, event);
+	mtx_unlock(&priv->async_events_mtx);
+}
+
+static void
+mlx5e_enable_async_events(struct mlx5e_priv *priv)
+{
+	set_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLE, &priv->state);
+}
+
+static void
+mlx5e_disable_async_events(struct mlx5e_priv *priv)
+{
+	mtx_lock(&priv->async_events_mtx);
+	clear_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLE, &priv->state);
+	mtx_unlock(&priv->async_events_mtx);
+}
+
+static const char *mlx5e_rq_stats_desc[] = {
+	MLX5E_RQ_STATS(MLX5E_STATS_DESC)
+};
+
+static int
+mlx5e_create_rq(struct mlx5e_channel *c,
+    struct mlx5e_rq_param *param,
+    struct mlx5e_rq *rq)
+{
+	struct mlx5e_priv *priv = c->priv;
+	struct mlx5_core_dev *mdev = priv->mdev;
+	char buffer[16];
+	void *rqc = param->rqc;
+	void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq);
+	int wq_sz;
+	int err;
+	int i;
+
+	/* Create DMA descriptor TAG */
+	if ((err = -bus_dma_tag_create(
+	    bus_get_dma_tag(mdev->pdev->dev.bsddev),
+	    1,				/* any alignment */
+	    0,				/* no boundary */
+	    BUS_SPACE_MAXADDR,		/* lowaddr */
+	    BUS_SPACE_MAXADDR,		/* highaddr */
+	    NULL, NULL,			/* filter, filterarg */
+	    MJUM16BYTES,		/* maxsize */
+	    1,				/* nsegments */
+	    MJUM16BYTES,		/* maxsegsize */
+	    0,				/* flags */
+	    NULL, NULL,			/* lockfunc, lockfuncarg */
+	    &rq->dma_tag)))
+		goto done;
+
+	err = mlx5_wq_ll_create(mdev, &param->wq, rqc_wq, &rq->wq,
+	    &rq->wq_ctrl);
+	if (err)
+		goto err_free_dma_tag;
+
+	rq->wq.db = &rq->wq.db[MLX5_RCV_DBR];
+
+	if (priv->params.hw_lro_en) {
+		rq->wqe_sz = priv->params.lro_wqe_sz;
+	} else {
+		rq->wqe_sz = MLX5E_SW2MB_MTU(priv->ifp->if_mtu);
+	}
+	if (rq->wqe_sz > MJUM16BYTES) {
+		err = -ENOMEM;
+		goto err_rq_wq_destroy;
+	} else if (rq->wqe_sz > MJUM9BYTES) {
+		rq->wqe_sz = MJUM16BYTES;
+	} else if (rq->wqe_sz > MJUMPAGESIZE) {
+		rq->wqe_sz = MJUM9BYTES;
+	} else if (rq->wqe_sz > MCLBYTES) {
+		rq->wqe_sz = MJUMPAGESIZE;
+	} else {
+		rq->wqe_sz = MCLBYTES;
+	}
+
+	wq_sz = mlx5_wq_ll_get_size(&rq->wq);
+	rq->mbuf = malloc(wq_sz * sizeof(rq->mbuf[0]), M_MLX5EN, M_WAITOK | M_ZERO);
+	if (rq->mbuf == NULL) {
+		err = -ENOMEM;
+		goto err_rq_wq_destroy;
+	}
+	for (i = 0; i != wq_sz; i++) {
+		struct mlx5e_rx_wqe *wqe = mlx5_wq_ll_get_wqe(&rq->wq, i);
+		uint32_t byte_count = rq->wqe_sz - MLX5E_NET_IP_ALIGN;
+
+		err = -bus_dmamap_create(rq->dma_tag, 0, &rq->mbuf[i].dma_map);
+		if (err != 0) {
+			while (i--)
+				bus_dmamap_destroy(rq->dma_tag, rq->mbuf[i].dma_map);
+			goto err_rq_mbuf_free;
+		}
+		wqe->data.lkey = c->mkey_be;
+		wqe->data.byte_count = cpu_to_be32(byte_count | MLX5_HW_START_PADDING);
+	}
+
+	rq->pdev = c->pdev;
+	rq->ifp = c->ifp;
+	rq->channel = c;
+	rq->ix = c->ix;
+
+	snprintf(buffer, sizeof(buffer), "rxstat%d", c->ix);
+	mlx5e_create_stats(&rq->stats.ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
+	    buffer, mlx5e_rq_stats_desc, MLX5E_RQ_STATS_NUM,
+	    rq->stats.arg);
+
+#ifdef HAVE_TURBO_LRO
+	if (tcp_tlro_init(&rq->lro, c->ifp, MLX5E_BUDGET_MAX) != 0)
+		rq->lro.mbuf = NULL;
+#else
+	if (tcp_lro_init(&rq->lro))
+		rq->lro.lro_cnt = 0;
+	else
+		rq->lro.ifp = c->ifp;
+#endif
+	return (0);
+
+err_rq_mbuf_free:
+	free(rq->mbuf, M_MLX5EN);
+err_rq_wq_destroy:
+	mlx5_wq_destroy(&rq->wq_ctrl);
+err_free_dma_tag:
+	bus_dma_tag_destroy(rq->dma_tag);
+done:
+	return (err);
+}
+
+static void
+mlx5e_destroy_rq(struct mlx5e_rq *rq)
+{
+	int wq_sz;
+	int i;
+
+	/* destroy all sysctl nodes */
+	sysctl_ctx_free(&rq->stats.ctx);
+
+	/* free leftover LRO packets, if any */
+#ifdef HAVE_TURBO_LRO
+	tcp_tlro_free(&rq->lro);
+#else
+	tcp_lro_free(&rq->lro);
+#endif
+	wq_sz = mlx5_wq_ll_get_size(&rq->wq);
+	for (i = 0; i != wq_sz; i++) {
+		if (rq->mbuf[i].mbuf != NULL) {
+			bus_dmamap_unload(rq->dma_tag,
+			    rq->mbuf[i].dma_map);
+			m_freem(rq->mbuf[i].mbuf);
+		}
+		bus_dmamap_destroy(rq->dma_tag, rq->mbuf[i].dma_map);
+	}
+	free(rq->mbuf, M_MLX5EN);
+	mlx5_wq_destroy(&rq->wq_ctrl);
+}
+
+static int
+mlx5e_enable_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param)
+{
+	struct mlx5e_channel *c = rq->channel;
+	struct mlx5e_priv *priv = c->priv;
+	struct mlx5_core_dev *mdev = priv->mdev;
+
+	void *in;
+	void *rqc;
+	void *wq;
+	int inlen;
+	int err;
+
+	inlen = MLX5_ST_SZ_BYTES(create_rq_in) +
+	    sizeof(u64) * rq->wq_ctrl.buf.npages;
+	in = mlx5_vzalloc(inlen);
+	if (in == NULL)
+		return (-ENOMEM);
+
+	rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
+	wq = MLX5_ADDR_OF(rqc, rqc, wq);
+
+	memcpy(rqc, param->rqc, sizeof(param->rqc));
+
+	MLX5_SET(rqc, rqc, cqn, c->rq.cq.mcq.cqn);
+	MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST);
+	MLX5_SET(rqc, rqc, flush_in_error_en, 1);
+	if (priv->counter_set_id >= 0)
+		MLX5_SET(rqc, rqc, counter_set_id, priv->counter_set_id);
+	MLX5_SET(wq, wq, log_wq_pg_sz, rq->wq_ctrl.buf.page_shift -
+	    PAGE_SHIFT);
+	MLX5_SET64(wq, wq, dbr_addr, rq->wq_ctrl.db.dma);
+
+	mlx5_fill_page_array(&rq->wq_ctrl.buf,
+	    (__be64 *) MLX5_ADDR_OF(wq, wq, pas));
+
+	err = mlx5_core_create_rq(mdev, in, inlen, &rq->rqn);
+
+	kvfree(in);
+
+	return (err);
+}
+
+static int
+mlx5e_modify_rq(struct mlx5e_rq *rq, int curr_state, int next_state)
+{
+	struct mlx5e_channel *c = rq->channel;
+	struct mlx5e_priv *priv = c->priv;
+	struct mlx5_core_dev *mdev = priv->mdev;
+
+	void *in;
+	void *rqc;
+	int inlen;
+	int err;
+
+	inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
+	in = mlx5_vzalloc(inlen);
+	if (in == NULL)
+		return (-ENOMEM);
+
+	rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
+
+	MLX5_SET(modify_rq_in, in, rqn, rq->rqn);
+	MLX5_SET(modify_rq_in, in, rq_state, curr_state);
+	MLX5_SET(rqc, rqc, state, next_state);
+
+	err = mlx5_core_modify_rq(mdev, in, inlen);
+
+	kvfree(in);
+
+	return (err);
+}
+
+static void
+mlx5e_disable_rq(struct mlx5e_rq *rq)
+{
+	struct mlx5e_channel *c = rq->channel;
+	struct mlx5e_priv *priv = c->priv;
+	struct mlx5_core_dev *mdev = priv->mdev;
+
+	mlx5_core_destroy_rq(mdev, rq->rqn);
+}
+
+static int
+mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq)
+{
+	struct mlx5e_channel *c = rq->channel;
+	struct mlx5e_priv *priv = c->priv;
+	struct mlx5_wq_ll *wq = &rq->wq;
+	int i;
+
+	for (i = 0; i < 1000; i++) {
+		if (wq->cur_sz >= priv->params.min_rx_wqes)
+			return (0);
+
+		msleep(4);
+	}
+	return (-ETIMEDOUT);
+}
+
+static int
+mlx5e_open_rq(struct mlx5e_channel *c,
+    struct mlx5e_rq_param *param,
+    struct mlx5e_rq *rq)
+{
+	int err;
+	int i;
+
+	err = mlx5e_create_rq(c, param, rq);
+	if (err)
+		return (err);
+
+	err = mlx5e_enable_rq(rq, param);
+	if (err)
+		goto err_destroy_rq;
+
+	err = mlx5e_modify_rq(rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY);
+	if (err)
+		goto err_disable_rq;
+
+	c->rq.enabled = 1;
+
+	/*
+	 * Test send queues, which will trigger
+	 * "mlx5e_post_rx_wqes()":
+	 */
+	for (i = 0; i != c->num_tc; i++)
+		mlx5e_send_nop(&c->sq[i], 1, true);
+	return (0);
+
+err_disable_rq:
+	mlx5e_disable_rq(rq);
+err_destroy_rq:
+	mlx5e_destroy_rq(rq);
+
+	return (err);
+}
+
+static void
+mlx5e_close_rq(struct mlx5e_rq *rq)
+{
+	rq->enabled = 0;
+	mlx5e_modify_rq(rq, MLX5_RQC_STATE_RDY, MLX5_RQC_STATE_ERR);
+}
+
+static void
+mlx5e_close_rq_wait(struct mlx5e_rq *rq)
+{
+	/* wait till RQ is empty */
+	while (!mlx5_wq_ll_is_empty(&rq->wq)) {
+		msleep(4);
+		rq->cq.mcq.comp(&rq->cq.mcq);
+	}
+
+	mlx5e_disable_rq(rq);
+	mlx5e_destroy_rq(rq);
+}
+
+static void
+mlx5e_free_sq_db(struct mlx5e_sq *sq)
+{
+	int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
+	int x;
+
+	for (x = 0; x != wq_sz; x++)
+		bus_dmamap_destroy(sq->dma_tag, sq->mbuf[x].dma_map);
+	free(sq->mbuf, M_MLX5EN);
+}
+
+static int
+mlx5e_alloc_sq_db(struct mlx5e_sq *sq)
+{
+	int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
+	int err;
+	int x;
+
+	sq->mbuf = malloc(wq_sz * sizeof(sq->mbuf[0]), M_MLX5EN, M_WAITOK | M_ZERO);
+	if (sq->mbuf == NULL)
+		return (-ENOMEM);
+
+	/* Create DMA descriptor MAPs */
+	for (x = 0; x != wq_sz; x++) {
+		err = -bus_dmamap_create(sq->dma_tag, 0, &sq->mbuf[x].dma_map);
+		if (err != 0) {
+			while (x--)
+				bus_dmamap_destroy(sq->dma_tag, sq->mbuf[x].dma_map);
+			free(sq->mbuf, M_MLX5EN);
+			return (err);
+		}
+	}
+	return (0);
+}
+
+static const char *mlx5e_sq_stats_desc[] = {
+	MLX5E_SQ_STATS(MLX5E_STATS_DESC)
+};
+
+static int
+mlx5e_create_sq(struct mlx5e_channel *c,
+    int tc,
+    struct mlx5e_sq_param *param,
+    struct mlx5e_sq *sq)
+{
+	struct mlx5e_priv *priv = c->priv;
+	struct mlx5_core_dev *mdev = priv->mdev;
+	char buffer[16];
+
+	void *sqc = param->sqc;
+	void *sqc_wq = MLX5_ADDR_OF(sqc, sqc, wq);
+	int err;
+
+	/* Create DMA descriptor TAG */
+	if ((err = -bus_dma_tag_create(
+	    bus_get_dma_tag(mdev->pdev->dev.bsddev),
+	    1,				/* any alignment */
+	    0,				/* no boundary */
+	    BUS_SPACE_MAXADDR,		/* lowaddr */
+	    BUS_SPACE_MAXADDR,		/* highaddr */
+	    NULL, NULL,			/* filter, filterarg */
+	    MLX5E_MAX_TX_PAYLOAD_SIZE,	/* maxsize */
+	    MLX5E_MAX_TX_MBUF_FRAGS,	/* nsegments */
+	    MLX5E_MAX_TX_MBUF_SIZE,	/* maxsegsize */
+	    0,				/* flags */
+	    NULL, NULL,			/* lockfunc, lockfuncarg */
+	    &sq->dma_tag)))
+		goto done;
+
+	err = mlx5_alloc_map_uar(mdev, &sq->uar);
+	if (err)
+		goto err_free_dma_tag;
+
+	err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, &sq->wq,
+	    &sq->wq_ctrl);
+	if (err)
+		goto err_unmap_free_uar;
+
+	sq->wq.db = &sq->wq.db[MLX5_SND_DBR];
+	sq->uar_map = sq->uar.map;
+	sq->uar_bf_map = sq->uar.bf_map;
+	sq->bf_buf_size = (1 << MLX5_CAP_GEN(mdev, log_bf_reg_size)) / 2;
+
+	err = mlx5e_alloc_sq_db(sq);
+	if (err)
+		goto err_sq_wq_destroy;
+
+	sq->pdev = c->pdev;
+	sq->mkey_be = c->mkey_be;
+	sq->channel = c;
+	sq->tc = tc;
+
+	sq->br = buf_ring_alloc(MLX5E_SQ_TX_QUEUE_SIZE, M_MLX5EN,
+	    M_WAITOK, &sq->lock);
+	if (sq->br == NULL) {
+		if_printf(c->ifp, "%s: Failed allocating sq drbr buffer\n",
+		    __func__);
+		err = -ENOMEM;
+		goto err_free_sq_db;
+	}
+
+	sq->sq_tq = taskqueue_create_fast("mlx5e_que", M_WAITOK,
+	    taskqueue_thread_enqueue, &sq->sq_tq);
+	if (sq->sq_tq == NULL) {
+		if_printf(c->ifp, "%s: Failed allocating taskqueue\n",
+		    __func__);
+		err = -ENOMEM;
+		goto err_free_drbr;
+	}
+
+	TASK_INIT(&sq->sq_task, 0, mlx5e_tx_que, sq);
+	taskqueue_start_threads(&sq->sq_tq, 1, PI_NET, "%s tx sq",
+	    c->ifp->if_xname);
+
+	snprintf(buffer, sizeof(buffer), "txstat%dtc%d", c->ix, tc);
+	mlx5e_create_stats(&sq->stats.ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
+	    buffer, mlx5e_sq_stats_desc, MLX5E_SQ_STATS_NUM,
+	    sq->stats.arg);
+
+	return (0);
+
+err_free_drbr:
+	buf_ring_free(sq->br, M_MLX5EN);
+err_free_sq_db:
+	mlx5e_free_sq_db(sq);
+err_sq_wq_destroy:
+	mlx5_wq_destroy(&sq->wq_ctrl);
+
+err_unmap_free_uar:
+	mlx5_unmap_free_uar(mdev, &sq->uar);
+
+err_free_dma_tag:
+	bus_dma_tag_destroy(sq->dma_tag);
+done:
+	return (err);
+}
+
+static void
+mlx5e_destroy_sq(struct mlx5e_sq *sq)
+{
+	struct mlx5e_channel *c = sq->channel;
+	struct mlx5e_priv *priv = c->priv;
+
+	/* destroy all sysctl nodes */
+	sysctl_ctx_free(&sq->stats.ctx);
+
+	mlx5e_free_sq_db(sq);
+	mlx5_wq_destroy(&sq->wq_ctrl);
+	mlx5_unmap_free_uar(priv->mdev, &sq->uar);
+	taskqueue_drain(sq->sq_tq, &sq->sq_task);
+	taskqueue_free(sq->sq_tq);
+	buf_ring_free(sq->br, M_MLX5EN);
+}
+
+static int
+mlx5e_enable_sq(struct mlx5e_sq *sq, struct mlx5e_sq_param *param)
+{
+	struct mlx5e_channel *c = sq->channel;
+	struct mlx5e_priv *priv = c->priv;
+	struct mlx5_core_dev *mdev = priv->mdev;
+
+	void *in;
+	void *sqc;
+	void *wq;
+	int inlen;
+	int err;
+
+	inlen = MLX5_ST_SZ_BYTES(create_sq_in) +
+	    sizeof(u64) * sq->wq_ctrl.buf.npages;
+	in = mlx5_vzalloc(inlen);
+	if (in == NULL)
+		return (-ENOMEM);
+
+	sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
+	wq = MLX5_ADDR_OF(sqc, sqc, wq);
+
+	memcpy(sqc, param->sqc, sizeof(param->sqc));
+
+	MLX5_SET(sqc, sqc, tis_num_0, priv->tisn[sq->tc]);
+	MLX5_SET(sqc, sqc, cqn, c->sq[sq->tc].cq.mcq.cqn);
+	MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RST);
+	MLX5_SET(sqc, sqc, tis_lst_sz, 1);
+	MLX5_SET(sqc, sqc, flush_in_error_en, 1);
+
+	MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC);
+	MLX5_SET(wq, wq, uar_page, sq->uar.index);
+	MLX5_SET(wq, wq, log_wq_pg_sz, sq->wq_ctrl.buf.page_shift -
+	    PAGE_SHIFT);
+	MLX5_SET64(wq, wq, dbr_addr, sq->wq_ctrl.db.dma);
+
+	mlx5_fill_page_array(&sq->wq_ctrl.buf,
+	    (__be64 *) MLX5_ADDR_OF(wq, wq, pas));
+
+	err = mlx5_core_create_sq(mdev, in, inlen, &sq->sqn);
+
+	kvfree(in);
+
+	return (err);
+}
+
+static int
+mlx5e_modify_sq(struct mlx5e_sq *sq, int curr_state, int next_state)
+{
+	struct mlx5e_channel *c = sq->channel;
+	struct mlx5e_priv *priv = c->priv;
+	struct mlx5_core_dev *mdev = priv->mdev;
+
+	void *in;
+	void *sqc;
+	int inlen;
+	int err;
+
+	inlen = MLX5_ST_SZ_BYTES(modify_sq_in);
+	in = mlx5_vzalloc(inlen);
+	if (in == NULL)
+		return (-ENOMEM);
+
+	sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx);
+
+	MLX5_SET(modify_sq_in, in, sqn, sq->sqn);
+	MLX5_SET(modify_sq_in, in, sq_state, curr_state);
+	MLX5_SET(sqc, sqc, state, next_state);
+
+	err = mlx5_core_modify_sq(mdev, in, inlen);
+
+	kvfree(in);
+
+	return (err);
+}
+
+static void
+mlx5e_disable_sq(struct mlx5e_sq *sq)
+{
+	struct mlx5e_channel *c = sq->channel;
+	struct mlx5e_priv *priv = c->priv;
+	struct mlx5_core_dev *mdev = priv->mdev;
+
+	mlx5_core_destroy_sq(mdev, sq->sqn);
+}
+
+static int
+mlx5e_open_sq(struct mlx5e_channel *c,
+    int tc,
+    struct mlx5e_sq_param *param,
+    struct mlx5e_sq *sq)
+{
+	int err;
+
+	err = mlx5e_create_sq(c, tc, param, sq);
+	if (err)
+		return (err);
+
+	err = mlx5e_enable_sq(sq, param);
+	if (err)
+		goto err_destroy_sq;
+
+	err = mlx5e_modify_sq(sq, MLX5_SQC_STATE_RST, MLX5_SQC_STATE_RDY);
+	if (err)
+		goto err_disable_sq;
+
+	atomic_store_rel_int(&sq->queue_state, MLX5E_SQ_READY);
+
+	return (0);
+
+err_disable_sq:
+	mlx5e_disable_sq(sq);
+err_destroy_sq:
+	mlx5e_destroy_sq(sq);
+
+	return (err);
+}
+
+static void
+mlx5e_close_sq(struct mlx5e_sq *sq)
+{
+
+	/* ensure hw is notified of all pending wqes */
+	if (mlx5e_sq_has_room_for(sq, 1))
+		mlx5e_send_nop(sq, 1, true);
+
+	mlx5e_modify_sq(sq, MLX5_SQC_STATE_RDY, MLX5_SQC_STATE_ERR);
+}
+
+static void
+mlx5e_close_sq_wait(struct mlx5e_sq *sq)
+{
+	/* wait till SQ is empty */
+	while (sq->cc != sq->pc) {
+		msleep(4);
+		sq->cq.mcq.comp(&sq->cq.mcq);
+	}
+
+	mlx5e_disable_sq(sq);
+	mlx5e_destroy_sq(sq);
+}
+
+static int
+mlx5e_create_cq(struct mlx5e_channel *c,
+    struct mlx5e_cq_param *param,
+    struct mlx5e_cq *cq,
+    mlx5e_cq_comp_t *comp)
+{
+	struct mlx5e_priv *priv = c->priv;
+	struct mlx5_core_dev *mdev = priv->mdev;
+	struct mlx5_core_cq *mcq = &cq->mcq;
+	int eqn_not_used;
+	int irqn;
+	int err;
+	u32 i;
+
+	param->wq.buf_numa_node = 0;
+	param->wq.db_numa_node = 0;
+	param->eq_ix = c->ix;
+
+	err = mlx5_cqwq_create(mdev, &param->wq, param->cqc, &cq->wq,
+	    &cq->wq_ctrl);
+	if (err)
+		return (err);
+
+	mlx5_vector2eqn(mdev, param->eq_ix, &eqn_not_used, &irqn);
+
+	mcq->cqe_sz = 64;
+	mcq->set_ci_db = cq->wq_ctrl.db.db;
+	mcq->arm_db = cq->wq_ctrl.db.db + 1;
+	*mcq->set_ci_db = 0;
+	*mcq->arm_db = 0;
+	mcq->vector = param->eq_ix;
+	mcq->comp = comp;
+	mcq->event = mlx5e_cq_error_event;
+	mcq->irqn = irqn;
+	mcq->uar = &priv->cq_uar;
+
+	for (i = 0; i < mlx5_cqwq_get_size(&cq->wq); i++) {
+		struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(&cq->wq, i);
+
+		cqe->op_own = 0xf1;
+	}
+
+	cq->channel = c;
+
+	return (0);
+}
+
+static void
+mlx5e_destroy_cq(struct mlx5e_cq *cq)
+{
+	mlx5_wq_destroy(&cq->wq_ctrl);
+}
+
+static int
+mlx5e_enable_cq(struct mlx5e_cq *cq, struct mlx5e_cq_param *param,
+    u8 moderation_mode)
+{
+	struct mlx5e_channel *c = cq->channel;
+	struct mlx5e_priv *priv = c->priv;
+	struct mlx5_core_dev *mdev = priv->mdev;
+	struct mlx5_core_cq *mcq = &cq->mcq;
+	void *in;
+	void *cqc;
+	int inlen;
+	int irqn_not_used;
+	int eqn;
+	int err;
+
+	inlen = MLX5_ST_SZ_BYTES(create_cq_in) +
+	    sizeof(u64) * cq->wq_ctrl.buf.npages;
+	in = mlx5_vzalloc(inlen);
+	if (in == NULL)
+		return (-ENOMEM);
+
+	cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context);
+
+	memcpy(cqc, param->cqc, sizeof(param->cqc));
+
+	mlx5_fill_page_array(&cq->wq_ctrl.buf,
+	    (__be64 *) MLX5_ADDR_OF(create_cq_in, in, pas));
+
+	mlx5_vector2eqn(mdev, param->eq_ix, &eqn, &irqn_not_used);
+
+	MLX5_SET(cqc, cqc, cq_period_mode, moderation_mode);
+	MLX5_SET(cqc, cqc, c_eqn, eqn);
+	MLX5_SET(cqc, cqc, uar_page, mcq->uar->index);
+	MLX5_SET(cqc, cqc, log_page_size, cq->wq_ctrl.buf.page_shift -
+	    PAGE_SHIFT);
+	MLX5_SET64(cqc, cqc, dbr_addr, cq->wq_ctrl.db.dma);
+
+	err = mlx5_core_create_cq(mdev, mcq, in, inlen);
+
+	kvfree(in);
+
+	if (err)
+		return (err);
+
+	mlx5e_cq_arm(cq);
+
+	return (0);
+}
+
+static void
+mlx5e_disable_cq(struct mlx5e_cq *cq)
+{
+	struct mlx5e_channel *c = cq->channel;
+	struct mlx5e_priv *priv = c->priv;
+	struct mlx5_core_dev *mdev = priv->mdev;
+
+	mlx5_core_destroy_cq(mdev, &cq->mcq);
+}
+
+static int
+mlx5e_open_cq(struct mlx5e_channel *c,
+    struct mlx5e_cq_param *param,
+    struct mlx5e_cq *cq,
+    mlx5e_cq_comp_t *comp,
+    u8 moderation_mode)
+{
+	int err;
+
+	err = mlx5e_create_cq(c, param, cq, comp);
+	if (err)
+		return (err);
+
+	err = mlx5e_enable_cq(cq, param, moderation_mode);
+	if (err)
+		goto err_destroy_cq;
+
+	return (0);
+
+err_destroy_cq:
+	mlx5e_destroy_cq(cq);
+
+	return (err);
+}
+
+static void
+mlx5e_close_cq(struct mlx5e_cq *cq)
+{
+	mlx5e_disable_cq(cq);
+	mlx5e_destroy_cq(cq);
+}
+
+static int
+mlx5e_open_tx_cqs(struct mlx5e_channel *c,
+    struct mlx5e_channel_param *cparam)
+{
+	int err;
+	int tc;
+
+	for (tc = 0; tc < c->num_tc; tc++) {
+		/* open completion queue */
+		err = mlx5e_open_cq(c, &cparam->tx_cq, &c->sq[tc].cq,
+		    &mlx5e_tx_cq_comp, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
+		if (err)
+			goto err_close_tx_cqs;
+	}
+	return (0);
+
+err_close_tx_cqs:
+	for (tc--; tc >= 0; tc--)
+		mlx5e_close_cq(&c->sq[tc].cq);
+
+	return (err);
+}
+
+static void
+mlx5e_close_tx_cqs(struct mlx5e_channel *c)
+{
+	int tc;
+
+	for (tc = 0; tc < c->num_tc; tc++)
+		mlx5e_close_cq(&c->sq[tc].cq);
+}
+
+static int
+mlx5e_open_sqs(struct mlx5e_channel *c,
+    struct mlx5e_channel_param *cparam)
+{
+	int err;
+	int tc;
+
+	for (tc = 0; tc < c->num_tc; tc++) {
+		err = mlx5e_open_sq(c, tc, &cparam->sq, &c->sq[tc]);
+		if (err)
+			goto err_close_sqs;
+	}
+
+	return (0);
+
+err_close_sqs:
+	for (tc--; tc >= 0; tc--) {
+		mlx5e_close_sq(&c->sq[tc]);
+		mlx5e_close_sq_wait(&c->sq[tc]);
+	}
+
+	return (err);
+}
+
+static void
+mlx5e_close_sqs(struct mlx5e_channel *c)
+{
+	int tc;
+
+	for (tc = 0; tc < c->num_tc; tc++)
+		mlx5e_close_sq(&c->sq[tc]);
+}
+
+static void
+mlx5e_close_sqs_wait(struct mlx5e_channel *c)
+{
+	int tc;
+
+	for (tc = 0; tc < c->num_tc; tc++)
+		mlx5e_close_sq_wait(&c->sq[tc]);
+}
+
+static void
+mlx5e_chan_mtx_init(struct mlx5e_channel *c)
+{
+	int tc;
+
+	mtx_init(&c->rq.mtx, "mlx5rx", MTX_NETWORK_LOCK, MTX_DEF);
+
+	for (tc = 0; tc < c->num_tc; tc++) {
+		mtx_init(&c->sq[tc].lock, "mlx5tx", MTX_NETWORK_LOCK, MTX_DEF);
+		mtx_init(&c->sq[tc].comp_lock, "mlx5comp", MTX_NETWORK_LOCK,
+		    MTX_DEF);
+	}
+}
+
+static void
+mlx5e_chan_mtx_destroy(struct mlx5e_channel *c)
+{
+	int tc;
+
+	mtx_destroy(&c->rq.mtx);
+
+	for (tc = 0; tc < c->num_tc; tc++) {
+		mtx_destroy(&c->sq[tc].lock);
+		mtx_destroy(&c->sq[tc].comp_lock);
+	}
+}
+
+static int
+mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
+    struct mlx5e_channel_param *cparam,
+    struct mlx5e_channel *volatile *cp)
+{
+	struct mlx5e_channel *c;
+	u8 rx_moderation_mode;
+	int err;
+
+	c = malloc(sizeof(*c), M_MLX5EN, M_WAITOK | M_ZERO);
+	if (c == NULL)
+		return (-ENOMEM);
+
+	c->priv = priv;
+	c->ix = ix;
+	c->cpu = 0;
+	c->pdev = &priv->mdev->pdev->dev;
+	c->ifp = priv->ifp;
+	c->mkey_be = cpu_to_be32(priv->mr.key);
+	c->num_tc = priv->num_tc;
+
+	/* init mutexes */
+	mlx5e_chan_mtx_init(c);
+
+	/* open transmit completion queue */
+	err = mlx5e_open_tx_cqs(c, cparam);
+	if (err)
+		goto err_free;
+
+	switch (priv->params.rx_cq_moderation_mode) {
+	case 0:
+		rx_moderation_mode = MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
+		break;
+	default:
+		if (MLX5_CAP_GEN(priv->mdev, cq_period_start_from_cqe))
+			rx_moderation_mode = MLX5_CQ_PERIOD_MODE_START_FROM_CQE;
+		else
+			rx_moderation_mode = MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
+		break;
+	}
+
+	/* open receive completion queue */
+	err = mlx5e_open_cq(c, &cparam->rx_cq, &c->rq.cq,
+	    &mlx5e_rx_cq_comp, rx_moderation_mode);
+	if (err)
+		goto err_close_tx_cqs;
+
+	err = mlx5e_open_sqs(c, cparam);
+	if (err)
+		goto err_close_rx_cq;
+
+	err = mlx5e_open_rq(c, &cparam->rq, &c->rq);
+	if (err)
+		goto err_close_sqs;
+
+	/* store channel pointer */
+	*cp = c;
+
+	/* poll receive queue initially */
+	c->rq.cq.mcq.comp(&c->rq.cq.mcq);
+
+	return (0);
+
+err_close_sqs:
+	mlx5e_close_sqs(c);
+	mlx5e_close_sqs_wait(c);
+
+err_close_rx_cq:
+	mlx5e_close_cq(&c->rq.cq);
+
+err_close_tx_cqs:
+	mlx5e_close_tx_cqs(c);
+
+err_free:
+	/* destroy mutexes */
+	mlx5e_chan_mtx_destroy(c);
+	free(c, M_MLX5EN);
+	return (err);
+}
+
+static void
+mlx5e_close_channel(struct mlx5e_channel *volatile *pp)
+{
+	struct mlx5e_channel *c = *pp;
+
+	/* check if channel is already closed */
+	if (c == NULL)
+		return;
+	mlx5e_close_rq(&c->rq);
+	mlx5e_close_sqs(c);
+}
+
+static void
+mlx5e_close_channel_wait(struct mlx5e_channel *volatile *pp)
+{
+	struct mlx5e_channel *c = *pp;
+
+	/* check if channel is already closed */
+	if (c == NULL)
+		return;
+	/* ensure channel pointer is no longer used */
+	*pp = NULL;
+
+	mlx5e_close_rq_wait(&c->rq);
+	mlx5e_close_sqs_wait(c);
+	mlx5e_close_cq(&c->rq.cq);
+	mlx5e_close_tx_cqs(c);
+	/* destroy mutexes */
+	mlx5e_chan_mtx_destroy(c);
+	free(c, M_MLX5EN);
+}
+
+static void
+mlx5e_build_rq_param(struct mlx5e_priv *priv,
+    struct mlx5e_rq_param *param)
+{
+	void *rqc = param->rqc;
+	void *wq = MLX5_ADDR_OF(rqc, rqc, wq);
+
+	MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_LINKED_LIST);
+	MLX5_SET(wq, wq, end_padding_mode, MLX5_WQ_END_PAD_MODE_ALIGN);
+	MLX5_SET(wq, wq, log_wq_stride, ilog2(sizeof(struct mlx5e_rx_wqe)));
+	MLX5_SET(wq, wq, log_wq_sz, priv->params.log_rq_size);
+	MLX5_SET(wq, wq, pd, priv->pdn);
+
+	param->wq.buf_numa_node = 0;
+	param->wq.db_numa_node = 0;
+	param->wq.linear = 1;
+}
+
+static void
+mlx5e_build_sq_param(struct mlx5e_priv *priv,
+    struct mlx5e_sq_param *param)
+{
+	void *sqc = param->sqc;
+	void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
+
+	MLX5_SET(wq, wq, log_wq_sz, priv->params.log_sq_size);
+	MLX5_SET(wq, wq, log_wq_stride, ilog2(MLX5_SEND_WQE_BB));
+	MLX5_SET(wq, wq, pd, priv->pdn);
+
+	param->wq.buf_numa_node = 0;
+	param->wq.db_numa_node = 0;
+	param->wq.linear = 1;
+}
+
+static void
+mlx5e_build_common_cq_param(struct mlx5e_priv *priv,
+    struct mlx5e_cq_param *param)
+{
+	void *cqc = param->cqc;
+
+	MLX5_SET(cqc, cqc, uar_page, priv->cq_uar.index);
+}
+
+static void
+mlx5e_build_rx_cq_param(struct mlx5e_priv *priv,
+    struct mlx5e_cq_param *param)
+{
+	void *cqc = param->cqc;
+
+	MLX5_SET(cqc, cqc, log_cq_size, priv->params.log_rq_size);
+	MLX5_SET(cqc, cqc, cq_period, priv->params.rx_cq_moderation_usec);
+	MLX5_SET(cqc, cqc, cq_max_count, priv->params.rx_cq_moderation_pkts);
+
+	mlx5e_build_common_cq_param(priv, param);
+}
+
+static void
+mlx5e_build_tx_cq_param(struct mlx5e_priv *priv,
+    struct mlx5e_cq_param *param)
+{
+	void *cqc = param->cqc;
+
+	MLX5_SET(cqc, cqc, log_cq_size, priv->params.log_sq_size);
+	MLX5_SET(cqc, cqc, cq_period, priv->params.tx_cq_moderation_usec);
+	MLX5_SET(cqc, cqc, cq_max_count, priv->params.tx_cq_moderation_pkts);
+
+	mlx5e_build_common_cq_param(priv, param);
+}
+
+static void
+mlx5e_build_channel_param(struct mlx5e_priv *priv,
+    struct mlx5e_channel_param *cparam)
+{
+	memset(cparam, 0, sizeof(*cparam));
+
+	mlx5e_build_rq_param(priv, &cparam->rq);
+	mlx5e_build_sq_param(priv, &cparam->sq);
+	mlx5e_build_rx_cq_param(priv, &cparam->rx_cq);
+	mlx5e_build_tx_cq_param(priv, &cparam->tx_cq);
+}
+
+static int
+mlx5e_open_channels(struct mlx5e_priv *priv)
+{
+	struct mlx5e_channel_param cparam;
+	void *ptr;
+	int err;
+	int i;
+	int j;
+
+	priv->channel = malloc(priv->params.num_channels *
+	    sizeof(struct mlx5e_channel *), M_MLX5EN, M_WAITOK | M_ZERO);
+	if (priv->channel == NULL)
+		return (-ENOMEM);
+
+	mlx5e_build_channel_param(priv, &cparam);
+	for (i = 0; i < priv->params.num_channels; i++) {
+		err = mlx5e_open_channel(priv, i, &cparam, &priv->channel[i]);
+		if (err)
+			goto err_close_channels;
+	}
+
+	for (j = 0; j < priv->params.num_channels; j++) {
+		err = mlx5e_wait_for_min_rx_wqes(&priv->channel[j]->rq);
+		if (err)
+			goto err_close_channels;
+	}
+
+	return (0);
+
+err_close_channels:
+	for (i--; i >= 0; i--) {
+		mlx5e_close_channel(&priv->channel[i]);
+		mlx5e_close_channel_wait(&priv->channel[i]);
+	}
+
+	/* remove "volatile" attribute from "channel" pointer */
+	ptr = __DECONST(void *, priv->channel);
+	priv->channel = NULL;
+
+	free(ptr, M_MLX5EN);
+
+	return (err);
+}
+
+static void
+mlx5e_close_channels(struct mlx5e_priv *priv)
+{
+	void *ptr;
+	int i;
+
+	if (priv->channel == NULL)
+		return;
+
+	for (i = 0; i < priv->params.num_channels; i++)
+		mlx5e_close_channel(&priv->channel[i]);
+	for (i = 0; i < priv->params.num_channels; i++)
+		mlx5e_close_channel_wait(&priv->channel[i]);
+
+	/* remove "volatile" attribute from "channel" pointer */
+	ptr = __DECONST(void *, priv->channel);
+	priv->channel = NULL;
+
+	free(ptr, M_MLX5EN);
+}
+
+static int
+mlx5e_open_tis(struct mlx5e_priv *priv, int tc)
+{
+	struct mlx5_core_dev *mdev = priv->mdev;
+	u32 in[MLX5_ST_SZ_DW(create_tis_in)];
+	void *tisc = MLX5_ADDR_OF(create_tis_in, in, ctx);
+
+	memset(in, 0, sizeof(in));
+
+	MLX5_SET(tisc, tisc, prio, tc);
+	MLX5_SET(tisc, tisc, transport_domain, priv->tdn);
+
+	return (mlx5_core_create_tis(mdev, in, sizeof(in), &priv->tisn[tc]));
+}
+
+static void
+mlx5e_close_tis(struct mlx5e_priv *priv, int tc)
+{
+	mlx5_core_destroy_tis(priv->mdev, priv->tisn[tc]);
+}
+
+static int
+mlx5e_open_tises(struct mlx5e_priv *priv)
+{
+	int num_tc = priv->num_tc;
+	int err;
+	int tc;
+
+	for (tc = 0; tc < num_tc; tc++) {
+		err = mlx5e_open_tis(priv, tc);
+		if (err)
+			goto err_close_tises;
+	}
+
+	return (0);
+
+err_close_tises:
+	for (tc--; tc >= 0; tc--)
+		mlx5e_close_tis(priv, tc);
+
+	return (err);
+}
+
+static void
+mlx5e_close_tises(struct mlx5e_priv *priv)
+{
+	int num_tc = priv->num_tc;
+	int tc;
+
+	for (tc = 0; tc < num_tc; tc++)
+		mlx5e_close_tis(priv, tc);
+}
+
+static int
+mlx5e_open_rqt(struct mlx5e_priv *priv)
+{
+	struct mlx5_core_dev *mdev = priv->mdev;
+	u32 *in;
+	u32 out[MLX5_ST_SZ_DW(create_rqt_out)];
+	void *rqtc;
+	int inlen;
+	int err;
+	int sz;
+	int i;
+
+	sz = 1 << priv->params.rx_hash_log_tbl_sz;
+
+	inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + sizeof(u32) * sz;
+	in = mlx5_vzalloc(inlen);
+	if (in == NULL)
+		return (-ENOMEM);
+	rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context);
+
+	MLX5_SET(rqtc, rqtc, rqt_actual_size, sz);
+	MLX5_SET(rqtc, rqtc, rqt_max_size, sz);
+
+	for (i = 0; i < sz; i++) {
+		int ix = i % priv->params.num_channels;
+
+		MLX5_SET(rqtc, rqtc, rq_num[i], priv->channel[ix]->rq.rqn);
+	}
+
+	MLX5_SET(create_rqt_in, in, opcode, MLX5_CMD_OP_CREATE_RQT);
+
+	memset(out, 0, sizeof(out));
+	err = mlx5_cmd_exec_check_status(mdev, in, inlen, out, sizeof(out));
+	if (!err)
+		priv->rqtn = MLX5_GET(create_rqt_out, out, rqtn);
+
+	kvfree(in);
+
+	return (err);
+}
+
+static void
+mlx5e_close_rqt(struct mlx5e_priv *priv)
+{
+	u32 in[MLX5_ST_SZ_DW(destroy_rqt_in)];
+	u32 out[MLX5_ST_SZ_DW(destroy_rqt_out)];
+
+	memset(in, 0, sizeof(in));
+
+	MLX5_SET(destroy_rqt_in, in, opcode, MLX5_CMD_OP_DESTROY_RQT);
+	MLX5_SET(destroy_rqt_in, in, rqtn, priv->rqtn);
+
+	mlx5_cmd_exec_check_status(priv->mdev, in, sizeof(in), out,
+	    sizeof(out));
+}
+
+static void
+mlx5e_build_tir_ctx(struct mlx5e_priv *priv, u32 * tirc, int tt)
+{
+	void *hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer);
+	__be32 *hkey;
+
+	MLX5_SET(tirc, tirc, transport_domain, priv->tdn);
+
+#define	ROUGH_MAX_L2_L3_HDR_SZ 256
+
+#define	MLX5_HASH_IP     (MLX5_HASH_FIELD_SEL_SRC_IP   |\
+			  MLX5_HASH_FIELD_SEL_DST_IP)
+
+#define	MLX5_HASH_ALL    (MLX5_HASH_FIELD_SEL_SRC_IP   |\
+			  MLX5_HASH_FIELD_SEL_DST_IP   |\
+			  MLX5_HASH_FIELD_SEL_L4_SPORT |\
+			  MLX5_HASH_FIELD_SEL_L4_DPORT)
+
+#define	MLX5_HASH_IP_IPSEC_SPI	(MLX5_HASH_FIELD_SEL_SRC_IP   |\
+				 MLX5_HASH_FIELD_SEL_DST_IP   |\
+				 MLX5_HASH_FIELD_SEL_IPSEC_SPI)
+
+	if (priv->params.hw_lro_en) {
+		MLX5_SET(tirc, tirc, lro_enable_mask,
+		    MLX5_TIRC_LRO_ENABLE_MASK_IPV4_LRO |
+		    MLX5_TIRC_LRO_ENABLE_MASK_IPV6_LRO);
+		MLX5_SET(tirc, tirc, lro_max_msg_sz,
+		    (priv->params.lro_wqe_sz -
+		    ROUGH_MAX_L2_L3_HDR_SZ) >> 8);
+		/* TODO: add the option to choose timer value dynamically */
+		MLX5_SET(tirc, tirc, lro_timeout_period_usecs,
+		    MLX5_CAP_ETH(priv->mdev,
+		    lro_timer_supported_periods[2]));
+	}
+	switch (tt) {
+	case MLX5E_TT_ANY:
+		MLX5_SET(tirc, tirc, disp_type,
+		    MLX5_TIRC_DISP_TYPE_DIRECT);
+		MLX5_SET(tirc, tirc, inline_rqn,
+		    priv->channel[0]->rq.rqn);
+		break;
+	default:
+		MLX5_SET(tirc, tirc, disp_type,
+		    MLX5_TIRC_DISP_TYPE_INDIRECT);
+		MLX5_SET(tirc, tirc, indirect_table,
+		    priv->rqtn);
+		MLX5_SET(tirc, tirc, rx_hash_fn,
+		    MLX5_TIRC_RX_HASH_FN_HASH_TOEPLITZ);
+		MLX5_SET(tirc, tirc, rx_hash_symmetric, 1);
+		hkey = (__be32 *) MLX5_ADDR_OF(tirc, tirc, rx_hash_toeplitz_key);
+		hkey[0] = cpu_to_be32(0xD181C62C);
+		hkey[1] = cpu_to_be32(0xF7F4DB5B);
+		hkey[2] = cpu_to_be32(0x1983A2FC);
+		hkey[3] = cpu_to_be32(0x943E1ADB);
+		hkey[4] = cpu_to_be32(0xD9389E6B);
+		hkey[5] = cpu_to_be32(0xD1039C2C);
+		hkey[6] = cpu_to_be32(0xA74499AD);
+		hkey[7] = cpu_to_be32(0x593D56D9);
+		hkey[8] = cpu_to_be32(0xF3253C06);
+		hkey[9] = cpu_to_be32(0x2ADC1FFC);
+		break;
+	}
+
+	switch (tt) {
+	case MLX5E_TT_IPV4_TCP:
+		MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
+		    MLX5_L3_PROT_TYPE_IPV4);
+		MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
+		    MLX5_L4_PROT_TYPE_TCP);
+		MLX5_SET(rx_hash_field_select, hfso, selected_fields,
+		    MLX5_HASH_ALL);
+		break;
+
+	case MLX5E_TT_IPV6_TCP:
+		MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
+		    MLX5_L3_PROT_TYPE_IPV6);
+		MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
+		    MLX5_L4_PROT_TYPE_TCP);
+		MLX5_SET(rx_hash_field_select, hfso, selected_fields,
+		    MLX5_HASH_ALL);
+		break;
+
+	case MLX5E_TT_IPV4_UDP:
+		MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
+		    MLX5_L3_PROT_TYPE_IPV4);
+		MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
+		    MLX5_L4_PROT_TYPE_UDP);
+		MLX5_SET(rx_hash_field_select, hfso, selected_fields,
+		    MLX5_HASH_ALL);
+		break;
+
+	case MLX5E_TT_IPV6_UDP:
+		MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
+		    MLX5_L3_PROT_TYPE_IPV6);
+		MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
+		    MLX5_L4_PROT_TYPE_UDP);
+		MLX5_SET(rx_hash_field_select, hfso, selected_fields,
+		    MLX5_HASH_ALL);
+		break;
+
+	case MLX5E_TT_IPV4_IPSEC_AH:
+		MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
+		    MLX5_L3_PROT_TYPE_IPV4);
+		MLX5_SET(rx_hash_field_select, hfso, selected_fields,
+		    MLX5_HASH_IP_IPSEC_SPI);
+		break;
+
+	case MLX5E_TT_IPV6_IPSEC_AH:
+		MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
+		    MLX5_L3_PROT_TYPE_IPV6);
+		MLX5_SET(rx_hash_field_select, hfso, selected_fields,
+		    MLX5_HASH_IP_IPSEC_SPI);
+		break;
+
+	case MLX5E_TT_IPV4_IPSEC_ESP:
+		MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
+		    MLX5_L3_PROT_TYPE_IPV4);
+		MLX5_SET(rx_hash_field_select, hfso, selected_fields,
+		    MLX5_HASH_IP_IPSEC_SPI);
+		break;
+
+	case MLX5E_TT_IPV6_IPSEC_ESP:
+		MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
+		    MLX5_L3_PROT_TYPE_IPV6);
+		MLX5_SET(rx_hash_field_select, hfso, selected_fields,
+		    MLX5_HASH_IP_IPSEC_SPI);
+		break;
+
+	case MLX5E_TT_IPV4:
+		MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
+		    MLX5_L3_PROT_TYPE_IPV4);
+		MLX5_SET(rx_hash_field_select, hfso, selected_fields,
+		    MLX5_HASH_IP);
+		break;
+
+	case MLX5E_TT_IPV6:
+		MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
+		    MLX5_L3_PROT_TYPE_IPV6);
+		MLX5_SET(rx_hash_field_select, hfso, selected_fields,
+		    MLX5_HASH_IP);
+		break;
+
+	default:
+		break;
+	}
+}
+
+static int
+mlx5e_open_tir(struct mlx5e_priv *priv, int tt)
+{
+	struct mlx5_core_dev *mdev = priv->mdev;
+	u32 *in;
+	void *tirc;
+	int inlen;
+	int err;
+
+	inlen = MLX5_ST_SZ_BYTES(create_tir_in);
+	in = mlx5_vzalloc(inlen);
+	if (in == NULL)
+		return (-ENOMEM);
+	tirc = MLX5_ADDR_OF(create_tir_in, in, tir_context);
+
+	mlx5e_build_tir_ctx(priv, tirc, tt);
+
+	err = mlx5_core_create_tir(mdev, in, inlen, &priv->tirn[tt]);
+
+	kvfree(in);
+
+	return (err);
+}
+
+static void
+mlx5e_close_tir(struct mlx5e_priv *priv, int tt)
+{
+	mlx5_core_destroy_tir(priv->mdev, priv->tirn[tt]);
+}
+
+static int
+mlx5e_open_tirs(struct mlx5e_priv *priv)
+{
+	int err;
+	int i;
+
+	for (i = 0; i < MLX5E_NUM_TT; i++) {
+		err = mlx5e_open_tir(priv, i);
+		if (err)
+			goto err_close_tirs;
+	}
+
+	return (0);
+
+err_close_tirs:
+	for (i--; i >= 0; i--)
+		mlx5e_close_tir(priv, i);
+
+	return (err);
+}
+
+static void
+mlx5e_close_tirs(struct mlx5e_priv *priv)
+{
+	int i;
+
+	for (i = 0; i < MLX5E_NUM_TT; i++)
+		mlx5e_close_tir(priv, i);
+}
+
+/*
+ * SW MTU does not include headers,
+ * HW MTU includes all headers and checksums.
+ */
+static int
+mlx5e_set_dev_port_mtu(struct ifnet *ifp, int sw_mtu)
+{
+	struct mlx5e_priv *priv = ifp->if_softc;
+	struct mlx5_core_dev *mdev = priv->mdev;
+	int hw_mtu;
+	int min_mtu;
+	int err;
+
+	/*
+	 * Trying to set MTU to zero, in order
+	 * to find out the FW's minimal MTU
+	 */
+	err = mlx5_set_port_mtu(mdev, 0);
+	if (err)
+		return (err);
+
+	err = mlx5_query_port_oper_mtu(mdev, &min_mtu);
+	if (err) {
+		if_printf(ifp, "Query port minimal MTU failed\n");
+		return (err);
+	}
+
+	if (sw_mtu < MLX5E_HW2SW_MTU(min_mtu)) {
+		ifp->if_mtu = sw_mtu;
+		return (0);
+	}
+
+	err = mlx5_set_port_mtu(mdev, MLX5E_SW2HW_MTU(sw_mtu));
+	if (err)
+		return (err);
+
+	err = mlx5_query_port_oper_mtu(mdev, &hw_mtu);
+	if (!err) {
+		ifp->if_mtu = MLX5E_HW2SW_MTU(hw_mtu);
+
+		if (ifp->if_mtu != sw_mtu) {
+			if_printf(ifp, "Port MTU %d is different than "
+			    "ifp mtu %d\n", sw_mtu, (int)ifp->if_mtu);
+		}
+	} else {
+		if_printf(ifp, "Query port MTU, after setting new "
+		    "MTU value, failed\n");
+		ifp->if_mtu = sw_mtu;
+	}
+	return (0);
+}
+
+int
+mlx5e_open_locked(struct ifnet *ifp)
+{
+	struct mlx5e_priv *priv = ifp->if_softc;
+	int err;
+
+	/* check if already opened */
+	if (test_bit(MLX5E_STATE_OPENED, &priv->state) != 0)
+		return (0);
+
+	err = mlx5e_open_tises(priv);
+	if (err) {
+		if_printf(ifp, "%s: mlx5e_open_tises failed, %d\n",
+		    __func__, err);
+		return (err);
+	}
+	err = mlx5_vport_alloc_q_counter(priv->mdev, &priv->counter_set_id);
+	if (err) {
+		if_printf(priv->ifp,
+		    "%s: mlx5_vport_alloc_q_counter failed: %d\n",
+		    __func__, err);
+		goto err_close_tises;
+	}
+	err = mlx5e_open_channels(priv);
+	if (err) {
+		if_printf(ifp, "%s: mlx5e_open_channels failed, %d\n",
+		    __func__, err);
+		goto err_dalloc_q_counter;
+	}
+	err = mlx5e_open_rqt(priv);
+	if (err) {
+		if_printf(ifp, "%s: mlx5e_open_rqt failed, %d\n",
+		    __func__, err);
+		goto err_close_channels;
+	}
+	err = mlx5e_open_tirs(priv);
+	if (err) {
+		if_printf(ifp, "%s: mlx5e_open_tir failed, %d\n",
+		    __func__, err);
+		goto err_close_rqls;
+	}
+	err = mlx5e_open_flow_table(priv);
+	if (err) {
+		if_printf(ifp, "%s: mlx5e_open_flow_table failed, %d\n",
+		    __func__, err);
+		goto err_close_tirs;
+	}
+	err = mlx5e_add_all_vlan_rules(priv);
+	if (err) {
+		if_printf(ifp, "%s: mlx5e_add_all_vlan_rules failed, %d\n",
+		    __func__, err);
+		goto err_close_flow_table;
+	}
+	set_bit(MLX5E_STATE_OPENED, &priv->state);
+
+	mlx5e_update_carrier(priv);
+	mlx5e_set_rx_mode_core(priv);
+
+	return (0);
+
+err_close_flow_table:
+	mlx5e_close_flow_table(priv);
+
+err_close_tirs:
+	mlx5e_close_tirs(priv);
+
+err_close_rqls:
+	mlx5e_close_rqt(priv);
+
+err_close_channels:
+	mlx5e_close_channels(priv);
+
+err_dalloc_q_counter:
+	mlx5_vport_dealloc_q_counter(priv->mdev, priv->counter_set_id);
+
+err_close_tises:
+	mlx5e_close_tises(priv);
+
+	return (err);
+}
+
+static void
+mlx5e_open(void *arg)
+{
+	struct mlx5e_priv *priv = arg;
+
+	PRIV_LOCK(priv);
+	if (mlx5_set_port_status(priv->mdev, MLX5_PORT_UP))
+		if_printf(priv->ifp,
+		    "%s: Setting port status to up failed\n",
+		    __func__);
+
+	mlx5e_open_locked(priv->ifp);
+	priv->ifp->if_drv_flags |= IFF_DRV_RUNNING;
+	PRIV_UNLOCK(priv);
+}
+
+int
+mlx5e_close_locked(struct ifnet *ifp)
+{
+	struct mlx5e_priv *priv = ifp->if_softc;
+
+	/* check if already closed */
+	if (test_bit(MLX5E_STATE_OPENED, &priv->state) == 0)
+		return (0);
+
+	clear_bit(MLX5E_STATE_OPENED, &priv->state);
+
+	mlx5e_set_rx_mode_core(priv);
+	mlx5e_del_all_vlan_rules(priv);
+	if_link_state_change(priv->ifp, LINK_STATE_DOWN);
+	mlx5e_close_flow_table(priv);
+	mlx5e_close_tirs(priv);
+	mlx5e_close_rqt(priv);
+	mlx5e_close_channels(priv);
+	mlx5_vport_dealloc_q_counter(priv->mdev, priv->counter_set_id);
+	mlx5e_close_tises(priv);
+
+	return (0);
+}
+
+#if (__FreeBSD_version >= 1100000)
+static uint64_t
+mlx5e_get_counter(struct ifnet *ifp, ift_counter cnt)
+{
+	struct mlx5e_priv *priv = ifp->if_softc;
+	u64 retval;
+
+	/* PRIV_LOCK(priv); XXX not allowed */
+	switch (cnt) {
+	case IFCOUNTER_IPACKETS:
+		retval = priv->stats.vport.rx_packets;
+		break;
+	case IFCOUNTER_IERRORS:
+		retval = priv->stats.vport.rx_error_packets;
+		break;
+	case IFCOUNTER_IQDROPS:
+		retval = priv->stats.vport.rx_out_of_buffer;
+		break;
+	case IFCOUNTER_OPACKETS:
+		retval = priv->stats.vport.tx_packets;
+		break;
+	case IFCOUNTER_OERRORS:
+		retval = priv->stats.vport.tx_error_packets;
+		break;
+	case IFCOUNTER_IBYTES:
+		retval = priv->stats.vport.rx_bytes;
+		break;
+	case IFCOUNTER_OBYTES:
+		retval = priv->stats.vport.tx_bytes;
+		break;
+	case IFCOUNTER_IMCASTS:
+		retval = priv->stats.vport.rx_multicast_packets;
+		break;
+	case IFCOUNTER_OMCASTS:
+		retval = priv->stats.vport.tx_multicast_packets;
+		break;
+	case IFCOUNTER_OQDROPS:
+		retval = priv->stats.vport.tx_queue_dropped;
+		break;
+	default:
+		retval = if_get_counter_default(ifp, cnt);
+		break;
+	}
+	/* PRIV_UNLOCK(priv); XXX not allowed */
+	return (retval);
+}
+#endif
+
+static void
+mlx5e_set_rx_mode(struct ifnet *ifp)
+{
+	struct mlx5e_priv *priv = ifp->if_softc;
+
+	schedule_work(&priv->set_rx_mode_work);
+}
+
+static int
+mlx5e_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
+{
+	struct mlx5e_priv *priv;
+	struct ifreq *ifr;
+	struct ifi2creq i2c;
+	int error = 0;
+	int mask = 0;
+	int size_read = 0;
+	int module_num;
+	int max_mtu;
+
+	priv = ifp->if_softc;
+
+	/* check if detaching */
+	if (priv == NULL || priv->gone != 0)
+		return (ENXIO);
+
+	switch (command) {
+	case SIOCSIFMTU:
+		ifr = (struct ifreq *)data;
+
+		PRIV_LOCK(priv);
+		mlx5_query_port_max_mtu(priv->mdev, &max_mtu);
+
+		if (ifr->ifr_mtu >= MLX5E_MTU_MIN &&
+		    ifr->ifr_mtu <= MIN(MLX5E_MTU_MAX, max_mtu)) {
+			int was_opened;
+
+			was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
+			if (was_opened)
+				mlx5e_close_locked(ifp);
+
+			/* set new MTU */
+			mlx5e_set_dev_port_mtu(ifp, ifr->ifr_mtu);
+
+			if (was_opened)
+				mlx5e_open_locked(ifp);
+		} else {
+			error = EINVAL;
+			if_printf(ifp, "Invalid MTU value. Min val: %d, Max val: %d\n",
+			    MLX5E_MTU_MIN, MIN(MLX5E_MTU_MAX, max_mtu));
+		}
+		PRIV_UNLOCK(priv);
+		break;
+	case SIOCSIFFLAGS:
+		if ((ifp->if_flags & IFF_UP) &&
+		    (ifp->if_drv_flags & IFF_DRV_RUNNING)) {
+			mlx5e_set_rx_mode(ifp);
+			break;
+		}
+		PRIV_LOCK(priv);
+		if (ifp->if_flags & IFF_UP) {
+			if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
+				if (test_bit(MLX5E_STATE_OPENED, &priv->state) == 0)
+					mlx5e_open_locked(ifp);
+				ifp->if_drv_flags |= IFF_DRV_RUNNING;
+				mlx5_set_port_status(priv->mdev, MLX5_PORT_UP);
+			}
+		} else {
+			if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
+				mlx5_set_port_status(priv->mdev,
+				    MLX5_PORT_DOWN);
+				if (test_bit(MLX5E_STATE_OPENED, &priv->state) != 0)
+					mlx5e_close_locked(ifp);
+				mlx5e_update_carrier(priv);
+				ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
+			}
+		}
+		PRIV_UNLOCK(priv);
+		break;
+	case SIOCADDMULTI:
+	case SIOCDELMULTI:
+		mlx5e_set_rx_mode(ifp);
+		break;
+	case SIOCSIFMEDIA:
+	case SIOCGIFMEDIA:
+	case SIOCGIFXMEDIA:
+		ifr = (struct ifreq *)data;
+		error = ifmedia_ioctl(ifp, ifr, &priv->media, command);
+		break;
+	case SIOCSIFCAP:
+		ifr = (struct ifreq *)data;
+		PRIV_LOCK(priv);
+		mask = ifr->ifr_reqcap ^ ifp->if_capenable;
+
+		if (mask & IFCAP_TXCSUM) {
+			ifp->if_capenable ^= IFCAP_TXCSUM;
+			ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
+
+			if (IFCAP_TSO4 & ifp->if_capenable &&
+			    !(IFCAP_TXCSUM & ifp->if_capenable)) {
+				ifp->if_capenable &= ~IFCAP_TSO4;
+				ifp->if_hwassist &= ~CSUM_IP_TSO;
+				if_printf(ifp,
+				    "tso4 disabled due to -txcsum.\n");
+			}
+		}
+		if (mask & IFCAP_TXCSUM_IPV6) {
+			ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
+			ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
+
+			if (IFCAP_TSO6 & ifp->if_capenable &&
+			    !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
+				ifp->if_capenable &= ~IFCAP_TSO6;
+				ifp->if_hwassist &= ~CSUM_IP6_TSO;
+				if_printf(ifp,
+				    "tso6 disabled due to -txcsum6.\n");
+			}
+		}
+		if (mask & IFCAP_RXCSUM)
+			ifp->if_capenable ^= IFCAP_RXCSUM;
+		if (mask & IFCAP_RXCSUM_IPV6)
+			ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
+		if (mask & IFCAP_TSO4) {
+			if (!(IFCAP_TSO4 & ifp->if_capenable) &&
+			    !(IFCAP_TXCSUM & ifp->if_capenable)) {
+				if_printf(ifp, "enable txcsum first.\n");
+				error = EAGAIN;
+				goto out;
+			}
+			ifp->if_capenable ^= IFCAP_TSO4;
+			ifp->if_hwassist ^= CSUM_IP_TSO;
+		}
+		if (mask & IFCAP_TSO6) {
+			if (!(IFCAP_TSO6 & ifp->if_capenable) &&
+			    !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
+				if_printf(ifp, "enable txcsum6 first.\n");
+				error = EAGAIN;
+				goto out;
+			}
+			ifp->if_capenable ^= IFCAP_TSO6;
+			ifp->if_hwassist ^= CSUM_IP6_TSO;
+		}
+		if (mask & IFCAP_VLAN_HWFILTER) {
+			if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
+				mlx5e_disable_vlan_filter(priv);
+			else
+				mlx5e_enable_vlan_filter(priv);
+
+			ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
+		}
+		if (mask & IFCAP_VLAN_HWTAGGING)
+			ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
+		if (mask & IFCAP_WOL_MAGIC)
+			ifp->if_capenable ^= IFCAP_WOL_MAGIC;
+
+		VLAN_CAPABILITIES(ifp);
+		/* turn off LRO means also turn of HW LRO - if it's on */
+		if (mask & IFCAP_LRO) {
+			int was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
+			bool need_restart = false;
+
+			ifp->if_capenable ^= IFCAP_LRO;
+			if (!(ifp->if_capenable & IFCAP_LRO)) {
+				if (priv->params.hw_lro_en) {
+					priv->params.hw_lro_en = false;
+					need_restart = true;
+					/* Not sure this is the correct way */
+					priv->params_ethtool.hw_lro = priv->params.hw_lro_en;
+				}
+			}
+			if (was_opened && need_restart) {
+				mlx5e_close_locked(ifp);
+				mlx5e_open_locked(ifp);
+			}
+		}
+out:
+		PRIV_UNLOCK(priv);
+		break;
+
+	case SIOCGI2C:
+		ifr = (struct ifreq *)data;
+
+		/*
+		 * Copy from the user-space address ifr_data to the
+		 * kernel-space address i2c
+		 */
+		error = copyin(ifr->ifr_data, &i2c, sizeof(i2c));
+		if (error)
+			break;
+
+		if (i2c.len > sizeof(i2c.data)) {
+			error = EINVAL;
+			break;
+		}
+
+		PRIV_LOCK(priv);
+		/* Get module_num which is required for the query_eeprom */
+		error = mlx5_query_module_num(priv->mdev, &module_num);
+		if (error) {
+			if_printf(ifp, "Query module num failed, eeprom "
+			    "reading is not supported\n");
+			goto err_i2c;
+		}
+
+		/*
+		 * Note that we ignore i2c.addr here. The driver hardcodes
+		 * the address to 0x50, while standard expects it to be 0xA0.
+		 */
+		error = mlx5_query_eeprom(priv->mdev,
+		    MLX5E_I2C_ADDR_LOW, MLX5E_EEPROM_LOW_PAGE,
+		    (uint32_t)i2c.offset, (uint32_t)i2c.len, module_num,
+		    (uint32_t *)i2c.data, &size_read);
+		if (error) {
+			if_printf(ifp, "Query eeprom failed, eeprom "
+			    "reading is not supported\n");
+			goto err_i2c;
+		}
+
+		if (i2c.len > MLX5_EEPROM_MAX_BYTES) {
+			error = mlx5_query_eeprom(priv->mdev,
+			    MLX5E_I2C_ADDR_LOW, MLX5E_EEPROM_LOW_PAGE,
+			    (uint32_t)(i2c.offset + size_read),
+			    (uint32_t)(i2c.len - size_read), module_num,
+			    (uint32_t *)(i2c.data + size_read), &size_read);
+		}
+		if (error) {
+			if_printf(ifp, "Query eeprom failed, eeprom "
+			    "reading is not supported\n");
+			goto err_i2c;
+		}
+
+		error = copyout(&i2c, ifr->ifr_data, sizeof(i2c));
+err_i2c:
+		PRIV_UNLOCK(priv);
+		break;
+
+	default:
+		error = ether_ioctl(ifp, command, data);
+		break;
+	}
+	return (error);
+}
+
+static int
+mlx5e_check_required_hca_cap(struct mlx5_core_dev *mdev)
+{
+	/*
+	 * TODO: uncoment once FW really sets all these bits if
+	 * (!mdev->caps.eth.rss_ind_tbl_cap || !mdev->caps.eth.csum_cap ||
+	 * !mdev->caps.eth.max_lso_cap || !mdev->caps.eth.vlan_cap ||
+	 * !(mdev->caps.gen.flags & MLX5_DEV_CAP_FLAG_SCQE_BRK_MOD)) return
+	 * -ENOTSUPP;
+	 */
+
+	/* TODO: add more must-to-have features */
+
+	return (0);
+}
+
+static void
+mlx5e_build_ifp_priv(struct mlx5_core_dev *mdev,
+    struct mlx5e_priv *priv,
+    int num_comp_vectors)
+{
+	/*
+	 * TODO: Consider link speed for setting "log_sq_size",
+	 * "log_rq_size" and "cq_moderation_xxx":
+	 */
+	priv->params.log_sq_size =
+	    MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE;
+	priv->params.log_rq_size =
+	    MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE;
+	priv->params.rx_cq_moderation_usec =
+	    MLX5_CAP_GEN(mdev, cq_period_start_from_cqe) ?
+	    MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC_FROM_CQE :
+	    MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC;
+	priv->params.rx_cq_moderation_mode =
+	    MLX5_CAP_GEN(mdev, cq_period_start_from_cqe) ? 1 : 0;
+	priv->params.rx_cq_moderation_pkts =
+	    MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_PKTS;
+	priv->params.tx_cq_moderation_usec =
+	    MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC;
+	priv->params.tx_cq_moderation_pkts =
+	    MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_PKTS;
+	priv->params.min_rx_wqes =
+	    MLX5E_PARAMS_DEFAULT_MIN_RX_WQES;
+	priv->params.rx_hash_log_tbl_sz =
+	    (order_base_2(num_comp_vectors) >
+	    MLX5E_PARAMS_DEFAULT_RX_HASH_LOG_TBL_SZ) ?
+	    order_base_2(num_comp_vectors) :
+	    MLX5E_PARAMS_DEFAULT_RX_HASH_LOG_TBL_SZ;
+	priv->params.num_tc = 1;
+	priv->params.default_vlan_prio = 0;
+	priv->counter_set_id = -1;
+
+	/*
+	 * hw lro is currently defaulted to off. when it won't anymore we
+	 * will consider the HW capability: "!!MLX5_CAP_ETH(mdev, lro_cap)"
+	 */
+	priv->params.hw_lro_en = false;
+	priv->params.lro_wqe_sz = MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ;
+
+	priv->mdev = mdev;
+	priv->params.num_channels = num_comp_vectors;
+	priv->order_base_2_num_channels = order_base_2(num_comp_vectors);
+	priv->queue_mapping_channel_mask =
+	    roundup_pow_of_two(num_comp_vectors) - 1;
+	priv->num_tc = priv->params.num_tc;
+	priv->default_vlan_prio = priv->params.default_vlan_prio;
+
+	INIT_WORK(&priv->update_stats_work, mlx5e_update_stats_work);
+	INIT_WORK(&priv->update_carrier_work, mlx5e_update_carrier_work);
+	INIT_WORK(&priv->set_rx_mode_work, mlx5e_set_rx_mode_work);
+}
+
+static int
+mlx5e_create_mkey(struct mlx5e_priv *priv, u32 pdn,
+    struct mlx5_core_mr *mr)
+{
+	struct ifnet *ifp = priv->ifp;
+	struct mlx5_core_dev *mdev = priv->mdev;
+	struct mlx5_create_mkey_mbox_in *in;
+	int err;
+
+	in = mlx5_vzalloc(sizeof(*in));
+	if (in == NULL) {
+		if_printf(ifp, "%s: failed to allocate inbox\n", __func__);
+		return (-ENOMEM);
+	}
+	in->seg.flags = MLX5_PERM_LOCAL_WRITE |
+	    MLX5_PERM_LOCAL_READ |
+	    MLX5_ACCESS_MODE_PA;
+	in->seg.flags_pd = cpu_to_be32(pdn | MLX5_MKEY_LEN64);
+	in->seg.qpn_mkey7_0 = cpu_to_be32(0xffffff << 8);
+
+	err = mlx5_core_create_mkey(mdev, mr, in, sizeof(*in), NULL, NULL,
+	    NULL);
+	if (err)
+		if_printf(ifp, "%s: mlx5_core_create_mkey failed, %d\n",
+		    __func__, err);
+
+	kvfree(in);
+
+	return (err);
+}
+
+static const char *mlx5e_vport_stats_desc[] = {
+	MLX5E_VPORT_STATS(MLX5E_STATS_DESC)
+};
+
+static const char *mlx5e_pport_stats_desc[] = {
+	MLX5E_PPORT_STATS(MLX5E_STATS_DESC)
+};
+
+static void
+mlx5e_priv_mtx_init(struct mlx5e_priv *priv)
+{
+	mtx_init(&priv->async_events_mtx, "mlx5async", MTX_NETWORK_LOCK, MTX_DEF);
+	sx_init(&priv->state_lock, "mlx5state");
+	callout_init_mtx(&priv->watchdog, &priv->async_events_mtx, 0);
+}
+
+static void
+mlx5e_priv_mtx_destroy(struct mlx5e_priv *priv)
+{
+	mtx_destroy(&priv->async_events_mtx);
+	sx_destroy(&priv->state_lock);
+}
+
+static int
+sysctl_firmware(SYSCTL_HANDLER_ARGS)
+{
+	/*
+	 * %d.%d%.d the string format.
+	 * fw_rev_{maj,min,sub} return u16, 2^16 = 65536.
+	 * We need at most 5 chars to store that.
+	 * It also has: two "." and NULL at the end, which means we need 18
+	 * (5*3 + 3) chars at most.
+	 */
+	char fw[18];
+	struct mlx5e_priv *priv = arg1;
+	int error;
+
+	snprintf(fw, sizeof(fw), "%d.%d.%d", fw_rev_maj(priv->mdev), fw_rev_min(priv->mdev),
+	    fw_rev_sub(priv->mdev));
+	error = sysctl_handle_string(oidp, fw, sizeof(fw), req);
+	return (error);
+}
+
+static void
+mlx5e_add_hw_stats(struct mlx5e_priv *priv)
+{
+	SYSCTL_ADD_PROC(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_hw),
+	    OID_AUTO, "fw_version", CTLTYPE_STRING | CTLFLAG_RD, priv, 0,
+	    sysctl_firmware, "A", "HCA firmware version");
+
+	SYSCTL_ADD_STRING(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_hw),
+	    OID_AUTO, "board_id", CTLFLAG_RD, priv->mdev->board_id, 0,
+	    "Board ID");
+}
+
+static void *
+mlx5e_create_ifp(struct mlx5_core_dev *mdev)
+{
+	static volatile int mlx5_en_unit;
+	struct ifnet *ifp;
+	struct mlx5e_priv *priv;
+	u8 dev_addr[ETHER_ADDR_LEN] __aligned(4);
+	struct sysctl_oid_list *child;
+	int ncv = mdev->priv.eq_table.num_comp_vectors;
+	char unit[16];
+	int err;
+	int i;
+	u32 eth_proto_cap;
+
+	if (mlx5e_check_required_hca_cap(mdev)) {
+		mlx5_core_dbg(mdev, "mlx5e_check_required_hca_cap() failed\n");
+		return (NULL);
+	}
+	priv = malloc(sizeof(*priv), M_MLX5EN, M_WAITOK | M_ZERO);
+	if (priv == NULL) {
+		mlx5_core_err(mdev, "malloc() failed\n");
+		return (NULL);
+	}
+	mlx5e_priv_mtx_init(priv);
+
+	ifp = priv->ifp = if_alloc(IFT_ETHER);
+	if (ifp == NULL) {
+		mlx5_core_err(mdev, "if_alloc() failed\n");
+		goto err_free_priv;
+	}
+	ifp->if_softc = priv;
+	if_initname(ifp, "mce", atomic_fetchadd_int(&mlx5_en_unit, 1));
+	ifp->if_mtu = ETHERMTU;
+	ifp->if_init = mlx5e_open;
+	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
+	ifp->if_ioctl = mlx5e_ioctl;
+	ifp->if_transmit = mlx5e_xmit;
+	ifp->if_qflush = if_qflush;
+#if (__FreeBSD_version >= 1100000)
+	ifp->if_get_counter = mlx5e_get_counter;
+#endif
+	ifp->if_snd.ifq_maxlen = ifqmaxlen;
+	/*
+         * Set driver features
+         */
+	ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_HWCSUM_IPV6;
+	ifp->if_capabilities |= IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING;
+	ifp->if_capabilities |= IFCAP_VLAN_HWCSUM | IFCAP_VLAN_HWFILTER;
+	ifp->if_capabilities |= IFCAP_LINKSTATE | IFCAP_JUMBO_MTU;
+	ifp->if_capabilities |= IFCAP_LRO;
+	ifp->if_capabilities |= IFCAP_TSO | IFCAP_VLAN_HWTSO;
+
+	/* set TSO limits so that we don't have to drop TX packets */
+	ifp->if_hw_tsomax = MLX5E_MAX_TX_PAYLOAD_SIZE - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
+	ifp->if_hw_tsomaxsegcount = MLX5E_MAX_TX_MBUF_FRAGS - 1 /* hdr */;
+	ifp->if_hw_tsomaxsegsize = MLX5E_MAX_TX_MBUF_SIZE;
+
+	ifp->if_capenable = ifp->if_capabilities;
+	ifp->if_hwassist = 0;
+	if (ifp->if_capenable & IFCAP_TSO)
+		ifp->if_hwassist |= CSUM_TSO;
+	if (ifp->if_capenable & IFCAP_TXCSUM)
+		ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP | CSUM_IP);
+	if (ifp->if_capenable & IFCAP_TXCSUM_IPV6)
+		ifp->if_hwassist |= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
+
+	/* ifnet sysctl tree */
+	sysctl_ctx_init(&priv->sysctl_ctx);
+	priv->sysctl_ifnet = SYSCTL_ADD_NODE(&priv->sysctl_ctx, SYSCTL_STATIC_CHILDREN(_dev),
+	    OID_AUTO, ifp->if_dname, CTLFLAG_RD, 0, "MLX5 ethernet - interface name");
+	if (priv->sysctl_ifnet == NULL) {
+		mlx5_core_err(mdev, "SYSCTL_ADD_NODE() failed\n");
+		goto err_free_sysctl;
+	}
+	snprintf(unit, sizeof(unit), "%d", ifp->if_dunit);
+	priv->sysctl_ifnet = SYSCTL_ADD_NODE(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
+	    OID_AUTO, unit, CTLFLAG_RD, 0, "MLX5 ethernet - interface unit");
+	if (priv->sysctl_ifnet == NULL) {
+		mlx5_core_err(mdev, "SYSCTL_ADD_NODE() failed\n");
+		goto err_free_sysctl;
+	}
+
+	/* HW sysctl tree */
+	child = SYSCTL_CHILDREN(device_get_sysctl_tree(mdev->pdev->dev.bsddev));
+	priv->sysctl_hw = SYSCTL_ADD_NODE(&priv->sysctl_ctx, child,
+	    OID_AUTO, "hw", CTLFLAG_RD, 0, "MLX5 ethernet dev hw");
+	if (priv->sysctl_hw == NULL) {
+		mlx5_core_err(mdev, "SYSCTL_ADD_NODE() failed\n");
+		goto err_free_sysctl;
+	}
+	mlx5e_build_ifp_priv(mdev, priv, ncv);
+	err = mlx5_alloc_map_uar(mdev, &priv->cq_uar);
+	if (err) {
+		if_printf(ifp, "%s: mlx5_alloc_map_uar failed, %d\n",
+		    __func__, err);
+		goto err_free_sysctl;
+	}
+	err = mlx5_core_alloc_pd(mdev, &priv->pdn);
+	if (err) {
+		if_printf(ifp, "%s: mlx5_core_alloc_pd failed, %d\n",
+		    __func__, err);
+		goto err_unmap_free_uar;
+	}
+	err = mlx5_alloc_transport_domain(mdev, &priv->tdn);
+	if (err) {
+		if_printf(ifp, "%s: mlx5_alloc_transport_domain failed, %d\n",
+		    __func__, err);
+		goto err_dealloc_pd;
+	}
+	err = mlx5e_create_mkey(priv, priv->pdn, &priv->mr);
+	if (err) {
+		if_printf(ifp, "%s: mlx5e_create_mkey failed, %d\n",
+		    __func__, err);
+		goto err_dealloc_transport_domain;
+	}
+	mlx5_query_nic_vport_mac_address(priv->mdev, 0, dev_addr);
+
+	/* set default MTU */
+	mlx5e_set_dev_port_mtu(ifp, ifp->if_mtu);
+
+	/* Set desc */
+	device_set_desc(mdev->pdev->dev.bsddev, mlx5e_version);
+
+	/* Set default media status */
+	priv->media_status_last = IFM_AVALID;
+	priv->media_active_last = IFM_ETHER | IFM_AUTO;
+
+	/* Pauseframes are enabled by default */
+	priv->params_ethtool.tx_pauseframe_control = 1;
+	priv->params_ethtool.rx_pauseframe_control = 1;
+
+	err = mlx5_query_port_proto_cap(mdev, &eth_proto_cap, MLX5_PTYS_EN);
+	if (err) {
+		eth_proto_cap = 0;
+		if_printf(ifp, "%s: Query port media capability failed, %d\n",
+		    __func__, err);
+	}
+
+	/* Setup supported medias */
+	ifmedia_init(&priv->media, IFM_IMASK | IFM_ETH_FMASK,
+	    mlx5e_media_change, mlx5e_media_status);
+
+	for (i = 0; i < MLX5E_LINK_MODES_NUMBER; ++i) {
+		if (mlx5e_mode_table[i].baudrate == 0)
+			continue;
+		if (MLX5E_PROT_MASK(i) & eth_proto_cap)
+			ifmedia_add(&priv->media,
+			    IFM_ETHER | mlx5e_mode_table[i].subtype |
+			    IFM_FDX, 0, NULL);
+	}
+
+	ifmedia_add(&priv->media, IFM_ETHER | IFM_AUTO, 0, NULL);
+	ifmedia_set(&priv->media, IFM_ETHER | IFM_AUTO);
+	ether_ifattach(ifp, dev_addr);
+
+	/* Register for VLAN events */
+	priv->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
+	    mlx5e_vlan_rx_add_vid, priv, EVENTHANDLER_PRI_FIRST);
+	priv->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
+	    mlx5e_vlan_rx_kill_vid, priv, EVENTHANDLER_PRI_FIRST);
+
+	/* Link is down by default */
+	if_link_state_change(ifp, LINK_STATE_DOWN);
+
+	mlx5e_enable_async_events(priv);
+
+	mlx5e_add_hw_stats(priv);
+
+	mlx5e_create_stats(&priv->stats.vport.ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
+	    "vstats", mlx5e_vport_stats_desc, MLX5E_VPORT_STATS_NUM,
+	    priv->stats.vport.arg);
+
+	mlx5e_create_stats(&priv->stats.pport.ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
+	    "pstats", mlx5e_pport_stats_desc, MLX5E_PPORT_STATS_NUM,
+	    priv->stats.pport.arg);
+
+	mlx5e_create_ethtool(priv);
+
+	mtx_lock(&priv->async_events_mtx);
+	mlx5e_update_stats(priv);
+	mtx_unlock(&priv->async_events_mtx);
+
+	return (priv);
+
+err_dealloc_transport_domain:
+	mlx5_dealloc_transport_domain(mdev, priv->tdn);
+
+err_dealloc_pd:
+	mlx5_core_dealloc_pd(mdev, priv->pdn);
+
+err_unmap_free_uar:
+	mlx5_unmap_free_uar(mdev, &priv->cq_uar);
+
+err_free_sysctl:
+	sysctl_ctx_free(&priv->sysctl_ctx);
+
+	if_free(ifp);
+
+err_free_priv:
+	mlx5e_priv_mtx_destroy(priv);
+	free(priv, M_MLX5EN);
+	return (NULL);
+}
+
+static void
+mlx5e_destroy_ifp(struct mlx5_core_dev *mdev, void *vpriv)
+{
+	struct mlx5e_priv *priv = vpriv;
+	struct ifnet *ifp = priv->ifp;
+
+	/* don't allow more IOCTLs */
+	priv->gone = 1;
+
+	/* XXX wait a bit to allow IOCTL handlers to complete */
+	pause("W", hz);
+
+	/* stop watchdog timer */
+	callout_drain(&priv->watchdog);
+
+	if (priv->vlan_attach != NULL)
+		EVENTHANDLER_DEREGISTER(vlan_config, priv->vlan_attach);
+	if (priv->vlan_detach != NULL)
+		EVENTHANDLER_DEREGISTER(vlan_unconfig, priv->vlan_detach);
+
+	/* make sure device gets closed */
+	PRIV_LOCK(priv);
+	mlx5e_close_locked(ifp);
+	PRIV_UNLOCK(priv);
+
+	/* unregister device */
+	ifmedia_removeall(&priv->media);
+	ether_ifdetach(ifp);
+	if_free(ifp);
+
+	/* destroy all remaining sysctl nodes */
+	if (priv->sysctl_debug)
+		sysctl_ctx_free(&priv->stats.port_stats_debug.ctx);
+	sysctl_ctx_free(&priv->stats.vport.ctx);
+	sysctl_ctx_free(&priv->stats.pport.ctx);
+	sysctl_ctx_free(&priv->sysctl_ctx);
+
+	mlx5_core_destroy_mkey(priv->mdev, &priv->mr);
+	mlx5_dealloc_transport_domain(priv->mdev, priv->tdn);
+	mlx5_core_dealloc_pd(priv->mdev, priv->pdn);
+	mlx5_unmap_free_uar(priv->mdev, &priv->cq_uar);
+	mlx5e_disable_async_events(priv);
+	flush_scheduled_work();
+	mlx5e_priv_mtx_destroy(priv);
+	free(priv, M_MLX5EN);
+}
+
+static void *
+mlx5e_get_ifp(void *vpriv)
+{
+	struct mlx5e_priv *priv = vpriv;
+
+	return (priv->ifp);
+}
+
+static struct mlx5_interface mlx5e_interface = {
+	.add = mlx5e_create_ifp,
+	.remove = mlx5e_destroy_ifp,
+	.event = mlx5e_async_event,
+	.protocol = MLX5_INTERFACE_PROTOCOL_ETH,
+	.get_dev = mlx5e_get_ifp,
+};
+
+void
+mlx5e_init(void)
+{
+	mlx5_register_interface(&mlx5e_interface);
+}
+
+void
+mlx5e_cleanup(void)
+{
+	mlx5_unregister_interface(&mlx5e_interface);
+}
+
+module_init_order(mlx5e_init, SI_ORDER_THIRD);
+module_exit_order(mlx5e_cleanup, SI_ORDER_THIRD);
+
+#if (__FreeBSD_version >= 1100000)
+MODULE_DEPEND(mlx5en, linuxkpi, 1, 1, 1);
+#endif
+MODULE_DEPEND(mlx5en, mlx5, 1, 1, 1);
+MODULE_VERSION(mlx5en, 1);
diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c b/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c
new file mode 100644
index 0000000..3e827bd
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_en/mlx5_en_rx.c
@@ -0,0 +1,343 @@
+/*-
+ * Copyright (c) 2015 Mellanox Technologies. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include "en.h"
+#include <machine/in_cksum.h>
+
+static inline int
+mlx5e_alloc_rx_wqe(struct mlx5e_rq *rq,
+    struct mlx5e_rx_wqe *wqe, u16 ix)
+{
+	bus_dma_segment_t segs[1];
+	struct mbuf *mb;
+	int nsegs;
+	int err;
+
+	if (rq->mbuf[ix].mbuf != NULL)
+		return (0);
+
+	mb = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, rq->wqe_sz);
+	if (unlikely(!mb))
+		return (-ENOMEM);
+
+	/* set initial mbuf length */
+	mb->m_pkthdr.len = mb->m_len = rq->wqe_sz;
+
+	/* get IP header aligned */
+	m_adj(mb, MLX5E_NET_IP_ALIGN);
+
+	err = -bus_dmamap_load_mbuf_sg(rq->dma_tag, rq->mbuf[ix].dma_map,
+	    mb, segs, &nsegs, BUS_DMA_NOWAIT);
+	if (err != 0)
+		goto err_free_mbuf;
+	if (unlikely(nsegs != 1)) {
+		bus_dmamap_unload(rq->dma_tag, rq->mbuf[ix].dma_map);
+		err = -ENOMEM;
+		goto err_free_mbuf;
+	}
+	wqe->data.addr = cpu_to_be64(segs[0].ds_addr);
+
+	rq->mbuf[ix].mbuf = mb;
+	rq->mbuf[ix].data = mb->m_data;
+
+	bus_dmamap_sync(rq->dma_tag, rq->mbuf[ix].dma_map,
+	    BUS_DMASYNC_PREREAD);
+	return (0);
+
+err_free_mbuf:
+	m_freem(mb);
+	return (err);
+}
+
+static void
+mlx5e_post_rx_wqes(struct mlx5e_rq *rq)
+{
+	if (unlikely(rq->enabled == 0))
+		return;
+
+	while (!mlx5_wq_ll_is_full(&rq->wq)) {
+		struct mlx5e_rx_wqe *wqe = mlx5_wq_ll_get_wqe(&rq->wq, rq->wq.head);
+
+		if (unlikely(mlx5e_alloc_rx_wqe(rq, wqe, rq->wq.head)))
+			break;
+
+		mlx5_wq_ll_push(&rq->wq, be16_to_cpu(wqe->next.next_wqe_index));
+	}
+
+	/* ensure wqes are visible to device before updating doorbell record */
+	wmb();
+
+	mlx5_wq_ll_update_db_record(&rq->wq);
+}
+
+static void
+mlx5e_lro_update_hdr(struct mbuf *mb, struct mlx5_cqe64 *cqe)
+{
+	/* TODO: consider vlans, ip options, ... */
+	struct ether_header *eh;
+	uint16_t eh_type;
+	struct ip6_hdr *ip6 = NULL;
+	struct ip *ip4 = NULL;
+	struct tcphdr *th;
+	uint32_t *ts_ptr;
+
+	eh = mtod(mb, struct ether_header *);
+	eh_type = ntohs(eh->ether_type);
+
+	u8 l4_hdr_type = get_cqe_l4_hdr_type(cqe);
+	int tcp_ack = ((CQE_L4_HDR_TYPE_TCP_ACK_NO_DATA == l4_hdr_type) ||
+	    (CQE_L4_HDR_TYPE_TCP_ACK_AND_DATA == l4_hdr_type));
+
+	/* TODO: consider vlan */
+	u16 tot_len = be32_to_cpu(cqe->byte_cnt) - ETHER_HDR_LEN;
+
+	switch (eh_type) {
+	case ETHERTYPE_IP:
+		ip4 = (struct ip *)(eh + 1);
+		th = (struct tcphdr *)(ip4 + 1);
+		break;
+	case ETHERTYPE_IPV6:
+		ip6 = (struct ip6_hdr *)(eh + 1);
+		th = (struct tcphdr *)(ip6 + 1);
+		break;
+	default:
+		return;
+	}
+
+	ts_ptr = (uint32_t *)(th + 1);
+
+	if (get_cqe_lro_tcppsh(cqe))
+		th->th_flags |= TH_PUSH;
+
+	if (tcp_ack) {
+		th->th_flags |= TH_ACK;
+		th->th_ack = cqe->lro_ack_seq_num;
+		th->th_win = cqe->lro_tcp_win;
+
+		/*
+		 * FreeBSD handles only 32bit aligned timestamp right after
+		 * the TCP hdr
+		 * +--------+--------+--------+--------+
+		 * |   NOP  |  NOP   |  TSopt |   10   |
+		 * +--------+--------+--------+--------+
+		 * |          TSval   timestamp        |
+		 * +--------+--------+--------+--------+
+		 * |          TSecr   timestamp        |
+		 * +--------+--------+--------+--------+
+		 */
+		if (get_cqe_lro_timestamp_valid(cqe) &&
+		    (__predict_true(*ts_ptr) == ntohl(TCPOPT_NOP << 24 |
+		    TCPOPT_NOP << 16 | TCPOPT_TIMESTAMP << 8 |
+		    TCPOLEN_TIMESTAMP))) {
+			/*
+			 * cqe->timestamp is 64bit long.
+			 * [0-31] - timestamp.
+			 * [32-64] - timestamp echo replay.
+			 */
+			ts_ptr[1] = *(uint32_t *)&cqe->timestamp;
+			ts_ptr[2] = *((uint32_t *)&cqe->timestamp + 1);
+		}
+	}
+	if (ip4) {
+		ip4->ip_ttl = cqe->lro_min_ttl;
+		ip4->ip_len = cpu_to_be16(tot_len);
+		ip4->ip_sum = 0;
+		ip4->ip_sum = in_cksum(mb, ip4->ip_hl << 2);
+	} else {
+		ip6->ip6_hlim = cqe->lro_min_ttl;
+		ip6->ip6_plen = cpu_to_be16(tot_len -
+		    sizeof(struct ip6_hdr));
+	}
+	/* TODO: handle tcp checksum */
+}
+
+static inline void
+mlx5e_build_rx_mbuf(struct mlx5_cqe64 *cqe,
+    struct mlx5e_rq *rq, struct mbuf *mb,
+    u32 cqe_bcnt)
+{
+	struct ifnet *ifp = rq->ifp;
+	int lro_num_seg;	/* HW LRO session aggregated packets counter */
+
+	lro_num_seg = be32_to_cpu(cqe->srqn) >> 24;
+	if (lro_num_seg > 1) {
+		mlx5e_lro_update_hdr(mb, cqe);
+		rq->stats.lro_packets++;
+		rq->stats.lro_bytes += cqe_bcnt;
+	}
+
+	mb->m_pkthdr.len = mb->m_len = cqe_bcnt;
+	/* check if a Toeplitz hash was computed */
+	if (cqe->rss_hash_type != 0)
+		mb->m_pkthdr.flowid = be32_to_cpu(cqe->rss_hash_result);
+	else
+		mb->m_pkthdr.flowid = rq->ix;
+
+	M_HASHTYPE_SET(mb, M_HASHTYPE_OPAQUE);
+	mb->m_pkthdr.rcvif = ifp;
+
+	if (likely(ifp->if_capenable & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) &&
+	    ((cqe->hds_ip_ext & (CQE_L2_OK | CQE_L3_OK | CQE_L4_OK)) ==
+	    (CQE_L2_OK | CQE_L3_OK | CQE_L4_OK))) {
+		mb->m_pkthdr.csum_flags =
+		    CSUM_IP_CHECKED | CSUM_IP_VALID |
+		    CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
+		mb->m_pkthdr.csum_data = htons(0xffff);
+	} else {
+		rq->stats.csum_none++;
+	}
+
+	if (cqe_has_vlan(cqe)) {
+		mb->m_pkthdr.ether_vtag = be16_to_cpu(cqe->vlan_info);
+		mb->m_flags |= M_VLANTAG;
+	}
+}
+
+static int
+mlx5e_poll_rx_cq(struct mlx5e_rq *rq, int budget)
+{
+#ifndef HAVE_TURBO_LRO
+	struct lro_entry *queued;
+#endif
+	int i;
+
+	for (i = 0; i < budget; i++) {
+		struct mlx5e_rx_wqe *wqe;
+		struct mlx5_cqe64 *cqe;
+		struct mbuf *mb;
+		__be16 wqe_counter_be;
+		u16 wqe_counter;
+		u32 byte_cnt;
+
+		cqe = mlx5e_get_cqe(&rq->cq);
+		if (!cqe)
+			break;
+
+		wqe_counter_be = cqe->wqe_counter;
+		wqe_counter = be16_to_cpu(wqe_counter_be);
+		wqe = mlx5_wq_ll_get_wqe(&rq->wq, wqe_counter);
+		byte_cnt = be32_to_cpu(cqe->byte_cnt);
+
+		bus_dmamap_sync(rq->dma_tag,
+		    rq->mbuf[wqe_counter].dma_map,
+		    BUS_DMASYNC_POSTREAD);
+
+		if (unlikely((cqe->op_own >> 4) != MLX5_CQE_RESP_SEND)) {
+			rq->stats.wqe_err++;
+			goto wq_ll_pop;
+		}
+
+		if (MHLEN >= byte_cnt &&
+		    (mb = m_gethdr(M_NOWAIT, MT_DATA)) != NULL) {
+			bcopy(rq->mbuf[wqe_counter].data, mtod(mb, caddr_t),
+			    byte_cnt);
+		} else {
+			mb = rq->mbuf[wqe_counter].mbuf;
+			rq->mbuf[wqe_counter].mbuf = NULL;	/* safety clear */
+
+			bus_dmamap_unload(rq->dma_tag,
+			    rq->mbuf[wqe_counter].dma_map);
+		}
+
+		mlx5e_build_rx_mbuf(cqe, rq, mb, byte_cnt);
+		rq->stats.packets++;
+#ifdef HAVE_TURBO_LRO
+		if (mb->m_pkthdr.csum_flags == 0 ||
+		    (rq->ifp->if_capenable & IFCAP_LRO) == 0 ||
+		    rq->lro.mbuf == NULL) {
+			/* normal input */
+			rq->ifp->if_input(rq->ifp, mb);
+		} else {
+			tcp_tlro_rx(&rq->lro, mb);
+		}
+#else
+		if (mb->m_pkthdr.csum_flags == 0 ||
+		    (rq->ifp->if_capenable & IFCAP_LRO) == 0 ||
+		    rq->lro.lro_cnt == 0 ||
+		    tcp_lro_rx(&rq->lro, mb, 0) != 0) {
+			rq->ifp->if_input(rq->ifp, mb);
+		}
+#endif
+wq_ll_pop:
+		mlx5_wq_ll_pop(&rq->wq, wqe_counter_be,
+		    &wqe->next.next_wqe_index);
+	}
+
+	mlx5_cqwq_update_db_record(&rq->cq.wq);
+
+	/* ensure cq space is freed before enabling more cqes */
+	wmb();
+#ifndef HAVE_TURBO_LRO
+	while ((queued = SLIST_FIRST(&rq->lro.lro_active)) != NULL) {
+		SLIST_REMOVE_HEAD(&rq->lro.lro_active, next);
+		tcp_lro_flush(&rq->lro, queued);
+	}
+#endif
+	return (i);
+}
+
+void
+mlx5e_rx_cq_comp(struct mlx5_core_cq *mcq)
+{
+	struct mlx5e_rq *rq = container_of(mcq, struct mlx5e_rq, cq.mcq);
+	int i = 0;
+
+#ifdef HAVE_PER_CQ_EVENT_PACKET
+	struct mbuf *mb = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, rq->wqe_sz);
+
+	if (mb != NULL) {
+		/* this code is used for debugging purpose only */
+		mb->m_pkthdr.len = mb->m_len = 15;
+		memset(mb->m_data, 255, 14);
+		mb->m_data[14] = rq->ix;
+		mb->m_pkthdr.rcvif = rq->ifp;
+		rq->ifp->if_input(rq->ifp, mb);
+	}
+#endif
+
+	mtx_lock(&rq->mtx);
+
+	/*
+	 * Polling the entire CQ without posting new WQEs results in
+	 * lack of receive WQEs during heavy traffic scenarios.
+	 */
+	while (1) {
+		if (mlx5e_poll_rx_cq(rq, MLX5E_RX_BUDGET_MAX) !=
+		    MLX5E_RX_BUDGET_MAX)
+			break;
+		i += MLX5E_RX_BUDGET_MAX;
+		if (i >= MLX5E_BUDGET_MAX)
+			break;
+		mlx5e_post_rx_wqes(rq);
+	}
+	mlx5e_post_rx_wqes(rq);
+	mlx5e_cq_arm(&rq->cq);
+#ifdef HAVE_TURBO_LRO
+	tcp_tlro_flush(&rq->lro, 1);
+#endif
+	mtx_unlock(&rq->mtx);
+}
diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c b/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c
new file mode 100644
index 0000000..0256fc7
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c
@@ -0,0 +1,492 @@
+/*-
+ * Copyright (c) 2015 Mellanox Technologies. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include "en.h"
+#include <machine/atomic.h>
+
+void
+mlx5e_send_nop(struct mlx5e_sq *sq, u32 ds_cnt, bool notify_hw)
+{
+	u16 pi = sq->pc & sq->wq.sz_m1;
+	struct mlx5e_tx_wqe *wqe = mlx5_wq_cyc_get_wqe(&sq->wq, pi);
+
+	memset(&wqe->ctrl, 0, sizeof(wqe->ctrl));
+
+	wqe->ctrl.opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | MLX5_OPCODE_NOP);
+	wqe->ctrl.qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt);
+	wqe->ctrl.fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE;
+
+	sq->mbuf[pi].mbuf = NULL;
+	sq->mbuf[pi].num_bytes = 0;
+	sq->mbuf[pi].num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS);
+	sq->pc += sq->mbuf[pi].num_wqebbs;
+	if (notify_hw)
+		mlx5e_tx_notify_hw(sq, wqe, 0);
+}
+
+#if (__FreeBSD_version >= 1100000)
+static uint32_t mlx5e_hash_value;
+
+static void
+mlx5e_hash_init(void *arg)
+{
+	mlx5e_hash_value = m_ether_tcpip_hash_init();
+}
+
+/* Make kernel call mlx5e_hash_init after the random stack finished initializing */
+SYSINIT(mlx5e_hash_init, SI_SUB_RANDOM, SI_ORDER_ANY, &mlx5e_hash_init, NULL);
+#endif
+
+static struct mlx5e_sq *
+mlx5e_select_queue(struct ifnet *ifp, struct mbuf *mb)
+{
+	struct mlx5e_priv *priv = ifp->if_softc;
+	u32 ch;
+	u32 tc;
+
+	/* check if channels are successfully opened */
+	if (unlikely(priv->channel == NULL))
+		return (NULL);
+
+	/* obtain VLAN information if present */
+	if (mb->m_flags & M_VLANTAG) {
+		tc = (mb->m_pkthdr.ether_vtag >> 13);
+		if (tc >= priv->num_tc)
+			tc = priv->default_vlan_prio;
+	} else {
+		tc = priv->default_vlan_prio;
+	}
+
+	ch = priv->params.num_channels;
+
+	/* check if flowid is set */
+	if (M_HASHTYPE_GET(mb) != M_HASHTYPE_NONE) {
+		ch = (mb->m_pkthdr.flowid % 128) % ch;
+	} else {
+#if (__FreeBSD_version >= 1100000)
+		ch = m_ether_tcpip_hash(MBUF_HASHFLAG_L3 |
+		    MBUF_HASHFLAG_L4, mb, mlx5e_hash_value) % ch;
+#else
+		/*
+		 * m_ether_tcpip_hash not present in stable, so just
+		 * throw unhashed mbufs on queue 0
+		 */
+		ch = 0;
+#endif
+	}
+
+	/* check if channel is allocated */
+	if (unlikely(priv->channel[ch] == NULL))
+		return (NULL);
+
+	return (&priv->channel[ch]->sq[tc]);
+}
+
+static inline u16
+mlx5e_get_inline_hdr_size(struct mlx5e_sq *sq, struct mbuf *mb)
+{
+	return (MIN(MLX5E_MAX_TX_INLINE, mb->m_len));
+}
+
+static int
+mlx5e_get_header_size(struct mbuf *mb)
+{
+	struct ether_vlan_header *eh;
+	struct tcphdr *th;
+	struct ip *ip;
+	int ip_hlen, tcp_hlen;
+	struct ip6_hdr *ip6;
+	uint16_t eth_type;
+	int eth_hdr_len;
+
+	eh = mtod(mb, struct ether_vlan_header *);
+	if (mb->m_len < ETHER_HDR_LEN)
+		return (0);
+	if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
+		eth_type = ntohs(eh->evl_proto);
+		eth_hdr_len = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
+	} else {
+		eth_type = ntohs(eh->evl_encap_proto);
+		eth_hdr_len = ETHER_HDR_LEN;
+	}
+	if (mb->m_len < eth_hdr_len)
+		return (0);
+	switch (eth_type) {
+	case ETHERTYPE_IP:
+		ip = (struct ip *)(mb->m_data + eth_hdr_len);
+		if (mb->m_len < eth_hdr_len + sizeof(*ip))
+			return (0);
+		if (ip->ip_p != IPPROTO_TCP)
+			return (0);
+		ip_hlen = ip->ip_hl << 2;
+		eth_hdr_len += ip_hlen;
+		break;
+	case ETHERTYPE_IPV6:
+		ip6 = (struct ip6_hdr *)(mb->m_data + eth_hdr_len);
+		if (mb->m_len < eth_hdr_len + sizeof(*ip6))
+			return (0);
+		if (ip6->ip6_nxt != IPPROTO_TCP)
+			return (0);
+		eth_hdr_len += sizeof(*ip6);
+		break;
+	default:
+		return (0);
+	}
+	if (mb->m_len < eth_hdr_len + sizeof(*th))
+		return (0);
+	th = (struct tcphdr *)(mb->m_data + eth_hdr_len);
+	tcp_hlen = th->th_off << 2;
+	eth_hdr_len += tcp_hlen;
+	if (mb->m_len < eth_hdr_len)
+		return (0);
+	return (eth_hdr_len);
+}
+
+/*
+ * The return value is not going back to the stack because of
+ * the drbr
+ */
+static int
+mlx5e_sq_xmit(struct mlx5e_sq *sq, struct mbuf **mbp)
+{
+	bus_dma_segment_t segs[MLX5E_MAX_TX_MBUF_FRAGS];
+	struct mlx5_wqe_data_seg *dseg;
+	struct mlx5e_tx_wqe *wqe;
+	struct ifnet *ifp;
+	int nsegs;
+	int err;
+	int x;
+	struct mbuf *mb = *mbp;
+	u16 ds_cnt;
+	u16 ihs;
+	u16 pi;
+	u8 opcode;
+
+	/*
+	 * Return ENOBUFS if the queue is full, this may trigger reinsertion
+	 * of the mbuf into the drbr (see mlx5e_xmit_locked)
+	 */
+	if (unlikely(!mlx5e_sq_has_room_for(sq, 2 * MLX5_SEND_WQE_MAX_WQEBBS))) {
+		return (ENOBUFS);
+	}
+
+	/* Align SQ edge with NOPs to avoid WQE wrap around */
+	pi = ((~sq->pc) & sq->wq.sz_m1);
+	if (pi < (MLX5_SEND_WQE_MAX_WQEBBS - 1)) {
+		/* Send one multi NOP message instead of many */
+		mlx5e_send_nop(sq, (pi + 1) * MLX5_SEND_WQEBB_NUM_DS, false);
+		pi = ((~sq->pc) & sq->wq.sz_m1);
+		if (pi < (MLX5_SEND_WQE_MAX_WQEBBS - 1)) {
+			m_freem(mb);
+			return (ENOMEM);
+		}
+	}
+
+	/* Setup local variables */
+	pi = sq->pc & sq->wq.sz_m1;
+	wqe = mlx5_wq_cyc_get_wqe(&sq->wq, pi);
+	ifp = sq->channel->ifp;
+
+	memset(wqe, 0, sizeof(*wqe));
+
+	/* Send a copy of the frame to the BPF listener, if any */
+	if (ifp != NULL && ifp->if_bpf != NULL)
+		ETHER_BPF_MTAP(ifp, mb);
+
+	if (mb->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TSO)) {
+		wqe->eth.cs_flags |= MLX5_ETH_WQE_L3_CSUM;
+	}
+	if (mb->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP | CSUM_UDP_IPV6 | CSUM_TCP_IPV6 | CSUM_TSO)) {
+		wqe->eth.cs_flags |= MLX5_ETH_WQE_L4_CSUM;
+	}
+	if (wqe->eth.cs_flags == 0) {
+		sq->stats.csum_offload_none++;
+	}
+	if (mb->m_pkthdr.csum_flags & CSUM_TSO) {
+		u32 payload_len;
+		u32 mss = mb->m_pkthdr.tso_segsz;
+		u32 num_pkts;
+
+		wqe->eth.mss = cpu_to_be16(mss);
+		opcode = MLX5_OPCODE_LSO;
+		ihs = mlx5e_get_header_size(mb);
+		payload_len = mb->m_pkthdr.len - ihs;
+		if (payload_len == 0)
+			num_pkts = 1;
+		else
+			num_pkts = DIV_ROUND_UP(payload_len, mss);
+		sq->mbuf[pi].num_bytes = payload_len + (num_pkts * ihs);
+
+		sq->stats.tso_packets++;
+		sq->stats.tso_bytes += payload_len;
+	} else {
+		opcode = MLX5_OPCODE_SEND;
+		ihs = mlx5e_get_inline_hdr_size(sq, mb);
+		sq->mbuf[pi].num_bytes = max_t (unsigned int,
+		    mb->m_pkthdr.len, ETHER_MIN_LEN - ETHER_CRC_LEN);
+	}
+	if (mb->m_flags & M_VLANTAG) {
+		struct ether_vlan_header *eh =
+		    (struct ether_vlan_header *)wqe->eth.inline_hdr_start;
+
+		/* Range checks */
+		if (ihs > (MLX5E_MAX_TX_INLINE - ETHER_VLAN_ENCAP_LEN))
+			ihs = (MLX5E_MAX_TX_INLINE - ETHER_VLAN_ENCAP_LEN);
+		else if (ihs < ETHER_HDR_LEN) {
+			err = EINVAL;
+			goto tx_drop;
+		}
+		m_copydata(mb, 0, ETHER_HDR_LEN, (caddr_t)eh);
+		m_adj(mb, ETHER_HDR_LEN);
+		/* Insert 4 bytes VLAN tag into data stream */
+		eh->evl_proto = eh->evl_encap_proto;
+		eh->evl_encap_proto = htons(ETHERTYPE_VLAN);
+		eh->evl_tag = htons(mb->m_pkthdr.ether_vtag);
+		/* Copy rest of header data, if any */
+		m_copydata(mb, 0, ihs - ETHER_HDR_LEN, (caddr_t)(eh + 1));
+		m_adj(mb, ihs - ETHER_HDR_LEN);
+		/* Extend header by 4 bytes */
+		ihs += ETHER_VLAN_ENCAP_LEN;
+	} else {
+		m_copydata(mb, 0, ihs, wqe->eth.inline_hdr_start);
+		m_adj(mb, ihs);
+	}
+
+	wqe->eth.inline_hdr_sz = cpu_to_be16(ihs);
+
+	ds_cnt = sizeof(*wqe) / MLX5_SEND_WQE_DS;
+	if (likely(ihs > sizeof(wqe->eth.inline_hdr_start))) {
+		ds_cnt += DIV_ROUND_UP(ihs - sizeof(wqe->eth.inline_hdr_start),
+		    MLX5_SEND_WQE_DS);
+	}
+	dseg = ((struct mlx5_wqe_data_seg *)&wqe->ctrl) + ds_cnt;
+
+	/* Trim off empty mbufs */
+	while (mb->m_len == 0) {
+		mb = m_free(mb);
+		/* Check if all data has been inlined */
+		if (mb == NULL)
+			goto skip_dma;
+	}
+
+	err = bus_dmamap_load_mbuf_sg(sq->dma_tag, sq->mbuf[pi].dma_map,
+	    mb, segs, &nsegs, BUS_DMA_NOWAIT);
+	if (err == EFBIG) {
+		/*
+		 * Update *mbp before defrag in case it was trimmed in the
+		 * loop above
+		 */
+		*mbp = mb;
+		/* Update statistics */
+		sq->stats.defragged++;
+		/* Too many mbuf fragments */
+		mb = m_defrag(*mbp, M_NOWAIT);
+		if (mb == NULL) {
+			mb = *mbp;
+			goto tx_drop;
+		}
+		/* Try again */
+		err = bus_dmamap_load_mbuf_sg(sq->dma_tag, sq->mbuf[pi].dma_map,
+		    mb, segs, &nsegs, BUS_DMA_NOWAIT);
+	}
+	/* Catch errors */
+	if (err != 0) {
+		goto tx_drop;
+	}
+	*mbp = mb;
+
+	for (x = 0; x != nsegs; x++) {
+		if (segs[x].ds_len == 0)
+			continue;
+		dseg->addr = cpu_to_be64((uint64_t)segs[x].ds_addr);
+		dseg->lkey = sq->mkey_be;
+		dseg->byte_count = cpu_to_be32((uint32_t)segs[x].ds_len);
+		dseg++;
+	}
+skip_dma:
+	ds_cnt = (dseg - ((struct mlx5_wqe_data_seg *)&wqe->ctrl));
+
+	wqe->ctrl.opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | opcode);
+	wqe->ctrl.qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt);
+	wqe->ctrl.fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE;
+
+	/* Store pointer to mbuf */
+	sq->mbuf[pi].mbuf = mb;
+	sq->mbuf[pi].num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS);
+	sq->pc += sq->mbuf[pi].num_wqebbs;
+
+	/* Make sure all mbuf data is written to RAM */
+	if (mb != NULL)
+		bus_dmamap_sync(sq->dma_tag, sq->mbuf[pi].dma_map, BUS_DMASYNC_PREWRITE);
+
+	mlx5e_tx_notify_hw(sq, wqe, 0);
+
+	sq->stats.packets++;
+	return (0);
+
+tx_drop:
+	sq->stats.dropped++;
+	*mbp = NULL;
+	m_freem(mb);
+	return err;
+}
+
+static void
+mlx5e_poll_tx_cq(struct mlx5e_sq *sq, int budget)
+{
+	u16 sqcc;
+
+	/*
+	 * sq->cc must be updated only after mlx5_cqwq_update_db_record(),
+	 * otherwise a cq overrun may occur
+	 */
+	sqcc = sq->cc;
+
+	while (budget--) {
+		struct mlx5_cqe64 *cqe;
+		struct mbuf *mb;
+		u16 ci;
+
+		cqe = mlx5e_get_cqe(&sq->cq);
+		if (!cqe)
+			break;
+
+		ci = sqcc & sq->wq.sz_m1;
+		mb = sq->mbuf[ci].mbuf;
+		sq->mbuf[ci].mbuf = NULL;	/* Safety clear */
+
+		if (mb == NULL) {
+			if (sq->mbuf[ci].num_bytes == 0) {
+				/* NOP */
+				sq->stats.nop++;
+			}
+		} else {
+			bus_dmamap_sync(sq->dma_tag, sq->mbuf[ci].dma_map,
+			    BUS_DMASYNC_POSTWRITE);
+			bus_dmamap_unload(sq->dma_tag, sq->mbuf[ci].dma_map);
+
+			/* Free transmitted mbuf */
+			m_freem(mb);
+		}
+		sqcc += sq->mbuf[ci].num_wqebbs;
+	}
+
+	mlx5_cqwq_update_db_record(&sq->cq.wq);
+
+	/* Ensure cq space is freed before enabling more cqes */
+	wmb();
+
+	sq->cc = sqcc;
+
+	if (atomic_cmpset_int(&sq->queue_state, MLX5E_SQ_FULL, MLX5E_SQ_READY))
+		taskqueue_enqueue(sq->sq_tq, &sq->sq_task);
+}
+
+static int
+mlx5e_xmit_locked(struct ifnet *ifp, struct mlx5e_sq *sq, struct mbuf *mb)
+{
+	struct mbuf *next;
+	int err = 0;
+
+	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
+		if (mb)
+			err = drbr_enqueue(ifp, sq->br, mb);
+		return (err);
+	}
+
+	if (mb != NULL)
+		/*
+		 * If we can't insert mbuf into drbr, try to xmit anyway.
+		 * We keep the error we got so we could return that after xmit.
+		 */
+		err = drbr_enqueue(ifp, sq->br, mb);
+
+	/* Process the queue */
+	while ((next = drbr_peek(ifp, sq->br)) != NULL) {
+		if (mlx5e_sq_xmit(sq, &next) != 0) {
+			if (next == NULL) {
+				drbr_advance(ifp, sq->br);
+			} else {
+				drbr_putback(ifp, sq->br, next);
+				atomic_store_rel_int(&sq->queue_state, MLX5E_SQ_FULL);
+			}
+			break;
+		}
+		drbr_advance(ifp, sq->br);
+		if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
+			break;
+	}
+	return (err);
+}
+
+int
+mlx5e_xmit(struct ifnet *ifp, struct mbuf *mb)
+{
+	struct mlx5e_sq *sq;
+	int ret;
+
+	sq = mlx5e_select_queue(ifp, mb);
+	if (unlikely(sq == NULL)) {
+		/* Invalid send queue */
+		m_freem(mb);
+		return (ENXIO);
+	}
+	if (mtx_trylock(&sq->lock)) {
+		ret = mlx5e_xmit_locked(ifp, sq, mb);
+		mtx_unlock(&sq->lock);
+	} else {
+		ret = drbr_enqueue(ifp, sq->br, mb);
+		taskqueue_enqueue(sq->sq_tq, &sq->sq_task);
+	}
+
+	return (ret);
+}
+
+void
+mlx5e_tx_cq_comp(struct mlx5_core_cq *mcq)
+{
+	struct mlx5e_sq *sq = container_of(mcq, struct mlx5e_sq, cq.mcq);
+
+	mtx_lock(&sq->comp_lock);
+	mlx5e_poll_tx_cq(sq, MLX5E_BUDGET_MAX);
+	mlx5e_cq_arm(&sq->cq);
+	mtx_unlock(&sq->comp_lock);
+}
+
+void
+mlx5e_tx_que(void *context, int pending)
+{
+	struct mlx5e_sq *sq = context;
+	struct ifnet *ifp = sq->channel->ifp;
+
+	if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
+		mtx_lock(&sq->lock);
+		if (!drbr_empty(ifp, sq->br))
+			mlx5e_xmit_locked(ifp, sq, NULL);
+		mtx_unlock(&sq->lock);
+	}
+}
diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_txrx.c b/sys/dev/mlx5/mlx5_en/mlx5_en_txrx.c
new file mode 100644
index 0000000..7836bfe
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_en/mlx5_en_txrx.c
@@ -0,0 +1,58 @@
+/*-
+ * Copyright (c) 2015 Mellanox Technologies. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include "en.h"
+
+struct mlx5_cqe64 *
+mlx5e_get_cqe(struct mlx5e_cq *cq)
+{
+	struct mlx5_cqe64 *cqe;
+
+	cqe = mlx5_cqwq_get_wqe(&cq->wq, mlx5_cqwq_get_ci(&cq->wq));
+
+	if ((cqe->op_own ^ mlx5_cqwq_get_wrap_cnt(&cq->wq)) & MLX5_CQE_OWNER_MASK)
+		return (NULL);
+
+	mlx5_cqwq_pop(&cq->wq);
+
+	/* ensure cqe content is read after cqe ownership bit */
+	rmb();
+
+	return (cqe);
+}
+
+void
+mlx5e_cq_error_event(struct mlx5_core_cq *mcq, int event)
+{
+	struct mlx5e_cq *cq = container_of(mcq, struct mlx5e_cq, mcq);
+	struct mlx5e_channel *c = cq->channel;
+	struct mlx5e_priv *priv = c->priv;
+	struct ifnet *ifp = priv->ifp;
+
+	if_printf(ifp, "%s: cqn=0x%.6x event=0x%.2x\n",
+	    __func__, mcq->cqn, event);
+}
diff --git a/sys/dev/mlx5/mlx5_en/tcp_tlro.c b/sys/dev/mlx5/mlx5_en/tcp_tlro.c
new file mode 100644
index 0000000..4b81904
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_en/tcp_tlro.c
@@ -0,0 +1,697 @@
+/*-
+ * Copyright (c) 2015 Mellanox Technologies. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include "opt_inet.h"
+#include "opt_inet6.h"
+
+#include <sys/param.h>
+#include <sys/libkern.h>
+#include <sys/mbuf.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+#include <sys/sysctl.h>
+#include <sys/malloc.h>
+#include <sys/kernel.h>
+#include <sys/endian.h>
+#include <sys/socket.h>
+#include <sys/sockopt.h>
+#include <sys/smp.h>
+
+#include <net/if.h>
+#include <net/if_var.h>
+#include <net/ethernet.h>
+
+#if defined(INET) || defined(INET6)
+#include <netinet/in.h>
+#endif
+
+#ifdef INET
+#include <netinet/ip.h>
+#endif
+
+#ifdef INET6
+#include <netinet/ip6.h>
+#endif
+
+#include <netinet/tcp_var.h>
+
+#include "tcp_tlro.h"
+
+#ifndef M_HASHTYPE_LRO_TCP
+#ifndef KLD_MODULE
+#warning "M_HASHTYPE_LRO_TCP is not defined"
+#endif
+#define	M_HASHTYPE_LRO_TCP 254
+#endif
+
+static SYSCTL_NODE(_net_inet_tcp, OID_AUTO, tlro,
+    CTLFLAG_RW, 0, "TCP turbo LRO parameters");
+
+static MALLOC_DEFINE(M_TLRO, "TLRO", "Turbo LRO");
+
+static int tlro_min_rate = 20;		/* Hz */
+
+SYSCTL_INT(_net_inet_tcp_tlro, OID_AUTO, min_rate, CTLFLAG_RWTUN,
+    &tlro_min_rate, 0, "Minimum serving rate in Hz");
+
+static int tlro_max_packet = IP_MAXPACKET;
+
+SYSCTL_INT(_net_inet_tcp_tlro, OID_AUTO, max_packet, CTLFLAG_RWTUN,
+    &tlro_max_packet, 0, "Maximum packet size in bytes");
+
+typedef struct {
+	uint32_t value;
+} __packed uint32_p_t;
+
+static uint16_t
+tcp_tlro_csum(const uint32_p_t *p, size_t l)
+{
+	const uint32_p_t *pend = p + (l / 4);
+	uint64_t cs;
+
+	for (cs = 0; p != pend; p++)
+		cs += le32toh(p->value);
+	while (cs > 0xffff)
+		cs = (cs >> 16) + (cs & 0xffff);
+	return (cs);
+}
+
+static void *
+tcp_tlro_get_header(const struct mbuf *m, const u_int off,
+    const u_int len)
+{
+	if (m->m_len < (off + len))
+		return (NULL);
+	return (mtod(m, char *) + off);
+}
+
+static uint8_t
+tcp_tlro_info_save_timestamp(struct tlro_mbuf_data *pinfo)
+{
+	struct tcphdr *tcp = pinfo->tcp;
+	uint32_t *ts_ptr;
+
+	if (tcp->th_off < ((TCPOLEN_TSTAMP_APPA + sizeof(*tcp)) >> 2))
+		return (0);
+
+	ts_ptr = (uint32_t *)(tcp + 1);
+	if (*ts_ptr != ntohl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) |
+	    (TCPOPT_TIMESTAMP << 8) | TCPOLEN_TIMESTAMP))
+		return (0);
+
+	/* Save timestamps */
+	pinfo->tcp_ts = ts_ptr[1];
+	pinfo->tcp_ts_reply = ts_ptr[2];
+	return (1);
+}
+
+static void
+tcp_tlro_info_restore_timestamp(struct tlro_mbuf_data *pinfoa,
+    struct tlro_mbuf_data *pinfob)
+{
+	struct tcphdr *tcp = pinfoa->tcp;
+	uint32_t *ts_ptr;
+
+	if (tcp->th_off < ((TCPOLEN_TSTAMP_APPA + sizeof(*tcp)) >> 2))
+		return;
+
+	ts_ptr = (uint32_t *)(tcp + 1);
+	if (*ts_ptr != ntohl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) |
+	    (TCPOPT_TIMESTAMP << 8) | TCPOLEN_TIMESTAMP))
+		return;
+
+	/* Restore timestamps */
+	ts_ptr[1] = pinfob->tcp_ts;
+	ts_ptr[2] = pinfob->tcp_ts_reply;
+}
+
+static void
+tcp_tlro_extract_header(struct tlro_mbuf_data *pinfo, struct mbuf *m, int seq)
+{
+	uint8_t *phdr = (uint8_t *)pinfo->buf;
+	struct ether_header *eh;
+	struct ether_vlan_header *vlan;
+#ifdef INET
+	struct ip *ip;
+#endif
+#ifdef INET6
+	struct ip6_hdr *ip6;
+#endif
+	struct tcphdr *tcp;
+	uint16_t etype;
+	int diff;
+	int off;
+
+	/* Fill in information */
+	pinfo->head = m;
+	pinfo->last_tick = ticks;
+	pinfo->sequence = seq;
+	pinfo->pprev = &m_last(m)->m_next;
+
+	off = sizeof(*eh);
+	if (m->m_len < off)
+		goto error;
+	eh = tcp_tlro_get_header(m, 0, sizeof(*eh));
+	if (eh == NULL)
+		goto error;
+	memcpy(phdr, &eh->ether_dhost, ETHER_ADDR_LEN);
+	phdr += ETHER_ADDR_LEN;
+	memcpy(phdr, &eh->ether_type, sizeof(eh->ether_type));
+	phdr += sizeof(eh->ether_type);
+	etype = ntohs(eh->ether_type);
+
+	if (etype == ETHERTYPE_VLAN) {
+		vlan = tcp_tlro_get_header(m, off, sizeof(*vlan));
+		if (vlan == NULL)
+			goto error;
+		memcpy(phdr, &vlan->evl_tag, sizeof(vlan->evl_tag) +
+		    sizeof(vlan->evl_proto));
+		phdr += sizeof(vlan->evl_tag) + sizeof(vlan->evl_proto);
+		etype = ntohs(vlan->evl_proto);
+		off += sizeof(*vlan) - sizeof(*eh);
+	}
+	switch (etype) {
+#ifdef INET
+	case ETHERTYPE_IP:
+		/*
+		 * Cannot LRO:
+		 * - Non-IP packets
+		 * - Fragmented packets
+		 * - Packets with IPv4 options
+		 * - Non-TCP packets
+		 */
+		ip = tcp_tlro_get_header(m, off, sizeof(*ip));
+		if (ip == NULL ||
+		    (ip->ip_off & htons(IP_MF | IP_OFFMASK)) != 0 ||
+		    (ip->ip_p != IPPROTO_TCP) ||
+		    (ip->ip_hl << 2) != sizeof(*ip))
+			goto error;
+
+		/* Legacy IP has a header checksum that needs to be correct */
+		if (!(m->m_pkthdr.csum_flags & CSUM_IP_CHECKED)) {
+			/* Verify IP header */
+			if (tcp_tlro_csum((uint32_p_t *)ip, sizeof(*ip)) != 0xFFFF)
+				m->m_pkthdr.csum_flags |= CSUM_IP_CHECKED;
+			else
+				m->m_pkthdr.csum_flags |= CSUM_IP_CHECKED |
+				    CSUM_IP_VALID;
+		}
+		/* Only accept valid checksums */
+		if (!(m->m_pkthdr.csum_flags & CSUM_IP_VALID) ||
+		    !(m->m_pkthdr.csum_flags & CSUM_DATA_VALID))
+			goto error;
+		memcpy(phdr, &ip->ip_src, sizeof(ip->ip_src) +
+		    sizeof(ip->ip_dst));
+		phdr += sizeof(ip->ip_src) + sizeof(ip->ip_dst);
+		if (M_HASHTYPE_GET(m) == M_HASHTYPE_LRO_TCP)
+			pinfo->ip_len = m->m_pkthdr.len - off;
+		else
+			pinfo->ip_len = ntohs(ip->ip_len);
+		pinfo->ip_hdrlen = sizeof(*ip);
+		pinfo->ip.v4 = ip;
+		pinfo->ip_version = 4;
+		off += sizeof(*ip);
+		break;
+#endif
+#ifdef INET6
+	case ETHERTYPE_IPV6:
+		/*
+		 * Cannot LRO:
+		 * - Non-IP packets
+		 * - Packets with IPv6 options
+		 * - Non-TCP packets
+		 */
+		ip6 = tcp_tlro_get_header(m, off, sizeof(*ip6));
+		if (ip6 == NULL || ip6->ip6_nxt != IPPROTO_TCP)
+			goto error;
+		if (!(m->m_pkthdr.csum_flags & CSUM_DATA_VALID))
+			goto error;
+		memcpy(phdr, &ip6->ip6_src, sizeof(struct in6_addr) +
+		    sizeof(struct in6_addr));
+		phdr += sizeof(struct in6_addr) + sizeof(struct in6_addr);
+		if (M_HASHTYPE_GET(m) == M_HASHTYPE_LRO_TCP)
+			pinfo->ip_len = m->m_pkthdr.len - off;
+		else
+			pinfo->ip_len = ntohs(ip6->ip6_plen) + sizeof(*ip6);
+		pinfo->ip_hdrlen = sizeof(*ip6);
+		pinfo->ip.v6 = ip6;
+		pinfo->ip_version = 6;
+		off += sizeof(*ip6);
+		break;
+#endif
+	default:
+		goto error;
+	}
+	tcp = tcp_tlro_get_header(m, off, sizeof(*tcp));
+	if (tcp == NULL)
+		goto error;
+	memcpy(phdr, &tcp->th_sport, sizeof(tcp->th_sport) +
+	    sizeof(tcp->th_dport));
+	phdr += sizeof(tcp->th_sport) +
+	    sizeof(tcp->th_dport);
+	/* Store TCP header length */
+	*phdr++ = tcp->th_off;
+	if (tcp->th_off < (sizeof(*tcp) >> 2))
+		goto error;
+
+	/* Compute offset to data payload */
+	pinfo->tcp_len = (tcp->th_off << 2);
+	off += pinfo->tcp_len;
+
+	/* Store more info */
+	pinfo->data_off = off;
+	pinfo->tcp = tcp;
+
+	/* Try to save timestamp, if any */
+	*phdr++ = tcp_tlro_info_save_timestamp(pinfo);
+
+	/* Verify offset and IP/TCP length */
+	if (off > m->m_pkthdr.len ||
+	    pinfo->ip_len < pinfo->tcp_len)
+		goto error;
+
+	/* Compute data payload length */
+	pinfo->data_len = (pinfo->ip_len - pinfo->tcp_len - pinfo->ip_hdrlen);
+
+	/* Trim any padded data */
+	diff = (m->m_pkthdr.len - off) - pinfo->data_len;
+	if (diff != 0) {
+		if (diff < 0)
+			goto error;
+		else
+			m_adj(m, -diff);
+	}
+	/* Compute header length */
+	pinfo->buf_length = phdr - (uint8_t *)pinfo->buf;
+	/* Zero-pad rest of buffer */
+	memset(phdr, 0, TLRO_MAX_HEADER - pinfo->buf_length);
+	return;
+error:
+	pinfo->buf_length = 0;
+}
+
+static int
+tcp_tlro_cmp64(const uint64_t *pa, const uint64_t *pb)
+{
+	int64_t diff = 0;
+	unsigned x;
+
+	for (x = 0; x != TLRO_MAX_HEADER / 8; x++) {
+		/*
+		 * NOTE: Endianness does not matter in this
+		 * comparisation:
+		 */
+		diff = pa[x] - pb[x];
+		if (diff != 0)
+			goto done;
+	}
+done:
+	if (diff < 0)
+		return (-1);
+	else if (diff > 0)
+		return (1);
+	return (0);
+}
+
+static int
+tcp_tlro_compare_header(const void *_ppa, const void *_ppb)
+{
+	const struct tlro_mbuf_ptr *ppa = _ppa;
+	const struct tlro_mbuf_ptr *ppb = _ppb;
+	struct tlro_mbuf_data *pinfoa = ppa->data;
+	struct tlro_mbuf_data *pinfob = ppb->data;
+	int ret;
+
+	ret = (pinfoa->head == NULL) - (pinfob->head == NULL);
+	if (ret != 0)
+		goto done;
+
+	ret = pinfoa->buf_length - pinfob->buf_length;
+	if (ret != 0)
+		goto done;
+	if (pinfoa->buf_length != 0) {
+		ret = tcp_tlro_cmp64(pinfoa->buf, pinfob->buf);
+		if (ret != 0)
+			goto done;
+		ret = ntohl(pinfoa->tcp->th_seq) - ntohl(pinfob->tcp->th_seq);
+		if (ret != 0)
+			goto done;
+		ret = ntohl(pinfoa->tcp->th_ack) - ntohl(pinfob->tcp->th_ack);
+		if (ret != 0)
+			goto done;
+		ret = pinfoa->sequence - pinfob->sequence;
+		if (ret != 0)
+			goto done;
+	}
+done:
+	return (ret);
+}
+
+static void
+tcp_tlro_sort(struct tlro_ctrl *tlro)
+{
+	if (tlro->curr == 0)
+		return;
+
+	qsort(tlro->mbuf, tlro->curr, sizeof(struct tlro_mbuf_ptr),
+	    &tcp_tlro_compare_header);
+}
+
+static int
+tcp_tlro_get_ticks(void)
+{
+	int to = tlro_min_rate;
+
+	if (to < 1)
+		to = 1;
+	to = hz / to;
+	if (to < 1)
+		to = 1;
+	return (to);
+}
+
+static void
+tcp_tlro_combine(struct tlro_ctrl *tlro, int force)
+{
+	struct tlro_mbuf_data *pinfoa;
+	struct tlro_mbuf_data *pinfob;
+	uint32_t cs;
+	int curr_ticks = ticks;
+	int ticks_limit = tcp_tlro_get_ticks();
+	unsigned x;
+	unsigned y;
+	unsigned z;
+	int temp;
+
+	if (tlro->curr == 0)
+		return;
+
+	for (y = 0; y != tlro->curr;) {
+		struct mbuf *m;
+
+		pinfoa = tlro->mbuf[y].data;
+		for (x = y + 1; x != tlro->curr; x++) {
+			pinfob = tlro->mbuf[x].data;
+			if (pinfoa->buf_length != pinfob->buf_length ||
+			    tcp_tlro_cmp64(pinfoa->buf, pinfob->buf) != 0)
+				break;
+		}
+		if (pinfoa->buf_length == 0) {
+			/* Forward traffic which cannot be combined */
+			for (z = y; z != x; z++) {
+				/* Just forward packets */
+				pinfob = tlro->mbuf[z].data;
+
+				m = pinfob->head;
+
+				/* Reset info structure */
+				pinfob->head = NULL;
+				pinfob->buf_length = 0;
+
+				/* Do stats */
+				tlro->lro_flushed++;
+
+				/* Input packet to network layer */
+				(*tlro->ifp->if_input) (tlro->ifp, m);
+			}
+			y = z;
+			continue;
+		}
+
+		/* Compute current checksum subtracted some header parts */
+		temp = (pinfoa->ip_len - pinfoa->ip_hdrlen);
+		cs = ((temp & 0xFF) << 8) + ((temp & 0xFF00) >> 8) +
+		    tcp_tlro_csum((uint32_p_t *)pinfoa->tcp, pinfoa->tcp_len);
+
+		/* Append all fragments into one block */
+		for (z = y + 1; z != x; z++) {
+
+			pinfob = tlro->mbuf[z].data;
+
+			/* Check for command packets */
+			if ((pinfoa->tcp->th_flags & ~(TH_ACK | TH_PUSH)) ||
+			    (pinfob->tcp->th_flags & ~(TH_ACK | TH_PUSH)))
+				break;
+
+			/* Check if there is enough space */
+			if ((pinfoa->ip_len + pinfob->data_len) > tlro_max_packet)
+				break;
+
+			/* Try to append the new segment */
+			temp = ntohl(pinfoa->tcp->th_seq) + pinfoa->data_len;
+			if (temp != (int)ntohl(pinfob->tcp->th_seq))
+				break;
+
+			temp = pinfob->ip_len - pinfob->ip_hdrlen;
+			cs += ((temp & 0xFF) << 8) + ((temp & 0xFF00) >> 8) +
+			    tcp_tlro_csum((uint32_p_t *)pinfob->tcp, pinfob->tcp_len);
+			/* Remove fields which appear twice */
+			cs += (IPPROTO_TCP << 8);
+			if (pinfob->ip_version == 4) {
+				cs += tcp_tlro_csum((uint32_p_t *)&pinfob->ip.v4->ip_src, 4);
+				cs += tcp_tlro_csum((uint32_p_t *)&pinfob->ip.v4->ip_dst, 4);
+			} else {
+				cs += tcp_tlro_csum((uint32_p_t *)&pinfob->ip.v6->ip6_src, 16);
+				cs += tcp_tlro_csum((uint32_p_t *)&pinfob->ip.v6->ip6_dst, 16);
+			}
+			/* Remainder computation */
+			while (cs > 0xffff)
+				cs = (cs >> 16) + (cs & 0xffff);
+
+			/* Update window and ack sequence number */
+			pinfoa->tcp->th_ack = pinfob->tcp->th_ack;
+			pinfoa->tcp->th_win = pinfob->tcp->th_win;
+
+			/* Check if we should restore the timestamp */
+			tcp_tlro_info_restore_timestamp(pinfoa, pinfob);
+
+			/* Accumulate TCP flags */
+			pinfoa->tcp->th_flags |= pinfob->tcp->th_flags;
+
+			/* update lengths */
+			pinfoa->ip_len += pinfob->data_len;
+			pinfoa->data_len += pinfob->data_len;
+
+			/* Clear mbuf pointer - packet is accumulated */
+			m = pinfob->head;
+
+			/* Reset info structure */
+			pinfob->head = NULL;
+			pinfob->buf_length = 0;
+
+			/* Append data to mbuf [y] */
+			m_adj(m, pinfob->data_off);
+			/* Delete mbuf tags, if any */
+			m_tag_delete_chain(m, NULL);
+			/* Clear packet header flag */
+			m->m_flags &= ~M_PKTHDR;
+
+			/* Concat mbuf(s) to end of list */
+			pinfoa->pprev[0] = m;
+			m = m_last(m);
+			pinfoa->pprev = &m->m_next;
+			pinfoa->head->m_pkthdr.len += pinfob->data_len;
+		}
+		/* Compute new TCP header checksum */
+		pinfoa->tcp->th_sum = 0;
+
+		temp = pinfoa->ip_len - pinfoa->ip_hdrlen;
+		cs = (cs ^ 0xFFFF) +
+		    tcp_tlro_csum((uint32_p_t *)pinfoa->tcp, pinfoa->tcp_len) +
+		    ((temp & 0xFF) << 8) + ((temp & 0xFF00) >> 8);
+
+		/* Remainder computation */
+		while (cs > 0xffff)
+			cs = (cs >> 16) + (cs & 0xffff);
+
+		/* Update new checksum */
+		pinfoa->tcp->th_sum = ~htole16(cs);
+
+		/* Update IP length, if any */
+		if (pinfoa->ip_version == 4) {
+			if (pinfoa->ip_len > IP_MAXPACKET) {
+				M_HASHTYPE_SET(pinfoa->head, M_HASHTYPE_LRO_TCP);
+				pinfoa->ip.v4->ip_len = htons(IP_MAXPACKET);
+			} else {
+				pinfoa->ip.v4->ip_len = htons(pinfoa->ip_len);
+			}
+		} else {
+			if (pinfoa->ip_len > (IP_MAXPACKET + sizeof(*pinfoa->ip.v6))) {
+				M_HASHTYPE_SET(pinfoa->head, M_HASHTYPE_LRO_TCP);
+				pinfoa->ip.v6->ip6_plen = htons(IP_MAXPACKET);
+			} else {
+				temp = pinfoa->ip_len - sizeof(*pinfoa->ip.v6);
+				pinfoa->ip.v6->ip6_plen = htons(temp);
+			}
+		}
+
+		temp = curr_ticks - pinfoa->last_tick;
+		/* Check if packet should be forwarded */
+		if (force != 0 || z != x || temp >= ticks_limit ||
+		    pinfoa->data_len == 0) {
+
+			/* Compute new IPv4 header checksum */
+			if (pinfoa->ip_version == 4) {
+				pinfoa->ip.v4->ip_sum = 0;
+				cs = tcp_tlro_csum((uint32_p_t *)pinfoa->ip.v4,
+				    sizeof(*pinfoa->ip.v4));
+				pinfoa->ip.v4->ip_sum = ~htole16(cs);
+			}
+			/* Forward packet */
+			m = pinfoa->head;
+
+			/* Reset info structure */
+			pinfoa->head = NULL;
+			pinfoa->buf_length = 0;
+
+			/* Do stats */
+			tlro->lro_flushed++;
+
+			/* Input packet to network layer */
+			(*tlro->ifp->if_input) (tlro->ifp, m);
+		}
+		y = z;
+	}
+
+	/* Cleanup all NULL heads */
+	for (y = 0; y != tlro->curr; y++) {
+		if (tlro->mbuf[y].data->head == NULL) {
+			for (z = y + 1; z != tlro->curr; z++) {
+				struct tlro_mbuf_ptr ptemp;
+				if (tlro->mbuf[z].data->head == NULL)
+					continue;
+				ptemp = tlro->mbuf[y];
+				tlro->mbuf[y] = tlro->mbuf[z];
+				tlro->mbuf[z] = ptemp;
+				y++;
+			}
+			break;
+		}
+	}
+	tlro->curr = y;
+}
+
+static void
+tcp_tlro_cleanup(struct tlro_ctrl *tlro)
+{
+	while (tlro->curr != 0 &&
+	    tlro->mbuf[tlro->curr - 1].data->head == NULL)
+		tlro->curr--;
+}
+
+void
+tcp_tlro_flush(struct tlro_ctrl *tlro, int force)
+{
+	if (tlro->curr == 0)
+		return;
+
+	tcp_tlro_sort(tlro);
+	tcp_tlro_cleanup(tlro);
+	tcp_tlro_combine(tlro, force);
+}
+
+int
+tcp_tlro_init(struct tlro_ctrl *tlro, struct ifnet *ifp,
+    int max_mbufs)
+{
+	ssize_t size;
+	uint32_t x;
+
+	/* Set zero defaults */
+	memset(tlro, 0, sizeof(*tlro));
+
+	/* Compute size needed for data */
+	size = (sizeof(struct tlro_mbuf_ptr) * max_mbufs) +
+	    (sizeof(struct tlro_mbuf_data) * max_mbufs);
+
+	/* Range check */
+	if (max_mbufs <= 0 || size <= 0 || ifp == NULL)
+		return (EINVAL);
+
+	/* Setup tlro control structure */
+	tlro->mbuf = malloc(size, M_TLRO, M_WAITOK | M_ZERO);
+	tlro->max = max_mbufs;
+	tlro->ifp = ifp;
+
+	/* Setup pointer array */
+	for (x = 0; x != tlro->max; x++) {
+		tlro->mbuf[x].data = ((struct tlro_mbuf_data *)
+		    &tlro->mbuf[max_mbufs]) + x;
+	}
+	return (0);
+}
+
+void
+tcp_tlro_free(struct tlro_ctrl *tlro)
+{
+	struct tlro_mbuf_data *pinfo;
+	struct mbuf *m;
+	uint32_t y;
+
+	/* Check if not setup */
+	if (tlro->mbuf == NULL)
+		return;
+	/* Free MBUF array and any leftover MBUFs */
+	for (y = 0; y != tlro->max; y++) {
+
+		pinfo = tlro->mbuf[y].data;
+
+		m = pinfo->head;
+
+		/* Reset info structure */
+		pinfo->head = NULL;
+		pinfo->buf_length = 0;
+
+		m_freem(m);
+	}
+	free(tlro->mbuf, M_TLRO);
+	/* Reset buffer */
+	memset(tlro, 0, sizeof(*tlro));
+}
+
+void
+tcp_tlro_rx(struct tlro_ctrl *tlro, struct mbuf *m)
+{
+	if (m->m_len > 0 && tlro->curr < tlro->max) {
+		/* do stats */
+		tlro->lro_queued++;
+
+		/* extract header */
+		tcp_tlro_extract_header(tlro->mbuf[tlro->curr++].data,
+		    m, tlro->sequence++);
+	} else if (tlro->ifp != NULL) {
+		/* do stats */
+		tlro->lro_flushed++;
+
+		/* input packet to network layer */
+		(*tlro->ifp->if_input) (tlro->ifp, m);
+	} else {
+		/* packet drop */
+		m_freem(m);
+	}
+}
diff --git a/sys/dev/mlx5/mlx5_en/tcp_tlro.h b/sys/dev/mlx5/mlx5_en/tcp_tlro.h
new file mode 100644
index 0000000..40ebdf6
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_en/tcp_tlro.h
@@ -0,0 +1,83 @@
+/*-
+ * Copyright (c) 2015 Mellanox Technologies. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _TCP_TLRO_H_
+#define	_TCP_TLRO_H_
+
+#define	TLRO_MAX_HEADER	64		/* bytes */
+
+struct ip;
+struct ip6_hdr;
+struct tcphdr;
+
+struct tlro_mbuf_data {
+	union {
+#ifdef INET
+		struct ip *v4;
+#endif
+#ifdef INET6
+		struct ip6_hdr *v6;
+#endif
+	} ip;
+	struct tcphdr *tcp;
+	struct mbuf *head;
+	struct mbuf **pprev;
+	int	last_tick;
+	int	sequence;
+	int	data_len;
+	int	data_off;
+	int	ip_hdrlen;
+	int	ip_len;
+	uint32_t tcp_ts;
+	uint32_t tcp_ts_reply;
+	uint16_t tcp_len;
+	uint8_t	ip_version;
+	uint8_t	buf_length;		/* in 32-bit words */
+	uint64_t buf[TLRO_MAX_HEADER / 8];
+} __aligned(256);
+
+struct tlro_mbuf_ptr {
+	struct tlro_mbuf_data *data;
+};
+
+/* NB: This is part of driver structs */
+struct tlro_ctrl {
+	struct ifnet *ifp;
+	struct tlro_mbuf_ptr *mbuf;
+	uint64_t lro_queued;
+	uint64_t lro_flushed;
+	uint32_t max;
+	uint32_t curr;
+	int	sequence;
+};
+
+int	tcp_tlro_init(struct tlro_ctrl *, struct ifnet *, int);
+void	tcp_tlro_free(struct tlro_ctrl *);
+void	tcp_tlro_flush(struct tlro_ctrl *, int);
+void	tcp_tlro_rx(struct tlro_ctrl *, struct mbuf *);
+
+#endif					/* _TCP_TLRO_H_ */
diff --git a/sys/dev/mlx5/mlx5_ifc.h b/sys/dev/mlx5/mlx5_ifc.h
new file mode 100644
index 0000000..461e5b0
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_ifc.h
@@ -0,0 +1,8491 @@
+/*-
+ * Copyright (c) 2013-2015, Mellanox Technologies, Ltd.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+
+   Autogenerated file.
+   Date: 2015-04-13 14:59
+   Source Document Name: Mellanox <Doc Name>
+   Source Document Version: 0.28
+   Generated by adb_to_c.py (EAT.ME Version: 1.0.70)
+*/
+#ifndef MLX5_IFC_H
+#define MLX5_IFC_H
+
+enum {
+	MLX5_EVENT_TYPE_COMP                                       = 0x0,
+	MLX5_EVENT_TYPE_PATH_MIG                                   = 0x1,
+	MLX5_EVENT_TYPE_COMM_EST                                   = 0x2,
+	MLX5_EVENT_TYPE_SQ_DRAINED                                 = 0x3,
+	MLX5_EVENT_TYPE_SRQ_LAST_WQE                               = 0x13,
+	MLX5_EVENT_TYPE_SRQ_RQ_LIMIT                               = 0x14,
+	MLX5_EVENT_TYPE_DCT_DRAINED                                = 0x1c,
+	MLX5_EVENT_TYPE_DCT_KEY_VIOLATION                          = 0x1d,
+	MLX5_EVENT_TYPE_CQ_ERROR                                   = 0x4,
+	MLX5_EVENT_TYPE_WQ_CATAS_ERROR                             = 0x5,
+	MLX5_EVENT_TYPE_PATH_MIG_FAILED                            = 0x7,
+	MLX5_EVENT_TYPE_PAGE_FAULT                                 = 0xc,
+	MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR                         = 0x10,
+	MLX5_EVENT_TYPE_WQ_ACCESS_ERROR                            = 0x11,
+	MLX5_EVENT_TYPE_SRQ_CATAS_ERROR                            = 0x12,
+	MLX5_EVENT_TYPE_INTERNAL_ERROR                             = 0x8,
+	MLX5_EVENT_TYPE_PORT_CHANGE                                = 0x9,
+	MLX5_EVENT_TYPE_GPIO_EVENT                                 = 0x15,
+	MLX5_EVENT_TYPE_CODING_PORT_MODULE_EVENT                   = 0x16,
+	MLX5_EVENT_TYPE_REMOTE_CONFIG                              = 0x19,
+	MLX5_EVENT_TYPE_DB_BF_CONGESTION                           = 0x1a,
+	MLX5_EVENT_TYPE_STALL_EVENT                                = 0x1b,
+	MLX5_EVENT_TYPE_DROPPED_PACKET_LOGGED_EVENT                = 0x1f,
+	MLX5_EVENT_TYPE_CMD                                        = 0xa,
+	MLX5_EVENT_TYPE_PAGE_REQUEST                               = 0xb,
+	MLX5_EVENT_TYPE_NIC_VPORT_CHANGE                           = 0xd
+};
+
+enum {
+	MLX5_MODIFY_TIR_BITMASK_LRO                                                                                                         = 0x0,
+	MLX5_MODIFY_TIR_BITMASK_INDIRECT_TABLE                                                                                              = 0x1,
+	MLX5_MODIFY_TIR_BITMASK_HASH                                                                                                        = 0x2,
+	MLX5_MODIFY_TIR_BITMASK_TUNNELED_OFFLOAD_EN                                                                                         = 0x3
+};
+
+enum {
+	MLX5_MODIFY_RQT_BITMASK_RQN_LIST          = 0x1,
+};
+
+enum {
+	MLX5_CMD_OP_QUERY_HCA_CAP                 = 0x100,
+	MLX5_CMD_OP_QUERY_ADAPTER                 = 0x101,
+	MLX5_CMD_OP_INIT_HCA                      = 0x102,
+	MLX5_CMD_OP_TEARDOWN_HCA                  = 0x103,
+	MLX5_CMD_OP_ENABLE_HCA                    = 0x104,
+	MLX5_CMD_OP_DISABLE_HCA                   = 0x105,
+	MLX5_CMD_OP_QUERY_PAGES                   = 0x107,
+	MLX5_CMD_OP_MANAGE_PAGES                  = 0x108,
+	MLX5_CMD_OP_SET_HCA_CAP                   = 0x109,
+	MLX5_CMD_OP_QUERY_ISSI                    = 0x10a,
+	MLX5_CMD_OP_SET_ISSI                      = 0x10b,
+	MLX5_CMD_OP_SET_DRIVER_VERSION            = 0x10d,
+	MLX5_CMD_OP_CREATE_MKEY                   = 0x200,
+	MLX5_CMD_OP_QUERY_MKEY                    = 0x201,
+	MLX5_CMD_OP_DESTROY_MKEY                  = 0x202,
+	MLX5_CMD_OP_QUERY_SPECIAL_CONTEXTS        = 0x203,
+	MLX5_CMD_OP_PAGE_FAULT_RESUME             = 0x204,
+	MLX5_CMD_OP_CREATE_EQ                     = 0x301,
+	MLX5_CMD_OP_DESTROY_EQ                    = 0x302,
+	MLX5_CMD_OP_QUERY_EQ                      = 0x303,
+	MLX5_CMD_OP_GEN_EQE                       = 0x304,
+	MLX5_CMD_OP_CREATE_CQ                     = 0x400,
+	MLX5_CMD_OP_DESTROY_CQ                    = 0x401,
+	MLX5_CMD_OP_QUERY_CQ                      = 0x402,
+	MLX5_CMD_OP_MODIFY_CQ                     = 0x403,
+	MLX5_CMD_OP_CREATE_QP                     = 0x500,
+	MLX5_CMD_OP_DESTROY_QP                    = 0x501,
+	MLX5_CMD_OP_RST2INIT_QP                   = 0x502,
+	MLX5_CMD_OP_INIT2RTR_QP                   = 0x503,
+	MLX5_CMD_OP_RTR2RTS_QP                    = 0x504,
+	MLX5_CMD_OP_RTS2RTS_QP                    = 0x505,
+	MLX5_CMD_OP_SQERR2RTS_QP                  = 0x506,
+	MLX5_CMD_OP_2ERR_QP                       = 0x507,
+	MLX5_CMD_OP_2RST_QP                       = 0x50a,
+	MLX5_CMD_OP_QUERY_QP                      = 0x50b,
+	MLX5_CMD_OP_SQD_RTS_QP                    = 0x50c,
+	MLX5_CMD_OP_INIT2INIT_QP                  = 0x50e,
+	MLX5_CMD_OP_CREATE_PSV                    = 0x600,
+	MLX5_CMD_OP_DESTROY_PSV                   = 0x601,
+	MLX5_CMD_OP_CREATE_SRQ                    = 0x700,
+	MLX5_CMD_OP_DESTROY_SRQ                   = 0x701,
+	MLX5_CMD_OP_QUERY_SRQ                     = 0x702,
+	MLX5_CMD_OP_ARM_RQ                        = 0x703,
+	MLX5_CMD_OP_CREATE_XRC_SRQ                = 0x705,
+	MLX5_CMD_OP_DESTROY_XRC_SRQ               = 0x706,
+	MLX5_CMD_OP_QUERY_XRC_SRQ                 = 0x707,
+	MLX5_CMD_OP_ARM_XRC_SRQ                   = 0x708,
+	MLX5_CMD_OP_CREATE_DCT                    = 0x710,
+	MLX5_CMD_OP_DESTROY_DCT                   = 0x711,
+	MLX5_CMD_OP_DRAIN_DCT                     = 0x712,
+	MLX5_CMD_OP_QUERY_DCT                     = 0x713,
+	MLX5_CMD_OP_ARM_DCT_FOR_KEY_VIOLATION     = 0x714,
+	MLX5_CMD_OP_SET_DC_CNAK_TRACE             = 0x715,
+	MLX5_CMD_OP_QUERY_DC_CNAK_TRACE           = 0x716,
+	MLX5_CMD_OP_QUERY_VPORT_STATE             = 0x750,
+	MLX5_CMD_OP_MODIFY_VPORT_STATE            = 0x751,
+	MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT       = 0x752,
+	MLX5_CMD_OP_MODIFY_ESW_VPORT_CONTEXT      = 0x753,
+	MLX5_CMD_OP_QUERY_NIC_VPORT_CONTEXT       = 0x754,
+	MLX5_CMD_OP_MODIFY_NIC_VPORT_CONTEXT      = 0x755,
+	MLX5_CMD_OP_QUERY_ROCE_ADDRESS            = 0x760,
+	MLX5_CMD_OP_SET_ROCE_ADDRESS              = 0x761,
+	MLX5_CMD_OP_QUERY_HCA_VPORT_CONTEXT       = 0x762,
+	MLX5_CMD_OP_MODIFY_HCA_VPORT_CONTEXT      = 0x763,
+	MLX5_CMD_OP_QUERY_HCA_VPORT_GID           = 0x764,
+	MLX5_CMD_OP_QUERY_HCA_VPORT_PKEY          = 0x765,
+	MLX5_CMD_OP_QUERY_VPORT_COUNTER           = 0x770,
+	MLX5_CMD_OP_ALLOC_Q_COUNTER               = 0x771,
+	MLX5_CMD_OP_DEALLOC_Q_COUNTER             = 0x772,
+	MLX5_CMD_OP_QUERY_Q_COUNTER               = 0x773,
+	MLX5_CMD_OP_ALLOC_PD                      = 0x800,
+	MLX5_CMD_OP_DEALLOC_PD                    = 0x801,
+	MLX5_CMD_OP_ALLOC_UAR                     = 0x802,
+	MLX5_CMD_OP_DEALLOC_UAR                   = 0x803,
+	MLX5_CMD_OP_CONFIG_INT_MODERATION         = 0x804,
+	MLX5_CMD_OP_ACCESS_REG                    = 0x805,
+	MLX5_CMD_OP_ATTACH_TO_MCG                 = 0x806,
+	MLX5_CMD_OP_DETACH_FROM_MCG               = 0x807,
+	MLX5_CMD_OP_GET_DROPPED_PACKET_LOG        = 0x80a,
+	MLX5_CMD_OP_MAD_IFC                       = 0x50d,
+	MLX5_CMD_OP_QUERY_MAD_DEMUX               = 0x80b,
+	MLX5_CMD_OP_SET_MAD_DEMUX                 = 0x80c,
+	MLX5_CMD_OP_NOP                           = 0x80d,
+	MLX5_CMD_OP_ALLOC_XRCD                    = 0x80e,
+	MLX5_CMD_OP_DEALLOC_XRCD                  = 0x80f,
+	MLX5_CMD_OP_SET_BURST_SIZE                = 0x812,
+	MLX5_CMD_OP_QUERY_BURST_SIZE              = 0x813,
+	MLX5_CMD_OP_ACTIVATE_TRACER               = 0x814,
+	MLX5_CMD_OP_DEACTIVATE_TRACER             = 0x815,
+	MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN        = 0x816,
+	MLX5_CMD_OP_DEALLOC_TRANSPORT_DOMAIN      = 0x817,
+	MLX5_CMD_OP_QUERY_CONG_STATUS             = 0x822,
+	MLX5_CMD_OP_MODIFY_CONG_STATUS            = 0x823,
+	MLX5_CMD_OP_QUERY_CONG_PARAMS             = 0x824,
+	MLX5_CMD_OP_MODIFY_CONG_PARAMS            = 0x825,
+	MLX5_CMD_OP_QUERY_CONG_STATISTICS         = 0x826,
+	MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT           = 0x827,
+	MLX5_CMD_OP_DELETE_VXLAN_UDP_DPORT        = 0x828,
+	MLX5_CMD_OP_SET_L2_TABLE_ENTRY            = 0x829,
+	MLX5_CMD_OP_QUERY_L2_TABLE_ENTRY          = 0x82a,
+	MLX5_CMD_OP_DELETE_L2_TABLE_ENTRY         = 0x82b,
+	MLX5_CMD_OP_SET_WOL_ROL                   = 0x830,
+	MLX5_CMD_OP_QUERY_WOL_ROL                 = 0x831,
+	MLX5_CMD_OP_CREATE_TIR                    = 0x900,
+	MLX5_CMD_OP_MODIFY_TIR                    = 0x901,
+	MLX5_CMD_OP_DESTROY_TIR                   = 0x902,
+	MLX5_CMD_OP_QUERY_TIR                     = 0x903,
+	MLX5_CMD_OP_CREATE_SQ                     = 0x904,
+	MLX5_CMD_OP_MODIFY_SQ                     = 0x905,
+	MLX5_CMD_OP_DESTROY_SQ                    = 0x906,
+	MLX5_CMD_OP_QUERY_SQ                      = 0x907,
+	MLX5_CMD_OP_CREATE_RQ                     = 0x908,
+	MLX5_CMD_OP_MODIFY_RQ                     = 0x909,
+	MLX5_CMD_OP_DESTROY_RQ                    = 0x90a,
+	MLX5_CMD_OP_QUERY_RQ                      = 0x90b,
+	MLX5_CMD_OP_CREATE_RMP                    = 0x90c,
+	MLX5_CMD_OP_MODIFY_RMP                    = 0x90d,
+	MLX5_CMD_OP_DESTROY_RMP                   = 0x90e,
+	MLX5_CMD_OP_QUERY_RMP                     = 0x90f,
+	MLX5_CMD_OP_CREATE_TIS                    = 0x912,
+	MLX5_CMD_OP_MODIFY_TIS                    = 0x913,
+	MLX5_CMD_OP_DESTROY_TIS                   = 0x914,
+	MLX5_CMD_OP_QUERY_TIS                     = 0x915,
+	MLX5_CMD_OP_CREATE_RQT                    = 0x916,
+	MLX5_CMD_OP_MODIFY_RQT                    = 0x917,
+	MLX5_CMD_OP_DESTROY_RQT                   = 0x918,
+	MLX5_CMD_OP_QUERY_RQT                     = 0x919,
+	MLX5_CMD_OP_SET_FLOW_TABLE_ROOT           = 0x92f,
+	MLX5_CMD_OP_CREATE_FLOW_TABLE             = 0x930,
+	MLX5_CMD_OP_DESTROY_FLOW_TABLE            = 0x931,
+	MLX5_CMD_OP_QUERY_FLOW_TABLE              = 0x932,
+	MLX5_CMD_OP_CREATE_FLOW_GROUP             = 0x933,
+	MLX5_CMD_OP_DESTROY_FLOW_GROUP            = 0x934,
+	MLX5_CMD_OP_QUERY_FLOW_GROUP              = 0x935,
+	MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY          = 0x936,
+	MLX5_CMD_OP_QUERY_FLOW_TABLE_ENTRY        = 0x937,
+	MLX5_CMD_OP_DELETE_FLOW_TABLE_ENTRY       = 0x938,
+	MLX5_CMD_OP_ALLOC_FLOW_COUNTER            = 0x939,
+	MLX5_CMD_OP_DEALLOC_FLOW_COUNTER          = 0x93a,
+	MLX5_CMD_OP_QUERY_FLOW_COUNTER            = 0x93b
+};
+
+enum {
+	MLX5_ICMD_CMDS_OPCODE_ICMD_OPCODE_QUERY_FW_INFO     = 0x8007,
+	MLX5_ICMD_CMDS_OPCODE_ICMD_QUERY_CAPABILITY         = 0x8400,
+	MLX5_ICMD_CMDS_OPCODE_ICMD_ACCESS_REGISTER          = 0x9001,
+	MLX5_ICMD_CMDS_OPCODE_ICMD_QUERY_VIRTUAL_MAC        = 0x9003,
+	MLX5_ICMD_CMDS_OPCODE_ICMD_SET_VIRTUAL_MAC          = 0x9004,
+	MLX5_ICMD_CMDS_OPCODE_ICMD_QUERY_WOL_ROL            = 0x9005,
+	MLX5_ICMD_CMDS_OPCODE_ICMD_SET_WOL_ROL              = 0x9006,
+	MLX5_ICMD_CMDS_OPCODE_ICMD_OCBB_INIT                = 0x9007,
+	MLX5_ICMD_CMDS_OPCODE_ICMD_OCBB_QUERY_HEADER_STATUS = 0x9008,
+	MLX5_ICMD_CMDS_OPCODE_ICMD_OCBB_QUERY_ETOC_STATUS   = 0x9009,
+	MLX5_ICMD_CMDS_OPCODE_ICMD_OCBB_SET_EVENT           = 0x900a,
+	MLX5_ICMD_CMDS_OPCODE_ICMD_OPCODE_INIT_OCSD         = 0xf004
+};
+
+struct mlx5_ifc_flow_table_fields_supported_bits {
+	u8         outer_dmac[0x1];
+	u8         outer_smac[0x1];
+	u8         outer_ether_type[0x1];
+	u8         reserved_0[0x1];
+	u8         outer_first_prio[0x1];
+	u8         outer_first_cfi[0x1];
+	u8         outer_first_vid[0x1];
+	u8         reserved_1[0x1];
+	u8         outer_second_prio[0x1];
+	u8         outer_second_cfi[0x1];
+	u8         outer_second_vid[0x1];
+	u8         outer_ipv6_flow_label[0x1];
+	u8         outer_sip[0x1];
+	u8         outer_dip[0x1];
+	u8         outer_frag[0x1];
+	u8         outer_ip_protocol[0x1];
+	u8         outer_ip_ecn[0x1];
+	u8         outer_ip_dscp[0x1];
+	u8         outer_udp_sport[0x1];
+	u8         outer_udp_dport[0x1];
+	u8         outer_tcp_sport[0x1];
+	u8         outer_tcp_dport[0x1];
+	u8         outer_tcp_flags[0x1];
+	u8         outer_gre_protocol[0x1];
+	u8         outer_gre_key[0x1];
+	u8         outer_vxlan_vni[0x1];
+	u8         reserved_2[0x5];
+	u8         source_eswitch_port[0x1];
+
+	u8         inner_dmac[0x1];
+	u8         inner_smac[0x1];
+	u8         inner_ether_type[0x1];
+	u8         reserved_3[0x1];
+	u8         inner_first_prio[0x1];
+	u8         inner_first_cfi[0x1];
+	u8         inner_first_vid[0x1];
+	u8         reserved_4[0x1];
+	u8         inner_second_prio[0x1];
+	u8         inner_second_cfi[0x1];
+	u8         inner_second_vid[0x1];
+	u8         inner_ipv6_flow_label[0x1];
+	u8         inner_sip[0x1];
+	u8         inner_dip[0x1];
+	u8         inner_frag[0x1];
+	u8         inner_ip_protocol[0x1];
+	u8         inner_ip_ecn[0x1];
+	u8         inner_ip_dscp[0x1];
+	u8         inner_udp_sport[0x1];
+	u8         inner_udp_dport[0x1];
+	u8         inner_tcp_sport[0x1];
+	u8         inner_tcp_dport[0x1];
+	u8         inner_tcp_flags[0x1];
+	u8         reserved_5[0x9];
+
+	u8         reserved_6[0x1f];
+	u8         source_sqn[0x1];
+
+	u8         reserved_7[0x20];
+};
+
+struct mlx5_ifc_flow_table_prop_layout_bits {
+	u8         ft_support[0x1];
+	u8         flow_tag[0x1];
+	u8         flow_counter[0x1];
+	u8         flow_modify_en[0x1];
+	u8         modify_root[0x1];
+	u8         reserved_0[0x1b];
+
+	u8         reserved_1[0x2];
+	u8         log_max_ft_size[0x6];
+	u8         reserved_2[0x10];
+	u8         max_ft_level[0x8];
+
+	u8         reserved_3[0x20];
+
+	u8         reserved_4[0x18];
+	u8         log_max_ft_num[0x8];
+
+	u8         reserved_5[0x10];
+	u8         log_max_flow_counter[0x8];
+	u8         log_max_destination[0x8];
+
+	u8         reserved_6[0x18];
+	u8         log_max_flow[0x8];
+
+	u8         reserved_7[0x40];
+
+	struct mlx5_ifc_flow_table_fields_supported_bits ft_field_support;
+
+	struct mlx5_ifc_flow_table_fields_supported_bits ft_field_bitmask_support;
+};
+
+struct mlx5_ifc_odp_per_transport_service_cap_bits {
+	u8         send[0x1];
+	u8         receive[0x1];
+	u8         write[0x1];
+	u8         read[0x1];
+	u8         atomic[0x1];
+	u8         srq_receive[0x1];
+	u8         reserved_0[0x1a];
+};
+
+struct mlx5_ifc_flow_counter_list_bits {
+	u8         reserved_0[0x10];
+	u8         flow_counter_id[0x10];
+
+	u8         reserved_1[0x20];
+};
+
+enum {
+	MLX5_FLOW_CONTEXT_DEST_TYPE_VPORT                    = 0x0,
+	MLX5_FLOW_CONTEXT_DEST_TYPE_FLOW_TABLE               = 0x1,
+	MLX5_FLOW_CONTEXT_DEST_TYPE_TIR                      = 0x2,
+};
+
+struct mlx5_ifc_dest_format_struct_bits {
+	u8         destination_type[0x8];
+	u8         destination_id[0x18];
+
+	u8         reserved_0[0x20];
+};
+
+struct mlx5_ifc_fte_match_set_lyr_2_4_bits {
+	u8         smac_47_16[0x20];
+
+	u8         smac_15_0[0x10];
+	u8         ethertype[0x10];
+
+	u8         dmac_47_16[0x20];
+
+	u8         dmac_15_0[0x10];
+	u8         first_prio[0x3];
+	u8         first_cfi[0x1];
+	u8         first_vid[0xc];
+
+	u8         ip_protocol[0x8];
+	u8         ip_dscp[0x6];
+	u8         ip_ecn[0x2];
+	u8         vlan_tag[0x1];
+	u8         reserved_0[0x1];
+	u8         frag[0x1];
+	u8         reserved_1[0x4];
+	u8         tcp_flags[0x9];
+
+	u8         tcp_sport[0x10];
+	u8         tcp_dport[0x10];
+
+	u8         reserved_2[0x20];
+
+	u8         udp_sport[0x10];
+	u8         udp_dport[0x10];
+
+	u8         src_ip[4][0x20];
+
+	u8         dst_ip[4][0x20];
+};
+
+struct mlx5_ifc_fte_match_set_misc_bits {
+	u8         reserved_0[0x8];
+	u8         source_sqn[0x18];
+
+	u8         reserved_1[0x10];
+	u8         source_port[0x10];
+
+	u8         outer_second_prio[0x3];
+	u8         outer_second_cfi[0x1];
+	u8         outer_second_vid[0xc];
+	u8         inner_second_prio[0x3];
+	u8         inner_second_cfi[0x1];
+	u8         inner_second_vid[0xc];
+
+	u8         outer_second_vlan_tag[0x1];
+	u8         inner_second_vlan_tag[0x1];
+	u8         reserved_2[0xe];
+	u8         gre_protocol[0x10];
+
+	u8         gre_key_h[0x18];
+	u8         gre_key_l[0x8];
+
+	u8         vxlan_vni[0x18];
+	u8         reserved_3[0x8];
+
+	u8         reserved_4[0x20];
+
+	u8         reserved_5[0xc];
+	u8         outer_ipv6_flow_label[0x14];
+
+	u8         reserved_6[0xc];
+	u8         inner_ipv6_flow_label[0x14];
+
+	u8         reserved_7[0xe0];
+};
+
+struct mlx5_ifc_cmd_pas_bits {
+	u8         pa_h[0x20];
+
+	u8         pa_l[0x14];
+	u8         reserved_0[0xc];
+};
+
+struct mlx5_ifc_uint64_bits {
+	u8         hi[0x20];
+
+	u8         lo[0x20];
+};
+
+struct mlx5_ifc_nodnic_ring_doorbell_bits {
+	u8         reserved_0[0x8];
+	u8         ring_pi[0x10];
+	u8         reserved_1[0x8];
+};
+
+enum {
+	MLX5_ADS_STAT_RATE_NO_LIMIT  = 0x0,
+	MLX5_ADS_STAT_RATE_2_5GBPS   = 0x7,
+	MLX5_ADS_STAT_RATE_10GBPS    = 0x8,
+	MLX5_ADS_STAT_RATE_30GBPS    = 0x9,
+	MLX5_ADS_STAT_RATE_5GBPS     = 0xa,
+	MLX5_ADS_STAT_RATE_20GBPS    = 0xb,
+	MLX5_ADS_STAT_RATE_40GBPS    = 0xc,
+	MLX5_ADS_STAT_RATE_60GBPS    = 0xd,
+	MLX5_ADS_STAT_RATE_80GBPS    = 0xe,
+	MLX5_ADS_STAT_RATE_120GBPS   = 0xf,
+};
+
+struct mlx5_ifc_ads_bits {
+	u8         fl[0x1];
+	u8         free_ar[0x1];
+	u8         reserved_0[0xe];
+	u8         pkey_index[0x10];
+
+	u8         reserved_1[0x8];
+	u8         grh[0x1];
+	u8         mlid[0x7];
+	u8         rlid[0x10];
+
+	u8         ack_timeout[0x5];
+	u8         reserved_2[0x3];
+	u8         src_addr_index[0x8];
+	u8         log_rtm[0x4];
+	u8         stat_rate[0x4];
+	u8         hop_limit[0x8];
+
+	u8         reserved_3[0x4];
+	u8         tclass[0x8];
+	u8         flow_label[0x14];
+
+	u8         rgid_rip[16][0x8];
+
+	u8         reserved_4[0x4];
+	u8         f_dscp[0x1];
+	u8         f_ecn[0x1];
+	u8         reserved_5[0x1];
+	u8         f_eth_prio[0x1];
+	u8         ecn[0x2];
+	u8         dscp[0x6];
+	u8         udp_sport[0x10];
+
+	u8         dei_cfi[0x1];
+	u8         eth_prio[0x3];
+	u8         sl[0x4];
+	u8         port[0x8];
+	u8         rmac_47_32[0x10];
+
+	u8         rmac_31_0[0x20];
+};
+
+struct mlx5_ifc_e_switch_cap_bits {
+	u8         vport_svlan_strip[0x1];
+	u8         vport_cvlan_strip[0x1];
+	u8         vport_svlan_insert[0x1];
+	u8         vport_cvlan_insert_if_not_exist[0x1];
+	u8         vport_cvlan_insert_overwrite[0x1];
+	u8         reserved_0[0x1b];
+
+	u8         reserved_1[0x7e0];
+};
+
+struct mlx5_ifc_flow_table_eswitch_cap_bits {
+	u8         reserved_0[0x200];
+
+	struct mlx5_ifc_flow_table_prop_layout_bits flow_table_properties_nic_esw_fdb;
+
+	struct mlx5_ifc_flow_table_prop_layout_bits flow_table_properties_esw_acl_ingress;
+
+	struct mlx5_ifc_flow_table_prop_layout_bits flow_table_properties_esw_acl_egress;
+
+	u8         reserved_1[0x7800];
+};
+
+struct mlx5_ifc_flow_table_nic_cap_bits {
+	u8         reserved_0[0x200];
+
+	struct mlx5_ifc_flow_table_prop_layout_bits flow_table_properties_nic_receive;
+
+	struct mlx5_ifc_flow_table_prop_layout_bits flow_table_properties_nic_receive_rdma;
+
+	struct mlx5_ifc_flow_table_prop_layout_bits flow_table_properties_nic_receive_sniffer;
+
+	struct mlx5_ifc_flow_table_prop_layout_bits flow_table_properties_nic_transmit;
+
+	struct mlx5_ifc_flow_table_prop_layout_bits flow_table_properties_nic_transmit_rdma;
+
+	struct mlx5_ifc_flow_table_prop_layout_bits flow_table_properties_nic_transmit_sniffer;
+
+	u8         reserved_1[0x7200];
+};
+
+struct mlx5_ifc_per_protocol_networking_offload_caps_bits {
+	u8         csum_cap[0x1];
+	u8         vlan_cap[0x1];
+	u8         lro_cap[0x1];
+	u8         lro_psh_flag[0x1];
+	u8         lro_time_stamp[0x1];
+	u8         lro_max_msg_sz_mode[0x2];
+	u8         reserved_0[0x2];
+	u8         self_lb_mc[0x1];
+	u8         self_lb_uc[0x1];
+	u8         max_lso_cap[0x5];
+	u8         multi_pkt_send_wqe[0x2];
+	u8         wqe_inline_mode[0x2];
+	u8         rss_ind_tbl_cap[0x4];
+	u8         reserved_1[0x3];
+	u8         tunnel_lso_const_out_ip_id[0x1];
+	u8         tunnel_lro_gre[0x1];
+	u8         tunnel_lro_vxlan[0x1];
+	u8         tunnel_statless_gre[0x1];
+	u8         tunnel_stateless_vxlan[0x1];
+
+	u8         reserved_2[0x20];
+
+	u8         reserved_3[0x10];
+	u8         lro_min_mss_size[0x10];
+
+	u8         reserved_4[0x120];
+
+	u8         lro_timer_supported_periods[4][0x20];
+
+	u8         reserved_5[0x600];
+};
+
+enum {
+	MLX5_ROCE_CAP_L3_TYPE_GRH   = 0x1,
+	MLX5_ROCE_CAP_L3_TYPE_IPV4  = 0x2,
+	MLX5_ROCE_CAP_L3_TYPE_IPV6  = 0x4,
+};
+
+struct mlx5_ifc_roce_cap_bits {
+	u8         roce_apm[0x1];
+	u8         eth_prio_primary_in_rts2rts[0x1];
+	u8         reserved_0[0x1e];
+
+	u8         reserved_1[0x60];
+
+	u8         reserved_2[0xc];
+	u8         l3_type[0x4];
+	u8         reserved_3[0x8];
+	u8         roce_version[0x8];
+
+	u8         reserved_4[0x10];
+	u8         r_roce_dest_udp_port[0x10];
+
+	u8         r_roce_max_src_udp_port[0x10];
+	u8         r_roce_min_src_udp_port[0x10];
+
+	u8         reserved_5[0x10];
+	u8         roce_address_table_size[0x10];
+
+	u8         reserved_6[0x700];
+};
+
+enum {
+	MLX5_ATOMIC_CAPS_ATOMIC_SIZE_QP_1_BYTE     = 0x1,
+	MLX5_ATOMIC_CAPS_ATOMIC_SIZE_QP_2_BYTES    = 0x2,
+	MLX5_ATOMIC_CAPS_ATOMIC_SIZE_QP_4_BYTES    = 0x4,
+	MLX5_ATOMIC_CAPS_ATOMIC_SIZE_QP_8_BYTES    = 0x8,
+	MLX5_ATOMIC_CAPS_ATOMIC_SIZE_QP_16_BYTES   = 0x10,
+	MLX5_ATOMIC_CAPS_ATOMIC_SIZE_QP_32_BYTES   = 0x20,
+	MLX5_ATOMIC_CAPS_ATOMIC_SIZE_QP_64_BYTES   = 0x40,
+	MLX5_ATOMIC_CAPS_ATOMIC_SIZE_QP_128_BYTES  = 0x80,
+	MLX5_ATOMIC_CAPS_ATOMIC_SIZE_QP_256_BYTES  = 0x100,
+};
+
+enum {
+	MLX5_ATOMIC_CAPS_ATOMIC_SIZE_DC_1_BYTE     = 0x1,
+	MLX5_ATOMIC_CAPS_ATOMIC_SIZE_DC_2_BYTES    = 0x2,
+	MLX5_ATOMIC_CAPS_ATOMIC_SIZE_DC_4_BYTES    = 0x4,
+	MLX5_ATOMIC_CAPS_ATOMIC_SIZE_DC_8_BYTES    = 0x8,
+	MLX5_ATOMIC_CAPS_ATOMIC_SIZE_DC_16_BYTES   = 0x10,
+	MLX5_ATOMIC_CAPS_ATOMIC_SIZE_DC_32_BYTES   = 0x20,
+	MLX5_ATOMIC_CAPS_ATOMIC_SIZE_DC_64_BYTES   = 0x40,
+	MLX5_ATOMIC_CAPS_ATOMIC_SIZE_DC_128_BYTES  = 0x80,
+	MLX5_ATOMIC_CAPS_ATOMIC_SIZE_DC_256_BYTES  = 0x100,
+};
+
+struct mlx5_ifc_atomic_caps_bits {
+	u8         reserved_0[0x40];
+
+	u8         atomic_req_endianess[0x1];
+	u8         reserved_1[0x1f];
+
+	u8         reserved_2[0x20];
+
+	u8         reserved_3[0x10];
+	u8         atomic_operations[0x10];
+
+	u8         reserved_4[0x10];
+	u8         atomic_size_qp[0x10];
+
+	u8         reserved_5[0x10];
+	u8         atomic_size_dc[0x10];
+
+	u8         reserved_6[0x720];
+};
+
+struct mlx5_ifc_odp_cap_bits {
+	u8         reserved_0[0x40];
+
+	u8         sig[0x1];
+	u8         reserved_1[0x1f];
+
+	u8         reserved_2[0x20];
+
+	struct mlx5_ifc_odp_per_transport_service_cap_bits rc_odp_caps;
+
+	struct mlx5_ifc_odp_per_transport_service_cap_bits uc_odp_caps;
+
+	struct mlx5_ifc_odp_per_transport_service_cap_bits ud_odp_caps;
+
+	struct mlx5_ifc_odp_per_transport_service_cap_bits xrc_odp_caps;
+
+	struct mlx5_ifc_odp_per_transport_service_cap_bits dc_odp_caps;
+
+	u8         reserved_3[0x6e0];
+};
+
+enum {
+	MLX5_CMD_HCA_CAP_GID_TABLE_SIZE_8_GID_ENTRIES    = 0x0,
+	MLX5_CMD_HCA_CAP_GID_TABLE_SIZE_16_GID_ENTRIES   = 0x1,
+	MLX5_CMD_HCA_CAP_GID_TABLE_SIZE_32_GID_ENTRIES   = 0x2,
+	MLX5_CMD_HCA_CAP_GID_TABLE_SIZE_64_GID_ENTRIES   = 0x3,
+	MLX5_CMD_HCA_CAP_GID_TABLE_SIZE_128_GID_ENTRIES  = 0x4,
+};
+
+enum {
+	MLX5_CMD_HCA_CAP_PKEY_TABLE_SIZE_128_ENTRIES  = 0x0,
+	MLX5_CMD_HCA_CAP_PKEY_TABLE_SIZE_256_ENTRIES  = 0x1,
+	MLX5_CMD_HCA_CAP_PKEY_TABLE_SIZE_512_ENTRIES  = 0x2,
+	MLX5_CMD_HCA_CAP_PKEY_TABLE_SIZE_1K_ENTRIES   = 0x3,
+	MLX5_CMD_HCA_CAP_PKEY_TABLE_SIZE_2K_ENTRIES   = 0x4,
+	MLX5_CMD_HCA_CAP_PKEY_TABLE_SIZE_4K_ENTRIES   = 0x5,
+};
+
+enum {
+	MLX5_CMD_HCA_CAP_PORT_TYPE_IB        = 0x0,
+	MLX5_CMD_HCA_CAP_PORT_TYPE_ETHERNET  = 0x1,
+};
+
+enum {
+	MLX5_CMD_HCA_CAP_CMDIF_CHECKSUM_DISABLED       = 0x0,
+	MLX5_CMD_HCA_CAP_CMDIF_CHECKSUM_INITIAL_STATE  = 0x1,
+	MLX5_CMD_HCA_CAP_CMDIF_CHECKSUM_ENABLED        = 0x3,
+};
+
+struct mlx5_ifc_cmd_hca_cap_bits {
+	u8         reserved_0[0x80];
+
+	u8         log_max_srq_sz[0x8];
+	u8         log_max_qp_sz[0x8];
+	u8         reserved_1[0xb];
+	u8         log_max_qp[0x5];
+
+	u8         reserved_2[0xb];
+	u8         log_max_srq[0x5];
+	u8         reserved_3[0x10];
+
+	u8         reserved_4[0x8];
+	u8         log_max_cq_sz[0x8];
+	u8         reserved_5[0xb];
+	u8         log_max_cq[0x5];
+
+	u8         log_max_eq_sz[0x8];
+	u8         reserved_6[0x2];
+	u8         log_max_mkey[0x6];
+	u8         reserved_7[0xc];
+	u8         log_max_eq[0x4];
+
+	u8         max_indirection[0x8];
+	u8         reserved_8[0x1];
+	u8         log_max_mrw_sz[0x7];
+	u8         reserved_9[0x2];
+	u8         log_max_bsf_list_size[0x6];
+	u8         reserved_10[0x2];
+	u8         log_max_klm_list_size[0x6];
+
+	u8         reserved_11[0xa];
+	u8         log_max_ra_req_dc[0x6];
+	u8         reserved_12[0xa];
+	u8         log_max_ra_res_dc[0x6];
+
+	u8         reserved_13[0xa];
+	u8         log_max_ra_req_qp[0x6];
+	u8         reserved_14[0xa];
+	u8         log_max_ra_res_qp[0x6];
+
+	u8         pad_cap[0x1];
+	u8         cc_query_allowed[0x1];
+	u8         cc_modify_allowed[0x1];
+	u8         reserved_15[0xd];
+	u8         gid_table_size[0x10];
+
+	u8         out_of_seq_cnt[0x1];
+	u8         vport_counters[0x1];
+	u8         reserved_16[0x4];
+	u8         max_qp_cnt[0xa];
+	u8         pkey_table_size[0x10];
+
+	u8         vport_group_manager[0x1];
+	u8         vhca_group_manager[0x1];
+	u8         ib_virt[0x1];
+	u8         eth_virt[0x1];
+	u8         reserved_17[0x1];
+	u8         ets[0x1];
+	u8         nic_flow_table[0x1];
+	u8         eswitch_flow_table[0x1];
+	u8         reserved_18[0x3];
+	u8         local_ca_ack_delay[0x5];
+	u8         port_module_event[0x1];
+	u8         reserved_19[0x5];
+	u8         port_type[0x2];
+	u8         num_ports[0x8];
+
+	u8         snapshot[0x1];
+	u8         reserved_20[0x2];
+	u8         log_max_msg[0x5];
+	u8         reserved_21[0x4];
+	u8         max_tc[0x4];
+	u8         reserved_22[0x6];
+	u8         rol_s[0x1];
+	u8         rol_g[0x1];
+	u8         reserved_23[0x1];
+	u8         wol_s[0x1];
+	u8         wol_g[0x1];
+	u8         wol_a[0x1];
+	u8         wol_b[0x1];
+	u8         wol_m[0x1];
+	u8         wol_u[0x1];
+	u8         wol_p[0x1];
+
+	u8         stat_rate_support[0x10];
+	u8         reserved_24[0xc];
+	u8         cqe_version[0x4];
+
+	u8         compact_address_vector[0x1];
+	u8         striding_rq[0x1];
+	u8         reserved_25[0xc];
+	u8         dc_cnak_trace[0x1];
+	u8         drain_sigerr[0x1];
+	u8         cmdif_checksum[0x2];
+	u8         sigerr_cqe[0x1];
+	u8         reserved_26[0x1];
+	u8         wq_signature[0x1];
+	u8         sctr_data_cqe[0x1];
+	u8         reserved_27[0x1];
+	u8         sho[0x1];
+	u8         tph[0x1];
+	u8         rf[0x1];
+	u8         dct[0x1];
+	u8         reserved_28[0x1];
+	u8         eth_net_offloads[0x1];
+	u8         roce[0x1];
+	u8         atomic[0x1];
+	u8         reserved_29[0x1];
+
+	u8         cq_oi[0x1];
+	u8         cq_resize[0x1];
+	u8         cq_moderation[0x1];
+	u8         reserved_30[0x3];
+	u8         cq_eq_remap[0x1];
+	u8         pg[0x1];
+	u8         block_lb_mc[0x1];
+	u8         exponential_backoff[0x1];
+	u8         scqe_break_moderation[0x1];
+	u8         cq_period_start_from_cqe[0x1];
+	u8         cd[0x1];
+	u8         atm[0x1];
+	u8         apm[0x1];
+	u8         reserved_31[0x7];
+	u8         qkv[0x1];
+	u8         pkv[0x1];
+	u8         reserved_32[0x4];
+	u8         xrc[0x1];
+	u8         ud[0x1];
+	u8         uc[0x1];
+	u8         rc[0x1];
+
+	u8         reserved_33[0xa];
+	u8         uar_sz[0x6];
+	u8         reserved_34[0x8];
+	u8         log_pg_sz[0x8];
+
+	u8         bf[0x1];
+	u8         driver_version[0x1];
+	u8         pad_tx_eth_packet[0x1];
+	u8         reserved_35[0x8];
+	u8         log_bf_reg_size[0x5];
+	u8         reserved_36[0x10];
+
+	u8         reserved_37[0x10];
+	u8         max_wqe_sz_sq[0x10];
+
+	u8         reserved_38[0x10];
+	u8         max_wqe_sz_rq[0x10];
+
+	u8         reserved_39[0x10];
+	u8         max_wqe_sz_sq_dc[0x10];
+
+	u8         reserved_40[0x7];
+	u8         max_qp_mcg[0x19];
+
+	u8         reserved_41[0x18];
+	u8         log_max_mcg[0x8];
+
+	u8         reserved_42[0x3];
+	u8         log_max_transport_domain[0x5];
+	u8         reserved_43[0x3];
+	u8         log_max_pd[0x5];
+	u8         reserved_44[0xb];
+	u8         log_max_xrcd[0x5];
+
+	u8         reserved_45[0x10];
+	u8         max_flow_counter[0x10];
+
+	u8         reserved_46[0x3];
+	u8         log_max_rq[0x5];
+	u8         reserved_47[0x3];
+	u8         log_max_sq[0x5];
+	u8         reserved_48[0x3];
+	u8         log_max_tir[0x5];
+	u8         reserved_49[0x3];
+	u8         log_max_tis[0x5];
+
+	u8         basic_cyclic_rcv_wqe[0x1];
+	u8         reserved_50[0x2];
+	u8         log_max_rmp[0x5];
+	u8         reserved_51[0x3];
+	u8         log_max_rqt[0x5];
+	u8         reserved_52[0x3];
+	u8         log_max_rqt_size[0x5];
+	u8         reserved_53[0x3];
+	u8         log_max_tis_per_sq[0x5];
+
+	u8         reserved_54[0x3];
+	u8         log_max_stride_sz_rq[0x5];
+	u8         reserved_55[0x3];
+	u8         log_min_stride_sz_rq[0x5];
+	u8         reserved_56[0x3];
+	u8         log_max_stride_sz_sq[0x5];
+	u8         reserved_57[0x3];
+	u8         log_min_stride_sz_sq[0x5];
+
+	u8         reserved_58[0x1b];
+	u8         log_max_wq_sz[0x5];
+
+	u8         nic_vport_change_event[0x1];
+	u8         reserved_59[0xa];
+	u8         log_max_vlan_list[0x5];
+	u8         reserved_60[0x3];
+	u8         log_max_current_mc_list[0x5];
+	u8         reserved_61[0x3];
+	u8         log_max_current_uc_list[0x5];
+
+	u8         reserved_62[0x80];
+
+	u8         reserved_63[0x3];
+	u8         log_max_l2_table[0x5];
+	u8         reserved_64[0x8];
+	u8         log_uar_page_sz[0x10];
+
+	u8         reserved_65[0x20];
+
+	u8         device_frequency[0x20];
+
+	u8         reserved_66[0xa0];
+
+	u8         log_max_atomic_size_qp[0x8];
+	u8         reserved_67[0x10];
+	u8         log_max_atomic_size_dc[0x8];
+
+	u8         reserved_68[0x1f];
+	u8         cqe_compression[0x1];
+
+	u8         cqe_compression_timeout[0x10];
+	u8         cqe_compression_max_num[0x10];
+
+	u8         reserved_69[0x220];
+};
+
+union mlx5_ifc_dest_format_struct_flow_counter_list_auto_bits {
+	struct mlx5_ifc_dest_format_struct_bits dest_format_struct;
+	struct mlx5_ifc_flow_counter_list_bits flow_counter_list;
+	u8         reserved_0[0x40];
+};
+
+struct mlx5_ifc_fte_match_param_bits {
+	struct mlx5_ifc_fte_match_set_lyr_2_4_bits outer_headers;
+
+	struct mlx5_ifc_fte_match_set_misc_bits misc_parameters;
+
+	struct mlx5_ifc_fte_match_set_lyr_2_4_bits inner_headers;
+
+	u8         reserved_0[0xa00];
+};
+
+enum {
+	MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_SRC_IP     = 0x0,
+	MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_DST_IP     = 0x1,
+	MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_L4_SPORT   = 0x2,
+	MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_L4_DPORT   = 0x3,
+	MLX5_RX_HASH_FIELD_SELECT_SELECTED_FIELDS_IPSEC_SPI  = 0x4,
+};
+
+struct mlx5_ifc_rx_hash_field_select_bits {
+	u8         l3_prot_type[0x1];
+	u8         l4_prot_type[0x1];
+	u8         selected_fields[0x1e];
+};
+
+enum {
+	MLX5_WQ_TYPE_LINKED_LIST                 = 0x0,
+	MLX5_WQ_TYPE_CYCLIC                      = 0x1,
+	MLX5_WQ_TYPE_STRQ_LINKED_LIST            = 0x2,
+	MLX5_WQ_TYPE_STRQ_CYCLIC                 = 0x3,
+};
+
+enum {
+	MLX5_WQ_END_PAD_MODE_NONE               = 0x0,
+	MLX5_WQ_END_PAD_MODE_ALIGN              = 0x1,
+};
+
+struct mlx5_ifc_wq_bits {
+	u8         wq_type[0x4];
+	u8         wq_signature[0x1];
+	u8         end_padding_mode[0x2];
+	u8         cd_slave[0x1];
+	u8         reserved_0[0x18];
+
+	u8         hds_skip_first_sge[0x1];
+	u8         log2_hds_buf_size[0x3];
+	u8         reserved_1[0x7];
+	u8         page_offset[0x5];
+	u8         lwm[0x10];
+
+	u8         reserved_2[0x8];
+	u8         pd[0x18];
+
+	u8         reserved_3[0x8];
+	u8         uar_page[0x18];
+
+	u8         dbr_addr[0x40];
+
+	u8         hw_counter[0x20];
+
+	u8         sw_counter[0x20];
+
+	u8         reserved_4[0xc];
+	u8         log_wq_stride[0x4];
+	u8         reserved_5[0x3];
+	u8         log_wq_pg_sz[0x5];
+	u8         reserved_6[0x3];
+	u8         log_wq_sz[0x5];
+
+	u8         reserved_7[0x15];
+	u8         single_wqe_log_num_of_strides[0x3];
+	u8         two_byte_shift_en[0x1];
+	u8         reserved_8[0x4];
+	u8         single_stride_log_num_of_bytes[0x3];
+
+	u8         reserved_9[0x4c0];
+
+	struct mlx5_ifc_cmd_pas_bits pas[0];
+};
+
+struct mlx5_ifc_rq_num_bits {
+	u8         reserved_0[0x8];
+	u8         rq_num[0x18];
+};
+
+struct mlx5_ifc_mac_address_layout_bits {
+	u8         reserved_0[0x10];
+	u8         mac_addr_47_32[0x10];
+
+	u8         mac_addr_31_0[0x20];
+};
+
+struct mlx5_ifc_cong_control_r_roce_ecn_np_bits {
+	u8         reserved_0[0xa0];
+
+	u8         min_time_between_cnps[0x20];
+
+	u8         reserved_1[0x12];
+	u8         cnp_dscp[0x6];
+	u8         reserved_2[0x4];
+	u8         cnp_prio_mode[0x1];
+	u8         cnp_802p_prio[0x3];
+
+	u8         reserved_3[0x720];
+};
+
+struct mlx5_ifc_cong_control_r_roce_ecn_rp_bits {
+	u8         reserved_0[0x60];
+
+	u8         reserved_1[0x4];
+	u8         clamp_tgt_rate[0x1];
+	u8         reserved_2[0x3];
+	u8         clamp_tgt_rate_after_time_inc[0x1];
+	u8         reserved_3[0x17];
+
+	u8         reserved_4[0x20];
+
+	u8         rpg_time_reset[0x20];
+
+	u8         rpg_byte_reset[0x20];
+
+	u8         rpg_threshold[0x20];
+
+	u8         rpg_max_rate[0x20];
+
+	u8         rpg_ai_rate[0x20];
+
+	u8         rpg_hai_rate[0x20];
+
+	u8         rpg_gd[0x20];
+
+	u8         rpg_min_dec_fac[0x20];
+
+	u8         rpg_min_rate[0x20];
+
+	u8         reserved_5[0xe0];
+
+	u8         rate_to_set_on_first_cnp[0x20];
+
+	u8         dce_tcp_g[0x20];
+
+	u8         dce_tcp_rtt[0x20];
+
+	u8         rate_reduce_monitor_period[0x20];
+
+	u8         reserved_6[0x20];
+
+	u8         initial_alpha_value[0x20];
+
+	u8         reserved_7[0x4a0];
+};
+
+struct mlx5_ifc_cong_control_802_1qau_rp_bits {
+	u8         reserved_0[0x80];
+
+	u8         rppp_max_rps[0x20];
+
+	u8         rpg_time_reset[0x20];
+
+	u8         rpg_byte_reset[0x20];
+
+	u8         rpg_threshold[0x20];
+
+	u8         rpg_max_rate[0x20];
+
+	u8         rpg_ai_rate[0x20];
+
+	u8         rpg_hai_rate[0x20];
+
+	u8         rpg_gd[0x20];
+
+	u8         rpg_min_dec_fac[0x20];
+
+	u8         rpg_min_rate[0x20];
+
+	u8         reserved_1[0x640];
+};
+
+enum {
+	MLX5_RESIZE_FIELD_SELECT_RESIZE_FIELD_SELECT_LOG_CQ_SIZE    = 0x1,
+	MLX5_RESIZE_FIELD_SELECT_RESIZE_FIELD_SELECT_PAGE_OFFSET    = 0x2,
+	MLX5_RESIZE_FIELD_SELECT_RESIZE_FIELD_SELECT_LOG_PAGE_SIZE  = 0x4,
+};
+
+struct mlx5_ifc_resize_field_select_bits {
+	u8         resize_field_select[0x20];
+};
+
+enum {
+	MLX5_MODIFY_FIELD_SELECT_MODIFY_FIELD_SELECT_CQ_PERIOD     = 0x1,
+	MLX5_MODIFY_FIELD_SELECT_MODIFY_FIELD_SELECT_CQ_MAX_COUNT  = 0x2,
+	MLX5_MODIFY_FIELD_SELECT_MODIFY_FIELD_SELECT_OI            = 0x4,
+	MLX5_MODIFY_FIELD_SELECT_MODIFY_FIELD_SELECT_C_EQN         = 0x8,
+};
+
+struct mlx5_ifc_modify_field_select_bits {
+	u8         modify_field_select[0x20];
+};
+
+struct mlx5_ifc_field_select_r_roce_np_bits {
+	u8         field_select_r_roce_np[0x20];
+};
+
+enum {
+	MLX5_FIELD_SELECT_R_ROCE_RP_FIELD_SELECT_R_ROCE_RP_CLAMP_TGT_RATE                 = 0x2,
+	MLX5_FIELD_SELECT_R_ROCE_RP_FIELD_SELECT_R_ROCE_RP_CLAMP_TGT_RATE_AFTER_TIME_INC  = 0x4,
+	MLX5_FIELD_SELECT_R_ROCE_RP_FIELD_SELECT_R_ROCE_RP_RPG_TIME_RESET                 = 0x8,
+	MLX5_FIELD_SELECT_R_ROCE_RP_FIELD_SELECT_R_ROCE_RP_RPG_BYTE_RESET                 = 0x10,
+	MLX5_FIELD_SELECT_R_ROCE_RP_FIELD_SELECT_R_ROCE_RP_RPG_THRESHOLD                  = 0x20,
+	MLX5_FIELD_SELECT_R_ROCE_RP_FIELD_SELECT_R_ROCE_RP_RPG_MAX_RATE                   = 0x40,
+	MLX5_FIELD_SELECT_R_ROCE_RP_FIELD_SELECT_R_ROCE_RP_RPG_AI_RATE                    = 0x80,
+	MLX5_FIELD_SELECT_R_ROCE_RP_FIELD_SELECT_R_ROCE_RP_RPG_HAI_RATE                   = 0x100,
+	MLX5_FIELD_SELECT_R_ROCE_RP_FIELD_SELECT_R_ROCE_RP_RPG_MIN_DEC_FAC                = 0x200,
+	MLX5_FIELD_SELECT_R_ROCE_RP_FIELD_SELECT_R_ROCE_RP_RPG_MIN_RATE                   = 0x400,
+	MLX5_FIELD_SELECT_R_ROCE_RP_FIELD_SELECT_R_ROCE_RP_RATE_TO_SET_ON_FIRST_CNP       = 0x800,
+	MLX5_FIELD_SELECT_R_ROCE_RP_FIELD_SELECT_R_ROCE_RP_DCE_TCP_G                      = 0x1000,
+	MLX5_FIELD_SELECT_R_ROCE_RP_FIELD_SELECT_R_ROCE_RP_DCE_TCP_RTT                    = 0x2000,
+	MLX5_FIELD_SELECT_R_ROCE_RP_FIELD_SELECT_R_ROCE_RP_RATE_REDUCE_MONITOR_PERIOD     = 0x4000,
+	MLX5_FIELD_SELECT_R_ROCE_RP_FIELD_SELECT_R_ROCE_RP_INITIAL_ALPHA_VALUE            = 0x8000,
+};
+
+struct mlx5_ifc_field_select_r_roce_rp_bits {
+	u8         field_select_r_roce_rp[0x20];
+};
+
+enum {
+	MLX5_FIELD_SELECT_802_1QAU_RP_FIELD_SELECT_8021QAURP_RPPP_MAX_RPS     = 0x4,
+	MLX5_FIELD_SELECT_802_1QAU_RP_FIELD_SELECT_8021QAURP_RPG_TIME_RESET   = 0x8,
+	MLX5_FIELD_SELECT_802_1QAU_RP_FIELD_SELECT_8021QAURP_RPG_BYTE_RESET   = 0x10,
+	MLX5_FIELD_SELECT_802_1QAU_RP_FIELD_SELECT_8021QAURP_RPG_THRESHOLD    = 0x20,
+	MLX5_FIELD_SELECT_802_1QAU_RP_FIELD_SELECT_8021QAURP_RPG_MAX_RATE     = 0x40,
+	MLX5_FIELD_SELECT_802_1QAU_RP_FIELD_SELECT_8021QAURP_RPG_AI_RATE      = 0x80,
+	MLX5_FIELD_SELECT_802_1QAU_RP_FIELD_SELECT_8021QAURP_RPG_HAI_RATE     = 0x100,
+	MLX5_FIELD_SELECT_802_1QAU_RP_FIELD_SELECT_8021QAURP_RPG_GD           = 0x200,
+	MLX5_FIELD_SELECT_802_1QAU_RP_FIELD_SELECT_8021QAURP_RPG_MIN_DEC_FAC  = 0x400,
+	MLX5_FIELD_SELECT_802_1QAU_RP_FIELD_SELECT_8021QAURP_RPG_MIN_RATE     = 0x800,
+};
+
+struct mlx5_ifc_field_select_802_1qau_rp_bits {
+	u8         field_select_8021qaurp[0x20];
+};
+
+struct mlx5_ifc_nodnic_ring_config_reg_bits {
+	u8         queue_address_63_32[0x20];
+
+	u8         queue_address_31_12[0x14];
+	u8         reserved_0[0x6];
+	u8         log_size[0x6];
+
+	struct mlx5_ifc_nodnic_ring_doorbell_bits doorbell;
+
+	u8         reserved_1[0x8];
+	u8         queue_number[0x18];
+
+	u8         q_key[0x20];
+
+	u8         reserved_2[0x10];
+	u8         pkey_index[0x10];
+
+	u8         reserved_3[0x40];
+};
+
+struct mlx5_ifc_nodnic_cq_arming_word_bits {
+	u8         reserved_0[0x8];
+	u8         cq_ci[0x10];
+	u8         reserved_1[0x8];
+};
+
+enum {
+	MLX5_NODNIC_EVENT_WORD_LINK_TYPE_INFINIBAND  = 0x0,
+	MLX5_NODNIC_EVENT_WORD_LINK_TYPE_ETHERNET    = 0x1,
+};
+
+enum {
+	MLX5_NODNIC_EVENT_WORD_PORT_STATE_DOWN        = 0x0,
+	MLX5_NODNIC_EVENT_WORD_PORT_STATE_INITIALIZE  = 0x1,
+	MLX5_NODNIC_EVENT_WORD_PORT_STATE_ARMED       = 0x2,
+	MLX5_NODNIC_EVENT_WORD_PORT_STATE_ACTIVE      = 0x3,
+};
+
+struct mlx5_ifc_nodnic_event_word_bits {
+	u8         driver_reset_needed[0x1];
+	u8         port_management_change_event[0x1];
+	u8         reserved_0[0x19];
+	u8         link_type[0x1];
+	u8         port_state[0x4];
+};
+
+struct mlx5_ifc_nic_vport_change_event_bits {
+	u8         reserved_0[0x10];
+	u8         vport_num[0x10];
+
+	u8         reserved_1[0xc0];
+};
+
+struct mlx5_ifc_pages_req_event_bits {
+	u8         reserved_0[0x10];
+	u8         function_id[0x10];
+
+	u8         num_pages[0x20];
+
+	u8         reserved_1[0xa0];
+};
+
+struct mlx5_ifc_cmd_inter_comp_event_bits {
+	u8         command_completion_vector[0x20];
+
+	u8         reserved_0[0xc0];
+};
+
+struct mlx5_ifc_stall_vl_event_bits {
+	u8         reserved_0[0x18];
+	u8         port_num[0x1];
+	u8         reserved_1[0x3];
+	u8         vl[0x4];
+
+	u8         reserved_2[0xa0];
+};
+
+struct mlx5_ifc_db_bf_congestion_event_bits {
+	u8         event_subtype[0x8];
+	u8         reserved_0[0x8];
+	u8         congestion_level[0x8];
+	u8         reserved_1[0x8];
+
+	u8         reserved_2[0xa0];
+};
+
+struct mlx5_ifc_gpio_event_bits {
+	u8         reserved_0[0x60];
+
+	u8         gpio_event_hi[0x20];
+
+	u8         gpio_event_lo[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_port_state_change_event_bits {
+	u8         reserved_0[0x40];
+
+	u8         port_num[0x4];
+	u8         reserved_1[0x1c];
+
+	u8         reserved_2[0x80];
+};
+
+struct mlx5_ifc_dropped_packet_logged_bits {
+	u8         reserved_0[0xe0];
+};
+
+enum {
+	MLX5_CQ_ERROR_SYNDROME_CQ_OVERRUN                 = 0x1,
+	MLX5_CQ_ERROR_SYNDROME_CQ_ACCESS_VIOLATION_ERROR  = 0x2,
+};
+
+struct mlx5_ifc_cq_error_bits {
+	u8         reserved_0[0x8];
+	u8         cqn[0x18];
+
+	u8         reserved_1[0x20];
+
+	u8         reserved_2[0x18];
+	u8         syndrome[0x8];
+
+	u8         reserved_3[0x80];
+};
+
+struct mlx5_ifc_rdma_page_fault_event_bits {
+	u8         bytes_commited[0x20];
+
+	u8         r_key[0x20];
+
+	u8         reserved_0[0x10];
+	u8         packet_len[0x10];
+
+	u8         rdma_op_len[0x20];
+
+	u8         rdma_va[0x40];
+
+	u8         reserved_1[0x5];
+	u8         rdma[0x1];
+	u8         write[0x1];
+	u8         requestor[0x1];
+	u8         qp_number[0x18];
+};
+
+struct mlx5_ifc_wqe_associated_page_fault_event_bits {
+	u8         bytes_committed[0x20];
+
+	u8         reserved_0[0x10];
+	u8         wqe_index[0x10];
+
+	u8         reserved_1[0x10];
+	u8         len[0x10];
+
+	u8         reserved_2[0x60];
+
+	u8         reserved_3[0x5];
+	u8         rdma[0x1];
+	u8         write_read[0x1];
+	u8         requestor[0x1];
+	u8         qpn[0x18];
+};
+
+enum {
+	MLX5_QP_EVENTS_TYPE_QP  = 0x0,
+	MLX5_QP_EVENTS_TYPE_RQ  = 0x1,
+	MLX5_QP_EVENTS_TYPE_SQ  = 0x2,
+};
+
+struct mlx5_ifc_qp_events_bits {
+	u8         reserved_0[0xa0];
+
+	u8         type[0x8];
+	u8         reserved_1[0x18];
+
+	u8         reserved_2[0x8];
+	u8         qpn_rqn_sqn[0x18];
+};
+
+struct mlx5_ifc_dct_events_bits {
+	u8         reserved_0[0xc0];
+
+	u8         reserved_1[0x8];
+	u8         dct_number[0x18];
+};
+
+struct mlx5_ifc_comp_event_bits {
+	u8         reserved_0[0xc0];
+
+	u8         reserved_1[0x8];
+	u8         cq_number[0x18];
+};
+
+struct mlx5_ifc_fw_version_bits {
+	u8         major[0x10];
+	u8         reserved_0[0x10];
+
+	u8         minor[0x10];
+	u8         subminor[0x10];
+
+	u8         second[0x8];
+	u8         minute[0x8];
+	u8         hour[0x8];
+	u8         reserved_1[0x8];
+
+	u8         year[0x10];
+	u8         month[0x8];
+	u8         day[0x8];
+};
+
+enum {
+	MLX5_QPC_STATE_RST        = 0x0,
+	MLX5_QPC_STATE_INIT       = 0x1,
+	MLX5_QPC_STATE_RTR        = 0x2,
+	MLX5_QPC_STATE_RTS        = 0x3,
+	MLX5_QPC_STATE_SQER       = 0x4,
+	MLX5_QPC_STATE_SQD        = 0x5,
+	MLX5_QPC_STATE_ERR        = 0x6,
+	MLX5_QPC_STATE_SUSPENDED  = 0x9,
+};
+
+enum {
+	MLX5_QPC_ST_RC            = 0x0,
+	MLX5_QPC_ST_UC            = 0x1,
+	MLX5_QPC_ST_UD            = 0x2,
+	MLX5_QPC_ST_XRC           = 0x3,
+	MLX5_QPC_ST_DCI           = 0x5,
+	MLX5_QPC_ST_QP0           = 0x7,
+	MLX5_QPC_ST_QP1           = 0x8,
+	MLX5_QPC_ST_RAW_DATAGRAM  = 0x9,
+	MLX5_QPC_ST_REG_UMR       = 0xc,
+};
+
+enum {
+	MLX5_QP_PM_ARMED            = 0x0,
+	MLX5_QP_PM_REARM            = 0x1,
+	MLX5_QPC_PM_STATE_RESERVED  = 0x2,
+	MLX5_QP_PM_MIGRATED         = 0x3,
+};
+
+enum {
+	MLX5_QPC_END_PADDING_MODE_SCATTER_AS_IS                = 0x0,
+	MLX5_QPC_END_PADDING_MODE_PAD_TO_CACHE_LINE_ALIGNMENT  = 0x1,
+};
+
+enum {
+	MLX5_QPC_MTU_256_BYTES        = 0x1,
+	MLX5_QPC_MTU_512_BYTES        = 0x2,
+	MLX5_QPC_MTU_1K_BYTES         = 0x3,
+	MLX5_QPC_MTU_2K_BYTES         = 0x4,
+	MLX5_QPC_MTU_4K_BYTES         = 0x5,
+	MLX5_QPC_MTU_RAW_ETHERNET_QP  = 0x7,
+};
+
+enum {
+	MLX5_QPC_ATOMIC_MODE_IB_SPEC     = 0x1,
+	MLX5_QPC_ATOMIC_MODE_ONLY_8B     = 0x2,
+	MLX5_QPC_ATOMIC_MODE_UP_TO_8B    = 0x3,
+	MLX5_QPC_ATOMIC_MODE_UP_TO_16B   = 0x4,
+	MLX5_QPC_ATOMIC_MODE_UP_TO_32B   = 0x5,
+	MLX5_QPC_ATOMIC_MODE_UP_TO_64B   = 0x6,
+	MLX5_QPC_ATOMIC_MODE_UP_TO_128B  = 0x7,
+	MLX5_QPC_ATOMIC_MODE_UP_TO_256B  = 0x8,
+};
+
+enum {
+	MLX5_QPC_CS_REQ_DISABLE    = 0x0,
+	MLX5_QPC_CS_REQ_UP_TO_32B  = 0x11,
+	MLX5_QPC_CS_REQ_UP_TO_64B  = 0x22,
+};
+
+enum {
+	MLX5_QPC_CS_RES_DISABLE    = 0x0,
+	MLX5_QPC_CS_RES_UP_TO_32B  = 0x1,
+	MLX5_QPC_CS_RES_UP_TO_64B  = 0x2,
+};
+
+struct mlx5_ifc_qpc_bits {
+	u8         state[0x4];
+	u8         reserved_0[0x4];
+	u8         st[0x8];
+	u8         reserved_1[0x3];
+	u8         pm_state[0x2];
+	u8         reserved_2[0x7];
+	u8         end_padding_mode[0x2];
+	u8         reserved_3[0x2];
+
+	u8         wq_signature[0x1];
+	u8         block_lb_mc[0x1];
+	u8         atomic_like_write_en[0x1];
+	u8         latency_sensitive[0x1];
+	u8         reserved_4[0x1];
+	u8         drain_sigerr[0x1];
+	u8         reserved_5[0x2];
+	u8         pd[0x18];
+
+	u8         mtu[0x3];
+	u8         log_msg_max[0x5];
+	u8         reserved_6[0x1];
+	u8         log_rq_size[0x4];
+	u8         log_rq_stride[0x3];
+	u8         no_sq[0x1];
+	u8         log_sq_size[0x4];
+	u8         reserved_7[0x6];
+	u8         rlky[0x1];
+	u8         reserved_8[0x4];
+
+	u8         counter_set_id[0x8];
+	u8         uar_page[0x18];
+
+	u8         reserved_9[0x8];
+	u8         user_index[0x18];
+
+	u8         reserved_10[0x3];
+	u8         log_page_size[0x5];
+	u8         remote_qpn[0x18];
+
+	struct mlx5_ifc_ads_bits primary_address_path;
+
+	struct mlx5_ifc_ads_bits secondary_address_path;
+
+	u8         log_ack_req_freq[0x4];
+	u8         reserved_11[0x4];
+	u8         log_sra_max[0x3];
+	u8         reserved_12[0x2];
+	u8         retry_count[0x3];
+	u8         rnr_retry[0x3];
+	u8         reserved_13[0x1];
+	u8         fre[0x1];
+	u8         cur_rnr_retry[0x3];
+	u8         cur_retry_count[0x3];
+	u8         reserved_14[0x5];
+
+	u8         reserved_15[0x20];
+
+	u8         reserved_16[0x8];
+	u8         next_send_psn[0x18];
+
+	u8         reserved_17[0x8];
+	u8         cqn_snd[0x18];
+
+	u8         reserved_18[0x40];
+
+	u8         reserved_19[0x8];
+	u8         last_acked_psn[0x18];
+
+	u8         reserved_20[0x8];
+	u8         ssn[0x18];
+
+	u8         reserved_21[0x8];
+	u8         log_rra_max[0x3];
+	u8         reserved_22[0x1];
+	u8         atomic_mode[0x4];
+	u8         rre[0x1];
+	u8         rwe[0x1];
+	u8         rae[0x1];
+	u8         reserved_23[0x1];
+	u8         page_offset[0x6];
+	u8         reserved_24[0x3];
+	u8         cd_slave_receive[0x1];
+	u8         cd_slave_send[0x1];
+	u8         cd_master[0x1];
+
+	u8         reserved_25[0x3];
+	u8         min_rnr_nak[0x5];
+	u8         next_rcv_psn[0x18];
+
+	u8         reserved_26[0x8];
+	u8         xrcd[0x18];
+
+	u8         reserved_27[0x8];
+	u8         cqn_rcv[0x18];
+
+	u8         dbr_addr[0x40];
+
+	u8         q_key[0x20];
+
+	u8         reserved_28[0x5];
+	u8         rq_type[0x3];
+	u8         srqn_rmpn[0x18];
+
+	u8         reserved_29[0x8];
+	u8         rmsn[0x18];
+
+	u8         hw_sq_wqebb_counter[0x10];
+	u8         sw_sq_wqebb_counter[0x10];
+
+	u8         hw_rq_counter[0x20];
+
+	u8         sw_rq_counter[0x20];
+
+	u8         reserved_30[0x20];
+
+	u8         reserved_31[0xf];
+	u8         cgs[0x1];
+	u8         cs_req[0x8];
+	u8         cs_res[0x8];
+
+	u8         dc_access_key[0x40];
+
+	u8         rdma_active[0x1];
+	u8         comm_est[0x1];
+	u8         suspended[0x1];
+	u8         reserved_32[0x5];
+	u8         send_msg_psn[0x18];
+
+	u8         reserved_33[0x8];
+	u8         rcv_msg_psn[0x18];
+
+	u8         rdma_va[0x40];
+
+	u8         rdma_key[0x20];
+
+	u8         reserved_34[0x20];
+};
+
+struct mlx5_ifc_roce_addr_layout_bits {
+	u8         source_l3_address[16][0x8];
+
+	u8         reserved_0[0x3];
+	u8         vlan_valid[0x1];
+	u8         vlan_id[0xc];
+	u8         source_mac_47_32[0x10];
+
+	u8         source_mac_31_0[0x20];
+
+	u8         reserved_1[0x14];
+	u8         roce_l3_type[0x4];
+	u8         roce_version[0x8];
+
+	u8         reserved_2[0x20];
+};
+
+struct mlx5_ifc_rdbc_bits {
+	u8         reserved_0[0x1c];
+	u8         type[0x4];
+
+	u8         reserved_1[0x20];
+
+	u8         reserved_2[0x8];
+	u8         psn[0x18];
+
+	u8         rkey[0x20];
+
+	u8         address[0x40];
+
+	u8         byte_count[0x20];
+
+	u8         reserved_3[0x20];
+
+	u8         atomic_resp[32][0x8];
+};
+
+enum {
+	MLX5_FLOW_CONTEXT_ACTION_ALLOW     = 0x1,
+	MLX5_FLOW_CONTEXT_ACTION_DROP      = 0x2,
+	MLX5_FLOW_CONTEXT_ACTION_FWD_DEST  = 0x4,
+	MLX5_FLOW_CONTEXT_ACTION_COUNT     = 0x8,
+};
+
+struct mlx5_ifc_flow_context_bits {
+	u8         reserved_0[0x20];
+
+	u8         group_id[0x20];
+
+	u8         reserved_1[0x8];
+	u8         flow_tag[0x18];
+
+	u8         reserved_2[0x10];
+	u8         action[0x10];
+
+	u8         reserved_3[0x8];
+	u8         destination_list_size[0x18];
+
+	u8         reserved_4[0x8];
+	u8         flow_counter_list_size[0x18];
+
+	u8         reserved_5[0x140];
+
+	struct mlx5_ifc_fte_match_param_bits match_value;
+
+	u8         reserved_6[0x600];
+
+	union mlx5_ifc_dest_format_struct_flow_counter_list_auto_bits destination[0];
+};
+
+enum {
+	MLX5_XRC_SRQC_STATE_GOOD   = 0x0,
+	MLX5_XRC_SRQC_STATE_ERROR  = 0x1,
+};
+
+struct mlx5_ifc_xrc_srqc_bits {
+	u8         state[0x4];
+	u8         log_xrc_srq_size[0x4];
+	u8         reserved_0[0x18];
+
+	u8         wq_signature[0x1];
+	u8         cont_srq[0x1];
+	u8         reserved_1[0x1];
+	u8         rlky[0x1];
+	u8         basic_cyclic_rcv_wqe[0x1];
+	u8         log_rq_stride[0x3];
+	u8         xrcd[0x18];
+
+	u8         page_offset[0x6];
+	u8         reserved_2[0x2];
+	u8         cqn[0x18];
+
+	u8         reserved_3[0x20];
+
+	u8         reserved_4[0x2];
+	u8         log_page_size[0x6];
+	u8         user_index[0x18];
+
+	u8         reserved_5[0x20];
+
+	u8         reserved_6[0x8];
+	u8         pd[0x18];
+
+	u8         lwm[0x10];
+	u8         wqe_cnt[0x10];
+
+	u8         reserved_7[0x40];
+
+	u8         db_record_addr_h[0x20];
+
+	u8         db_record_addr_l[0x1e];
+	u8         reserved_8[0x2];
+
+	u8         reserved_9[0x80];
+};
+
+struct mlx5_ifc_traffic_counter_bits {
+	u8         packets[0x40];
+
+	u8         octets[0x40];
+};
+
+struct mlx5_ifc_tisc_bits {
+	u8         reserved_0[0xc];
+	u8         prio[0x4];
+	u8         reserved_1[0x10];
+
+	u8         reserved_2[0x100];
+
+	u8         reserved_3[0x8];
+	u8         transport_domain[0x18];
+
+	u8         reserved_4[0x3c0];
+};
+
+enum {
+	MLX5_TIRC_DISP_TYPE_DIRECT    = 0x0,
+	MLX5_TIRC_DISP_TYPE_INDIRECT  = 0x1,
+};
+
+enum {
+	MLX5_TIRC_LRO_ENABLE_MASK_IPV4_LRO  = 0x1,
+	MLX5_TIRC_LRO_ENABLE_MASK_IPV6_LRO  = 0x2,
+};
+
+enum {
+	MLX5_TIRC_RX_HASH_FN_HASH_NONE           = 0x0,
+	MLX5_TIRC_RX_HASH_FN_HASH_INVERTED_XOR8  = 0x1,
+	MLX5_TIRC_RX_HASH_FN_HASH_TOEPLITZ       = 0x2,
+};
+
+enum {
+	MLX5_TIRC_SELF_LB_EN_ENABLE_UNICAST    = 0x1,
+	MLX5_TIRC_SELF_LB_EN_ENABLE_MULTICAST  = 0x2,
+};
+
+struct mlx5_ifc_tirc_bits {
+	u8         reserved_0[0x20];
+
+	u8         disp_type[0x4];
+	u8         reserved_1[0x1c];
+
+	u8         reserved_2[0x40];
+
+	u8         reserved_3[0x4];
+	u8         lro_timeout_period_usecs[0x10];
+	u8         lro_enable_mask[0x4];
+	u8         lro_max_msg_sz[0x8];
+
+	u8         reserved_4[0x40];
+
+	u8         reserved_5[0x8];
+	u8         inline_rqn[0x18];
+
+	u8         rx_hash_symmetric[0x1];
+	u8         reserved_6[0x1];
+	u8         tunneled_offload_en[0x1];
+	u8         reserved_7[0x5];
+	u8         indirect_table[0x18];
+
+	u8         rx_hash_fn[0x4];
+	u8         reserved_8[0x2];
+	u8         self_lb_en[0x2];
+	u8         transport_domain[0x18];
+
+	u8         rx_hash_toeplitz_key[10][0x20];
+
+	struct mlx5_ifc_rx_hash_field_select_bits rx_hash_field_selector_outer;
+
+	struct mlx5_ifc_rx_hash_field_select_bits rx_hash_field_selector_inner;
+
+	u8         reserved_9[0x4c0];
+};
+
+enum {
+	MLX5_SRQC_STATE_GOOD   = 0x0,
+	MLX5_SRQC_STATE_ERROR  = 0x1,
+};
+
+struct mlx5_ifc_srqc_bits {
+	u8         state[0x4];
+	u8         log_srq_size[0x4];
+	u8         reserved_0[0x18];
+
+	u8         wq_signature[0x1];
+	u8         cont_srq[0x1];
+	u8         reserved_1[0x1];
+	u8         rlky[0x1];
+	u8         reserved_2[0x1];
+	u8         log_rq_stride[0x3];
+	u8         xrcd[0x18];
+
+	u8         page_offset[0x6];
+	u8         reserved_3[0x2];
+	u8         cqn[0x18];
+
+	u8         reserved_4[0x20];
+
+	u8         reserved_5[0x2];
+	u8         log_page_size[0x6];
+	u8         reserved_6[0x18];
+
+	u8         reserved_7[0x20];
+
+	u8         reserved_8[0x8];
+	u8         pd[0x18];
+
+	u8         lwm[0x10];
+	u8         wqe_cnt[0x10];
+
+	u8         reserved_9[0x40];
+
+	u8         db_record_addr_h[0x20];
+
+	u8         db_record_addr_l[0x1e];
+	u8         reserved_10[0x2];
+
+	u8         reserved_11[0x80];
+};
+
+enum {
+	MLX5_SQC_STATE_RST  = 0x0,
+	MLX5_SQC_STATE_RDY  = 0x1,
+	MLX5_SQC_STATE_ERR  = 0x3,
+};
+
+struct mlx5_ifc_sqc_bits {
+	u8         rlky[0x1];
+	u8         cd_master[0x1];
+	u8         fre[0x1];
+	u8         flush_in_error_en[0x1];
+	u8         allow_multi_pkt_send_wqe[0x1];
+	u8         min_wqe_inline_mode[0x3];
+	u8         state[0x4];
+	u8         reserved_0[0x14];
+
+	u8         reserved_1[0x8];
+	u8         user_index[0x18];
+
+	u8         reserved_2[0x8];
+	u8         cqn[0x18];
+
+	u8         reserved_3[0xa0];
+
+	u8         tis_lst_sz[0x10];
+	u8         reserved_4[0x10];
+
+	u8         reserved_5[0x40];
+
+	u8         reserved_6[0x8];
+	u8         tis_num_0[0x18];
+
+	struct mlx5_ifc_wq_bits wq;
+};
+
+struct mlx5_ifc_rqtc_bits {
+	u8         reserved_0[0xa0];
+
+	u8         reserved_1[0x10];
+	u8         rqt_max_size[0x10];
+
+	u8         reserved_2[0x10];
+	u8         rqt_actual_size[0x10];
+
+	u8         reserved_3[0x6a0];
+
+	struct mlx5_ifc_rq_num_bits rq_num[0];
+};
+
+enum {
+	MLX5_RQC_RQ_TYPE_MEMORY_RQ_INLINE      = 0x0,
+	MLX5_RQC_RQ_TYPE_MEMORY_RQ_RMP         = 0x1,
+};
+
+enum {
+	MLX5_RQC_STATE_RST  = 0x0,
+	MLX5_RQC_STATE_RDY  = 0x1,
+	MLX5_RQC_STATE_ERR  = 0x3,
+};
+
+struct mlx5_ifc_rqc_bits {
+	u8         rlky[0x1];
+	u8         reserved_0[0x2];
+	u8         vlan_strip_disable[0x1];
+	u8         mem_rq_type[0x4];
+	u8         state[0x4];
+	u8         reserved_1[0x1];
+	u8         flush_in_error_en[0x1];
+	u8         reserved_2[0x12];
+
+	u8         reserved_3[0x8];
+	u8         user_index[0x18];
+
+	u8         reserved_4[0x8];
+	u8         cqn[0x18];
+
+	u8         counter_set_id[0x8];
+	u8         reserved_5[0x18];
+
+	u8         reserved_6[0x8];
+	u8         rmpn[0x18];
+
+	u8         reserved_7[0xe0];
+
+	struct mlx5_ifc_wq_bits wq;
+};
+
+enum {
+	MLX5_RMPC_STATE_RDY  = 0x1,
+	MLX5_RMPC_STATE_ERR  = 0x3,
+};
+
+struct mlx5_ifc_rmpc_bits {
+	u8         reserved_0[0x8];
+	u8         state[0x4];
+	u8         reserved_1[0x14];
+
+	u8         basic_cyclic_rcv_wqe[0x1];
+	u8         reserved_2[0x1f];
+
+	u8         reserved_3[0x140];
+
+	struct mlx5_ifc_wq_bits wq;
+};
+
+enum {
+	MLX5_NIC_VPORT_CONTEXT_ALLOWED_LIST_TYPE_CURRENT_UC_MAC_ADDRESS  = 0x0,
+	MLX5_NIC_VPORT_CONTEXT_ALLOWED_LIST_TYPE_CURRENT_MC_MAC_ADDRESS  = 0x1,
+	MLX5_NIC_VPORT_CONTEXT_ALLOWED_LIST_TYPE_VLAN_LIST               = 0x2,
+};
+
+struct mlx5_ifc_nic_vport_context_bits {
+	u8         reserved_0[0x5];
+	u8         min_wqe_inline_mode[0x3];
+	u8         reserved_1[0x17];
+	u8         roce_en[0x1];
+
+	u8         arm_change_event[0x1];
+	u8         reserved_2[0x1a];
+	u8         event_on_mtu[0x1];
+	u8         event_on_promisc_change[0x1];
+	u8         event_on_vlan_change[0x1];
+	u8         event_on_mc_address_change[0x1];
+	u8         event_on_uc_address_change[0x1];
+
+	u8         reserved_3[0xe0];
+
+	u8         reserved_4[0x10];
+	u8         mtu[0x10];
+
+	u8         system_image_guid[0x40];
+
+	u8         port_guid[0x40];
+
+	u8         node_guid[0x40];
+
+	u8         reserved_5[0x140];
+
+	u8         qkey_violation_counter[0x10];
+	u8         reserved_6[0x10];
+
+	u8         reserved_7[0x420];
+
+	u8         promisc_uc[0x1];
+	u8         promisc_mc[0x1];
+	u8         promisc_all[0x1];
+	u8         reserved_8[0x2];
+	u8         allowed_list_type[0x3];
+	u8         reserved_9[0xc];
+	u8         allowed_list_size[0xc];
+
+	struct mlx5_ifc_mac_address_layout_bits permanent_address;
+
+	u8         reserved_10[0x20];
+
+	u8         current_uc_mac_address[0][0x40];
+};
+
+enum {
+	MLX5_ACCESS_MODE_PA        = 0x0,
+	MLX5_ACCESS_MODE_MTT       = 0x1,
+	MLX5_ACCESS_MODE_KLM       = 0x2,
+};
+
+struct mlx5_ifc_mkc_bits {
+	u8         reserved_0[0x1];
+	u8         free[0x1];
+	u8         reserved_1[0xd];
+	u8         small_fence_on_rdma_read_response[0x1];
+	u8         umr_en[0x1];
+	u8         a[0x1];
+	u8         rw[0x1];
+	u8         rr[0x1];
+	u8         lw[0x1];
+	u8         lr[0x1];
+	u8         access_mode[0x2];
+	u8         reserved_2[0x8];
+
+	u8         qpn[0x18];
+	u8         mkey_7_0[0x8];
+
+	u8         reserved_3[0x20];
+
+	u8         length64[0x1];
+	u8         bsf_en[0x1];
+	u8         sync_umr[0x1];
+	u8         reserved_4[0x2];
+	u8         expected_sigerr_count[0x1];
+	u8         reserved_5[0x1];
+	u8         en_rinval[0x1];
+	u8         pd[0x18];
+
+	u8         start_addr[0x40];
+
+	u8         len[0x40];
+
+	u8         bsf_octword_size[0x20];
+
+	u8         reserved_6[0x80];
+
+	u8         translations_octword_size[0x20];
+
+	u8         reserved_7[0x1b];
+	u8         log_page_size[0x5];
+
+	u8         reserved_8[0x20];
+};
+
+struct mlx5_ifc_pkey_bits {
+	u8         reserved_0[0x10];
+	u8         pkey[0x10];
+};
+
+struct mlx5_ifc_array128_auto_bits {
+	u8         array128_auto[16][0x8];
+};
+
+enum {
+	MLX5_HCA_VPORT_CONTEXT_FIELD_SELECT_PORT_GUID           = 0x0,
+	MLX5_HCA_VPORT_CONTEXT_FIELD_SELECT_NODE_GUID           = 0x1,
+	MLX5_HCA_VPORT_CONTEXT_FIELD_SELECT_VPORT_STATE_POLICY  = 0x2,
+};
+
+enum {
+	MLX5_HCA_VPORT_CONTEXT_PORT_PHYSICAL_STATE_SLEEP                      = 0x1,
+	MLX5_HCA_VPORT_CONTEXT_PORT_PHYSICAL_STATE_POLLING                    = 0x2,
+	MLX5_HCA_VPORT_CONTEXT_PORT_PHYSICAL_STATE_DISABLED                   = 0x3,
+	MLX5_HCA_VPORT_CONTEXT_PORT_PHYSICAL_STATE_PORTCONFIGURATIONTRAINING  = 0x4,
+	MLX5_HCA_VPORT_CONTEXT_PORT_PHYSICAL_STATE_LINKUP                     = 0x5,
+	MLX5_HCA_VPORT_CONTEXT_PORT_PHYSICAL_STATE_LINKERRORRECOVERY          = 0x6,
+	MLX5_HCA_VPORT_CONTEXT_PORT_PHYSICAL_STATE_PHYTEST                    = 0x7,
+};
+
+enum {
+	MLX5_HCA_VPORT_CONTEXT_VPORT_STATE_POLICY_DOWN    = 0x0,
+	MLX5_HCA_VPORT_CONTEXT_VPORT_STATE_POLICY_UP      = 0x1,
+	MLX5_HCA_VPORT_CONTEXT_VPORT_STATE_POLICY_FOLLOW  = 0x2,
+};
+
+enum {
+	MLX5_HCA_VPORT_CONTEXT_PORT_STATE_DOWN    = 0x1,
+	MLX5_HCA_VPORT_CONTEXT_PORT_STATE_INIT    = 0x2,
+	MLX5_HCA_VPORT_CONTEXT_PORT_STATE_ARM     = 0x3,
+	MLX5_HCA_VPORT_CONTEXT_PORT_STATE_ACTIVE  = 0x4,
+};
+
+enum {
+	MLX5_HCA_VPORT_CONTEXT_VPORT_STATE_DOWN    = 0x1,
+	MLX5_HCA_VPORT_CONTEXT_VPORT_STATE_INIT    = 0x2,
+	MLX5_HCA_VPORT_CONTEXT_VPORT_STATE_ARM     = 0x3,
+	MLX5_HCA_VPORT_CONTEXT_VPORT_STATE_ACTIVE  = 0x4,
+};
+
+struct mlx5_ifc_hca_vport_context_bits {
+	u8         field_select[0x20];
+
+	u8         reserved_0[0xe0];
+
+	u8         sm_virt_aware[0x1];
+	u8         has_smi[0x1];
+	u8         has_raw[0x1];
+	u8         grh_required[0x1];
+	u8         reserved_1[0xc];
+	u8         port_physical_state[0x4];
+	u8         vport_state_policy[0x4];
+	u8         port_state[0x4];
+	u8         vport_state[0x4];
+
+	u8         reserved_2[0x20];
+
+	u8         system_image_guid[0x40];
+
+	u8         port_guid[0x40];
+
+	u8         node_guid[0x40];
+
+	u8         cap_mask1[0x20];
+
+	u8         cap_mask1_field_select[0x20];
+
+	u8         cap_mask2[0x20];
+
+	u8         cap_mask2_field_select[0x20];
+
+	u8         reserved_3[0x80];
+
+	u8         lid[0x10];
+	u8         reserved_4[0x4];
+	u8         init_type_reply[0x4];
+	u8         lmc[0x3];
+	u8         subnet_timeout[0x5];
+
+	u8         sm_lid[0x10];
+	u8         sm_sl[0x4];
+	u8         reserved_5[0xc];
+
+	u8         qkey_violation_counter[0x10];
+	u8         pkey_violation_counter[0x10];
+
+	u8         reserved_6[0xca0];
+};
+
+union mlx5_ifc_hca_cap_union_bits {
+	struct mlx5_ifc_cmd_hca_cap_bits cmd_hca_cap;
+	struct mlx5_ifc_odp_cap_bits odp_cap;
+	struct mlx5_ifc_atomic_caps_bits atomic_caps;
+	struct mlx5_ifc_roce_cap_bits roce_cap;
+	struct mlx5_ifc_per_protocol_networking_offload_caps_bits per_protocol_networking_offload_caps;
+	struct mlx5_ifc_flow_table_nic_cap_bits flow_table_nic_cap;
+	struct mlx5_ifc_flow_table_eswitch_cap_bits flow_table_eswitch_cap;
+	struct mlx5_ifc_e_switch_cap_bits e_switch_cap;
+	u8         reserved_0[0x8000];
+};
+
+struct mlx5_ifc_esw_vport_context_bits {
+	u8         reserved_0[0x3];
+	u8         vport_svlan_strip[0x1];
+	u8         vport_cvlan_strip[0x1];
+	u8         vport_svlan_insert[0x1];
+	u8         vport_cvlan_insert[0x2];
+	u8         reserved_1[0x18];
+
+	u8         reserved_2[0x20];
+
+	u8         svlan_cfi[0x1];
+	u8         svlan_pcp[0x3];
+	u8         svlan_id[0xc];
+	u8         cvlan_cfi[0x1];
+	u8         cvlan_pcp[0x3];
+	u8         cvlan_id[0xc];
+
+	u8         reserved_3[0x7a0];
+};
+
+enum {
+	MLX5_EQC_STATUS_OK                = 0x0,
+	MLX5_EQC_STATUS_EQ_WRITE_FAILURE  = 0xa,
+};
+
+enum {
+	MLX5_EQ_STATE_ARMED = 0x9,
+	MLX5_EQ_STATE_FIRED = 0xa,
+};
+
+struct mlx5_ifc_eqc_bits {
+	u8         status[0x4];
+	u8         reserved_0[0x9];
+	u8         ec[0x1];
+	u8         oi[0x1];
+	u8         reserved_1[0x5];
+	u8         st[0x4];
+	u8         reserved_2[0x8];
+
+	u8         reserved_3[0x20];
+
+	u8         reserved_4[0x14];
+	u8         page_offset[0x6];
+	u8         reserved_5[0x6];
+
+	u8         reserved_6[0x3];
+	u8         log_eq_size[0x5];
+	u8         uar_page[0x18];
+
+	u8         reserved_7[0x20];
+
+	u8         reserved_8[0x18];
+	u8         intr[0x8];
+
+	u8         reserved_9[0x3];
+	u8         log_page_size[0x5];
+	u8         reserved_10[0x18];
+
+	u8         reserved_11[0x60];
+
+	u8         reserved_12[0x8];
+	u8         consumer_counter[0x18];
+
+	u8         reserved_13[0x8];
+	u8         producer_counter[0x18];
+
+	u8         reserved_14[0x80];
+};
+
+enum {
+	MLX5_DCTC_STATE_ACTIVE    = 0x0,
+	MLX5_DCTC_STATE_DRAINING  = 0x1,
+	MLX5_DCTC_STATE_DRAINED   = 0x2,
+};
+
+enum {
+	MLX5_DCTC_CS_RES_DISABLE    = 0x0,
+	MLX5_DCTC_CS_RES_NA         = 0x1,
+	MLX5_DCTC_CS_RES_UP_TO_64B  = 0x2,
+};
+
+enum {
+	MLX5_DCTC_MTU_256_BYTES  = 0x1,
+	MLX5_DCTC_MTU_512_BYTES  = 0x2,
+	MLX5_DCTC_MTU_1K_BYTES   = 0x3,
+	MLX5_DCTC_MTU_2K_BYTES   = 0x4,
+	MLX5_DCTC_MTU_4K_BYTES   = 0x5,
+};
+
+struct mlx5_ifc_dctc_bits {
+	u8         reserved_0[0x4];
+	u8         state[0x4];
+	u8         reserved_1[0x18];
+
+	u8         reserved_2[0x8];
+	u8         user_index[0x18];
+
+	u8         reserved_3[0x8];
+	u8         cqn[0x18];
+
+	u8         counter_set_id[0x8];
+	u8         atomic_mode[0x4];
+	u8         rre[0x1];
+	u8         rwe[0x1];
+	u8         rae[0x1];
+	u8         atomic_like_write_en[0x1];
+	u8         latency_sensitive[0x1];
+	u8         rlky[0x1];
+	u8         reserved_4[0xe];
+
+	u8         reserved_5[0x8];
+	u8         cs_res[0x8];
+	u8         reserved_6[0x3];
+	u8         min_rnr_nak[0x5];
+	u8         reserved_7[0x8];
+
+	u8         reserved_8[0x8];
+	u8         srqn[0x18];
+
+	u8         reserved_9[0x8];
+	u8         pd[0x18];
+
+	u8         tclass[0x8];
+	u8         reserved_10[0x4];
+	u8         flow_label[0x14];
+
+	u8         dc_access_key[0x40];
+
+	u8         reserved_11[0x5];
+	u8         mtu[0x3];
+	u8         port[0x8];
+	u8         pkey_index[0x10];
+
+	u8         reserved_12[0x8];
+	u8         my_addr_index[0x8];
+	u8         reserved_13[0x8];
+	u8         hop_limit[0x8];
+
+	u8         dc_access_key_violation_count[0x20];
+
+	u8         reserved_14[0x14];
+	u8         dei_cfi[0x1];
+	u8         eth_prio[0x3];
+	u8         ecn[0x2];
+	u8         dscp[0x6];
+
+	u8         reserved_15[0x40];
+};
+
+enum {
+	MLX5_CQC_STATUS_OK             = 0x0,
+	MLX5_CQC_STATUS_CQ_OVERFLOW    = 0x9,
+	MLX5_CQC_STATUS_CQ_WRITE_FAIL  = 0xa,
+};
+
+enum {
+	CQE_SIZE_64                = 0x0,
+	CQE_SIZE_128               = 0x1,
+};
+
+enum {
+	MLX5_CQ_PERIOD_MODE_START_FROM_EQE  = 0x0,
+	MLX5_CQ_PERIOD_MODE_START_FROM_CQE  = 0x1,
+};
+
+enum {
+	MLX5_CQ_STATE_SOLICITED_ARMED                     = 0x6,
+	MLX5_CQ_STATE_ARMED                               = 0x9,
+	MLX5_CQ_STATE_FIRED                               = 0xa,
+};
+
+struct mlx5_ifc_cqc_bits {
+	u8         status[0x4];
+	u8         reserved_0[0x4];
+	u8         cqe_sz[0x3];
+	u8         cc[0x1];
+	u8         reserved_1[0x1];
+	u8         scqe_break_moderation_en[0x1];
+	u8         oi[0x1];
+	u8         cq_period_mode[0x2];
+	u8         cqe_compression_en[0x1];
+	u8         mini_cqe_res_format[0x2];
+	u8         st[0x4];
+	u8         reserved_2[0x8];
+
+	u8         reserved_3[0x20];
+
+	u8         reserved_4[0x14];
+	u8         page_offset[0x6];
+	u8         reserved_5[0x6];
+
+	u8         reserved_6[0x3];
+	u8         log_cq_size[0x5];
+	u8         uar_page[0x18];
+
+	u8         reserved_7[0x4];
+	u8         cq_period[0xc];
+	u8         cq_max_count[0x10];
+
+	u8         reserved_8[0x18];
+	u8         c_eqn[0x8];
+
+	u8         reserved_9[0x3];
+	u8         log_page_size[0x5];
+	u8         reserved_10[0x18];
+
+	u8         reserved_11[0x20];
+
+	u8         reserved_12[0x8];
+	u8         last_notified_index[0x18];
+
+	u8         reserved_13[0x8];
+	u8         last_solicit_index[0x18];
+
+	u8         reserved_14[0x8];
+	u8         consumer_counter[0x18];
+
+	u8         reserved_15[0x8];
+	u8         producer_counter[0x18];
+
+	u8         reserved_16[0x40];
+
+	u8         dbr_addr[0x40];
+};
+
+union mlx5_ifc_cong_control_roce_ecn_auto_bits {
+	struct mlx5_ifc_cong_control_802_1qau_rp_bits cong_control_802_1qau_rp;
+	struct mlx5_ifc_cong_control_r_roce_ecn_rp_bits cong_control_r_roce_ecn_rp;
+	struct mlx5_ifc_cong_control_r_roce_ecn_np_bits cong_control_r_roce_ecn_np;
+	u8         reserved_0[0x800];
+};
+
+struct mlx5_ifc_query_adapter_param_block_bits {
+	u8         reserved_0[0xc0];
+
+	u8         reserved_1[0x8];
+	u8         ieee_vendor_id[0x18];
+
+	u8         reserved_2[0x10];
+	u8         vsd_vendor_id[0x10];
+
+	u8         vsd[208][0x8];
+
+	u8         vsd_contd_psid[16][0x8];
+};
+
+union mlx5_ifc_modify_field_select_resize_field_select_auto_bits {
+	struct mlx5_ifc_modify_field_select_bits modify_field_select;
+	struct mlx5_ifc_resize_field_select_bits resize_field_select;
+	u8         reserved_0[0x20];
+};
+
+union mlx5_ifc_field_select_802_1_r_roce_auto_bits {
+	struct mlx5_ifc_field_select_802_1qau_rp_bits field_select_802_1qau_rp;
+	struct mlx5_ifc_field_select_r_roce_rp_bits field_select_r_roce_rp;
+	struct mlx5_ifc_field_select_r_roce_np_bits field_select_r_roce_np;
+	u8         reserved_0[0x20];
+};
+
+struct mlx5_ifc_bufferx_reg_bits {
+	u8         reserved_0[0x6];
+	u8         lossy[0x1];
+	u8         epsb[0x1];
+	u8         reserved_1[0xc];
+	u8         size[0xc];
+
+	u8         xoff_threshold[0x10];
+	u8         xon_threshold[0x10];
+};
+
+struct mlx5_ifc_config_item_bits {
+	u8         valid[0x2];
+	u8         reserved_0[0x2];
+	u8         header_type[0x2];
+	u8         reserved_1[0x2];
+	u8         default_location[0x1];
+	u8         reserved_2[0x7];
+	u8         version[0x4];
+	u8         reserved_3[0x3];
+	u8         length[0x9];
+
+	u8         type[0x20];
+
+	u8         reserved_4[0x10];
+	u8         crc16[0x10];
+};
+
+struct mlx5_ifc_nodnic_port_config_reg_bits {
+	struct mlx5_ifc_nodnic_event_word_bits event;
+
+	u8         network_en[0x1];
+	u8         dma_en[0x1];
+	u8         promisc_en[0x1];
+	u8         promisc_multicast_en[0x1];
+	u8         reserved_0[0x17];
+	u8         receive_filter_en[0x5];
+
+	u8         reserved_1[0x10];
+	u8         mac_47_32[0x10];
+
+	u8         mac_31_0[0x20];
+
+	u8         receive_filters_mgid_mac[64][0x8];
+
+	u8         gid[16][0x8];
+
+	u8         reserved_2[0x10];
+	u8         lid[0x10];
+
+	u8         reserved_3[0xc];
+	u8         sm_sl[0x4];
+	u8         sm_lid[0x10];
+
+	u8         completion_address_63_32[0x20];
+
+	u8         completion_address_31_12[0x14];
+	u8         reserved_4[0x6];
+	u8         log_cq_size[0x6];
+
+	u8         working_buffer_address_63_32[0x20];
+
+	u8         working_buffer_address_31_12[0x14];
+	u8         reserved_5[0xc];
+
+	struct mlx5_ifc_nodnic_cq_arming_word_bits arm_cq;
+
+	u8         pkey_index[0x10];
+	u8         pkey[0x10];
+
+	struct mlx5_ifc_nodnic_ring_config_reg_bits send_ring0;
+
+	struct mlx5_ifc_nodnic_ring_config_reg_bits send_ring1;
+
+	struct mlx5_ifc_nodnic_ring_config_reg_bits receive_ring0;
+
+	struct mlx5_ifc_nodnic_ring_config_reg_bits receive_ring1;
+
+	u8         reserved_6[0x400];
+};
+
+union mlx5_ifc_event_auto_bits {
+	struct mlx5_ifc_comp_event_bits comp_event;
+	struct mlx5_ifc_dct_events_bits dct_events;
+	struct mlx5_ifc_qp_events_bits qp_events;
+	struct mlx5_ifc_wqe_associated_page_fault_event_bits wqe_associated_page_fault_event;
+	struct mlx5_ifc_rdma_page_fault_event_bits rdma_page_fault_event;
+	struct mlx5_ifc_cq_error_bits cq_error;
+	struct mlx5_ifc_dropped_packet_logged_bits dropped_packet_logged;
+	struct mlx5_ifc_port_state_change_event_bits port_state_change_event;
+	struct mlx5_ifc_gpio_event_bits gpio_event;
+	struct mlx5_ifc_db_bf_congestion_event_bits db_bf_congestion_event;
+	struct mlx5_ifc_stall_vl_event_bits stall_vl_event;
+	struct mlx5_ifc_cmd_inter_comp_event_bits cmd_inter_comp_event;
+	struct mlx5_ifc_pages_req_event_bits pages_req_event;
+	struct mlx5_ifc_nic_vport_change_event_bits nic_vport_change_event;
+	u8         reserved_0[0xe0];
+};
+
+struct mlx5_ifc_health_buffer_bits {
+	u8         reserved_0[0x100];
+
+	u8         assert_existptr[0x20];
+
+	u8         assert_callra[0x20];
+
+	u8         reserved_1[0x40];
+
+	u8         fw_version[0x20];
+
+	u8         hw_id[0x20];
+
+	u8         reserved_2[0x20];
+
+	u8         irisc_index[0x8];
+	u8         synd[0x8];
+	u8         ext_synd[0x10];
+};
+
+struct mlx5_ifc_register_loopback_control_bits {
+	u8         no_lb[0x1];
+	u8         reserved_0[0x7];
+	u8         port[0x8];
+	u8         reserved_1[0x10];
+
+	u8         reserved_2[0x60];
+};
+
+struct mlx5_ifc_icmd_set_wol_rol_out_bits {
+	u8         reserved_0[0x40];
+
+	u8         reserved_1[0x10];
+	u8         rol_mode[0x8];
+	u8         wol_mode[0x8];
+};
+
+struct mlx5_ifc_icmd_set_wol_rol_in_bits {
+	u8         reserved_0[0x40];
+
+	u8         rol_mode_valid[0x1];
+	u8         wol_mode_valid[0x1];
+	u8         reserved_1[0xe];
+	u8         rol_mode[0x8];
+	u8         wol_mode[0x8];
+
+	u8         reserved_2[0x7a0];
+};
+
+struct mlx5_ifc_icmd_set_virtual_mac_in_bits {
+	u8         virtual_mac_en[0x1];
+	u8         mac_aux_v[0x1];
+	u8         reserved_0[0x1e];
+
+	u8         reserved_1[0x40];
+
+	struct mlx5_ifc_mac_address_layout_bits virtual_mac;
+
+	u8         reserved_2[0x760];
+};
+
+struct mlx5_ifc_icmd_query_virtual_mac_out_bits {
+	u8         virtual_mac_en[0x1];
+	u8         mac_aux_v[0x1];
+	u8         reserved_0[0x1e];
+
+	struct mlx5_ifc_mac_address_layout_bits permanent_mac;
+
+	struct mlx5_ifc_mac_address_layout_bits virtual_mac;
+
+	u8         reserved_1[0x760];
+};
+
+struct mlx5_ifc_icmd_query_fw_info_out_bits {
+	struct mlx5_ifc_fw_version_bits fw_version;
+
+	u8         reserved_0[0x10];
+	u8         hash_signature[0x10];
+
+	u8         psid[16][0x8];
+
+	u8         reserved_1[0x6e0];
+};
+
+struct mlx5_ifc_icmd_query_cap_in_bits {
+	u8         reserved_0[0x10];
+	u8         capability_group[0x10];
+};
+
+struct mlx5_ifc_icmd_query_cap_general_bits {
+	u8         nv_access[0x1];
+	u8         fw_info_psid[0x1];
+	u8         reserved_0[0x1e];
+
+	u8         reserved_1[0x16];
+	u8         rol_s[0x1];
+	u8         rol_g[0x1];
+	u8         reserved_2[0x1];
+	u8         wol_s[0x1];
+	u8         wol_g[0x1];
+	u8         wol_a[0x1];
+	u8         wol_b[0x1];
+	u8         wol_m[0x1];
+	u8         wol_u[0x1];
+	u8         wol_p[0x1];
+};
+
+struct mlx5_ifc_icmd_ocbb_query_header_stats_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         reserved_1[0x7e0];
+};
+
+struct mlx5_ifc_icmd_ocbb_query_etoc_stats_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         reserved_1[0x7e0];
+};
+
+struct mlx5_ifc_icmd_ocbb_init_in_bits {
+	u8         address_hi[0x20];
+
+	u8         address_lo[0x20];
+
+	u8         reserved_0[0x7c0];
+};
+
+struct mlx5_ifc_icmd_init_ocsd_in_bits {
+	u8         reserved_0[0x20];
+
+	u8         address_hi[0x20];
+
+	u8         address_lo[0x20];
+
+	u8         reserved_1[0x7a0];
+};
+
+struct mlx5_ifc_icmd_access_reg_out_bits {
+	u8         reserved_0[0x11];
+	u8         status[0x7];
+	u8         reserved_1[0x8];
+
+	u8         register_id[0x10];
+	u8         reserved_2[0x10];
+
+	u8         reserved_3[0x40];
+
+	u8         reserved_4[0x5];
+	u8         len[0xb];
+	u8         reserved_5[0x10];
+
+	u8         register_data[0][0x20];
+};
+
+enum {
+	MLX5_ICMD_ACCESS_REG_IN_METHOD_QUERY  = 0x1,
+	MLX5_ICMD_ACCESS_REG_IN_METHOD_WRITE  = 0x2,
+};
+
+struct mlx5_ifc_icmd_access_reg_in_bits {
+	u8         constant_1[0x5];
+	u8         constant_2[0xb];
+	u8         reserved_0[0x10];
+
+	u8         register_id[0x10];
+	u8         reserved_1[0x1];
+	u8         method[0x7];
+	u8         constant_3[0x8];
+
+	u8         reserved_2[0x40];
+
+	u8         constant_4[0x5];
+	u8         len[0xb];
+	u8         reserved_3[0x10];
+
+	u8         register_data[0][0x20];
+};
+
+struct mlx5_ifc_teardown_hca_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+enum {
+	MLX5_TEARDOWN_HCA_IN_PROFILE_GRACEFUL_CLOSE  = 0x0,
+	MLX5_TEARDOWN_HCA_IN_PROFILE_PANIC_CLOSE     = 0x1,
+};
+
+struct mlx5_ifc_teardown_hca_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x10];
+	u8         profile[0x10];
+
+	u8         reserved_3[0x20];
+};
+
+struct mlx5_ifc_suspend_qp_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_suspend_qp_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x8];
+	u8         qpn[0x18];
+
+	u8         reserved_3[0x20];
+};
+
+struct mlx5_ifc_sqerr2rts_qp_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_sqerr2rts_qp_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x8];
+	u8         qpn[0x18];
+
+	u8         reserved_3[0x20];
+
+	u8         opt_param_mask[0x20];
+
+	u8         reserved_4[0x20];
+
+	struct mlx5_ifc_qpc_bits qpc;
+
+	u8         reserved_5[0x80];
+};
+
+struct mlx5_ifc_sqd2rts_qp_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_sqd2rts_qp_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x8];
+	u8         qpn[0x18];
+
+	u8         reserved_3[0x20];
+
+	u8         opt_param_mask[0x20];
+
+	u8         reserved_4[0x20];
+
+	struct mlx5_ifc_qpc_bits qpc;
+
+	u8         reserved_5[0x80];
+};
+
+struct mlx5_ifc_snapshot_cap_bits {
+	u8         reserved_0[0x1d];
+	u8         suspend_qp_uc[0x1];
+	u8         suspend_qp_ud[0x1];
+	u8         suspend_qp_rc[0x1];
+
+	u8         reserved_1[0x1c];
+	u8         restore_pd[0x1];
+	u8         restore_uar[0x1];
+	u8         restore_mkey[0x1];
+	u8         restore_qp[0x1];
+
+	u8         reserved_2[0x1e];
+	u8         named_mkey[0x1];
+	u8         named_qp[0x1];
+
+	u8         reserved_3[0x7a0];
+};
+
+struct mlx5_ifc_set_wol_rol_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_set_wol_rol_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         rol_mode_valid[0x1];
+	u8         wol_mode_valid[0x1];
+	u8         reserved_2[0xe];
+	u8         rol_mode[0x8];
+	u8         wol_mode[0x8];
+
+	u8         reserved_3[0x20];
+};
+
+struct mlx5_ifc_set_roce_address_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_set_roce_address_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         roce_address_index[0x10];
+	u8         reserved_2[0x10];
+
+	u8         reserved_3[0x20];
+
+	struct mlx5_ifc_roce_addr_layout_bits roce_address;
+};
+
+struct mlx5_ifc_set_rdb_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_set_rdb_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x8];
+	u8         qpn[0x18];
+
+	u8         reserved_3[0x18];
+	u8         rdb_list_size[0x8];
+
+	struct mlx5_ifc_rdbc_bits rdb_context[0];
+};
+
+struct mlx5_ifc_set_mad_demux_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+enum {
+	MLX5_SET_MAD_DEMUX_IN_DEMUX_MODE_PASS_ALL   = 0x0,
+	MLX5_SET_MAD_DEMUX_IN_DEMUX_MODE_SELECTIVE  = 0x2,
+};
+
+struct mlx5_ifc_set_mad_demux_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x20];
+
+	u8         reserved_3[0x6];
+	u8         demux_mode[0x2];
+	u8         reserved_4[0x18];
+};
+
+struct mlx5_ifc_set_l2_table_entry_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_set_l2_table_entry_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x60];
+
+	u8         reserved_3[0x8];
+	u8         table_index[0x18];
+
+	u8         reserved_4[0x20];
+
+	u8         reserved_5[0x13];
+	u8         vlan_valid[0x1];
+	u8         vlan[0xc];
+
+	struct mlx5_ifc_mac_address_layout_bits mac_address;
+
+	u8         reserved_6[0xc0];
+};
+
+struct mlx5_ifc_set_issi_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_set_issi_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x10];
+	u8         current_issi[0x10];
+
+	u8         reserved_3[0x20];
+};
+
+struct mlx5_ifc_set_hca_cap_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_set_hca_cap_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x40];
+
+	union mlx5_ifc_hca_cap_union_bits capability;
+};
+
+struct mlx5_ifc_set_flow_table_root_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_set_flow_table_root_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         other_vport[0x1];
+	u8         reserved_2[0xf];
+	u8         vport_number[0x10];
+
+	u8         reserved_3[0x20];
+
+	u8         table_type[0x8];
+	u8         reserved_4[0x18];
+
+	u8         reserved_5[0x8];
+	u8         table_id[0x18];
+
+	u8         reserved_6[0x140];
+};
+
+struct mlx5_ifc_set_fte_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_set_fte_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         other_vport[0x1];
+	u8         reserved_2[0xf];
+	u8         vport_number[0x10];
+
+	u8         reserved_3[0x20];
+
+	u8         table_type[0x8];
+	u8         reserved_4[0x18];
+
+	u8         reserved_5[0x8];
+	u8         table_id[0x18];
+
+	u8         reserved_6[0x18];
+	u8         modify_enable_mask[0x8];
+
+	u8         reserved_7[0x20];
+
+	u8         flow_index[0x20];
+
+	u8         reserved_8[0xe0];
+
+	struct mlx5_ifc_flow_context_bits flow_context;
+};
+
+struct mlx5_ifc_set_driver_version_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_set_driver_version_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x40];
+
+	u8         driver_version[64][0x8];
+};
+
+struct mlx5_ifc_set_dc_cnak_trace_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_set_dc_cnak_trace_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         enable[0x1];
+	u8         reserved_2[0x1f];
+
+	u8         reserved_3[0x160];
+
+	struct mlx5_ifc_cmd_pas_bits pas;
+};
+
+struct mlx5_ifc_set_burst_size_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_set_burst_size_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x20];
+
+	u8         reserved_3[0x9];
+	u8         device_burst_size[0x17];
+};
+
+struct mlx5_ifc_rts2rts_qp_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_rts2rts_qp_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x8];
+	u8         qpn[0x18];
+
+	u8         reserved_3[0x20];
+
+	u8         opt_param_mask[0x20];
+
+	u8         reserved_4[0x20];
+
+	struct mlx5_ifc_qpc_bits qpc;
+
+	u8         reserved_5[0x80];
+};
+
+struct mlx5_ifc_rtr2rts_qp_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_rtr2rts_qp_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x8];
+	u8         qpn[0x18];
+
+	u8         reserved_3[0x20];
+
+	u8         opt_param_mask[0x20];
+
+	u8         reserved_4[0x20];
+
+	struct mlx5_ifc_qpc_bits qpc;
+
+	u8         reserved_5[0x80];
+};
+
+struct mlx5_ifc_rst2init_qp_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_rst2init_qp_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x8];
+	u8         qpn[0x18];
+
+	u8         reserved_3[0x20];
+
+	u8         opt_param_mask[0x20];
+
+	u8         reserved_4[0x20];
+
+	struct mlx5_ifc_qpc_bits qpc;
+
+	u8         reserved_5[0x80];
+};
+
+struct mlx5_ifc_resume_qp_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_resume_qp_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x8];
+	u8         qpn[0x18];
+
+	u8         reserved_3[0x20];
+};
+
+struct mlx5_ifc_query_xrc_srq_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+
+	struct mlx5_ifc_xrc_srqc_bits xrc_srq_context_entry;
+
+	u8         reserved_2[0x600];
+
+	u8         pas[0][0x40];
+};
+
+struct mlx5_ifc_query_xrc_srq_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x8];
+	u8         xrc_srqn[0x18];
+
+	u8         reserved_3[0x20];
+};
+
+struct mlx5_ifc_query_wol_rol_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x10];
+	u8         rol_mode[0x8];
+	u8         wol_mode[0x8];
+
+	u8         reserved_2[0x20];
+};
+
+struct mlx5_ifc_query_wol_rol_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x40];
+};
+
+enum {
+	MLX5_QUERY_VPORT_STATE_OUT_STATE_DOWN  = 0x0,
+	MLX5_QUERY_VPORT_STATE_OUT_STATE_UP    = 0x1,
+};
+
+struct mlx5_ifc_query_vport_state_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x20];
+
+	u8         reserved_2[0x18];
+	u8         admin_state[0x4];
+	u8         state[0x4];
+};
+
+enum {
+	MLX5_QUERY_VPORT_STATE_IN_OP_MOD_VNIC_VPORT  = 0x0,
+	MLX5_QUERY_VPORT_STATE_IN_OP_MOD_ESW_VPORT   = 0x1,
+	MLX5_QUERY_VPORT_STATE_IN_OP_MOD_UPLINK      = 0x2,
+};
+
+struct mlx5_ifc_query_vport_state_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         other_vport[0x1];
+	u8         reserved_2[0xf];
+	u8         vport_number[0x10];
+
+	u8         reserved_3[0x20];
+};
+
+struct mlx5_ifc_query_vport_counter_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+
+	struct mlx5_ifc_traffic_counter_bits received_errors;
+
+	struct mlx5_ifc_traffic_counter_bits transmit_errors;
+
+	struct mlx5_ifc_traffic_counter_bits received_ib_unicast;
+
+	struct mlx5_ifc_traffic_counter_bits transmitted_ib_unicast;
+
+	struct mlx5_ifc_traffic_counter_bits received_ib_multicast;
+
+	struct mlx5_ifc_traffic_counter_bits transmitted_ib_multicast;
+
+	struct mlx5_ifc_traffic_counter_bits received_eth_broadcast;
+
+	struct mlx5_ifc_traffic_counter_bits transmitted_eth_broadcast;
+
+	struct mlx5_ifc_traffic_counter_bits received_eth_unicast;
+
+	struct mlx5_ifc_traffic_counter_bits transmitted_eth_unicast;
+
+	struct mlx5_ifc_traffic_counter_bits received_eth_multicast;
+
+	struct mlx5_ifc_traffic_counter_bits transmitted_eth_multicast;
+
+	u8         reserved_2[0xa00];
+};
+
+enum {
+	MLX5_QUERY_VPORT_COUNTER_IN_OP_MOD_VPORT_COUNTERS  = 0x0,
+};
+
+struct mlx5_ifc_query_vport_counter_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         other_vport[0x1];
+	u8         reserved_2[0xb];
+	u8         port_num[0x4];
+	u8         vport_number[0x10];
+
+	u8         reserved_3[0x60];
+
+	u8         clear[0x1];
+	u8         reserved_4[0x1f];
+
+	u8         reserved_5[0x20];
+};
+
+struct mlx5_ifc_query_tis_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+
+	struct mlx5_ifc_tisc_bits tis_context;
+};
+
+struct mlx5_ifc_query_tis_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x8];
+	u8         tisn[0x18];
+
+	u8         reserved_3[0x20];
+};
+
+struct mlx5_ifc_query_tir_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0xc0];
+
+	struct mlx5_ifc_tirc_bits tir_context;
+};
+
+struct mlx5_ifc_query_tir_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x8];
+	u8         tirn[0x18];
+
+	u8         reserved_3[0x20];
+};
+
+struct mlx5_ifc_query_srq_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+
+	struct mlx5_ifc_srqc_bits srq_context_entry;
+
+	u8         reserved_2[0x600];
+
+	u8         pas[0][0x40];
+};
+
+struct mlx5_ifc_query_srq_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x8];
+	u8         srqn[0x18];
+
+	u8         reserved_3[0x20];
+};
+
+struct mlx5_ifc_query_sq_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0xc0];
+
+	struct mlx5_ifc_sqc_bits sq_context;
+};
+
+struct mlx5_ifc_query_sq_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x8];
+	u8         sqn[0x18];
+
+	u8         reserved_3[0x20];
+};
+
+struct mlx5_ifc_query_special_contexts_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x20];
+
+	u8         resd_lkey[0x20];
+};
+
+struct mlx5_ifc_query_special_contexts_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x40];
+};
+
+struct mlx5_ifc_query_rqt_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0xc0];
+
+	struct mlx5_ifc_rqtc_bits rqt_context;
+};
+
+struct mlx5_ifc_query_rqt_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x8];
+	u8         rqtn[0x18];
+
+	u8         reserved_3[0x20];
+};
+
+struct mlx5_ifc_query_rq_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0xc0];
+
+	struct mlx5_ifc_rqc_bits rq_context;
+};
+
+struct mlx5_ifc_query_rq_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x8];
+	u8         rqn[0x18];
+
+	u8         reserved_3[0x20];
+};
+
+struct mlx5_ifc_query_roce_address_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+
+	struct mlx5_ifc_roce_addr_layout_bits roce_address;
+};
+
+struct mlx5_ifc_query_roce_address_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         roce_address_index[0x10];
+	u8         reserved_2[0x10];
+
+	u8         reserved_3[0x20];
+};
+
+struct mlx5_ifc_query_rmp_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0xc0];
+
+	struct mlx5_ifc_rmpc_bits rmp_context;
+};
+
+struct mlx5_ifc_query_rmp_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x8];
+	u8         rmpn[0x18];
+
+	u8         reserved_3[0x20];
+};
+
+struct mlx5_ifc_query_rdb_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x20];
+
+	u8         reserved_2[0x18];
+	u8         rdb_list_size[0x8];
+
+	struct mlx5_ifc_rdbc_bits rdb_context[0];
+};
+
+struct mlx5_ifc_query_rdb_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x8];
+	u8         qpn[0x18];
+
+	u8         reserved_3[0x20];
+};
+
+struct mlx5_ifc_query_qp_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+
+	u8         opt_param_mask[0x20];
+
+	u8         reserved_2[0x20];
+
+	struct mlx5_ifc_qpc_bits qpc;
+
+	u8         reserved_3[0x80];
+
+	u8         pas[0][0x40];
+};
+
+struct mlx5_ifc_query_qp_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x8];
+	u8         qpn[0x18];
+
+	u8         reserved_3[0x20];
+};
+
+struct mlx5_ifc_query_q_counter_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+
+	u8         rx_write_requests[0x20];
+
+	u8         reserved_2[0x20];
+
+	u8         rx_read_requests[0x20];
+
+	u8         reserved_3[0x20];
+
+	u8         rx_atomic_requests[0x20];
+
+	u8         reserved_4[0x20];
+
+	u8         rx_dct_connect[0x20];
+
+	u8         reserved_5[0x20];
+
+	u8         out_of_buffer[0x20];
+
+	u8         reserved_6[0x20];
+
+	u8         out_of_sequence[0x20];
+
+	u8         reserved_7[0x620];
+};
+
+struct mlx5_ifc_query_q_counter_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x80];
+
+	u8         clear[0x1];
+	u8         reserved_3[0x1f];
+
+	u8         reserved_4[0x18];
+	u8         counter_set_id[0x8];
+};
+
+struct mlx5_ifc_query_pages_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x10];
+	u8         function_id[0x10];
+
+	u8         num_pages[0x20];
+};
+
+enum {
+	MLX5_BOOT_PAGES                           = 0x1,
+	MLX5_INIT_PAGES                           = 0x2,
+	MLX5_POST_INIT_PAGES                      = 0x3,
+};
+
+struct mlx5_ifc_query_pages_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x10];
+	u8         function_id[0x10];
+
+	u8         reserved_3[0x20];
+};
+
+struct mlx5_ifc_query_nic_vport_context_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+
+	struct mlx5_ifc_nic_vport_context_bits nic_vport_context;
+};
+
+struct mlx5_ifc_query_nic_vport_context_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         other_vport[0x1];
+	u8         reserved_2[0xf];
+	u8         vport_number[0x10];
+
+	u8         reserved_3[0x5];
+	u8         allowed_list_type[0x3];
+	u8         reserved_4[0x18];
+};
+
+struct mlx5_ifc_query_mkey_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+
+	struct mlx5_ifc_mkc_bits memory_key_mkey_entry;
+
+	u8         reserved_2[0x600];
+
+	u8         bsf0_klm0_pas_mtt0_1[16][0x8];
+
+	u8         bsf1_klm1_pas_mtt2_3[16][0x8];
+};
+
+struct mlx5_ifc_query_mkey_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x8];
+	u8         mkey_index[0x18];
+
+	u8         pg_access[0x1];
+	u8         reserved_3[0x1f];
+};
+
+struct mlx5_ifc_query_mad_demux_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+
+	u8         mad_dumux_parameters_block[0x20];
+};
+
+struct mlx5_ifc_query_mad_demux_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x40];
+};
+
+struct mlx5_ifc_query_l2_table_entry_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0xa0];
+
+	u8         reserved_2[0x13];
+	u8         vlan_valid[0x1];
+	u8         vlan[0xc];
+
+	struct mlx5_ifc_mac_address_layout_bits mac_address;
+
+	u8         reserved_3[0xc0];
+};
+
+struct mlx5_ifc_query_l2_table_entry_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x60];
+
+	u8         reserved_3[0x8];
+	u8         table_index[0x18];
+
+	u8         reserved_4[0x140];
+};
+
+struct mlx5_ifc_query_issi_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x10];
+	u8         current_issi[0x10];
+
+	u8         reserved_2[0xa0];
+
+	u8         supported_issi_reserved[76][0x8];
+	u8         supported_issi_dw0[0x20];
+};
+
+struct mlx5_ifc_query_issi_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x40];
+};
+
+struct mlx5_ifc_query_hca_vport_pkey_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+
+	struct mlx5_ifc_pkey_bits pkey[0];
+};
+
+struct mlx5_ifc_query_hca_vport_pkey_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         other_vport[0x1];
+	u8         reserved_2[0xb];
+	u8         port_num[0x4];
+	u8         vport_number[0x10];
+
+	u8         reserved_3[0x10];
+	u8         pkey_index[0x10];
+};
+
+struct mlx5_ifc_query_hca_vport_gid_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x20];
+
+	u8         gids_num[0x10];
+	u8         reserved_2[0x10];
+
+	struct mlx5_ifc_array128_auto_bits gid[0];
+};
+
+struct mlx5_ifc_query_hca_vport_gid_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         other_vport[0x1];
+	u8         reserved_2[0xb];
+	u8         port_num[0x4];
+	u8         vport_number[0x10];
+
+	u8         reserved_3[0x10];
+	u8         gid_index[0x10];
+};
+
+struct mlx5_ifc_query_hca_vport_context_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+
+	struct mlx5_ifc_hca_vport_context_bits hca_vport_context;
+};
+
+struct mlx5_ifc_query_hca_vport_context_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         other_vport[0x1];
+	u8         reserved_2[0xb];
+	u8         port_num[0x4];
+	u8         vport_number[0x10];
+
+	u8         reserved_3[0x20];
+};
+
+struct mlx5_ifc_query_hca_cap_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+
+	union mlx5_ifc_hca_cap_union_bits capability;
+};
+
+struct mlx5_ifc_query_hca_cap_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x40];
+};
+
+struct mlx5_ifc_query_flow_table_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x80];
+
+	u8         reserved_2[0x8];
+	u8         level[0x8];
+	u8         reserved_3[0x8];
+	u8         log_size[0x8];
+
+	u8         reserved_4[0x120];
+};
+
+struct mlx5_ifc_query_flow_table_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         other_vport[0x1];
+	u8         reserved_2[0xf];
+	u8         vport_number[0x10];
+
+	u8         reserved_3[0x20];
+
+	u8         table_type[0x8];
+	u8         reserved_4[0x18];
+
+	u8         reserved_5[0x8];
+	u8         table_id[0x18];
+
+	u8         reserved_6[0x140];
+};
+
+struct mlx5_ifc_query_fte_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x1c0];
+
+	struct mlx5_ifc_flow_context_bits flow_context;
+};
+
+struct mlx5_ifc_query_fte_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         other_vport[0x1];
+	u8         reserved_2[0xf];
+	u8         vport_number[0x10];
+
+	u8         reserved_3[0x20];
+
+	u8         table_type[0x8];
+	u8         reserved_4[0x18];
+
+	u8         reserved_5[0x8];
+	u8         table_id[0x18];
+
+	u8         reserved_6[0x40];
+
+	u8         flow_index[0x20];
+
+	u8         reserved_7[0xe0];
+};
+
+enum {
+	MLX5_QUERY_FLOW_GROUP_OUT_MATCH_CRITERIA_ENABLE_OUTER_HEADERS    = 0x0,
+	MLX5_QUERY_FLOW_GROUP_OUT_MATCH_CRITERIA_ENABLE_MISC_PARAMETERS  = 0x1,
+	MLX5_QUERY_FLOW_GROUP_OUT_MATCH_CRITERIA_ENABLE_INNER_HEADERS    = 0x2,
+};
+
+struct mlx5_ifc_query_flow_group_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0xa0];
+
+	u8         start_flow_index[0x20];
+
+	u8         reserved_2[0x20];
+
+	u8         end_flow_index[0x20];
+
+	u8         reserved_3[0xa0];
+
+	u8         reserved_4[0x18];
+	u8         match_criteria_enable[0x8];
+
+	struct mlx5_ifc_fte_match_param_bits match_criteria;
+
+	u8         reserved_5[0xe00];
+};
+
+struct mlx5_ifc_query_flow_group_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         other_vport[0x1];
+	u8         reserved_2[0xf];
+	u8         vport_number[0x10];
+
+	u8         reserved_3[0x20];
+
+	u8         table_type[0x8];
+	u8         reserved_4[0x18];
+
+	u8         reserved_5[0x8];
+	u8         table_id[0x18];
+
+	u8         group_id[0x20];
+
+	u8         reserved_6[0x120];
+};
+
+struct mlx5_ifc_query_flow_counter_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+
+	struct mlx5_ifc_traffic_counter_bits flow_statistics;
+
+	u8         reserved_2[0x700];
+};
+
+struct mlx5_ifc_query_flow_counter_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x80];
+
+	u8         clear[0x1];
+	u8         reserved_3[0x1f];
+
+	u8         reserved_4[0x10];
+	u8         flow_counter_id[0x10];
+};
+
+struct mlx5_ifc_query_esw_vport_context_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+
+	struct mlx5_ifc_esw_vport_context_bits esw_vport_context;
+};
+
+struct mlx5_ifc_query_esw_vport_context_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         other_vport[0x1];
+	u8         reserved_2[0xf];
+	u8         vport_number[0x10];
+
+	u8         reserved_3[0x20];
+};
+
+struct mlx5_ifc_query_eq_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+
+	struct mlx5_ifc_eqc_bits eq_context_entry;
+
+	u8         reserved_2[0x40];
+
+	u8         event_bitmask[0x40];
+
+	u8         reserved_3[0x580];
+
+	u8         pas[0][0x40];
+};
+
+struct mlx5_ifc_query_eq_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x18];
+	u8         eq_number[0x8];
+
+	u8         reserved_3[0x20];
+};
+
+struct mlx5_ifc_query_dct_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+
+	struct mlx5_ifc_dctc_bits dct_context_entry;
+
+	u8         reserved_2[0x180];
+};
+
+struct mlx5_ifc_query_dct_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x8];
+	u8         dctn[0x18];
+
+	u8         reserved_3[0x20];
+};
+
+struct mlx5_ifc_query_dc_cnak_trace_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         enable[0x1];
+	u8         reserved_1[0x1f];
+
+	u8         reserved_2[0x160];
+
+	struct mlx5_ifc_cmd_pas_bits pas;
+};
+
+struct mlx5_ifc_query_dc_cnak_trace_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x40];
+};
+
+struct mlx5_ifc_query_cq_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+
+	struct mlx5_ifc_cqc_bits cq_context;
+
+	u8         reserved_2[0x600];
+
+	u8         pas[0][0x40];
+};
+
+struct mlx5_ifc_query_cq_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x8];
+	u8         cqn[0x18];
+
+	u8         reserved_3[0x20];
+};
+
+struct mlx5_ifc_query_cong_status_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x20];
+
+	u8         enable[0x1];
+	u8         tag_enable[0x1];
+	u8         reserved_2[0x1e];
+};
+
+struct mlx5_ifc_query_cong_status_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x18];
+	u8         priority[0x4];
+	u8         cong_protocol[0x4];
+
+	u8         reserved_3[0x20];
+};
+
+struct mlx5_ifc_query_cong_statistics_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+
+	u8         cur_flows[0x20];
+
+	u8         sum_flows[0x20];
+
+	u8         cnp_ignored_high[0x20];
+
+	u8         cnp_ignored_low[0x20];
+
+	u8         cnp_handled_high[0x20];
+
+	u8         cnp_handled_low[0x20];
+
+	u8         reserved_2[0x100];
+
+	u8         time_stamp_high[0x20];
+
+	u8         time_stamp_low[0x20];
+
+	u8         accumulators_period[0x20];
+
+	u8         ecn_marked_roce_packets_high[0x20];
+
+	u8         ecn_marked_roce_packets_low[0x20];
+
+	u8         cnps_sent_high[0x20];
+
+	u8         cnps_sent_low[0x20];
+
+	u8         reserved_3[0x560];
+};
+
+struct mlx5_ifc_query_cong_statistics_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         clear[0x1];
+	u8         reserved_2[0x1f];
+
+	u8         reserved_3[0x20];
+};
+
+struct mlx5_ifc_query_cong_params_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+
+	union mlx5_ifc_cong_control_roce_ecn_auto_bits congestion_parameters;
+};
+
+struct mlx5_ifc_query_cong_params_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x1c];
+	u8         cong_protocol[0x4];
+
+	u8         reserved_3[0x20];
+};
+
+struct mlx5_ifc_query_burst_size_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x20];
+
+	u8         reserved_2[0x9];
+	u8         device_burst_size[0x17];
+};
+
+struct mlx5_ifc_query_burst_size_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x40];
+};
+
+struct mlx5_ifc_query_adapter_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+
+	struct mlx5_ifc_query_adapter_param_block_bits query_adapter_struct;
+};
+
+struct mlx5_ifc_query_adapter_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x40];
+};
+
+struct mlx5_ifc_qp_2rst_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_qp_2rst_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x8];
+	u8         qpn[0x18];
+
+	u8         reserved_3[0x20];
+};
+
+struct mlx5_ifc_qp_2err_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_qp_2err_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x8];
+	u8         qpn[0x18];
+
+	u8         reserved_3[0x20];
+};
+
+struct mlx5_ifc_page_fault_resume_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_page_fault_resume_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         error[0x1];
+	u8         reserved_2[0x4];
+	u8         rdma[0x1];
+	u8         read_write[0x1];
+	u8         req_res[0x1];
+	u8         qpn[0x18];
+
+	u8         reserved_3[0x20];
+};
+
+struct mlx5_ifc_nop_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_nop_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x40];
+};
+
+struct mlx5_ifc_modify_vport_state_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+enum {
+	MLX5_MODIFY_VPORT_STATE_IN_OP_MOD_NIC_VPORT  = 0x0,
+	MLX5_MODIFY_VPORT_STATE_IN_OP_MOD_ESW_VPORT  = 0x1,
+	MLX5_MODIFY_VPORT_STATE_IN_OP_MOD_UPLINK     = 0x2,
+};
+
+enum {
+	MLX5_MODIFY_VPORT_STATE_IN_ADMIN_STATE_DOWN    = 0x0,
+	MLX5_MODIFY_VPORT_STATE_IN_ADMIN_STATE_UP      = 0x1,
+	MLX5_MODIFY_VPORT_STATE_IN_ADMIN_STATE_FOLLOW  = 0x2,
+};
+
+struct mlx5_ifc_modify_vport_state_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         other_vport[0x1];
+	u8         reserved_2[0xf];
+	u8         vport_number[0x10];
+
+	u8         reserved_3[0x18];
+	u8         admin_state[0x4];
+	u8         reserved_4[0x4];
+};
+
+struct mlx5_ifc_modify_tis_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_modify_tis_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x8];
+	u8         tisn[0x18];
+
+	u8         reserved_3[0x20];
+
+	u8         modify_bitmask[0x40];
+
+	u8         reserved_4[0x40];
+
+	struct mlx5_ifc_tisc_bits ctx;
+};
+
+struct mlx5_ifc_modify_tir_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_modify_tir_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x8];
+	u8         tirn[0x18];
+
+	u8         reserved_3[0x20];
+
+	u8         modify_bitmask[0x40];
+
+	u8         reserved_4[0x40];
+
+	struct mlx5_ifc_tirc_bits tir_context;
+};
+
+struct mlx5_ifc_modify_sq_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_modify_sq_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         sq_state[0x4];
+	u8         reserved_2[0x4];
+	u8         sqn[0x18];
+
+	u8         reserved_3[0x20];
+
+	u8         modify_bitmask[0x40];
+
+	u8         reserved_4[0x40];
+
+	struct mlx5_ifc_sqc_bits ctx;
+};
+
+struct mlx5_ifc_modify_rqt_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_modify_rqt_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x8];
+	u8         rqtn[0x18];
+
+	u8         reserved_3[0x20];
+
+	u8         modify_bitmask[0x40];
+
+	u8         reserved_4[0x40];
+
+	struct mlx5_ifc_rqtc_bits ctx;
+};
+
+struct mlx5_ifc_modify_rq_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_modify_rq_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         rq_state[0x4];
+	u8         reserved_2[0x4];
+	u8         rqn[0x18];
+
+	u8         reserved_3[0x20];
+
+	u8         modify_bitmask[0x40];
+
+	u8         reserved_4[0x40];
+
+	struct mlx5_ifc_rqc_bits ctx;
+};
+
+struct mlx5_ifc_modify_rmp_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_rmp_bitmask_bits {
+	u8	   reserved[0x20];
+
+	u8         reserved1[0x1f];
+	u8         lwm[0x1];
+};
+
+struct mlx5_ifc_modify_rmp_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         rmp_state[0x4];
+	u8         reserved_2[0x4];
+	u8         rmpn[0x18];
+
+	u8         reserved_3[0x20];
+
+	struct mlx5_ifc_rmp_bitmask_bits bitmask;
+
+	u8         reserved_4[0x40];
+
+	struct mlx5_ifc_rmpc_bits ctx;
+};
+
+struct mlx5_ifc_modify_nic_vport_context_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_modify_nic_vport_field_select_bits {
+	u8         reserved_0[0x18];
+	u8         min_wqe_inline_mode[0x1];
+	u8         mtu[0x1];
+	u8         change_event[0x1];
+	u8         promisc[0x1];
+	u8         permanent_address[0x1];
+	u8         addresses_list[0x1];
+	u8         roce_en[0x1];
+	u8         reserved_1[0x1];
+};
+
+struct mlx5_ifc_modify_nic_vport_context_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         other_vport[0x1];
+	u8         reserved_2[0xf];
+	u8         vport_number[0x10];
+
+	struct mlx5_ifc_modify_nic_vport_field_select_bits field_select;
+
+	u8         reserved_3[0x780];
+
+	struct mlx5_ifc_nic_vport_context_bits nic_vport_context;
+};
+
+struct mlx5_ifc_modify_hca_vport_context_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_modify_hca_vport_context_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         other_vport[0x1];
+	u8         reserved_2[0xb];
+	u8         port_num[0x4];
+	u8         vport_number[0x10];
+
+	u8         reserved_3[0x20];
+
+	struct mlx5_ifc_hca_vport_context_bits hca_vport_context;
+};
+
+struct mlx5_ifc_modify_esw_vport_context_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_modify_esw_vport_context_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         other_vport[0x1];
+	u8         reserved_2[0xf];
+	u8         vport_number[0x10];
+
+	u8         field_select[0x20];
+
+	struct mlx5_ifc_esw_vport_context_bits esw_vport_context;
+};
+
+struct mlx5_ifc_modify_cq_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+enum {
+	MLX5_MODIFY_CQ_IN_OP_MOD_MODIFY_CQ  = 0x0,
+	MLX5_MODIFY_CQ_IN_OP_MOD_RESIZE_CQ  = 0x1,
+};
+
+struct mlx5_ifc_modify_cq_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x8];
+	u8         cqn[0x18];
+
+	union mlx5_ifc_modify_field_select_resize_field_select_auto_bits modify_field_select_resize_field_select;
+
+	struct mlx5_ifc_cqc_bits cq_context;
+
+	u8         reserved_3[0x600];
+
+	u8         pas[0][0x40];
+};
+
+struct mlx5_ifc_modify_cong_status_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_modify_cong_status_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x18];
+	u8         priority[0x4];
+	u8         cong_protocol[0x4];
+
+	u8         enable[0x1];
+	u8         tag_enable[0x1];
+	u8         reserved_3[0x1e];
+};
+
+struct mlx5_ifc_modify_cong_params_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_modify_cong_params_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x1c];
+	u8         cong_protocol[0x4];
+
+	union mlx5_ifc_field_select_802_1_r_roce_auto_bits field_select;
+
+	u8         reserved_3[0x80];
+
+	union mlx5_ifc_cong_control_roce_ecn_auto_bits congestion_parameters;
+};
+
+struct mlx5_ifc_manage_pages_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         output_num_entries[0x20];
+
+	u8         reserved_1[0x20];
+
+	u8         pas[0][0x40];
+};
+
+enum {
+	MLX5_PAGES_CANT_GIVE                            = 0x0,
+	MLX5_PAGES_GIVE                                 = 0x1,
+	MLX5_PAGES_TAKE                                 = 0x2,
+};
+
+struct mlx5_ifc_manage_pages_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x10];
+	u8         function_id[0x10];
+
+	u8         input_num_entries[0x20];
+
+	u8         pas[0][0x40];
+};
+
+struct mlx5_ifc_mad_ifc_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+
+	u8         response_mad_packet[256][0x8];
+};
+
+struct mlx5_ifc_mad_ifc_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         remote_lid[0x10];
+	u8         reserved_2[0x8];
+	u8         port[0x8];
+
+	u8         reserved_3[0x20];
+
+	u8         mad[256][0x8];
+};
+
+struct mlx5_ifc_init_hca_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+enum {
+	MLX5_INIT_HCA_IN_OP_MOD_INIT      = 0x0,
+	MLX5_INIT_HCA_IN_OP_MOD_PRE_INIT  = 0x1,
+};
+
+struct mlx5_ifc_init_hca_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x40];
+};
+
+struct mlx5_ifc_init2rtr_qp_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_init2rtr_qp_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x8];
+	u8         qpn[0x18];
+
+	u8         reserved_3[0x20];
+
+	u8         opt_param_mask[0x20];
+
+	u8         reserved_4[0x20];
+
+	struct mlx5_ifc_qpc_bits qpc;
+
+	u8         reserved_5[0x80];
+};
+
+struct mlx5_ifc_init2init_qp_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_init2init_qp_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x8];
+	u8         qpn[0x18];
+
+	u8         reserved_3[0x20];
+
+	u8         opt_param_mask[0x20];
+
+	u8         reserved_4[0x20];
+
+	struct mlx5_ifc_qpc_bits qpc;
+
+	u8         reserved_5[0x80];
+};
+
+struct mlx5_ifc_get_dropped_packet_log_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+
+	u8         packet_headers_log[128][0x8];
+
+	u8         packet_syndrome[64][0x8];
+};
+
+struct mlx5_ifc_get_dropped_packet_log_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x40];
+};
+
+struct mlx5_ifc_gen_eqe_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x18];
+	u8         eq_number[0x8];
+
+	u8         reserved_3[0x20];
+
+	u8         eqe[64][0x8];
+};
+
+struct mlx5_ifc_gen_eq_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_enable_hca_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x20];
+};
+
+struct mlx5_ifc_enable_hca_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x10];
+	u8         function_id[0x10];
+
+	u8         reserved_3[0x20];
+};
+
+struct mlx5_ifc_drain_dct_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_drain_dct_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x8];
+	u8         dctn[0x18];
+
+	u8         reserved_3[0x20];
+};
+
+struct mlx5_ifc_disable_hca_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x20];
+};
+
+struct mlx5_ifc_disable_hca_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x10];
+	u8         function_id[0x10];
+
+	u8         reserved_3[0x20];
+};
+
+struct mlx5_ifc_detach_from_mcg_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_detach_from_mcg_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x8];
+	u8         qpn[0x18];
+
+	u8         reserved_3[0x20];
+
+	u8         multicast_gid[16][0x8];
+};
+
+struct mlx5_ifc_destroy_xrc_srq_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_destroy_xrc_srq_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x8];
+	u8         xrc_srqn[0x18];
+
+	u8         reserved_3[0x20];
+};
+
+struct mlx5_ifc_destroy_tis_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_destroy_tis_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x8];
+	u8         tisn[0x18];
+
+	u8         reserved_3[0x20];
+};
+
+struct mlx5_ifc_destroy_tir_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_destroy_tir_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x8];
+	u8         tirn[0x18];
+
+	u8         reserved_3[0x20];
+};
+
+struct mlx5_ifc_destroy_srq_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_destroy_srq_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x8];
+	u8         srqn[0x18];
+
+	u8         reserved_3[0x20];
+};
+
+struct mlx5_ifc_destroy_sq_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_destroy_sq_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x8];
+	u8         sqn[0x18];
+
+	u8         reserved_3[0x20];
+};
+
+struct mlx5_ifc_destroy_rqt_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_destroy_rqt_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x8];
+	u8         rqtn[0x18];
+
+	u8         reserved_3[0x20];
+};
+
+struct mlx5_ifc_destroy_rq_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_destroy_rq_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x8];
+	u8         rqn[0x18];
+
+	u8         reserved_3[0x20];
+};
+
+struct mlx5_ifc_destroy_rmp_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_destroy_rmp_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x8];
+	u8         rmpn[0x18];
+
+	u8         reserved_3[0x20];
+};
+
+struct mlx5_ifc_destroy_qp_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_destroy_qp_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x8];
+	u8         qpn[0x18];
+
+	u8         reserved_3[0x20];
+};
+
+struct mlx5_ifc_destroy_psv_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_destroy_psv_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x8];
+	u8         psvn[0x18];
+
+	u8         reserved_3[0x20];
+};
+
+struct mlx5_ifc_destroy_mkey_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_destroy_mkey_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x8];
+	u8         mkey_index[0x18];
+
+	u8         reserved_3[0x20];
+};
+
+struct mlx5_ifc_destroy_flow_table_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_destroy_flow_table_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         other_vport[0x1];
+	u8         reserved_2[0xf];
+	u8         vport_number[0x10];
+
+	u8         reserved_3[0x20];
+
+	u8         table_type[0x8];
+	u8         reserved_4[0x18];
+
+	u8         reserved_5[0x8];
+	u8         table_id[0x18];
+
+	u8         reserved_6[0x140];
+};
+
+struct mlx5_ifc_destroy_flow_group_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_destroy_flow_group_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         other_vport[0x1];
+	u8         reserved_2[0xf];
+	u8         vport_number[0x10];
+
+	u8         reserved_3[0x20];
+
+	u8         table_type[0x8];
+	u8         reserved_4[0x18];
+
+	u8         reserved_5[0x8];
+	u8         table_id[0x18];
+
+	u8         group_id[0x20];
+
+	u8         reserved_6[0x120];
+};
+
+struct mlx5_ifc_destroy_eq_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_destroy_eq_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x18];
+	u8         eq_number[0x8];
+
+	u8         reserved_3[0x20];
+};
+
+struct mlx5_ifc_destroy_dct_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_destroy_dct_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x8];
+	u8         dctn[0x18];
+
+	u8         reserved_3[0x20];
+};
+
+struct mlx5_ifc_destroy_cq_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_destroy_cq_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x8];
+	u8         cqn[0x18];
+
+	u8         reserved_3[0x20];
+};
+
+struct mlx5_ifc_delete_vxlan_udp_dport_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_delete_vxlan_udp_dport_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x20];
+
+	u8         reserved_3[0x10];
+	u8         vxlan_udp_port[0x10];
+};
+
+struct mlx5_ifc_delete_l2_table_entry_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_delete_l2_table_entry_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x60];
+
+	u8         reserved_3[0x8];
+	u8         table_index[0x18];
+
+	u8         reserved_4[0x140];
+};
+
+struct mlx5_ifc_delete_fte_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_delete_fte_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         other_vport[0x1];
+	u8         reserved_2[0xf];
+	u8         vport_number[0x10];
+
+	u8         reserved_3[0x20];
+
+	u8         table_type[0x8];
+	u8         reserved_4[0x18];
+
+	u8         reserved_5[0x8];
+	u8         table_id[0x18];
+
+	u8         reserved_6[0x40];
+
+	u8         flow_index[0x20];
+
+	u8         reserved_7[0xe0];
+};
+
+struct mlx5_ifc_dealloc_xrcd_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_dealloc_xrcd_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x8];
+	u8         xrcd[0x18];
+
+	u8         reserved_3[0x20];
+};
+
+struct mlx5_ifc_dealloc_uar_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_dealloc_uar_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x8];
+	u8         uar[0x18];
+
+	u8         reserved_3[0x20];
+};
+
+struct mlx5_ifc_dealloc_transport_domain_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_dealloc_transport_domain_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x8];
+	u8         transport_domain[0x18];
+
+	u8         reserved_3[0x20];
+};
+
+struct mlx5_ifc_dealloc_q_counter_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_dealloc_q_counter_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x18];
+	u8         counter_set_id[0x8];
+
+	u8         reserved_3[0x20];
+};
+
+struct mlx5_ifc_dealloc_pd_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_dealloc_pd_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x8];
+	u8         pd[0x18];
+
+	u8         reserved_3[0x20];
+};
+
+struct mlx5_ifc_dealloc_flow_counter_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_dealloc_flow_counter_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x10];
+	u8         flow_counter_id[0x10];
+
+	u8         reserved_3[0x20];
+};
+
+struct mlx5_ifc_deactivate_tracer_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_deactivate_tracer_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         mkey[0x20];
+
+	u8         reserved_2[0x20];
+};
+
+struct mlx5_ifc_create_xrc_srq_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x8];
+	u8         xrc_srqn[0x18];
+
+	u8         reserved_2[0x20];
+};
+
+struct mlx5_ifc_create_xrc_srq_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x40];
+
+	struct mlx5_ifc_xrc_srqc_bits xrc_srq_context_entry;
+
+	u8         reserved_3[0x600];
+
+	u8         pas[0][0x40];
+};
+
+struct mlx5_ifc_create_tis_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x8];
+	u8         tisn[0x18];
+
+	u8         reserved_2[0x20];
+};
+
+struct mlx5_ifc_create_tis_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0xc0];
+
+	struct mlx5_ifc_tisc_bits ctx;
+};
+
+struct mlx5_ifc_create_tir_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x8];
+	u8         tirn[0x18];
+
+	u8         reserved_2[0x20];
+};
+
+struct mlx5_ifc_create_tir_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0xc0];
+
+	struct mlx5_ifc_tirc_bits tir_context;
+};
+
+struct mlx5_ifc_create_srq_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x8];
+	u8         srqn[0x18];
+
+	u8         reserved_2[0x20];
+};
+
+struct mlx5_ifc_create_srq_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x40];
+
+	struct mlx5_ifc_srqc_bits srq_context_entry;
+
+	u8         reserved_3[0x600];
+
+	u8         pas[0][0x40];
+};
+
+struct mlx5_ifc_create_sq_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x8];
+	u8         sqn[0x18];
+
+	u8         reserved_2[0x20];
+};
+
+struct mlx5_ifc_create_sq_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0xc0];
+
+	struct mlx5_ifc_sqc_bits ctx;
+};
+
+struct mlx5_ifc_create_rqt_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x8];
+	u8         rqtn[0x18];
+
+	u8         reserved_2[0x20];
+};
+
+struct mlx5_ifc_create_rqt_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0xc0];
+
+	struct mlx5_ifc_rqtc_bits rqt_context;
+};
+
+struct mlx5_ifc_create_rq_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x8];
+	u8         rqn[0x18];
+
+	u8         reserved_2[0x20];
+};
+
+struct mlx5_ifc_create_rq_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0xc0];
+
+	struct mlx5_ifc_rqc_bits ctx;
+};
+
+struct mlx5_ifc_create_rmp_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x8];
+	u8         rmpn[0x18];
+
+	u8         reserved_2[0x20];
+};
+
+struct mlx5_ifc_create_rmp_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0xc0];
+
+	struct mlx5_ifc_rmpc_bits ctx;
+};
+
+struct mlx5_ifc_create_qp_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x8];
+	u8         qpn[0x18];
+
+	u8         reserved_2[0x20];
+};
+
+struct mlx5_ifc_create_qp_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x40];
+
+	u8         opt_param_mask[0x20];
+
+	u8         reserved_3[0x20];
+
+	struct mlx5_ifc_qpc_bits qpc;
+
+	u8         reserved_4[0x80];
+
+	u8         pas[0][0x40];
+};
+
+struct mlx5_ifc_create_psv_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+
+	u8         reserved_2[0x8];
+	u8         psv0_index[0x18];
+
+	u8         reserved_3[0x8];
+	u8         psv1_index[0x18];
+
+	u8         reserved_4[0x8];
+	u8         psv2_index[0x18];
+
+	u8         reserved_5[0x8];
+	u8         psv3_index[0x18];
+};
+
+struct mlx5_ifc_create_psv_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         num_psv[0x4];
+	u8         reserved_2[0x4];
+	u8         pd[0x18];
+
+	u8         reserved_3[0x20];
+};
+
+struct mlx5_ifc_create_mkey_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x8];
+	u8         mkey_index[0x18];
+
+	u8         reserved_2[0x20];
+};
+
+struct mlx5_ifc_create_mkey_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x20];
+
+	u8         pg_access[0x1];
+	u8         reserved_3[0x1f];
+
+	struct mlx5_ifc_mkc_bits memory_key_mkey_entry;
+
+	u8         reserved_4[0x80];
+
+	u8         translations_octword_actual_size[0x20];
+
+	u8         reserved_5[0x560];
+
+	u8         klm_pas_mtt[0][0x20];
+};
+
+struct mlx5_ifc_create_flow_table_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x8];
+	u8         table_id[0x18];
+
+	u8         reserved_2[0x20];
+};
+
+struct mlx5_ifc_create_flow_table_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         other_vport[0x1];
+	u8         reserved_2[0xf];
+	u8         vport_number[0x10];
+
+	u8         reserved_3[0x20];
+
+	u8         table_type[0x8];
+	u8         reserved_4[0x18];
+
+	u8         reserved_5[0x20];
+
+	u8         reserved_6[0x8];
+	u8         level[0x8];
+	u8         reserved_7[0x8];
+	u8         log_size[0x8];
+
+	u8         reserved_8[0x120];
+};
+
+struct mlx5_ifc_create_flow_group_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x8];
+	u8         group_id[0x18];
+
+	u8         reserved_2[0x20];
+};
+
+enum {
+	MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_OUTER_HEADERS    = 0x0,
+	MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_MISC_PARAMETERS  = 0x1,
+	MLX5_CREATE_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_INNER_HEADERS    = 0x2,
+};
+
+struct mlx5_ifc_create_flow_group_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         other_vport[0x1];
+	u8         reserved_2[0xf];
+	u8         vport_number[0x10];
+
+	u8         reserved_3[0x20];
+
+	u8         table_type[0x8];
+	u8         reserved_4[0x18];
+
+	u8         reserved_5[0x8];
+	u8         table_id[0x18];
+
+	u8         reserved_6[0x20];
+
+	u8         start_flow_index[0x20];
+
+	u8         reserved_7[0x20];
+
+	u8         end_flow_index[0x20];
+
+	u8         reserved_8[0xa0];
+
+	u8         reserved_9[0x18];
+	u8         match_criteria_enable[0x8];
+
+	struct mlx5_ifc_fte_match_param_bits match_criteria;
+
+	u8         reserved_10[0xe00];
+};
+
+struct mlx5_ifc_create_eq_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x18];
+	u8         eq_number[0x8];
+
+	u8         reserved_2[0x20];
+};
+
+struct mlx5_ifc_create_eq_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x40];
+
+	struct mlx5_ifc_eqc_bits eq_context_entry;
+
+	u8         reserved_3[0x40];
+
+	u8         event_bitmask[0x40];
+
+	u8         reserved_4[0x580];
+
+	u8         pas[0][0x40];
+};
+
+struct mlx5_ifc_create_dct_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x8];
+	u8         dctn[0x18];
+
+	u8         reserved_2[0x20];
+};
+
+struct mlx5_ifc_create_dct_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x40];
+
+	struct mlx5_ifc_dctc_bits dct_context_entry;
+
+	u8         reserved_3[0x180];
+};
+
+struct mlx5_ifc_create_cq_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x8];
+	u8         cqn[0x18];
+
+	u8         reserved_2[0x20];
+};
+
+struct mlx5_ifc_create_cq_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x40];
+
+	struct mlx5_ifc_cqc_bits cq_context;
+
+	u8         reserved_3[0x600];
+
+	u8         pas[0][0x40];
+};
+
+struct mlx5_ifc_config_int_moderation_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x4];
+	u8         min_delay[0xc];
+	u8         int_vector[0x10];
+
+	u8         reserved_2[0x20];
+};
+
+enum {
+	MLX5_CONFIG_INT_MODERATION_IN_OP_MOD_WRITE  = 0x0,
+	MLX5_CONFIG_INT_MODERATION_IN_OP_MOD_READ   = 0x1,
+};
+
+struct mlx5_ifc_config_int_moderation_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x4];
+	u8         min_delay[0xc];
+	u8         int_vector[0x10];
+
+	u8         reserved_3[0x20];
+};
+
+struct mlx5_ifc_attach_to_mcg_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_attach_to_mcg_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x8];
+	u8         qpn[0x18];
+
+	u8         reserved_3[0x20];
+
+	u8         multicast_gid[16][0x8];
+};
+
+struct mlx5_ifc_arm_xrc_srq_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+enum {
+	MLX5_ARM_XRC_SRQ_IN_OP_MOD_XRC_SRQ  = 0x1,
+};
+
+struct mlx5_ifc_arm_xrc_srq_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x8];
+	u8         xrc_srqn[0x18];
+
+	u8         reserved_3[0x10];
+	u8         lwm[0x10];
+};
+
+struct mlx5_ifc_arm_rq_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+enum {
+	MLX5_ARM_RQ_IN_OP_MOD_SRQ  = 0x1,
+};
+
+struct mlx5_ifc_arm_rq_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x8];
+	u8         srq_number[0x18];
+
+	u8         reserved_3[0x10];
+	u8         lwm[0x10];
+};
+
+struct mlx5_ifc_arm_dct_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_arm_dct_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x8];
+	u8         dctn[0x18];
+
+	u8         reserved_3[0x20];
+};
+
+struct mlx5_ifc_alloc_xrcd_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x8];
+	u8         xrcd[0x18];
+
+	u8         reserved_2[0x20];
+};
+
+struct mlx5_ifc_alloc_xrcd_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x40];
+};
+
+struct mlx5_ifc_alloc_uar_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x8];
+	u8         uar[0x18];
+
+	u8         reserved_2[0x20];
+};
+
+struct mlx5_ifc_alloc_uar_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x40];
+};
+
+struct mlx5_ifc_alloc_transport_domain_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x8];
+	u8         transport_domain[0x18];
+
+	u8         reserved_2[0x20];
+};
+
+struct mlx5_ifc_alloc_transport_domain_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x40];
+};
+
+struct mlx5_ifc_alloc_q_counter_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x18];
+	u8         counter_set_id[0x8];
+
+	u8         reserved_2[0x20];
+};
+
+struct mlx5_ifc_alloc_q_counter_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x40];
+};
+
+struct mlx5_ifc_alloc_pd_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x8];
+	u8         pd[0x18];
+
+	u8         reserved_2[0x20];
+};
+
+struct mlx5_ifc_alloc_pd_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x40];
+};
+
+struct mlx5_ifc_alloc_flow_counter_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x10];
+	u8         flow_counter_id[0x10];
+
+	u8         reserved_2[0x20];
+};
+
+struct mlx5_ifc_alloc_flow_counter_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x40];
+};
+
+struct mlx5_ifc_add_vxlan_udp_dport_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_add_vxlan_udp_dport_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x20];
+
+	u8         reserved_3[0x10];
+	u8         vxlan_udp_port[0x10];
+};
+
+struct mlx5_ifc_activate_tracer_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+};
+
+struct mlx5_ifc_activate_tracer_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         mkey[0x20];
+
+	u8         reserved_2[0x20];
+};
+
+struct mlx5_ifc_access_register_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         reserved_1[0x40];
+
+	u8         register_data[0][0x20];
+};
+
+enum {
+	MLX5_ACCESS_REGISTER_IN_OP_MOD_WRITE  = 0x0,
+	MLX5_ACCESS_REGISTER_IN_OP_MOD_READ   = 0x1,
+};
+
+struct mlx5_ifc_access_register_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         reserved_2[0x10];
+	u8         register_id[0x10];
+
+	u8         argument[0x20];
+
+	u8         register_data[0][0x20];
+};
+
+struct mlx5_ifc_sltp_reg_bits {
+	u8         status[0x4];
+	u8         version[0x4];
+	u8         local_port[0x8];
+	u8         pnat[0x2];
+	u8         reserved_0[0x2];
+	u8         lane[0x4];
+	u8         reserved_1[0x8];
+
+	u8         reserved_2[0x20];
+
+	u8         reserved_3[0x7];
+	u8         polarity[0x1];
+	u8         ob_tap0[0x8];
+	u8         ob_tap1[0x8];
+	u8         ob_tap2[0x8];
+
+	u8         reserved_4[0xc];
+	u8         ob_preemp_mode[0x4];
+	u8         ob_reg[0x8];
+	u8         ob_bias[0x8];
+
+	u8         reserved_5[0x20];
+};
+
+struct mlx5_ifc_slrp_reg_bits {
+	u8         status[0x4];
+	u8         version[0x4];
+	u8         local_port[0x8];
+	u8         pnat[0x2];
+	u8         reserved_0[0x2];
+	u8         lane[0x4];
+	u8         reserved_1[0x8];
+
+	u8         ib_sel[0x2];
+	u8         reserved_2[0x11];
+	u8         dp_sel[0x1];
+	u8         dp90sel[0x4];
+	u8         mix90phase[0x8];
+
+	u8         ffe_tap0[0x8];
+	u8         ffe_tap1[0x8];
+	u8         ffe_tap2[0x8];
+	u8         ffe_tap3[0x8];
+
+	u8         ffe_tap4[0x8];
+	u8         ffe_tap5[0x8];
+	u8         ffe_tap6[0x8];
+	u8         ffe_tap7[0x8];
+
+	u8         ffe_tap8[0x8];
+	u8         mixerbias_tap_amp[0x8];
+	u8         reserved_3[0x7];
+	u8         ffe_tap_en[0x9];
+
+	u8         ffe_tap_offset0[0x8];
+	u8         ffe_tap_offset1[0x8];
+	u8         slicer_offset0[0x10];
+
+	u8         mixer_offset0[0x10];
+	u8         mixer_offset1[0x10];
+
+	u8         mixerbgn_inp[0x8];
+	u8         mixerbgn_inn[0x8];
+	u8         mixerbgn_refp[0x8];
+	u8         mixerbgn_refn[0x8];
+
+	u8         sel_slicer_lctrl_h[0x1];
+	u8         sel_slicer_lctrl_l[0x1];
+	u8         reserved_4[0x1];
+	u8         ref_mixer_vreg[0x5];
+	u8         slicer_gctrl[0x8];
+	u8         lctrl_input[0x8];
+	u8         mixer_offset_cm1[0x8];
+
+	u8         common_mode[0x6];
+	u8         reserved_5[0x1];
+	u8         mixer_offset_cm0[0x9];
+	u8         reserved_6[0x7];
+	u8         slicer_offset_cm[0x9];
+};
+
+struct mlx5_ifc_slrg_reg_bits {
+	u8         status[0x4];
+	u8         version[0x4];
+	u8         local_port[0x8];
+	u8         pnat[0x2];
+	u8         reserved_0[0x2];
+	u8         lane[0x4];
+	u8         reserved_1[0x8];
+
+	u8         time_to_link_up[0x10];
+	u8         reserved_2[0xc];
+	u8         grade_lane_speed[0x4];
+
+	u8         grade_version[0x8];
+	u8         grade[0x18];
+
+	u8         reserved_3[0x4];
+	u8         height_grade_type[0x4];
+	u8         height_grade[0x18];
+
+	u8         height_dz[0x10];
+	u8         height_dv[0x10];
+
+	u8         reserved_4[0x10];
+	u8         height_sigma[0x10];
+
+	u8         reserved_5[0x20];
+
+	u8         reserved_6[0x4];
+	u8         phase_grade_type[0x4];
+	u8         phase_grade[0x18];
+
+	u8         reserved_7[0x8];
+	u8         phase_eo_pos[0x8];
+	u8         reserved_8[0x8];
+	u8         phase_eo_neg[0x8];
+
+	u8         ffe_set_tested[0x10];
+	u8         test_errors_per_lane[0x10];
+};
+
+struct mlx5_ifc_pvlc_reg_bits {
+	u8         reserved_0[0x8];
+	u8         local_port[0x8];
+	u8         reserved_1[0x10];
+
+	u8         reserved_2[0x1c];
+	u8         vl_hw_cap[0x4];
+
+	u8         reserved_3[0x1c];
+	u8         vl_admin[0x4];
+
+	u8         reserved_4[0x1c];
+	u8         vl_operational[0x4];
+};
+
+struct mlx5_ifc_pude_reg_bits {
+	u8         swid[0x8];
+	u8         local_port[0x8];
+	u8         reserved_0[0x4];
+	u8         admin_status[0x4];
+	u8         reserved_1[0x4];
+	u8         oper_status[0x4];
+
+	u8         reserved_2[0x60];
+};
+
+enum {
+	MLX5_PTYS_REG_PROTO_MASK_INFINIBAND  = 0x1,
+	MLX5_PTYS_REG_PROTO_MASK_ETHERNET    = 0x4,
+};
+
+struct mlx5_ifc_ptys_reg_bits {
+	u8         reserved_0[0x8];
+	u8         local_port[0x8];
+	u8         reserved_1[0xd];
+	u8         proto_mask[0x3];
+
+	u8         reserved_2[0x40];
+
+	u8         eth_proto_capability[0x20];
+
+	u8         ib_link_width_capability[0x10];
+	u8         ib_proto_capability[0x10];
+
+	u8         reserved_3[0x20];
+
+	u8         eth_proto_admin[0x20];
+
+	u8         ib_link_width_admin[0x10];
+	u8         ib_proto_admin[0x10];
+
+	u8         reserved_4[0x20];
+
+	u8         eth_proto_oper[0x20];
+
+	u8         ib_link_width_oper[0x10];
+	u8         ib_proto_oper[0x10];
+
+	u8         reserved_5[0x20];
+
+	u8         eth_proto_lp_advertise[0x20];
+
+	u8         reserved_6[0x60];
+};
+
+struct mlx5_ifc_ptas_reg_bits {
+	u8         reserved_0[0x20];
+
+	u8         algorithm_options[0x10];
+	u8         reserved_1[0x4];
+	u8         repetitions_mode[0x4];
+	u8         num_of_repetitions[0x8];
+
+	u8         grade_version[0x8];
+	u8         height_grade_type[0x4];
+	u8         phase_grade_type[0x4];
+	u8         height_grade_weight[0x8];
+	u8         phase_grade_weight[0x8];
+
+	u8         gisim_measure_bits[0x10];
+	u8         adaptive_tap_measure_bits[0x10];
+
+	u8         ber_bath_high_error_threshold[0x10];
+	u8         ber_bath_mid_error_threshold[0x10];
+
+	u8         ber_bath_low_error_threshold[0x10];
+	u8         one_ratio_high_threshold[0x10];
+
+	u8         one_ratio_high_mid_threshold[0x10];
+	u8         one_ratio_low_mid_threshold[0x10];
+
+	u8         one_ratio_low_threshold[0x10];
+	u8         ndeo_error_threshold[0x10];
+
+	u8         mixer_offset_step_size[0x10];
+	u8         reserved_2[0x8];
+	u8         mix90_phase_for_voltage_bath[0x8];
+
+	u8         mixer_offset_start[0x10];
+	u8         mixer_offset_end[0x10];
+
+	u8         reserved_3[0x15];
+	u8         ber_test_time[0xb];
+};
+
+struct mlx5_ifc_pspa_reg_bits {
+	u8         swid[0x8];
+	u8         local_port[0x8];
+	u8         sub_port[0x8];
+	u8         reserved_0[0x8];
+
+	u8         reserved_1[0x20];
+};
+
+struct mlx5_ifc_ppsc_reg_bits {
+	u8         reserved_0[0x8];
+	u8         local_port[0x8];
+	u8         reserved_1[0x10];
+
+	u8         reserved_2[0x60];
+
+	u8         reserved_3[0x1c];
+	u8         wrps_admin[0x4];
+
+	u8         reserved_4[0x1c];
+	u8         wrps_status[0x4];
+
+	u8         up_th_vld[0x1];
+	u8         down_th_vld[0x1];
+	u8         reserved_5[0x6];
+	u8         up_threshold[0x8];
+	u8         reserved_6[0x8];
+	u8         down_threshold[0x8];
+
+	u8         reserved_7[0x20];
+
+	u8         reserved_8[0x1c];
+	u8         srps_admin[0x4];
+
+	u8         reserved_9[0x60];
+};
+
+struct mlx5_ifc_pplr_reg_bits {
+	u8         reserved_0[0x8];
+	u8         local_port[0x8];
+	u8         reserved_1[0x10];
+
+	u8         reserved_2[0x8];
+	u8         lb_cap[0x8];
+	u8         reserved_3[0x8];
+	u8         lb_en[0x8];
+};
+
+struct mlx5_ifc_pplm_reg_bits {
+	u8         reserved_0[0x8];
+	u8         local_port[0x8];
+	u8         reserved_1[0x10];
+
+	u8         reserved_2[0x20];
+
+	u8         port_profile_mode[0x8];
+	u8         static_port_profile[0x8];
+	u8         active_port_profile[0x8];
+	u8         reserved_3[0x8];
+
+	u8         retransmission_active[0x8];
+	u8         fec_mode_active[0x18];
+
+	u8         reserved_4[0x10];
+	u8         v_100g_fec_override_cap[0x4];
+	u8         v_50g_fec_override_cap[0x4];
+	u8         v_25g_fec_override_cap[0x4];
+	u8         v_10g_40g_fec_override_cap[0x4];
+
+	u8         reserved_5[0x10];
+	u8         v_100g_fec_override_admin[0x4];
+	u8         v_50g_fec_override_admin[0x4];
+	u8         v_25g_fec_override_admin[0x4];
+	u8         v_10g_40g_fec_override_admin[0x4];
+};
+
+struct mlx5_ifc_ppll_reg_bits {
+	u8         num_pll_groups[0x8];
+	u8         pll_group[0x8];
+	u8         reserved_0[0x4];
+	u8         num_plls[0x4];
+	u8         reserved_1[0x8];
+
+	u8         reserved_2[0x1f];
+	u8         ae[0x1];
+
+	u8         pll_status[4][0x40];
+};
+
+struct mlx5_ifc_ppad_reg_bits {
+	u8         reserved_0[0x3];
+	u8         single_mac[0x1];
+	u8         reserved_1[0x4];
+	u8         local_port[0x8];
+	u8         mac_47_32[0x10];
+
+	u8         mac_31_0[0x20];
+
+	u8         reserved_2[0x40];
+};
+
+struct mlx5_ifc_pmtu_reg_bits {
+	u8         reserved_0[0x8];
+	u8         local_port[0x8];
+	u8         reserved_1[0x10];
+
+	u8         max_mtu[0x10];
+	u8         reserved_2[0x10];
+
+	u8         admin_mtu[0x10];
+	u8         reserved_3[0x10];
+
+	u8         oper_mtu[0x10];
+	u8         reserved_4[0x10];
+};
+
+struct mlx5_ifc_pmpr_reg_bits {
+	u8         reserved_0[0x8];
+	u8         module[0x8];
+	u8         reserved_1[0x10];
+
+	u8         reserved_2[0x18];
+	u8         attenuation_5g[0x8];
+
+	u8         reserved_3[0x18];
+	u8         attenuation_7g[0x8];
+
+	u8         reserved_4[0x18];
+	u8         attenuation_12g[0x8];
+};
+
+struct mlx5_ifc_pmpe_reg_bits {
+	u8         reserved_0[0x8];
+	u8         module[0x8];
+	u8         reserved_1[0xc];
+	u8         module_status[0x4];
+
+	u8         reserved_2[0x14];
+	u8         error_type[0x4];
+	u8         reserved_3[0x8];
+
+	u8         reserved_4[0x40];
+};
+
+struct mlx5_ifc_pmpc_reg_bits {
+	u8         module_state_updated[32][0x8];
+};
+
+struct mlx5_ifc_pmlpn_reg_bits {
+	u8         reserved_0[0x4];
+	u8         mlpn_status[0x4];
+	u8         local_port[0x8];
+	u8         reserved_1[0x10];
+
+	u8         e[0x1];
+	u8         reserved_2[0x1f];
+};
+
+struct mlx5_ifc_pmlp_reg_bits {
+	u8         rxtx[0x1];
+	u8         reserved_0[0x7];
+	u8         local_port[0x8];
+	u8         reserved_1[0x8];
+	u8         width[0x8];
+
+	u8         lane0_module_mapping[0x20];
+
+	u8         lane1_module_mapping[0x20];
+
+	u8         lane2_module_mapping[0x20];
+
+	u8         lane3_module_mapping[0x20];
+
+	u8         reserved_2[0x160];
+};
+
+struct mlx5_ifc_pmaos_reg_bits {
+	u8         reserved_0[0x8];
+	u8         module[0x8];
+	u8         reserved_1[0x4];
+	u8         admin_status[0x4];
+	u8         reserved_2[0x4];
+	u8         oper_status[0x4];
+
+	u8         ase[0x1];
+	u8         ee[0x1];
+	u8         reserved_3[0x12];
+	u8         error_type[0x4];
+	u8         reserved_4[0x6];
+	u8         e[0x2];
+
+	u8         reserved_5[0x40];
+};
+
+struct mlx5_ifc_plpc_reg_bits {
+	u8         reserved_0[0x4];
+	u8         profile_id[0xc];
+	u8         reserved_1[0x4];
+	u8         proto_mask[0x4];
+	u8         reserved_2[0x8];
+
+	u8         reserved_3[0x10];
+	u8         lane_speed[0x10];
+
+	u8         reserved_4[0x17];
+	u8         lpbf[0x1];
+	u8         fec_mode_policy[0x8];
+
+	u8         retransmission_capability[0x8];
+	u8         fec_mode_capability[0x18];
+
+	u8         retransmission_support_admin[0x8];
+	u8         fec_mode_support_admin[0x18];
+
+	u8         retransmission_request_admin[0x8];
+	u8         fec_mode_request_admin[0x18];
+
+	u8         reserved_5[0x80];
+};
+
+struct mlx5_ifc_pll_status_data_bits {
+	u8         reserved_0[0x1];
+	u8         lock_cal[0x1];
+	u8         lock_status[0x2];
+	u8         reserved_1[0x2];
+	u8         algo_f_ctrl[0xa];
+	u8         analog_algo_num_var[0x6];
+	u8         f_ctrl_measure[0xa];
+
+	u8         reserved_2[0x2];
+	u8         analog_var[0x6];
+	u8         reserved_3[0x2];
+	u8         high_var[0x6];
+	u8         reserved_4[0x2];
+	u8         low_var[0x6];
+	u8         reserved_5[0x2];
+	u8         mid_val[0x6];
+};
+
+struct mlx5_ifc_plib_reg_bits {
+	u8         reserved_0[0x8];
+	u8         local_port[0x8];
+	u8         reserved_1[0x8];
+	u8         ib_port[0x8];
+
+	u8         reserved_2[0x60];
+};
+
+struct mlx5_ifc_plbf_reg_bits {
+	u8         reserved_0[0x8];
+	u8         local_port[0x8];
+	u8         reserved_1[0xd];
+	u8         lbf_mode[0x3];
+
+	u8         reserved_2[0x20];
+};
+
+struct mlx5_ifc_pipg_reg_bits {
+	u8         reserved_0[0x8];
+	u8         local_port[0x8];
+	u8         reserved_1[0x10];
+
+	u8         dic[0x1];
+	u8         reserved_2[0x19];
+	u8         ipg[0x4];
+	u8         reserved_3[0x2];
+};
+
+struct mlx5_ifc_pifr_reg_bits {
+	u8         reserved_0[0x8];
+	u8         local_port[0x8];
+	u8         reserved_1[0x10];
+
+	u8         reserved_2[0xe0];
+
+	u8         port_filter[8][0x20];
+
+	u8         port_filter_update_en[8][0x20];
+};
+
+struct mlx5_ifc_phys_layer_cntrs_bits {
+	u8         time_since_last_clear_high[0x20];
+
+	u8         time_since_last_clear_low[0x20];
+
+	u8         symbol_errors_high[0x20];
+
+	u8         symbol_errors_low[0x20];
+
+	u8         sync_headers_errors_high[0x20];
+
+	u8         sync_headers_errors_low[0x20];
+
+	u8         edpl_bip_errors_lane0_high[0x20];
+
+	u8         edpl_bip_errors_lane0_low[0x20];
+
+	u8         edpl_bip_errors_lane1_high[0x20];
+
+	u8         edpl_bip_errors_lane1_low[0x20];
+
+	u8         edpl_bip_errors_lane2_high[0x20];
+
+	u8         edpl_bip_errors_lane2_low[0x20];
+
+	u8         edpl_bip_errors_lane3_high[0x20];
+
+	u8         edpl_bip_errors_lane3_low[0x20];
+
+	u8         fc_fec_corrected_blocks_lane0_high[0x20];
+
+	u8         fc_fec_corrected_blocks_lane0_low[0x20];
+
+	u8         fc_fec_corrected_blocks_lane1_high[0x20];
+
+	u8         fc_fec_corrected_blocks_lane1_low[0x20];
+
+	u8         fc_fec_corrected_blocks_lane2_high[0x20];
+
+	u8         fc_fec_corrected_blocks_lane2_low[0x20];
+
+	u8         fc_fec_corrected_blocks_lane3_high[0x20];
+
+	u8         fc_fec_corrected_blocks_lane3_low[0x20];
+
+	u8         fc_fec_uncorrectable_blocks_lane0_high[0x20];
+
+	u8         fc_fec_uncorrectable_blocks_lane0_low[0x20];
+
+	u8         fc_fec_uncorrectable_blocks_lane1_high[0x20];
+
+	u8         fc_fec_uncorrectable_blocks_lane1_low[0x20];
+
+	u8         fc_fec_uncorrectable_blocks_lane2_high[0x20];
+
+	u8         fc_fec_uncorrectable_blocks_lane2_low[0x20];
+
+	u8         fc_fec_uncorrectable_blocks_lane3_high[0x20];
+
+	u8         fc_fec_uncorrectable_blocks_lane3_low[0x20];
+
+	u8         rs_fec_corrected_blocks_high[0x20];
+
+	u8         rs_fec_corrected_blocks_low[0x20];
+
+	u8         rs_fec_uncorrectable_blocks_high[0x20];
+
+	u8         rs_fec_uncorrectable_blocks_low[0x20];
+
+	u8         rs_fec_no_errors_blocks_high[0x20];
+
+	u8         rs_fec_no_errors_blocks_low[0x20];
+
+	u8         rs_fec_single_error_blocks_high[0x20];
+
+	u8         rs_fec_single_error_blocks_low[0x20];
+
+	u8         rs_fec_corrected_symbols_total_high[0x20];
+
+	u8         rs_fec_corrected_symbols_total_low[0x20];
+
+	u8         rs_fec_corrected_symbols_lane0_high[0x20];
+
+	u8         rs_fec_corrected_symbols_lane0_low[0x20];
+
+	u8         rs_fec_corrected_symbols_lane1_high[0x20];
+
+	u8         rs_fec_corrected_symbols_lane1_low[0x20];
+
+	u8         rs_fec_corrected_symbols_lane2_high[0x20];
+
+	u8         rs_fec_corrected_symbols_lane2_low[0x20];
+
+	u8         rs_fec_corrected_symbols_lane3_high[0x20];
+
+	u8         rs_fec_corrected_symbols_lane3_low[0x20];
+
+	u8         link_down_events[0x20];
+
+	u8         successful_recovery_events[0x20];
+
+	u8         reserved_0[0x180];
+};
+
+struct mlx5_ifc_phrr_reg_bits {
+	u8         clr[0x1];
+	u8         reserved_0[0x7];
+	u8         local_port[0x8];
+	u8         reserved_1[0x10];
+
+	u8         hist_group[0x8];
+	u8         reserved_2[0x10];
+	u8         hist_id[0x8];
+
+	u8         reserved_3[0x40];
+
+	u8         time_since_last_clear_high[0x20];
+
+	u8         time_since_last_clear_low[0x20];
+
+	u8         bin[10][0x20];
+};
+
+struct mlx5_ifc_phbr_for_prio_reg_bits {
+	u8         reserved_0[0x18];
+	u8         prio[0x8];
+};
+
+struct mlx5_ifc_phbr_for_port_tclass_reg_bits {
+	u8         reserved_0[0x18];
+	u8         tclass[0x8];
+};
+
+struct mlx5_ifc_phbr_binding_reg_bits {
+	u8         opcode[0x4];
+	u8         reserved_0[0x4];
+	u8         local_port[0x8];
+	u8         pnat[0x2];
+	u8         reserved_1[0xe];
+
+	u8         hist_group[0x8];
+	u8         reserved_2[0x10];
+	u8         hist_id[0x8];
+
+	u8         reserved_3[0x10];
+	u8         hist_type[0x10];
+
+	u8         hist_parameters[0x20];
+
+	u8         hist_min_value[0x20];
+
+	u8         hist_max_value[0x20];
+
+	u8         sample_time[0x20];
+};
+
+enum {
+	MLX5_PFCC_REG_PPAN_DISABLED  = 0x0,
+	MLX5_PFCC_REG_PPAN_ENABLED   = 0x1,
+};
+
+struct mlx5_ifc_pfcc_reg_bits {
+	u8         reserved_0[0x8];
+	u8         local_port[0x8];
+	u8         pnat[0x2];
+	u8         reserved_1[0xc];
+	u8         shl_cap[0x1];
+	u8         shl_opr[0x1];
+
+	u8         ppan[0x4];
+	u8         reserved_2[0x4];
+	u8         prio_mask_tx[0x8];
+	u8         reserved_3[0x8];
+	u8         prio_mask_rx[0x8];
+
+	u8         pptx[0x1];
+	u8         aptx[0x1];
+	u8         reserved_4[0x6];
+	u8         pfctx[0x8];
+	u8         reserved_5[0x10];
+
+	u8         pprx[0x1];
+	u8         aprx[0x1];
+	u8         reserved_6[0x6];
+	u8         pfcrx[0x8];
+	u8         reserved_7[0x10];
+
+	u8         reserved_8[0x80];
+};
+
+struct mlx5_ifc_pelc_reg_bits {
+	u8         op[0x4];
+	u8         reserved_0[0x4];
+	u8         local_port[0x8];
+	u8         reserved_1[0x10];
+
+	u8         op_admin[0x8];
+	u8         op_capability[0x8];
+	u8         op_request[0x8];
+	u8         op_active[0x8];
+
+	u8         admin[0x40];
+
+	u8         capability[0x40];
+
+	u8         request[0x40];
+
+	u8         active[0x40];
+
+	u8         reserved_2[0x80];
+};
+
+struct mlx5_ifc_peir_reg_bits {
+	u8         reserved_0[0x8];
+	u8         local_port[0x8];
+	u8         reserved_1[0x10];
+
+	u8         reserved_2[0xc];
+	u8         error_count[0x4];
+	u8         reserved_3[0x10];
+
+	u8         reserved_4[0xc];
+	u8         lane[0x4];
+	u8         reserved_5[0x8];
+	u8         error_type[0x8];
+};
+
+struct mlx5_ifc_pcap_reg_bits {
+	u8         reserved_0[0x8];
+	u8         local_port[0x8];
+	u8         reserved_1[0x10];
+
+	u8         port_capability_mask[4][0x20];
+};
+
+struct mlx5_ifc_pbmc_reg_bits {
+	u8         reserved_0[0x8];
+	u8         local_port[0x8];
+	u8         reserved_1[0x10];
+
+	u8         xoff_timer_value[0x10];
+	u8         xoff_refresh[0x10];
+
+	u8         reserved_2[0x10];
+	u8         port_buffer_size[0x10];
+
+	struct mlx5_ifc_bufferx_reg_bits buffer[10];
+
+	u8         reserved_3[0x40];
+
+	u8         port_shared_buffer[0x40];
+};
+
+struct mlx5_ifc_paos_reg_bits {
+	u8         swid[0x8];
+	u8         local_port[0x8];
+	u8         reserved_0[0x4];
+	u8         admin_status[0x4];
+	u8         reserved_1[0x4];
+	u8         oper_status[0x4];
+
+	u8         ase[0x1];
+	u8         ee[0x1];
+	u8         reserved_2[0x1c];
+	u8         e[0x2];
+
+	u8         reserved_3[0x40];
+};
+
+struct mlx5_ifc_pamp_reg_bits {
+	u8         reserved_0[0x8];
+	u8         opamp_group[0x8];
+	u8         reserved_1[0xc];
+	u8         opamp_group_type[0x4];
+
+	u8         start_index[0x10];
+	u8         reserved_2[0x4];
+	u8         num_of_indices[0xc];
+
+	u8         index_data[18][0x10];
+};
+
+struct mlx5_ifc_link_level_retrans_cntr_grp_date_bits {
+	u8         llr_rx_cells_high[0x20];
+
+	u8         llr_rx_cells_low[0x20];
+
+	u8         llr_rx_error_high[0x20];
+
+	u8         llr_rx_error_low[0x20];
+
+	u8         llr_rx_crc_error_high[0x20];
+
+	u8         llr_rx_crc_error_low[0x20];
+
+	u8         llr_tx_cells_high[0x20];
+
+	u8         llr_tx_cells_low[0x20];
+
+	u8         llr_tx_ret_cells_high[0x20];
+
+	u8         llr_tx_ret_cells_low[0x20];
+
+	u8         llr_tx_ret_events_high[0x20];
+
+	u8         llr_tx_ret_events_low[0x20];
+
+	u8         reserved_0[0x640];
+};
+
+struct mlx5_ifc_lane_2_module_mapping_bits {
+	u8         reserved_0[0x6];
+	u8         rx_lane[0x2];
+	u8         reserved_1[0x6];
+	u8         tx_lane[0x2];
+	u8         reserved_2[0x8];
+	u8         module[0x8];
+};
+
+struct mlx5_ifc_eth_per_traffic_class_layout_bits {
+	u8         transmit_queue_high[0x20];
+
+	u8         transmit_queue_low[0x20];
+
+	u8         reserved_0[0x780];
+};
+
+struct mlx5_ifc_eth_per_traffic_class_cong_layout_bits {
+	u8         no_buffer_discard_uc_high[0x20];
+
+	u8         no_buffer_discard_uc_low[0x20];
+
+	u8         wred_discard_high[0x20];
+
+	u8         wred_discard_low[0x20];
+
+	u8         reserved_0[0x740];
+};
+
+struct mlx5_ifc_eth_per_prio_grp_data_layout_bits {
+	u8         rx_octets_high[0x20];
+
+	u8         rx_octets_low[0x20];
+
+	u8         reserved_0[0xc0];
+
+	u8         rx_frames_high[0x20];
+
+	u8         rx_frames_low[0x20];
+
+	u8         tx_octets_high[0x20];
+
+	u8         tx_octets_low[0x20];
+
+	u8         reserved_1[0xc0];
+
+	u8         tx_frames_high[0x20];
+
+	u8         tx_frames_low[0x20];
+
+	u8         rx_pause_high[0x20];
+
+	u8         rx_pause_low[0x20];
+
+	u8         rx_pause_duration_high[0x20];
+
+	u8         rx_pause_duration_low[0x20];
+
+	u8         tx_pause_high[0x20];
+
+	u8         tx_pause_low[0x20];
+
+	u8         tx_pause_duration_high[0x20];
+
+	u8         tx_pause_duration_low[0x20];
+
+	u8         rx_pause_transition_high[0x20];
+
+	u8         rx_pause_transition_low[0x20];
+
+	u8         reserved_2[0x400];
+};
+
+struct mlx5_ifc_eth_extended_cntrs_grp_data_layout_bits {
+	u8         port_transmit_wait_high[0x20];
+
+	u8         port_transmit_wait_low[0x20];
+
+	u8         ecn_marked_high[0x20];
+
+	u8         ecn_marked_low[0x20];
+
+	u8         no_buffer_discard_mc_high[0x20];
+
+	u8         no_buffer_discard_mc_low[0x20];
+
+	u8         reserved_0[0x700];
+};
+
+struct mlx5_ifc_eth_802_3_cntrs_grp_data_layout_bits {
+	u8         a_frames_transmitted_ok_high[0x20];
+
+	u8         a_frames_transmitted_ok_low[0x20];
+
+	u8         a_frames_received_ok_high[0x20];
+
+	u8         a_frames_received_ok_low[0x20];
+
+	u8         a_frame_check_sequence_errors_high[0x20];
+
+	u8         a_frame_check_sequence_errors_low[0x20];
+
+	u8         a_alignment_errors_high[0x20];
+
+	u8         a_alignment_errors_low[0x20];
+
+	u8         a_octets_transmitted_ok_high[0x20];
+
+	u8         a_octets_transmitted_ok_low[0x20];
+
+	u8         a_octets_received_ok_high[0x20];
+
+	u8         a_octets_received_ok_low[0x20];
+
+	u8         a_multicast_frames_xmitted_ok_high[0x20];
+
+	u8         a_multicast_frames_xmitted_ok_low[0x20];
+
+	u8         a_broadcast_frames_xmitted_ok_high[0x20];
+
+	u8         a_broadcast_frames_xmitted_ok_low[0x20];
+
+	u8         a_multicast_frames_received_ok_high[0x20];
+
+	u8         a_multicast_frames_received_ok_low[0x20];
+
+	u8         a_broadcast_frames_recieved_ok_high[0x20];
+
+	u8         a_broadcast_frames_recieved_ok_low[0x20];
+
+	u8         a_in_range_length_errors_high[0x20];
+
+	u8         a_in_range_length_errors_low[0x20];
+
+	u8         a_out_of_range_length_field_high[0x20];
+
+	u8         a_out_of_range_length_field_low[0x20];
+
+	u8         a_frame_too_long_errors_high[0x20];
+
+	u8         a_frame_too_long_errors_low[0x20];
+
+	u8         a_symbol_error_during_carrier_high[0x20];
+
+	u8         a_symbol_error_during_carrier_low[0x20];
+
+	u8         a_mac_control_frames_transmitted_high[0x20];
+
+	u8         a_mac_control_frames_transmitted_low[0x20];
+
+	u8         a_mac_control_frames_received_high[0x20];
+
+	u8         a_mac_control_frames_received_low[0x20];
+
+	u8         a_unsupported_opcodes_received_high[0x20];
+
+	u8         a_unsupported_opcodes_received_low[0x20];
+
+	u8         a_pause_mac_ctrl_frames_received_high[0x20];
+
+	u8         a_pause_mac_ctrl_frames_received_low[0x20];
+
+	u8         a_pause_mac_ctrl_frames_transmitted_high[0x20];
+
+	u8         a_pause_mac_ctrl_frames_transmitted_low[0x20];
+
+	u8         reserved_0[0x300];
+};
+
+struct mlx5_ifc_eth_3635_cntrs_grp_data_layout_bits {
+	u8         dot3stats_alignment_errors_high[0x20];
+
+	u8         dot3stats_alignment_errors_low[0x20];
+
+	u8         dot3stats_fcs_errors_high[0x20];
+
+	u8         dot3stats_fcs_errors_low[0x20];
+
+	u8         dot3stats_single_collision_frames_high[0x20];
+
+	u8         dot3stats_single_collision_frames_low[0x20];
+
+	u8         dot3stats_multiple_collision_frames_high[0x20];
+
+	u8         dot3stats_multiple_collision_frames_low[0x20];
+
+	u8         dot3stats_sqe_test_errors_high[0x20];
+
+	u8         dot3stats_sqe_test_errors_low[0x20];
+
+	u8         dot3stats_deferred_transmissions_high[0x20];
+
+	u8         dot3stats_deferred_transmissions_low[0x20];
+
+	u8         dot3stats_late_collisions_high[0x20];
+
+	u8         dot3stats_late_collisions_low[0x20];
+
+	u8         dot3stats_excessive_collisions_high[0x20];
+
+	u8         dot3stats_excessive_collisions_low[0x20];
+
+	u8         dot3stats_internal_mac_transmit_errors_high[0x20];
+
+	u8         dot3stats_internal_mac_transmit_errors_low[0x20];
+
+	u8         dot3stats_carrier_sense_errors_high[0x20];
+
+	u8         dot3stats_carrier_sense_errors_low[0x20];
+
+	u8         dot3stats_frame_too_longs_high[0x20];
+
+	u8         dot3stats_frame_too_longs_low[0x20];
+
+	u8         dot3stats_internal_mac_receive_errors_high[0x20];
+
+	u8         dot3stats_internal_mac_receive_errors_low[0x20];
+
+	u8         dot3stats_symbol_errors_high[0x20];
+
+	u8         dot3stats_symbol_errors_low[0x20];
+
+	u8         dot3control_in_unknown_opcodes_high[0x20];
+
+	u8         dot3control_in_unknown_opcodes_low[0x20];
+
+	u8         dot3in_pause_frames_high[0x20];
+
+	u8         dot3in_pause_frames_low[0x20];
+
+	u8         dot3out_pause_frames_high[0x20];
+
+	u8         dot3out_pause_frames_low[0x20];
+
+	u8         reserved_0[0x3c0];
+};
+
+struct mlx5_ifc_eth_2863_cntrs_grp_data_layout_bits {
+	u8         if_in_octets_high[0x20];
+
+	u8         if_in_octets_low[0x20];
+
+	u8         if_in_ucast_pkts_high[0x20];
+
+	u8         if_in_ucast_pkts_low[0x20];
+
+	u8         if_in_discards_high[0x20];
+
+	u8         if_in_discards_low[0x20];
+
+	u8         if_in_errors_high[0x20];
+
+	u8         if_in_errors_low[0x20];
+
+	u8         if_in_unknown_protos_high[0x20];
+
+	u8         if_in_unknown_protos_low[0x20];
+
+	u8         if_out_octets_high[0x20];
+
+	u8         if_out_octets_low[0x20];
+
+	u8         if_out_ucast_pkts_high[0x20];
+
+	u8         if_out_ucast_pkts_low[0x20];
+
+	u8         if_out_discards_high[0x20];
+
+	u8         if_out_discards_low[0x20];
+
+	u8         if_out_errors_high[0x20];
+
+	u8         if_out_errors_low[0x20];
+
+	u8         if_in_multicast_pkts_high[0x20];
+
+	u8         if_in_multicast_pkts_low[0x20];
+
+	u8         if_in_broadcast_pkts_high[0x20];
+
+	u8         if_in_broadcast_pkts_low[0x20];
+
+	u8         if_out_multicast_pkts_high[0x20];
+
+	u8         if_out_multicast_pkts_low[0x20];
+
+	u8         if_out_broadcast_pkts_high[0x20];
+
+	u8         if_out_broadcast_pkts_low[0x20];
+
+	u8         reserved_0[0x480];
+};
+
+struct mlx5_ifc_eth_2819_cntrs_grp_data_layout_bits {
+	u8         ether_stats_drop_events_high[0x20];
+
+	u8         ether_stats_drop_events_low[0x20];
+
+	u8         ether_stats_octets_high[0x20];
+
+	u8         ether_stats_octets_low[0x20];
+
+	u8         ether_stats_pkts_high[0x20];
+
+	u8         ether_stats_pkts_low[0x20];
+
+	u8         ether_stats_broadcast_pkts_high[0x20];
+
+	u8         ether_stats_broadcast_pkts_low[0x20];
+
+	u8         ether_stats_multicast_pkts_high[0x20];
+
+	u8         ether_stats_multicast_pkts_low[0x20];
+
+	u8         ether_stats_crc_align_errors_high[0x20];
+
+	u8         ether_stats_crc_align_errors_low[0x20];
+
+	u8         ether_stats_undersize_pkts_high[0x20];
+
+	u8         ether_stats_undersize_pkts_low[0x20];
+
+	u8         ether_stats_oversize_pkts_high[0x20];
+
+	u8         ether_stats_oversize_pkts_low[0x20];
+
+	u8         ether_stats_fragments_high[0x20];
+
+	u8         ether_stats_fragments_low[0x20];
+
+	u8         ether_stats_jabbers_high[0x20];
+
+	u8         ether_stats_jabbers_low[0x20];
+
+	u8         ether_stats_collisions_high[0x20];
+
+	u8         ether_stats_collisions_low[0x20];
+
+	u8         ether_stats_pkts64octets_high[0x20];
+
+	u8         ether_stats_pkts64octets_low[0x20];
+
+	u8         ether_stats_pkts65to127octets_high[0x20];
+
+	u8         ether_stats_pkts65to127octets_low[0x20];
+
+	u8         ether_stats_pkts128to255octets_high[0x20];
+
+	u8         ether_stats_pkts128to255octets_low[0x20];
+
+	u8         ether_stats_pkts256to511octets_high[0x20];
+
+	u8         ether_stats_pkts256to511octets_low[0x20];
+
+	u8         ether_stats_pkts512to1023octets_high[0x20];
+
+	u8         ether_stats_pkts512to1023octets_low[0x20];
+
+	u8         ether_stats_pkts1024to1518octets_high[0x20];
+
+	u8         ether_stats_pkts1024to1518octets_low[0x20];
+
+	u8         ether_stats_pkts1519to2047octets_high[0x20];
+
+	u8         ether_stats_pkts1519to2047octets_low[0x20];
+
+	u8         ether_stats_pkts2048to4095octets_high[0x20];
+
+	u8         ether_stats_pkts2048to4095octets_low[0x20];
+
+	u8         ether_stats_pkts4096to8191octets_high[0x20];
+
+	u8         ether_stats_pkts4096to8191octets_low[0x20];
+
+	u8         ether_stats_pkts8192to10239octets_high[0x20];
+
+	u8         ether_stats_pkts8192to10239octets_low[0x20];
+
+	u8         reserved_0[0x280];
+};
+
+struct mlx5_ifc_ib_portcntrs_attribute_grp_data_bits {
+	u8         symbol_error_counter[0x10];
+	u8         link_error_recovery_counter[0x8];
+	u8         link_downed_counter[0x8];
+
+	u8         port_rcv_errors[0x10];
+	u8         port_rcv_remote_physical_errors[0x10];
+
+	u8         port_rcv_switch_relay_errors[0x10];
+	u8         port_xmit_discards[0x10];
+
+	u8         port_xmit_constraint_errors[0x8];
+	u8         port_rcv_constraint_errors[0x8];
+	u8         reserved_0[0x8];
+	u8         local_link_integrity_errors[0x4];
+	u8         excessive_buffer_overrun_errors[0x4];
+
+	u8         reserved_1[0x10];
+	u8         vl_15_dropped[0x10];
+
+	u8         port_xmit_data[0x20];
+
+	u8         port_rcv_data[0x20];
+
+	u8         port_xmit_pkts[0x20];
+
+	u8         port_rcv_pkts[0x20];
+
+	u8         port_xmit_wait[0x20];
+
+	u8         reserved_2[0x680];
+};
+
+struct mlx5_ifc_trc_tlb_reg_bits {
+	u8         reserved_0[0x80];
+
+	u8         tlb_addr[0][0x40];
+};
+
+struct mlx5_ifc_trc_read_fifo_reg_bits {
+	u8         reserved_0[0x10];
+	u8         requested_event_num[0x10];
+
+	u8         reserved_1[0x20];
+
+	u8         reserved_2[0x10];
+	u8         acual_event_num[0x10];
+
+	u8         reserved_3[0x20];
+
+	u8         event[0][0x40];
+};
+
+struct mlx5_ifc_trc_lock_reg_bits {
+	u8         reserved_0[0x1f];
+	u8         lock[0x1];
+
+	u8         reserved_1[0x60];
+};
+
+struct mlx5_ifc_trc_filter_reg_bits {
+	u8         status[0x1];
+	u8         reserved_0[0xf];
+	u8         filter_index[0x10];
+
+	u8         reserved_1[0x20];
+
+	u8         filter_val[0x20];
+
+	u8         reserved_2[0x1a0];
+};
+
+struct mlx5_ifc_trc_event_reg_bits {
+	u8         status[0x1];
+	u8         reserved_0[0xf];
+	u8         event_index[0x10];
+
+	u8         reserved_1[0x20];
+
+	u8         event_id[0x20];
+
+	u8         event_selector_val[0x10];
+	u8         event_selector_size[0x10];
+
+	u8         reserved_2[0x180];
+};
+
+struct mlx5_ifc_trc_conf_reg_bits {
+	u8         limit_en[0x1];
+	u8         reserved_0[0x3];
+	u8         dump_mode[0x4];
+	u8         reserved_1[0x15];
+	u8         state[0x3];
+
+	u8         reserved_2[0x20];
+
+	u8         limit_event_index[0x20];
+
+	u8         mkey[0x20];
+
+	u8         fifo_ready_ev_num[0x20];
+
+	u8         reserved_3[0x160];
+};
+
+struct mlx5_ifc_trc_cap_reg_bits {
+	u8         reserved_0[0x18];
+	u8         dump_mode[0x8];
+
+	u8         reserved_1[0x20];
+
+	u8         num_of_events[0x10];
+	u8         num_of_filters[0x10];
+
+	u8         fifo_size[0x20];
+
+	u8         tlb_size[0x10];
+	u8         event_size[0x10];
+
+	u8         reserved_2[0x160];
+};
+
+struct mlx5_ifc_set_node_in_bits {
+	u8         node_description[64][0x8];
+};
+
+struct mlx5_ifc_register_power_settings_bits {
+	u8         reserved_0[0x18];
+	u8         power_settings_level[0x8];
+
+	u8         reserved_1[0x60];
+};
+
+struct mlx5_ifc_register_host_endianess_bits {
+	u8         he[0x1];
+	u8         reserved_0[0x1f];
+
+	u8         reserved_1[0x60];
+};
+
+struct mlx5_ifc_register_diag_buffer_ctrl_bits {
+	u8         physical_address[0x40];
+};
+
+struct mlx5_ifc_qtct_reg_bits {
+	u8         reserved_0[0x8];
+	u8         port_number[0x8];
+	u8         reserved_1[0xd];
+	u8         prio[0x3];
+
+	u8         reserved_2[0x1d];
+	u8         tclass[0x3];
+};
+
+struct mlx5_ifc_qpdp_reg_bits {
+	u8         reserved_0[0x8];
+	u8         port_number[0x8];
+	u8         reserved_1[0x10];
+
+	u8         reserved_2[0x1d];
+	u8         pprio[0x3];
+};
+
+struct mlx5_ifc_port_info_ro_fields_param_bits {
+	u8         reserved_0[0x8];
+	u8         port[0x8];
+	u8         max_gid[0x10];
+
+	u8         reserved_1[0x20];
+
+	u8         port_guid[0x40];
+};
+
+struct mlx5_ifc_nvqc_reg_bits {
+	u8         type[0x20];
+
+	u8         reserved_0[0x18];
+	u8         version[0x4];
+	u8         reserved_1[0x2];
+	u8         support_wr[0x1];
+	u8         support_rd[0x1];
+};
+
+struct mlx5_ifc_nvia_reg_bits {
+	u8         reserved_0[0x1d];
+	u8         target[0x3];
+
+	u8         reserved_1[0x20];
+};
+
+struct mlx5_ifc_nvdi_reg_bits {
+	struct mlx5_ifc_config_item_bits configuration_item_header;
+};
+
+struct mlx5_ifc_nvda_reg_bits {
+	struct mlx5_ifc_config_item_bits configuration_item_header;
+
+	u8         configuration_item_data[0x20];
+};
+
+struct mlx5_ifc_node_info_ro_fields_param_bits {
+	u8         system_image_guid[0x40];
+
+	u8         reserved_0[0x40];
+
+	u8         node_guid[0x40];
+
+	u8         reserved_1[0x10];
+	u8         max_pkey[0x10];
+
+	u8         reserved_2[0x20];
+};
+
+struct mlx5_ifc_ets_tcn_config_reg_bits {
+	u8         g[0x1];
+	u8         b[0x1];
+	u8         r[0x1];
+	u8         reserved_0[0x9];
+	u8         group[0x4];
+	u8         reserved_1[0x9];
+	u8         bw_allocation[0x7];
+
+	u8         reserved_2[0xc];
+	u8         max_bw_units[0x4];
+	u8         reserved_3[0x8];
+	u8         max_bw_value[0x8];
+};
+
+struct mlx5_ifc_ets_global_config_reg_bits {
+	u8         reserved_0[0x2];
+	u8         r[0x1];
+	u8         reserved_1[0x1d];
+
+	u8         reserved_2[0xc];
+	u8         max_bw_units[0x4];
+	u8         reserved_3[0x8];
+	u8         max_bw_value[0x8];
+};
+
+struct mlx5_ifc_nodnic_mac_filters_bits {
+	struct mlx5_ifc_mac_address_layout_bits mac_filter0;
+
+	struct mlx5_ifc_mac_address_layout_bits mac_filter1;
+
+	struct mlx5_ifc_mac_address_layout_bits mac_filter2;
+
+	struct mlx5_ifc_mac_address_layout_bits mac_filter3;
+
+	struct mlx5_ifc_mac_address_layout_bits mac_filter4;
+
+	u8         reserved_0[0xc0];
+};
+
+struct mlx5_ifc_nodnic_gid_filters_bits {
+	u8         mgid_filter0[16][0x8];
+
+	u8         mgid_filter1[16][0x8];
+
+	u8         mgid_filter2[16][0x8];
+
+	u8         mgid_filter3[16][0x8];
+};
+
+enum {
+	MLX5_NODNIC_CONFIG_REG_NUM_PORTS_SINGLE_PORT  = 0x0,
+	MLX5_NODNIC_CONFIG_REG_NUM_PORTS_DUAL_PORT    = 0x1,
+};
+
+enum {
+	MLX5_NODNIC_CONFIG_REG_CQE_FORMAT_LEGACY_CQE  = 0x0,
+	MLX5_NODNIC_CONFIG_REG_CQE_FORMAT_NEW_CQE     = 0x1,
+};
+
+struct mlx5_ifc_nodnic_config_reg_bits {
+	u8         no_dram_nic_revision[0x8];
+	u8         hardware_format[0x8];
+	u8         support_receive_filter[0x1];
+	u8         support_promisc_filter[0x1];
+	u8         support_promisc_multicast_filter[0x1];
+	u8         reserved_0[0x2];
+	u8         log_working_buffer_size[0x3];
+	u8         log_pkey_table_size[0x4];
+	u8         reserved_1[0x3];
+	u8         num_ports[0x1];
+
+	u8         reserved_2[0x2];
+	u8         log_max_ring_size[0x6];
+	u8         reserved_3[0x18];
+
+	u8         lkey[0x20];
+
+	u8         cqe_format[0x4];
+	u8         reserved_4[0x1c];
+
+	u8         node_guid[0x40];
+
+	u8         reserved_5[0x740];
+
+	struct mlx5_ifc_nodnic_port_config_reg_bits port1_settings;
+
+	struct mlx5_ifc_nodnic_port_config_reg_bits port2_settings;
+};
+
+struct mlx5_ifc_vlan_layout_bits {
+	u8         reserved_0[0x14];
+	u8         vlan[0xc];
+
+	u8         reserved_1[0x20];
+};
+
+struct mlx5_ifc_umr_pointer_desc_argument_bits {
+	u8         reserved_0[0x20];
+
+	u8         mkey[0x20];
+
+	u8         addressh_63_32[0x20];
+
+	u8         addressl_31_0[0x20];
+};
+
+struct mlx5_ifc_ud_adrs_vector_bits {
+	u8         dc_key[0x40];
+
+	u8         ext[0x1];
+	u8         reserved_0[0x7];
+	u8         destination_qp_dct[0x18];
+
+	u8         static_rate[0x4];
+	u8         sl_eth_prio[0x4];
+	u8         fl[0x1];
+	u8         mlid[0x7];
+	u8         rlid_udp_sport[0x10];
+
+	u8         reserved_1[0x20];
+
+	u8         rmac_47_16[0x20];
+
+	u8         rmac_15_0[0x10];
+	u8         tclass[0x8];
+	u8         hop_limit[0x8];
+
+	u8         reserved_2[0x1];
+	u8         grh[0x1];
+	u8         reserved_3[0x2];
+	u8         src_addr_index[0x8];
+	u8         flow_label[0x14];
+
+	u8         rgid_rip[16][0x8];
+};
+
+struct mlx5_ifc_port_module_event_bits {
+	u8         reserved_0[0x8];
+	u8         module[0x8];
+	u8         reserved_1[0xc];
+	u8         module_status[0x4];
+
+	u8         reserved_2[0x14];
+	u8         error_type[0x4];
+	u8         reserved_3[0x8];
+
+	u8         reserved_4[0xa0];
+};
+
+struct mlx5_ifc_icmd_control_bits {
+	u8         opcode[0x10];
+	u8         status[0x8];
+	u8         reserved_0[0x7];
+	u8         busy[0x1];
+};
+
+struct mlx5_ifc_eqe_bits {
+	u8         reserved_0[0x8];
+	u8         event_type[0x8];
+	u8         reserved_1[0x8];
+	u8         event_sub_type[0x8];
+
+	u8         reserved_2[0xe0];
+
+	union mlx5_ifc_event_auto_bits event_data;
+
+	u8         reserved_3[0x10];
+	u8         signature[0x8];
+	u8         reserved_4[0x7];
+	u8         owner[0x1];
+};
+
+enum {
+	MLX5_CMD_QUEUE_ENTRY_TYPE_PCIE_CMD_IF_TRANSPORT  = 0x7,
+};
+
+struct mlx5_ifc_cmd_queue_entry_bits {
+	u8         type[0x8];
+	u8         reserved_0[0x18];
+
+	u8         input_length[0x20];
+
+	u8         input_mailbox_pointer_63_32[0x20];
+
+	u8         input_mailbox_pointer_31_9[0x17];
+	u8         reserved_1[0x9];
+
+	u8         command_input_inline_data[16][0x8];
+
+	u8         command_output_inline_data[16][0x8];
+
+	u8         output_mailbox_pointer_63_32[0x20];
+
+	u8         output_mailbox_pointer_31_9[0x17];
+	u8         reserved_2[0x9];
+
+	u8         output_length[0x20];
+
+	u8         token[0x8];
+	u8         signature[0x8];
+	u8         reserved_3[0x8];
+	u8         status[0x7];
+	u8         ownership[0x1];
+};
+
+struct mlx5_ifc_cmd_out_bits {
+	u8         status[0x8];
+	u8         reserved_0[0x18];
+
+	u8         syndrome[0x20];
+
+	u8         command_output[0x20];
+};
+
+struct mlx5_ifc_cmd_in_bits {
+	u8         opcode[0x10];
+	u8         reserved_0[0x10];
+
+	u8         reserved_1[0x10];
+	u8         op_mod[0x10];
+
+	u8         command[0][0x20];
+};
+
+struct mlx5_ifc_cmd_if_box_bits {
+	u8         mailbox_data[512][0x8];
+
+	u8         reserved_0[0x180];
+
+	u8         next_pointer_63_32[0x20];
+
+	u8         next_pointer_31_10[0x16];
+	u8         reserved_1[0xa];
+
+	u8         block_number[0x20];
+
+	u8         reserved_2[0x8];
+	u8         token[0x8];
+	u8         ctrl_signature[0x8];
+	u8         signature[0x8];
+};
+
+struct mlx5_ifc_mtt_bits {
+	u8         ptag_63_32[0x20];
+
+	u8         ptag_31_8[0x18];
+	u8         reserved_0[0x6];
+	u8         wr_en[0x1];
+	u8         rd_en[0x1];
+};
+
+struct mlx5_ifc_vendor_specific_cap_bits {
+	u8         type[0x8];
+	u8         length[0x8];
+	u8         next_pointer[0x8];
+	u8         capability_id[0x8];
+
+	u8         status[0x3];
+	u8         reserved_0[0xd];
+	u8         space[0x10];
+
+	u8         counter[0x20];
+
+	u8         semaphore[0x20];
+
+	u8         flag[0x1];
+	u8         reserved_1[0x1];
+	u8         address[0x1e];
+
+	u8         data[0x20];
+};
+
+enum {
+	MLX5_INITIAL_SEG_NIC_INTERFACE_FULL_DRIVER  = 0x0,
+	MLX5_INITIAL_SEG_NIC_INTERFACE_DISABLED     = 0x1,
+	MLX5_INITIAL_SEG_NIC_INTERFACE_NO_DRAM_NIC  = 0x2,
+};
+
+enum {
+	MLX5_INITIAL_SEG_NIC_INTERFACE_SUPPORTED_FULL_DRIVER  = 0x0,
+	MLX5_INITIAL_SEG_NIC_INTERFACE_SUPPORTED_DISABLED     = 0x1,
+	MLX5_INITIAL_SEG_NIC_INTERFACE_SUPPORTED_NO_DRAM_NIC  = 0x2,
+};
+
+enum {
+	MLX5_HEALTH_SYNDR_FW_ERR                                      = 0x1,
+	MLX5_HEALTH_SYNDR_IRISC_ERR                                   = 0x7,
+	MLX5_HEALTH_SYNDR_HW_UNRECOVERABLE_ERR                        = 0x8,
+	MLX5_HEALTH_SYNDR_CRC_ERR                                     = 0x9,
+	MLX5_HEALTH_SYNDR_FETCH_PCI_ERR                               = 0xa,
+	MLX5_HEALTH_SYNDR_HW_FTL_ERR                                  = 0xb,
+	MLX5_HEALTH_SYNDR_ASYNC_EQ_OVERRUN_ERR                        = 0xc,
+	MLX5_HEALTH_SYNDR_EQ_ERR                                      = 0xd,
+	MLX5_HEALTH_SYNDR_EQ_INV                                      = 0xe,
+	MLX5_HEALTH_SYNDR_FFSER_ERR                                   = 0xf,
+	MLX5_HEALTH_SYNDR_HIGH_TEMP                                   = 0x10,
+};
+
+struct mlx5_ifc_initial_seg_bits {
+	u8         fw_rev_minor[0x10];
+	u8         fw_rev_major[0x10];
+
+	u8         cmd_interface_rev[0x10];
+	u8         fw_rev_subminor[0x10];
+
+	u8         reserved_0[0x40];
+
+	u8         cmdq_phy_addr_63_32[0x20];
+
+	u8         cmdq_phy_addr_31_12[0x14];
+	u8         reserved_1[0x2];
+	u8         nic_interface[0x2];
+	u8         log_cmdq_size[0x4];
+	u8         log_cmdq_stride[0x4];
+
+	u8         command_doorbell_vector[0x20];
+
+	u8         reserved_2[0xf00];
+
+	u8         initializing[0x1];
+	u8         reserved_3[0x4];
+	u8         nic_interface_supported[0x3];
+	u8         reserved_4[0x18];
+
+	struct mlx5_ifc_health_buffer_bits health_buffer;
+
+	u8         no_dram_nic_offset[0x20];
+
+	u8         reserved_5[0x6de0];
+
+	u8         internal_timer_h[0x20];
+
+	u8         internal_timer_l[0x20];
+
+	u8         reserved_6[0x20];
+
+	u8         reserved_7[0x1f];
+	u8         clear_int[0x1];
+
+	u8         health_syndrome[0x8];
+	u8         health_counter[0x18];
+
+	u8         reserved_8[0x17fc0];
+};
+
+union mlx5_ifc_icmd_interface_document_bits {
+	struct mlx5_ifc_fw_version_bits fw_version;
+	struct mlx5_ifc_icmd_access_reg_in_bits icmd_access_reg_in;
+	struct mlx5_ifc_icmd_access_reg_out_bits icmd_access_reg_out;
+	struct mlx5_ifc_icmd_init_ocsd_in_bits icmd_init_ocsd_in;
+	struct mlx5_ifc_icmd_ocbb_init_in_bits icmd_ocbb_init_in;
+	struct mlx5_ifc_icmd_ocbb_query_etoc_stats_out_bits icmd_ocbb_query_etoc_stats_out;
+	struct mlx5_ifc_icmd_ocbb_query_header_stats_out_bits icmd_ocbb_query_header_stats_out;
+	struct mlx5_ifc_icmd_query_cap_general_bits icmd_query_cap_general;
+	struct mlx5_ifc_icmd_query_cap_in_bits icmd_query_cap_in;
+	struct mlx5_ifc_icmd_query_fw_info_out_bits icmd_query_fw_info_out;
+	struct mlx5_ifc_icmd_query_virtual_mac_out_bits icmd_query_virtual_mac_out;
+	struct mlx5_ifc_icmd_set_virtual_mac_in_bits icmd_set_virtual_mac_in;
+	struct mlx5_ifc_icmd_set_wol_rol_in_bits icmd_set_wol_rol_in;
+	struct mlx5_ifc_icmd_set_wol_rol_out_bits icmd_set_wol_rol_out;
+	u8         reserved_0[0x42c0];
+};
+
+union mlx5_ifc_eth_cntrs_grp_data_layout_auto_bits {
+	struct mlx5_ifc_eth_802_3_cntrs_grp_data_layout_bits eth_802_3_cntrs_grp_data_layout;
+	struct mlx5_ifc_eth_2863_cntrs_grp_data_layout_bits eth_2863_cntrs_grp_data_layout;
+	struct mlx5_ifc_eth_2819_cntrs_grp_data_layout_bits eth_2819_cntrs_grp_data_layout;
+	struct mlx5_ifc_eth_3635_cntrs_grp_data_layout_bits eth_3635_cntrs_grp_data_layout;
+	struct mlx5_ifc_eth_extended_cntrs_grp_data_layout_bits eth_extended_cntrs_grp_data_layout;
+	struct mlx5_ifc_eth_per_prio_grp_data_layout_bits eth_per_prio_grp_data_layout;
+	struct mlx5_ifc_phys_layer_cntrs_bits phys_layer_cntrs;
+	u8         reserved_0[0x7c0];
+};
+
+struct mlx5_ifc_ppcnt_reg_bits {
+	u8         swid[0x8];
+	u8         local_port[0x8];
+	u8         pnat[0x2];
+	u8         reserved_0[0x8];
+	u8         grp[0x6];
+
+	u8         clr[0x1];
+	u8         reserved_1[0x1c];
+	u8         prio_tc[0x3];
+
+	union mlx5_ifc_eth_cntrs_grp_data_layout_auto_bits counter_set;
+};
+
+union mlx5_ifc_ports_control_registers_document_bits {
+	struct mlx5_ifc_ib_portcntrs_attribute_grp_data_bits ib_portcntrs_attribute_grp_data;
+	struct mlx5_ifc_bufferx_reg_bits bufferx_reg;
+	struct mlx5_ifc_eth_2819_cntrs_grp_data_layout_bits eth_2819_cntrs_grp_data_layout;
+	struct mlx5_ifc_eth_2863_cntrs_grp_data_layout_bits eth_2863_cntrs_grp_data_layout;
+	struct mlx5_ifc_eth_3635_cntrs_grp_data_layout_bits eth_3635_cntrs_grp_data_layout;
+	struct mlx5_ifc_eth_802_3_cntrs_grp_data_layout_bits eth_802_3_cntrs_grp_data_layout;
+	struct mlx5_ifc_eth_extended_cntrs_grp_data_layout_bits eth_extended_cntrs_grp_data_layout;
+	struct mlx5_ifc_eth_per_prio_grp_data_layout_bits eth_per_prio_grp_data_layout;
+	struct mlx5_ifc_eth_per_traffic_class_cong_layout_bits eth_per_traffic_class_cong_layout;
+	struct mlx5_ifc_eth_per_traffic_class_layout_bits eth_per_traffic_class_layout;
+	struct mlx5_ifc_lane_2_module_mapping_bits lane_2_module_mapping;
+	struct mlx5_ifc_link_level_retrans_cntr_grp_date_bits link_level_retrans_cntr_grp_date;
+	struct mlx5_ifc_pamp_reg_bits pamp_reg;
+	struct mlx5_ifc_paos_reg_bits paos_reg;
+	struct mlx5_ifc_pbmc_reg_bits pbmc_reg;
+	struct mlx5_ifc_pcap_reg_bits pcap_reg;
+	struct mlx5_ifc_peir_reg_bits peir_reg;
+	struct mlx5_ifc_pelc_reg_bits pelc_reg;
+	struct mlx5_ifc_pfcc_reg_bits pfcc_reg;
+	struct mlx5_ifc_phbr_binding_reg_bits phbr_binding_reg;
+	struct mlx5_ifc_phbr_for_port_tclass_reg_bits phbr_for_port_tclass_reg;
+	struct mlx5_ifc_phbr_for_prio_reg_bits phbr_for_prio_reg;
+	struct mlx5_ifc_phrr_reg_bits phrr_reg;
+	struct mlx5_ifc_phys_layer_cntrs_bits phys_layer_cntrs;
+	struct mlx5_ifc_pifr_reg_bits pifr_reg;
+	struct mlx5_ifc_pipg_reg_bits pipg_reg;
+	struct mlx5_ifc_plbf_reg_bits plbf_reg;
+	struct mlx5_ifc_plib_reg_bits plib_reg;
+	struct mlx5_ifc_pll_status_data_bits pll_status_data;
+	struct mlx5_ifc_plpc_reg_bits plpc_reg;
+	struct mlx5_ifc_pmaos_reg_bits pmaos_reg;
+	struct mlx5_ifc_pmlp_reg_bits pmlp_reg;
+	struct mlx5_ifc_pmlpn_reg_bits pmlpn_reg;
+	struct mlx5_ifc_pmpc_reg_bits pmpc_reg;
+	struct mlx5_ifc_pmpe_reg_bits pmpe_reg;
+	struct mlx5_ifc_pmpr_reg_bits pmpr_reg;
+	struct mlx5_ifc_pmtu_reg_bits pmtu_reg;
+	struct mlx5_ifc_ppad_reg_bits ppad_reg;
+	struct mlx5_ifc_ppcnt_reg_bits ppcnt_reg;
+	struct mlx5_ifc_ppll_reg_bits ppll_reg;
+	struct mlx5_ifc_pplm_reg_bits pplm_reg;
+	struct mlx5_ifc_pplr_reg_bits pplr_reg;
+	struct mlx5_ifc_ppsc_reg_bits ppsc_reg;
+	struct mlx5_ifc_pspa_reg_bits pspa_reg;
+	struct mlx5_ifc_ptas_reg_bits ptas_reg;
+	struct mlx5_ifc_ptys_reg_bits ptys_reg;
+	struct mlx5_ifc_pude_reg_bits pude_reg;
+	struct mlx5_ifc_pvlc_reg_bits pvlc_reg;
+	struct mlx5_ifc_slrg_reg_bits slrg_reg;
+	struct mlx5_ifc_slrp_reg_bits slrp_reg;
+	struct mlx5_ifc_sltp_reg_bits sltp_reg;
+	u8         reserved_0[0x7880];
+};
+
+union mlx5_ifc_debug_enhancements_document_bits {
+	struct mlx5_ifc_health_buffer_bits health_buffer;
+	u8         reserved_0[0x200];
+};
+
+union mlx5_ifc_no_dram_nic_document_bits {
+	struct mlx5_ifc_nodnic_config_reg_bits nodnic_config_reg;
+	struct mlx5_ifc_nodnic_cq_arming_word_bits nodnic_cq_arming_word;
+	struct mlx5_ifc_nodnic_event_word_bits nodnic_event_word;
+	struct mlx5_ifc_nodnic_gid_filters_bits nodnic_gid_filters;
+	struct mlx5_ifc_nodnic_mac_filters_bits nodnic_mac_filters;
+	struct mlx5_ifc_nodnic_port_config_reg_bits nodnic_port_config_reg;
+	struct mlx5_ifc_nodnic_ring_config_reg_bits nodnic_ring_config_reg;
+	struct mlx5_ifc_nodnic_ring_doorbell_bits nodnic_ring_doorbell;
+	u8         reserved_0[0x3160];
+};
+
+union mlx5_ifc_uplink_pci_interface_document_bits {
+	struct mlx5_ifc_initial_seg_bits initial_seg;
+	struct mlx5_ifc_vendor_specific_cap_bits vendor_specific_cap;
+	u8         reserved_0[0x20120];
+};
+
+
+#endif /* MLX5_IFC_H */
diff --git a/sys/dev/mlx5/mlx5_rdma_if.h b/sys/dev/mlx5/mlx5_rdma_if.h
new file mode 100644
index 0000000..8bb90cce
--- /dev/null
+++ b/sys/dev/mlx5/mlx5_rdma_if.h
@@ -0,0 +1,31 @@
+/*-
+ * Copyright (c) 2013-2015, Mellanox Technologies, Ltd.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef MLX5_RDMA_IF_H
+#define MLX5_RDMA_IF_H
+
+#endif /* MLX5_RDMA_IF_H */
diff --git a/sys/dev/mlx5/qp.h b/sys/dev/mlx5/qp.h
new file mode 100644
index 0000000..f8a7575
--- /dev/null
+++ b/sys/dev/mlx5/qp.h
@@ -0,0 +1,599 @@
+/*-
+ * Copyright (c) 2013-2015, Mellanox Technologies, Ltd.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef MLX5_QP_H
+#define MLX5_QP_H
+
+#include <dev/mlx5/device.h>
+#include <dev/mlx5/driver.h>
+#include <dev/mlx5/mlx5_ifc.h>
+
+#define MLX5_INVALID_LKEY	0x100
+#define MLX5_SIG_WQE_SIZE	(MLX5_SEND_WQE_BB * 5)
+#define MLX5_DIF_SIZE		8
+#define MLX5_STRIDE_BLOCK_OP	0x400
+#define MLX5_CPY_GRD_MASK	0xc0
+#define MLX5_CPY_APP_MASK	0x30
+#define MLX5_CPY_REF_MASK	0x0f
+#define MLX5_BSF_INC_REFTAG	(1 << 6)
+#define MLX5_BSF_INL_VALID	(1 << 15)
+#define MLX5_BSF_REFRESH_DIF	(1 << 14)
+#define MLX5_BSF_REPEAT_BLOCK	(1 << 7)
+#define MLX5_BSF_APPTAG_ESCAPE	0x1
+#define MLX5_BSF_APPREF_ESCAPE	0x2
+
+enum mlx5_qp_optpar {
+	MLX5_QP_OPTPAR_ALT_ADDR_PATH		= 1 << 0,
+	MLX5_QP_OPTPAR_RRE			= 1 << 1,
+	MLX5_QP_OPTPAR_RAE			= 1 << 2,
+	MLX5_QP_OPTPAR_RWE			= 1 << 3,
+	MLX5_QP_OPTPAR_PKEY_INDEX		= 1 << 4,
+	MLX5_QP_OPTPAR_Q_KEY			= 1 << 5,
+	MLX5_QP_OPTPAR_RNR_TIMEOUT		= 1 << 6,
+	MLX5_QP_OPTPAR_PRIMARY_ADDR_PATH	= 1 << 7,
+	MLX5_QP_OPTPAR_SRA_MAX			= 1 << 8,
+	MLX5_QP_OPTPAR_RRA_MAX			= 1 << 9,
+	MLX5_QP_OPTPAR_PM_STATE			= 1 << 10,
+	MLX5_QP_OPTPAR_RETRY_COUNT		= 1 << 12,
+	MLX5_QP_OPTPAR_RNR_RETRY		= 1 << 13,
+	MLX5_QP_OPTPAR_ACK_TIMEOUT		= 1 << 14,
+	MLX5_QP_OPTPAR_PRI_PORT			= 1 << 16,
+	MLX5_QP_OPTPAR_SRQN			= 1 << 18,
+	MLX5_QP_OPTPAR_CQN_RCV			= 1 << 19,
+	MLX5_QP_OPTPAR_DC_HS			= 1 << 20,
+	MLX5_QP_OPTPAR_DC_KEY			= 1 << 21,
+};
+
+enum mlx5_qp_state {
+	MLX5_QP_STATE_RST			= 0,
+	MLX5_QP_STATE_INIT			= 1,
+	MLX5_QP_STATE_RTR			= 2,
+	MLX5_QP_STATE_RTS			= 3,
+	MLX5_QP_STATE_SQER			= 4,
+	MLX5_QP_STATE_SQD			= 5,
+	MLX5_QP_STATE_ERR			= 6,
+	MLX5_QP_STATE_SQ_DRAINING		= 7,
+	MLX5_QP_STATE_SUSPENDED			= 9,
+	MLX5_QP_NUM_STATE
+};
+
+enum {
+	MLX5_QP_ST_RC				= 0x0,
+	MLX5_QP_ST_UC				= 0x1,
+	MLX5_QP_ST_UD				= 0x2,
+	MLX5_QP_ST_XRC				= 0x3,
+	MLX5_QP_ST_MLX				= 0x4,
+	MLX5_QP_ST_DCI				= 0x5,
+	MLX5_QP_ST_DCT				= 0x6,
+	MLX5_QP_ST_QP0				= 0x7,
+	MLX5_QP_ST_QP1				= 0x8,
+	MLX5_QP_ST_RAW_ETHERTYPE		= 0x9,
+	MLX5_QP_ST_RAW_IPV6			= 0xa,
+	MLX5_QP_ST_SNIFFER			= 0xb,
+	MLX5_QP_ST_SYNC_UMR			= 0xe,
+	MLX5_QP_ST_PTP_1588			= 0xd,
+	MLX5_QP_ST_REG_UMR			= 0xc,
+	MLX5_QP_ST_MAX
+};
+
+enum {
+	MLX5_NON_ZERO_RQ	= 0 << 24,
+	MLX5_SRQ_RQ		= 1 << 24,
+	MLX5_CRQ_RQ		= 2 << 24,
+	MLX5_ZERO_LEN_RQ	= 3 << 24
+};
+
+enum {
+	/* params1 */
+	MLX5_QP_BIT_SRE				= 1 << 15,
+	MLX5_QP_BIT_SWE				= 1 << 14,
+	MLX5_QP_BIT_SAE				= 1 << 13,
+	/* params2 */
+	MLX5_QP_BIT_RRE				= 1 << 15,
+	MLX5_QP_BIT_RWE				= 1 << 14,
+	MLX5_QP_BIT_RAE				= 1 << 13,
+	MLX5_QP_BIT_RIC				= 1 <<	4,
+};
+
+enum {
+	MLX5_WQE_CTRL_CQ_UPDATE		= 2 << 2,
+	MLX5_WQE_CTRL_CQ_UPDATE_AND_EQE	= 3 << 2,
+	MLX5_WQE_CTRL_SOLICITED		= 1 << 1,
+};
+
+enum {
+	MLX5_SEND_WQE_DS	= 16,
+	MLX5_SEND_WQE_BB	= 64,
+};
+
+#define MLX5_SEND_WQEBB_NUM_DS	(MLX5_SEND_WQE_BB / MLX5_SEND_WQE_DS)
+
+enum {
+	MLX5_SEND_WQE_MAX_WQEBBS	= 16,
+};
+
+enum {
+	MLX5_WQE_FMR_PERM_LOCAL_READ	= 1 << 27,
+	MLX5_WQE_FMR_PERM_LOCAL_WRITE	= 1 << 28,
+	MLX5_WQE_FMR_PERM_REMOTE_READ	= 1 << 29,
+	MLX5_WQE_FMR_PERM_REMOTE_WRITE	= 1 << 30,
+	MLX5_WQE_FMR_PERM_ATOMIC	= 1 << 31
+};
+
+enum {
+	MLX5_FENCE_MODE_NONE			= 0 << 5,
+	MLX5_FENCE_MODE_INITIATOR_SMALL		= 1 << 5,
+	MLX5_FENCE_MODE_STRONG_ORDERING		= 3 << 5,
+	MLX5_FENCE_MODE_SMALL_AND_FENCE		= 4 << 5,
+};
+
+enum {
+	MLX5_QP_LAT_SENSITIVE	= 1 << 28,
+	MLX5_QP_BLOCK_MCAST	= 1 << 30,
+	MLX5_QP_ENABLE_SIG	= 1 << 31,
+};
+
+enum {
+	MLX5_RCV_DBR	= 0,
+	MLX5_SND_DBR	= 1,
+};
+
+enum {
+	MLX5_FLAGS_INLINE	= 1<<7,
+	MLX5_FLAGS_CHECK_FREE   = 1<<5,
+};
+
+struct mlx5_wqe_fmr_seg {
+	__be32			flags;
+	__be32			mem_key;
+	__be64			buf_list;
+	__be64			start_addr;
+	__be64			reg_len;
+	__be32			offset;
+	__be32			page_size;
+	u32			reserved[2];
+};
+
+struct mlx5_wqe_ctrl_seg {
+	__be32			opmod_idx_opcode;
+	__be32			qpn_ds;
+	u8			signature;
+	u8			rsvd[2];
+	u8			fm_ce_se;
+	__be32			imm;
+};
+
+enum {
+	MLX5_ETH_WQE_L3_INNER_CSUM	= 1 << 4,
+	MLX5_ETH_WQE_L4_INNER_CSUM	= 1 << 5,
+	MLX5_ETH_WQE_L3_CSUM		= 1 << 6,
+	MLX5_ETH_WQE_L4_CSUM		= 1 << 7,
+};
+
+struct mlx5_wqe_eth_seg {
+	u8		rsvd0[4];
+	u8		cs_flags;
+	u8		rsvd1;
+	__be16		mss;
+	__be32		rsvd2;
+	__be16		inline_hdr_sz;
+	u8		inline_hdr_start[2];
+};
+
+struct mlx5_wqe_xrc_seg {
+	__be32			xrc_srqn;
+	u8			rsvd[12];
+};
+
+struct mlx5_wqe_masked_atomic_seg {
+	__be64			swap_add;
+	__be64			compare;
+	__be64			swap_add_mask;
+	__be64			compare_mask;
+};
+
+struct mlx5_av {
+	union {
+		struct {
+			__be32	qkey;
+			__be32	reserved;
+		} qkey;
+		__be64	dc_key;
+	} key;
+	__be32	dqp_dct;
+	u8	stat_rate_sl;
+	u8	fl_mlid;
+	union {
+		__be16	rlid;
+		__be16  udp_sport;
+	};
+	u8	reserved0[4];
+	u8	rmac[6];
+	u8	tclass;
+	u8	hop_limit;
+	__be32	grh_gid_fl;
+	u8	rgid[16];
+};
+
+struct mlx5_wqe_datagram_seg {
+	struct mlx5_av	av;
+};
+
+struct mlx5_wqe_raddr_seg {
+	__be64			raddr;
+	__be32			rkey;
+	u32			reserved;
+};
+
+struct mlx5_wqe_atomic_seg {
+	__be64			swap_add;
+	__be64			compare;
+};
+
+struct mlx5_wqe_data_seg {
+	__be32			byte_count;
+	__be32			lkey;
+	__be64			addr;
+};
+
+struct mlx5_wqe_umr_ctrl_seg {
+	u8		flags;
+	u8		rsvd0[3];
+	__be16		klm_octowords;
+	__be16		bsf_octowords;
+	__be64		mkey_mask;
+	u8		rsvd1[32];
+};
+
+struct mlx5_seg_set_psv {
+	__be32		psv_num;
+	__be16		syndrome;
+	__be16		status;
+	__be32		transient_sig;
+	__be32		ref_tag;
+};
+
+struct mlx5_seg_get_psv {
+	u8		rsvd[19];
+	u8		num_psv;
+	__be32		l_key;
+	__be64		va;
+	__be32		psv_index[4];
+};
+
+struct mlx5_seg_check_psv {
+	u8		rsvd0[2];
+	__be16		err_coalescing_op;
+	u8		rsvd1[2];
+	__be16		xport_err_op;
+	u8		rsvd2[2];
+	__be16		xport_err_mask;
+	u8		rsvd3[7];
+	u8		num_psv;
+	__be32		l_key;
+	__be64		va;
+	__be32		psv_index[4];
+};
+
+struct mlx5_rwqe_sig {
+	u8	rsvd0[4];
+	u8	signature;
+	u8	rsvd1[11];
+};
+
+struct mlx5_wqe_signature_seg {
+	u8	rsvd0[4];
+	u8	signature;
+	u8	rsvd1[11];
+};
+
+struct mlx5_wqe_inline_seg {
+	__be32	byte_count;
+};
+
+enum mlx5_sig_type {
+	MLX5_DIF_CRC = 0x1,
+	MLX5_DIF_IPCS = 0x2,
+};
+
+struct mlx5_bsf_inl {
+	__be16		vld_refresh;
+	__be16		dif_apptag;
+	__be32		dif_reftag;
+	u8		sig_type;
+	u8		rp_inv_seed;
+	u8		rsvd[3];
+	u8		dif_inc_ref_guard_check;
+	__be16		dif_app_bitmask_check;
+};
+
+struct mlx5_bsf {
+	struct mlx5_bsf_basic {
+		u8		bsf_size_sbs;
+		u8		check_byte_mask;
+		union {
+			u8	copy_byte_mask;
+			u8	bs_selector;
+			u8	rsvd_wflags;
+		} wire;
+		union {
+			u8	bs_selector;
+			u8	rsvd_mflags;
+		} mem;
+		__be32		raw_data_size;
+		__be32		w_bfs_psv;
+		__be32		m_bfs_psv;
+	} basic;
+	struct mlx5_bsf_ext {
+		__be32		t_init_gen_pro_size;
+		__be32		rsvd_epi_size;
+		__be32		w_tfs_psv;
+		__be32		m_tfs_psv;
+	} ext;
+	struct mlx5_bsf_inl	w_inl;
+	struct mlx5_bsf_inl	m_inl;
+};
+
+struct mlx5_klm {
+	__be32		bcount;
+	__be32		key;
+	__be64		va;
+};
+
+struct mlx5_stride_block_entry {
+	__be16		stride;
+	__be16		bcount;
+	__be32		key;
+	__be64		va;
+};
+
+struct mlx5_stride_block_ctrl_seg {
+	__be32		bcount_per_cycle;
+	__be32		op;
+	__be32		repeat_count;
+	u16		rsvd;
+	__be16		num_entries;
+};
+
+struct mlx5_core_qp {
+	struct mlx5_core_rsc_common	common; /* must be first */
+	void (*event)		(struct mlx5_core_qp *, int);
+	int			qpn;
+	struct mlx5_rsc_debug	*dbg;
+	int			pid;
+};
+
+struct mlx5_qp_path {
+	u8			fl_free_ar;
+	u8			rsvd3;
+	__be16			pkey_index;
+	u8			rsvd0;
+	u8			grh_mlid;
+	__be16			rlid;
+	u8			ackto_lt;
+	u8			mgid_index;
+	u8			static_rate;
+	u8			hop_limit;
+	__be32			tclass_flowlabel;
+	union {
+		u8		rgid[16];
+		u8		rip[16];
+	};
+	u8			f_dscp_ecn_prio;
+	u8			ecn_dscp;
+	__be16			udp_sport;
+	u8			dci_cfi_prio_sl;
+	u8			port;
+	u8			rmac[6];
+};
+
+struct mlx5_qp_context {
+	__be32			flags;
+	__be32			flags_pd;
+	u8			mtu_msgmax;
+	u8			rq_size_stride;
+	__be16			sq_crq_size;
+	__be32			qp_counter_set_usr_page;
+	__be32			wire_qpn;
+	__be32			log_pg_sz_remote_qpn;
+	struct			mlx5_qp_path pri_path;
+	struct			mlx5_qp_path alt_path;
+	__be32			params1;
+	u8			reserved2[4];
+	__be32			next_send_psn;
+	__be32			cqn_send;
+	u8			reserved3[8];
+	__be32			last_acked_psn;
+	__be32			ssn;
+	__be32			params2;
+	__be32			rnr_nextrecvpsn;
+	__be32			xrcd;
+	__be32			cqn_recv;
+	__be64			db_rec_addr;
+	__be32			qkey;
+	__be32			rq_type_srqn;
+	__be32			rmsn;
+	__be16			hw_sq_wqe_counter;
+	__be16			sw_sq_wqe_counter;
+	__be16			hw_rcyclic_byte_counter;
+	__be16			hw_rq_counter;
+	__be16			sw_rcyclic_byte_counter;
+	__be16			sw_rq_counter;
+	u8			rsvd0[5];
+	u8			cgs;
+	u8			cs_req;
+	u8			cs_res;
+	__be64			dc_access_key;
+	u8			rsvd1[24];
+};
+
+struct mlx5_create_qp_mbox_in {
+	struct mlx5_inbox_hdr	hdr;
+	__be32			input_qpn;
+	u8			rsvd0[4];
+	__be32			opt_param_mask;
+	u8			rsvd1[4];
+	struct mlx5_qp_context	ctx;
+	u8			rsvd3[16];
+	__be64			pas[0];
+};
+
+struct mlx5_create_qp_mbox_out {
+	struct mlx5_outbox_hdr	hdr;
+	__be32			qpn;
+	u8			rsvd0[4];
+};
+
+struct mlx5_destroy_qp_mbox_in {
+	struct mlx5_inbox_hdr	hdr;
+	__be32			qpn;
+	u8			rsvd0[4];
+};
+
+struct mlx5_destroy_qp_mbox_out {
+	struct mlx5_outbox_hdr	hdr;
+	u8			rsvd0[8];
+};
+
+struct mlx5_modify_qp_mbox_in {
+	struct mlx5_inbox_hdr	hdr;
+	__be32			qpn;
+	u8			rsvd1[4];
+	__be32			optparam;
+	u8			rsvd0[4];
+	struct mlx5_qp_context	ctx;
+};
+
+struct mlx5_modify_qp_mbox_out {
+	struct mlx5_outbox_hdr	hdr;
+	u8			rsvd0[8];
+};
+
+struct mlx5_query_qp_mbox_in {
+	struct mlx5_inbox_hdr	hdr;
+	__be32			qpn;
+	u8			rsvd[4];
+};
+
+struct mlx5_query_qp_mbox_out {
+	struct mlx5_outbox_hdr	hdr;
+	u8			rsvd1[8];
+	__be32			optparam;
+	u8			rsvd0[4];
+	struct mlx5_qp_context	ctx;
+	u8			rsvd2[16];
+	__be64			pas[0];
+};
+
+struct mlx5_conf_sqp_mbox_in {
+	struct mlx5_inbox_hdr	hdr;
+	__be32			qpn;
+	u8			rsvd[3];
+	u8			type;
+};
+
+struct mlx5_conf_sqp_mbox_out {
+	struct mlx5_outbox_hdr	hdr;
+	u8			rsvd[8];
+};
+
+static inline struct mlx5_core_qp *__mlx5_qp_lookup(struct mlx5_core_dev *dev, u32 qpn)
+{
+	return radix_tree_lookup(&dev->priv.qp_table.tree, qpn);
+}
+
+static inline struct mlx5_core_mr *__mlx5_mr_lookup(struct mlx5_core_dev *dev, u32 key)
+{
+	return radix_tree_lookup(&dev->priv.mr_table.tree, key);
+}
+
+int mlx5_core_create_qp(struct mlx5_core_dev *dev,
+			struct mlx5_core_qp *qp,
+			struct mlx5_create_qp_mbox_in *in,
+			int inlen);
+int mlx5_core_qp_modify(struct mlx5_core_dev *dev, enum mlx5_qp_state cur_state,
+			enum mlx5_qp_state new_state,
+			struct mlx5_modify_qp_mbox_in *in, int sqd_event,
+			struct mlx5_core_qp *qp);
+int mlx5_core_destroy_qp(struct mlx5_core_dev *dev,
+			 struct mlx5_core_qp *qp);
+int mlx5_core_qp_query(struct mlx5_core_dev *dev, struct mlx5_core_qp *qp,
+		       struct mlx5_query_qp_mbox_out *out, int outlen);
+
+int mlx5_core_xrcd_alloc(struct mlx5_core_dev *dev, u32 *xrcdn);
+int mlx5_core_xrcd_dealloc(struct mlx5_core_dev *dev, u32 xrcdn);
+void mlx5_init_qp_table(struct mlx5_core_dev *dev);
+void mlx5_cleanup_qp_table(struct mlx5_core_dev *dev);
+int mlx5_debug_qp_add(struct mlx5_core_dev *dev, struct mlx5_core_qp *qp);
+void mlx5_debug_qp_remove(struct mlx5_core_dev *dev, struct mlx5_core_qp *qp);
+
+static inline const char *mlx5_qp_type_str(int type)
+{
+	switch (type) {
+	case MLX5_QP_ST_RC: return "RC";
+	case MLX5_QP_ST_UC: return "C";
+	case MLX5_QP_ST_UD: return "UD";
+	case MLX5_QP_ST_XRC: return "XRC";
+	case MLX5_QP_ST_MLX: return "MLX";
+	case MLX5_QP_ST_QP0: return "QP0";
+	case MLX5_QP_ST_QP1: return "QP1";
+	case MLX5_QP_ST_RAW_ETHERTYPE: return "RAW_ETHERTYPE";
+	case MLX5_QP_ST_RAW_IPV6: return "RAW_IPV6";
+	case MLX5_QP_ST_SNIFFER: return "SNIFFER";
+	case MLX5_QP_ST_SYNC_UMR: return "SYNC_UMR";
+	case MLX5_QP_ST_PTP_1588: return "PTP_1588";
+	case MLX5_QP_ST_REG_UMR: return "REG_UMR";
+	default: return "Invalid transport type";
+	}
+}
+
+static inline const char *mlx5_qp_state_str(int state)
+{
+	switch (state) {
+	case MLX5_QP_STATE_RST:
+	return "RST";
+	case MLX5_QP_STATE_INIT:
+	return "INIT";
+	case MLX5_QP_STATE_RTR:
+	return "RTR";
+	case MLX5_QP_STATE_RTS:
+	return "RTS";
+	case MLX5_QP_STATE_SQER:
+	return "SQER";
+	case MLX5_QP_STATE_SQD:
+	return "SQD";
+	case MLX5_QP_STATE_ERR:
+	return "ERR";
+	case MLX5_QP_STATE_SQ_DRAINING:
+	return "SQ_DRAINING";
+	case MLX5_QP_STATE_SUSPENDED:
+	return "SUSPENDED";
+	default: return "Invalid QP state";
+	}
+}
+
+#endif /* MLX5_QP_H */
diff --git a/sys/dev/mlx5/srq.h b/sys/dev/mlx5/srq.h
new file mode 100644
index 0000000..c621e0a
--- /dev/null
+++ b/sys/dev/mlx5/srq.h
@@ -0,0 +1,36 @@
+/*-
+ * Copyright (c) 2013-2015, Mellanox Technologies, Ltd.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef MLX5_SRQ_H
+#define MLX5_SRQ_H
+
+#include <dev/mlx5/driver.h>
+
+void mlx5_init_srq_table(struct mlx5_core_dev *dev);
+void mlx5_cleanup_srq_table(struct mlx5_core_dev *dev);
+
+#endif /* MLX5_SRQ_H */
diff --git a/sys/dev/mlx5/vport.h b/sys/dev/mlx5/vport.h
new file mode 100644
index 0000000..7d79c59
--- /dev/null
+++ b/sys/dev/mlx5/vport.h
@@ -0,0 +1,74 @@
+/*-
+ * Copyright (c) 2013-2015, Mellanox Technologies, Ltd.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef __MLX5_VPORT_H__
+#define __MLX5_VPORT_H__
+
+#include <dev/mlx5/driver.h>
+int mlx5_vport_alloc_q_counter(struct mlx5_core_dev *mdev,
+			       int *counter_set_id);
+int mlx5_vport_dealloc_q_counter(struct mlx5_core_dev *mdev,
+				 int counter_set_id);
+int mlx5_vport_query_out_of_rx_buffer(struct mlx5_core_dev *mdev,
+				      int counter_set_id,
+				      u32 *out_of_rx_buffer);
+
+u8 mlx5_query_vport_state(struct mlx5_core_dev *mdev, u8 opmod);
+int mlx5_query_nic_vport_mac_address(struct mlx5_core_dev *mdev,
+				     u32 vport, u8 *addr);
+int mlx5_set_nic_vport_current_mac(struct mlx5_core_dev *mdev, int vport,
+				   bool other_vport, u8 *addr);
+int mlx5_set_nic_vport_permanent_mac(struct mlx5_core_dev *mdev, int vport,
+				     u8 *addr);
+int mlx5_nic_vport_enable_roce(struct mlx5_core_dev *mdev);
+int mlx5_nic_vport_disable_roce(struct mlx5_core_dev *mdev);
+int mlx5_query_nic_vport_system_image_guid(struct mlx5_core_dev *mdev,
+					   u64 *system_image_guid);
+int mlx5_query_nic_vport_node_guid(struct mlx5_core_dev *mdev, u64 *node_guid);
+int mlx5_query_nic_vport_port_guid(struct mlx5_core_dev *mdev, u64 *port_guid);
+int mlx5_query_nic_vport_qkey_viol_cntr(struct mlx5_core_dev *mdev,
+					u16 *qkey_viol_cntr);
+int mlx5_query_hca_vport_node_guid(struct mlx5_core_dev *mdev, u64 *node_guid);
+int mlx5_query_hca_vport_system_image_guid(struct mlx5_core_dev *mdev,
+					   u64 *system_image_guid);
+int mlx5_query_hca_vport_context(struct mlx5_core_dev *mdev,
+				 u8 port_num, u8 vport_num, u32 *out,
+				 int outlen);
+int mlx5_query_hca_vport_pkey(struct mlx5_core_dev *dev, u8 other_vport,
+			      u8 port_num, u16 vf_num, u16 pkey_index,
+			      u16 *pkey);
+int mlx5_query_hca_vport_gid(struct mlx5_core_dev *dev, u8 port_num,
+			     u16 vport_num, u16 gid_index, union ib_gid *gid);
+int mlx5_set_eswitch_cvlan_info(struct mlx5_core_dev *mdev, u8 vport,
+				u8 insert_mode, u8 strip_mode,
+				u16 vlan, u8 cfi, u8 pcp);
+int mlx5_query_vport_counter(struct mlx5_core_dev *dev,
+			     u8 port_num, u16 vport_num,
+			     void *out, int out_size);
+int mlx5_get_vport_counters(struct mlx5_core_dev *dev, u8 port_num,
+			    struct mlx5_vport_counters *vc);
+#endif /* __MLX5_VPORT_H__ */
diff --git a/sys/modules/mlx5/Makefile b/sys/modules/mlx5/Makefile
new file mode 100644
index 0000000..4c19c4c
--- /dev/null
+++ b/sys/modules/mlx5/Makefile
@@ -0,0 +1,36 @@
+# $FreeBSD$
+.PATH:	${.CURDIR}/../../dev/mlx5/mlx5_core ${.CURDIR}/../../ofed/include/linux
+
+KMOD=mlx5
+SRCS= \
+mlx5_alloc.c \
+mlx5_cmd.c \
+mlx5_cq.c \
+mlx5_eq.c \
+mlx5_flow_table.c \
+mlx5_fw.c \
+mlx5_health.c \
+mlx5_mad.c \
+mlx5_main.c \
+mlx5_mcg.c \
+mlx5_mr.c \
+mlx5_pagealloc.c \
+mlx5_pd.c \
+mlx5_port.c \
+mlx5_qp.c \
+mlx5_srq.c \
+mlx5_transobj.c \
+mlx5_uar.c \
+mlx5_vport.c \
+mlx5_wq.c \
+device_if.h bus_if.h vnode_if.h pci_if.h \
+        opt_inet.h opt_inet6.h opt_random.h
+
+SRCS+= linux_compat.c linux_radix.c linux_idr.c
+
+CFLAGS+= -I${.CURDIR}/../../ofed/include
+CFLAGS+= -I${.CURDIR}/../../compat/linuxkpi/common/include
+
+.include <bsd.kmod.mk>
+
+CFLAGS+= -Wno-cast-qual -Wno-pointer-arith ${GCC_MS_EXTENSIONS}
diff --git a/sys/modules/mlx5en/Makefile b/sys/modules/mlx5en/Makefile
new file mode 100644
index 0000000..b112bf55
--- /dev/null
+++ b/sys/modules/mlx5en/Makefile
@@ -0,0 +1,29 @@
+# $FreeBSD$
+.PATH:	${.CURDIR}/../../dev/mlx5/mlx5_en
+
+KMOD=mlx5en
+SRCS= \
+mlx5_en_ethtool.c \
+mlx5_en_main.c \
+mlx5_en_tx.c \
+mlx5_en_flow_table.c \
+mlx5_en_rx.c \
+mlx5_en_txrx.c \
+device_if.h bus_if.h vnode_if.h pci_if.h \
+        opt_inet.h opt_inet6.h
+
+.if defined(HAVE_TURBO_LRO)
+CFLAGS+= -DHAVE_TURBO_LRO
+SRCS+= tcp_tlro.c
+.endif
+
+.if defined(HAVE_PER_CQ_EVENT_PACKET)
+CFLAGS+= -DHAVE_PER_CQ_EVENT_PACKET
+.endif
+
+CFLAGS+= -I${.CURDIR}/../../ofed/include
+CFLAGS+= -I${.CURDIR}/../../compat/linuxkpi/common/include
+
+.include <bsd.kmod.mk>
+
+CFLAGS+= -Wno-cast-qual -Wno-pointer-arith ${GCC_MS_EXTENSIONS}
diff --git a/sys/ofed/drivers/net/mlx4/cmd.c b/sys/ofed/drivers/net/mlx4/cmd.c
index 581651f..f0805e8 100644
--- a/sys/ofed/drivers/net/mlx4/cmd.c
+++ b/sys/ofed/drivers/net/mlx4/cmd.c
@@ -1784,6 +1784,7 @@ static int mlx4_master_immediate_activate_vlan_qos(struct mlx4_priv *priv,
 				mlx4_warn((&priv->dev),
 					  "No vlan resources slave %d, port %d\n",
 					  slave, port);
+				kfree(work);
 				return err;
 			}
 		} else {
diff --git a/sys/sys/_stdint.h b/sys/sys/_stdint.h
index d0f9249..a0fe0ad 100644
--- a/sys/sys/_stdint.h
+++ b/sys/sys/_stdint.h
@@ -78,5 +78,13 @@ typedef	__intptr_t		intptr_t;
 typedef	__uintptr_t		uintptr_t;
 #define	_UINTPTR_T_DECLARED
 #endif
+#ifndef _INTMAX_T_DECLARED
+typedef	__intmax_t		intmax_t;
+#define	_INTMAX_T_DECLARED
+#endif
+#ifndef _UINTMAX_T_DECLARED
+typedef	__uintmax_t		uintmax_t;
+#define	_UINTMAX_T_DECLARED
+#endif
 
 #endif /* !_SYS__STDINT_H_ */
diff --git a/sys/sys/stdint.h b/sys/sys/stdint.h
index 762e879..ec3698b 100644
--- a/sys/sys/stdint.h
+++ b/sys/sys/stdint.h
@@ -55,15 +55,6 @@ typedef	__uint_fast16_t		uint_fast16_t;
 typedef	__uint_fast32_t		uint_fast32_t;
 typedef	__uint_fast64_t		uint_fast64_t;
 
-#ifndef _INTMAX_T_DECLARED
-typedef	__intmax_t		intmax_t;
-#define	_INTMAX_T_DECLARED
-#endif
-#ifndef _UINTMAX_T_DECLARED
-typedef	__uintmax_t		uintmax_t;
-#define	_UINTMAX_T_DECLARED
-#endif
-
 /* GNU and Darwin define this and people seem to think it's portable */
 #if defined(UINTPTR_MAX) && defined(UINT64_MAX) && (UINTPTR_MAX == UINT64_MAX)
 #define	__WORDSIZE		64
diff --git a/tests/sys/kern/Makefile b/tests/sys/kern/Makefile
index c345e5d..5328b99 100644
--- a/tests/sys/kern/Makefile
+++ b/tests/sys/kern/Makefile
@@ -1,5 +1,9 @@
 # $FreeBSD$
 
+SRCTOP=		${.CURDIR:H:H:H}
+OBJTOP=		${.OBJDIR:H:H:H}
+TESTSRC=	${SRCTOP}/contrib/netbsd-tests/kernel
+
 TESTSDIR=	${TESTSBASE}/sys/kern
 
 ATF_TESTS_C+=	kern_descrip_test
@@ -10,8 +14,14 @@ TEST_METADATA.unix_seqpacket_test+=	timeout="15"
 LDADD.ptrace_test+=			-lpthread
 LDADD.unix_seqpacket_test+=		-lpthread
 
+NETBSD_ATF_TESTS_C+=	lockf_test
+
 WARNS?=	5
 
+TESTS_SUBDIRS+=	acct
 TESTS_SUBDIRS+=	execve
+TESTS_SUBDIRS+=	pipe
+
+.include <netbsd-tests.test.mk>
 
 .include <bsd.test.mk>
diff --git a/tests/sys/kern/acct/Makefile b/tests/sys/kern/acct/Makefile
new file mode 100644
index 0000000..966fe9a
--- /dev/null
+++ b/tests/sys/kern/acct/Makefile
@@ -0,0 +1,17 @@
+# $FreeBSD$
+
+TESTSDIR=	${TESTSBASE}/sys/kern/acct
+
+ATF_TESTS_C=	acct_test
+
+CFLAGS+=	-I${.OBJDIR}
+
+CLEANFILES+=	convert.c
+
+DPSRCS.acct_test=	convert.c
+
+convert.c: ${SRCTOP}/sys/kern/kern_acct.c
+	sed -n -e 's/log(/syslog(/g' \
+	       -e '/FLOAT_CONVERSION_START/,/FLOAT_CONVERSION_END/p' ${.ALLSRC} >>${.TARGET}
+
+.include <bsd.test.mk>
diff --git a/tests/sys/kern/acct/acct_test.c b/tests/sys/kern/acct/acct_test.c
new file mode 100644
index 0000000..2bf221c
--- /dev/null
+++ b/tests/sys/kern/acct/acct_test.c
@@ -0,0 +1,237 @@
+/*-
+ * Copyright (c) 2011 Giorgos Keramidas. All rights reserved.
+ * Copyright (c) 2007 Diomidis Spinellis. All rights reserved.
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/types.h>
+
+#include <assert.h>
+#include <errno.h>
+#include <float.h>
+#include <limits.h>
+#include <math.h>
+#include <stdio.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <strings.h>
+#include <syslog.h>
+#include <time.h>
+
+#include <atf-c.h>
+
+#define KASSERT(val, msg) assert(val)
+
+typedef u_int32_t comp_t;
+
+#define AHZ 1000000
+
+#include "convert.c"
+
+union cf {
+	comp_t c;
+	float f;
+};
+
+static void
+check_result(const char *name, float expected, union cf v)
+{
+	double eps;
+
+	eps = fabs(expected - v.f) / expected;
+	ATF_CHECK(eps <= FLT_EPSILON);
+	if (eps > FLT_EPSILON) {
+		printf("Error in %s\n", name);
+		printf("Got      0x%08x %12g\n", v.c, v.f);
+		v.f = expected;
+		printf("Expected 0x%08x %12g (%.15lg)\n", v.c, v.f, expected);
+		printf("Epsilon=%lg, rather than %g\n", eps, FLT_EPSILON);
+	}
+}
+
+/*
+ * Test case for encoding {0 sec, 0 usec} within a reasonable epsilon.
+ */
+
+ATF_TC_WITHOUT_HEAD(encode_tv_zero);
+ATF_TC_BODY(encode_tv_zero, tc)
+{
+	union cf v;
+	struct timeval tv;
+
+	tv.tv_sec = 0;
+	tv.tv_usec = 0;
+	v.c = encode_timeval(tv);
+	ATF_CHECK(fabs(v.f - 0.0) < FLT_EPSILON);
+}
+
+/*
+ * Test case for encoding a random long number.
+ */
+
+ATF_TC_WITHOUT_HEAD(encode_long);
+ATF_TC_BODY(encode_long, tc)
+{
+	union cf v;
+	long l;
+
+	l = random();
+	v.c = encode_long(l);
+	check_result(atf_tc_get_ident(tc), l, v);
+}
+
+/*
+ * Test case for encoding a small number of seconds {1 sec, 0 usec}.
+ */
+
+ATF_TC_WITHOUT_HEAD(encode_tv_only_sec);
+ATF_TC_BODY(encode_tv_only_sec, tc)
+{
+	union cf v;
+	struct timeval tv;
+
+	tv.tv_sec = 1;
+	tv.tv_usec = 0;
+	v.c = encode_timeval(tv);
+	check_result(atf_tc_get_ident(tc),
+	    (float)tv.tv_sec * AHZ + tv.tv_usec, v);
+}
+
+/*
+ * Test case for encoding a small number of usec {0 sec, 1 usec}.
+ */
+
+ATF_TC_WITHOUT_HEAD(encode_tv_only_usec);
+ATF_TC_BODY(encode_tv_only_usec, tc)
+{
+	union cf v;
+	struct timeval tv;
+
+	tv.tv_sec = 0;
+	tv.tv_usec = 1;
+	v.c = encode_timeval(tv);
+	check_result(atf_tc_get_ident(tc),
+	    (float)tv.tv_sec * AHZ + tv.tv_usec, v);
+}
+
+/*
+ * Test case for encoding a large number of usec {1 sec, 999.999 usec}.
+ */
+
+ATF_TC_WITHOUT_HEAD(encode_tv_many_usec);
+ATF_TC_BODY(encode_tv_many_usec, tc)
+{
+	union cf v;
+	struct timeval tv;
+
+	tv.tv_sec = 1;
+	tv.tv_usec = 999999L;
+	v.c = encode_timeval(tv);
+	check_result(atf_tc_get_ident(tc),
+	    (float)tv.tv_sec * AHZ + tv.tv_usec, v);
+}
+
+/*
+ * Test case for encoding a huge number of usec {1 sec, 1.000.000 usec} that
+ * overflows the usec counter and should show up as an increase in timeval's
+ * seconds instead.
+ */
+
+ATF_TC_WITHOUT_HEAD(encode_tv_usec_overflow);
+ATF_TC_BODY(encode_tv_usec_overflow, tc)
+{
+	union cf v;
+	struct timeval tv;
+
+	tv.tv_sec = 1;
+	tv.tv_usec = 1000000L;
+	v.c = encode_timeval(tv);
+	check_result(atf_tc_get_ident(tc),
+	    (float)tv.tv_sec * AHZ + tv.tv_usec, v);
+}
+
+/*
+ * Test case for encoding a very large number of seconds, one that is very
+ * near to the limit of 32-bit signed values.  With a usec value of 999.999
+ * microseconds this should result in the largest value we can represent with
+ * a timeval struct.
+ */
+
+ATF_TC_WITHOUT_HEAD(encode_tv_upper_limit);
+ATF_TC_BODY(encode_tv_upper_limit, tc)
+{
+	union cf v;
+	struct timeval tv;
+
+	tv.tv_sec = 2147483647L;
+	tv.tv_usec = 999999L;
+	v.c = encode_timeval(tv);
+	check_result(atf_tc_get_ident(tc),
+	    (float)tv.tv_sec * AHZ + tv.tv_usec, v);
+}
+
+/*
+ * Test case for encoding a million random timeval objects, and checking that
+ * the conversion does not diverge too much from the expected values.
+ */
+
+ATF_TC_WITHOUT_HEAD(encode_tv_random_million);
+ATF_TC_BODY(encode_tv_random_million, tc)
+{
+	union cf v;
+	struct timeval tv;
+	long k;
+
+	atf_tc_expect_fail("the testcase violates FLT_EPSILON");
+
+	ATF_REQUIRE_MSG(unsetenv("TZ") == 0, "unsetting TZ failed; errno=%d", errno);
+
+	for (k = 1; k < 1000000L; k++) {
+		tv.tv_sec = random();
+		tv.tv_usec = (random() % 1000000L);
+		v.c = encode_timeval(tv);
+		check_result(atf_tc_get_ident(tc),
+		    (float)tv.tv_sec * AHZ + tv.tv_usec, v);
+	}
+}
+
+/* ---------------------------------------------------------------------
+ * Main.
+ * --------------------------------------------------------------------- */
+
+ATF_TP_ADD_TCS(tp)
+{
+
+	ATF_TP_ADD_TC(tp, encode_long);
+	ATF_TP_ADD_TC(tp, encode_tv_zero);
+	ATF_TP_ADD_TC(tp, encode_tv_only_sec);
+	ATF_TP_ADD_TC(tp, encode_tv_only_usec);
+	ATF_TP_ADD_TC(tp, encode_tv_many_usec);
+	ATF_TP_ADD_TC(tp, encode_tv_usec_overflow);
+	ATF_TP_ADD_TC(tp, encode_tv_upper_limit);
+	ATF_TP_ADD_TC(tp, encode_tv_random_million);
+
+	return atf_no_error();
+}
diff --git a/tests/sys/kern/pipe/Makefile b/tests/sys/kern/pipe/Makefile
new file mode 100644
index 0000000..bcd2d07
--- /dev/null
+++ b/tests/sys/kern/pipe/Makefile
@@ -0,0 +1,16 @@
+# $FreeBSD$
+
+TESTSDIR=	${TESTSBASE}/sys/kern/pipe
+
+PLAIN_TESTS_C+= big_pipe_test
+PLAIN_TESTS_C+= pipe_fstat_bug_test
+PLAIN_TESTS_C+= pipe_ino_test
+PLAIN_TESTS_C+= pipe_overcommit1_test
+PLAIN_TESTS_C+= pipe_overcommit2_test
+PLAIN_TESTS_C+= pipe_reverse2_test
+PLAIN_TESTS_C+= pipe_reverse_test
+PLAIN_TESTS_C+= pipe_wraparound_test
+
+WARNS?=		6
+
+.include <bsd.test.mk>
diff --git a/tests/sys/kern/pipe/big_pipe_test.c b/tests/sys/kern/pipe/big_pipe_test.c
new file mode 100644
index 0000000..fa5687e
--- /dev/null
+++ b/tests/sys/kern/pipe/big_pipe_test.c
@@ -0,0 +1,88 @@
+#include <sys/select.h>
+#include <err.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#define BIG_PIPE_SIZE  64*1024 /* From sys/pipe.h */
+
+/*
+ * Test for the non-blocking big pipe bug (write(2) returning
+ * EAGAIN while select(2) returns the descriptor as ready for write).
+ *
+ * $FreeBSD$
+ */
+
+static void
+write_frame(int fd, char *buf, unsigned long buflen)
+{
+	fd_set wfd;
+	int i;
+
+	while (buflen) {
+		FD_ZERO(&wfd);
+		FD_SET(fd, &wfd);
+		i = select(fd+1, NULL, &wfd, NULL, NULL);
+		if (i < 0)
+			err(1, "select failed");
+		if (i != 1) {
+			errx(1, "select returned unexpected value %d\n", i);
+			exit(1);
+		}
+		i = write(fd, buf, buflen);
+		if (i < 0) {
+			if (errno != EAGAIN)
+				warn("write failed");
+			exit(1);
+		}
+		buf += i;
+		buflen -= i;
+	}
+}
+
+int
+main(void)
+{
+	/* any value over PIPE_SIZE should do */
+	char buf[BIG_PIPE_SIZE];
+	int i, flags, fd[2];
+
+	if (pipe(fd) < 0)
+		errx(1, "pipe failed");
+
+	flags = fcntl(fd[1], F_GETFL);
+	if (flags == -1 || fcntl(fd[1], F_SETFL, flags|O_NONBLOCK) == -1) {
+		printf("fcntl failed: %s\n", strerror(errno));
+		exit(1);
+	}
+
+	switch (fork()) {
+	case -1:
+		err(1, "fork failed: %s\n", strerror(errno));
+		break;
+	case 0:
+		close(fd[1]);
+		for (;;) {
+			/* Any small size should do */
+			i = read(fd[0], buf, 256);
+			if (i == 0)
+				break;
+			if (i < 0)
+				err(1, "read");
+		}
+		exit(0);
+	default:
+		break;
+	}
+
+	close(fd[0]);
+	memset(buf, 0, sizeof buf);
+	for (i = 0; i < 1000; i++)
+		write_frame(fd[1], buf, sizeof buf);
+
+	printf("ok\n");
+	exit(0);
+}
diff --git a/tests/sys/kern/pipe/pipe_fstat_bug_test.c b/tests/sys/kern/pipe/pipe_fstat_bug_test.c
new file mode 100644
index 0000000..ae0b309
--- /dev/null
+++ b/tests/sys/kern/pipe/pipe_fstat_bug_test.c
@@ -0,0 +1,138 @@
+/*
+Copyright (C) 2004 Michael J. Silbersack. All rights reserved.
+ 
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+1. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGE.
+*/
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/wait.h>
+#include <assert.h>
+#include <err.h>
+#include <errno.h>
+#include <inttypes.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+/*
+ * $FreeBSD$
+ * The goal of this program is to see if fstat reports the correct
+ * data count for a pipe.  Prior to revision 1.172 of sys_pipe.c,
+ * 0 would be returned once the pipe entered direct write mode.
+ *
+ * Linux (2.6) always returns zero, so it's not a valuable platform
+ * for comparison.
+ */
+
+int
+main(void)
+{
+	char buffer[32768], buffer2[32768], go[] = "go", go2[] = "go2";
+	int desc[2], ipc_coord[2];
+	ssize_t error;
+	int successes = 0;
+	struct stat status;
+	pid_t new_pid;
+
+	error = pipe(desc);
+	if (error == -1)
+		err(1, "Couldn't allocate data pipe");
+
+	error = pipe(ipc_coord);
+	if (error == -1)
+		err(1, "Couldn't allocate IPC coordination pipe");
+
+	new_pid = fork();
+	assert(new_pid != -1);
+
+	close(new_pid == 0 ? desc[0] : desc[1]);
+
+#define	SYNC_R(i, _buf) do {	\
+	int _error = errno; \
+	warnx("%d: waiting for synchronization", __LINE__); \
+	if (read(ipc_coord[i], &_buf, sizeof(_buf)) != sizeof(_buf)) \
+		err(1, "failed to synchronize (%s)", (i == 0 ? "parent" : "child")); \
+	errno = _error; \
+	} while(0)
+
+#define	SYNC_W(i, _buf) do {	\
+	int _error = errno; \
+	warnx("%d: sending synchronization", __LINE__); \
+	if (write(ipc_coord[i], &_buf, sizeof(_buf)) != sizeof(_buf)) \
+		err(1, "failed to synchronize (%s)", (i == 0 ? "child" : "parent")); \
+	errno = _error; \
+	} while(0)
+
+#define	WRITE(s) do { 							\
+	ssize_t _size; 							\
+	if ((_size = write(desc[1], &buffer, s)) != s)			\
+		warn("short write; wrote %zd, expected %d", _size, s);	\
+	} while(0)
+
+	if (new_pid == 0) {
+
+		SYNC_R(0, go);
+		WRITE(145);
+		SYNC_W(0, go2);
+
+		SYNC_R(0, go);
+		WRITE(2048);
+		SYNC_W(0, go2);
+
+		SYNC_R(0, go);
+		WRITE(4096);
+		SYNC_W(0, go2);
+
+		SYNC_R(0, go);
+		WRITE(8191);
+		SYNC_W(0, go2);
+
+		SYNC_R(0, go);
+		SYNC_W(0, go2); /* XXX: why is this required? */
+		WRITE(8192);
+		SYNC_W(0, go2);
+
+		close(ipc_coord[0]);
+		close(ipc_coord[1]);
+
+		_exit(0);
+	}
+
+	while (successes < 5) {
+		SYNC_W(1, go);
+		SYNC_R(1, go2);
+		fstat(desc[0], &status);
+		error = read(desc[0], &buffer2, sizeof(buffer2));
+
+		if (status.st_size != error)
+			err(1, "FAILURE: stat size %jd read size %zd",
+			    (intmax_t)status.st_size, error);
+		if (error > 0) {
+			printf("SUCCESS at stat size %jd read size %zd\n",
+			    (intmax_t)status.st_size, error);
+			successes++;
+		}
+	}
+
+	exit(0);
+}
diff --git a/tools/regression/pipe/pipe-ino.c b/tests/sys/kern/pipe/pipe_ino_test.c
index f739cc9..94692aa 100644
--- a/tools/regression/pipe/pipe-ino.c
+++ b/tests/sys/kern/pipe/pipe_ino_test.c
@@ -40,7 +40,7 @@
 #include <unistd.h>
 
 int
-main(int argc, char **argv)
+main(void)
 {
 	int pipefd[2];
 	struct stat st1, st2;
@@ -52,12 +52,11 @@ main(int argc, char **argv)
 		err(1, "FAIL: fstat st1");
 	if (fstat(pipefd[1], &st2) == -1)
 		err(1, "FAIL: fstat st2");
-	if (st1.st_dev != st2.st_dev || st1.st_dev == 0 || st2.st_dev == 0) {
-		errx(1, "FAIL: wrong dev number %d %d",
-		    st1.st_dev, st2.st_dev);
-	}
+	if (st1.st_dev != st2.st_dev || st1.st_dev == 0 || st2.st_dev == 0)
+		errx(1, "FAIL: wrong dev number %d %d", st1.st_dev, st2.st_dev);
 	if (st1.st_ino == st2.st_ino)
 		errx(1, "FAIL: inode numbers are equal: %d", st1.st_ino);
+
 	close(pipefd[0]);
 	close(pipefd[1]);
 	printf("PASS\n");
diff --git a/tools/regression/pipe/pipe-overcommit1.c b/tests/sys/kern/pipe/pipe_overcommit1_test.c
index c468658..f8f881d 100644
--- a/tools/regression/pipe/pipe-overcommit1.c
+++ b/tests/sys/kern/pipe/pipe_overcommit1_test.c
@@ -26,8 +26,10 @@
  * DAMAGE.
  */
 
+#include <sys/param.h>
 #include <stdio.h>
 #include <stdlib.h>
+#include <unistd.h>
 
 /*
  * $FreeBSD$
@@ -35,13 +37,16 @@
  * that using up all pipe memory doesn't cause a panic.
  */
 
-int main (void)
-
+int
+main(void)
 {
-	int i, returnval;
-	int pipes[10000];
-	for (i = 0; i < 10000; i++) {
+	int pipes[10000], returnval;
+	unsigned int i;
+
+	for (i = 0; i < nitems(pipes); i++) {
 		returnval = pipe(&pipes[i]);
 	}
 	printf("PASS\n");
+
+	exit(0);
 }
diff --git a/tools/regression/pipe/pipe-overcommit2.c b/tests/sys/kern/pipe/pipe_overcommit2_test.c
index 5979585..ee1e714 100644
--- a/tools/regression/pipe/pipe-overcommit2.c
+++ b/tests/sys/kern/pipe/pipe_overcommit2_test.c
@@ -26,9 +26,13 @@
  * DAMAGE.
  */
 
+#include <sys/param.h>
+#include <err.h>
+#include <errno.h>
+#include <fcntl.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <fcntl.h>
+#include <unistd.h>
 
 /*
  * $FreeBSD$
@@ -37,15 +41,21 @@
  * limit for that user has been exceeded.
  */
 
-int main (int argc, void *argv[])
-
+int
+main(void)
 {
-	int i, returnval, lastfd;
-	int pipes[10000];
+	char template[] = "pipe.XXXXXXXXXX";
+	int lastfd, pipes[10000], returnval;
+	unsigned int i;
+
+	lastfd = -1;
 
-	for (i = 0; i < 100000; i++) {
-		returnval = open(argv[0], O_RDONLY);
-		if (returnval < 1)
+	if (mkstemp(template) == -1)
+		err(1, "mkstemp failed");
+
+	for (i = 0; i < nitems(pipes); i++) {
+		returnval = open(template, O_RDONLY);
+		if (returnval == -1 && (errno == ENFILE || errno == EMFILE))
 			break; /* All descriptors exhausted. */
 		else
 			lastfd = returnval;
@@ -66,4 +76,8 @@ int main (int argc, void *argv[])
 		returnval = pipe(&pipes[i]);
 	}
 	printf("PASS\n");
+
+	unlink(template);
+
+	exit(0);
 }
diff --git a/tools/regression/pipe/pipe-reverse2.c b/tests/sys/kern/pipe/pipe_reverse2_test.c
index cd54190..4666d95 100644
--- a/tools/regression/pipe/pipe-reverse2.c
+++ b/tests/sys/kern/pipe/pipe_reverse2_test.c
@@ -26,17 +26,17 @@
  * $FreeBSD$
  */
 
-#include	<sys/select.h>
+#include <sys/select.h>
 
-#include	<err.h>
-#include	<stdio.h>
-#include	<unistd.h>
+#include <err.h>
+#include <stdio.h>
+#include <unistd.h>
 
 /*
  * Check that pipes can be selected for writing in the reverse direction.
  */
 int
-main(int argc, char *argv[])
+main(void)
 {
 	int pip[2];
 	fd_set set;
diff --git a/tests/sys/kern/pipe/pipe_reverse_test.c b/tests/sys/kern/pipe/pipe_reverse_test.c
new file mode 100644
index 0000000..fd4518d
--- /dev/null
+++ b/tests/sys/kern/pipe/pipe_reverse_test.c
@@ -0,0 +1,149 @@
+/*
+Copyright (C) 2004 Michael J. Silbersack. All rights reserved.
+ 
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+1. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGE.
+*/
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/wait.h>
+#include <assert.h>
+#include <err.h>
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+/*
+ * $FreeBSD$
+ * This program simply tests writing through the reverse direction of
+ * a pipe.  Nothing too fancy, it's only needed because most pipe-using
+ * programs never touch the reverse direction (it doesn't exist on
+ * Linux.)
+ */
+
+int
+main(void)
+{
+	char buffer[65535], buffer2[65535], go[] = "go", go2[] = "go2";
+	int desc[2], ipc_coord[2];
+	size_t i;
+	ssize_t total;
+	int buggy, error;
+	pid_t new_pid;
+
+	buggy = 0;
+	total = 0;
+
+	error = pipe(desc);
+	if (error == -1)
+		err(1, "Couldn't allocate data pipe");
+
+	error = pipe(ipc_coord);
+	if (error == -1)
+		err(1, "Couldn't allocate IPC coordination pipe");
+
+	buffer[0] = 'A';
+
+	for (i = 1; i < (int)sizeof(buffer); i++) {
+		buffer[i] = buffer[i - 1] + 1;
+		if (buffer[i] > 'Z')
+			buffer[i] = 'A';
+	}
+
+	new_pid = fork();
+	assert(new_pid != -1);
+
+#define	SYNC_R(i, _buf) do {	\
+	int _error = errno; \
+	warnx("%d: waiting for synchronization", __LINE__); \
+	if (read(ipc_coord[i], &_buf, sizeof(_buf)) != sizeof(_buf)) \
+		err(1, "failed to synchronize (%s)", (i == 0 ? "parent" : "child")); \
+	errno = _error; \
+	} while(0)
+
+#define	SYNC_W(i, _buf) do {	\
+	int _error = errno; \
+	warnx("%d: sending synchronization", __LINE__); \
+	if (write(ipc_coord[i], &_buf, sizeof(_buf)) != sizeof(_buf)) \
+		err(1, "failed to synchronize (%s)", (i == 0 ? "child" : "parent")); \
+	errno = _error; \
+	} while(0)
+
+#define	WRITE(s) do { 							\
+	ssize_t _size; 							\
+	if ((_size = write(desc[1], &buffer[total], s)) != s)		\
+		warn("short write; wrote %zd, expected %d", _size, s);	\
+	total += _size;							\
+	} while(0)
+
+	if (new_pid == 0) {
+		SYNC_R(0, go);
+		for (i = 0; i < 8; i++)
+			WRITE(4096);
+
+		SYNC_W(0, go2);
+		SYNC_R(0, go);
+
+		for (i = 0; i < 2; i++)
+			WRITE(4096);
+
+		SYNC_W(0, go2);
+
+		_exit(0);
+	}
+
+	SYNC_W(1, go);
+	SYNC_R(1, go2);
+
+	error = read(desc[0], &buffer2, 8 * 4096);
+	total += error;
+	printf("Read %d bytes\n", error);
+
+	SYNC_W(1, go);
+	SYNC_R(1, go2);
+
+	error = read(desc[0], &buffer2[total], 2 * 4096);
+	total += error;
+	printf("Read %d bytes, done\n", error);
+
+	if (memcmp(buffer, buffer2, total) != 0) {
+		for (i = 0; i < (size_t)total; i++) {
+			if (buffer[i] != buffer2[i]) {
+				buggy = 1;
+				printf("Location %zu input: %hhx "
+				    "output: %hhx\n",
+				    i, buffer[i], buffer2[i]);
+			}
+		}
+	}
+
+	waitpid(new_pid, NULL, 0);
+
+	if ((buggy == 1) || (total != 10 * 4096))
+		errx(1, "FAILED");
+	else
+		printf("SUCCESS\n");
+
+	exit(0);
+}
diff --git a/tests/sys/kern/pipe/pipe_wraparound_test.c b/tests/sys/kern/pipe/pipe_wraparound_test.c
new file mode 100644
index 0000000..1ee0203
--- /dev/null
+++ b/tests/sys/kern/pipe/pipe_wraparound_test.c
@@ -0,0 +1,140 @@
+/*
+Copyright (C) 2004 Michael J. Silbersack. All rights reserved.
+ 
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+1. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGE.
+*/
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/wait.h>
+#include <assert.h>
+#include <err.h>
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+/*
+ * $FreeBSD$
+ * This program tests to make sure that wraparound writes and reads
+ * are working, assuming that 16K socket buffers are used.  In order
+ * to really stress the pipe code with this test, kernel modifications
+ * nay be necessary.
+ */
+
+int main (void)
+{
+	char buffer[32768], buffer2[32768], go[] = "go", go2[] = "go2";
+	int desc[2], ipc_coord[2];
+	ssize_t error, total;
+	int buggy, i;
+	pid_t new_pid;
+
+	buggy = 0;
+	total = 0;
+
+	error = pipe(desc);
+	if (error == -1)
+		err(1, "Couldn't allocate data pipe");
+
+	error = pipe(ipc_coord);
+	if (error == -1)
+		err(1, "Couldn't allocate IPC coordination pipe");
+
+	buffer[0] = 'A';
+
+	for (i = 1; i < (int)sizeof(buffer); i++) {
+		buffer[i] = buffer[i - 1] + 1;
+		if (buffer[i] > 'Z')
+			buffer[i] = 'A';
+	}
+
+	new_pid = fork();
+	assert(new_pid != -1);
+
+#define	SYNC_R(i, _buf) do {	\
+	int _error = errno; \
+	warnx("%d: waiting for synchronization", __LINE__); \
+	if (read(ipc_coord[i], &_buf, sizeof(_buf)) != sizeof(_buf)) \
+		err(1, "failed to synchronize (%s)", (i == 0 ? "parent" : "child")); \
+	errno = _error; \
+	} while(0)
+
+#define	SYNC_W(i, _buf) do {	\
+	int _error = errno; \
+	warnx("%d: sending synchronization", __LINE__); \
+	if (write(ipc_coord[i], &_buf, sizeof(_buf)) != sizeof(_buf)) \
+		err(1, "failed to synchronize (%s)", (i == 0 ? "child" : "parent")); \
+	errno = _error; \
+	} while(0)
+
+#define	WRITE(s) do { 							\
+	ssize_t _size; 							\
+	if ((_size = write(desc[1], &buffer[total], s)) != s)		\
+		warn("short write; wrote %zd, expected %d", _size, s);	\
+	total += _size;							\
+	} while(0)
+
+	if (new_pid == 0) {
+		WRITE(4096);
+		WRITE(4096);
+		WRITE(4000);
+		SYNC_W(0, go2);
+
+		SYNC_R(0, go);
+		WRITE(3000);
+		WRITE(3000);
+		SYNC_W(0, go2);
+
+		_exit(0);
+	}
+
+	SYNC_R(1, go2);
+	error = read(desc[0], &buffer2, 8192);
+	total += error;
+	printf("Read %zd bytes\n", error);
+	SYNC_W(1, go);
+	SYNC_R(1, go2);
+	error = read(desc[0], &buffer2[total], 16384);
+	total += error;
+	printf("Read %zd bytes, done\n", error);
+
+	if (memcmp(buffer, buffer2, total) != 0) {
+		for (i = 0; i < total; i++) {
+			if (buffer[i] != buffer2[i]) {
+				buggy = 1;
+				printf("Location %d input: %hhx output: %hhx\n",
+				    i, buffer[i], buffer2[i]);
+			}
+		}
+	}
+
+	waitpid(new_pid, NULL, 0);
+
+	if (buggy)
+		errx(1, "FAILURE");
+
+	printf("SUCCESS\n");
+
+	exit(0);
+}
diff --git a/tools/regression/acct/Makefile b/tools/regression/acct/Makefile
deleted file mode 100644
index 581cc5f..0000000
--- a/tools/regression/acct/Makefile
+++ /dev/null
@@ -1,22 +0,0 @@
-#
-# $FreeBSD$
-#
-# "make" will compile the acct test programs
-#
-
-PROG=	pack
-SRCS=	pack.c
-MAN=
-CFLAGS+= -I${.OBJDIR}
-
-pack.o:	convert.c
-
-CLEANFILES+= convert.c
-convert.c: ../../../sys/kern/kern_acct.c
-	sed -n -e 's/log(/syslog(/g' \
-	       -e '/FLOAT_CONVERSION_START/,/FLOAT_CONVERSION_END/p' $? >$@
-
-regress: pack regress.t
-	${.CURDIR}/regress.t
-
-.include <bsd.prog.mk>
diff --git a/tools/regression/acct/pack.c b/tools/regression/acct/pack.c
deleted file mode 100644
index d1d3085..0000000
--- a/tools/regression/acct/pack.c
+++ /dev/null
@@ -1,108 +0,0 @@
-/*-
- * Copyright (c) 2007 Diomidis Spinellis. All rights reserved.
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include <sys/types.h>
-
-#include <assert.h>
-#include <float.h>
-#include <limits.h>
-#include <math.h>
-#include <stdio.h>
-#include <stdint.h>
-#include <stdlib.h>
-#include <strings.h>
-#include <syslog.h>
-#include <time.h>
-
-#define KASSERT(val, msg) assert(val)
-
-typedef u_int32_t comp_t;
-
-#define AHZ 1000000
-
-#include "convert.c"
-
-static int nerr;
-
-union cf {
-	comp_t c;
-	float f;
-};
-
-static void
-check_result(const char *name, float expected, union cf v)
-{
-	double eps;
-
-	eps = fabs(expected - v.f) / expected;
-	if (eps > FLT_EPSILON) {
-		printf("Error in %s\n", name);
-		printf("Got      0x%08x %12g\n", v.c, v.f);
-		v.f = expected;
-		printf("Expected 0x%08x %12g (%.15lg)\n", v.c, v.f, expected);
-		printf("Epsilon=%lg, rather than %g\n", eps, FLT_EPSILON);
-		nerr++;
-	}
-}
-
-int
-main(int argc, char *argv[])
-{
-	union cf v;
-	long l;
-	int i, end;
-	struct timeval tv;
-
-	if (argc == 2) {
-		/* Loop test */
-		end = atoi(argv[1]);
-		for (i = 0; i < end; i++) {
-			tv.tv_sec = random();
-			tv.tv_usec = (random() % 1000000);
-			v.c = encode_timeval(tv);
-			check_result("encode_timeval",
-			    (float)tv.tv_sec * AHZ + tv.tv_usec, v);
-			l = random();
-			v.c = encode_long(l);
-			check_result("encode_long", l, v);
-		}
-	} else if (argc == 3) {
-		/* Single-value timeval/long test */
-		tv.tv_sec = atol(argv[1]);
-		tv.tv_usec = atol(argv[2]);
-		v.c = encode_timeval(tv);
-		check_result("encode_timeval",
-		    (float)tv.tv_sec * AHZ + tv.tv_usec, v);
-		v.c = encode_long(tv.tv_sec);
-		check_result("encode_long", tv.tv_sec, v);
-	} else {
-		fprintf(stderr, "usage:\n%s repetitions\n%s sec usec\n",
-		    argv[0], argv[0]);
-		return (1);
-	}
-	return (nerr);
-}
diff --git a/tools/regression/acct/regress.t b/tools/regression/acct/regress.t
deleted file mode 100755
index 3f63c3a..0000000
--- a/tools/regression/acct/regress.t
+++ /dev/null
@@ -1,39 +0,0 @@
-#!/bin/sh
-#
-# $FreeBSD$
-#
-
-if test -z "${DIR}" ; then
-	DIR=$( make -V .OBJDIR )
-fi
-if test -z "${DIR}" ; then
-	DIR=$( dirname $0 )
-fi
-
-check()
-{
-	NUM=$1
-	shift
-	if $DIR/pack $*
-	then
-		echo "ok $NUM"
-	else
-		echo "not ok $NUM"
-	fi
-}
-
-(cd $DIR ; make pack) >/dev/null 2>&1
-
-echo 1..7
-
-check 1 0 0
-check 2 0 1
-check 3 1 0
-check 4 1 999999
-check 5 1 1000000
-check 6 2147483647 999999
-check 7 10000000
-
-(cd $DIR ; make clean) >/dev/null 2>&1
-
-exit 0
diff --git a/tools/regression/lib/libc/locale/Makefile b/tools/regression/lib/libc/locale/Makefile
deleted file mode 100644
index ebbd537..0000000
--- a/tools/regression/lib/libc/locale/Makefile
+++ /dev/null
@@ -1,27 +0,0 @@
-# $FreeBSD$
-
-TESTS=	test-mbrtowc	\
-	test-wcrtomb	\
-	test-mbsnrtowcs	\
-	test-mbsrtowcs	\
-	test-wcsnrtombs	\
-	test-wcsrtombs	\
-	test-btowc	\
-	test-mbrlen	\
-	test-mbtowc	\
-	test-wctomb	\
-	test-mbstowcs	\
-	test-wcstombs	\
-	test-mblen	\
-	test-iswctype	\
-	test-towctrans \
-	test-c16rtomb \
-	test-mbrtoc16
-
-.PHONY: tests
-tests: ${TESTS}
-	for p in ${TESTS}; do ${.OBJDIR}/$$p; done
-
-.PHONY: clean
-clean:
-	-rm -f ${TESTS}
diff --git a/tools/regression/lib/libc/locale/test-btowc.t b/tools/regression/lib/libc/locale/test-btowc.t
deleted file mode 100644
index 8bdfd03..0000000
--- a/tools/regression/lib/libc/locale/test-btowc.t
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/bin/sh
-# $FreeBSD$
-
-cd `dirname $0`
-
-executable=`basename $0 .t`
-
-make $executable 2>&1 > /dev/null
-
-exec ./$executable
diff --git a/tools/regression/lib/libc/locale/test-iswctype.t b/tools/regression/lib/libc/locale/test-iswctype.t
deleted file mode 100644
index 8bdfd03..0000000
--- a/tools/regression/lib/libc/locale/test-iswctype.t
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/bin/sh
-# $FreeBSD$
-
-cd `dirname $0`
-
-executable=`basename $0 .t`
-
-make $executable 2>&1 > /dev/null
-
-exec ./$executable
diff --git a/tools/regression/lib/libc/locale/test-mblen.t b/tools/regression/lib/libc/locale/test-mblen.t
deleted file mode 100644
index 8bdfd03..0000000
--- a/tools/regression/lib/libc/locale/test-mblen.t
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/bin/sh
-# $FreeBSD$
-
-cd `dirname $0`
-
-executable=`basename $0 .t`
-
-make $executable 2>&1 > /dev/null
-
-exec ./$executable
diff --git a/tools/regression/lib/libc/locale/test-mbrlen.t b/tools/regression/lib/libc/locale/test-mbrlen.t
deleted file mode 100644
index 8bdfd03..0000000
--- a/tools/regression/lib/libc/locale/test-mbrlen.t
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/bin/sh
-# $FreeBSD$
-
-cd `dirname $0`
-
-executable=`basename $0 .t`
-
-make $executable 2>&1 > /dev/null
-
-exec ./$executable
diff --git a/tools/regression/lib/libc/locale/test-mbrtoc16.c b/tools/regression/lib/libc/locale/test-mbrtoc16.c
deleted file mode 100644
index f709a9c..0000000
--- a/tools/regression/lib/libc/locale/test-mbrtoc16.c
+++ /dev/null
@@ -1,195 +0,0 @@
-/*-
- * Copyright (c) 2002 Tim J. Robbins
- * All rights reserved.
- *
- * Copyright (c) 2013 Ed Schouten <ed@FreeBSD.org>
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-/*
- * Test program for mbrtoc16() as specified by ISO/IEC 9899:2011.
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include <assert.h>
-#include <errno.h>
-#include <limits.h>
-#include <locale.h>
-#include <stdio.h>
-#include <string.h>
-#include <uchar.h>
-
-int
-main(int argc, char *argv[])
-{
-	mbstate_t s;
-	size_t len;
-	char16_t c16;
-
-	/*
-	 * C/POSIX locale.
-	 */
-
-	printf("1..1\n");
-
-	/* Null wide character, internal state. */
-	assert(mbrtoc16(&c16, "", 1, NULL) == 0);
-	assert(c16 == 0);
-
-	/* Null wide character. */
-	memset(&s, 0, sizeof(s));
-	assert(mbrtoc16(&c16, "", 1, &s) == 0);
-	assert(c16 == 0);
-
-	/* Latin letter A, internal state. */
-	assert(mbrtoc16(NULL, 0, 0, NULL) == 0);
-	assert(mbrtoc16(&c16, "A", 1, NULL) == 1);
-	assert(c16 == L'A');
-
-	/* Latin letter A. */
-	memset(&s, 0, sizeof(s));
-	assert(mbrtoc16(&c16, "A", 1, &s) == 1);
-	assert(c16 == L'A');
-
-	/* Incomplete character sequence. */
-	c16 = L'z';
-	memset(&s, 0, sizeof(s));
-	assert(mbrtoc16(&c16, "", 0, &s) == (size_t)-2);
-	assert(c16 == L'z');
-
-	/* Check that mbrtoc16() doesn't access the buffer when n == 0. */
-	c16 = L'z';
-	memset(&s, 0, sizeof(s));
-	assert(mbrtoc16(&c16, "", 0, &s) == (size_t)-2);
-	assert(c16 == L'z');
-
-	/* Check that mbrtoc16() doesn't read ahead too aggressively. */
-	memset(&s, 0, sizeof(s));
-	assert(mbrtoc16(&c16, "AB", 2, &s) == 1);
-	assert(c16 == L'A');
-	assert(mbrtoc16(&c16, "C", 1, &s) == 1);
-	assert(c16 == L'C');
-
-	/*
-	 * ISO-8859-1.
-	 */
-
-	assert(strcmp(setlocale(LC_CTYPE, "en_US.ISO8859-1"),
-	    "en_US.ISO8859-1") == 0);
-
-	/* Currency sign. */
-	memset(&s, 0, sizeof(s));
-	assert(mbrtoc16(&c16, "\xa4", 1, &s) == 1);
-	assert(c16 == 0xa4);
-
-	/*
-	 * ISO-8859-15.
-	 */
-
-	assert(strcmp(setlocale(LC_CTYPE, "en_US.ISO8859-15"),
-	    "en_US.ISO8859-15") == 0);
-
-	/* Euro sign. */
-	memset(&s, 0, sizeof(s));
-	assert(mbrtoc16(&c16, "\xa4", 1, &s) == 1);
-	assert(c16 == 0x20ac);
-
-	/*
-	 * UTF-8.
-	 */
-
-	assert(strcmp(setlocale(LC_CTYPE, "en_US.UTF-8"), "en_US.UTF-8") == 0);
-
-	/* Null wide character, internal state. */
-	assert(mbrtoc16(NULL, 0, 0, NULL) == 0);
-	assert(mbrtoc16(&c16, "", 1, NULL) == 0);
-	assert(c16 == 0);
-
-	/* Null wide character. */
-	memset(&s, 0, sizeof(s));
-	assert(mbrtoc16(&c16, "", 1, &s) == 0);
-	assert(c16 == 0);
-
-	/* Latin letter A, internal state. */
-	assert(mbrtoc16(NULL, 0, 0, NULL) == 0);
-	assert(mbrtoc16(&c16, "A", 1, NULL) == 1);
-	assert(c16 == L'A');
-
-	/* Latin letter A. */
-	memset(&s, 0, sizeof(s));
-	assert(mbrtoc16(&c16, "A", 1, &s) == 1);
-	assert(c16 == L'A');
-
-	/* Incomplete character sequence (zero length). */
-	c16 = L'z';
-	memset(&s, 0, sizeof(s));
-	assert(mbrtoc16(&c16, "", 0, &s) == (size_t)-2);
-	assert(c16 == L'z');
-
-	/* Incomplete character sequence (truncated double-byte). */
-	memset(&s, 0, sizeof(s));
-	c16 = 0;
-	assert(mbrtoc16(&c16, "\xc3", 1, &s) == (size_t)-2);
-
-	/* Same as above, but complete. */
-	memset(&s, 0, sizeof(s));
-	c16 = 0;
-	assert(mbrtoc16(&c16, "\xc3\x84", 2, &s) == 2);
-	assert(c16 == 0xc4);
-
-	/* Test restarting behaviour. */
-	memset(&s, 0, sizeof(s));
-	c16 = 0;
-	assert(mbrtoc16(&c16, "\xc3", 1, &s) == (size_t)-2);
-	assert(c16 == 0);
-	assert(mbrtoc16(&c16, "\xb7", 1, &s) == 1);
-	assert(c16 == 0xf7);
-
-	/* Surrogate pair. */
-	memset(&s, 0, sizeof(s));
-	c16 = 0;
-	assert(mbrtoc16(&c16, "\xf0\x9f\x92\xa9", 4, &s) == 4);
-	assert(c16 == 0xd83d);
-	assert(mbrtoc16(&c16, "", 0, &s) == (size_t)-3);
-	assert(c16 == 0xdca9);
-
-	/* Letter e with acute, precomposed. */
-	memset(&s, 0, sizeof(s));
-	c16 = 0;
-	assert(mbrtoc16(&c16, "\xc3\xa9", 2, &s) == 2);
-	assert(c16 == 0xe9);
-
-	/* Letter e with acute, combined. */
-	memset(&s, 0, sizeof(s));
-	c16 = 0;
-	assert(mbrtoc16(&c16, "\x65\xcc\x81", 3, &s) == 1);
-	assert(c16 == 0x65);
-	assert(mbrtoc16(&c16, "\xcc\x81", 2, &s) == 2);
-	assert(c16 == 0x301);
-
-	printf("ok 1 - mbrtoc16()\n");
-
-	return (0);
-}
diff --git a/tools/regression/lib/libc/locale/test-mbrtowc.t b/tools/regression/lib/libc/locale/test-mbrtowc.t
deleted file mode 100644
index 8bdfd03..0000000
--- a/tools/regression/lib/libc/locale/test-mbrtowc.t
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/bin/sh
-# $FreeBSD$
-
-cd `dirname $0`
-
-executable=`basename $0 .t`
-
-make $executable 2>&1 > /dev/null
-
-exec ./$executable
diff --git a/tools/regression/lib/libc/locale/test-mbsnrtowcs.t b/tools/regression/lib/libc/locale/test-mbsnrtowcs.t
deleted file mode 100644
index 8bdfd03..0000000
--- a/tools/regression/lib/libc/locale/test-mbsnrtowcs.t
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/bin/sh
-# $FreeBSD$
-
-cd `dirname $0`
-
-executable=`basename $0 .t`
-
-make $executable 2>&1 > /dev/null
-
-exec ./$executable
diff --git a/tools/regression/lib/libc/locale/test-mbsrtowcs.t b/tools/regression/lib/libc/locale/test-mbsrtowcs.t
deleted file mode 100644
index 8bdfd03..0000000
--- a/tools/regression/lib/libc/locale/test-mbsrtowcs.t
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/bin/sh
-# $FreeBSD$
-
-cd `dirname $0`
-
-executable=`basename $0 .t`
-
-make $executable 2>&1 > /dev/null
-
-exec ./$executable
diff --git a/tools/regression/lib/libc/locale/test-mbstowcs.t b/tools/regression/lib/libc/locale/test-mbstowcs.t
deleted file mode 100644
index 8bdfd03..0000000
--- a/tools/regression/lib/libc/locale/test-mbstowcs.t
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/bin/sh
-# $FreeBSD$
-
-cd `dirname $0`
-
-executable=`basename $0 .t`
-
-make $executable 2>&1 > /dev/null
-
-exec ./$executable
diff --git a/tools/regression/lib/libc/locale/test-mbtowc.t b/tools/regression/lib/libc/locale/test-mbtowc.t
deleted file mode 100644
index 8bdfd03..0000000
--- a/tools/regression/lib/libc/locale/test-mbtowc.t
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/bin/sh
-# $FreeBSD$
-
-cd `dirname $0`
-
-executable=`basename $0 .t`
-
-make $executable 2>&1 > /dev/null
-
-exec ./$executable
diff --git a/tools/regression/lib/libc/locale/test-towctrans.t b/tools/regression/lib/libc/locale/test-towctrans.t
deleted file mode 100644
index 8bdfd03..0000000
--- a/tools/regression/lib/libc/locale/test-towctrans.t
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/bin/sh
-# $FreeBSD$
-
-cd `dirname $0`
-
-executable=`basename $0 .t`
-
-make $executable 2>&1 > /dev/null
-
-exec ./$executable
diff --git a/tools/regression/lib/libc/locale/test-wcrtomb.t b/tools/regression/lib/libc/locale/test-wcrtomb.t
deleted file mode 100644
index 8bdfd03..0000000
--- a/tools/regression/lib/libc/locale/test-wcrtomb.t
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/bin/sh
-# $FreeBSD$
-
-cd `dirname $0`
-
-executable=`basename $0 .t`
-
-make $executable 2>&1 > /dev/null
-
-exec ./$executable
diff --git a/tools/regression/lib/libc/locale/test-wcsnrtombs.t b/tools/regression/lib/libc/locale/test-wcsnrtombs.t
deleted file mode 100644
index 8bdfd03..0000000
--- a/tools/regression/lib/libc/locale/test-wcsnrtombs.t
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/bin/sh
-# $FreeBSD$
-
-cd `dirname $0`
-
-executable=`basename $0 .t`
-
-make $executable 2>&1 > /dev/null
-
-exec ./$executable
diff --git a/tools/regression/lib/libc/locale/test-wcsrtombs.t b/tools/regression/lib/libc/locale/test-wcsrtombs.t
deleted file mode 100644
index 8bdfd03..0000000
--- a/tools/regression/lib/libc/locale/test-wcsrtombs.t
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/bin/sh
-# $FreeBSD$
-
-cd `dirname $0`
-
-executable=`basename $0 .t`
-
-make $executable 2>&1 > /dev/null
-
-exec ./$executable
diff --git a/tools/regression/lib/libc/locale/test-wcstombs.t b/tools/regression/lib/libc/locale/test-wcstombs.t
deleted file mode 100644
index 8bdfd03..0000000
--- a/tools/regression/lib/libc/locale/test-wcstombs.t
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/bin/sh
-# $FreeBSD$
-
-cd `dirname $0`
-
-executable=`basename $0 .t`
-
-make $executable 2>&1 > /dev/null
-
-exec ./$executable
diff --git a/tools/regression/lib/libc/locale/test-wctomb.t b/tools/regression/lib/libc/locale/test-wctomb.t
deleted file mode 100644
index 8bdfd03..0000000
--- a/tools/regression/lib/libc/locale/test-wctomb.t
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/bin/sh
-# $FreeBSD$
-
-cd `dirname $0`
-
-executable=`basename $0 .t`
-
-make $executable 2>&1 > /dev/null
-
-exec ./$executable
diff --git a/tools/regression/lib/libc/net/Makefile b/tools/regression/lib/libc/net/Makefile
deleted file mode 100644
index 4f19edf..0000000
--- a/tools/regression/lib/libc/net/Makefile
+++ /dev/null
@@ -1,12 +0,0 @@
-# $FreeBSD$
-
-TESTS=	test-ether test-eui64_aton test-eui64_ntoa
-CFLAGS+= -g -Wall
-
-.PHONY: tests
-tests: ${TESTS}
-	for p in ${TESTS}; do ${.OBJDIR}/$$p; done
-
-.PHONY: clean
-clean:
-	-rm -f ${TESTS}
diff --git a/tools/regression/lib/libc/net/test-ether.c b/tools/regression/lib/libc/net/test-ether.c
deleted file mode 100644
index 7408101..0000000
--- a/tools/regression/lib/libc/net/test-ether.c
+++ /dev/null
@@ -1,235 +0,0 @@
-/*-
- * Copyright (c) 2007 Robert N. M. Watson
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * $FreeBSD$
- */
-
-#include <sys/types.h>
-
-#include <net/ethernet.h>
-
-#include <stdio.h>
-#include <string.h>
-
-static int testnum;
-
-#define	OK()	do {							\
-	printf("ok %d %s\n", testnum, __func__);			\
-	return;								\
-} while (0)
-
-#define	NOTOK(why)	do {						\
-	printf("not ok %d %s # %s\n", testnum, __func__, why);		\
-	return;								\
-} while (0)
-
-#define	TODO()	NOTOK("TODO")
-
-static const char		*ether_line_string =
-				    "01:23:45:67:89:ab ether_line_hostname";
-static const char		*ether_line_hostname = "ether_line_hostname";
-static const struct ether_addr	 ether_line_addr = {
-	{ 0x01, 0x23, 0x45, 0x67, 0x89, 0xab }
-};
-
-static void
-test_ether_line(void)
-{
-	struct ether_addr e;
-	char hostname[256];
-
-	testnum++;
-	if (ether_line(ether_line_string, &e, hostname) != 0)
-		NOTOK("returned error");
-	if (bcmp(&e, &ether_line_addr, ETHER_ADDR_LEN) != 0)
-		NOTOK("bad address");
-	if (strcmp(hostname, ether_line_hostname) != 0) {
-		printf("hostname: %s\n", hostname);
-		NOTOK("bad hostname");
-	}
-	OK();
-}
-
-static const char		*ether_line_bad_1_string = "x";
-
-static void
-test_ether_line_bad_1(void)
-{
-	struct ether_addr e;
-	char hostname[256];
-
-	testnum++;
-	if (ether_line(ether_line_bad_1_string, &e, hostname) == 0)
-		NOTOK("returned success");
-	OK();
-}
-
-static const char		*ether_line_bad_2_string = "x x";
-
-static void
-test_ether_line_bad_2(void)
-{
-	struct ether_addr e;
-	char hostname[256];
-
-	testnum++;
-	if (ether_line(ether_line_bad_2_string, &e, hostname) == 0)
-		NOTOK("returned success");
-	OK();
-}
-
-static const char		*ether_aton_string = "01:23:45:67:89:ab";
-static const struct ether_addr	 ether_aton_addr = {
-	{ 0x01, 0x23, 0x45, 0x67, 0x89, 0xab }
-};
-
-static void
-test_ether_aton_r(void)
-{
-	struct ether_addr e, *ep;
-
-	testnum++;
-	ep = ether_aton_r(ether_aton_string, &e);
-	if (ep == NULL)
-		NOTOK("returned NULL");
-	if (ep != &e)
-		NOTOK("returned different pointer");
-	if (bcmp(&e, &ether_aton_addr, ETHER_ADDR_LEN) != 0)
-		NOTOK("bad address");
-	OK();
-}
-
-static const char		*ether_aton_bad_string = "x";
-
-static void
-test_ether_aton_r_bad(void)
-{
-	struct ether_addr e, *ep;
-
-	testnum++;
-	ep = ether_aton_r(ether_aton_bad_string, &e);
-	if (ep == &e)
-		NOTOK("returned success");
-	if (ep != NULL)
-		NOTOK("returned different pointer");
-	OK();
-}
-
-static void
-test_ether_aton(void)
-{
-	struct ether_addr *ep;
-
-	testnum++;
-	ep = ether_aton(ether_aton_string);
-	if (ep == NULL)
-		NOTOK("returned NULL");
-	if (bcmp(ep, &ether_aton_addr, ETHER_ADDR_LEN) != 0)
-		NOTOK("bad address");
-	OK();
-}
-
-static void
-test_ether_aton_bad(void)
-{
-	struct ether_addr *ep;
-
-	testnum++;
-	ep = ether_aton(ether_aton_bad_string);
-	if (ep != NULL)
-		NOTOK("returned success");
-	OK();
-}
-
-static const char		*ether_ntoa_string = "01:23:45:67:89:ab";
-static const struct ether_addr	 ether_ntoa_addr = {
-	{ 0x01, 0x23, 0x45, 0x67, 0x89, 0xab }
-};
-
-static void
-test_ether_ntoa_r(void)
-{
-	char buf[256], *cp;
-
-	testnum++;
-	cp = ether_ntoa_r(&ether_ntoa_addr, buf);
-	if (cp == NULL)
-		NOTOK("returned NULL");
-	if (cp != buf)
-		NOTOK("returned different pointer");
-	if (strcmp(cp, ether_ntoa_string) != 0)
-		NOTOK("bad string");
-	OK();
-}
-
-static void
-test_ether_ntoa(void)
-{
-	char *cp;
-
-	testnum++;
-	cp = ether_ntoa(&ether_ntoa_addr);
-	if (cp == NULL)
-		NOTOK("returned NULL");
-	if (strcmp(cp, ether_ntoa_string) != 0)
-		NOTOK("bad string");
-	OK();
-}
-
-static void
-test_ether_ntohost(void)
-{
-
-	testnum++;
-	TODO();
-}
-
-static void
-test_ether_hostton(void)
-{
-
-	testnum++;
-	TODO();
-}
-
-int
-main(int argc, char *argv[])
-{
-
-	printf("1..11\n");
-
-	test_ether_line();
-	test_ether_line_bad_1();
-	test_ether_line_bad_2();
-	test_ether_aton_r();
-	test_ether_aton_r_bad();
-	test_ether_aton();
-	test_ether_aton_bad();
-	test_ether_ntoa_r();
-	test_ether_ntoa();
-	test_ether_ntohost();
-	test_ether_hostton();
-	return (0);
-}
diff --git a/tools/regression/lib/libc/net/test-ether.t b/tools/regression/lib/libc/net/test-ether.t
deleted file mode 100644
index 8bdfd03..0000000
--- a/tools/regression/lib/libc/net/test-ether.t
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/bin/sh
-# $FreeBSD$
-
-cd `dirname $0`
-
-executable=`basename $0 .t`
-
-make $executable 2>&1 > /dev/null
-
-exec ./$executable
diff --git a/tools/regression/lib/libc/net/test-eui64_aton.t b/tools/regression/lib/libc/net/test-eui64_aton.t
deleted file mode 100644
index 8bdfd03..0000000
--- a/tools/regression/lib/libc/net/test-eui64_aton.t
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/bin/sh
-# $FreeBSD$
-
-cd `dirname $0`
-
-executable=`basename $0 .t`
-
-make $executable 2>&1 > /dev/null
-
-exec ./$executable
diff --git a/tools/regression/lib/libc/net/test-eui64_ntoa.t b/tools/regression/lib/libc/net/test-eui64_ntoa.t
deleted file mode 100644
index 8bdfd03..0000000
--- a/tools/regression/lib/libc/net/test-eui64_ntoa.t
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/bin/sh
-# $FreeBSD$
-
-cd `dirname $0`
-
-executable=`basename $0 .t`
-
-make $executable 2>&1 > /dev/null
-
-exec ./$executable
diff --git a/tools/regression/pipe/Makefile b/tools/regression/pipe/Makefile
deleted file mode 100644
index 8a5f83a..0000000
--- a/tools/regression/pipe/Makefile
+++ /dev/null
@@ -1,16 +0,0 @@
-#
-# $FreeBSD$
-#
-# "make" then "make regress".
-#
-PROG=	bigpipetest
-MAN=
-
-regress:
-	@if ./bigpipetest; then \
-		echo "PASS"; \
-	else \
-		echo "FAIL"; \
-	fi
-
-.include <bsd.prog.mk>
diff --git a/tools/regression/pipe/bigpipetest.c b/tools/regression/pipe/bigpipetest.c
deleted file mode 100644
index c5983fa..0000000
--- a/tools/regression/pipe/bigpipetest.c
+++ /dev/null
@@ -1,82 +0,0 @@
-#include <stdio.h>
-#include <fcntl.h>
-#include <unistd.h>
-#include <stdlib.h>
-#include <sys/select.h>
-#include <string.h>
-#include <errno.h>
-
-#define BIG_PIPE_SIZE  64*1024 /* From sys/pipe.h */
-
-/*
- * Test for the non-blocking big pipe bug (write(2) returning
- * EAGAIN while select(2) returns the descriptor as ready for write).
- *
- * $FreeBSD$
- */
-
-void write_frame(int fd, char *buf, unsigned long buflen)
-{
-    fd_set wfd;
-    int i;
-
-    while (buflen) {
-	FD_ZERO(&wfd);
-	FD_SET(fd, &wfd);
-	i = select(fd+1, NULL, &wfd, NULL, NULL);
-	if (i < 0) {
-	    perror("select");
-	    exit(1);
-	}
-	if (i != 1) {
-	    fprintf(stderr, "select returned unexpected value %d\n", i);
-	    exit(1);
-	}
-        i = write(fd, buf, buflen);
-	if (i < 0) {
-	    if (errno != EAGAIN)
-		perror("write");
-	    exit(1);
-	}
-        buf += i;
-        buflen -= i;
-    }
-}
-
-int main()
-{
-    char buf[BIG_PIPE_SIZE];	/* any value over PIPE_SIZE should do */
-    int i, flags, fd[2];
-
-    printf("1..1\n");
-
-    if (pipe(fd) < 0) { perror("pipe"); exit(1); }
-
-    flags = fcntl(fd[1], F_GETFL);
-    if (flags == -1 || fcntl(fd[1], F_SETFL, flags|O_NONBLOCK) == -1) {
-	perror("fcntl");
-	exit(1);
-    }
-
-    switch (fork()) {
-	case -1:
-	    perror("fork");
-	    exit(1);
-	case 0:
-	    close(fd[1]);
-	    for (;;) {
-		i = read(fd[0], buf, 256); /* any small size should do */
-		if (i == 0) break;
-		if (i < 0) { perror("read"); exit(1); }
-	    }
-	    exit(0);
-	default:
-	    break;
-    }
-
-    close(fd[0]);
-    memset(buf, 0, sizeof buf);
-    for (i = 0; i < 1000; i++) write_frame(fd[1], buf, sizeof buf);
-    printf("ok 1\n");
-    exit(0);
-}
diff --git a/tools/regression/pipe/bigpipetest.t b/tools/regression/pipe/bigpipetest.t
deleted file mode 100644
index 8bdfd03..0000000
--- a/tools/regression/pipe/bigpipetest.t
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/bin/sh
-# $FreeBSD$
-
-cd `dirname $0`
-
-executable=`basename $0 .t`
-
-make $executable 2>&1 > /dev/null
-
-exec ./$executable
diff --git a/tools/regression/pipe/pipe-fstatbug.c b/tools/regression/pipe/pipe-fstatbug.c
deleted file mode 100644
index 724c2d8..0000000
--- a/tools/regression/pipe/pipe-fstatbug.c
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
-Copyright (C) 2004 Michael J. Silbersack. All rights reserved.
- 
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-1. Redistributions of source code must retain the above copyright
-   notice, this list of conditions and the following disclaimer.
-2. Redistributions in binary form must reproduce the above copyright
-   notice, this list of conditions and the following disclaimer in the
-   documentation and/or other materials provided with the distribution.
-
-THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
-FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
-OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
-HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
-OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
-SUCH DAMAGE.
-*/
-
-#include <stdio.h>
-#include <unistd.h>
-#include <sys/stat.h>
-
-/*
- * $FreeBSD$
- * The goal of this program is to see if fstat reports the correct
- * data count for a pipe.  Prior to revision 1.172 of sys_pipe.c,
- * 0 would be returned once the pipe entered direct write mode.
- *
- * Linux (2.6) always returns zero, so it's not a valuable platform
- * for comparison.
- */
-
-int main (void)
-{
-char buffer[32768], buffer2[32768];
-int desc[2];
-int error, successes = 0;
-struct stat status;
-pid_t new_pid;
-
-error = pipe(desc);
-
-if (error)
-	err(0, "Couldn't allocate fds\n");
-
-new_pid = fork();
-
-if (new_pid == 0) {
-	write(desc[1], &buffer, 145);
-	usleep(1000000);
-	write(desc[1], &buffer, 2048);
-	usleep(1000000);
-	write(desc[1], &buffer, 4096);
-	usleep(1000000);
-	write(desc[1], &buffer, 8191);
-	usleep(1000000);
-	write(desc[1], &buffer, 8192);
-	usleep(1000000);
-} else {
-	while (successes < 5) {
-		usleep(3000);
-		fstat(desc[0], &status);
-		error = read(desc[0], &buffer2, 32768);
-		if (status.st_size != error)
-			err(0, "FAILURE: stat size %d read size %d\n", (int)status.st_size, error);
-		if (error > 0) {
-			printf("SUCCESS at stat size %d read size %d\n", (int)status.st_size, error);
-			successes++;
-			/* Sleep to avoid the natural race in reading st_size. */
-			usleep(1000000);
-		}
-	}
-}
-
-}
diff --git a/tools/regression/pipe/pipe-reverse.c b/tools/regression/pipe/pipe-reverse.c
deleted file mode 100644
index 702c516..0000000
--- a/tools/regression/pipe/pipe-reverse.c
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
-Copyright (C) 2004 Michael J. Silbersack. All rights reserved.
- 
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-1. Redistributions of source code must retain the above copyright
-   notice, this list of conditions and the following disclaimer.
-2. Redistributions in binary form must reproduce the above copyright
-   notice, this list of conditions and the following disclaimer in the
-   documentation and/or other materials provided with the distribution.
-
-THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
-FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
-OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
-HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
-OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
-SUCH DAMAGE.
-*/
-
-#include <stdio.h>
-#include <unistd.h>
-#include <sys/stat.h>
-
-/*
- * $FreeBSD$
- * This program simply tests writing through the reverse direction of
- * a pipe.  Nothing too fancy, it's only needed because most pipe-using
- * programs never touch the reverse direction (it doesn't exist on
- * Linux.)
- */
-
-int main (void)
-{
-char buffer[65535], buffer2[65535];
-int desc[2];
-int buggy, error, i, successes, total;
-struct stat status;
-pid_t new_pid;
-
-buggy = 0;
-total = 0;
-
-error = pipe(desc);
-
-if (error)
-	err(0, "Couldn't allocate fds\n");
-
-buffer[0] = 'A';
-
-for (i = 1; i < 65535; i++) {
-	buffer[i] = buffer[i - 1] + 1;
-	if (buffer[i] > 'Z')
-		buffer[i] = 'A';
-	}
-
-new_pid = fork();
-
-if (new_pid == 0) {
-	error = write(desc[0], &buffer, 4096);
-	total += error;
-	error = write(desc[0], &buffer[total], 4096);
-	total += error;
-	error = write(desc[0], &buffer[total], 4096);
-	total += error;
-	error = write(desc[0], &buffer[total], 4096);
-	total += error;
-	error = write(desc[0], &buffer[total], 4096);
-	total += error;
-	error = write(desc[0], &buffer[total], 4096);
-	total += error;
-	error = write(desc[0], &buffer[total], 4096);
-	total += error;
-	error = write(desc[0], &buffer[total], 4096);
-	total += error;
-	printf("Wrote %d bytes, sleeping\n", total);
-	usleep(1000000);
-	error = write(desc[0], &buffer[total], 4096);
-	total += error;
-	error = write(desc[0], &buffer[total], 4096);
-	total += error;
-	printf("Wrote another 8192 bytes, %d total, done\n", total);
-} else {
-	usleep(500000);
-	error = read(desc[1], &buffer2, 32768);
-	total += error;
-	printf("Read %d bytes, going back to sleep\n", error);
-	usleep(1000000);
-	error = read(desc[1], &buffer2[total], 8192);
-	total += error;
-	printf("Read %d bytes, done\n", error);
-
-	for (i = 0; i < total; i++) {
-		if (buffer[i] != buffer2[i]) {
-			buggy = 1;
-			printf("Location %d input: %hhx output: %hhx\n",
-					i, buffer[i], buffer2[i]);
-		}
-	}
-
-if ((buggy == 1) || (total != 40960))
-	printf("FAILURE\n");
-else
-	printf("SUCCESS\n");
-
-}
-
-}
diff --git a/tools/regression/pipe/pipe-wraparound.c b/tools/regression/pipe/pipe-wraparound.c
deleted file mode 100644
index 5cf09fa..0000000
--- a/tools/regression/pipe/pipe-wraparound.c
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
-Copyright (C) 2004 Michael J. Silbersack. All rights reserved.
- 
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-1. Redistributions of source code must retain the above copyright
-   notice, this list of conditions and the following disclaimer.
-2. Redistributions in binary form must reproduce the above copyright
-   notice, this list of conditions and the following disclaimer in the
-   documentation and/or other materials provided with the distribution.
-
-THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
-FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
-OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
-HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
-OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
-SUCH DAMAGE.
-*/
-
-#include <stdio.h>
-#include <unistd.h>
-#include <sys/stat.h>
-
-/*
- * $FreeBSD$
- * This program tests to make sure that wraparound writes and reads
- * are working, assuming that 16K socket buffers are used.  In order
- * to really stress the pipe code with this test, kernel modifications
- * nay be necessary.
- */
-
-int main (void)
-{
-char buffer[32768], buffer2[32768];
-int desc[2];
-int buggy, error, i, successes, total;
-struct stat status;
-pid_t new_pid;
-
-buggy = 0;
-total = 0;
-
-error = pipe(desc);
-
-if (error)
-	err(0, "Couldn't allocate fds\n");
-
-buffer[0] = 'A';
-
-for (i = 1; i < 32768; i++) {
-	buffer[i] = buffer[i - 1] + 1;
-	if (buffer[i] > 'Z')
-		buffer[i] = 'A';
-	}
-
-new_pid = fork();
-
-if (new_pid == 0) {
-	error = write(desc[1], &buffer, 4096);
-	total += error;
-	error = write(desc[1], &buffer[total], 4096);
-	total += error;
-	error = write(desc[1], &buffer[total], 4000);
-	total += error;
-	printf("Wrote %d bytes, sleeping\n", total);
-	usleep(1000000);
-	error = write(desc[1], &buffer[total], 3000);
-	total += error;
-	error = write(desc[1], &buffer[total], 3000);
-	total += error;
-	printf("Wrote another 6000 bytes, %d total, done\n", total);
-} else {
-	usleep(500000);
-	error = read(desc[0], &buffer2, 8192);
-	total += error;
-	printf("Read %d bytes, going back to sleep\n", error);
-	usleep(1000000);
-	error = read(desc[0], &buffer2[total], 16384);
-	total += error;
-	printf("Read %d bytes, done\n", error);
-
-	for (i = 0; i < total; i++) {
-		if (buffer[i] != buffer2[i]) {
-			buggy = 1;
-			printf("Location %d input: %hhx output: %hhx\n",
-					i, buffer[i], buffer2[i]);
-		}
-	}
-
-if (buggy)
-	printf("FAILURE\n");
-else
-	printf("SUCCESS\n");
-}
-
-
-}