MFC r330653:

Add kernel and userspace code to dump the firmware state of supported
ConnectX-4/5 devices in mlx5core.

The dump is obtained by reading a predefined register map from the
non-destructive crspace, accessible by the vendor-specific PCIe
capability (VSC). The dump is stored in preallocated kernel memory and
managed by the mlx5tool(8), which communicates with the driver using a
character device node.

The utility allows to store the dump in format
    <address> <value>
into a file, to reset the dump content, and to manually initiate the
dump.

A call to mlx5_fwdump() should be added at the places where a dump
must be fetched automatically. The most likely place is right before a
firmware reset request.

Submitted by:	kib@
Sponsored by:	Mellanox Technologies

4 files changed, 325 insertions, 0 deletions

diff --git a/usr.sbin/Makefile b/usr.sbin/Makefile
index b4fa1b1..da186af 100644
--- a/usr.sbin/Makefile
+++ b/usr.sbin/Makefile
@@ -124,6 +124,7 @@ SUBDIR.${MK_BSDINSTALL}+=	bsdinstall
 SUBDIR.${MK_BSNMP}+=	bsnmpd
 SUBDIR.${MK_CTM}+=	ctm
 SUBDIR.${MK_CXGBETOOL}+=	cxgbetool
+SUBDIR.${MK_MLX5TOOL}+=	mlx5tool
 SUBDIR.${MK_DIALOG}+=	bsdconfig
 SUBDIR.${MK_EFI}+=	efivar efidp
 SUBDIR.${MK_FLOPPY}+=	fdcontrol
diff --git a/usr.sbin/mlx5tool/Makefile b/usr.sbin/mlx5tool/Makefile
new file mode 100644
index 0000000..bf5ec06
--- /dev/null
+++ b/usr.sbin/mlx5tool/Makefile
@@ -0,0 +1,7 @@
+# $FreeBSD$
+
+PROG=	mlx5tool
+MAN=	mlx5tool.8
+WARNS?= 6
+
+.include <bsd.prog.mk>
diff --git a/usr.sbin/mlx5tool/mlx5tool.8 b/usr.sbin/mlx5tool/mlx5tool.8
new file mode 100644
index 0000000..057bb31
--- /dev/null
+++ b/usr.sbin/mlx5tool/mlx5tool.8
@@ -0,0 +1,97 @@
+.\"
+.\" Copyright (c) 2018 Mellanox Technologies
+.\" All rights reserved.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\" 
+.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.\" $FreeBSD$
+.\"
+.Dd March 8, 2018
+.Dt mlx5tool 8
+.Os
+.Sh NAME
+.Nm mlx5tool
+.Nd Utility for managing Connect-X 4/5 Mellanox network adapters
+.Sh SYNOPSIS
+.Nm
+.Fl d Ar domain:bus:slot:func
+.Fl e
+.Nm
+.Fl d Ar domain:bus:slot:func
+.Fl r
+.Nm
+.Fl d Ar domain:bus:slot:func
+.Fl o Ar file
+.Fl w
+.Sh DESCRIPTION
+The
+.Nm
+utility is provided for management of the Connect-X 4 and 5 network adapters
+in the aspects not covered by the generic
+.Xr ifconfig 8
+command, mostly related to the PCIe attachment and internal card working.
+The utility executes commands on specific adapter, which is addressed using
+.Em device:bus:slot:function
+conventions of the PCIe buses.
+You can match adapters ethernet name and addresses using the
+.X pciconf 8
+utility.
+The address is passed as an argument of the
+.Fl d
+option, which must be specified for each invocation.
+.Pp
+When the driver detects the malfunctioning of the hardware, or by user
+request, it is possible to create
+.Em firmware dump ,
+which contains debugging information about internal device state, for
+analysis of the failure by the Mellanox support team.
+.Pp
+The following commands are currently implemented:
+.Bl -tag -width indent
+.It Fl e
+Take the snapshot of the firmware registers state and store it in the
+kernel buffer.
+The buffer must be empty, in other words, no dumps should be written so
+far, or existing dump cleared with the
+.Fl r
+command for the specified device.
+.It Fl r
+Clear the stored firmware dump, preparing the kernel buffer for
+the next dump.
+.It Fl w
+Fetche the stored firmware dump and writes it into the file specified
+by the
+.Fl o
+option argument.
+.El
+.Sh FILES
+The
+.Pa /dev/mlx5ctl
+.Xr devfs 5
+node is used to pass commands to the driver.
+.Sh SEE ALSO
+.Xr mlx5en 4 ,
+.Xr mlx5ib 4 ,
+.Xr mlx5io 4 ,
+.Xr ifconfig 8
+and
+.Xr pciconf 8 .
diff --git a/usr.sbin/mlx5tool/mlx5tool.c b/usr.sbin/mlx5tool/mlx5tool.c
new file mode 100644
index 0000000..d52534f
--- /dev/null
+++ b/usr.sbin/mlx5tool/mlx5tool.c
@@ -0,0 +1,220 @@
+/*-
+ * Copyright (c) 2018, Mellanox Technologies, Ltd.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/ioctl.h>
+#include <dev/mlx5/mlx5io.h>
+#include <ctype.h>
+#include <err.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <paths.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+/* stolen from pciconf.c: parsesel() */
+static int
+parse_pci_addr(const char *addrstr, struct mlx5_fwdump_addr *addr)
+{
+	char *eppos;
+	unsigned long selarr[4];
+	int i;
+
+	if (strncmp(addrstr, "pci", 3) == 0) {
+		addrstr += 3;
+		i = 0;
+		while (isdigit(*addrstr) && i < 4) {
+			selarr[i++] = strtoul(addrstr, &eppos, 10);
+			addrstr = eppos;
+			if (*addrstr == ':')
+				addrstr++;
+		}
+		if (i > 0 && *addrstr == '\0') {
+			addr->func = (i > 2) ? selarr[--i] : 0;
+			addr->slot = (i > 0) ? selarr[--i] : 0;
+			addr->bus = (i > 0) ? selarr[--i] : 0;
+			addr->domain = (i > 0) ? selarr[--i] : 0;
+			return (0);
+		}
+	}
+	warnx("invalid pci address %s", addrstr);
+	return (1);
+}
+
+static int
+mlx5tool_save_dump(int ctldev, const struct mlx5_fwdump_addr *addr,
+    const char *dumpname)
+{
+	struct mlx5_fwdump_get fdg;
+	struct mlx5_fwdump_reg *rege;
+	FILE *dump;
+	size_t cnt;
+	int error, res;
+
+	if (dumpname == NULL)
+		dump = stdout;
+	else
+		dump = fopen(dumpname, "w");
+	if (dump == NULL) {
+		warn("open %s", dumpname);
+		return (1);
+	}
+	res = 1;
+	memset(&fdg, 0, sizeof(fdg));
+	fdg.devaddr = *addr;
+	error = ioctl(ctldev, MLX5_FWDUMP_GET, &fdg);
+	if (error != 0) {
+		warn("MLX5_FWDUMP_GET dumpsize");
+		goto out;
+	}
+	rege = calloc(fdg.reg_filled, sizeof(*rege));
+	if (rege == NULL) {
+		warn("alloc rege");
+		goto out;
+	}
+	fdg.buf = rege;
+	fdg.reg_cnt = fdg.reg_filled;
+	error = ioctl(ctldev, MLX5_FWDUMP_GET, &fdg);
+	if (error != 0) {
+		if (errno == ENOENT)
+			warnx("no dump recorded");
+		else
+			warn("MLX5_FWDUMP_GET dump fetch");
+		goto out;
+	}
+	for (cnt = 0; cnt < fdg.reg_cnt; cnt++, rege++)
+		fprintf(dump, "0x%08x\t0x%08x\n", rege->addr, rege->val);
+	res = 0;
+out:
+	if (dump != stdout)
+		fclose(dump);
+	return (res);
+}
+
+static int
+mlx5tool_dump_reset(int ctldev, const struct mlx5_fwdump_addr *addr)
+{
+
+	if (ioctl(ctldev, MLX5_FWDUMP_RESET, addr) == -1) {
+		warn("MLX5_FWDUMP_RESET");
+		return (1);
+	}
+	return (0);
+}
+
+static int
+mlx5tool_dump_force(int ctldev, const struct mlx5_fwdump_addr *addr)
+{
+
+	if (ioctl(ctldev, MLX5_FWDUMP_FORCE, addr) == -1) {
+		warn("MLX5_FWDUMP_FORCE");
+		return (1);
+	}
+	return (0);
+}
+
+static void
+usage(void)
+{
+
+	fprintf(stderr,
+	    "Usage: mlx5tool -d pci<d:b:s:f> [-w -o dump.file | -r | -e]\n");
+	fprintf(stderr, "\t-w - write firmware dump to the specified file\n");
+	fprintf(stderr, "\t-r - reset dump\n");
+	fprintf(stderr, "\t-e - force dump\n");
+	exit(1);
+}
+
+enum mlx5_action {
+	ACTION_DUMP_GET,
+	ACTION_DUMP_RESET,
+	ACTION_DUMP_FORCE,
+	ACTION_NONE,
+};
+
+int
+main(int argc, char *argv[])
+{
+	struct mlx5_fwdump_addr addr;
+	char *dumpname;
+	char *addrstr;
+	int c, ctldev, res;
+	enum mlx5_action act;
+
+	act = ACTION_NONE;
+	addrstr = NULL;
+	dumpname = NULL;
+	while ((c = getopt(argc, argv, "d:eho:rw")) != -1) {
+		switch (c) {
+		case 'd':
+			addrstr = optarg;
+			break;
+		case 'w':
+			act = ACTION_DUMP_GET;
+			break;
+		case 'e':
+			act= ACTION_DUMP_FORCE;
+			break;
+		case 'o':
+			dumpname = optarg;
+			break;
+		case 'r':
+			act = ACTION_DUMP_RESET;
+			break;
+		case 'h':
+		default:
+			usage();
+		}
+	}
+	if (act == ACTION_NONE || (dumpname != NULL && act != ACTION_DUMP_GET))
+		usage();
+	if (parse_pci_addr(addrstr, &addr) != 0)
+		exit(1);
+
+	ctldev = open(MLX5_DEV_PATH, O_RDWR);
+	if (ctldev == -1)
+		err(1, "open "MLX5_DEV_PATH);
+	switch (act) {
+	case ACTION_DUMP_GET:
+		res = mlx5tool_save_dump(ctldev, &addr, dumpname);
+		break;
+	case ACTION_DUMP_RESET:
+		res = mlx5tool_dump_reset(ctldev, &addr);
+		break;
+	case ACTION_DUMP_FORCE:
+		res = mlx5tool_dump_force(ctldev, &addr);
+		break;
+	default:
+		res = 0;
+		break;
+	}
+	close(ctldev);
+	exit(res);
+}