changeset 22368:da4207e17ba1

11961 add DDI UFM support to the nvme driver Reviewed by: Robert Mustacchi <rm@joyent.com> Reviewed by: Jordan Hendricks <jordan.hendricks@joyent.com> Reviewed by: Paul Winder <paul@winders.demon.co.uk> Reviewed by: Andrew Stormont <astormont@racktopsystems.com> Approved by: Dan McDonald <danmcd@joyent.com>
author Rob Johnston <rob.johnston@joyent.com>
date Mon, 19 Aug 2019 20:54:40 +0000
parents 3e54f247288e
children 5653fa724c62 13d2cba2e231
files usr/src/uts/common/io/nvme/nvme.c usr/src/uts/common/io/nvme/nvme_var.h usr/src/uts/common/os/ddi_ufm.c usr/src/uts/common/sys/nvme.h
diffstat 4 files changed, 174 insertions(+), 5 deletions(-) [+]
line wrap: on
line diff
--- a/usr/src/uts/common/io/nvme/nvme.c	Wed Nov 13 12:32:26 2019 -0600
+++ b/usr/src/uts/common/io/nvme/nvme.c	Mon Aug 19 20:54:40 2019 +0000
@@ -201,6 +201,14 @@
  * device.
  *
  *
+ * DDI UFM Support
+ *
+ * The driver supports the DDI UFM framework for reporting information about
+ * the device's firmware image and slot configuration. This data can be
+ * queried by userland software via ioctls to the ufm driver. For more
+ * information, see ddi_ufm(9E).
+ *
+ *
  * Driver Configuration:
  *
  * The following driver properties can be changed to control some aspects of the
@@ -247,6 +255,7 @@
 #include <sys/conf.h>
 #include <sys/devops.h>
 #include <sys/ddi.h>
+#include <sys/ddi_ufm.h>
 #include <sys/sunddi.h>
 #include <sys/sunndi.h>
 #include <sys/bitmap.h>
@@ -386,10 +395,24 @@
 
 static void nvme_prepare_devid(nvme_t *, uint32_t);
 
+/* DDI UFM callbacks */
+static int nvme_ufm_fill_image(ddi_ufm_handle_t *, void *, uint_t,
+    ddi_ufm_image_t *);
+static int nvme_ufm_fill_slot(ddi_ufm_handle_t *, void *, uint_t, uint_t,
+    ddi_ufm_slot_t *);
+static int nvme_ufm_getcaps(ddi_ufm_handle_t *, void *, ddi_ufm_cap_t *);
+
 static int nvme_open(dev_t *, int, int, cred_t *);
 static int nvme_close(dev_t, int, int, cred_t *);
 static int nvme_ioctl(dev_t, int, intptr_t, int, cred_t *, int *);
 
+static ddi_ufm_ops_t nvme_ufm_ops = {
+	NULL,
+	nvme_ufm_fill_image,
+	nvme_ufm_fill_slot,
+	nvme_ufm_getcaps
+};
+
 #define	NVME_MINOR_INST_SHIFT	9
 #define	NVME_MINOR(inst, nsid)	(((inst) << NVME_MINOR_INST_SHIFT) | (nsid))
 #define	NVME_MINOR_INST(minor)	((minor) >> NVME_MINOR_INST_SHIFT)
@@ -3352,6 +3375,18 @@
 		goto fail;
 
 	/*
+	 * Initialize the driver with the UFM subsystem
+	 */
+	if (ddi_ufm_init(dip, DDI_UFM_CURRENT_VERSION, &nvme_ufm_ops,
+	    &nvme->n_ufmh, nvme) != 0) {
+		dev_err(dip, CE_WARN, "!failed to initialize UFM subsystem");
+		goto fail;
+	}
+	mutex_init(&nvme->n_fwslot_mutex, NULL, MUTEX_DRIVER, NULL);
+	ddi_ufm_update(nvme->n_ufmh);
+	nvme->n_progress |= NVME_UFM_INIT;
+
+	/*
 	 * Attach the blkdev driver for each namespace.
 	 */
 	for (i = 0; i != nvme->n_namespace_count; i++) {
@@ -3444,6 +3479,10 @@
 		kmem_free(nvme->n_ns, sizeof (nvme_namespace_t) *
 		    nvme->n_namespace_count);
 	}
+	if (nvme->n_progress & NVME_UFM_INIT) {
+		ddi_ufm_fini(nvme->n_ufmh);
+		mutex_destroy(&nvme->n_fwslot_mutex);
+	}
 
 	if (nvme->n_progress & NVME_INTERRUPTS)
 		nvme_release_interrupts(nvme);
@@ -4354,6 +4393,18 @@
 	return (rv);
 }
 
+static void
+nvme_ufm_update(nvme_t *nvme)
+{
+	mutex_enter(&nvme->n_fwslot_mutex);
+	ddi_ufm_update(nvme->n_ufmh);
+	if (nvme->n_fwslot != NULL) {
+		kmem_free(nvme->n_fwslot, sizeof (nvme_fwslot_log_t));
+		nvme->n_fwslot = NULL;
+	}
+	mutex_exit(&nvme->n_fwslot_mutex);
+}
+
 static int
 nvme_ioctl_firmware_download(nvme_t *nvme, int nsid, nvme_ioctl_t *nioc,
     int mode, cred_t *cred_p)
@@ -4406,6 +4457,12 @@
 		len -= copylen;
 	}
 
+	/*
+	 * Let the DDI UFM subsystem know that the firmware information for
+	 * this device has changed.
+	 */
+	nvme_ufm_update(nvme);
+
 	return (rv);
 }
 
@@ -4454,6 +4511,12 @@
 
 	nioc->n_arg = ((uint64_t)cqe.cqe_sf.sf_sct << 16) | cqe.cqe_sf.sf_sc;
 
+	/*
+	 * Let the DDI UFM subsystem know that the firmware information for
+	 * this device has changed.
+	 */
+	nvme_ufm_update(nvme);
+
 	return (rv);
 }
 
@@ -4567,3 +4630,90 @@
 
 	return (rv);
 }
+
+/*
+ * DDI UFM Callbacks
+ */
+static int
+nvme_ufm_fill_image(ddi_ufm_handle_t *ufmh, void *arg, uint_t imgno,
+    ddi_ufm_image_t *img)
+{
+	nvme_t *nvme = arg;
+
+	if (imgno != 0)
+		return (EINVAL);
+
+	ddi_ufm_image_set_desc(img, "Firmware");
+	ddi_ufm_image_set_nslots(img, nvme->n_idctl->id_frmw.fw_nslot);
+
+	return (0);
+}
+
+/*
+ * Fill out firmware slot information for the requested slot.  The firmware
+ * slot information is gathered by requesting the Firmware Slot Information log
+ * page.  The format of the page is described in section 5.10.1.3.
+ *
+ * We lazily cache the log page on the first call and then invalidate the cache
+ * data after a successful firmware download or firmware commit command.
+ * The cached data is protected by a mutex as the state can change
+ * asynchronous to this callback.
+ */
+static int
+nvme_ufm_fill_slot(ddi_ufm_handle_t *ufmh, void *arg, uint_t imgno,
+    uint_t slotno, ddi_ufm_slot_t *slot)
+{
+	nvme_t *nvme = arg;
+	void *log = NULL;
+	size_t bufsize;
+	ddi_ufm_attr_t attr = 0;
+	char fw_ver[NVME_FWVER_SZ + 1];
+	int ret;
+
+	if (imgno > 0 || slotno > (nvme->n_idctl->id_frmw.fw_nslot - 1))
+		return (EINVAL);
+
+	mutex_enter(&nvme->n_fwslot_mutex);
+	if (nvme->n_fwslot == NULL) {
+		ret = nvme_get_logpage(nvme, B_TRUE, &log, &bufsize,
+		    NVME_LOGPAGE_FWSLOT, 0);
+		if (ret != DDI_SUCCESS ||
+		    bufsize != sizeof (nvme_fwslot_log_t)) {
+			if (log != NULL)
+				kmem_free(log, bufsize);
+			mutex_exit(&nvme->n_fwslot_mutex);
+			return (EIO);
+		}
+		nvme->n_fwslot = (nvme_fwslot_log_t *)log;
+	}
+
+	/*
+	 * NVMe numbers firmware slots starting at 1
+	 */
+	if (slotno == (nvme->n_fwslot->fw_afi - 1))
+		attr |= DDI_UFM_ATTR_ACTIVE;
+
+	if (slotno != 0 || nvme->n_idctl->id_frmw.fw_readonly == 0)
+		attr |= DDI_UFM_ATTR_WRITEABLE;
+
+	if (nvme->n_fwslot->fw_frs[slotno][0] == '\0') {
+		attr |= DDI_UFM_ATTR_EMPTY;
+	} else {
+		(void) strncpy(fw_ver, nvme->n_fwslot->fw_frs[slotno],
+		    NVME_FWVER_SZ);
+		fw_ver[NVME_FWVER_SZ] = '\0';
+		ddi_ufm_slot_set_version(slot, fw_ver);
+	}
+	mutex_exit(&nvme->n_fwslot_mutex);
+
+	ddi_ufm_slot_set_attrs(slot, attr);
+
+	return (0);
+}
+
+static int
+nvme_ufm_getcaps(ddi_ufm_handle_t *ufmh, void *arg, ddi_ufm_cap_t *caps)
+{
+	*caps = DDI_UFM_CAP_REPORT;
+	return (0);
+}
--- a/usr/src/uts/common/io/nvme/nvme_var.h	Wed Nov 13 12:32:26 2019 -0600
+++ b/usr/src/uts/common/io/nvme/nvme_var.h	Mon Aug 19 20:54:40 2019 +0000
@@ -12,7 +12,7 @@
 /*
  * Copyright 2018 Nexenta Systems, Inc.
  * Copyright 2016 The MathWorks, Inc. All rights reserved.
- * Copyright 2017 Joyent, Inc.
+ * Copyright 2019 Joyent, Inc.
  * Copyright 2019 Western Digital Corporation.
  */
 
@@ -38,6 +38,7 @@
 #define	NVME_ADMIN_QUEUE		0x4
 #define	NVME_CTRL_LIMITS		0x8
 #define	NVME_INTERRUPTS			0x10
+#define	NVME_UFM_INIT			0x20
 
 #define	NVME_MIN_ADMIN_QUEUE_LEN	16
 #define	NVME_MIN_IO_QUEUE_LEN		16
@@ -242,6 +243,12 @@
 	uint32_t n_vendor_event;
 	uint32_t n_unknown_event;
 
+	/* DDI UFM handle */
+	ddi_ufm_handle_t *n_ufmh;
+	/* Cached Firmware Slot Information log page */
+	nvme_fwslot_log_t *n_fwslot;
+	/* Lock protecting the cached firmware slot info */
+	kmutex_t n_fwslot_mutex;
 };
 
 struct nvme_namespace {
--- a/usr/src/uts/common/os/ddi_ufm.c	Wed Nov 13 12:32:26 2019 -0600
+++ b/usr/src/uts/common/os/ddi_ufm.c	Mon Aug 19 20:54:40 2019 +0000
@@ -184,7 +184,10 @@
 		if (ret != 0)
 			goto cache_fail;
 
-		ASSERT(img->ufmi_desc != NULL && img->ufmi_nslots != 0);
+		if (img->ufmi_desc == NULL || img->ufmi_nslots == 0) {
+			ret = EIO;
+			goto cache_fail;
+		}
 
 		img->ufmi_slots =
 		    kmem_zalloc((sizeof (ddi_ufm_slot_t) * img->ufmi_nslots),
--- a/usr/src/uts/common/sys/nvme.h	Wed Nov 13 12:32:26 2019 -0600
+++ b/usr/src/uts/common/sys/nvme.h	Mon Aug 19 20:54:40 2019 +0000
@@ -436,13 +436,22 @@
 	uint8_t hl_rsvd2[512 - 192];
 } nvme_health_log_t;
 
+/*
+ * The NVMe spec allows for up to seven firmware slots.
+ */
+#define	NVME_MAX_FWSLOTS	7
+#define	NVME_FWVER_SZ		8
+
 typedef struct {
-	uint8_t fw_afi:3;		/* Active Firmware Slot */
+	/* Active Firmware Slot */
+	uint8_t fw_afi:3;
 	uint8_t fw_rsvd1:1;
-	uint8_t fw_next:3;		/* Next Active Firmware Slot */
+	/* Next Active Firmware Slot */
+	uint8_t fw_next:3;
 	uint8_t fw_rsvd2:1;
 	uint8_t fw_rsvd3[7];
-	char fw_frs[7][8];		/* Firmware Revision / Slot */
+	/* Firmware Revision / Slot */
+	char fw_frs[NVME_MAX_FWSLOTS][NVME_FWVER_SZ];
 	uint8_t fw_rsvd4[512 - 64];
 } nvme_fwslot_log_t;