--- a/usr/src/uts/sun4v/io/vsw.c	Mon Jul 06 15:04:34 2009 -0700
+++ b/usr/src/uts/sun4v/io/vsw.c	Mon Jul 06 16:54:01 2009 -0700
@@ -80,7 +80,7 @@
 static	int vsw_unattach(vsw_t *vswp);
 static	int vsw_get_md_physname(vsw_t *, md_t *, mde_cookie_t, char *);
 static	int vsw_get_md_smodes(vsw_t *, md_t *, mde_cookie_t, uint8_t *);
-static	int vsw_mod_cleanup(void);
+void vsw_destroy_rxpools(void *);
 
 /* MDEG routines */
 static	int vsw_mdeg_register(vsw_t *vswp);
@@ -186,6 +186,8 @@
 int	vsw_ldc_delay = 1000;		/* 1 ms delay for ldc_close() */
 boolean_t vsw_ldc_rxthr_enabled = B_TRUE;	/* LDC Rx thread enabled */
 boolean_t vsw_ldc_txthr_enabled = B_TRUE;	/* LDC Tx thread enabled */
+int	vsw_rxpool_cleanup_delay = 100000;	/* 100ms */
+
 
 uint32_t	vsw_fdb_nchains = 8;	/* # of chains in fdb hash table */
 uint32_t	vsw_vlan_nchains = 4;	/* # of chains in vlan id hash table */
@@ -352,7 +354,6 @@
  * Linked list of "vsw_t" structures - one per instance.
  */
 vsw_t		*vsw_head = NULL;
-vio_mblk_pool_t	*vsw_rx_poolp = NULL;
 krwlock_t	vsw_rw;
 
 /*
@@ -485,10 +486,6 @@
 {
 	int status;
 
-	status = vsw_mod_cleanup();
-	if (status != 0)
-		return (status);
-
 	status = mod_remove(&modlinkage);
 	if (status != 0)
 		return (status);
@@ -594,6 +591,17 @@
 
 	progress |= PROG_taskq;
 
+	(void) snprintf(qname, TASKQ_NAMELEN, "vsw_rxp_taskq%d",
+	    vswp->instance);
+	if ((vswp->rxp_taskq = ddi_taskq_create(vswp->dip, qname, 1,
+	    TASKQ_DEFAULTPRI, 0)) == NULL) {
+		cmn_err(CE_WARN, "!vsw%d: Unable to create rxp task queue",
+		    vswp->instance);
+		goto vsw_attach_fail;
+	}
+
+	progress |= PROG_rxp_taskq;
+
 	/* prevent auto-detaching */
 	if (ddi_prop_update_int(DDI_DEV_T_NONE, vswp->dip,
 	    DDI_NO_AUTODETACH, 1) != DDI_SUCCESS) {
@@ -716,7 +724,6 @@
 static int
 vsw_unattach(vsw_t *vswp)
 {
-	vio_mblk_pool_t		*poolp, *npoolp;
 	vsw_attach_progress_t	progress;
 
 	progress = vswp->attach_progress;
@@ -747,24 +754,6 @@
 	if (progress & PROG_mdreg) {
 		vsw_mdeg_unregister(vswp);
 		vsw_detach_ports(vswp);
-
-		/*
-		 * At this point, we attempt to free receive mblk pools that
-		 * couldn't be destroyed when the ports were detached; if this
-		 * attempt also fails, we hook up the pool(s) to the module so
-		 * they can be cleaned up in _fini().
-		 */
-		poolp = vswp->rxh;
-		while (poolp != NULL) {
-			npoolp = vswp->rxh = poolp->nextp;
-			if (vio_destroy_mblks(poolp) != 0) {
-				WRITE_ENTER(&vsw_rw);
-				poolp->nextp = vsw_rx_poolp;
-				vsw_rx_poolp = poolp;
-				RW_EXIT(&vsw_rw);
-			}
-			poolp = npoolp;
-		}
 		progress &= ~PROG_mdreg;
 	}
 
@@ -796,6 +785,17 @@
 	}
 
 	/*
+	 * We now destroy the taskq used to clean up rx mblk pools that
+	 * couldn't be destroyed when the ports/channels were detached.
+	 * We implicitly wait for those tasks to complete in
+	 * ddi_taskq_destroy().
+	 */
+	if (progress & PROG_rxp_taskq) {
+		ddi_taskq_destroy(vswp->rxp_taskq);
+		progress &= ~PROG_rxp_taskq;
+	}
+
+	/*
 	 * By now any pending tasks have finished and the underlying
 	 * ldc's have been destroyed, so its safe to delete the control
 	 * message taskq.
@@ -844,32 +844,19 @@
 	return (0);
 }
 
-/*
- * one time cleanup.
- */
-static int
-vsw_mod_cleanup(void)
+void
+vsw_destroy_rxpools(void *arg)
 {
-	vio_mblk_pool_t		*poolp, *npoolp;
-
-	/*
-	 * If any rx mblk pools are still in use, return
-	 * error and stop the module from unloading.
-	 */
-	WRITE_ENTER(&vsw_rw);
-	poolp = vsw_rx_poolp;
+	vio_mblk_pool_t	*poolp = (vio_mblk_pool_t *)arg;
+	vio_mblk_pool_t	*npoolp;
+
 	while (poolp != NULL) {
-		npoolp = vsw_rx_poolp = poolp->nextp;
-		if (vio_destroy_mblks(poolp) != 0) {
-			vsw_rx_poolp = poolp;
-			RW_EXIT(&vsw_rw);
-			return (EBUSY);
+		npoolp =  poolp->nextp;
+		while (vio_destroy_mblks(poolp) != 0) {
+			drv_usecwait(vsw_rxpool_cleanup_delay);
 		}
 		poolp = npoolp;
 	}
-	RW_EXIT(&vsw_rw);
-
-	return (0);
 }
 
 /*

--- a/usr/src/uts/sun4v/io/vsw_ldc.c	Mon Jul 06 15:04:34 2009 -0700
+++ b/usr/src/uts/sun4v/io/vsw_ldc.c	Mon Jul 06 16:54:01 2009 -0700
@@ -209,7 +209,7 @@
 extern void vsw_publish_macaddr(vsw_t *vswp, vsw_port_t *portp);
 extern int vsw_mac_client_init(vsw_t *vswp, vsw_port_t *port, int type);
 extern void vsw_mac_client_cleanup(vsw_t *vswp, vsw_port_t *port, int type);
-
+extern void vsw_destroy_rxpools(void *arg);
 
 #define	VSW_NUM_VMPOOLS		3	/* number of vio mblk pools */
 
@@ -865,6 +865,7 @@
 	vsw_ldc_list_t	*ldcl = &port->p_ldclist;
 	int 		rv;
 	int		retries = 0;
+	vio_mblk_pool_t *fvmp = NULL;
 
 	prev_ldcp = ldcl->head;
 	for (; (ldcp = prev_ldcp) != NULL; prev_ldcp = ldcp->ldc_next) {
@@ -931,13 +932,17 @@
 
 
 	/*
-	 * Most likely some mblks are still in use and
-	 * have not been returned to the pool. These mblks are
-	 * added to the pool that is maintained in the device instance.
-	 * Another attempt will be made to destroy the pool
-	 * when the device detaches.
+	 * If we can't destroy all the rx pools for this channel, dispatch
+	 * a task to retry and clean up those rx pools. Note that we don't
+	 * need to wait for the task to complete. If the vsw device itself
+	 * gets detached (vsw_detach()), it will wait for the task to complete
+	 * implicitly in ddi_taskq_destroy().
 	 */
-	vio_destroy_multipools(&ldcp->vmp, &vswp->rxh);
+	vio_destroy_multipools(&ldcp->vmp, &fvmp);
+	if (fvmp != NULL) {
+		(void) ddi_taskq_dispatch(vswp->rxp_taskq,
+		    vsw_destroy_rxpools, fvmp, DDI_SLEEP);
+	}
 
 	/* unlink it from the list */
 	prev_ldcp = ldcp->ldc_next;
@@ -1606,11 +1611,22 @@
 	vsw_t		*vswp = ldcp->ldc_vswp;
 	vsw_port_t	*port;
 	vsw_ldc_list_t	*ldcl;
+	vio_mblk_pool_t *fvmp = NULL;
 
 	D1(vswp, "%s: enter", __func__);
 
-	/* free receive mblk pools for the channel */
-	vio_destroy_multipools(&ldcp->vmp, &vswp->rxh);
+	/*
+	 * If we can't destroy all the rx pools for this channel, dispatch
+	 * a task to retry and clean up those rx pools. Note that we don't
+	 * need to wait for the task to complete. If the vsw device itself
+	 * gets detached (vsw_detach()), it will wait for the task to complete
+	 * implicitly in ddi_taskq_destroy().
+	 */
+	vio_destroy_multipools(&ldcp->vmp, &fvmp);
+	if (fvmp != NULL) {
+		(void) ddi_taskq_dispatch(vswp->rxp_taskq,
+		    vsw_destroy_rxpools, fvmp, DDI_SLEEP);
+	}
 
 	port = ldcp->ldc_port;
 	ldcl = &port->p_ldclist;

--- a/usr/src/uts/sun4v/sys/vsw.h	Mon Jul 06 15:04:34 2009 -0700
+++ b/usr/src/uts/sun4v/sys/vsw.h	Mon Jul 06 16:54:01 2009 -0700
@@ -99,9 +99,10 @@
 	PROG_fdb = 0x04,
 	PROG_mfdb = 0x08,
 	PROG_taskq = 0x10,
-	PROG_swmode = 0x20,
-	PROG_macreg = 0x40,
-	PROG_mdreg = 0x80
+	PROG_rxp_taskq = 0x20,
+	PROG_swmode = 0x40,
+	PROG_macreg = 0x80,
+	PROG_mdreg = 0x100
 } vsw_attach_progress_t;
 
 /*
@@ -143,7 +144,7 @@
 	mod_hash_t		*mfdb;		/* multicast FDB */
 	krwlock_t		mfdbrw;		/* rwlock for mFDB */
 
-	vio_mblk_pool_t		*rxh;		/* Receive pool handle */
+	ddi_taskq_t		*rxp_taskq;	/* VIO rx pool taskq */
 	void			(*vsw_switch_frame)
 					(struct vsw *, mblk_t *, int,
 					vsw_port_t *, mac_resource_handle_t);