Mercurial > illumos > illumos-gate

changeset 12574:0d8f6d488501

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
6946555 Hot lock in ibt_map_mem_iov / ibt_unmap_mem_iov 6950116 Need to optimize hermon_ci_unmap_mem_iov() for RDS performance
author Bill Taylor <William.Taylor@Oracle.COM>
date Mon, 07 Jun 2010 09:54:51 -0700
parents fb4ef506980f
children 6d803152496b
files usr/src/uts/common/io/ib/adapters/hermon/hermon_ci.c usr/src/uts/common/io/ib/ibtl/ibtl_chan.c usr/src/uts/common/io/ib/ibtl/ibtl_cq.c usr/src/uts/common/io/ib/ibtl/ibtl_hca.c usr/src/uts/common/io/ib/ibtl/ibtl_impl.c usr/src/uts/common/io/ib/ibtl/ibtl_mem.c usr/src/uts/common/io/ib/ibtl/ibtl_qp.c usr/src/uts/common/io/ib/ibtl/ibtl_srq.c usr/src/uts/common/sys/ib/ibtl/impl/ibtl.h
diffstat 9 files changed, 70 insertions(+), 143 deletions(-) [+]
line wrap: on
line diff
--- a/usr/src/uts/common/io/ib/adapters/hermon/hermon_ci.c	Mon Jun 07 08:54:25 2010 -0700
+++ b/usr/src/uts/common/io/ib/adapters/hermon/hermon_ci.c	Mon Jun 07 09:54:51 2010 -0700
@@ -2856,23 +2856,15 @@
  * Unmap the memory
  *    Context: Can be called from interrupt or base context.
  */
-/* ARGSUSED */
 static ibt_status_t
 hermon_ci_unmap_mem_iov(ibc_hca_hdl_t hca, ibc_mi_hdl_t mi_hdl)
 {
 	int		status, i;
 	hermon_state_t	*state;
 
-	/* Check for valid HCA handle */
-	if (hca == NULL)
-		return (IBT_HCA_HDL_INVALID);
-
 	state = (hermon_state_t *)hca;
 
-	if (mi_hdl == NULL)
-		return (IBT_MI_HDL_INVALID);
-
-	for (i = 0; i < mi_hdl->imh_len; i++) {
+	for (i = mi_hdl->imh_len; --i >= 0; ) {
 		status = ddi_dma_unbind_handle(mi_hdl->imh_dmahandle[i]);
 		if (status != DDI_SUCCESS)
 			HERMON_WARNING(state, "failed to unbind DMA mapping");
--- a/usr/src/uts/common/io/ib/ibtl/ibtl_chan.c	Mon Jun 07 08:54:25 2010 -0700
+++ b/usr/src/uts/common/io/ib/ibtl/ibtl_chan.c	Mon Jun 07 09:54:51 2010 -0700
@@ -19,8 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
- * Use is subject to license terms.
+ * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
  */
 
 /*
@@ -1225,9 +1224,7 @@
 	}
 
 	/* Update the PDs Resource Count per HCA Device. */
-	mutex_enter(&hca_hdl->ha_mutex);
-	hca_hdl->ha_pd_cnt++;
-	mutex_exit(&hca_hdl->ha_mutex);
+	atomic_inc_32(&hca_hdl->ha_pd_cnt);
 
 	return (retval);
 }
@@ -1265,9 +1262,7 @@
 	}
 
 	/* Update the PDs Resource Count per HCA Device. */
-	mutex_enter(&hca_hdl->ha_mutex);
-	hca_hdl->ha_pd_cnt--;
-	mutex_exit(&hca_hdl->ha_mutex);
+	atomic_dec_32(&hca_hdl->ha_pd_cnt);
 
 	return (retval);
 }
@@ -1316,9 +1311,7 @@
 	}
 
 	/* Update the AHs Resource Count per HCA Device. */
-	mutex_enter(&hca_hdl->ha_mutex);
-	hca_hdl->ha_ah_cnt++;
-	mutex_exit(&hca_hdl->ha_mutex);
+	atomic_inc_32(&hca_hdl->ha_ah_cnt);
 
 	return (retval);
 }
@@ -1356,9 +1349,7 @@
 	}
 
 	/* Update the AHs Resource Count per HCA Device. */
-	mutex_enter(&hca_hdl->ha_mutex);
-	hca_hdl->ha_ah_cnt--;
-	mutex_exit(&hca_hdl->ha_mutex);
+	atomic_dec_32(&hca_hdl->ha_ah_cnt);
 
 	return (retval);
 }
--- a/usr/src/uts/common/io/ib/ibtl/ibtl_cq.c	Mon Jun 07 08:54:25 2010 -0700
+++ b/usr/src/uts/common/io/ib/ibtl/ibtl_cq.c	Mon Jun 07 09:54:51 2010 -0700
@@ -19,8 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
- * Use is subject to license terms.
+ * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
  */
 
 #include <sys/ib/ibtl/impl/ibtl.h>
@@ -155,9 +154,7 @@
 	mutex_init(&ibt_cq->cq_mutex, NULL, MUTEX_DEFAULT, NULL);
 
 	/* Update the cq resource count */
-	mutex_enter(&hca_hdl->ha_mutex);
-	hca_hdl->ha_cq_cnt++;
-	mutex_exit(&hca_hdl->ha_mutex);
+	atomic_inc_32(&hca_hdl->ha_cq_cnt);
 
 	return (IBT_SUCCESS);
 }
@@ -190,9 +187,7 @@
 	ibtl_free_cq_async_check(ibt_cq);
 
 	/* Update the cq resource count */
-	mutex_enter(&ibt_hca->ha_mutex);
-	ibt_hca->ha_cq_cnt--;
-	mutex_exit(&ibt_hca->ha_mutex);
+	atomic_dec_32(&ibt_hca->ha_cq_cnt);
 
 	return (status);
 }
--- a/usr/src/uts/common/io/ib/ibtl/ibtl_hca.c	Mon Jun 07 08:54:25 2010 -0700
+++ b/usr/src/uts/common/io/ib/ibtl/ibtl_hca.c	Mon Jun 07 09:54:51 2010 -0700
@@ -19,12 +19,9 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
- * Use is subject to license terms.
+ * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
  */
 
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
-
 /*
  * ibtl_hca.c
  *
@@ -149,9 +146,6 @@
 	/* Create a new HCA Info entity. */
 	hca_infop = kmem_zalloc(sizeof (ibtl_hca_t), KM_SLEEP);
 
-	/* Initialize HCA Mutex. */
-	mutex_init(&hca_infop->ha_mutex, NULL, MUTEX_DEFAULT, NULL);
-
 	/* Update the HCA Info entity */
 	hca_infop->ha_hca_devp  = hca_devp;	/* HCA Device Info */
 	hca_infop->ha_clnt_devp = ibt_hdl;	/* Client Info */
@@ -176,6 +170,16 @@
 	return (IBT_SUCCESS);
 }
 
+static char *ibtl_close_error_fmt = "IBT CLOSE HCA failed: %d '%s' "
+	"resources not yet freed by client '%s'\n";
+
+#define	IBTL_CLOSE_RESOURCE_CHECK(counter, resource_type) \
+	if ((cntr = atomic_add_32_nv(&(counter), 0)) != 0) {		\
+		cmn_err(CE_CONT, ibtl_close_error_fmt,			\
+		    cntr, resource_type,				\
+		    hca_hdl->ha_clnt_devp->clnt_modinfop->mi_clnt_name); \
+	}								\
+	error |= cntr
 
 /*
  * Function:
@@ -197,6 +201,7 @@
 	ibtl_hca_devinfo_t	*hca_devp, *tmp_devp;
 	ibtl_hca_t		**hcapp;
 	ibtl_clnt_t		*clntp = hca_hdl->ha_clnt_devp;
+	uint32_t		cntr, error;
 
 	IBTF_DPRINTF_L3(ibtf_hca, "ibt_close_hca(%p)", hca_hdl);
 
@@ -219,22 +224,22 @@
 		return (IBT_HCA_HDL_INVALID);
 	}
 
-	mutex_enter(&hca_hdl->ha_mutex);
-
 	/* Make sure resources have been freed. */
-	if (hca_hdl->ha_qp_cnt | hca_hdl->ha_cq_cnt | hca_hdl->ha_eec_cnt |
-	    hca_hdl->ha_ah_cnt | hca_hdl->ha_mr_cnt | hca_hdl->ha_mw_cnt |
-	    hca_hdl->ha_pd_cnt | hca_hdl->ha_fmr_pool_cnt |
-	    hca_hdl->ha_ma_cnt) {
-		IBTF_DPRINTF_L2(ibtf_hca, "ibt_close_hca: "
-		    "some resources have not been freed by '%s': hca_hdl = %p",
-		    hca_hdl->ha_clnt_devp->clnt_modinfop->mi_clnt_name,
-		    hca_hdl);
-		mutex_exit(&hca_hdl->ha_mutex);
+	error = 0;
+	IBTL_CLOSE_RESOURCE_CHECK(hca_hdl->ha_qp_cnt, "QP/Channel");
+	IBTL_CLOSE_RESOURCE_CHECK(hca_hdl->ha_eec_cnt, "EEC");
+	IBTL_CLOSE_RESOURCE_CHECK(hca_hdl->ha_cq_cnt, "CQ");
+	IBTL_CLOSE_RESOURCE_CHECK(hca_hdl->ha_pd_cnt, "Protection Domain");
+	IBTL_CLOSE_RESOURCE_CHECK(hca_hdl->ha_ah_cnt, "AH");
+	IBTL_CLOSE_RESOURCE_CHECK(hca_hdl->ha_mr_cnt, "Memory Region");
+	IBTL_CLOSE_RESOURCE_CHECK(hca_hdl->ha_mw_cnt, "Memory Window");
+	IBTL_CLOSE_RESOURCE_CHECK(hca_hdl->ha_qpn_cnt, "QPN");
+	IBTL_CLOSE_RESOURCE_CHECK(hca_hdl->ha_srq_cnt, "SRQ");
+	IBTL_CLOSE_RESOURCE_CHECK(hca_hdl->ha_fmr_pool_cnt, "FMR Pool");
+	if (error) {
 		mutex_exit(&ibtl_clnt_list_mutex);
 		return (IBT_HCA_RESOURCES_NOT_FREED);
 	}
-	mutex_exit(&hca_hdl->ha_mutex);	/* ok to drop this now */
 
 	/* we are now committed to closing the HCA */
 	hca_hdl->ha_flags |= IBTL_HA_CLOSING;
@@ -281,9 +286,6 @@
 	*hcapp = hca_hdl->ha_clnt_link;		/* remove us */
 	mutex_exit(&ibtl_clnt_list_mutex);
 
-	/* Un-Initialize HCA Mutex. */
-	mutex_destroy(&hca_hdl->ha_mutex);
-
 	/* Free memory for this HCA Handle */
 	ibtl_free_hca_async_check(hca_hdl);
 
--- a/usr/src/uts/common/io/ib/ibtl/ibtl_impl.c	Mon Jun 07 08:54:25 2010 -0700
+++ b/usr/src/uts/common/io/ib/ibtl/ibtl_impl.c	Mon Jun 07 09:54:51 2010 -0700
@@ -393,6 +393,9 @@
 		IBTF_DPRINTF_L2(ibtf, "ibt_detach: "
 		    "ERROR: Client '%s' has not closed all of its HCAs",
 		    ibt_hdl->clnt_modinfop->mi_clnt_name);
+		cmn_err(CE_CONT, "IBT DETACH failed: resources not yet "
+		    "freed by client '%s'\n",
+		    ibt_hdl->clnt_modinfop->mi_clnt_name);
 		return (IBT_HCA_RESOURCES_NOT_FREED);
 	}
 
@@ -401,6 +404,9 @@
 		IBTF_DPRINTF_L2(ibtf, "ibt_detach: client '%s' still has "
 		    "services or subnet_notices registered",
 		    ibt_hdl->clnt_modinfop->mi_clnt_name);
+		cmn_err(CE_CONT, "IBT DETACH failed: resources not yet "
+		    "freed by client '%s'\n",
+		    ibt_hdl->clnt_modinfop->mi_clnt_name);
 		return (IBT_HCA_RESOURCES_NOT_FREED);
 	}
 
--- a/usr/src/uts/common/io/ib/ibtl/ibtl_mem.c	Mon Jun 07 08:54:25 2010 -0700
+++ b/usr/src/uts/common/io/ib/ibtl/ibtl_mem.c	Mon Jun 07 09:54:51 2010 -0700
@@ -19,8 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
- * Use is subject to license terms.
+ * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
  */
 
 
@@ -74,9 +73,7 @@
 	    mr_hdl_p, mem_desc);
 	if (status == IBT_SUCCESS) {
 		mem_desc->md_vaddr = vaddr;
-		mutex_enter(&hca_hdl->ha_mutex);
-		hca_hdl->ha_mr_cnt++;
-		mutex_exit(&hca_hdl->ha_mutex);
+		atomic_inc_32(&hca_hdl->ha_mr_cnt);
 	}
 
 	return (status);
@@ -121,9 +118,7 @@
 	    IBTL_HCA2CIHCA(hca_hdl), pd, mem_bpattr, bp, IBTL_HCA2CLNT(hca_hdl),
 	    mr_hdl_p, mem_desc);
 	if (status == IBT_SUCCESS) {
-		mutex_enter(&hca_hdl->ha_mutex);
-		hca_hdl->ha_mr_cnt++;
-		mutex_exit(&hca_hdl->ha_mutex);
+		atomic_inc_32(&hca_hdl->ha_mr_cnt);
 	}
 
 	return (status);
@@ -184,9 +179,7 @@
 	status = IBTL_HCA2CIHCAOPS_P(hca_hdl)->ibc_deregister_mr(
 	    IBTL_HCA2CIHCA(hca_hdl), mr_hdl);
 	if (status == IBT_SUCCESS) {
-		mutex_enter(&hca_hdl->ha_mutex);
-		hca_hdl->ha_mr_cnt--;
-		mutex_exit(&hca_hdl->ha_mutex);
+		atomic_dec_32(&hca_hdl->ha_mr_cnt);
 	}
 	return (status);
 }
@@ -239,9 +232,7 @@
 		    "Re-registration Failed: %d", status);
 
 		/* we lost one memory region resource */
-		mutex_enter(&hca_hdl->ha_mutex);
-		hca_hdl->ha_mr_cnt--;
-		mutex_exit(&hca_hdl->ha_mutex);
+		atomic_dec_32(&hca_hdl->ha_mr_cnt);
 	}
 
 	return (status);
@@ -297,9 +288,7 @@
 		    "Re-registration Mem Failed: %d", status);
 
 		/* we lost one memory region resource */
-		mutex_enter(&hca_hdl->ha_mutex);
-		hca_hdl->ha_mr_cnt--;
-		mutex_exit(&hca_hdl->ha_mutex);
+		atomic_dec_32(&hca_hdl->ha_mr_cnt);
 	}
 	return (status);
 }
@@ -341,9 +330,7 @@
 	    IBTL_HCA2CIHCA(hca_hdl), mr_hdl, pd, mem_sattr,
 	    IBTL_HCA2CLNT(hca_hdl), mr_hdl_p, mem_desc);
 	if (status == IBT_SUCCESS) {
-		mutex_enter(&hca_hdl->ha_mutex);
-		hca_hdl->ha_mr_cnt++;
-		mutex_exit(&hca_hdl->ha_mutex);
+		atomic_inc_32(&hca_hdl->ha_mr_cnt);
 	}
 	return (status);
 }
@@ -420,9 +407,7 @@
 	 * a good reason to have local MW state at this point, so we won't.
 	 */
 	if (status == IBT_SUCCESS) {
-		mutex_enter(&hca_hdl->ha_mutex);
-		hca_hdl->ha_mw_cnt++;
-		mutex_exit(&hca_hdl->ha_mutex);
+		atomic_inc_32(&hca_hdl->ha_mw_cnt);
 	}
 	return (status);
 }
@@ -481,9 +466,7 @@
 	    IBTL_HCA2CIHCA(hca_hdl), mw_hdl);
 
 	if (status == IBT_SUCCESS) {
-		mutex_enter(&hca_hdl->ha_mutex);
-		hca_hdl->ha_mw_cnt--;
-		mutex_exit(&hca_hdl->ha_mutex);
+		atomic_dec_32(&hca_hdl->ha_mw_cnt);
 	}
 	return (status);
 }
@@ -525,11 +508,8 @@
 	    IBTL_HCA2CIHCA(hca_hdl), va_attrs,
 	    NULL, /* IBTL_HCA2MODI_P(hca_hdl)->mi_reserved */
 	    paddr_list_len, reg_req, ma_hdl_p);
-	if (status == IBT_SUCCESS) {
-		mutex_enter(&hca_hdl->ha_mutex);
-		hca_hdl->ha_ma_cnt++;
-		mutex_exit(&hca_hdl->ha_mutex);
-	}
+	/* Not doing reference counting, which adversely effects performance */
+
 	return (status);
 }
 
@@ -557,11 +537,7 @@
 
 	status = (IBTL_HCA2CIHCAOPS_P(hca_hdl)->ibc_unmap_mem_area(
 	    IBTL_HCA2CIHCA(hca_hdl), ma_hdl));
-	if (status == IBT_SUCCESS) {
-		mutex_enter(&hca_hdl->ha_mutex);
-		hca_hdl->ha_ma_cnt--;
-		mutex_exit(&hca_hdl->ha_mutex);
-	}
+	/* Not doing reference counting, which adversely effects performance */
 
 	return (status);
 }
@@ -594,11 +570,7 @@
 
 	status = IBTL_HCA2CIHCAOPS_P(hca_hdl)->ibc_map_mem_iov(
 	    IBTL_HCA2CIHCA(hca_hdl), iov_attr, wr, mi_hdl_p);
-	if (status == IBT_SUCCESS) {
-		mutex_enter(&hca_hdl->ha_mutex);
-		hca_hdl->ha_ma_cnt++;
-		mutex_exit(&hca_hdl->ha_mutex);
-	}
+	/* Not doing reference counting, which adversely effects performance */
 
 	return (status);
 }
@@ -627,11 +599,7 @@
 
 	status = (IBTL_HCA2CIHCAOPS_P(hca_hdl)->ibc_unmap_mem_iov(
 	    IBTL_HCA2CIHCA(hca_hdl), mi_hdl));
-	if (status == IBT_SUCCESS) {
-		mutex_enter(&hca_hdl->ha_mutex);
-		hca_hdl->ha_ma_cnt--;
-		mutex_exit(&hca_hdl->ha_mutex);
-	}
+	/* Not doing reference counting, which adversely effects performance */
 
 	return (status);
 }
@@ -720,9 +688,7 @@
 	    IBTL_HCA2CIHCA(hca_hdl), pd, flags, phys_buf_list_sz, mr_hdl_p,
 	    mem_desc_p);
 	if (status == IBT_SUCCESS) {
-		mutex_enter(&hca_hdl->ha_mutex);
-		hca_hdl->ha_mr_cnt++;
-		mutex_exit(&hca_hdl->ha_mutex);
+		atomic_inc_32(&hca_hdl->ha_mr_cnt);
 	}
 
 	return (status);
@@ -759,9 +725,7 @@
 	    NULL, /* IBTL_HCA2MODI_P(hca_hdl)->mi_reserved */
 	    mr_hdl_p, mem_desc_p);
 	if (status == IBT_SUCCESS) {
-		mutex_enter(&hca_hdl->ha_mutex);
-		hca_hdl->ha_mr_cnt++;
-		mutex_exit(&hca_hdl->ha_mutex);
+		atomic_inc_32(&hca_hdl->ha_mr_cnt);
 	}
 
 	return (status);
@@ -805,9 +769,7 @@
 		    "Re-registration Mem Failed: %d", status);
 
 		/* we lost one memory region resource */
-		mutex_enter(&hca_hdl->ha_mutex);
-		hca_hdl->ha_mr_cnt--;
-		mutex_exit(&hca_hdl->ha_mutex);
+		atomic_dec_32(&hca_hdl->ha_mr_cnt);
 
 	}
 	return (status);
@@ -842,9 +804,7 @@
 	}
 
 	/* Update the FMR resource count */
-	mutex_enter(&hca_hdl->ha_mutex);
-	hca_hdl->ha_fmr_pool_cnt++;
-	mutex_exit(&hca_hdl->ha_mutex);
+	atomic_inc_32(&hca_hdl->ha_fmr_pool_cnt);
 
 	return (status);
 }
@@ -873,9 +833,7 @@
 		return (status);
 	}
 
-	mutex_enter(&hca_hdl->ha_mutex);
-	hca_hdl->ha_fmr_pool_cnt--;
-	mutex_exit(&hca_hdl->ha_mutex);
+	atomic_dec_32(&hca_hdl->ha_fmr_pool_cnt);
 
 	return (status);
 }
--- a/usr/src/uts/common/io/ib/ibtl/ibtl_qp.c	Mon Jun 07 08:54:25 2010 -0700
+++ b/usr/src/uts/common/io/ib/ibtl/ibtl_qp.c	Mon Jun 07 09:54:51 2010 -0700
@@ -185,9 +185,7 @@
 	mutex_init(&chanp->ch_cm_mutex, NULL, MUTEX_DEFAULT, NULL);
 	cv_init(&chanp->ch_cm_cv, NULL, CV_DEFAULT, NULL);
 
-	mutex_enter(&hca_hdl->ha_mutex);
-	hca_hdl->ha_qp_cnt++;
-	mutex_exit(&hca_hdl->ha_mutex);
+	atomic_inc_32(&hca_hdl->ha_qp_cnt);
 
 	IBTF_DPRINTF_L2(ibtf_qp, "ibt_alloc_qp: SUCCESS: qp %p owned by '%s'",
 	    chanp, hca_hdl->ha_clnt_devp->clnt_name);
@@ -393,9 +391,7 @@
 	chanp->ch_qp.qp_flags = qp_attrp->qp_flags;
 	chanp->ch_qp.qp_pd_hdl = qp_attrp->qp_pd_hdl;
 
-	mutex_enter(&hca_hdl->ha_mutex);
-	hca_hdl->ha_qp_cnt++;
-	mutex_exit(&hca_hdl->ha_mutex);
+	atomic_inc_32(&hca_hdl->ha_qp_cnt);
 
 	*ibt_qp_p = chanp;
 
@@ -701,9 +697,7 @@
 				mutex_enter(&ibtl_clnt_list_mutex);
 				ibtl_hca->ha_qpn_cnt++;
 				mutex_exit(&ibtl_clnt_list_mutex);
-				mutex_enter(&ibtl_hca->ha_mutex);
-				ibtl_hca->ha_qp_cnt--;
-				mutex_exit(&ibtl_hca->ha_mutex);
+				atomic_dec_32(&ibtl_hca->ha_qp_cnt);
 				IBTF_DPRINTF_L3(ibtf_qp, "ibt_free_qp(%p) - "
 				    "SUCCESS", ibt_qp);
 			} else
@@ -724,9 +718,7 @@
 		/* effectively, this is kmem_free(ibt_qp); */
 		ibtl_free_qp_async_check(&ibt_qp->ch_qp);
 
-		mutex_enter(&ibtl_hca->ha_mutex);
-		ibtl_hca->ha_qp_cnt--;
-		mutex_exit(&ibtl_hca->ha_mutex);
+		atomic_dec_32(&ibtl_hca->ha_qp_cnt);
 		IBTF_DPRINTF_L3(ibtf_qp, "ibt_free_qp(%p) - SUCCESS", ibt_qp);
 	} else {
 		IBTF_DPRINTF_L2(ibtf_qp, "ibt_free_qp: "
--- a/usr/src/uts/common/io/ib/ibtl/ibtl_srq.c	Mon Jun 07 08:54:25 2010 -0700
+++ b/usr/src/uts/common/io/ib/ibtl/ibtl_srq.c	Mon Jun 07 09:54:51 2010 -0700
@@ -2,9 +2,8 @@
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
- * Common Development and Distribution License, Version 1.0 only
- * (the "License").  You may not use this file except in compliance
- * with the License.
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or http://www.opensolaris.org/os/licensing.
@@ -20,10 +19,8 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2004 Sun Microsystems, Inc.  All rights reserved.
- * Use is subject to license terms.
+ * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
  */
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
 
 #include <sys/ib/ibtl/impl/ibtl.h>
 
@@ -85,9 +82,7 @@
 	_NOTE(NOW_VISIBLE_TO_OTHER_THREADS(ibt_srq->srq_hca))
 
 	/* Update the srq resource count */
-	mutex_enter(&hca_hdl->ha_mutex);
-	hca_hdl->ha_srq_cnt++;
-	mutex_exit(&hca_hdl->ha_mutex);
+	atomic_inc_32(&hca_hdl->ha_srq_cnt);
 
 	return (IBT_SUCCESS);
 }
@@ -117,9 +112,7 @@
 	ibtl_free_srq_async_check(ibt_srq);
 
 	/* Update the srq resource count */
-	mutex_enter(&ibt_hca->ha_mutex);
-	ibt_hca->ha_srq_cnt--;
-	mutex_exit(&ibt_hca->ha_mutex);
+	atomic_dec_32(&ibt_hca->ha_srq_cnt);
 
 	return (status);
 }
--- a/usr/src/uts/common/sys/ib/ibtl/impl/ibtl.h	Mon Jun 07 08:54:25 2010 -0700
+++ b/usr/src/uts/common/sys/ib/ibtl/impl/ibtl.h	Mon Jun 07 09:54:51 2010 -0700
@@ -19,8 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
- * Use is subject to license terms.
+ * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
  */
 
 #ifndef _SYS_IB_IBTL_IMPL_IBTL_H
@@ -210,9 +209,9 @@
 	ibtl_hca_devinfo_t	*ha_hca_devp;	/* CI HCA device structure. */
 	ibtl_clnt_t		*ha_clnt_devp;	/* Client state struct */
 	void			*ha_clnt_private;
-	kmutex_t		ha_mutex;	/* Mutex to protect resource */
-						/* counters. */
 	int			ha_flags;	/* misc. flags */
+
+	/* The following counters are accessed with atomic operations. */
 	uint32_t		ha_qp_cnt;	/* QP resource counter */
 	uint32_t		ha_eec_cnt;	/* EEC resource counter */
 	uint32_t		ha_cq_cnt;	/* CQ resource counter */
@@ -225,7 +224,6 @@
 	ibtl_async_flags_t	ha_async_flags;	/* see *_async_flags above */
 	uint32_t		ha_async_cnt;	/* #asyncs in progress */
 	uint32_t		ha_fmr_pool_cnt; /* FMR Pool resource count */
-	uint32_t		ha_ma_cnt;	/* Mem Area resource count */
 } ibtl_hca_t;
 
 /* ha_flags values */