Mercurial > illumos > illumos-gate

changeset 10326:8e3fbeec2d76

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
6868411 NFS provider generates error on ci_remote on RDMA operations
author Siddheshwar Mahesh <Siddheshwar.Mahesh@Sun.COM>
date Mon, 17 Aug 2009 13:26:49 -0500
parents 111393108d23
children 9cfc17131b4f
files usr/src/lib/libdtrace/common/nfs.d usr/src/uts/common/rpc/rpc_rdma.h usr/src/uts/common/rpc/rpcib.c usr/src/uts/common/rpc/svc.h usr/src/uts/common/rpc/svc_rdma.c
diffstat 5 files changed, 93 insertions(+), 29 deletions(-) [+]
line wrap: on
line diff
--- a/usr/src/lib/libdtrace/common/nfs.d	Mon Aug 17 13:11:01 2009 -0400
+++ b/usr/src/lib/libdtrace/common/nfs.d	Mon Aug 17 13:26:49 2009 -0500
@@ -28,6 +28,9 @@
 #pragma	D depends_on library net.d
 #pragma	D depends_on module genunix
 
+inline int T_RDMA = 4;
+#pragma D binding "1.5" T_RDMA
+
 typedef struct nfsv4opinfo {
 	uint64_t noi_xid;	/* unique transation ID */
 	cred_t *noi_cred;	/* credentials for operation */
@@ -40,28 +43,29 @@
 
 #pragma D binding "1.5" translator
 translator conninfo_t < struct svc_req *P > {
-	ci_protocol = P->rq_xprt->xp_master->xp_netid == "tcp" ? "ipv4" :
-	    P->rq_xprt->xp_master->xp_netid == "udp" ? "ipv4" :
-	    P->rq_xprt->xp_master->xp_netid == "tcp6" ? "ipv6" :
-	    P->rq_xprt->xp_master->xp_netid == "udp6" ? "ipv6" :
+	ci_protocol = P->rq_xprt->xp_xpc.xpc_type == T_RDMA ? "rdma" :
+	    P->rq_xprt->xp_xpc.xpc_netid == "tcp" ? "ipv4" :
+	    P->rq_xprt->xp_xpc.xpc_netid == "udp" ? "ipv4" :
+	    P->rq_xprt->xp_xpc.xpc_netid == "tcp6" ? "ipv6" :
+	    P->rq_xprt->xp_xpc.xpc_netid == "udp6" ? "ipv6" :
 	    "<unknown>";
 
-	ci_local = (P->rq_xprt->xp_master->xp_netid == "tcp" ||
-	    P->rq_xprt->xp_master->xp_netid == "udp") ?
+	ci_local = (P->rq_xprt->xp_xpc.xpc_netid == "tcp" ||
+	    P->rq_xprt->xp_xpc.xpc_netid == "udp") ?
 	    inet_ntoa(&((struct sockaddr_in *)
 	    P->rq_xprt->xp_xpc.xpc_lcladdr.buf)->sin_addr.S_un.S_addr) :
-	    (P->rq_xprt->xp_master->xp_netid == "tcp6" ||
-	    P->rq_xprt->xp_master->xp_netid == "udp6") ?
+	    (P->rq_xprt->xp_xpc.xpc_netid == "tcp6" ||
+	    P->rq_xprt->xp_xpc.xpc_netid == "udp6") ?
 	    inet_ntoa6(&((struct sockaddr_in6 *)
 	    P->rq_xprt->xp_xpc.xpc_lcladdr.buf)->sin6_addr) :
 	    "unknown";
 
-	ci_remote = (P->rq_xprt->xp_master->xp_netid == "tcp" ||
-	    P->rq_xprt->xp_master->xp_netid == "udp") ?
+	ci_remote = (P->rq_xprt->xp_xpc.xpc_netid == "tcp" ||
+	    P->rq_xprt->xp_xpc.xpc_netid == "udp") ?
 	    inet_ntoa(&((struct sockaddr_in *)
 	    P->rq_xprt->xp_xpc.xpc_rtaddr.buf)->sin_addr.S_un.S_addr) :
-	    (P->rq_xprt->xp_master->xp_netid == "tcp6" ||
-	    P->rq_xprt->xp_master->xp_netid == "udp6") ?
+	    (P->rq_xprt->xp_xpc.xpc_netid == "tcp6" ||
+	    P->rq_xprt->xp_xpc.xpc_netid == "udp6") ?
 	    inet_ntoa6(&((struct sockaddr_in6 *)
 	    P->rq_xprt->xp_xpc.xpc_rtaddr.buf)->sin6_addr) :
 	    "unknown";
@@ -69,15 +73,27 @@
 
 #pragma D binding "1.5" translator
 translator conninfo_t < struct compound_state *P > {
-	ci_protocol = P->req->rq_xprt->xp_master->xp_netid == "tcp" ? "ipv4" :
-	    P->req->rq_xprt->xp_master->xp_netid == "tcp6" ? "ipv6" :
+	ci_protocol = P->req->rq_xprt->xp_xpc.xpc_type == T_RDMA ? "rdma" :
+	    P->req->rq_xprt->xp_xpc.xpc_netid == "tcp" ? "ipv4" :
+	    P->req->rq_xprt->xp_xpc.xpc_netid == "tcp6" ? "ipv6" :
 	    "<unknown>";
 
-	ci_local = inet_ntoa6(&((conn_t *)P->req->rq_xprt->xp_xpc.
-	    xpc_wq->q_next->q_ptr)->connua_v6addr.connua_laddr);
+	ci_local = (P->req->rq_xprt->xp_xpc.xpc_netid == "tcp") ?
+	    inet_ntoa(&((struct sockaddr_in *)
+	    P->req->rq_xprt->xp_xpc.xpc_lcladdr.buf)->sin_addr.S_un.S_addr) :
+	    (P->req->rq_xprt->xp_xpc.xpc_netid == "tcp6") ?
+	    inet_ntoa6(&((struct sockaddr_in6 *)
+	    P->req->rq_xprt->xp_xpc.xpc_lcladdr.buf)->sin6_addr) :
+	    "unknown";
 
-	ci_remote = inet_ntoa6(&((conn_t *)P->req->rq_xprt->xp_xpc.
-	    xpc_wq->q_next->q_ptr)->connua_v6addr.connua_faddr);
+	ci_remote = (P->req->rq_xprt->xp_xpc.xpc_netid == "tcp") ?
+	    inet_ntoa(&((struct sockaddr_in *)
+	    P->req->rq_xprt->xp_xpc.xpc_rtaddr.buf)->sin_addr.S_un.S_addr) :
+	    (P->req->rq_xprt->xp_xpc.xpc_netid == "tcp6") ?
+	    inet_ntoa6(&((struct sockaddr_in6 *)
+	    P->req->rq_xprt->xp_xpc.xpc_rtaddr.buf)->sin6_addr) :
+	    "unknown";
+
 };
 
 #pragma D binding "1.5" translator
--- a/usr/src/uts/common/rpc/rpc_rdma.h	Mon Aug 17 13:11:01 2009 -0400
+++ b/usr/src/uts/common/rpc/rpc_rdma.h	Mon Aug 17 13:26:49 2009 -0500
@@ -342,6 +342,7 @@
  */
 typedef struct conn {
 	rdma_mod_t	*c_rdmamod;	/* RDMA transport info for conn */
+	char 		*c_netid;	/* tcp or tcp6 token */
 	struct netbuf	c_raddr;	/* remote address */
 	struct netbuf	c_laddr;	/* local address */
 	int		c_ref;		/* no. of clients of connection */
--- a/usr/src/uts/common/rpc/rpcib.c	Mon Aug 17 13:11:01 2009 -0400
+++ b/usr/src/uts/common/rpc/rpcib.c	Mon Aug 17 13:26:49 2009 -0500
@@ -217,6 +217,9 @@
 static uint64_t max_unsignaled_rws = 5;
 int nfs_rdma_port = NFS_RDMA_PORT;
 
+#define	RIBNETID_TCP	"tcp"
+#define	RIBNETID_TCP6	"tcp6"
+
 /*
  * rib_stat: private data pointer used when registering
  *	with the IBTF.  It is returned to the consumer
@@ -2095,6 +2098,9 @@
 	if (conn->c_laddr.buf != NULL) {
 		kmem_free(conn->c_laddr.buf, conn->c_laddr.len);
 	}
+	if (conn->c_netid != NULL) {
+		kmem_free(conn->c_netid, (strlen(conn->c_netid) + 1));
+	}
 
 	/*
 	 * Credit control cleanup.
@@ -3042,20 +3048,36 @@
 		switch (ipinfo.src_addr.family) {
 		case AF_INET:
 
+			conn->c_netid = kmem_zalloc(strlen(RIBNETID_TCP) + 1,
+			    KM_SLEEP);
+			(void) strcpy(conn->c_netid, RIBNETID_TCP);
+
 			conn->c_raddr.maxlen =
 			    conn->c_raddr.len = sin_size;
 			conn->c_raddr.buf = kmem_zalloc(sin_size, KM_SLEEP);
 
 			s = (struct sockaddr_in *)conn->c_raddr.buf;
 			s->sin_family = AF_INET;
-
 			bcopy((void *)&ipinfo.src_addr.un.ip4addr,
 			    &s->sin_addr, in_size);
 
+			conn->c_laddr.maxlen =
+			    conn->c_laddr.len = sin_size;
+			conn->c_laddr.buf = kmem_zalloc(sin_size, KM_SLEEP);
+
+			s = (struct sockaddr_in *)conn->c_laddr.buf;
+			s->sin_family = AF_INET;
+			bcopy((void *)&ipinfo.dst_addr.un.ip4addr,
+			    &s->sin_addr, in_size);
+
 			break;
 
 		case AF_INET6:
 
+			conn->c_netid = kmem_zalloc(strlen(RIBNETID_TCP6) + 1,
+			    KM_SLEEP);
+			(void) strcpy(conn->c_netid, RIBNETID_TCP6);
+
 			conn->c_raddr.maxlen =
 			    conn->c_raddr.len = sin6_size;
 			conn->c_raddr.buf = kmem_zalloc(sin6_size, KM_SLEEP);
@@ -3066,6 +3088,16 @@
 			    &s6->sin6_addr,
 			    sizeof (struct in6_addr));
 
+			conn->c_laddr.maxlen =
+			    conn->c_laddr.len = sin6_size;
+			conn->c_laddr.buf = kmem_zalloc(sin6_size, KM_SLEEP);
+
+			s6 = (struct sockaddr_in6 *)conn->c_laddr.buf;
+			s6->sin6_family = AF_INET6;
+			bcopy((void *)&ipinfo.dst_addr.un.ip6addr,
+			    &s6->sin6_addr,
+			    sizeof (struct in6_addr));
+
 			break;
 
 		default:
@@ -4339,6 +4371,14 @@
 	bcopy(s_addr_buf, cn->c_laddr.buf, s_addr_len);
 	cn->c_laddr.len = cn->c_laddr.maxlen = s_addr_len;
 
+	if (rpt->srcip.family == AF_INET) {
+		cn->c_netid = kmem_zalloc(strlen(RIBNETID_TCP) + 1, KM_SLEEP);
+		(void) strcpy(cn->c_netid, RIBNETID_TCP);
+	} else {
+		cn->c_netid = kmem_zalloc(strlen(RIBNETID_TCP6) + 1, KM_SLEEP);
+		(void) strcpy(cn->c_netid, RIBNETID_TCP6);
+	}
+
 	/*
 	 * Add to conn list.
 	 * We had given up the READER lock. In the time since then,
--- a/usr/src/uts/common/rpc/svc.h	Mon Aug 17 13:11:01 2009 -0400
+++ b/usr/src/uts/common/rpc/svc.h	Mon Aug 17 13:26:49 2009 -0500
@@ -390,6 +390,7 @@
 	int		xpc_msg_size;	/* TSDU or TIDU size		*/
 	struct netbuf	xpc_rtaddr;	/* remote transport address	*/
 	struct netbuf	xpc_lcladdr;	/* local transport address	*/
+	char		*xpc_netid;	/* network token		*/
 	SVC_CALLOUT_TABLE *xpc_sct;
 } __SVCXPRT_COMMON;
 
@@ -402,6 +403,7 @@
 #define	xp_rtaddr	xp_xpc.xpc_rtaddr
 #define	xp_lcladdr	xp_xpc.xpc_lcladdr
 #define	xp_sct		xp_xpc.xpc_sct
+#define	xp_netid	xp_xpc.xpc_netid
 
 struct __svcmasterxprt {
 	SVCMASTERXPRT 	*xp_next;	/* Next transport in the list	*/
@@ -416,7 +418,6 @@
 	kmutex_t	xp_thread_lock;	/* Thread count lock		*/
 	void		(*xp_closeproc)(const SVCMASTERXPRT *);
 					/* optional; see comments above	*/
-	char		*xp_netid;	/* network token		*/
 	struct netbuf	xp_addrmask;	/* address mask			*/
 
 	caddr_t		xp_p2;		/* private: for use by svc ops  */
@@ -501,7 +502,7 @@
 #ifdef _KERNEL
 #define	svc_getcaller(x) (&(x)->xp_rtaddr.buf)
 #define	svc_getaddrmask(x) (&(x)->xp_master->xp_addrmask)
-#define	svc_getnetid(x) ((x)->xp_master->xp_netid)
+#define	svc_getnetid(x) ((x)->xp_netid)
 #endif	/* _KERNEL */
 
 /*
--- a/usr/src/uts/common/rpc/svc_rdma.c	Mon Aug 17 13:11:01 2009 -0400
+++ b/usr/src/uts/common/rpc/svc_rdma.c	Mon Aug 17 13:26:49 2009 -0500
@@ -264,12 +264,6 @@
 		q->q_ptr = &rd->rd_xprt;
 		xprt->xp_netid = NULL;
 
-		if (netid != NULL) {
-			xprt->xp_netid = kmem_alloc(strlen(netid) + 1,
-			    KM_SLEEP);
-			(void) strcpy(xprt->xp_netid, netid);
-		}
-
 		xprt->xp_addrmask.maxlen =
 		    xprt->xp_addrmask.len = sizeof (struct sockaddr_in);
 		xprt->xp_addrmask.buf =
@@ -340,7 +334,6 @@
 
 	mutex_destroy(&xprt->xp_req_lock);
 	mutex_destroy(&xprt->xp_thread_lock);
-	kmem_free(xprt->xp_netid, strlen(xprt->xp_netid) + 1);
 	kmem_free(rd, sizeof (*rd));
 	kmem_free(xprt->xp_addrmask.buf, xprt->xp_addrmask.maxlen);
 	kmem_free(xprt, sizeof (*xprt));
@@ -398,7 +391,6 @@
 {
 	XDR	*xdrs;
 	CONN	*conn;
-
 	rdma_recv_data_t	*rdp = (rdma_recv_data_t *)mp->b_rptr;
 	struct clone_rdma_data *crdp;
 	struct clist	*cl = NULL;
@@ -552,6 +544,20 @@
 	clone_xprt->xp_rtaddr.buf = conn->c_raddr.buf;
 	clone_xprt->xp_rtaddr.len = conn->c_raddr.len;
 	clone_xprt->xp_rtaddr.maxlen = conn->c_raddr.len;
+
+	clone_xprt->xp_lcladdr.buf = conn->c_laddr.buf;
+	clone_xprt->xp_lcladdr.len = conn->c_laddr.len;
+	clone_xprt->xp_lcladdr.maxlen = conn->c_laddr.len;
+
+	/*
+	 * In case of RDMA, connection management is
+	 * entirely done in rpcib module and netid in the
+	 * SVCMASTERXPRT is NULL. Initialize the clone netid
+	 * from the connection.
+	 */
+
+	clone_xprt->xp_netid = conn->c_netid;
+
 	clone_xprt->xp_xid = xid;
 	crdp->conn = conn;