changeset 13812:903032120792

3179 vmem_xalloc needs to check for overflow Reviewed by: Richard Lowe <richlowe@richlowe.net> Approved by: Robert Mustacchi <rm@joyent.com>
author Bryan Cantrill <bryan@joyent.com>
date Wed, 12 Sep 2012 10:54:28 -0400
parents 4dadf1a8e003
children 57f9d55c2a31
files usr/src/uts/common/os/vmem.c
diffstat 1 files changed, 16 insertions(+), 0 deletions(-) [+]
line wrap: on
line diff
--- a/usr/src/uts/common/os/vmem.c	Wed Aug 08 09:59:53 2012 +0300
+++ b/usr/src/uts/common/os/vmem.c	Wed Sep 12 10:54:28 2012 -0400
@@ -25,6 +25,7 @@
 
 /*
  * Copyright (c) 2012 by Delphix. All rights reserved.
+ * Copyright (c) 2012, Joyent, Inc. All rights reserved.
  */
 
 /*
@@ -1065,6 +1066,21 @@
 			aneeded = MAX(size + aphase, vmp->vm_min_import);
 			asize = P2ROUNDUP(aneeded, aquantum);
 
+			if (asize < size) {
+				/*
+				 * The rounding induced overflow; return NULL
+				 * if we are permitted to fail the allocation
+				 * (and explicitly panic if we aren't).
+				 */
+				if ((vmflag & VM_NOSLEEP) &&
+				    !(vmflag & VM_PANIC)) {
+					mutex_exit(&vmp->vm_lock);
+					return (NULL);
+				}
+
+				panic("vmem_xalloc(): size overflow");
+			}
+
 			/*
 			 * Determine how many segment structures we'll consume.
 			 * The calculation must be precise because if we're