--- a/usr/src/uts/common/vm/anon.h	Thu Jul 29 16:10:08 2010 -0600
+++ b/usr/src/uts/common/vm/anon.h	Thu Jul 29 18:49:19 2010 -0700
@@ -426,7 +426,7 @@
 extern void	anon_decref(struct anon *);
 extern int	non_anon(struct anon_hdr *, ulong_t, u_offset_t *, size_t *);
 extern pgcnt_t	anon_pages(struct anon_hdr *, ulong_t, pgcnt_t);
-extern int	anon_swap_adjust(pgcnt_t);
+extern int	anon_swap_adjust(pgcnt_t, pgcnt_t, int);
 extern void	anon_swap_restore(pgcnt_t);
 extern struct	anon_hdr *anon_create(pgcnt_t, int);
 extern void	anon_release(struct anon_hdr *, pgcnt_t);

--- a/usr/src/uts/common/vm/page.h	Thu Jul 29 16:10:08 2010 -0600
+++ b/usr/src/uts/common/vm/page.h	Thu Jul 29 18:49:19 2010 -0700
@@ -823,7 +823,7 @@
 void	page_settoxic(page_t *, uchar_t);
 
 int	page_mem_avail(pgcnt_t);
-int	page_reclaim_mem(pgcnt_t, pgcnt_t, int);
+int	page_reclaim_mem(pgcnt_t, pgcnt_t, int, int);
 
 void page_set_props(page_t *, uint_t);
 void page_clr_all_props(page_t *);

--- a/usr/src/uts/common/vm/seg_spt.c	Thu Jul 29 16:10:08 2010 -0600
+++ b/usr/src/uts/common/vm/seg_spt.c	Thu Jul 29 18:49:19 2010 -0700
@@ -399,7 +399,7 @@
 	    tnf_opaque, addr, addr, tnf_ulong, len, seg->s_size);
 #endif
 	if ((sptcargs->flags & SHM_PAGEABLE) == 0) {
-		if (err = anon_swap_adjust(npages))
+		if (err = anon_swap_adjust(npages, segspt_minfree, 60))
 			return (err);
 	}
 	err = ENOMEM;

--- a/usr/src/uts/common/vm/vm_anon.c	Thu Jul 29 16:10:08 2010 -0600
+++ b/usr/src/uts/common/vm/vm_anon.c	Thu Jul 29 18:49:19 2010 -0700
@@ -863,7 +863,7 @@
 		}
 
 		mutex_exit(&anoninfo_lock);
-		(void) page_reclaim_mem(mswap_pages, floor_pages, 0);
+		(void) page_reclaim_mem(mswap_pages, floor_pages, 0, 60);
 		mutex_enter(&anoninfo_lock);
 	}
 
@@ -3450,12 +3450,20 @@
 }
 
 /*
- * Move reserved phys swap into memory swap (unreserve phys swap
- * and reserve mem swap by the same amount).
- * Used by segspt when it needs to lock reserved swap npages in memory
+ * When memory is locked the corresponding swap is unreserved.
+ * Swap for locked memory is "locked" which means that it can
+ * not be unreserved until its memory is unlocked.
+ * If there is enough of mem swap reserved, nothing is done
+ * because availrmem was already decremented (when mem swap
+ * was reserved).
+ * If there is not enough of mem swap reserved, move reserved disk
+ * swap into memory swap (unreserve phys swap and reserve mem swap
+ * by the same amount) and decrement availrmem. The availrmem needs
+ * to be decremented because pages are locked!.
+ * Used by segspt and mlock when memory is locked.
  */
 int
-anon_swap_adjust(pgcnt_t npages)
+anon_swap_adjust(pgcnt_t npages, pgcnt_t limit, int cnt)
 {
 	pgcnt_t unlocked_mem_swap;
 
@@ -3473,7 +3481,7 @@
 		 * if there is not enough unlocked mem swap we take missing
 		 * amount from phys swap and give it to mem swap
 		 */
-		if (!page_reclaim_mem(adjusted_swap, segspt_minfree, 1)) {
+		if (!page_reclaim_mem(adjusted_swap, limit, 1, cnt)) {
 			mutex_exit(&anoninfo_lock);
 			return (ENOMEM);
 		}
@@ -3495,8 +3503,9 @@
 }
 
 /*
- * 'unlocked' reserved mem swap so when it is unreserved it
- * can be moved back phys (disk) swap
+ * When memory is unlocked make its "locked" swap freeable.
+ * The unlocked mem swap is unreserved (and availrmem is decremented)
+ * in anon_unresvmem().
  */
 void
 anon_swap_restore(pgcnt_t npages)

--- a/usr/src/uts/common/vm/vm_page.c	Thu Jul 29 16:10:08 2010 -0600
+++ b/usr/src/uts/common/vm/vm_page.c	Thu Jul 29 18:49:19 2010 -0700
@@ -109,12 +109,13 @@
  * pages_useclaim,pages_claimed : These two variables track the
  * claim adjustments because of the protection changes on a segvn segment.
  *
- * All these globals are protected by the same lock which protects availrmem.
+ * The pages_locked is protected by pages_locked_lock lock.
+ * All other globals are protected by the same lock which protects availrmem.
  */
 pgcnt_t pages_locked = 0;
 pgcnt_t pages_useclaim = 0;
 pgcnt_t pages_claimed = 0;
-
+kmutex_t pages_locked_lock;
 
 /*
  * new_freemem_lock protects freemem, freemem_wait & freemem_cv.
@@ -3097,18 +3098,27 @@
 		 * We are doing a modified version of page_pp_unlock here.
 		 */
 		if ((pp->p_lckcnt != 0) || (pp->p_cowcnt != 0)) {
-			mutex_enter(&freemem_lock);
 			if (pp->p_lckcnt != 0) {
-				availrmem++;
+				/*
+				 * Page has not been unlocked via
+				 * page_pp_unlock(). Therefore
+				 * anon_swap_restore() is called to unlock
+				 * swap for this page so its swap can be
+				 * unreserved.
+				 */
+				anon_swap_restore(1);
+				mutex_enter(&pages_locked_lock);
 				pages_locked--;
+				mutex_exit(&pages_locked_lock);
 				pp->p_lckcnt = 0;
 			}
 			if (pp->p_cowcnt != 0) {
-				availrmem += pp->p_cowcnt;
+				anon_swap_restore(pp->p_cowcnt);
+				mutex_enter(&pages_locked_lock);
 				pages_locked -= pp->p_cowcnt;
+				mutex_exit(&pages_locked_lock);
 				pp->p_cowcnt = 0;
 			}
-			mutex_exit(&freemem_lock);
 		}
 		/*
 		 * Put the page on the "free" list.
@@ -3826,20 +3836,20 @@
 	 * Acquire the "freemem_lock" for availrmem.
 	 */
 	if (cow) {
-		mutex_enter(&freemem_lock);
-		if ((availrmem > pages_pp_maximum) &&
-		    (pp->p_cowcnt < (ushort_t)PAGE_LOCK_MAXIMUM)) {
-			availrmem--;
-			pages_locked++;
-			mutex_exit(&freemem_lock);
-			r = 1;
-			if (++pp->p_cowcnt == (ushort_t)PAGE_LOCK_MAXIMUM) {
-				cmn_err(CE_WARN,
-				    "COW lock limit reached on pfn 0x%lx",
-				    page_pptonum(pp));
+		if (pp->p_cowcnt < (ushort_t)PAGE_LOCK_MAXIMUM) {
+			if (!anon_swap_adjust(1, pages_pp_maximum, 0)) {
+				mutex_enter(&pages_locked_lock);
+				pages_locked++;
+				mutex_exit(&pages_locked_lock);
+				r = 1;
+				if (++pp->p_cowcnt ==
+				    (ushort_t)PAGE_LOCK_MAXIMUM) {
+					cmn_err(CE_WARN,
+					    "COW lock limit on pfn 0x%lx",
+					page_pptonum(pp));
+				}
 			}
-		} else
-			mutex_exit(&freemem_lock);
+		}
 	} else {
 		if (pp->p_lckcnt) {
 			if (pp->p_lckcnt < (ushort_t)PAGE_LOCK_MAXIMUM) {
@@ -3857,14 +3867,13 @@
 				++pp->p_lckcnt;
 				r = 1;
 			} else {
-				mutex_enter(&freemem_lock);
-				if (availrmem > pages_pp_maximum) {
-					availrmem--;
+				if (!anon_swap_adjust(1, pages_pp_maximum, 0)) {
+					mutex_enter(&pages_locked_lock);
 					pages_locked++;
+					mutex_exit(&pages_locked_lock);
 					++pp->p_lckcnt;
 					r = 1;
 				}
-				mutex_exit(&freemem_lock);
 			}
 		}
 	}
@@ -3895,19 +3904,19 @@
 	 */
 	if (cow) {
 		if (pp->p_cowcnt) {
-			mutex_enter(&freemem_lock);
+			anon_swap_restore(1);
 			pp->p_cowcnt--;
-			availrmem++;
+			mutex_enter(&pages_locked_lock);
 			pages_locked--;
-			mutex_exit(&freemem_lock);
+			mutex_exit(&pages_locked_lock);
 		}
 	} else {
 		if (pp->p_lckcnt && --pp->p_lckcnt == 0) {
 			if (!kernel) {
-				mutex_enter(&freemem_lock);
-				availrmem++;
+				anon_swap_restore(1);
+				mutex_enter(&pages_locked_lock);
 				pages_locked--;
-				mutex_exit(&freemem_lock);
+				mutex_exit(&pages_locked_lock);
 			}
 		}
 	}
@@ -5768,12 +5777,11 @@
 	return (1);
 }
 
-#define	MAX_CNT	60	/* max num of iterations */
 /*
  * Reclaim/reserve availrmem for npages.
  * If there is not enough memory start reaping seg, kmem caches.
  * Start pageout scanner (via page_needfree()).
- * Exit after ~ MAX_CNT s regardless of how much memory has been released.
+ * Exit after max_cnt tries regardless of how much memory has been released.
  * Note: There is no guarantee that any availrmem will be freed as
  * this memory typically is locked (kernel heap) or reserved for swap.
  * Also due to memory fragmentation kmem allocator may not be able
@@ -5781,7 +5789,7 @@
  * freeing slab or a page).
  */
 int
-page_reclaim_mem(pgcnt_t npages, pgcnt_t epages, int adjust)
+page_reclaim_mem(pgcnt_t npages, pgcnt_t epages, int adjust, int max_cnt)
 {
 	int	i = 0;
 	int	ret = 0;
@@ -5791,7 +5799,7 @@
 	mutex_enter(&freemem_lock);
 	old_availrmem = availrmem - 1;
 	while ((availrmem < tune.t_minarmem + npages + epages) &&
-	    (old_availrmem < availrmem) && (i++ < MAX_CNT)) {
+	    (old_availrmem < availrmem) && (i++ < max_cnt)) {
 		old_availrmem = availrmem;
 		deficit = tune.t_minarmem + npages + epages - availrmem;
 		mutex_exit(&freemem_lock);