changeset 12920:80a39220b451

6948098 freemem_lock contention causes poor concurrent DISM MC_LOCK/MC_UNLOCK performance 6946742 sfmmu_mlspl_enter lock contention on mml_table 6956454 ani_free_pool lock contention with multi processed ISM
author Pavel Tatashin <Pavel.Tatashin@Sun.COM>
date Mon, 26 Jul 2010 07:24:36 -0700
parents eeab51820f81
children 0e5521492f8c
files usr/src/uts/common/os/clock.c usr/src/uts/common/os/cpu.c usr/src/uts/common/sys/cpuvar.h usr/src/uts/common/vm/anon.h usr/src/uts/common/vm/seg_spt.c usr/src/uts/common/vm/vm_anon.c usr/src/uts/common/vm/vm_swap.c usr/src/uts/i86pc/vm/vm_dep.h usr/src/uts/intel/ia32/ml/i86_subr.s usr/src/uts/sfmmu/vm/hat_sfmmu.c usr/src/uts/sfmmu/vm/hat_sfmmu.h usr/src/uts/sun4/cpu/cpu_module.c usr/src/uts/sun4/os/startup.c usr/src/uts/sun4/vm/sfmmu.c usr/src/uts/sun4/vm/vm_dep.h usr/src/uts/sun4u/cpu/common_asm.s usr/src/uts/sun4u/sys/machsystm.h usr/src/uts/sun4v/cpu/common_asm.s usr/src/uts/sun4v/sys/machsystm.h
diffstat 19 files changed, 238 insertions(+), 163 deletions(-) [+]
line wrap: on
line diff
--- a/usr/src/uts/common/os/clock.c	Mon Jul 26 07:07:43 2010 -0700
+++ b/usr/src/uts/common/os/clock.c	Mon Jul 26 07:24:36 2010 -0700
@@ -398,7 +398,6 @@
 	uint_t	w_io;
 	cpu_t	*cp;
 	cpupart_t *cpupart;
-	extern void set_anoninfo();
 	extern	void	set_freemem();
 	void	(*funcp)();
 	int32_t ltemp;
@@ -411,7 +410,6 @@
 	if (panicstr)
 		return;
 
-	set_anoninfo();
 	/*
 	 * Make sure that 'freemem' do not drift too far from the truth
 	 */
@@ -839,6 +837,8 @@
 
 			maxswap = k_anoninfo.ani_mem_resv +
 			    k_anoninfo.ani_max +avail;
+			/* Update ani_free */
+			set_anoninfo();
 			free = k_anoninfo.ani_free + avail;
 			resv = k_anoninfo.ani_phys_resv +
 			    k_anoninfo.ani_mem_resv;
--- a/usr/src/uts/common/os/cpu.c	Mon Jul 26 07:07:43 2010 -0700
+++ b/usr/src/uts/common/os/cpu.c	Mon Jul 26 07:24:36 2010 -0700
@@ -19,8 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
- * Use is subject to license terms.
+ * Copyright (c) 1991, 2010, Oracle and/or its affiliates. All rights reserved.
  */
 
 /*
@@ -91,8 +90,8 @@
 
 /*
  * cpu_lock protects ncpus, ncpus_online, cpu_flag, cpu_list, cpu_active,
- * and dispatch queue reallocations.  The lock ordering with respect to
- * related locks is:
+ * max_cpu_seqid_ever, and dispatch queue reallocations.  The lock ordering with
+ * respect to related locks is:
  *
  *	cpu_lock --> thread_free_lock  --->  p_lock  --->  thread_lock()
  *
@@ -133,6 +132,13 @@
  */
 processorid_t max_cpuid = NCPU - 1;
 
+/*
+ * Maximum cpu_seqid was given. This number can only grow and never shrink. It
+ * can be used to optimize NCPU loops to avoid going through CPUs which were
+ * never on-line.
+ */
+processorid_t max_cpu_seqid_ever = 0;
+
 int ncpus = 1;
 int ncpus_online = 1;
 
@@ -1758,6 +1764,10 @@
 		continue;
 	CPUSET_ADD(cpu_seqid_inuse, seqid);
 	cp->cpu_seqid = seqid;
+
+	if (seqid > max_cpu_seqid_ever)
+		max_cpu_seqid_ever = seqid;
+
 	ASSERT(ncpus < max_ncpus);
 	ncpus++;
 	cp->cpu_cache_offset = KMEM_CPU_CACHE_OFFSET(cp->cpu_seqid);
--- a/usr/src/uts/common/sys/cpuvar.h	Mon Jul 26 07:07:43 2010 -0700
+++ b/usr/src/uts/common/sys/cpuvar.h	Mon Jul 26 07:24:36 2010 -0700
@@ -20,8 +20,7 @@
  */
 
 /*
- * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
- * Use is subject to license terms.
+ * Copyright (c) 1989, 2010, Oracle and/or its affiliates. All rights reserved.
  */
 
 #ifndef _SYS_CPUVAR_H
@@ -564,6 +563,7 @@
 extern processorid_t	max_cpuid;	/* maximum CPU number */
 extern struct cpu	*cpu_inmotion;	/* offline or partition move target */
 extern cpu_t		*clock_cpu_list;
+extern processorid_t	max_cpu_seqid_ever;	/* maximum seqid ever given */
 
 #if defined(__i386) || defined(__amd64)
 extern struct cpu *curcpup(void);
--- a/usr/src/uts/common/vm/anon.h	Mon Jul 26 07:07:43 2010 -0700
+++ b/usr/src/uts/common/vm/anon.h	Mon Jul 26 07:24:36 2010 -0700
@@ -198,25 +198,28 @@
  */
 
 typedef	struct	ani_free {
-	kmutex_t	ani_lock;
 	pgcnt_t		ani_count;
-	uchar_t		pad[64 - sizeof (kmutex_t) - sizeof (pgcnt_t)];
+	uchar_t		pad[64 - sizeof (pgcnt_t)];
 			/* XXX 64 = cacheline size */
 } ani_free_t;
 
-#define	ANI_MAX_POOL	128
-extern	ani_free_t	ani_free_pool[];
+#define	ANI_MAX_POOL	(NCPU_P2)
+extern	ani_free_t	*ani_free_pool;
 
+/*
+ * Since each CPU has its own bucket in ani_free_pool, there should be no
+ * contention here.
+ */
 #define	ANI_ADD(inc)	{ \
-	ani_free_t	*anifp; \
-	int		index; \
-	index = (CPU->cpu_id & (ANI_MAX_POOL - 1)); \
-	anifp = &ani_free_pool[index]; \
-	mutex_enter(&anifp->ani_lock); \
-	anifp->ani_count += inc; \
-	mutex_exit(&anifp->ani_lock); \
+	pgcnt_t	*ani_countp; \
+	int	index; \
+	index = (CPU->cpu_seqid & (ANI_MAX_POOL - 1)); \
+	ani_countp = &ani_free_pool[index].ani_count; \
+	atomic_add_long(ani_countp, inc); \
 }
 
+extern void	set_anoninfo(void);
+
 /*
  * Anon array pointers are allocated in chunks. Each chunk
  * has PAGESIZE/sizeof(u_long *) of anon pointers.
--- a/usr/src/uts/common/vm/seg_spt.c	Mon Jul 26 07:07:43 2010 -0700
+++ b/usr/src/uts/common/vm/seg_spt.c	Mon Jul 26 07:24:36 2010 -0700
@@ -19,8 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
- * Use is subject to license terms.
+ * Copyright (c) 1993, 2010, Oracle and/or its affiliates. All rights reserved.
  */
 
 #include <sys/param.h>
@@ -2487,19 +2486,59 @@
 	return (unlocked);
 }
 
+extern	u_longlong_t randtick(void);
+/* number of locks to reserve/skip by spt_lockpages() and spt_unlockpages() */
+#define	NLCK	(NCPU_P2)
+/* Random number with a range [0, n-1], n must be power of two */
+#define	RAND_P2(n)	\
+	((((long)curthread >> PTR24_LSB) ^ (long)randtick()) & ((n) - 1))
+
 int
 spt_lockpages(struct seg *seg, pgcnt_t anon_index, pgcnt_t npages,
     page_t **ppa, ulong_t *lockmap, size_t pos,
     rctl_qty_t *locked)
 {
-	struct shm_data *shmd = seg->s_data;
-	struct spt_data *sptd = shmd->shm_sptseg->s_data;
+	struct	shm_data *shmd = seg->s_data;
+	struct	spt_data *sptd = shmd->shm_sptseg->s_data;
 	ulong_t	i;
 	int	kernel;
+	pgcnt_t	nlck = 0;
+	int	rv = 0;
+	int	use_reserved = 1;
 
 	/* return the number of bytes actually locked */
 	*locked = 0;
+
+	/*
+	 * To avoid contention on freemem_lock, availrmem and pages_locked
+	 * global counters are updated only every nlck locked pages instead of
+	 * every time.  Reserve nlck locks up front and deduct from this
+	 * reservation for each page that requires a lock.  When the reservation
+	 * is consumed, reserve again.  nlck is randomized, so the competing
+	 * threads do not fall into a cyclic lock contention pattern. When
+	 * memory is low, the lock ahead is disabled, and instead page_pp_lock()
+	 * is used to lock pages.
+	 */
 	for (i = 0; i < npages; anon_index++, pos++, i++) {
+		if (nlck == 0 && use_reserved == 1) {
+			nlck = NLCK + RAND_P2(NLCK);
+			/* if fewer loops left, decrease nlck */
+			nlck = MIN(nlck, npages - i);
+			/*
+			 * Reserve nlck locks up front and deduct from this
+			 * reservation for each page that requires a lock.  When
+			 * the reservation is consumed, reserve again.
+			 */
+			mutex_enter(&freemem_lock);
+			if ((availrmem - nlck) < pages_pp_maximum) {
+				/* Do not do advance memory reserves */
+				use_reserved = 0;
+			} else {
+				availrmem	-= nlck;
+				pages_locked	+= nlck;
+			}
+			mutex_exit(&freemem_lock);
+		}
 		if (!(shmd->shm_vpage[anon_index] & DISM_PG_LOCKED)) {
 			if (sptd->spt_ppa_lckcnt[anon_index] <
 			    (ushort_t)DISM_LOCK_MAX) {
@@ -2511,13 +2550,17 @@
 					    anon_index << PAGESHIFT);
 				}
 				kernel = (sptd->spt_ppa &&
-				    sptd->spt_ppa[anon_index]) ? 1 : 0;
-				if (!page_pp_lock(ppa[i], 0, kernel)) {
+				    sptd->spt_ppa[anon_index]);
+				if (!page_pp_lock(ppa[i], 0, kernel ||
+				    use_reserved)) {
 					sptd->spt_ppa_lckcnt[anon_index]--;
-					return (EAGAIN);
+					rv = EAGAIN;
+					break;
 				}
 				/* if this is a newly locked page, count it */
 				if (ppa[i]->p_lckcnt == 1) {
+					if (kernel == 0 && use_reserved == 1)
+						nlck--;
 					*locked += PAGESIZE;
 				}
 				shmd->shm_lckpgs++;
@@ -2527,6 +2570,86 @@
 			}
 		}
 	}
+	/* Return unused lock reservation */
+	if (nlck != 0 && use_reserved == 1) {
+		mutex_enter(&freemem_lock);
+		availrmem	+= nlck;
+		pages_locked	-= nlck;
+		mutex_exit(&freemem_lock);
+	}
+
+	return (rv);
+}
+
+int
+spt_unlockpages(struct seg *seg, pgcnt_t anon_index, pgcnt_t npages,
+    rctl_qty_t *unlocked)
+{
+	struct shm_data	*shmd = seg->s_data;
+	struct spt_data	*sptd = shmd->shm_sptseg->s_data;
+	struct anon_map	*amp = sptd->spt_amp;
+	struct anon 	*ap;
+	struct vnode 	*vp;
+	u_offset_t 	off;
+	struct page	*pp;
+	int		kernel;
+	anon_sync_obj_t	cookie;
+	ulong_t		i;
+	pgcnt_t		nlck = 0;
+	pgcnt_t		nlck_limit = NLCK;
+
+	ANON_LOCK_ENTER(&amp->a_rwlock, RW_READER);
+	for (i = 0; i < npages; i++, anon_index++) {
+		if (shmd->shm_vpage[anon_index] & DISM_PG_LOCKED) {
+			anon_array_enter(amp, anon_index, &cookie);
+			ap = anon_get_ptr(amp->ahp, anon_index);
+			ASSERT(ap);
+
+			swap_xlate(ap, &vp, &off);
+			anon_array_exit(&cookie);
+			pp = page_lookup(vp, off, SE_SHARED);
+			ASSERT(pp);
+			/*
+			 * availrmem is decremented only for pages which are not
+			 * in seg pcache, for pages in seg pcache availrmem was
+			 * decremented in _dismpagelock()
+			 */
+			kernel = (sptd->spt_ppa && sptd->spt_ppa[anon_index]);
+			ASSERT(pp->p_lckcnt > 0);
+
+			/*
+			 * lock page but do not change availrmem, we do it
+			 * ourselves every nlck loops.
+			 */
+			page_pp_unlock(pp, 0, 1);
+			if (pp->p_lckcnt == 0) {
+				if (kernel == 0)
+					nlck++;
+				*unlocked += PAGESIZE;
+			}
+			page_unlock(pp);
+			shmd->shm_vpage[anon_index] &= ~DISM_PG_LOCKED;
+			sptd->spt_ppa_lckcnt[anon_index]--;
+			shmd->shm_lckpgs--;
+		}
+
+		/*
+		 * To reduce freemem_lock contention, do not update availrmem
+		 * until at least NLCK pages have been unlocked.
+		 * 1. No need to update if nlck is zero
+		 * 2. Always update if the last iteration
+		 */
+		if (nlck > 0 && (nlck == nlck_limit || i == npages - 1)) {
+			mutex_enter(&freemem_lock);
+			availrmem	+= nlck;
+			pages_locked	-= nlck;
+			mutex_exit(&freemem_lock);
+			nlck = 0;
+			nlck_limit = NLCK + RAND_P2(NLCK);
+		}
+	}
+	ANON_LOCK_EXIT(&amp->a_rwlock);
+
 	return (0);
 }
 
@@ -2646,17 +2769,8 @@
 		kmem_free(ppa, ((sizeof (page_t *)) * a_npages));
 
 	} else if (op == MC_UNLOCK) { /* unlock */
-		struct anon_map *amp;
-		struct anon 	*ap;
-		struct vnode 	*vp;
-		u_offset_t 	off;
-		struct page	*pp;
-		int		kernel;
-		anon_sync_obj_t cookie;
-		rctl_qty_t	unlocked = 0;
 		page_t		**ppa;
 
-		amp = sptd->spt_amp;
 		mutex_enter(&sptd->spt_lock);
 		if (shmd->shm_lckpgs == 0) {
 			mutex_exit(&sptd->spt_lock);
@@ -2669,37 +2783,7 @@
 			sptd->spt_flags |= DISM_PPA_CHANGED;
 
 		mutex_enter(&sp->shm_mlock);
-		ANON_LOCK_ENTER(&amp->a_rwlock, RW_READER);
-		for (i = 0; i < npages; i++, an_idx++) {
-			if (shmd->shm_vpage[an_idx] & DISM_PG_LOCKED) {
-				anon_array_enter(amp, an_idx, &cookie);
-				ap = anon_get_ptr(amp->ahp, an_idx);
-				ASSERT(ap);
-
-				swap_xlate(ap, &vp, &off);
-				anon_array_exit(&cookie);
-				pp = page_lookup(vp, off, SE_SHARED);
-				ASSERT(pp);
-				/*
-				 * the availrmem is decremented only for
-				 * pages which are not in seg pcache,
-				 * for pages in seg pcache availrmem was
-				 * decremented in _dismpagelock() (if
-				 * they were not locked here)
-				 */
-				kernel = (sptd->spt_ppa &&
-				    sptd->spt_ppa[an_idx]) ? 1 : 0;
-				ASSERT(pp->p_lckcnt > 0);
-				page_pp_unlock(pp, 0, kernel);
-				if (pp->p_lckcnt == 0)
-					unlocked += PAGESIZE;
-				page_unlock(pp);
-				shmd->shm_vpage[an_idx] &= ~DISM_PG_LOCKED;
-				sptd->spt_ppa_lckcnt[an_idx]--;
-				shmd->shm_lckpgs--;
-			}
-		}
-		ANON_LOCK_EXIT(&amp->a_rwlock);
+		sts = spt_unlockpages(seg, an_idx, npages, &unlocked);
 		if ((ppa = sptd->spt_ppa) != NULL)
 			sptd->spt_flags |= DISM_PPA_CHANGED;
 		mutex_exit(&sptd->spt_lock);
--- a/usr/src/uts/common/vm/vm_anon.c	Mon Jul 26 07:07:43 2010 -0700
+++ b/usr/src/uts/common/vm/vm_anon.c	Mon Jul 26 07:24:36 2010 -0700
@@ -129,7 +129,7 @@
 
 kmutex_t	anoninfo_lock;
 struct		k_anoninfo k_anoninfo;
-ani_free_t	ani_free_pool[ANI_MAX_POOL];
+ani_free_t	*ani_free_pool;
 pad_mutex_t	anon_array_lock[ANON_LOCKSIZE];
 kcondvar_t	anon_array_cv[ANON_LOCKSIZE];
 
@@ -238,6 +238,10 @@
 	    NULL, NULL, 0);
 	swap_maxcontig = (1024 * 1024) >> PAGESHIFT;	/* 1MB of pages */
 
+	tmp = kmem_zalloc((ANI_MAX_POOL * sizeof (ani_free_t)) + 63, KM_SLEEP);
+	/* Round ani_free_pool to cacheline boundary to avoid false sharing. */
+	ani_free_pool = (ani_free_t *)P2ROUNDUP((uintptr_t)tmp, 64);
+
 	anon_vp = vn_alloc(KM_SLEEP);
 	vn_setops(anon_vp, swap_vnodeops);
 	anon_vp->v_type = VREG;
@@ -727,19 +731,35 @@
 
 
 /*
- * Called from clock handler to sync ani_free value.
+ * Called to sync ani_free value.
  */
 
 void
 set_anoninfo(void)
 {
-	int	ix;
-	pgcnt_t	total = 0;
-
-	for (ix = 0; ix < ANI_MAX_POOL; ix++) {
-		total += ani_free_pool[ix].ani_count;
+	processorid_t	ix, max_seqid;
+	pgcnt_t		total = 0;
+	static clock_t	last_time;
+	clock_t		new_time;
+
+	if (ani_free_pool == NULL)
+		return;
+
+	/*
+	 * Recompute ani_free at most once per tick. Use max_cpu_seqid_ever to
+	 * identify the maximum number of CPUs were ever online.
+	 */
+	new_time = ddi_get_lbolt();
+	if (new_time > last_time) {
+
+		max_seqid = max_cpu_seqid_ever;
+		ASSERT(ANI_MAX_POOL > max_seqid);
+		for (ix = 0; ix <= max_seqid; ix++)
+			total += ani_free_pool[ix].ani_count;
+
+		last_time = new_time;
+		k_anoninfo.ani_free = total;
 	}
-	k_anoninfo.ani_free = total;
 }
 
 /*
--- a/usr/src/uts/common/vm/vm_swap.c	Mon Jul 26 07:07:43 2010 -0700
+++ b/usr/src/uts/common/vm/vm_swap.c	Mon Jul 26 07:24:36 2010 -0700
@@ -484,6 +484,8 @@
 		ai.ani_max = (k_anoninfo.ani_max +
 		    k_anoninfo.ani_mem_resv) + avail;
 
+		/* Update ani_free */
+		set_anoninfo();
 		ai.ani_free = k_anoninfo.ani_free + avail;
 
 		ai.ani_resv = k_anoninfo.ani_phys_resv +
@@ -821,6 +823,8 @@
 			return (EOVERFLOW);
 		ai.ani_max = s;
 
+		/* Update ani_free */
+		set_anoninfo();
 		s = k_anoninfo.ani_free + avail;
 		if (s > UINT32_MAX)
 			return (EOVERFLOW);
--- a/usr/src/uts/i86pc/vm/vm_dep.h	Mon Jul 26 07:07:43 2010 -0700
+++ b/usr/src/uts/i86pc/vm/vm_dep.h	Mon Jul 26 07:24:36 2010 -0700
@@ -19,8 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
- * Use is subject to license terms.
+ * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
  */
 /*
  * Copyright (c) 2010, Intel Corporation.
@@ -55,7 +54,7 @@
  * correct tick value.  The proper routine to use is tsc_read().
  */
 
-extern hrtime_t		randtick();
+extern u_longlong_t	randtick();
 extern uint_t page_create_update_flags_x86(uint_t);
 
 extern size_t plcnt_sz(size_t);
--- a/usr/src/uts/intel/ia32/ml/i86_subr.s	Mon Jul 26 07:07:43 2010 -0700
+++ b/usr/src/uts/intel/ia32/ml/i86_subr.s	Mon Jul 26 07:24:36 2010 -0700
@@ -888,7 +888,7 @@
  * is called by callers who do not need to have a guarenteed
  * correct tick value.  The proper routine to use is tsc_read().
  */
-hrtime_t
+u_longlong_t
 randtick(void)
 {
 	return (0);
--- a/usr/src/uts/sfmmu/vm/hat_sfmmu.c	Mon Jul 26 07:07:43 2010 -0700
+++ b/usr/src/uts/sfmmu/vm/hat_sfmmu.c	Mon Jul 26 07:24:36 2010 -0700
@@ -736,36 +736,6 @@
 static void	sfmmu_ismhat_enter(sfmmu_t *, int);
 static void	sfmmu_ismhat_exit(sfmmu_t *, int);
 
-/*
- * Array of mutexes protecting a page's mapping list and p_nrm field.
- *
- * The hash function looks complicated, but is made up so that:
- *
- * "pp" not shifted, so adjacent pp values will hash to different cache lines
- *  (8 byte alignment * 8 bytes/mutes == 64 byte coherency subblock)
- *
- * "pp" >> mml_shift, incorporates more source bits into the hash result
- *
- *  "& (mml_table_size - 1), should be faster than using remainder "%"
- *
- * Hopefully, mml_table, mml_table_size and mml_shift are all in the same
- * cacheline, since they get declared next to each other below. We'll trust
- * ld not to do something random.
- */
-#ifdef	DEBUG
-int mlist_hash_debug = 0;
-#define	MLIST_HASH(pp)	(mlist_hash_debug ? &mml_table[0] : \
-	&mml_table[((uintptr_t)(pp) + \
-	((uintptr_t)(pp) >> mml_shift)) & (mml_table_sz - 1)])
-#else	/* !DEBUG */
-#define	MLIST_HASH(pp)   &mml_table[ \
-	((uintptr_t)(pp) + ((uintptr_t)(pp) >> mml_shift)) & (mml_table_sz - 1)]
-#endif	/* !DEBUG */
-
-kmutex_t		*mml_table;
-uint_t			mml_table_sz;	/* must be a power of 2 */
-uint_t			mml_shift;	/* log2(mml_table_sz) + 3 for align */
-
 kpm_hlk_t	*kpmp_table;
 uint_t		kpmp_table_sz;	/* must be a power of 2 */
 uchar_t		kpmp_shift;
@@ -794,13 +764,19 @@
 	((uintptr_t)(pp) >> (PP_SHIFT + SPL_SHIFT)) ^ \
 	((uintptr_t)(pp) >> (PP_SHIFT + SPL_SHIFT * 2)) ^ \
 	((uintptr_t)(pp) >> (PP_SHIFT + SPL_SHIFT * 3))) & \
-	(SPL_TABLE_SIZE - 1))
+	SPL_MASK)
 
 #define	SPL_HASH(pp)    \
-	(&sfmmu_page_lock[SPL_INDEX(pp) & SPL_MASK].pad_mutex)
+	(&sfmmu_page_lock[SPL_INDEX(pp)].pad_mutex)
 
 static	pad_mutex_t	sfmmu_page_lock[SPL_TABLE_SIZE];
 
+/* Array of mutexes protecting a page's mapping list and p_nrm field. */
+
+#define	MML_TABLE_SIZE	SPL_TABLE_SIZE
+#define	MLIST_HASH(pp)	(&mml_table[SPL_INDEX(pp)].pad_mutex)
+
+static pad_mutex_t	mml_table[MML_TABLE_SIZE];
 
 /*
  * hat_unload_callback() will group together callbacks in order
@@ -1458,8 +1434,8 @@
 	 * initialize the array of mutexes protecting a page's mapping
 	 * list and p_nrm field.
 	 */
-	for (i = 0; i < mml_table_sz; i++)
-		mutex_init(&mml_table[i], NULL, MUTEX_DEFAULT, NULL);
+	for (i = 0; i < MML_TABLE_SIZE; i++)
+		mutex_init(&mml_table[i].pad_mutex, NULL, MUTEX_DEFAULT, NULL);
 
 	if (kpm_enable) {
 		for (i = 0; i < kpmp_table_sz; i++) {
--- a/usr/src/uts/sfmmu/vm/hat_sfmmu.h	Mon Jul 26 07:07:43 2010 -0700
+++ b/usr/src/uts/sfmmu/vm/hat_sfmmu.h	Mon Jul 26 07:24:36 2010 -0700
@@ -19,8 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
- * Use is subject to license terms.
+ * Copyright (c) 1987, 2010, Oracle and/or its affiliates. All rights reserved.
  */
 
 /*
@@ -2364,9 +2363,6 @@
 extern int		khmehash_num;
 extern struct hmehash_bucket *uhme_hash;
 extern struct hmehash_bucket *khme_hash;
-extern kmutex_t		*mml_table;
-extern uint_t		mml_table_sz;
-extern uint_t		mml_shift;
 extern uint_t		hblk_alloc_dynamic;
 extern struct tsbmiss	tsbmiss_area[NCPU];
 extern struct kpmtsbm	kpmtsbm_area[NCPU];
--- a/usr/src/uts/sun4/cpu/cpu_module.c	Mon Jul 26 07:07:43 2010 -0700
+++ b/usr/src/uts/sun4/cpu/cpu_module.c	Mon Jul 26 07:24:36 2010 -0700
@@ -19,8 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
- * Use is subject to license terms.
+ * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
  */
 
 #include <sys/cpu_module.h>
@@ -140,6 +139,10 @@
 gettick(void)
 { return (0); }
 
+u_longlong_t
+randtick(void)
+{ return (0); }
+
 uint64_t
 gettick_counter(void)
 { return (0); }
--- a/usr/src/uts/sun4/os/startup.c	Mon Jul 26 07:07:43 2010 -0700
+++ b/usr/src/uts/sun4/os/startup.c	Mon Jul 26 07:24:36 2010 -0700
@@ -1063,7 +1063,7 @@
 		cmn_err(CE_PANIC,
 		    "no more nucleus memory after page free lists alloc");
 
-	if (ndata_alloc_hat(&ndata, npages) != 0)
+	if (ndata_alloc_hat(&ndata) != 0)
 		cmn_err(CE_PANIC, "no more nucleus memory after hat alloc");
 
 	if (ndata_alloc_memseg(&ndata, boot_physavail_len) != 0)
--- a/usr/src/uts/sun4/vm/sfmmu.c	Mon Jul 26 07:07:43 2010 -0700
+++ b/usr/src/uts/sun4/vm/sfmmu.c	Mon Jul 26 07:24:36 2010 -0700
@@ -20,8 +20,7 @@
  */
 
 /*
- * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
- * Use is subject to license terms.
+ * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  */
 
 #include <sys/types.h>
@@ -907,36 +906,10 @@
  * Allocate hat structs from the nucleus data memory.
  */
 int
-ndata_alloc_hat(struct memlist *ndata, pgcnt_t npages)
+ndata_alloc_hat(struct memlist *ndata)
 {
-	size_t	mml_alloc_sz;
 	size_t	cb_alloc_sz;
 
-	/*
-	 * For the page mapping list mutex array we allocate one mutex
-	 * for every 128 pages (1 MB) with a minimum of 64 entries and
-	 * a maximum of 8K entries. For the initial computation npages
-	 * is rounded up (ie. 1 << highbit(npages * 1.5 / 128))
-	 *
-	 * mml_shift is roughly log2(mml_table_sz) + 3 for MLIST_HASH
-	 */
-	mml_table_sz = 1 << highbit((npages * 3) / 256);
-	if (mml_table_sz < 64)
-		mml_table_sz = 64;
-	else if (mml_table_sz > 8192)
-		mml_table_sz = 8192;
-	mml_shift = highbit(mml_table_sz) + 3;
-
-	PRM_DEBUG(mml_table_sz);
-	PRM_DEBUG(mml_shift);
-
-	mml_alloc_sz = mml_table_sz * sizeof (kmutex_t);
-
-	mml_table = ndata_alloc(ndata, mml_alloc_sz, ecache_alignsize);
-	if (mml_table == NULL)
-		return (-1);
-	PRM_DEBUG(mml_table);
-
 	cb_alloc_sz = sfmmu_max_cb_id * sizeof (struct sfmmu_callback);
 	PRM_DEBUG(cb_alloc_sz);
 	sfmmu_cb_table = ndata_alloc(ndata, cb_alloc_sz, ecache_alignsize);
--- a/usr/src/uts/sun4/vm/vm_dep.h	Mon Jul 26 07:07:43 2010 -0700
+++ b/usr/src/uts/sun4/vm/vm_dep.h	Mon Jul 26 07:24:36 2010 -0700
@@ -19,8 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
- * Use is subject to license terms.
+ * Copyright (c) 1995, 2010, Oracle and/or its affiliates. All rights reserved.
  */
 
 /*
@@ -40,8 +39,8 @@
 
 #define	GETTICK()	gettick()
 
-/* #define for keeping code architecturally neutral */
-#define	randtick()	gettick()
+/* tick value that should be used for random values */
+extern u_longlong_t randtick(void);
 
 /*
  * Per page size free lists. Allocated dynamically.
--- a/usr/src/uts/sun4u/cpu/common_asm.s	Mon Jul 26 07:07:43 2010 -0700
+++ b/usr/src/uts/sun4u/cpu/common_asm.s	Mon Jul 26 07:24:36 2010 -0700
@@ -19,8 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
- * Use is subject to license terms.
+ * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved.
  */
 
 #if !defined(lint)
@@ -381,12 +380,18 @@
 gettick(void)
 { return (0); }
 
+u_longlong_t
+randtick(void)
+{ return (0); }
+
 #else	/* lint */
 
 	ENTRY(gettick)
+	ALTENTRY(randtick)
 	GET_NATIVE_TIME(%o0, %o2, %o3)
 	retl
 	nop
+	SET_SIZE(randtick)
 	SET_SIZE(gettick)
 
 #endif	/* lint */
--- a/usr/src/uts/sun4u/sys/machsystm.h	Mon Jul 26 07:07:43 2010 -0700
+++ b/usr/src/uts/sun4u/sys/machsystm.h	Mon Jul 26 07:24:36 2010 -0700
@@ -19,8 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
- * Use is subject to license terms.
+ * Copyright (c) 1993, 2010, Oracle and/or its affiliates. All rights reserved.
  */
 
 #ifndef _SYS_MACHSYSTM_H
@@ -274,7 +273,7 @@
 extern int ndata_alloc_page_freelists(struct memlist *, int);
 extern int ndata_alloc_dmv(struct memlist *);
 extern int ndata_alloc_tsbs(struct memlist *, pgcnt_t);
-extern int ndata_alloc_hat(struct memlist *, pgcnt_t);
+extern int ndata_alloc_hat(struct memlist *);
 extern int ndata_alloc_kpm(struct memlist *, pgcnt_t);
 extern int ndata_alloc_page_mutexs(struct memlist *ndata);
 
--- a/usr/src/uts/sun4v/cpu/common_asm.s	Mon Jul 26 07:07:43 2010 -0700
+++ b/usr/src/uts/sun4v/cpu/common_asm.s	Mon Jul 26 07:24:36 2010 -0700
@@ -19,8 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
- * Use is subject to license terms.
+ * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  */
 
 #if !defined(lint)
@@ -191,12 +190,18 @@
 gettick(void)
 { return (0); }
 
+u_longlong_t
+randtick(void)
+{ return (0); }
+
 #else   /* lint */
 
 	ENTRY(gettick)
+	ALTENTRY(randtick)
 	GET_NATIVE_TIME(%o0,%o2,%o3,__LINE__)
 	retl
 	  nop
+	SET_SIZE(randtick)
 	SET_SIZE(gettick)
 
 #endif  /* lint */
--- a/usr/src/uts/sun4v/sys/machsystm.h	Mon Jul 26 07:07:43 2010 -0700
+++ b/usr/src/uts/sun4v/sys/machsystm.h	Mon Jul 26 07:24:36 2010 -0700
@@ -19,8 +19,7 @@
  * CDDL HEADER END
  */
 /*
- * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
- * Use is subject to license terms.
+ * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  */
 
 #ifndef _SYS_MACHSYSTM_H
@@ -275,7 +274,7 @@
 extern int ndata_alloc_page_freelists(struct memlist *, int);
 extern int ndata_alloc_dmv(struct memlist *);
 extern int ndata_alloc_tsbs(struct memlist *, pgcnt_t);
-extern int ndata_alloc_hat(struct memlist *, pgcnt_t);
+extern int ndata_alloc_hat(struct memlist *);
 extern int ndata_alloc_kpm(struct memlist *, pgcnt_t);
 extern int ndata_alloc_page_mutexs(struct memlist *ndata);