2 files changed, 39 insertions, 55 deletions

diff --git a/sys/powerpc/aim/mmu_oea.c b/sys/powerpc/aim/mmu_oea.c
index 3880f51f19c5..4cd5f75cf293 100644
--- a/sys/powerpc/aim/mmu_oea.c
+++ b/sys/powerpc/aim/mmu_oea.c
@@ -1729,6 +1729,10 @@ moea_remove_all(mmu_t mmu, vm_page_t m)
 		moea_pvo_remove(pvo, -1);
 		PMAP_UNLOCK(pmap);
 	}
+	if ((m->flags & PG_WRITEABLE) && moea_is_modified(mmu, m)) {
+		moea_attr_clear(m, LPTE_CHG);
+		vm_page_dirty(m);
+	}
 	vm_page_flag_clear(m, PG_WRITEABLE);
 }
 
@@ -2203,10 +2207,8 @@ moea_query_bit(vm_page_t m, int ptebit)
 	struct	pvo_entry *pvo;
 	struct	pte *pt;
 
-#if 0
 	if (moea_attr_fetch(m) & ptebit)
 		return (TRUE);
-#endif
 
 	LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) {
 		MOEA_PVO_CHECK(pvo);	/* sanity check */
diff --git a/sys/powerpc/aim/mmu_oea64.c b/sys/powerpc/aim/mmu_oea64.c
index 834ba3bd8aba..db4127594d11 100644
--- a/sys/powerpc/aim/mmu_oea64.c
+++ b/sys/powerpc/aim/mmu_oea64.c
@@ -172,6 +172,7 @@ va_to_vsid(pmap_t pm, vm_offset_t va)
 	return ((pm->pm_sr[(uintptr_t)va >> ADDR_SR_SHFT]) & SR_VSID_MASK);
 }
 
+#define	PTESYNC()	__asm __volatile("ptesync");
 #define	TLBSYNC()	__asm __volatile("tlbsync; ptesync");
 #define	SYNC()		__asm __volatile("sync");
 #define	EIEIO()		__asm __volatile("eieio");
@@ -194,6 +195,7 @@ TLBIE(pmap_t pmap, vm_offset_t va) {
 	vpn = (uint64_t)(va & ADDR_PIDX);
 	if (pmap != NULL)
 		vpn |= (va_to_vsid(pmap,va) << 28);
+	vpn &= ~(0xffffULL << 48);
 
 	vpn_hi = (uint32_t)(vpn >> 32);
 	vpn_lo = (uint32_t)vpn;
@@ -201,8 +203,7 @@ TLBIE(pmap_t pmap, vm_offset_t va) {
 	mtx_lock_spin(&tlbie_mutex);
 	__asm __volatile("\
 	    mfmsr %0; \
-	    clrldi %1,%0,49; \
-	    mtmsr %1; \
+	    mr %1, %0; \
 	    insrdi %1,%5,1,0; \
 	    mtmsrd %1; \
 	    ptesync; \
@@ -215,7 +216,8 @@ TLBIE(pmap_t pmap, vm_offset_t va) {
 	    eieio; \
 	    tlbsync; \
 	    ptesync;" 
-	: "=r"(msr), "=r"(scratch) : "r"(vpn_hi), "r"(vpn_lo), "r"(32), "r"(1));
+	: "=r"(msr), "=r"(scratch) : "r"(vpn_hi), "r"(vpn_lo), "r"(32), "r"(1)
+	    : "memory");
 	mtx_unlock_spin(&tlbie_mutex);
 }
 
@@ -226,13 +228,13 @@ TLBIE(pmap_t pmap, vm_offset_t va) {
 #define	VSID_TO_SR(vsid)	((vsid) & 0xf)
 #define	VSID_TO_HASH(vsid)	(((vsid) >> 4) & 0xfffff)
 
-#define	PVO_PTEGIDX_MASK	0x007		/* which PTEG slot */
-#define	PVO_PTEGIDX_VALID	0x008		/* slot is valid */
-#define	PVO_WIRED		0x010		/* PVO entry is wired */
-#define	PVO_MANAGED		0x020		/* PVO entry is managed */
-#define	PVO_BOOTSTRAP		0x080		/* PVO entry allocated during
+#define	PVO_PTEGIDX_MASK	0x007UL		/* which PTEG slot */
+#define	PVO_PTEGIDX_VALID	0x008UL		/* slot is valid */
+#define	PVO_WIRED		0x010UL		/* PVO entry is wired */
+#define	PVO_MANAGED		0x020UL		/* PVO entry is managed */
+#define	PVO_BOOTSTRAP		0x080UL		/* PVO entry allocated during
 						   bootstrap */
-#define PVO_FAKE		0x100		/* fictitious phys page */
+#define PVO_FAKE		0x100UL		/* fictitious phys page */
 #define	PVO_VADDR(pvo)		((pvo)->pvo_vaddr & ~ADDR_POFF)
 #define PVO_ISFAKE(pvo)		((pvo)->pvo_vaddr & PVO_FAKE)
 #define	PVO_PTEGIDX_GET(pvo)	((pvo)->pvo_vaddr & PVO_PTEGIDX_MASK)
@@ -512,23 +514,6 @@ moea64_attr_save(vm_page_t m, u_int64_t ptebit)
 	m->md.mdpg_attrs |= ptebit;
 }
 
-static __inline int
-moea64_pte_compare(const struct lpte *pt, const struct lpte *pvo_pt)
-{
-	if (pt->pte_hi == pvo_pt->pte_hi)
-		return (1);
-
-	return (0);
-}
-
-static __inline int
-moea64_pte_match(struct lpte *pt, uint64_t vsid, vm_offset_t va, int which)
-{
-	return (pt->pte_hi & ~LPTE_VALID) ==
-	    ((vsid << LPTE_VSID_SHIFT) |
-	    ((uint64_t)(va >> ADDR_API_SHFT64) & LPTE_API) | which);
-}
-
 static __inline void
 moea64_pte_create(struct lpte *pt, uint64_t vsid, vm_offset_t va, 
     uint64_t pte_lo)
@@ -583,7 +568,7 @@ moea64_pte_set(struct lpte *pt, struct lpte *pvo_pt)
 	pt->pte_lo = pvo_pt->pte_lo;
 	EIEIO();
 	pt->pte_hi = pvo_pt->pte_hi;
-	SYNC();
+	PTESYNC();
 	moea64_pte_valid++;
 }
 
@@ -602,7 +587,6 @@ moea64_pte_unset(struct lpte *pt, struct lpte *pvo_pt, pmap_t pmap, vm_offset_t
 	 * Invalidate the pte.
 	 */
 	pt->pte_hi &= ~LPTE_VALID;
-
 	TLBIE(pmap,va);
 
 	/*
@@ -621,6 +605,8 @@ moea64_pte_change(struct lpte *pt, struct lpte *pvo_pt, pmap_t pmap, vm_offset_t
 	 */
 	moea64_pte_unset(pt, pvo_pt, pmap, va);
 	moea64_pte_set(pt, pvo_pt);
+	if (pmap == kernel_pmap)
+		isync();
 }
 
 static __inline uint64_t
@@ -701,7 +687,7 @@ moea64_bridge_cpu_bootstrap(mmu_t mmup, int ap)
 	for (i = 0; i < 16; i++) {
 		mtsrin(i << ADDR_SR_SHFT, kernel_pmap->pm_sr[i]);
 	}
-	__asm __volatile ("sync; mtsdr1 %0; isync"
+	__asm __volatile ("ptesync; mtsdr1 %0; isync"
 	    :: "r"((u_int)moea64_pteg_table 
 		     | (32 - cntlzw(moea64_pteg_mask >> 11))));
 	tlbia();
@@ -1135,7 +1121,7 @@ void moea64_set_scratchpage_pa(int which, vm_offset_t pa) {
 	EIEIO();
 
 	moea64_scratchpage_pte[which]->pte_hi |= LPTE_VALID;
-	TLBIE(kernel_pmap, moea64_scratchpage_va[which]);
+	PTESYNC(); isync();
 }
 
 void
@@ -1155,8 +1141,6 @@ moea64_copy_page(mmu_t mmu, vm_page_t msrc, vm_page_t mdst)
 	kcopy((void *)moea64_scratchpage_va[0], 
 	    (void *)moea64_scratchpage_va[1], PAGE_SIZE);
 
-	__syncicache((void *)moea64_scratchpage_va[1],PAGE_SIZE);
-
 	mtx_unlock(&moea64_scratchpage_mtx);
 }
 
@@ -1174,8 +1158,6 @@ moea64_zero_page_area(mmu_t mmu, vm_page_t m, int off, int size)
 
 	moea64_set_scratchpage_pa(0,pa);
 	bzero((caddr_t)moea64_scratchpage_va[0] + off, size);
-	__syncicache((void *)moea64_scratchpage_va[0],PAGE_SIZE);
-
 	mtx_unlock(&moea64_scratchpage_mtx);
 }
 
@@ -1266,9 +1248,6 @@ moea64_enter_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
 	error = moea64_pvo_enter(pmap, zone, pvo_head, va, VM_PAGE_TO_PHYS(m),
 	    pte_lo, pvo_flags);
 
-	if (pmap == kernel_pmap)
-		TLBIE(pmap, va);
-
 	/*
 	 * Flush the page from the instruction cache if this page is
 	 * mapped executable and cacheable.
@@ -1281,6 +1260,7 @@ moea64_enter_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
 static void
 moea64_syncicache(pmap_t pmap, vm_offset_t va, vm_offset_t pa, vm_size_t sz)
 {
+
 	/*
 	 * This is much trickier than on older systems because
 	 * we can't sync the icache on physical addresses directly
@@ -1303,8 +1283,9 @@ moea64_syncicache(pmap_t pmap, vm_offset_t va, vm_offset_t pa, vm_size_t sz)
 
 		mtx_lock(&moea64_scratchpage_mtx);
 
-		moea64_set_scratchpage_pa(1,pa);
-		__syncicache((void *)moea64_scratchpage_va[1], sz);
+		moea64_set_scratchpage_pa(1,pa & ~ADDR_POFF);
+		__syncicache((void *)(moea64_scratchpage_va[1] + 
+		    (va & ADDR_POFF)), sz);
 
 		mtx_unlock(&moea64_scratchpage_mtx);
 	}
@@ -1441,8 +1422,6 @@ moea64_uma_page_alloc(uma_zone_t zone, int bytes, u_int8_t *flags, int wait)
 	    &moea64_pvo_kunmanaged, va,  VM_PAGE_TO_PHYS(m), LPTE_M, 
 	    PVO_WIRED | PVO_BOOTSTRAP);
 
-	TLBIE(kernel_pmap, va);
-
 	if (needed_lock)
 		PMAP_UNLOCK(kernel_pmap);
 	
@@ -1531,7 +1510,7 @@ moea64_remove_write(mmu_t mmu, vm_page_t m)
 				lo |= pvo->pvo_pte.lpte.pte_lo;
 				pvo->pvo_pte.lpte.pte_lo &= ~LPTE_CHG;
 				moea64_pte_change(pt, &pvo->pvo_pte.lpte,
-				    pvo->pvo_pmap, pvo->pvo_vaddr);
+				    pvo->pvo_pmap, PVO_VADDR(pvo));
 			}
 			UNLOCK_TABLE();
 		}
@@ -1590,8 +1569,6 @@ moea64_kenter(mmu_t mmu, vm_offset_t va, vm_offset_t pa)
 	    &moea64_pvo_kunmanaged, va, pa, pte_lo, 
 	    PVO_WIRED | VM_PROT_EXECUTE);
 
-	TLBIE(kernel_pmap, va);
-
 	if (error != 0 && error != ENOENT)
 		panic("moea64_kenter: failed to enter va %#x pa %#x: %d", va,
 		    pa, error);
@@ -1823,7 +1800,7 @@ moea64_protect(mmu_t mmu, pmap_t pm, vm_offset_t sva, vm_offset_t eva,
 		 */
 		if (pt != NULL) {
 			moea64_pte_change(pt, &pvo->pvo_pte.lpte, 
-			    pvo->pvo_pmap, pvo->pvo_vaddr);
+			    pvo->pvo_pmap, PVO_VADDR(pvo));
 			if ((pvo->pvo_pte.lpte.pte_lo & 
 			    (LPTE_I | LPTE_G | LPTE_NOEXEC)) == 0) {
 				moea64_syncicache(pm, sva,
@@ -1927,6 +1904,10 @@ moea64_remove_all(mmu_t mmu, vm_page_t m)
 		moea64_pvo_remove(pvo, -1);
 		PMAP_UNLOCK(pmap);
 	}
+	if ((m->flags & PG_WRITEABLE) && moea64_is_modified(mmu, m)) {
+		moea64_attr_clear(m, LPTE_CHG);
+		vm_page_dirty(m);
+	}
 	vm_page_flag_clear(m, PG_WRITEABLE);
 }
 
@@ -2107,7 +2088,7 @@ moea64_pvo_enter(pmap_t pm, uma_zone_t zone, struct pvo_head *pvo_head,
 		first = 1;
 	LIST_INSERT_HEAD(pvo_head, pvo, pvo_vlink);
 
-	if (pvo->pvo_pte.lpte.pte_lo & PVO_WIRED)
+	if (pvo->pvo_vaddr & PVO_WIRED)
 		pm->pm_stats.wired_count++;
 	pm->pm_stats.resident_count++;
 
@@ -2122,6 +2103,9 @@ moea64_pvo_enter(pmap_t pm, uma_zone_t zone, struct pvo_head *pvo_head,
 		moea64_pte_overflow++;
 	}
 
+	if (pm == kernel_pmap)
+		isync();
+
 	UNLOCK_TABLE();
 
 	return (first ? ENOENT : 0);
@@ -2140,7 +2124,7 @@ moea64_pvo_remove(struct pvo_entry *pvo, int pteidx)
 	pt = moea64_pvo_to_pte(pvo, pteidx);
 	if (pt != NULL) {
 		moea64_pte_unset(pt, &pvo->pvo_pte.lpte, pvo->pvo_pmap,
-		    pvo->pvo_vaddr);
+		    PVO_VADDR(pvo));
 		PVO_PTEGIDX_CLR(pvo);
 	} else {
 		moea64_pte_overflow--;
@@ -2151,7 +2135,7 @@ moea64_pvo_remove(struct pvo_entry *pvo, int pteidx)
 	 * Update our statistics.
 	 */
 	pvo->pvo_pmap->pm_stats.resident_count--;
-	if (pvo->pvo_pte.lpte.pte_lo & PVO_WIRED)
+	if (pvo->pvo_vaddr & PVO_WIRED)
 		pvo->pvo_pmap->pm_stats.wired_count--;
 
 	/*
@@ -2178,7 +2162,7 @@ moea64_pvo_remove(struct pvo_entry *pvo, int pteidx)
 	 */
 	LIST_REMOVE(pvo, pvo_olink);
 	if (!(pvo->pvo_vaddr & PVO_BOOTSTRAP))
-		uma_zfree(pvo->pvo_vaddr & PVO_MANAGED ? moea64_mpvo_zone :
+		uma_zfree((pvo->pvo_vaddr & PVO_MANAGED) ? moea64_mpvo_zone :
 		    moea64_upvo_zone, pvo);
 	moea64_pvo_entries--;
 	moea64_pvo_remove_calls++;
@@ -2237,8 +2221,8 @@ moea64_pvo_to_pte(const struct pvo_entry *pvo, int pteidx)
 		int		ptegidx;
 		uint64_t	vsid;
 
-		vsid = va_to_vsid(pvo->pvo_pmap, pvo->pvo_vaddr);
-		ptegidx = va_to_pteg(vsid, pvo->pvo_vaddr);
+		vsid = va_to_vsid(pvo->pvo_pmap, PVO_VADDR(pvo));
+		ptegidx = va_to_pteg(vsid, PVO_VADDR(pvo));
 		pteidx = moea64_pvo_pte_index(pvo, ptegidx);
 	}
 
@@ -2325,10 +2309,8 @@ moea64_query_bit(vm_page_t m, u_int64_t ptebit)
 	struct	pvo_entry *pvo;
 	struct	lpte *pt;
 
-#if 0
 	if (moea64_attr_fetch(m) & ptebit)
 		return (TRUE);
-#endif
 
 	LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) {
 		MOEA_PVO_CHECK(pvo);	/* sanity check */