cache UVM_OBJ_IS_VNODE in pqflags yamt-pagecache
authoryamt <yamt@NetBSD.org>
Sun, 13 Nov 2011 01:18:02 +0000
branchyamt-pagecache
changeset 280312 136c67680137
parent 280311 8bf6560152c4
child 280313 5c2c51792488
cache UVM_OBJ_IS_VNODE in pqflags
sys/uvm/uvm_aobj.c
sys/uvm/uvm_page.c
sys/uvm/uvm_page.h
sys/uvm/uvm_page_status.c
--- a/sys/uvm/uvm_aobj.c	Sat Nov 12 04:12:52 2011 +0000
+++ b/sys/uvm/uvm_aobj.c	Sun Nov 13 01:18:02 2011 +0000
@@ -1,4 +1,4 @@
-/*	$NetBSD: uvm_aobj.c,v 1.116.2.3 2011/11/06 22:05:00 yamt Exp $	*/
+/*	$NetBSD: uvm_aobj.c,v 1.116.2.4 2011/11/13 01:18:02 yamt Exp $	*/
 
 /*
  * Copyright (c) 1998 Chuck Silvers, Charles D. Cranor and
@@ -38,7 +38,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_aobj.c,v 1.116.2.3 2011/11/06 22:05:00 yamt Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_aobj.c,v 1.116.2.4 2011/11/13 01:18:02 yamt Exp $");
 
 #include "opt_uvmhist.h"
 
@@ -909,7 +909,6 @@
 				    NULL, UVM_FLAG_COLORMATCH|UVM_PGA_ZERO);
 				if (ptmp) {
 					/* new page */
-					ptmp->pqflags |= PQ_AOBJ;
 					ptmp->flags &= ~PG_FAKE;
 					uvm_pagemarkdirty(ptmp,
 					    UVM_PAGE_STATUS_UNKNOWN);
@@ -1018,13 +1017,6 @@
 				}
 
 				/*
-				 * safe with PQ's unlocked: because we just
-				 * alloc'd the page
-				 */
-
-				ptmp->pqflags |= PQ_AOBJ;
-
-				/*
 				 * got new page ready for I/O.  break pps while
 				 * loop.  pps[lcv] is still NULL.
 				 */
--- a/sys/uvm/uvm_page.c	Sat Nov 12 04:12:52 2011 +0000
+++ b/sys/uvm/uvm_page.c	Sun Nov 13 01:18:02 2011 +0000
@@ -1,4 +1,4 @@
-/*	$NetBSD: uvm_page.c,v 1.178.2.4 2011/11/12 02:54:04 yamt Exp $	*/
+/*	$NetBSD: uvm_page.c,v 1.178.2.5 2011/11/13 01:18:02 yamt Exp $	*/
 
 /*
  * Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -66,7 +66,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_page.c,v 1.178.2.4 2011/11/12 02:54:04 yamt Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_page.c,v 1.178.2.5 2011/11/13 01:18:02 yamt Exp $");
 
 #include "opt_ddb.h"
 #include "opt_uvmhist.h"
@@ -169,8 +169,6 @@
 uvm_pageinsert_list(struct uvm_object *uobj, struct vm_page *pg,
     struct vm_page *where)
 {
-	const bool isvnode = UVM_OBJ_IS_VNODE(uobj);
-	const bool isaobj = UVM_OBJ_IS_AOBJ(uobj);
 
 	KASSERT(uobj == pg->uobject);
 	KASSERT(mutex_owned(uobj->vmobjlock));
@@ -178,15 +176,17 @@
 	KASSERT(where == NULL || (where->flags & PG_TABLED));
 	KASSERT(where == NULL || (where->uobject == uobj));
 
-	if (isvnode || isaobj) {
+	if ((pg->pqflags & PQ_STAT) != 0) {
 		struct uvm_cpu *ucpu;
 		const unsigned int status = uvm_pagegetdirty(pg);
+		const bool isaobj = (pg->pqflags & PQ_AOBJ) != 0;
 
 		kpreempt_disable();
 		ucpu = curcpu()->ci_data.cpu_uvm;
 		ucpu->pagestate[isaobj][status]++;
 		kpreempt_enable();
-		if (isvnode) {
+		if (!isaobj) {
+			KASSERT((pg->pqflags & PQ_FILE) != 0);
 			if (uobj->uo_npages == 0) {
 				struct vnode *vp = (struct vnode *)uobj;
 
@@ -248,22 +248,22 @@
 static inline void
 uvm_pageremove_list(struct uvm_object *uobj, struct vm_page *pg)
 {
-	const bool isvnode = UVM_OBJ_IS_VNODE(uobj);
-	const bool isaobj = UVM_OBJ_IS_AOBJ(uobj);
 
 	KASSERT(uobj == pg->uobject);
 	KASSERT(mutex_owned(uobj->vmobjlock));
 	KASSERT(pg->flags & PG_TABLED);
 
-	if (isvnode || isaobj) {
+	if ((pg->pqflags & PQ_STAT) != 0) {
 		struct uvm_cpu *ucpu;
 		const unsigned int status = uvm_pagegetdirty(pg);
+		const bool isaobj = (pg->pqflags & PQ_AOBJ) != 0;
 
 		kpreempt_disable();
 		ucpu = curcpu()->ci_data.cpu_uvm;
 		ucpu->pagestate[isaobj][status]--;
 		kpreempt_enable();
-		if (isvnode) {
+		if (!isaobj) {
+			KASSERT((pg->pqflags & PQ_FILE) != 0);
 			if (uobj->uo_npages == 1) {
 				struct vnode *vp = (struct vnode *)uobj;
 
@@ -1324,18 +1324,27 @@
 	 * otherwise we race with uvm_pglistalloc.
 	 */
 	pg->pqflags = 0;
-	if (anon) {
-		ucpu->pagestate[1][UVM_PAGE_STATUS_CLEAN]++;
-	}
 	mutex_spin_exit(&uvm_fpageqlock);
 	if (anon) {
 		anon->an_page = pg;
 		pg->pqflags = PQ_ANON;
 		atomic_inc_uint(&uvmexp.anonpages);
+		kpreempt_disable();
+		ucpu = curcpu()->ci_data.cpu_uvm;
+		ucpu->pagestate[1][UVM_PAGE_STATUS_CLEAN]++;
+		kpreempt_enable();
 	} else {
 		if (obj) {
 			int error;
 
+			/*
+			 * set PQ_FILE|PQ_AOBJ before the first uvm_pageinsert.
+			 */
+			if (UVM_OBJ_IS_VNODE(obj)) {
+				pg->pqflags |= PQ_FILE;
+			} else {
+				pg->pqflags |= PQ_AOBJ;
+			}
 			error = uvm_pageinsert(obj, pg);
 			if (error != 0) {
 #if 1
--- a/sys/uvm/uvm_page.h	Sat Nov 12 04:12:52 2011 +0000
+++ b/sys/uvm/uvm_page.h	Sun Nov 13 01:18:02 2011 +0000
@@ -1,4 +1,4 @@
-/*	$NetBSD: uvm_page.h,v 1.73.2.3 2011/11/11 10:34:24 yamt Exp $	*/
+/*	$NetBSD: uvm_page.h,v 1.73.2.4 2011/11/13 01:18:02 yamt Exp $	*/
 
 /*
  * Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -194,6 +194,8 @@
 					   uvm_object */
 #define PQ_SWAPBACKED	(PQ_ANON|PQ_AOBJ)
 #define PQ_READAHEAD	0x0008	/* read-ahead but has not been "hit" yet */
+#define	PQ_FILE		0x0010		/* file backed (non-anonymous) */
+#define	PQ_STAT		(PQ_ANON|PQ_AOBJ|PQ_FILE)
 
 #define PQ_PRIVATE1	0x0100
 #define PQ_PRIVATE2	0x0200
@@ -205,7 +207,7 @@
 #define PQ_PRIVATE8	0x8000
 
 #define	UVM_PQFLAGBITS \
-	"\20\1FREE\2ANON\3AOBJ\4READAHEAD" \
+	"\20\1FREE\2ANON\3AOBJ\4READAHEAD\5FILE" \
 	"\11PRIVATE1\12PRIVATE2\13PRIVATE3\14PRIVATE4" \
 	"\15PRIVATE5\16PRIVATE6\17PRIVATE7\20PRIVATE8"
 
--- a/sys/uvm/uvm_page_status.c	Sat Nov 12 04:12:52 2011 +0000
+++ b/sys/uvm/uvm_page_status.c	Sun Nov 13 01:18:02 2011 +0000
@@ -1,4 +1,4 @@
-/*	$NetBSD: uvm_page_status.c,v 1.1.2.3 2011/11/12 02:54:04 yamt Exp $	*/
+/*	$NetBSD: uvm_page_status.c,v 1.1.2.4 2011/11/13 01:18:02 yamt Exp $	*/
 
 /*-
  * Copyright (c)2011 YAMAMOTO Takashi,
@@ -27,7 +27,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: uvm_page_status.c,v 1.1.2.3 2011/11/12 02:54:04 yamt Exp $");
+__KERNEL_RCSID(0, "$NetBSD: uvm_page_status.c,v 1.1.2.4 2011/11/13 01:18:02 yamt Exp $");
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -125,15 +125,9 @@
 	pg->flags |= newstatus;
 	KASSERT(uobj == NULL || ((pg->flags & PG_CLEAN) == 0) ==
 	    radix_tree_get_tag(&uobj->uo_pages, idx, UVM_PAGE_DIRTY_TAG));
-	if (uobj != NULL) {
-		const bool isvnode = UVM_OBJ_IS_VNODE(uobj);
-		const bool isaobj = UVM_OBJ_IS_AOBJ(uobj);
-
-		if (isvnode || isaobj) {
-			stat_update(isaobj, oldstatus, newstatus);
-		}
-	} else {
-		stat_update(true, oldstatus, newstatus);
+	if ((pg->pqflags & PQ_STAT) != 0) {
+		stat_update((pg->pqflags & PQ_SWAPBACKED) != 0,
+		    oldstatus, newstatus);
 	}
 }