- Make __cpu_simple_lock and similar real functions and patch at runtime. trunk
authorad <ad@NetBSD.org>
Thu, 20 Dec 2007 23:46:10 +0000
branchtrunk
changeset 164838 49d10ecc8f3e
parent 164837 1de695484144
child 164839 875886e4bd08
- Make __cpu_simple_lock and similar real functions and patch at runtime. - Remove old x86 atomic ops. - Drop text alignment back to 16 on i386 (really, this time). - Minor cleanup.
common/lib/libc/arch/i386/atomic/atomic.S
distrib/sets/lists/comp/md.amd64
distrib/sets/lists/comp/md.i386
sys/arch/amd64/amd64/cpufunc.S
sys/arch/amd64/amd64/ipifuncs.c
sys/arch/amd64/amd64/lock_stubs.S
sys/arch/amd64/include/Makefile
sys/arch/amd64/include/asm.h
sys/arch/amd64/include/atomic.h
sys/arch/amd64/include/profile.h
sys/arch/i386/i386/lock_stubs.S
sys/arch/i386/i386/spl.S
sys/arch/i386/include/Makefile
sys/arch/i386/include/asm.h
sys/arch/i386/include/atomic.h
sys/arch/i386/include/profile.h
sys/arch/x86/include/lock.h
sys/arch/x86/x86/bus_space.c
sys/arch/x86/x86/patch.c
sys/arch/x86/x86/pmap.c
sys/arch/xen/x86/hypervisor_machdep.c
sys/arch/xen/x86/intr.c
--- a/common/lib/libc/arch/i386/atomic/atomic.S	Thu Dec 20 23:32:24 2007 +0000
+++ b/common/lib/libc/arch/i386/atomic/atomic.S	Thu Dec 20 23:46:10 2007 +0000
@@ -1,4 +1,4 @@
-/*	$NetBSD: atomic.S,v 1.6 2007/12/20 01:27:24 ad Exp $	*/
+/*	$NetBSD: atomic.S,v 1.7 2007/12/20 23:46:10 ad Exp $	*/
 
 /*-
  * Copyright (c) 2007 The NetBSD Foundation, Inc.
@@ -217,7 +217,7 @@
 	popl	%ebx
 	popl	%edi
 	ret
-	.space	16, 0x90
+	.space	32, 0x90
 END(_atomic_cas_cx8_end)
 
 NENTRY(sse2_lfence)
@@ -235,6 +235,20 @@
 	.long	.Lpatch1, .Lpatch2, .Lpatch3, .Lpatch4, .Lpatch5
 	.long	.Lpatch6, .Lpatch7, .Lpatch8, .Lpatch9, .Lpatch10
 	.long	.Lpatch12, .Lpatch13, .Lpatch14, .Lpatch15, 0
+#else
+NENTRY(_atomic_cas_64)
+	pushl	%edi
+	pushl	%ebx
+	movl	12(%esp), %edi
+	movl	16(%esp), %eax
+	movl	20(%esp), %edx
+	movl	24(%esp), %ebx
+	movl	28(%esp), %ecx
+	lock
+	cmpxchg8b (%edi)
+	popl	%ebx
+	popl	%edi
+	ret
 #endif	/* _KERNEL */
 
 ALIAS(atomic_add_32,_atomic_add_32)
--- a/distrib/sets/lists/comp/md.amd64	Thu Dec 20 23:32:24 2007 +0000
+++ b/distrib/sets/lists/comp/md.amd64	Thu Dec 20 23:46:10 2007 +0000
@@ -1,9 +1,9 @@
-# $NetBSD: md.amd64,v 1.27 2007/12/02 18:10:20 bouyer Exp $
+# $NetBSD: md.amd64,v 1.28 2007/12/20 23:46:10 ad Exp $
 ./usr/include/amd64				comp-c-include
 ./usr/include/amd64/ansi.h			comp-c-include
 ./usr/include/amd64/aout_machdep.h		comp-c-include
 ./usr/include/amd64/asm.h			comp-c-include
-./usr/include/amd64/atomic.h			comp-c-include
+./usr/include/amd64/atomic.h			comp-obsolete		obsolete
 ./usr/include/amd64/autoconf.h			comp-obsolete		obsolete
 ./usr/include/amd64/bootinfo.h			comp-c-include
 ./usr/include/amd64/bswap.h			comp-c-include
--- a/distrib/sets/lists/comp/md.i386	Thu Dec 20 23:32:24 2007 +0000
+++ b/distrib/sets/lists/comp/md.i386	Thu Dec 20 23:46:10 2007 +0000
@@ -1,4 +1,4 @@
-# $NetBSD: md.i386,v 1.101 2007/11/14 17:59:44 ad Exp $
+# $NetBSD: md.i386,v 1.102 2007/12/20 23:46:10 ad Exp $
 ./usr/include/emmintrin.h			comp-c-include		gcccmds
 ./usr/include/i386				comp-c-include
 ./usr/include/i386/_G_config.h			comp-obsolete		obsolete
@@ -6,7 +6,7 @@
 ./usr/include/i386/aout_machdep.h		comp-c-include
 ./usr/include/i386/apmvar.h			comp-c-include
 ./usr/include/i386/asm.h			comp-c-include
-./usr/include/i386/atomic.h			comp-c-include
+./usr/include/i386/atomic.h			comp-obsolete		obsolete
 ./usr/include/i386/autoconf.h			comp-obsolete		obsolete
 ./usr/include/i386/bioscall.h			comp-c-include
 ./usr/include/i386/bootinfo.h			comp-c-include
--- a/sys/arch/amd64/amd64/cpufunc.S	Thu Dec 20 23:32:24 2007 +0000
+++ b/sys/arch/amd64/amd64/cpufunc.S	Thu Dec 20 23:46:10 2007 +0000
@@ -1,4 +1,4 @@
-/*	$NetBSD: cpufunc.S,v 1.4 2007/12/06 14:40:28 ad Exp $	*/
+/*	$NetBSD: cpufunc.S,v 1.5 2007/12/20 23:46:10 ad Exp $	*/
 
 /*-
  * Copyright (c) 1998, 2007 The NetBSD Foundation, Inc.
@@ -238,33 +238,6 @@
 	int	$0x03		/* paranoid, not 'int3' */
 	ret
 
-NENTRY(x86_atomic_testset_ul)
-	movq	%rsi, %rax
-	xchgq	%rax, (%rdi)
-	ret
-
-NENTRY(x86_atomic_testset_i)
-	movl	%esi, %eax
-	xchgl	%eax, (%rdi)
-	ret
-
-NENTRY(x86_atomic_testset_b)
-	movl	%esi, %eax
-	xchgb	%al, (%rdi)
-	andl	$0xff, %eax
-	ret
-
-NENTRY(x86_atomic_setbits_l)
-	lock
-	orq	%rsi, (%rdi)
-	ret
-
-NENTRY(x86_atomic_clearbits_l)
-	notq	%rsi
-	lock
-	andq	%rsi, (%rdi)
-	ret
-
 NENTRY(x86_curcpu)
 	movq	%gs:(CPU_INFO_SELF), %rax
 	ret
--- a/sys/arch/amd64/amd64/ipifuncs.c	Thu Dec 20 23:32:24 2007 +0000
+++ b/sys/arch/amd64/amd64/ipifuncs.c	Thu Dec 20 23:46:10 2007 +0000
@@ -1,4 +1,4 @@
-/*	$NetBSD: ipifuncs.c,v 1.13 2007/12/18 07:17:10 joerg Exp $ */
+/*	$NetBSD: ipifuncs.c,v 1.14 2007/12/20 23:46:10 ad Exp $ */
 
 /*-
  * Copyright (c) 2000 The NetBSD Foundation, Inc.
@@ -40,7 +40,7 @@
 
 
 #include <sys/cdefs.h>			/* RCS ID & Copyright macro defns */
-__KERNEL_RCSID(0, "$NetBSD: ipifuncs.c,v 1.13 2007/12/18 07:17:10 joerg Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ipifuncs.c,v 1.14 2007/12/20 23:46:10 ad Exp $");
 
 /*
  * Interprocessor interrupt handlers.
@@ -56,7 +56,6 @@
 #include <uvm/uvm_extern.h>
 
 #include <machine/intr.h>
-#include <machine/atomic.h>
 #include <machine/cpuvar.h>
 #include <machine/i82093var.h>
 #include <machine/i82489reg.h>
--- a/sys/arch/amd64/amd64/lock_stubs.S	Thu Dec 20 23:32:24 2007 +0000
+++ b/sys/arch/amd64/amd64/lock_stubs.S	Thu Dec 20 23:46:10 2007 +0000
@@ -1,4 +1,4 @@
-/*	$NetBSD: lock_stubs.S,v 1.10 2007/11/28 18:19:21 ad Exp $	*/
+/*	$NetBSD: lock_stubs.S,v 1.11 2007/12/20 23:46:10 ad Exp $	*/
 
 /*-
  * Copyright (c) 2006, 2007 The NetBSD Foundation, Inc.
@@ -58,8 +58,7 @@
 #endif
 
 #define	END(name,a)	.align	a; LABEL(name)
-
-#define	LOCK(name)	LABEL(name) lock
+#define	LOCK(num)	.Lpatch/**/num: lock
 
 #ifndef LOCKDEBUG
 
@@ -70,10 +69,10 @@
  */
 	.align	64
 
-NENTRY(mutex_enter)				/* 0x0000, 25 bytes */
+NENTRY(mutex_enter)
 	movq	CPUVAR(CURLWP), %rcx
 	xorq	%rax, %rax
-	LOCK(lockpatch1)
+	LOCK(1)
 	cmpxchgq %rcx, MTX_OWNER(%rdi)
 	jnz,pn	_C_LABEL(mutex_vector_enter)
 	ret
@@ -87,7 +86,7 @@
  * on multiprocessor systems, and comments in arch/x86/include/lock.h about
  * memory ordering on Intel x86 systems.
  */
-NENTRY(mutex_exit)				/* 0x0020, 24 bytes */
+NENTRY(mutex_exit)
 	movq	CPUVAR(CURLWP), %rax
 	xorq	%rdx, %rdx
 	cmpxchgq %rdx, MTX_OWNER(%rdi)
@@ -113,7 +112,7 @@
 	cmovgl	%esi, %ecx
 	movl	%ecx, CPU_INFO_ILEVEL(%r8)	/* splraiseipl() */
 #if defined(FULL)
-	LOCK(lockpatch11)
+	LOCK(11)
 	cmpxchgb %ah, MTX_LOCK(%rdi)		/* lock */
 	jnz,pn	_C_LABEL(mutex_spin_retry)	/* failed; hard case */
 #endif
@@ -180,7 +179,7 @@
  *
  * Acquire one hold on a RW lock.
  */
-NENTRY(rw_enter)				/* 0x00c0, 62 bytes */
+NENTRY(rw_enter)
 	cmpl	$RW_READER, %esi
 	jne	2f
 
@@ -191,7 +190,7 @@
 	testb	$(RW_WRITE_LOCKED|RW_WRITE_WANTED), %al
 	leaq	RW_READ_INCR(%rax), %rdx 
 	jnz,pn	_C_LABEL(rw_vector_enter)
-	LOCK(lockpatch2)
+	LOCK(2)
 	cmpxchgq %rdx, RW_OWNER(%rdi)
 	jnz,pn	1b
 	ret
@@ -202,7 +201,7 @@
 2:	movq	CPUVAR(CURLWP), %rcx
 	xorq	%rax, %rax
 	orq	$RW_WRITE_LOCKED, %rcx
-	LOCK(lockpatch3)
+	LOCK(3)
 	cmpxchgq %rcx, RW_OWNER(%rdi)
 	jnz,pn	_C_LABEL(rw_vector_enter)
 	ret
@@ -212,7 +211,7 @@
  *
  * Release one hold on a RW lock.
  */
-NENTRY(rw_exit)					/* 0x0100, 64 bytes */
+NENTRY(rw_exit)
 	movq	RW_OWNER(%rdi), %rax
 	testb	$RW_WRITE_LOCKED, %al
 	jnz	2f
@@ -225,7 +224,7 @@
 	cmpq	$RW_READ_INCR, %rax
 	leaq	-RW_READ_INCR(%rax), %rdx
 	jb,pn	3f
-	LOCK(lockpatch4)
+	LOCK(4)
 	cmpxchgq %rdx, RW_OWNER(%rdi)
 	jnz,pn	1b
 	ret
@@ -236,7 +235,7 @@
 2:	leaq	-RW_WRITE_LOCKED(%rax), %rdx
 	subq	CPUVAR(CURLWP), %rdx
 	jnz,pn	3f
-	LOCK(lockpatch5)
+	LOCK(5)
 	cmpxchgq %rdx, RW_OWNER(%rdi)
 	jnz	3f
 	ret
@@ -246,58 +245,49 @@
 #endif	/* LOCKDEBUG */
 
 /*
- * int _lock_cas(uintptr_t *val, uintptr_t old, uintptr_t new);
- *
- * Perform an atomic compare-and-set operation.
+ * Spinlocks.
  */
-NENTRY(_lock_cas)				/* 0x0140, 19 bytes */
-	movq	%rsi, %rax
-	LOCK(lockpatch6)
-	cmpxchgq %rdx, (%rdi)
-	movq	$0, %rax
-	setz	%al				/* = 1 if success */
+NENTRY(__cpu_simple_lock_init)
+	movb	$0, (%rdi)
 	ret
 
-/*
- * Memory barrier operations, may be patched at runtime.
- */
-	.align	8
-ENTRY(mb_read)
-	LOCK(lockpatch7)
-	addq	$0, 0(%rsp)
+NENTRY(__cpu_simple_lock)
+	movl	$0x0100, %eax
+1:
+	LOCK(6)
+	cmpxchgb %ah, (%rdi)
+	jnz	2f
 	ret
-END(mb_read_end, 8)
-
-ENTRY(mb_write)
+2:
+	movl	$0x0100, %eax
+	pause
 	nop
-	ret
-END(mb_write_end, 8)
+	nop
+	cmpb	$0, (%rdi)
+	je	1b
+	jmp	2b
 
-ENTRY(mb_memory)
-	LOCK(lockpatch8)
-	addq	$0, 0(%rsp)
+NENTRY(__cpu_simple_unlock)
+	movb	$0, (%rdi)
 	ret
-END(mb_memory_end, 8)
 
-ENTRY(sse2_mb_read)
-	lfence
+NENTRY(__cpu_simple_lock_try)
+	movl	$0x0100, %eax
+	LOCK(7)
+	cmpxchgb %ah, (%rdi)
+	movl	$0, %eax
+	setz	%al	
 	ret
-END(sse2_mb_read_end, 8)
-
-ENTRY(sse2_mb_memory)
-	mfence
-	ret
-END(sse2_mb_memory_end, 8)
 
 /*
  * Patchpoints to replace with NOP when ncpu == 1.
  */
 #ifndef LOCKDEBUG
 LABEL(x86_lockpatch)
-	.quad	lockpatch1, lockpatch2, lockpatch3, lockpatch4
-	.quad	lockpatch5, lockpatch6, lockpatch7, lockpatch8
+	.quad	.Lpatch1, .Lpatch2, .Lpatch3, .Lpatch4
+	.quad	.Lpatch5, .Lpatch6, .Lpatch7
 #ifdef FULL
-	.quad	lockpatch11
+	.quad	.Lpatch11
 #endif
 	.quad	0
 #endif
--- a/sys/arch/amd64/include/Makefile	Thu Dec 20 23:32:24 2007 +0000
+++ b/sys/arch/amd64/include/Makefile	Thu Dec 20 23:46:10 2007 +0000
@@ -1,8 +1,8 @@
-#	$NetBSD: Makefile,v 1.8 2007/02/09 21:55:01 ad Exp $
+#	$NetBSD: Makefile,v 1.9 2007/12/20 23:46:12 ad Exp $
 
 INCSDIR= /usr/include/amd64
 
-INCS=	ansi.h aout_machdep.h asm.h atomic.h \
+INCS=	ansi.h aout_machdep.h asm.h \
 	bootinfo.h bswap.h bus.h byte_swap.h \
 	cdefs.h cpu.h cpufunc.h \
 	disklabel.h \
--- a/sys/arch/amd64/include/asm.h	Thu Dec 20 23:32:24 2007 +0000
+++ b/sys/arch/amd64/include/asm.h	Thu Dec 20 23:46:10 2007 +0000
@@ -1,4 +1,4 @@
-/*	$NetBSD: asm.h,v 1.10 2007/11/29 22:03:48 ad Exp $	*/
+/*	$NetBSD: asm.h,v 1.11 2007/12/20 23:46:12 ad Exp $	*/
 
 /*-
  * Copyright (c) 1990 The Regents of the University of California.
@@ -61,7 +61,7 @@
 /* let kernels and others override entrypoint alignment */
 #ifndef _ALIGN_TEXT
 # ifdef _STANDALONE
-#  define _ALIGN_TEXT .align 3
+#  define _ALIGN_TEXT .align 4
 # else
 #  define _ALIGN_TEXT .align 16
 # endif
--- a/sys/arch/amd64/include/atomic.h	Thu Dec 20 23:32:24 2007 +0000
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,111 +0,0 @@
-/*	$NetBSD: atomic.h,v 1.9 2007/10/17 19:53:03 garbled Exp $	*/
-
-/*
- * Copyright 2002 (c) Wasabi Systems, Inc.
- * All rights reserved.
- *
- * Written by Frank van der Linden for Wasabi Systems, Inc.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. All advertising materials mentioning features or use of this software
- *    must display the following acknowledgement:
- *      This product includes software developed for the NetBSD Project by
- *      Wasabi Systems, Inc.
- * 4. The name of Wasabi Systems, Inc. may not be used to endorse
- *    or promote products derived from this software without specific prior
- *    written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
- * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
- * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL WASABI SYSTEMS, INC
- * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef _ATOMIC_H
-#define _ATOMIC_H
-
-#ifndef _LOCORE
-#if defined(_KERNEL) && !defined(__GNUC__)
-
-unsigned long	x86_atomic_testset_ul(volatile uint32_t *, unsigned long);
-int		x86_atomic_testset_i(volatile int *, int);
-uint8_t		x86_atomic_testset_b(volatile uint8_t *, uint8_t);
-void		x86_atomic_setbits_l(volatile uint32_t *, unsigned long);
-void		x86_atomic_clearbits_l(volatile uint32_t *, unsigned long);
-
-uint64_t	x86_atomic_testset_u64(volatile uint64_t *, uint64_t);
-void		x86_atomic_setbits_u64(volatile uint64_t *, uint64_t);
-void		x86_atomic_clearbits_u64(volatile uint64_t *, uint64_t);
-
-#else
-
-static __inline u_int64_t
-x86_atomic_testset_u64(volatile u_int64_t *ptr, u_int64_t val) {
-    __asm volatile ("xchgq %0,(%2)" :"=r" (val):"0" (val),"r" (ptr));
-    return val;
-}
-
-static __inline u_int32_t
-x86_atomic_testset_u32(volatile u_int32_t *ptr, u_int32_t val) {
-    __asm volatile ("xchgl %0,(%2)" :"=r" (val):"0" (val),"r" (ptr));
-    return val;
-}
-
-static __inline int32_t
-x86_atomic_testset_i32(volatile int32_t *ptr, int32_t val) {
-    __asm volatile ("xchgl %0,(%2)" :"=r" (val):"0" (val),"r" (ptr));
-    return val;
-}
-
-static __inline uint8_t
-x86_atomic_testset_u8(volatile uint8_t *ptr, uint8_t val) {
-    __asm volatile ("xchgb %0,(%2)" :"=r" (val):"0" (val),"r" (ptr));
-    return val;
-}
-
-static __inline void
-x86_atomic_setbits_u32(volatile u_int32_t *ptr, u_int32_t bits) {
-    __asm volatile("lock ; orl %1,%0" :  "=m" (*ptr) : "ir" (bits));
-}
-
-static __inline void
-x86_atomic_clearbits_u32(volatile u_int32_t *ptr, u_int32_t bits) {
-    __asm volatile("lock ; andl %1,%0" :  "=m" (*ptr) : "ir" (~bits));
-}
-
-static __inline void
-x86_atomic_setbits_u64(volatile u_int64_t *ptr, u_int64_t bits) {
-    __asm volatile("lock ; orq %1,%0" :  "=m" (*ptr) : "ir" (bits));
-}
-
-static __inline void
-x86_atomic_clearbits_u64(volatile u_int64_t *ptr, u_int64_t bits) {
-    __asm volatile("lock ; andq %1,%0" :  "=m" (*ptr) : "ir" (~bits));
-}
-
-#define x86_atomic_testset_ul	x86_atomic_testset_u32
-#define x86_atomic_testset_i	x86_atomic_testset_i32
-#define x86_atomic_testset_b	x86_atomic_testset_u8
-#define x86_atomic_setbits_l	x86_atomic_setbits_u32
-#define x86_atomic_setbits_ul	x86_atomic_setbits_u32
-#define x86_atomic_clearbits_l	x86_atomic_clearbits_u32
-#define x86_atomic_clearbits_ul	x86_atomic_clearbits_u32
-
-#endif	/* _KERNEL && !__GNUC__ */
-#endif	/* _LOCORE */
-
-#endif
--- a/sys/arch/amd64/include/profile.h	Thu Dec 20 23:32:24 2007 +0000
+++ b/sys/arch/amd64/include/profile.h	Thu Dec 20 23:46:10 2007 +0000
@@ -1,4 +1,4 @@
-/*	$NetBSD: profile.h,v 1.11 2007/11/24 18:55:41 bouyer Exp $	*/
+/*	$NetBSD: profile.h,v 1.12 2007/12/20 23:46:12 ad Exp $	*/
 
 /*
  * Copyright (c) 1992, 1993
@@ -36,7 +36,9 @@
 #include "opt_xen.h"
 #endif
 
-#include <machine/atomic.h>
+#ifdef _KERNEL
+#include <machine/lock.h>
+#endif
 
 #define	_MCOUNT_DECL void _mcount
 
@@ -86,11 +88,7 @@
 static inline void
 MCOUNT_ENTER_MP(void)
 {
-	while (x86_atomic_testset_b(&__mcount_lock, __SIMPLELOCK_LOCKED)
-	    != __SIMPLELOCK_UNLOCKED) {
-		while (__mcount_lock == __SIMPLELOCK_LOCKED)
-			;
-	}
+	__cpu_simple_lock(&__mcount_lock);
 	__insn_barrier();
 }
 
--- a/sys/arch/i386/i386/lock_stubs.S	Thu Dec 20 23:32:24 2007 +0000
+++ b/sys/arch/i386/i386/lock_stubs.S	Thu Dec 20 23:46:10 2007 +0000
@@ -1,4 +1,4 @@
-/*	$NetBSD: lock_stubs.S,v 1.11 2007/12/11 23:06:13 lukem Exp $	*/
+/*	$NetBSD: lock_stubs.S,v 1.12 2007/12/20 23:46:10 ad Exp $	*/
 
 /*-
  * Copyright (c) 2006, 2007 The NetBSD Foundation, Inc.
@@ -44,7 +44,7 @@
  */
 
 #include <machine/asm.h>
-__KERNEL_RCSID(0, "$NetBSD: lock_stubs.S,v 1.11 2007/12/11 23:06:13 lukem Exp $");
+__KERNEL_RCSID(0, "$NetBSD: lock_stubs.S,v 1.12 2007/12/20 23:46:10 ad Exp $");
 
 #include "opt_multiprocessor.h"
 #include "opt_lockdebug.h"
@@ -60,7 +60,7 @@
 
 #define	ALIGN64		.align	64
 #define	ALIGN32		.align	32
-#define	LOCK(name)	LABEL(name)	lock
+#define	LOCK(num)	.Lpatch/**/num:	lock
 #define	END(name,a)	.align	a; LABEL(name)
 
 #if !defined(LOCKDEBUG)
@@ -76,7 +76,7 @@
 	movl	4(%esp), %edx
 	movl	CPUVAR(CURLWP), %ecx
 	xorl	%eax, %eax
-	LOCK(lockpatch1)
+	LOCK(1)
 	cmpxchgl %ecx, MTX_OWNER(%edx)
 	jnz,pn	_C_LABEL(mutex_vector_enter)
 	ret
@@ -115,7 +115,7 @@
 	testb	$(RW_WRITE_LOCKED|RW_WRITE_WANTED), %al
 	leal	RW_READ_INCR(%eax), %ecx
 	jnz,pn	_C_LABEL(rw_vector_enter)
-	LOCK(lockpatch2)
+	LOCK(2)
 	cmpxchgl %ecx, RW_OWNER(%edx)
 	jnz,pn	1b
 	ret
@@ -126,7 +126,7 @@
 2:	movl	CPUVAR(CURLWP), %ecx
 	xorl	%eax, %eax
 	orl	$RW_WRITE_LOCKED, %ecx
-	LOCK(lockpatch3)
+	LOCK(3)
 	cmpxchgl %ecx, RW_OWNER(%edx)
 	jnz,pn	_C_LABEL(rw_vector_enter)
 	ret
@@ -150,7 +150,7 @@
 	cmpl	$RW_READ_INCR, %eax
 	leal	-RW_READ_INCR(%eax), %ecx
 	jb,pn	3f
-	LOCK(lockpatch4)
+	LOCK(4)
 	cmpxchgl %ecx, RW_OWNER(%edx)
 	jnz,pn	1b
 	ret
@@ -161,7 +161,7 @@
 2:	leal	-RW_WRITE_LOCKED(%eax), %ecx
 	subl	CPUVAR(CURLWP), %ecx
 	jnz,pn	3f
-	LOCK(lockpatch5)
+	LOCK(5)
 	cmpxchgl %ecx, RW_OWNER(%edx)
 	jnz,pn	3f
 	ret
@@ -191,7 +191,7 @@
 2:
 #ifdef FULL
 	mov	$0x0100, %eax			/* new + expected value */
-	LOCK(lockpatch11)
+	LOCK(11)
 	cmpxchgb %ah, MTX_LOCK(%edx)		/* lock it */
 	jnz,pn	_C_LABEL(mutex_spin_retry)
 #endif
@@ -268,61 +268,54 @@
 #endif	/* !LOCKDEBUG */
 
 /*
- * int _lock_cas(uintptr_t *val, uintptr_t old, uintptr_t new);
- *
- * Perform an atomic compare-and-set operation.
+ * Spinlocks.
  */
-ENTRY(_lock_cas)
-	mov	4(%esp),%edx
-	movl	8(%esp), %eax
-	movl	12(%esp), %ecx
-	LOCK(lockpatch6)
-	cmpxchgl %ecx, (%edx)
-	movl	$0, %eax
-	setz	%al
+NENTRY(__cpu_simple_lock_init)
+	movl	4(%esp), %edx
+	movb	$0, (%edx)
+	ret
+NENTRY(__cpu_simple_lock)
+	movl	4(%esp), %edx
+	movl	$0x0100, %eax
+1:
+	LOCK(6)
+	cmpxchgb %ah, (%edx)
+	jnz	2f
+	ret
+2:
+	movl	$0x0100, %eax
+	pause
+	nop
+	nop
+	cmpb	$0, (%edx)
+	je	1b
+	jmp	2b
+
+NENTRY(__cpu_simple_unlock)
+	movl	4(%esp), %edx
+	movb	$0, (%edx)
 	ret
 
-/*
- * Memory barrier operations, may be patched at runtime.
- */
-	.align	8
-
-ENTRY(mb_read)
-	LOCK(lockpatch7)
-	addl	$0, 0(%esp)
-	ret
-END(mb_read_end, 8)
-
-ENTRY(mb_write)
-	nop
+NENTRY(__cpu_simple_lock_try)
+	movl	4(%esp), %edx
+	movl	$0x0100, %eax
+	LOCK(7)
+	cmpxchgb %ah, (%edx)
+	movl	$0, %eax
+	setz	%al	
 	ret
-END(mb_write_end, 8)
-
-ENTRY(mb_memory)
-	LOCK(lockpatch8)
-	addl	$0, 0(%esp)
-	ret
-END(mb_memory_end, 8)
-
-ENTRY(sse2_mb_read)
-	lfence
-	ret
-END(sse2_mb_read_end, 8)
-
-ENTRY(sse2_mb_memory)
-	mfence
-	ret
-END(sse2_mb_memory_end, 8)
 
 /*
  * Patchpoints to replace with NOP when ncpu == 1.
  */
+	.data
+
 #ifndef LOCKDEBUG
 LABEL(x86_lockpatch)
-	.long	lockpatch1, lockpatch2, lockpatch3, lockpatch4
-	.long	lockpatch5, lockpatch6, lockpatch7, lockpatch8
+	.long	.Lpatch1, .Lpatch2, .Lpatch3, .Lpatch4
+	.long	.Lpatch5, .Lpatch6, .Lpatch7
 #if defined(FULL) && !defined(__XEN__)
-	.long	lockpatch11
+	.long	.Lpatch11
 #endif
 	.long	0
 #endif
--- a/sys/arch/i386/i386/spl.S	Thu Dec 20 23:32:24 2007 +0000
+++ b/sys/arch/i386/i386/spl.S	Thu Dec 20 23:46:10 2007 +0000
@@ -1,4 +1,4 @@
-/*	$NetBSD: spl.S,v 1.23 2007/12/11 23:06:14 lukem Exp $	*/
+/*	$NetBSD: spl.S,v 1.24 2007/12/20 23:46:11 ad Exp $	*/
 
 /*
  * Copyright (c) 1998, 2007 The NetBSD Foundation, Inc.
@@ -37,7 +37,7 @@
  */
 
 #include <machine/asm.h>
-__KERNEL_RCSID(0, "$NetBSD: spl.S,v 1.23 2007/12/11 23:06:14 lukem Exp $");
+__KERNEL_RCSID(0, "$NetBSD: spl.S,v 1.24 2007/12/20 23:46:11 ad Exp $");
 
 #include "opt_vm86.h"
 #include "opt_ddb.h"
@@ -69,8 +69,6 @@
  * i686_spllower().  This must use pushf/cli/popf as it is used
  * early in boot where interrupts are disabled via eflags/IE.
  */
-	.align 64
-
 ENTRY(spllower)
 	movl	4(%esp), %ecx
 	cmpl	CPUVAR(ILEVEL), %ecx
@@ -113,10 +111,6 @@
 	movl	CPUVAR(IPENDING),%eax
 	testl	%eax,CPUVAR(IUNMASK)(,%ecx,4)	/* deferred interrupts? */
 	movl	%eax,%ebx
-	/*
-	 * On the P4 this jump is cheaper than patching in junk
-	 * using cmovnz.  Is cmpxchg expensive if it fails?
-	 */
 	jnz,pn	2f
 	cmpxchg8b CPUVAR(ISTATE)		/* swap in new ilevel */
 	jnz,pn	0b
--- a/sys/arch/i386/include/Makefile	Thu Dec 20 23:32:24 2007 +0000
+++ b/sys/arch/i386/include/Makefile	Thu Dec 20 23:46:10 2007 +0000
@@ -1,8 +1,8 @@
-#	$NetBSD: Makefile,v 1.35 2007/11/14 17:55:00 ad Exp $
+#	$NetBSD: Makefile,v 1.36 2007/12/20 23:46:12 ad Exp $
 
 INCSDIR= /usr/include/i386
 
-INCS=	ansi.h aout_machdep.h apmvar.h asm.h atomic.h \
+INCS=	ansi.h aout_machdep.h apmvar.h asm.h \
 	bioscall.h bootinfo.h bswap.h byte_swap.h bus.h \
 	cdefs.h cpu.h cpufunc.h cputypes.h \
 	disklabel.h \
--- a/sys/arch/i386/include/asm.h	Thu Dec 20 23:32:24 2007 +0000
+++ b/sys/arch/i386/include/asm.h	Thu Dec 20 23:46:10 2007 +0000
@@ -1,4 +1,4 @@
-/*	$NetBSD: asm.h,v 1.36 2007/12/12 14:58:20 yamt Exp $	*/
+/*	$NetBSD: asm.h,v 1.37 2007/12/20 23:46:12 ad Exp $	*/
 
 /*-
  * Copyright (c) 1990 The Regents of the University of California.
@@ -124,15 +124,15 @@
 #ifdef _STANDALONE
 #define ALIGN_DATA	.align	4
 #define ALIGN_TEXT	.align	4	/* 4-byte boundaries */
-#define SUPERALIGN_TEXT	.align	32	/* 32-byte boundaries */
+#define SUPERALIGN_TEXT	.align	16	/* 15-byte boundaries */
 #elif defined __ELF__
 #define ALIGN_DATA	.align	4
-#define ALIGN_TEXT	.align	32	/* 32-byte boundaries */
-#define SUPERALIGN_TEXT	.align	32	/* 32-byte boundaries */
+#define ALIGN_TEXT	.align	16	/* 16-byte boundaries */
+#define SUPERALIGN_TEXT	.align	16	/* 16-byte boundaries */
 #else
 #define ALIGN_DATA	.align	2
-#define ALIGN_TEXT	.align	5	/* 32-byte boundaries */
-#define SUPERALIGN_TEXT	.align	5	/* 32-byte boundaries */
+#define ALIGN_TEXT	.align	4	/* 16-byte boundaries */
+#define SUPERALIGN_TEXT	.align	4	/* 16-byte boundaries */
 #endif /* __ELF__ */
 
 #define _ALIGN_TEXT ALIGN_TEXT
--- a/sys/arch/i386/include/atomic.h	Thu Dec 20 23:32:24 2007 +0000
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,91 +0,0 @@
-/*	$NetBSD: atomic.h,v 1.13 2007/10/17 19:54:56 garbled Exp $	*/
-
-/*-
- * Copyright (c) 2000, 2007 The NetBSD Foundation, Inc.
- * All rights reserved.
- *
- * This code is derived from software contributed to The NetBSD Foundation
- * by RedBack Networks Inc.
- *
- * Author: Bill Sommerfeld
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. All advertising materials mentioning features or use of this software
- *    must display the following acknowledgement:
- *        This product includes software developed by the NetBSD
- *        Foundation, Inc. and its contributors.
- * 4. Neither the name of The NetBSD Foundation nor the names of its
- *    contributors may be used to endorse or promote products derived
- *    from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
- * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
- * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
- * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
- * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef _I386_ATOMIC_H_
-#define _I386_ATOMIC_H_
-
-#ifndef _LOCORE
-#if defined(_KERNEL) && !defined(__GNUC__)
-
-unsigned long	x86_atomic_testset_ul(volatile uint32_t *, unsigned long);
-int		x86_atomic_testset_i(volatile int *, int);
-uint8_t		x86_atomic_testset_b(volatile uint8_t *, uint8_t);
-void		x86_atomic_setbits_l(volatile uint32_t *, unsigned long);
-void		x86_atomic_clearbits_l(volatile uint32_t *, unsigned long);
-
-#else
-
-static __inline unsigned long x86_atomic_testset_ul(volatile uint32_t *,
-    unsigned long);
-static __inline unsigned long
-x86_atomic_testset_ul(volatile uint32_t *__ptr, unsigned long __val) {
-	__asm volatile ("xchgl %0,(%2)" :"=r" (__val):"0" (__val),"r" (__ptr));
-	return __val;
-}
-
-static __inline int x86_atomic_testset_i(volatile int *, int);
-static __inline int
-x86_atomic_testset_i(volatile int *__ptr, int __val) {
-	__asm volatile ("xchgl %0,(%2)" :"=r" (__val):"0" (__val),"r" (__ptr));
-	return __val;
-}
-
-static __inline uint8_t x86_atomic_testset_b(volatile uint8_t *, uint8_t);
-static __inline uint8_t
-x86_atomic_testset_b(volatile uint8_t *__ptr, uint8_t __val) {
-	__asm volatile ("xchgb %0,(%2)" :"=A" (__val):"0" (__val),"r" (__ptr));
-	return __val;
-}
-
-static __inline void x86_atomic_setbits_l(volatile uint32_t *, unsigned long);
-static __inline void
-x86_atomic_setbits_l(volatile uint32_t *__ptr, unsigned long __bits) {
-	__asm volatile("lock ; orl %1,%0" :  "=m" (*__ptr) : "ir" (__bits));
-}
-
-static __inline void x86_atomic_clearbits_l(volatile uint32_t *, unsigned long);
-static __inline void
-x86_atomic_clearbits_l(volatile uint32_t *__ptr, unsigned long __bits) {
-	__asm volatile("lock ; andl %1,%0" :  "=m" (*__ptr) : "ir" (~__bits));
-}
-#endif	/* _KERNEL && !__GNUC__ */
-#endif	/* _LOCORE */
-
-#endif
--- a/sys/arch/i386/include/profile.h	Thu Dec 20 23:32:24 2007 +0000
+++ b/sys/arch/i386/include/profile.h	Thu Dec 20 23:46:10 2007 +0000
@@ -1,4 +1,4 @@
-/*	$NetBSD: profile.h,v 1.32 2007/10/17 19:54:57 garbled Exp $	*/
+/*	$NetBSD: profile.h,v 1.33 2007/12/20 23:46:13 ad Exp $	*/
 
 /*
  * Copyright (c) 1992, 1993
@@ -37,7 +37,7 @@
 
 #ifdef _KERNEL
 #include <machine/cpufunc.h>
-#include <machine/atomic.h>
+#include <machine/lock.h>
 #endif
 
 #define	_MCOUNT_DECL static __inline void _mcount
@@ -89,11 +89,7 @@
 static inline void
 MCOUNT_ENTER_MP(void)
 {
-	while (x86_atomic_testset_b(&__mcount_lock, __SIMPLELOCK_LOCKED)
-	    != __SIMPLELOCK_UNLOCKED) {
-		while (__mcount_lock == __SIMPLELOCK_LOCKED)
-			;
-	}
+	__cpu_simple_lock(&__mcount_lock);
 	__insn_barrier();
 }
 
--- a/sys/arch/x86/include/lock.h	Thu Dec 20 23:32:24 2007 +0000
+++ b/sys/arch/x86/include/lock.h	Thu Dec 20 23:46:10 2007 +0000
@@ -1,4 +1,4 @@
-/*	$NetBSD: lock.h,v 1.19 2007/11/07 16:02:27 ad Exp $	*/
+/*	$NetBSD: lock.h,v 1.20 2007/12/20 23:46:11 ad Exp $	*/
 
 /*-
  * Copyright (c) 2000, 2006 The NetBSD Foundation, Inc.
@@ -43,12 +43,6 @@
 #ifndef _X86_LOCK_H_
 #define	_X86_LOCK_H_
 
-#ifdef _KERNEL
-#include <machine/cpufunc.h>
-#endif
-#include <machine/atomic.h>
-
-
 static __inline int
 __SIMPLELOCK_LOCKED_P(__cpu_simple_lock_t *__ptr)
 {
@@ -75,6 +69,20 @@
 	*__ptr = __SIMPLELOCK_UNLOCKED;
 }
 
+#ifdef _KERNEL
+
+#include <machine/cpufunc.h>
+
+void	__cpu_simple_lock_init(__cpu_simple_lock_t *);
+void	__cpu_simple_lock(__cpu_simple_lock_t *);
+int	__cpu_simple_lock_try(__cpu_simple_lock_t *);
+void	__cpu_simple_unlock(__cpu_simple_lock_t *);
+
+#define	SPINLOCK_SPIN_HOOK	/* nothing */
+#define	SPINLOCK_BACKOFF_HOOK	x86_pause()
+
+#else
+
 static __inline void __cpu_simple_lock_init(__cpu_simple_lock_t *)
 	__attribute__((__unused__));
 static __inline void __cpu_simple_lock(__cpu_simple_lock_t *)
@@ -92,32 +100,28 @@
 	__insn_barrier();
 }
 
+static __inline int
+__cpu_simple_lock_try(__cpu_simple_lock_t *lockp)
+{
+	uint8_t val;
+
+	val = __SIMPLELOCK_LOCKED;
+	__asm volatile ("xchgb %0,(%2)" : 
+	    "=r" (val)
+	    :"0" (val), "r" (lockp));
+	__insn_barrier();
+	return val == __SIMPLELOCK_UNLOCKED;
+}
+
 static __inline void
 __cpu_simple_lock(__cpu_simple_lock_t *lockp)
 {
 
-	while (x86_atomic_testset_b(lockp, __SIMPLELOCK_LOCKED)
-	    != __SIMPLELOCK_UNLOCKED) {
-		do {
-#ifdef _KERNEL
-			x86_pause();
-#endif /* _KERNEL */
-		} while (*lockp == __SIMPLELOCK_LOCKED);
-	}
+	while (!__cpu_simple_lock_try(lockp))
+		/* nothing */;
 	__insn_barrier();
 }
 
-static __inline int
-__cpu_simple_lock_try(__cpu_simple_lock_t *lockp)
-{
-	int r = (x86_atomic_testset_b(lockp, __SIMPLELOCK_LOCKED)
-	    == __SIMPLELOCK_UNLOCKED);
-
-	__insn_barrier();
-
-	return (r);
-}
-
 /*
  * Note on x86 memory ordering
  *
@@ -178,13 +182,6 @@
 	*lockp = __SIMPLELOCK_UNLOCKED;
 }
 
-#define	SPINLOCK_SPIN_HOOK	/* nothing */
-#define	SPINLOCK_BACKOFF_HOOK	x86_pause()
-
-#ifdef _KERNEL
-void	mb_read(void);
-void	mb_write(void);
-void	mb_memory(void);
 #endif	/* _KERNEL */
 
 #endif /* _X86_LOCK_H_ */
--- a/sys/arch/x86/x86/bus_space.c	Thu Dec 20 23:32:24 2007 +0000
+++ b/sys/arch/x86/x86/bus_space.c	Thu Dec 20 23:46:10 2007 +0000
@@ -1,4 +1,4 @@
-/*	$NetBSD: bus_space.c,v 1.13 2007/11/28 16:44:47 ad Exp $	*/
+/*	$NetBSD: bus_space.c,v 1.14 2007/12/20 23:46:11 ad Exp $	*/
 
 /*-
  * Copyright (c) 1996, 1997, 1998 The NetBSD Foundation, Inc.
@@ -38,7 +38,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: bus_space.c,v 1.13 2007/11/28 16:44:47 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: bus_space.c,v 1.14 2007/12/20 23:46:11 ad Exp $");
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -52,7 +52,6 @@
 #include <machine/bus.h>
 #include <machine/pio.h>
 #include <machine/isa_machdep.h>
-#include <machine/atomic.h>
 
 #ifdef XEN
 #include <xen/hypervisor.h>
--- a/sys/arch/x86/x86/patch.c	Thu Dec 20 23:32:24 2007 +0000
+++ b/sys/arch/x86/x86/patch.c	Thu Dec 20 23:46:10 2007 +0000
@@ -1,4 +1,4 @@
-/*	$NetBSD: patch.c,v 1.10 2007/12/20 01:27:25 ad Exp $	*/
+/*	$NetBSD: patch.c,v 1.11 2007/12/20 23:46:11 ad Exp $	*/
 
 /*-
  * Copyright (c) 2007 The NetBSD Foundation, Inc.
@@ -41,7 +41,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: patch.c,v 1.10 2007/12/20 01:27:25 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: patch.c,v 1.11 2007/12/20 23:46:11 ad Exp $");
 
 #include "opt_lockdebug.h"
 
@@ -75,15 +75,6 @@
 void	sse2_mfence(void);
 void	sse2_mfence_end(void);
 
-void	mb_read(void);
-void	mb_read_end(void);
-void	mb_memory(void);
-void	mb_memory_end(void);
-void	sse2_mb_read(void);
-void	sse2_mb_read_end(void);
-void	sse2_mb_memory(void);
-void	sse2_mb_memory_end(void);
-
 void	_atomic_cas_64(void);
 void	_atomic_cas_64_end(void);
 void	_atomic_cas_cx8(void);
@@ -176,16 +167,6 @@
 	} else if ((cpu_feature & CPUID_SSE2) != 0) {
 		/* Faster memory barriers. */
 		patchfunc(
-		    sse2_mb_read, sse2_mb_read_end,
-		    mb_read, mb_read_end,
-		    NULL
-		);
-		patchfunc(
-		    sse2_mb_memory, sse2_mb_memory_end,
-		    mb_memory, mb_memory_end,
-		    NULL
-		);
-		patchfunc(
 		    sse2_lfence, sse2_lfence_end,
 		    membar_consumer, membar_consumer_end,
 		    NULL
--- a/sys/arch/x86/x86/pmap.c	Thu Dec 20 23:32:24 2007 +0000
+++ b/sys/arch/x86/x86/pmap.c	Thu Dec 20 23:46:10 2007 +0000
@@ -1,4 +1,4 @@
-/*	$NetBSD: pmap.c,v 1.14 2007/12/13 21:22:15 bouyer Exp $	*/
+/*	$NetBSD: pmap.c,v 1.15 2007/12/20 23:46:11 ad Exp $	*/
 
 /*
  * Copyright (c) 2007 Manuel Bouyer.
@@ -154,7 +154,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.14 2007/12/13 21:22:15 bouyer Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.15 2007/12/20 23:46:11 ad Exp $");
 
 #include "opt_user_ldt.h"
 #include "opt_lockdebug.h"
@@ -172,16 +172,15 @@
 #include <sys/user.h>
 #include <sys/kernel.h>
 #include <sys/atomic.h>
+#include <sys/cpu.h>
+#include <sys/intr.h>
 
 #include <uvm/uvm.h>
 
 #include <dev/isa/isareg.h>
 
-#include <machine/atomic.h>
-#include <machine/cpu.h>
 #include <machine/specialreg.h>
 #include <machine/gdt.h>
-#include <machine/intr.h>
 #include <machine/isa_machdep.h>
 #include <machine/cpuvar.h>
 
@@ -539,9 +538,7 @@
 static struct pool_cache pmap_pdp_cache;
 
 int	pmap_pdp_ctor(void *, void *, int);
-#ifdef XEN
-void   pmap_pdp_dtor(void *, void *);
-#endif
+void	pmap_pdp_dtor(void *, void *);
 
 void *vmmap; /* XXX: used by mem.c... it should really uvm_map_reserve it */
 
@@ -1361,18 +1358,13 @@
 	 * initialize caches.
 	 */
 
-	pool_cache_bootstrap(&pmap_cache, sizeof(struct pmap), 0, 0, 0, "pmappl",
-	    &pool_allocator_nointr, IPL_NONE, NULL, NULL, NULL);
-#ifdef XEN
-	pool_cache_bootstrap(&pmap_pdp_cache, PAGE_SIZE, 0, 0, 0, "pdppl",
-	    &pool_allocator_nointr, IPL_NONE,
-	    pmap_pdp_ctor, pmap_pdp_dtor, NULL);
-#else
-	pool_cache_bootstrap(&pmap_pdp_cache, PAGE_SIZE, 0, 0, 0, "pdppl",
-	    &pool_allocator_nointr, IPL_NONE, pmap_pdp_ctor, NULL, NULL);
-#endif
-	pool_cache_bootstrap(&pmap_pv_cache, sizeof(struct pv_entry), 0, 0, 0,
-	    "pvpl", &pool_allocator_meta, IPL_NONE, NULL, NULL, NULL);
+	pool_cache_bootstrap(&pmap_cache, sizeof(struct pmap), 0, 0, 0,
+	    "pmappl", NULL, IPL_NONE, NULL, NULL, NULL);
+	pool_cache_bootstrap(&pmap_pdp_cache, PAGE_SIZE, 0, 0, 0,
+	    "pdppl", NULL, IPL_NONE, pmap_pdp_ctor, pmap_pdp_dtor, NULL);
+	pool_cache_bootstrap(&pmap_pv_cache, sizeof(struct pv_entry), 0, 0,
+	    PR_LARGECACHE, "pvpl", &pool_allocator_meta, IPL_NONE, NULL,
+	    NULL, NULL);
 
 	/*
 	 * ensure the TLB is sync'd with reality by flushing it...
@@ -1862,7 +1854,6 @@
 	return (0);
 }
 
-#ifdef XEN
 /*
  * pmap_pdp_dtor: destructor for the PDP cache.
  */
@@ -1870,6 +1861,7 @@
 void
 pmap_pdp_dtor(void *arg, void *object)
 {
+#ifdef XEN
 	int s = splvm();
 	/* Set page RW again */
 	pt_entry_t *pte = kvtopte((vaddr_t)object);
@@ -1877,9 +1869,8 @@
 	xpq_queue_invlpg((vaddr_t)object);
 	xpq_flush_queue();
 	splx(s);
+#endif  /* XEN */
 }
-#endif  /* XEN */
-
 
 /*
  * pmap_create: create a pmap
--- a/sys/arch/xen/x86/hypervisor_machdep.c	Thu Dec 20 23:32:24 2007 +0000
+++ b/sys/arch/xen/x86/hypervisor_machdep.c	Thu Dec 20 23:46:10 2007 +0000
@@ -1,4 +1,4 @@
-/*	$NetBSD: hypervisor_machdep.c,v 1.3 2007/12/12 22:16:32 bouyer Exp $	*/
+/*	$NetBSD: hypervisor_machdep.c,v 1.4 2007/12/20 23:46:11 ad Exp $	*/
 
 /*
  *
@@ -59,7 +59,7 @@
 
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: hypervisor_machdep.c,v 1.3 2007/12/12 22:16:32 bouyer Exp $");
+__KERNEL_RCSID(0, "$NetBSD: hypervisor_machdep.c,v 1.4 2007/12/20 23:46:11 ad Exp $");
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -67,7 +67,6 @@
 #include <xen/xen.h>
 #include <xen/hypervisor.h>
 #include <xen/evtchn.h>
-#include <machine/atomic.h>
 
 #include "opt_xen.h"
 
--- a/sys/arch/xen/x86/intr.c	Thu Dec 20 23:32:24 2007 +0000
+++ b/sys/arch/xen/x86/intr.c	Thu Dec 20 23:46:10 2007 +0000
@@ -103,7 +103,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: intr.c,v 1.15 2007/12/03 15:34:29 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: intr.c,v 1.16 2007/12/20 23:46:12 ad Exp $");
 
 #include "opt_multiprocessor.h"
 #include "opt_xen.h"
@@ -118,12 +118,11 @@
 #include <sys/malloc.h>
 #include <sys/proc.h>
 #include <sys/errno.h>
+#include <sys/cpu.h>
 
 #include <uvm/uvm_extern.h>
 
-#include <machine/atomic.h>
 #include <machine/i8259.h>
-#include <machine/cpu.h>
 #include <machine/pio.h>
 #include <xen/evtchn.h>