vendor/opensolaris/20100818vendor/opensolaris

3 files changed, 2094 insertions, 0 deletions

diff --git a/common/atomic/amd64/atomic.s b/common/atomic/amd64/atomic.s
new file mode 100644
index 000000000000..4b0d66e4db20
--- /dev/null
+++ b/common/atomic/amd64/atomic.s
@@ -0,0 +1,573 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
+ */
+
+	.file	"atomic.s"
+
+#include <sys/asm_linkage.h>
+
+#if defined(_KERNEL)
+	/*
+	 * Legacy kernel interfaces; they will go away (eventually).
+	 */
+	ANSI_PRAGMA_WEAK2(cas8,atomic_cas_8,function)
+	ANSI_PRAGMA_WEAK2(cas32,atomic_cas_32,function)
+	ANSI_PRAGMA_WEAK2(cas64,atomic_cas_64,function)
+	ANSI_PRAGMA_WEAK2(caslong,atomic_cas_ulong,function)
+	ANSI_PRAGMA_WEAK2(casptr,atomic_cas_ptr,function)
+	ANSI_PRAGMA_WEAK2(atomic_and_long,atomic_and_ulong,function)
+	ANSI_PRAGMA_WEAK2(atomic_or_long,atomic_or_ulong,function)
+#endif
+
+	ENTRY(atomic_inc_8)
+	ALTENTRY(atomic_inc_uchar)
+	lock
+	incb	(%rdi)
+	ret
+	SET_SIZE(atomic_inc_uchar)
+	SET_SIZE(atomic_inc_8)
+
+	ENTRY(atomic_inc_16)
+	ALTENTRY(atomic_inc_ushort)
+	lock
+	incw	(%rdi)
+	ret
+	SET_SIZE(atomic_inc_ushort)
+	SET_SIZE(atomic_inc_16)
+
+	ENTRY(atomic_inc_32)
+	ALTENTRY(atomic_inc_uint)
+	lock
+	incl	(%rdi)
+	ret
+	SET_SIZE(atomic_inc_uint)
+	SET_SIZE(atomic_inc_32)
+
+	ENTRY(atomic_inc_64)
+	ALTENTRY(atomic_inc_ulong)
+	lock
+	incq	(%rdi)
+	ret
+	SET_SIZE(atomic_inc_ulong)
+	SET_SIZE(atomic_inc_64)
+
+	ENTRY(atomic_inc_8_nv)
+	ALTENTRY(atomic_inc_uchar_nv)
+	xorl	%eax, %eax	/ clear upper bits of %eax return register
+	incb	%al		/ %al = 1
+	lock
+	  xaddb	%al, (%rdi)	/ %al = old value, (%rdi) = new value
+	incb	%al		/ return new value
+	ret
+	SET_SIZE(atomic_inc_uchar_nv)
+	SET_SIZE(atomic_inc_8_nv)
+
+	ENTRY(atomic_inc_16_nv)
+	ALTENTRY(atomic_inc_ushort_nv)
+	xorl	%eax, %eax	/ clear upper bits of %eax return register
+	incw	%ax		/ %ax = 1
+	lock
+	  xaddw	%ax, (%rdi)	/ %ax = old value, (%rdi) = new value
+	incw	%ax		/ return new value
+	ret
+	SET_SIZE(atomic_inc_ushort_nv)
+	SET_SIZE(atomic_inc_16_nv)
+
+	ENTRY(atomic_inc_32_nv)
+	ALTENTRY(atomic_inc_uint_nv)
+	xorl	%eax, %eax	/ %eax = 0
+	incl	%eax		/ %eax = 1
+	lock
+	  xaddl	%eax, (%rdi)	/ %eax = old value, (%rdi) = new value
+	incl	%eax		/ return new value
+	ret
+	SET_SIZE(atomic_inc_uint_nv)
+	SET_SIZE(atomic_inc_32_nv)
+
+	ENTRY(atomic_inc_64_nv)
+	ALTENTRY(atomic_inc_ulong_nv)
+	xorq	%rax, %rax	/ %rax = 0
+	incq	%rax		/ %rax = 1
+	lock
+	  xaddq	%rax, (%rdi)	/ %rax = old value, (%rdi) = new value
+	incq	%rax		/ return new value
+	ret
+	SET_SIZE(atomic_inc_ulong_nv)
+	SET_SIZE(atomic_inc_64_nv)
+
+	ENTRY(atomic_dec_8)
+	ALTENTRY(atomic_dec_uchar)
+	lock
+	decb	(%rdi)
+	ret
+	SET_SIZE(atomic_dec_uchar)
+	SET_SIZE(atomic_dec_8)
+
+	ENTRY(atomic_dec_16)
+	ALTENTRY(atomic_dec_ushort)
+	lock
+	decw	(%rdi)
+	ret
+	SET_SIZE(atomic_dec_ushort)
+	SET_SIZE(atomic_dec_16)
+
+	ENTRY(atomic_dec_32)
+	ALTENTRY(atomic_dec_uint)
+	lock
+	decl	(%rdi)
+	ret
+	SET_SIZE(atomic_dec_uint)
+	SET_SIZE(atomic_dec_32)
+
+	ENTRY(atomic_dec_64)
+	ALTENTRY(atomic_dec_ulong)
+	lock
+	decq	(%rdi)
+	ret
+	SET_SIZE(atomic_dec_ulong)
+	SET_SIZE(atomic_dec_64)
+
+	ENTRY(atomic_dec_8_nv)
+	ALTENTRY(atomic_dec_uchar_nv)
+	xorl	%eax, %eax	/ clear upper bits of %eax return register
+	decb	%al		/ %al = -1
+	lock
+	  xaddb	%al, (%rdi)	/ %al = old value, (%rdi) = new value
+	decb	%al		/ return new value
+	ret
+	SET_SIZE(atomic_dec_uchar_nv)
+	SET_SIZE(atomic_dec_8_nv)
+
+	ENTRY(atomic_dec_16_nv)
+	ALTENTRY(atomic_dec_ushort_nv)
+	xorl	%eax, %eax	/ clear upper bits of %eax return register
+	decw	%ax		/ %ax = -1
+	lock
+	  xaddw	%ax, (%rdi)	/ %ax = old value, (%rdi) = new value
+	decw	%ax		/ return new value
+	ret
+	SET_SIZE(atomic_dec_ushort_nv)
+	SET_SIZE(atomic_dec_16_nv)
+
+	ENTRY(atomic_dec_32_nv)
+	ALTENTRY(atomic_dec_uint_nv)
+	xorl	%eax, %eax	/ %eax = 0
+	decl	%eax		/ %eax = -1
+	lock
+	  xaddl	%eax, (%rdi)	/ %eax = old value, (%rdi) = new value
+	decl	%eax		/ return new value
+	ret
+	SET_SIZE(atomic_dec_uint_nv)
+	SET_SIZE(atomic_dec_32_nv)
+
+	ENTRY(atomic_dec_64_nv)
+	ALTENTRY(atomic_dec_ulong_nv)
+	xorq	%rax, %rax	/ %rax = 0
+	decq	%rax		/ %rax = -1
+	lock
+	  xaddq	%rax, (%rdi)	/ %rax = old value, (%rdi) = new value
+	decq	%rax		/ return new value
+	ret
+	SET_SIZE(atomic_dec_ulong_nv)
+	SET_SIZE(atomic_dec_64_nv)
+
+	ENTRY(atomic_add_8)
+	ALTENTRY(atomic_add_char)
+	lock
+	addb	%sil, (%rdi)
+	ret
+	SET_SIZE(atomic_add_char)
+	SET_SIZE(atomic_add_8)
+
+	ENTRY(atomic_add_16)
+	ALTENTRY(atomic_add_short)
+	lock
+	addw	%si, (%rdi)
+	ret
+	SET_SIZE(atomic_add_short)
+	SET_SIZE(atomic_add_16)
+
+	ENTRY(atomic_add_32)
+	ALTENTRY(atomic_add_int)
+	lock
+	addl	%esi, (%rdi)
+	ret
+	SET_SIZE(atomic_add_int)
+	SET_SIZE(atomic_add_32)
+
+	ENTRY(atomic_add_64)
+	ALTENTRY(atomic_add_ptr)
+	ALTENTRY(atomic_add_long)
+	lock
+	addq	%rsi, (%rdi)
+	ret
+	SET_SIZE(atomic_add_long)
+	SET_SIZE(atomic_add_ptr)
+	SET_SIZE(atomic_add_64)
+
+	ENTRY(atomic_or_8)
+	ALTENTRY(atomic_or_uchar)
+	lock
+	orb	%sil, (%rdi)
+	ret
+	SET_SIZE(atomic_or_uchar)
+	SET_SIZE(atomic_or_8)
+
+	ENTRY(atomic_or_16)
+	ALTENTRY(atomic_or_ushort)
+	lock
+	orw	%si, (%rdi)
+	ret
+	SET_SIZE(atomic_or_ushort)
+	SET_SIZE(atomic_or_16)
+
+	ENTRY(atomic_or_32)
+	ALTENTRY(atomic_or_uint)
+	lock
+	orl	%esi, (%rdi)
+	ret
+	SET_SIZE(atomic_or_uint)
+	SET_SIZE(atomic_or_32)
+
+	ENTRY(atomic_or_64)
+	ALTENTRY(atomic_or_ulong)
+	lock
+	orq	%rsi, (%rdi)
+	ret
+	SET_SIZE(atomic_or_ulong)
+	SET_SIZE(atomic_or_64)
+
+	ENTRY(atomic_and_8)
+	ALTENTRY(atomic_and_uchar)
+	lock
+	andb	%sil, (%rdi)
+	ret
+	SET_SIZE(atomic_and_uchar)
+	SET_SIZE(atomic_and_8)
+
+	ENTRY(atomic_and_16)
+	ALTENTRY(atomic_and_ushort)
+	lock
+	andw	%si, (%rdi)
+	ret
+	SET_SIZE(atomic_and_ushort)
+	SET_SIZE(atomic_and_16)
+
+	ENTRY(atomic_and_32)
+	ALTENTRY(atomic_and_uint)
+	lock
+	andl	%esi, (%rdi)
+	ret
+	SET_SIZE(atomic_and_uint)
+	SET_SIZE(atomic_and_32)
+
+	ENTRY(atomic_and_64)
+	ALTENTRY(atomic_and_ulong)
+	lock
+	andq	%rsi, (%rdi)
+	ret
+	SET_SIZE(atomic_and_ulong)
+	SET_SIZE(atomic_and_64)
+
+	ENTRY(atomic_add_8_nv)
+	ALTENTRY(atomic_add_char_nv)
+	movzbl	%sil, %eax		/ %al = delta addend, clear upper bits
+	lock
+	  xaddb	%sil, (%rdi)		/ %sil = old value, (%rdi) = sum
+	addb	%sil, %al		/ new value = original value + delta
+	ret
+	SET_SIZE(atomic_add_char_nv)
+	SET_SIZE(atomic_add_8_nv)
+
+	ENTRY(atomic_add_16_nv)
+	ALTENTRY(atomic_add_short_nv)
+	movzwl	%si, %eax		/ %ax = delta addend, clean upper bits
+	lock
+	  xaddw	%si, (%rdi)		/ %si = old value, (%rdi) = sum
+	addw	%si, %ax		/ new value = original value + delta
+	ret
+	SET_SIZE(atomic_add_short_nv)
+	SET_SIZE(atomic_add_16_nv)
+
+	ENTRY(atomic_add_32_nv)
+	ALTENTRY(atomic_add_int_nv)
+	mov	%esi, %eax		/ %eax = delta addend
+	lock
+	  xaddl	%esi, (%rdi)		/ %esi = old value, (%rdi) = sum
+	add	%esi, %eax		/ new value = original value + delta
+	ret
+	SET_SIZE(atomic_add_int_nv)
+	SET_SIZE(atomic_add_32_nv)
+
+	ENTRY(atomic_add_64_nv)
+	ALTENTRY(atomic_add_ptr_nv)
+	ALTENTRY(atomic_add_long_nv)
+	mov	%rsi, %rax		/ %rax = delta addend
+	lock
+	  xaddq	%rsi, (%rdi)		/ %rsi = old value, (%rdi) = sum
+	addq	%rsi, %rax		/ new value = original value + delta
+	ret
+	SET_SIZE(atomic_add_long_nv)
+	SET_SIZE(atomic_add_ptr_nv)
+	SET_SIZE(atomic_add_64_nv)
+
+	ENTRY(atomic_and_8_nv)
+	ALTENTRY(atomic_and_uchar_nv)
+	movb	(%rdi), %al	/ %al = old value
+1:
+	movb	%sil, %cl
+	andb	%al, %cl	/ %cl = new value
+	lock
+	cmpxchgb %cl, (%rdi)	/ try to stick it in
+	jne	1b
+	movzbl	%cl, %eax	/ return new value
+	ret
+	SET_SIZE(atomic_and_uchar_nv)
+	SET_SIZE(atomic_and_8_nv)
+
+	ENTRY(atomic_and_16_nv)
+	ALTENTRY(atomic_and_ushort_nv)
+	movw	(%rdi), %ax	/ %ax = old value
+1:
+	movw	%si, %cx
+	andw	%ax, %cx	/ %cx = new value
+	lock
+	cmpxchgw %cx, (%rdi)	/ try to stick it in
+	jne	1b
+	movzwl	%cx, %eax	/ return new value
+	ret
+	SET_SIZE(atomic_and_ushort_nv)
+	SET_SIZE(atomic_and_16_nv)
+
+	ENTRY(atomic_and_32_nv)
+	ALTENTRY(atomic_and_uint_nv)
+	movl	(%rdi), %eax
+1:
+	movl	%esi, %ecx
+	andl	%eax, %ecx
+	lock
+	cmpxchgl %ecx, (%rdi)
+	jne	1b
+	movl	%ecx, %eax
+	ret
+	SET_SIZE(atomic_and_uint_nv)
+	SET_SIZE(atomic_and_32_nv)
+
+	ENTRY(atomic_and_64_nv)
+	ALTENTRY(atomic_and_ulong_nv)
+	movq	(%rdi), %rax
+1:
+	movq	%rsi, %rcx
+	andq	%rax, %rcx
+	lock
+	cmpxchgq %rcx, (%rdi)
+	jne	1b
+	movq	%rcx, %rax
+	ret
+	SET_SIZE(atomic_and_ulong_nv)
+	SET_SIZE(atomic_and_64_nv)
+
+	ENTRY(atomic_or_8_nv)
+	ALTENTRY(atomic_or_uchar_nv)
+	movb	(%rdi), %al	/ %al = old value
+1:
+	movb	%sil, %cl
+	orb	%al, %cl	/ %cl = new value
+	lock
+	cmpxchgb %cl, (%rdi)	/ try to stick it in
+	jne	1b
+	movzbl	%cl, %eax	/ return new value
+	ret
+	SET_SIZE(atomic_or_uchar_nv)
+	SET_SIZE(atomic_or_8_nv)
+
+	ENTRY(atomic_or_16_nv)
+	ALTENTRY(atomic_or_ushort_nv)
+	movw	(%rdi), %ax	/ %ax = old value
+1:
+	movw	%si, %cx
+	orw	%ax, %cx	/ %cx = new value
+	lock
+	cmpxchgw %cx, (%rdi)	/ try to stick it in
+	jne	1b
+	movzwl	%cx, %eax	/ return new value
+	ret
+	SET_SIZE(atomic_or_ushort_nv)
+	SET_SIZE(atomic_or_16_nv)
+
+	ENTRY(atomic_or_32_nv)
+	ALTENTRY(atomic_or_uint_nv)
+	movl	(%rdi), %eax
+1:
+	movl	%esi, %ecx
+	orl	%eax, %ecx
+	lock
+	cmpxchgl %ecx, (%rdi)
+	jne	1b
+	movl	%ecx, %eax
+	ret
+	SET_SIZE(atomic_or_uint_nv)
+	SET_SIZE(atomic_or_32_nv)
+
+	ENTRY(atomic_or_64_nv)
+	ALTENTRY(atomic_or_ulong_nv)
+	movq	(%rdi), %rax
+1:
+	movq	%rsi, %rcx
+	orq	%rax, %rcx
+	lock
+	cmpxchgq %rcx, (%rdi)
+	jne	1b
+	movq	%rcx, %rax
+	ret
+	SET_SIZE(atomic_or_ulong_nv)
+	SET_SIZE(atomic_or_64_nv)
+
+	ENTRY(atomic_cas_8)
+	ALTENTRY(atomic_cas_uchar)
+	movzbl	%sil, %eax
+	lock
+	cmpxchgb %dl, (%rdi)
+	ret
+	SET_SIZE(atomic_cas_uchar)
+	SET_SIZE(atomic_cas_8)
+
+	ENTRY(atomic_cas_16)
+	ALTENTRY(atomic_cas_ushort)
+	movzwl	%si, %eax
+	lock
+	cmpxchgw %dx, (%rdi)
+	ret
+	SET_SIZE(atomic_cas_ushort)
+	SET_SIZE(atomic_cas_16)
+
+	ENTRY(atomic_cas_32)
+	ALTENTRY(atomic_cas_uint)
+	movl	%esi, %eax
+	lock
+	cmpxchgl %edx, (%rdi)
+	ret
+	SET_SIZE(atomic_cas_uint)
+	SET_SIZE(atomic_cas_32)
+
+	ENTRY(atomic_cas_64)
+	ALTENTRY(atomic_cas_ulong)
+	ALTENTRY(atomic_cas_ptr)
+	movq	%rsi, %rax
+	lock
+	cmpxchgq %rdx, (%rdi)
+	ret
+	SET_SIZE(atomic_cas_ptr)
+	SET_SIZE(atomic_cas_ulong)
+	SET_SIZE(atomic_cas_64)
+
+	ENTRY(atomic_swap_8)
+	ALTENTRY(atomic_swap_uchar)
+	movzbl	%sil, %eax
+	lock
+	xchgb %al, (%rdi)
+	ret
+	SET_SIZE(atomic_swap_uchar)
+	SET_SIZE(atomic_swap_8)
+
+	ENTRY(atomic_swap_16)
+	ALTENTRY(atomic_swap_ushort)
+	movzwl	%si, %eax
+	lock
+	xchgw %ax, (%rdi)
+	ret
+	SET_SIZE(atomic_swap_ushort)
+	SET_SIZE(atomic_swap_16)
+
+	ENTRY(atomic_swap_32)
+	ALTENTRY(atomic_swap_uint)
+	movl	%esi, %eax
+	lock
+	xchgl %eax, (%rdi)
+	ret
+	SET_SIZE(atomic_swap_uint)
+	SET_SIZE(atomic_swap_32)
+
+	ENTRY(atomic_swap_64)
+	ALTENTRY(atomic_swap_ulong)
+	ALTENTRY(atomic_swap_ptr)
+	movq	%rsi, %rax
+	lock
+	xchgq %rax, (%rdi)
+	ret
+	SET_SIZE(atomic_swap_ptr)
+	SET_SIZE(atomic_swap_ulong)
+	SET_SIZE(atomic_swap_64)
+
+	ENTRY(atomic_set_long_excl)
+	xorl	%eax, %eax
+	lock
+	btsq	%rsi, (%rdi)
+	jnc	1f
+	decl	%eax			/ return -1
+1:
+	ret
+	SET_SIZE(atomic_set_long_excl)
+
+	ENTRY(atomic_clear_long_excl)
+	xorl	%eax, %eax
+	lock
+	btrq	%rsi, (%rdi)
+	jc	1f
+	decl	%eax			/ return -1
+1:
+	ret
+	SET_SIZE(atomic_clear_long_excl)
+
+#if !defined(_KERNEL)
+
+	/*
+	 * NOTE: membar_enter, and membar_exit are identical routines. 
+	 * We define them separately, instead of using an ALTENTRY
+	 * definitions to alias them together, so that DTrace and
+	 * debuggers will see a unique address for them, allowing 
+	 * more accurate tracing.
+	*/
+
+	ENTRY(membar_enter)
+	mfence
+	ret
+	SET_SIZE(membar_enter)
+
+	ENTRY(membar_exit)
+	mfence
+	ret
+	SET_SIZE(membar_exit)
+
+	ENTRY(membar_producer)
+	sfence
+	ret
+	SET_SIZE(membar_producer)
+
+	ENTRY(membar_consumer)
+	lfence
+	ret
+	SET_SIZE(membar_consumer)
+
+#endif	/* !_KERNEL */
diff --git a/common/atomic/i386/atomic.s b/common/atomic/i386/atomic.s
new file mode 100644
index 000000000000..4fa525ba20af
--- /dev/null
+++ b/common/atomic/i386/atomic.s
@@ -0,0 +1,720 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+	.file	"atomic.s"
+
+#include <sys/asm_linkage.h>
+
+#if defined(_KERNEL)
+	/*
+	 * Legacy kernel interfaces; they will go away (eventually).
+	 */
+	ANSI_PRAGMA_WEAK2(cas8,atomic_cas_8,function)
+	ANSI_PRAGMA_WEAK2(cas32,atomic_cas_32,function)
+	ANSI_PRAGMA_WEAK2(cas64,atomic_cas_64,function)
+	ANSI_PRAGMA_WEAK2(caslong,atomic_cas_ulong,function)
+	ANSI_PRAGMA_WEAK2(casptr,atomic_cas_ptr,function)
+	ANSI_PRAGMA_WEAK2(atomic_and_long,atomic_and_ulong,function)
+	ANSI_PRAGMA_WEAK2(atomic_or_long,atomic_or_ulong,function)
+#endif
+
+	ENTRY(atomic_inc_8)
+	ALTENTRY(atomic_inc_uchar)
+	movl	4(%esp), %eax
+	lock
+	incb	(%eax)
+	ret
+	SET_SIZE(atomic_inc_uchar)
+	SET_SIZE(atomic_inc_8)
+
+	ENTRY(atomic_inc_16)
+	ALTENTRY(atomic_inc_ushort)
+	movl	4(%esp), %eax
+	lock
+	incw	(%eax)
+	ret
+	SET_SIZE(atomic_inc_ushort)
+	SET_SIZE(atomic_inc_16)
+
+	ENTRY(atomic_inc_32)
+	ALTENTRY(atomic_inc_uint)
+	ALTENTRY(atomic_inc_ulong)
+	movl	4(%esp), %eax
+	lock
+	incl	(%eax)
+	ret
+	SET_SIZE(atomic_inc_ulong)
+	SET_SIZE(atomic_inc_uint)
+	SET_SIZE(atomic_inc_32)
+
+	ENTRY(atomic_inc_8_nv)
+	ALTENTRY(atomic_inc_uchar_nv)
+	movl	4(%esp), %edx	/ %edx = target address
+	xorl	%eax, %eax	/ clear upper bits of %eax
+	incb	%al		/ %al = 1
+	lock
+	  xaddb	%al, (%edx)	/ %al = old value, inc (%edx)
+	incb	%al	/ return new value
+	ret
+	SET_SIZE(atomic_inc_uchar_nv)
+	SET_SIZE(atomic_inc_8_nv)
+
+	ENTRY(atomic_inc_16_nv)
+	ALTENTRY(atomic_inc_ushort_nv)
+	movl	4(%esp), %edx	/ %edx = target address
+	xorl	%eax, %eax	/ clear upper bits of %eax
+	incw	%ax		/ %ax = 1
+	lock
+	  xaddw	%ax, (%edx)	/ %ax = old value, inc (%edx)
+	incw	%ax		/ return new value
+	ret
+	SET_SIZE(atomic_inc_ushort_nv)
+	SET_SIZE(atomic_inc_16_nv)
+
+	ENTRY(atomic_inc_32_nv)
+	ALTENTRY(atomic_inc_uint_nv)
+	ALTENTRY(atomic_inc_ulong_nv)
+	movl	4(%esp), %edx	/ %edx = target address
+	xorl	%eax, %eax	/ %eax = 0
+	incl	%eax		/ %eax = 1
+	lock
+	  xaddl	%eax, (%edx)	/ %eax = old value, inc (%edx)
+	incl	%eax		/ return new value
+	ret
+	SET_SIZE(atomic_inc_ulong_nv)
+	SET_SIZE(atomic_inc_uint_nv)
+	SET_SIZE(atomic_inc_32_nv)
+
+	/*
+	 * NOTE: If atomic_inc_64 and atomic_inc_64_nv are ever
+	 * separated, you need to also edit the libc i386 platform
+	 * specific mapfile and remove the NODYNSORT attribute
+	 * from atomic_inc_64_nv.
+	 */
+	ENTRY(atomic_inc_64)
+	ALTENTRY(atomic_inc_64_nv)
+	pushl	%edi
+	pushl	%ebx
+	movl	12(%esp), %edi	/ %edi = target address
+	movl	(%edi), %eax
+	movl	4(%edi), %edx	/ %edx:%eax = old value
+1:
+	xorl	%ebx, %ebx
+	xorl	%ecx, %ecx
+	incl	%ebx		/ %ecx:%ebx = 1
+	addl	%eax, %ebx
+	adcl	%edx, %ecx	/ add in the carry from inc
+	lock
+	cmpxchg8b (%edi)	/ try to stick it in
+	jne	1b
+	movl	%ebx, %eax
+	movl	%ecx, %edx	/ return new value
+	popl	%ebx
+	popl	%edi
+	ret
+	SET_SIZE(atomic_inc_64_nv)
+	SET_SIZE(atomic_inc_64)
+
+	ENTRY(atomic_dec_8)
+	ALTENTRY(atomic_dec_uchar)
+	movl	4(%esp), %eax
+	lock
+	decb	(%eax)
+	ret
+	SET_SIZE(atomic_dec_uchar)
+	SET_SIZE(atomic_dec_8)
+
+	ENTRY(atomic_dec_16)
+	ALTENTRY(atomic_dec_ushort)
+	movl	4(%esp), %eax
+	lock
+	decw	(%eax)
+	ret
+	SET_SIZE(atomic_dec_ushort)
+	SET_SIZE(atomic_dec_16)
+
+	ENTRY(atomic_dec_32)
+	ALTENTRY(atomic_dec_uint)
+	ALTENTRY(atomic_dec_ulong)
+	movl	4(%esp), %eax
+	lock
+	decl	(%eax)
+	ret
+	SET_SIZE(atomic_dec_ulong)
+	SET_SIZE(atomic_dec_uint)
+	SET_SIZE(atomic_dec_32)
+
+	ENTRY(atomic_dec_8_nv)
+	ALTENTRY(atomic_dec_uchar_nv)
+	movl	4(%esp), %edx	/ %edx = target address
+	xorl	%eax, %eax	/ zero upper bits of %eax
+	decb	%al		/ %al = -1
+	lock
+	  xaddb	%al, (%edx)	/ %al = old value, dec (%edx)
+	decb	%al		/ return new value
+	ret
+	SET_SIZE(atomic_dec_uchar_nv)
+	SET_SIZE(atomic_dec_8_nv)
+
+	ENTRY(atomic_dec_16_nv)
+	ALTENTRY(atomic_dec_ushort_nv)
+	movl	4(%esp), %edx	/ %edx = target address
+	xorl	%eax, %eax	/ zero upper bits of %eax
+	decw	%ax		/ %ax = -1
+	lock
+	  xaddw	%ax, (%edx)	/ %ax = old value, dec (%edx)
+	decw	%ax		/ return new value
+	ret
+	SET_SIZE(atomic_dec_ushort_nv)
+	SET_SIZE(atomic_dec_16_nv)
+
+	ENTRY(atomic_dec_32_nv)
+	ALTENTRY(atomic_dec_uint_nv)
+	ALTENTRY(atomic_dec_ulong_nv)
+	movl	4(%esp), %edx	/ %edx = target address
+	xorl	%eax, %eax	/ %eax = 0
+	decl	%eax		/ %eax = -1
+	lock
+	  xaddl	%eax, (%edx)	/ %eax = old value, dec (%edx)
+	decl	%eax		/ return new value
+	ret
+	SET_SIZE(atomic_dec_ulong_nv)
+	SET_SIZE(atomic_dec_uint_nv)
+	SET_SIZE(atomic_dec_32_nv)
+
+	/*
+	 * NOTE: If atomic_dec_64 and atomic_dec_64_nv are ever
+	 * separated, it is important to edit the libc i386 platform
+	 * specific mapfile and remove the NODYNSORT attribute
+	 * from atomic_dec_64_nv.
+	 */
+	ENTRY(atomic_dec_64)
+	ALTENTRY(atomic_dec_64_nv)
+	pushl	%edi
+	pushl	%ebx
+	movl	12(%esp), %edi	/ %edi = target address
+	movl	(%edi), %eax
+	movl	4(%edi), %edx	/ %edx:%eax = old value
+1:
+	xorl	%ebx, %ebx
+	xorl	%ecx, %ecx
+	not	%ecx
+	not	%ebx		/ %ecx:%ebx = -1
+	addl	%eax, %ebx
+	adcl	%edx, %ecx	/ add in the carry from inc
+	lock
+	cmpxchg8b (%edi)	/ try to stick it in
+	jne	1b
+	movl	%ebx, %eax
+	movl	%ecx, %edx	/ return new value
+	popl	%ebx
+	popl	%edi
+	ret
+	SET_SIZE(atomic_dec_64_nv)
+	SET_SIZE(atomic_dec_64)
+
+	ENTRY(atomic_add_8)
+	ALTENTRY(atomic_add_char)
+	movl	4(%esp), %eax
+	movl	8(%esp), %ecx
+	lock
+	addb	%cl, (%eax)
+	ret
+	SET_SIZE(atomic_add_char)
+	SET_SIZE(atomic_add_8)
+
+	ENTRY(atomic_add_16)
+	ALTENTRY(atomic_add_short)
+	movl	4(%esp), %eax
+	movl	8(%esp), %ecx
+	lock
+	addw	%cx, (%eax)
+	ret
+	SET_SIZE(atomic_add_short)
+	SET_SIZE(atomic_add_16)
+
+	ENTRY(atomic_add_32)
+	ALTENTRY(atomic_add_int)
+	ALTENTRY(atomic_add_ptr)
+	ALTENTRY(atomic_add_long)
+	movl	4(%esp), %eax
+	movl	8(%esp), %ecx
+	lock
+	addl	%ecx, (%eax)
+	ret
+	SET_SIZE(atomic_add_long)
+	SET_SIZE(atomic_add_ptr)
+	SET_SIZE(atomic_add_int)
+	SET_SIZE(atomic_add_32)
+
+	ENTRY(atomic_or_8)
+	ALTENTRY(atomic_or_uchar)
+	movl	4(%esp), %eax
+	movb	8(%esp), %cl
+	lock
+	orb	%cl, (%eax)
+	ret
+	SET_SIZE(atomic_or_uchar)
+	SET_SIZE(atomic_or_8)
+
+	ENTRY(atomic_or_16)
+	ALTENTRY(atomic_or_ushort)
+	movl	4(%esp), %eax
+	movw	8(%esp), %cx
+	lock
+	orw	%cx, (%eax)
+	ret
+	SET_SIZE(atomic_or_ushort)
+	SET_SIZE(atomic_or_16)
+
+	ENTRY(atomic_or_32)
+	ALTENTRY(atomic_or_uint)
+	ALTENTRY(atomic_or_ulong)
+	movl	4(%esp), %eax
+	movl	8(%esp), %ecx
+	lock
+	orl	%ecx, (%eax)
+	ret
+	SET_SIZE(atomic_or_ulong)
+	SET_SIZE(atomic_or_uint)
+	SET_SIZE(atomic_or_32)
+
+	ENTRY(atomic_and_8)
+	ALTENTRY(atomic_and_uchar)
+	movl	4(%esp), %eax
+	movb	8(%esp), %cl
+	lock
+	andb	%cl, (%eax)
+	ret
+	SET_SIZE(atomic_and_uchar)
+	SET_SIZE(atomic_and_8)
+
+	ENTRY(atomic_and_16)
+	ALTENTRY(atomic_and_ushort)
+	movl	4(%esp), %eax
+	movw	8(%esp), %cx
+	lock
+	andw	%cx, (%eax)
+	ret
+	SET_SIZE(atomic_and_ushort)
+	SET_SIZE(atomic_and_16)
+
+	ENTRY(atomic_and_32)
+	ALTENTRY(atomic_and_uint)
+	ALTENTRY(atomic_and_ulong)
+	movl	4(%esp), %eax
+	movl	8(%esp), %ecx
+	lock
+	andl	%ecx, (%eax)
+	ret
+	SET_SIZE(atomic_and_ulong)
+	SET_SIZE(atomic_and_uint)
+	SET_SIZE(atomic_and_32)
+
+	ENTRY(atomic_add_8_nv)
+	ALTENTRY(atomic_add_char_nv)
+	movl	4(%esp), %edx	/ %edx = target address
+	movb	8(%esp), %cl	/ %cl = delta
+	movzbl	%cl, %eax	/ %al = delta, zero extended
+	lock
+	  xaddb	%cl, (%edx)	/ %cl = old value, (%edx) = sum
+	addb	%cl, %al	/ return old value plus delta
+	ret
+	SET_SIZE(atomic_add_char_nv)
+	SET_SIZE(atomic_add_8_nv)
+
+	ENTRY(atomic_add_16_nv)
+	ALTENTRY(atomic_add_short_nv)
+	movl	4(%esp), %edx	/ %edx = target address
+	movw	8(%esp), %cx	/ %cx = delta
+	movzwl	%cx, %eax	/ %ax = delta, zero extended
+	lock
+	  xaddw	%cx, (%edx)	/ %cx = old value, (%edx) = sum
+	addw	%cx, %ax	/ return old value plus delta
+	ret
+	SET_SIZE(atomic_add_short_nv)
+	SET_SIZE(atomic_add_16_nv)
+
+	ENTRY(atomic_add_32_nv)
+	ALTENTRY(atomic_add_int_nv)
+	ALTENTRY(atomic_add_ptr_nv)
+	ALTENTRY(atomic_add_long_nv)
+	movl	4(%esp), %edx	/ %edx = target address
+	movl	8(%esp), %eax	/ %eax = delta
+	movl	%eax, %ecx	/ %ecx = delta
+	lock
+	  xaddl	%eax, (%edx)	/ %eax = old value, (%edx) = sum
+	addl	%ecx, %eax	/ return old value plus delta
+	ret
+	SET_SIZE(atomic_add_long_nv)
+	SET_SIZE(atomic_add_ptr_nv)
+	SET_SIZE(atomic_add_int_nv)
+	SET_SIZE(atomic_add_32_nv)
+
+	/*
+	 * NOTE: If atomic_add_64 and atomic_add_64_nv are ever
+	 * separated, it is important to edit the libc i386 platform
+	 * specific mapfile and remove the NODYNSORT attribute
+	 * from atomic_add_64_nv.
+	 */
+	ENTRY(atomic_add_64)
+	ALTENTRY(atomic_add_64_nv)
+	pushl	%edi
+	pushl	%ebx
+	movl	12(%esp), %edi	/ %edi = target address
+	movl	(%edi), %eax
+	movl	4(%edi), %edx	/ %edx:%eax = old value
+1:
+	movl	16(%esp), %ebx
+	movl	20(%esp), %ecx	/ %ecx:%ebx = delta
+	addl	%eax, %ebx
+	adcl	%edx, %ecx	/ %ecx:%ebx = new value
+	lock
+	cmpxchg8b (%edi)	/ try to stick it in
+	jne	1b
+	movl	%ebx, %eax
+	movl	%ecx, %edx	/ return new value
+	popl	%ebx
+	popl	%edi
+	ret
+	SET_SIZE(atomic_add_64_nv)
+	SET_SIZE(atomic_add_64)
+
+	ENTRY(atomic_or_8_nv)
+	ALTENTRY(atomic_or_uchar_nv)
+	movl	4(%esp), %edx	/ %edx = target address
+	movb	(%edx), %al	/ %al = old value
+1:
+	movl	8(%esp), %ecx	/ %ecx = delta
+	orb	%al, %cl	/ %cl = new value
+	lock
+	cmpxchgb %cl, (%edx)	/ try to stick it in
+	jne	1b
+	movzbl	%cl, %eax	/ return new value
+	ret
+	SET_SIZE(atomic_or_uchar_nv)
+	SET_SIZE(atomic_or_8_nv)
+
+	ENTRY(atomic_or_16_nv)
+	ALTENTRY(atomic_or_ushort_nv)
+	movl	4(%esp), %edx	/ %edx = target address
+	movw	(%edx), %ax	/ %ax = old value
+1:
+	movl	8(%esp), %ecx	/ %ecx = delta
+	orw	%ax, %cx	/ %cx = new value
+	lock
+	cmpxchgw %cx, (%edx)	/ try to stick it in
+	jne	1b
+	movzwl	%cx, %eax	/ return new value
+	ret
+	SET_SIZE(atomic_or_ushort_nv)
+	SET_SIZE(atomic_or_16_nv)
+
+	ENTRY(atomic_or_32_nv)
+	ALTENTRY(atomic_or_uint_nv)
+	ALTENTRY(atomic_or_ulong_nv)
+	movl	4(%esp), %edx	/ %edx = target address
+	movl	(%edx), %eax	/ %eax = old value
+1:
+	movl	8(%esp), %ecx	/ %ecx = delta
+	orl	%eax, %ecx	/ %ecx = new value
+	lock
+	cmpxchgl %ecx, (%edx)	/ try to stick it in
+	jne	1b
+	movl	%ecx, %eax	/ return new value
+	ret
+	SET_SIZE(atomic_or_ulong_nv)
+	SET_SIZE(atomic_or_uint_nv)
+	SET_SIZE(atomic_or_32_nv)
+
+	/*
+	 * NOTE: If atomic_or_64 and atomic_or_64_nv are ever
+	 * separated, it is important to edit the libc i386 platform
+	 * specific mapfile and remove the NODYNSORT attribute
+	 * from atomic_or_64_nv.
+	 */
+	ENTRY(atomic_or_64)
+	ALTENTRY(atomic_or_64_nv)
+	pushl	%edi
+	pushl	%ebx
+	movl	12(%esp), %edi	/ %edi = target address
+	movl	(%edi), %eax
+	movl	4(%edi), %edx	/ %edx:%eax = old value
+1:
+	movl	16(%esp), %ebx
+	movl	20(%esp), %ecx	/ %ecx:%ebx = delta
+	orl	%eax, %ebx
+	orl	%edx, %ecx	/ %ecx:%ebx = new value
+	lock
+	cmpxchg8b (%edi)	/ try to stick it in
+	jne	1b
+	movl	%ebx, %eax
+	movl	%ecx, %edx	/ return new value
+	popl	%ebx
+	popl	%edi
+	ret
+	SET_SIZE(atomic_or_64_nv)
+	SET_SIZE(atomic_or_64)
+
+	ENTRY(atomic_and_8_nv)
+	ALTENTRY(atomic_and_uchar_nv)
+	movl	4(%esp), %edx	/ %edx = target address
+	movb	(%edx), %al	/ %al = old value
+1:
+	movl	8(%esp), %ecx	/ %ecx = delta
+	andb	%al, %cl	/ %cl = new value
+	lock
+	cmpxchgb %cl, (%edx)	/ try to stick it in
+	jne	1b
+	movzbl	%cl, %eax	/ return new value
+	ret
+	SET_SIZE(atomic_and_uchar_nv)
+	SET_SIZE(atomic_and_8_nv)
+
+	ENTRY(atomic_and_16_nv)
+	ALTENTRY(atomic_and_ushort_nv)
+	movl	4(%esp), %edx	/ %edx = target address
+	movw	(%edx), %ax	/ %ax = old value
+1:
+	movl	8(%esp), %ecx	/ %ecx = delta
+	andw	%ax, %cx	/ %cx = new value
+	lock
+	cmpxchgw %cx, (%edx)	/ try to stick it in
+	jne	1b
+	movzwl	%cx, %eax	/ return new value
+	ret
+	SET_SIZE(atomic_and_ushort_nv)
+	SET_SIZE(atomic_and_16_nv)
+
+	ENTRY(atomic_and_32_nv)
+	ALTENTRY(atomic_and_uint_nv)
+	ALTENTRY(atomic_and_ulong_nv)
+	movl	4(%esp), %edx	/ %edx = target address
+	movl	(%edx), %eax	/ %eax = old value
+1:
+	movl	8(%esp), %ecx	/ %ecx = delta
+	andl	%eax, %ecx	/ %ecx = new value
+	lock
+	cmpxchgl %ecx, (%edx)	/ try to stick it in
+	jne	1b
+	movl	%ecx, %eax	/ return new value
+	ret
+	SET_SIZE(atomic_and_ulong_nv)
+	SET_SIZE(atomic_and_uint_nv)
+	SET_SIZE(atomic_and_32_nv)
+
+	/*
+	 * NOTE: If atomic_and_64 and atomic_and_64_nv are ever
+	 * separated, it is important to edit the libc i386 platform
+	 * specific mapfile and remove the NODYNSORT attribute
+	 * from atomic_and_64_nv.
+	 */
+	ENTRY(atomic_and_64)
+	ALTENTRY(atomic_and_64_nv)
+	pushl	%edi
+	pushl	%ebx
+	movl	12(%esp), %edi	/ %edi = target address
+	movl	(%edi), %eax
+	movl	4(%edi), %edx	/ %edx:%eax = old value
+1:
+	movl	16(%esp), %ebx
+	movl	20(%esp), %ecx	/ %ecx:%ebx = delta
+	andl	%eax, %ebx
+	andl	%edx, %ecx	/ %ecx:%ebx = new value
+	lock
+	cmpxchg8b (%edi)	/ try to stick it in
+	jne	1b
+	movl	%ebx, %eax
+	movl	%ecx, %edx	/ return new value
+	popl	%ebx
+	popl	%edi
+	ret
+	SET_SIZE(atomic_and_64_nv)
+	SET_SIZE(atomic_and_64)
+
+	ENTRY(atomic_cas_8)
+	ALTENTRY(atomic_cas_uchar)
+	movl	4(%esp), %edx
+	movzbl	8(%esp), %eax
+	movb	12(%esp), %cl
+	lock
+	cmpxchgb %cl, (%edx)
+	ret
+	SET_SIZE(atomic_cas_uchar)
+	SET_SIZE(atomic_cas_8)
+
+	ENTRY(atomic_cas_16)
+	ALTENTRY(atomic_cas_ushort)
+	movl	4(%esp), %edx
+	movzwl	8(%esp), %eax
+	movw	12(%esp), %cx
+	lock
+	cmpxchgw %cx, (%edx)
+	ret
+	SET_SIZE(atomic_cas_ushort)
+	SET_SIZE(atomic_cas_16)
+
+	ENTRY(atomic_cas_32)
+	ALTENTRY(atomic_cas_uint)
+	ALTENTRY(atomic_cas_ulong)
+	ALTENTRY(atomic_cas_ptr)
+	movl	4(%esp), %edx
+	movl	8(%esp), %eax
+	movl	12(%esp), %ecx
+	lock
+	cmpxchgl %ecx, (%edx)
+	ret
+	SET_SIZE(atomic_cas_ptr)
+	SET_SIZE(atomic_cas_ulong)
+	SET_SIZE(atomic_cas_uint)
+	SET_SIZE(atomic_cas_32)
+
+	ENTRY(atomic_cas_64)
+	pushl	%ebx
+	pushl	%esi
+	movl	12(%esp), %esi
+	movl	16(%esp), %eax
+	movl	20(%esp), %edx
+	movl	24(%esp), %ebx
+	movl	28(%esp), %ecx
+	lock
+	cmpxchg8b (%esi)
+	popl	%esi
+	popl	%ebx
+	ret
+	SET_SIZE(atomic_cas_64)
+
+	ENTRY(atomic_swap_8)
+	ALTENTRY(atomic_swap_uchar)
+	movl	4(%esp), %edx
+	movzbl	8(%esp), %eax
+	lock
+	xchgb	%al, (%edx)
+	ret
+	SET_SIZE(atomic_swap_uchar)
+	SET_SIZE(atomic_swap_8)
+
+	ENTRY(atomic_swap_16)
+	ALTENTRY(atomic_swap_ushort)
+	movl	4(%esp), %edx
+	movzwl	8(%esp), %eax
+	lock
+	xchgw	%ax, (%edx)
+	ret
+	SET_SIZE(atomic_swap_ushort)
+	SET_SIZE(atomic_swap_16)
+
+	ENTRY(atomic_swap_32)
+	ALTENTRY(atomic_swap_uint)
+	ALTENTRY(atomic_swap_ptr)
+	ALTENTRY(atomic_swap_ulong)
+	movl	4(%esp), %edx
+	movl	8(%esp), %eax
+	lock
+	xchgl	%eax, (%edx)
+	ret
+	SET_SIZE(atomic_swap_ulong)
+	SET_SIZE(atomic_swap_ptr)
+	SET_SIZE(atomic_swap_uint)
+	SET_SIZE(atomic_swap_32)
+
+	ENTRY(atomic_swap_64)
+	pushl	%esi
+	pushl	%ebx
+	movl	12(%esp), %esi
+	movl	16(%esp), %ebx
+	movl	20(%esp), %ecx
+	movl	(%esi), %eax
+	movl	4(%esi), %edx	/ %edx:%eax = old value
+1:
+	lock
+	cmpxchg8b (%esi)
+	jne	1b
+	popl	%ebx
+	popl	%esi
+	ret
+	SET_SIZE(atomic_swap_64)
+
+	ENTRY(atomic_set_long_excl)
+	movl	4(%esp), %edx	/ %edx = target address
+	movl	8(%esp), %ecx	/ %ecx = bit id
+	xorl	%eax, %eax
+	lock
+	btsl	%ecx, (%edx)
+	jnc	1f
+	decl	%eax		/ return -1
+1:
+	ret
+	SET_SIZE(atomic_set_long_excl)
+
+	ENTRY(atomic_clear_long_excl)
+	movl	4(%esp), %edx	/ %edx = target address
+	movl	8(%esp), %ecx	/ %ecx = bit id
+	xorl	%eax, %eax
+	lock
+	btrl	%ecx, (%edx)
+	jc	1f
+	decl	%eax		/ return -1
+1:
+	ret
+	SET_SIZE(atomic_clear_long_excl)
+
+#if !defined(_KERNEL)
+
+	/*
+	 * NOTE: membar_enter, membar_exit, membar_producer, and 
+	 * membar_consumer are all identical routines. We define them
+	 * separately, instead of using ALTENTRY definitions to alias them
+	 * together, so that DTrace and debuggers will see a unique address
+	 * for them, allowing more accurate tracing.
+	*/
+
+
+	ENTRY(membar_enter)
+	lock
+	xorl	$0, (%esp)
+	ret
+	SET_SIZE(membar_enter)
+
+	ENTRY(membar_exit)
+	lock
+	xorl	$0, (%esp)
+	ret
+	SET_SIZE(membar_exit)
+
+	ENTRY(membar_producer)
+	lock
+	xorl	$0, (%esp)
+	ret
+	SET_SIZE(membar_producer)
+
+	ENTRY(membar_consumer)
+	lock
+	xorl	$0, (%esp)
+	ret
+	SET_SIZE(membar_consumer)
+
+#endif	/* !_KERNEL */
diff --git a/common/atomic/sparc/atomic.s b/common/atomic/sparc/atomic.s
new file mode 100644
index 000000000000..8aa240efa297
--- /dev/null
+++ b/common/atomic/sparc/atomic.s
@@ -0,0 +1,801 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+	.file	"atomic.s"
+
+#include <sys/asm_linkage.h>
+
+#if defined(_KERNEL)
+	/*
+	 * Legacy kernel interfaces; they will go away (eventually).
+	 */
+	ANSI_PRAGMA_WEAK2(cas8,atomic_cas_8,function)
+	ANSI_PRAGMA_WEAK2(cas32,atomic_cas_32,function)
+	ANSI_PRAGMA_WEAK2(cas64,atomic_cas_64,function)
+	ANSI_PRAGMA_WEAK2(caslong,atomic_cas_ulong,function)
+	ANSI_PRAGMA_WEAK2(casptr,atomic_cas_ptr,function)
+	ANSI_PRAGMA_WEAK2(atomic_and_long,atomic_and_ulong,function)
+	ANSI_PRAGMA_WEAK2(atomic_or_long,atomic_or_ulong,function)
+	ANSI_PRAGMA_WEAK2(swapl,atomic_swap_32,function)
+#endif
+
+	/*
+	 * NOTE: If atomic_inc_8 and atomic_inc_8_nv are ever
+	 * separated, you need to also edit the libc sparc platform
+	 * specific mapfile and remove the NODYNSORT attribute
+	 * from atomic_inc_8_nv.
+	 */
+	ENTRY(atomic_inc_8)
+	ALTENTRY(atomic_inc_8_nv)
+	ALTENTRY(atomic_inc_uchar)
+	ALTENTRY(atomic_inc_uchar_nv)
+	ba	add_8
+	  add	%g0, 1, %o1
+	SET_SIZE(atomic_inc_uchar_nv)
+	SET_SIZE(atomic_inc_uchar)
+	SET_SIZE(atomic_inc_8_nv)
+	SET_SIZE(atomic_inc_8)
+
+	/*
+	 * NOTE: If atomic_dec_8 and atomic_dec_8_nv are ever
+	 * separated, you need to also edit the libc sparc platform
+	 * specific mapfile and remove the NODYNSORT attribute
+	 * from atomic_dec_8_nv.
+	 */
+	ENTRY(atomic_dec_8)
+	ALTENTRY(atomic_dec_8_nv)
+	ALTENTRY(atomic_dec_uchar)
+	ALTENTRY(atomic_dec_uchar_nv)
+	ba	add_8
+	  sub	%g0, 1, %o1
+	SET_SIZE(atomic_dec_uchar_nv)
+	SET_SIZE(atomic_dec_uchar)
+	SET_SIZE(atomic_dec_8_nv)
+	SET_SIZE(atomic_dec_8)
+
+	/*
+	 * NOTE: If atomic_add_8 and atomic_add_8_nv are ever
+	 * separated, you need to also edit the libc sparc platform
+	 * specific mapfile and remove the NODYNSORT attribute
+	 * from atomic_add_8_nv.
+	 */
+	ENTRY(atomic_add_8)
+	ALTENTRY(atomic_add_8_nv)
+	ALTENTRY(atomic_add_char)
+	ALTENTRY(atomic_add_char_nv)
+add_8:
+	and	%o0, 0x3, %o4		! %o4 = byte offset, left-to-right
+	xor	%o4, 0x3, %g1		! %g1 = byte offset, right-to-left
+	sll	%g1, 3, %g1		! %g1 = bit offset, right-to-left
+	set	0xff, %o3		! %o3 = mask
+	sll	%o3, %g1, %o3		! %o3 = shifted to bit offset
+	sll	%o1, %g1, %o1		! %o1 = shifted to bit offset
+	and	%o1, %o3, %o1		! %o1 = single byte value
+	andn	%o0, 0x3, %o0		! %o0 = word address
+	ld	[%o0], %o2		! read old value
+1:
+	add	%o2, %o1, %o5		! add value to the old value
+	and	%o5, %o3, %o5		! clear other bits
+	andn	%o2, %o3, %o4		! clear target bits
+	or	%o4, %o5, %o5		! insert the new value
+	cas	[%o0], %o2, %o5
+	cmp	%o2, %o5
+	bne,a,pn %icc, 1b
+	  mov	%o5, %o2		! %o2 = old value
+	add	%o2, %o1, %o5
+	and	%o5, %o3, %o5
+	retl
+	srl	%o5, %g1, %o0		! %o0 = new value
+	SET_SIZE(atomic_add_char_nv)
+	SET_SIZE(atomic_add_char)
+	SET_SIZE(atomic_add_8_nv)
+	SET_SIZE(atomic_add_8)
+
+	/*
+	 * NOTE: If atomic_inc_16 and atomic_inc_16_nv are ever
+	 * separated, you need to also edit the libc sparc platform
+	 * specific mapfile and remove the NODYNSORT attribute
+	 * from atomic_inc_16_nv.
+	 */
+	ENTRY(atomic_inc_16)
+	ALTENTRY(atomic_inc_16_nv)
+	ALTENTRY(atomic_inc_ushort)
+	ALTENTRY(atomic_inc_ushort_nv)
+	ba	add_16
+	  add	%g0, 1, %o1
+	SET_SIZE(atomic_inc_ushort_nv)
+	SET_SIZE(atomic_inc_ushort)
+	SET_SIZE(atomic_inc_16_nv)
+	SET_SIZE(atomic_inc_16)
+
+	/*
+	 * NOTE: If atomic_dec_16 and atomic_dec_16_nv are ever
+	 * separated, you need to also edit the libc sparc platform
+	 * specific mapfile and remove the NODYNSORT attribute
+	 * from atomic_dec_16_nv.
+	 */
+	ENTRY(atomic_dec_16)
+	ALTENTRY(atomic_dec_16_nv)
+	ALTENTRY(atomic_dec_ushort)
+	ALTENTRY(atomic_dec_ushort_nv)
+	ba	add_16
+	  sub	%g0, 1, %o1
+	SET_SIZE(atomic_dec_ushort_nv)
+	SET_SIZE(atomic_dec_ushort)
+	SET_SIZE(atomic_dec_16_nv)
+	SET_SIZE(atomic_dec_16)
+
+	/*
+	 * NOTE: If atomic_add_16 and atomic_add_16_nv are ever
+	 * separated, you need to also edit the libc sparc platform
+	 * specific mapfile and remove the NODYNSORT attribute
+	 * from atomic_add_16_nv.
+	 */
+	ENTRY(atomic_add_16)
+	ALTENTRY(atomic_add_16_nv)
+	ALTENTRY(atomic_add_short)
+	ALTENTRY(atomic_add_short_nv)
+add_16:
+	and	%o0, 0x2, %o4		! %o4 = byte offset, left-to-right
+	xor	%o4, 0x2, %g1		! %g1 = byte offset, right-to-left
+	sll	%o4, 3, %o4		! %o4 = bit offset, left-to-right
+	sll	%g1, 3, %g1		! %g1 = bit offset, right-to-left
+	sethi	%hi(0xffff0000), %o3	! %o3 = mask
+	srl	%o3, %o4, %o3		! %o3 = shifted to bit offset
+	sll	%o1, %g1, %o1		! %o1 = shifted to bit offset
+	and	%o1, %o3, %o1		! %o1 = single short value
+	andn	%o0, 0x2, %o0		! %o0 = word address
+	! if low-order bit is 1, we will properly get an alignment fault here
+	ld	[%o0], %o2		! read old value
+1:
+	add	%o1, %o2, %o5		! add value to the old value
+	and	%o5, %o3, %o5		! clear other bits
+	andn	%o2, %o3, %o4		! clear target bits
+	or	%o4, %o5, %o5		! insert the new value
+	cas	[%o0], %o2, %o5
+	cmp	%o2, %o5
+	bne,a,pn %icc, 1b
+	  mov	%o5, %o2		! %o2 = old value
+	add	%o1, %o2, %o5
+	and	%o5, %o3, %o5
+	retl
+	srl	%o5, %g1, %o0		! %o0 = new value
+	SET_SIZE(atomic_add_short_nv)
+	SET_SIZE(atomic_add_short)
+	SET_SIZE(atomic_add_16_nv)
+	SET_SIZE(atomic_add_16)
+
+	/*
+	 * NOTE: If atomic_inc_32 and atomic_inc_32_nv are ever
+	 * separated, you need to also edit the libc sparc platform
+	 * specific mapfile and remove the NODYNSORT attribute
+	 * from atomic_inc_32_nv.
+	 */
+	ENTRY(atomic_inc_32)
+	ALTENTRY(atomic_inc_32_nv)
+	ALTENTRY(atomic_inc_uint)
+	ALTENTRY(atomic_inc_uint_nv)
+	ALTENTRY(atomic_inc_ulong)
+	ALTENTRY(atomic_inc_ulong_nv)
+	ba	add_32
+	  add	%g0, 1, %o1
+	SET_SIZE(atomic_inc_ulong_nv)
+	SET_SIZE(atomic_inc_ulong)
+	SET_SIZE(atomic_inc_uint_nv)
+	SET_SIZE(atomic_inc_uint)
+	SET_SIZE(atomic_inc_32_nv)
+	SET_SIZE(atomic_inc_32)
+
+	/*
+	 * NOTE: If atomic_dec_32 and atomic_dec_32_nv are ever
+	 * separated, you need to also edit the libc sparc platform
+	 * specific mapfile and remove the NODYNSORT attribute
+	 * from atomic_dec_32_nv.
+	 */
+	ENTRY(atomic_dec_32)
+	ALTENTRY(atomic_dec_32_nv)
+	ALTENTRY(atomic_dec_uint)
+	ALTENTRY(atomic_dec_uint_nv)
+	ALTENTRY(atomic_dec_ulong)
+	ALTENTRY(atomic_dec_ulong_nv)
+	ba	add_32
+	  sub	%g0, 1, %o1
+	SET_SIZE(atomic_dec_ulong_nv)
+	SET_SIZE(atomic_dec_ulong)
+	SET_SIZE(atomic_dec_uint_nv)
+	SET_SIZE(atomic_dec_uint)
+	SET_SIZE(atomic_dec_32_nv)
+	SET_SIZE(atomic_dec_32)
+
+	/*
+	 * NOTE: If atomic_add_32 and atomic_add_32_nv are ever
+	 * separated, you need to also edit the libc sparc platform
+	 * specific mapfile and remove the NODYNSORT attribute
+	 * from atomic_add_32_nv.
+	 */
+	ENTRY(atomic_add_32)
+	ALTENTRY(atomic_add_32_nv)
+	ALTENTRY(atomic_add_int)
+	ALTENTRY(atomic_add_int_nv)
+	ALTENTRY(atomic_add_ptr)
+	ALTENTRY(atomic_add_ptr_nv)
+	ALTENTRY(atomic_add_long)
+	ALTENTRY(atomic_add_long_nv)
+add_32:
+	ld	[%o0], %o2
+1:
+	add	%o2, %o1, %o3
+	cas	[%o0], %o2, %o3
+	cmp	%o2, %o3
+	bne,a,pn %icc, 1b
+	  mov	%o3, %o2
+	retl
+	add	%o2, %o1, %o0		! return new value
+	SET_SIZE(atomic_add_long_nv)
+	SET_SIZE(atomic_add_long)
+	SET_SIZE(atomic_add_ptr_nv)
+	SET_SIZE(atomic_add_ptr)
+	SET_SIZE(atomic_add_int_nv)
+	SET_SIZE(atomic_add_int)
+	SET_SIZE(atomic_add_32_nv)
+	SET_SIZE(atomic_add_32)
+
+	/*
+	 * NOTE: If atomic_inc_64 and atomic_inc_64_nv are ever
+	 * separated, you need to also edit the libc sparc platform
+	 * specific mapfile and remove the NODYNSORT attribute
+	 * from atomic_inc_64_nv.
+	 */
+	ENTRY(atomic_inc_64)
+	ALTENTRY(atomic_inc_64_nv)
+	ba	add_64
+	  add	%g0, 1, %o1
+	SET_SIZE(atomic_inc_64_nv)
+	SET_SIZE(atomic_inc_64)
+
+	/*
+	 * NOTE: If atomic_dec_64 and atomic_dec_64_nv are ever
+	 * separated, you need to also edit the libc sparc platform
+	 * specific mapfile and remove the NODYNSORT attribute
+	 * from atomic_dec_64_nv.
+	 */
+	ENTRY(atomic_dec_64)
+	ALTENTRY(atomic_dec_64_nv)
+	ba	add_64
+	  sub	%g0, 1, %o1
+	SET_SIZE(atomic_dec_64_nv)
+	SET_SIZE(atomic_dec_64)
+
+	/*
+	 * NOTE: If atomic_add_64 and atomic_add_64_nv are ever
+	 * separated, you need to also edit the libc sparc platform
+	 * specific mapfile and remove the NODYNSORT attribute
+	 * from atomic_add_64_nv.
+	 */
+	ENTRY(atomic_add_64)
+	ALTENTRY(atomic_add_64_nv)
+	sllx	%o1, 32, %o1		! upper 32 in %o1, lower in %o2
+	srl	%o2, 0, %o2
+	add	%o1, %o2, %o1		! convert 2 32-bit args into 1 64-bit
+add_64:
+	ldx	[%o0], %o2
+1:
+	add	%o2, %o1, %o3
+	casx	[%o0], %o2, %o3
+	cmp	%o2, %o3
+	bne,a,pn %xcc, 1b
+	  mov	%o3, %o2
+	add	%o2, %o1, %o1		! return lower 32-bits in %o1
+	retl
+	srlx	%o1, 32, %o0		! return upper 32-bits in %o0
+	SET_SIZE(atomic_add_64_nv)
+	SET_SIZE(atomic_add_64)
+
+	/*
+	 * NOTE: If atomic_or_8 and atomic_or_8_nv are ever
+	 * separated, you need to also edit the libc sparc platform
+	 * specific mapfile and remove the NODYNSORT attribute
+	 * from atomic_or_8_nv.
+	 */
+	ENTRY(atomic_or_8)
+	ALTENTRY(atomic_or_8_nv)
+	ALTENTRY(atomic_or_uchar)
+	ALTENTRY(atomic_or_uchar_nv)
+	and	%o0, 0x3, %o4		! %o4 = byte offset, left-to-right
+	xor	%o4, 0x3, %g1		! %g1 = byte offset, right-to-left
+	sll	%g1, 3, %g1		! %g1 = bit offset, right-to-left
+	set	0xff, %o3		! %o3 = mask
+	sll	%o3, %g1, %o3		! %o3 = shifted to bit offset
+	sll	%o1, %g1, %o1		! %o1 = shifted to bit offset
+	and	%o1, %o3, %o1		! %o1 = single byte value
+	andn	%o0, 0x3, %o0		! %o0 = word address
+	ld	[%o0], %o2		! read old value
+1:
+	or	%o2, %o1, %o5		! or in the new value
+	cas	[%o0], %o2, %o5
+	cmp	%o2, %o5
+	bne,a,pn %icc, 1b
+	  mov	%o5, %o2		! %o2 = old value
+	or	%o2, %o1, %o5
+	and	%o5, %o3, %o5
+	retl
+	srl	%o5, %g1, %o0		! %o0 = new value
+	SET_SIZE(atomic_or_uchar_nv)
+	SET_SIZE(atomic_or_uchar)
+	SET_SIZE(atomic_or_8_nv)
+	SET_SIZE(atomic_or_8)
+
+	/*
+	 * NOTE: If atomic_or_16 and atomic_or_16_nv are ever
+	 * separated, you need to also edit the libc sparc platform
+	 * specific mapfile and remove the NODYNSORT attribute
+	 * from atomic_or_16_nv.
+	 */
+	ENTRY(atomic_or_16)
+	ALTENTRY(atomic_or_16_nv)
+	ALTENTRY(atomic_or_ushort)
+	ALTENTRY(atomic_or_ushort_nv)
+	and	%o0, 0x2, %o4		! %o4 = byte offset, left-to-right
+	xor	%o4, 0x2, %g1		! %g1 = byte offset, right-to-left
+	sll	%o4, 3, %o4		! %o4 = bit offset, left-to-right
+	sll	%g1, 3, %g1		! %g1 = bit offset, right-to-left
+	sethi	%hi(0xffff0000), %o3	! %o3 = mask
+	srl	%o3, %o4, %o3		! %o3 = shifted to bit offset
+	sll	%o1, %g1, %o1		! %o1 = shifted to bit offset
+	and	%o1, %o3, %o1		! %o1 = single short value
+	andn	%o0, 0x2, %o0		! %o0 = word address
+	! if low-order bit is 1, we will properly get an alignment fault here
+	ld	[%o0], %o2		! read old value
+1:
+	or	%o2, %o1, %o5		! or in the new value
+	cas	[%o0], %o2, %o5
+	cmp	%o2, %o5
+	bne,a,pn %icc, 1b
+	  mov	%o5, %o2		! %o2 = old value
+	or	%o2, %o1, %o5		! or in the new value
+	and	%o5, %o3, %o5
+	retl
+	srl	%o5, %g1, %o0		! %o0 = new value
+	SET_SIZE(atomic_or_ushort_nv)
+	SET_SIZE(atomic_or_ushort)
+	SET_SIZE(atomic_or_16_nv)
+	SET_SIZE(atomic_or_16)
+
+	/*
+	 * NOTE: If atomic_or_32 and atomic_or_32_nv are ever
+	 * separated, you need to also edit the libc sparc platform
+	 * specific mapfile and remove the NODYNSORT attribute
+	 * from atomic_or_32_nv.
+	 */
+	ENTRY(atomic_or_32)
+	ALTENTRY(atomic_or_32_nv)
+	ALTENTRY(atomic_or_uint)
+	ALTENTRY(atomic_or_uint_nv)
+	ALTENTRY(atomic_or_ulong)
+	ALTENTRY(atomic_or_ulong_nv)
+	ld	[%o0], %o2
+1:
+	or	%o2, %o1, %o3
+	cas	[%o0], %o2, %o3
+	cmp	%o2, %o3
+	bne,a,pn %icc, 1b
+	  mov	%o3, %o2
+	retl
+	or	%o2, %o1, %o0		! return new value
+	SET_SIZE(atomic_or_ulong_nv)
+	SET_SIZE(atomic_or_ulong)
+	SET_SIZE(atomic_or_uint_nv)
+	SET_SIZE(atomic_or_uint)
+	SET_SIZE(atomic_or_32_nv)
+	SET_SIZE(atomic_or_32)
+
+	/*
+	 * NOTE: If atomic_or_64 and atomic_or_64_nv are ever
+	 * separated, you need to also edit the libc sparc platform
+	 * specific mapfile and remove the NODYNSORT attribute
+	 * from atomic_or_64_nv.
+	 */
+	ENTRY(atomic_or_64)
+	ALTENTRY(atomic_or_64_nv)
+	sllx	%o1, 32, %o1		! upper 32 in %o1, lower in %o2
+	srl	%o2, 0, %o2
+	add	%o1, %o2, %o1		! convert 2 32-bit args into 1 64-bit
+	ldx	[%o0], %o2
+1:
+	or	%o2, %o1, %o3
+	casx	[%o0], %o2, %o3
+	cmp	%o2, %o3
+	bne,a,pn %xcc, 1b
+	  mov	%o3, %o2
+	or	%o2, %o1, %o1		! return lower 32-bits in %o1
+	retl
+	srlx	%o1, 32, %o0		! return upper 32-bits in %o0
+	SET_SIZE(atomic_or_64_nv)
+	SET_SIZE(atomic_or_64)
+
+	/*
+	 * NOTE: If atomic_and_8 and atomic_and_8_nv are ever
+	 * separated, you need to also edit the libc sparc platform
+	 * specific mapfile and remove the NODYNSORT attribute
+	 * from atomic_and_8_nv.
+	 */
+	ENTRY(atomic_and_8)
+	ALTENTRY(atomic_and_8_nv)
+	ALTENTRY(atomic_and_uchar)
+	ALTENTRY(atomic_and_uchar_nv)
+	and	%o0, 0x3, %o4		! %o4 = byte offset, left-to-right
+	xor	%o4, 0x3, %g1		! %g1 = byte offset, right-to-left
+	sll	%g1, 3, %g1		! %g1 = bit offset, right-to-left
+	set	0xff, %o3		! %o3 = mask
+	sll	%o3, %g1, %o3		! %o3 = shifted to bit offset
+	sll	%o1, %g1, %o1		! %o1 = shifted to bit offset
+	orn	%o1, %o3, %o1		! all ones in other bytes
+	andn	%o0, 0x3, %o0		! %o0 = word address
+	ld	[%o0], %o2		! read old value
+1:
+	and	%o2, %o1, %o5		! and in the new value
+	cas	[%o0], %o2, %o5
+	cmp	%o2, %o5
+	bne,a,pn %icc, 1b
+	  mov	%o5, %o2		! %o2 = old value
+	and	%o2, %o1, %o5
+	and	%o5, %o3, %o5
+	retl
+	srl	%o5, %g1, %o0		! %o0 = new value
+	SET_SIZE(atomic_and_uchar_nv)
+	SET_SIZE(atomic_and_uchar)
+	SET_SIZE(atomic_and_8_nv)
+	SET_SIZE(atomic_and_8)
+
+	/*
+	 * NOTE: If atomic_and_16 and atomic_and_16_nv are ever
+	 * separated, you need to also edit the libc sparc platform
+	 * specific mapfile and remove the NODYNSORT attribute
+	 * from atomic_and_16_nv.
+	 */
+	ENTRY(atomic_and_16)
+	ALTENTRY(atomic_and_16_nv)
+	ALTENTRY(atomic_and_ushort)
+	ALTENTRY(atomic_and_ushort_nv)
+	and	%o0, 0x2, %o4		! %o4 = byte offset, left-to-right
+	xor	%o4, 0x2, %g1		! %g1 = byte offset, right-to-left
+	sll	%o4, 3, %o4		! %o4 = bit offset, left-to-right
+	sll	%g1, 3, %g1		! %g1 = bit offset, right-to-left
+	sethi	%hi(0xffff0000), %o3	! %o3 = mask
+	srl	%o3, %o4, %o3		! %o3 = shifted to bit offset
+	sll	%o1, %g1, %o1		! %o1 = shifted to bit offset
+	orn	%o1, %o3, %o1		! all ones in the other half
+	andn	%o0, 0x2, %o0		! %o0 = word address
+	! if low-order bit is 1, we will properly get an alignment fault here
+	ld	[%o0], %o2		! read old value
+1:
+	and	%o2, %o1, %o5		! and in the new value
+	cas	[%o0], %o2, %o5
+	cmp	%o2, %o5
+	bne,a,pn %icc, 1b
+	  mov	%o5, %o2		! %o2 = old value
+	and	%o2, %o1, %o5
+	and	%o5, %o3, %o5
+	retl
+	srl	%o5, %g1, %o0		! %o0 = new value
+	SET_SIZE(atomic_and_ushort_nv)
+	SET_SIZE(atomic_and_ushort)
+	SET_SIZE(atomic_and_16_nv)
+	SET_SIZE(atomic_and_16)
+
+	/*
+	 * NOTE: If atomic_and_32 and atomic_and_32_nv are ever
+	 * separated, you need to also edit the libc sparc platform
+	 * specific mapfile and remove the NODYNSORT attribute
+	 * from atomic_and_32_nv.
+	 */
+	ENTRY(atomic_and_32)
+	ALTENTRY(atomic_and_32_nv)
+	ALTENTRY(atomic_and_uint)
+	ALTENTRY(atomic_and_uint_nv)
+	ALTENTRY(atomic_and_ulong)
+	ALTENTRY(atomic_and_ulong_nv)
+	ld	[%o0], %o2
+1:
+	and	%o2, %o1, %o3
+	cas	[%o0], %o2, %o3
+	cmp	%o2, %o3
+	bne,a,pn %icc, 1b
+	  mov	%o3, %o2
+	retl
+	and	%o2, %o1, %o0		! return new value
+	SET_SIZE(atomic_and_ulong_nv)
+	SET_SIZE(atomic_and_ulong)
+	SET_SIZE(atomic_and_uint_nv)
+	SET_SIZE(atomic_and_uint)
+	SET_SIZE(atomic_and_32_nv)
+	SET_SIZE(atomic_and_32)
+
+	/*
+	 * NOTE: If atomic_and_64 and atomic_and_64_nv are ever
+	 * separated, you need to also edit the libc sparc platform
+	 * specific mapfile and remove the NODYNSORT attribute
+	 * from atomic_and_64_nv.
+	 */
+	ENTRY(atomic_and_64)
+	ALTENTRY(atomic_and_64_nv)
+	sllx	%o1, 32, %o1		! upper 32 in %o1, lower in %o2
+	srl	%o2, 0, %o2
+	add	%o1, %o2, %o1		! convert 2 32-bit args into 1 64-bit
+	ldx	[%o0], %o2
+1:
+	and	%o2, %o1, %o3
+	casx	[%o0], %o2, %o3
+	cmp	%o2, %o3
+	bne,a,pn %xcc, 1b
+	  mov	%o3, %o2
+	and	%o2, %o1, %o1		! return lower 32-bits in %o1
+	retl
+	srlx	%o1, 32, %o0		! return upper 32-bits in %o0
+	SET_SIZE(atomic_and_64_nv)
+	SET_SIZE(atomic_and_64)
+
+	ENTRY(atomic_cas_8)
+	ALTENTRY(atomic_cas_uchar)
+	and	%o0, 0x3, %o4		! %o4 = byte offset, left-to-right
+	xor	%o4, 0x3, %g1		! %g1 = byte offset, right-to-left
+	sll	%g1, 3, %g1		! %g1 = bit offset, right-to-left
+	set	0xff, %o3		! %o3 = mask
+	sll	%o3, %g1, %o3		! %o3 = shifted to bit offset
+	sll	%o1, %g1, %o1		! %o1 = shifted to bit offset
+	and	%o1, %o3, %o1		! %o1 = single byte value
+	sll	%o2, %g1, %o2		! %o2 = shifted to bit offset
+	and	%o2, %o3, %o2		! %o2 = single byte value
+	andn	%o0, 0x3, %o0		! %o0 = word address
+	ld	[%o0], %o4		! read old value
+1:
+	andn	%o4, %o3, %o4		! clear target bits
+	or	%o4, %o2, %o5		! insert the new value
+	or	%o4, %o1, %o4		! insert the comparison value
+	cas	[%o0], %o4, %o5
+	cmp	%o4, %o5		! did we succeed?
+	be,pt	%icc, 2f
+	  and	%o5, %o3, %o4		! isolate the old value
+	cmp	%o1, %o4		! should we have succeeded?
+	be,a,pt	%icc, 1b		! yes, try again
+	  mov	%o5, %o4		! %o4 = old value
+2:
+	retl
+	srl	%o4, %g1, %o0		! %o0 = old value
+	SET_SIZE(atomic_cas_uchar)
+	SET_SIZE(atomic_cas_8)
+
+	ENTRY(atomic_cas_16)
+	ALTENTRY(atomic_cas_ushort)
+	and	%o0, 0x2, %o4		! %o4 = byte offset, left-to-right
+	xor	%o4, 0x2, %g1		! %g1 = byte offset, right-to-left
+	sll	%o4, 3, %o4		! %o4 = bit offset, left-to-right
+	sll	%g1, 3, %g1		! %g1 = bit offset, right-to-left
+	sethi	%hi(0xffff0000), %o3	! %o3 = mask
+	srl	%o3, %o4, %o3		! %o3 = shifted to bit offset
+	sll	%o1, %g1, %o1		! %o1 = shifted to bit offset
+	and	%o1, %o3, %o1		! %o1 = single short value
+	sll	%o2, %g1, %o2		! %o2 = shifted to bit offset
+	and	%o2, %o3, %o2		! %o2 = single short value
+	andn	%o0, 0x2, %o0		! %o0 = word address
+	! if low-order bit is 1, we will properly get an alignment fault here
+	ld	[%o0], %o4		! read old value
+1:
+	andn	%o4, %o3, %o4		! clear target bits
+	or	%o4, %o2, %o5		! insert the new value
+	or	%o4, %o1, %o4		! insert the comparison value
+	cas	[%o0], %o4, %o5
+	cmp	%o4, %o5		! did we succeed?
+	be,pt	%icc, 2f
+	  and	%o5, %o3, %o4		! isolate the old value
+	cmp	%o1, %o4		! should we have succeeded?
+	be,a,pt	%icc, 1b		! yes, try again
+	  mov	%o5, %o4		! %o4 = old value
+2:
+	retl
+	srl	%o4, %g1, %o0		! %o0 = old value
+	SET_SIZE(atomic_cas_ushort)
+	SET_SIZE(atomic_cas_16)
+
+	ENTRY(atomic_cas_32)
+	ALTENTRY(atomic_cas_uint)
+	ALTENTRY(atomic_cas_ptr)
+	ALTENTRY(atomic_cas_ulong)
+	cas	[%o0], %o1, %o2
+	retl
+	mov	%o2, %o0
+	SET_SIZE(atomic_cas_ulong)
+	SET_SIZE(atomic_cas_ptr)
+	SET_SIZE(atomic_cas_uint)
+	SET_SIZE(atomic_cas_32)
+
+	ENTRY(atomic_cas_64)
+	sllx	%o1, 32, %o1		! cmp's upper 32 in %o1, lower in %o2
+	srl	%o2, 0, %o2		! convert 2 32-bit args into 1 64-bit
+	add	%o1, %o2, %o1
+	sllx	%o3, 32, %o2		! newval upper 32 in %o3, lower in %o4
+	srl	%o4, 0, %o4		! setup %o2 to have newval
+	add	%o2, %o4, %o2
+	casx	[%o0], %o1, %o2
+	srl	%o2, 0, %o1		! return lower 32-bits in %o1
+	retl
+	srlx	%o2, 32, %o0		! return upper 32-bits in %o0
+	SET_SIZE(atomic_cas_64)
+
+	ENTRY(atomic_swap_8)
+	ALTENTRY(atomic_swap_uchar)
+	and	%o0, 0x3, %o4		! %o4 = byte offset, left-to-right
+	xor	%o4, 0x3, %g1		! %g1 = byte offset, right-to-left
+	sll	%g1, 3, %g1		! %g1 = bit offset, right-to-left
+	set	0xff, %o3		! %o3 = mask
+	sll	%o3, %g1, %o3		! %o3 = shifted to bit offset
+	sll	%o1, %g1, %o1		! %o1 = shifted to bit offset
+	and	%o1, %o3, %o1		! %o1 = single byte value
+	andn	%o0, 0x3, %o0		! %o0 = word address
+	ld	[%o0], %o2		! read old value
+1:
+	andn	%o2, %o3, %o5		! clear target bits
+	or	%o5, %o1, %o5		! insert the new value
+	cas	[%o0], %o2, %o5
+	cmp	%o2, %o5
+	bne,a,pn %icc, 1b
+	  mov	%o5, %o2		! %o2 = old value
+	and	%o5, %o3, %o5
+	retl
+	srl	%o5, %g1, %o0		! %o0 = old value
+	SET_SIZE(atomic_swap_uchar)
+	SET_SIZE(atomic_swap_8)
+
+	ENTRY(atomic_swap_16)
+	ALTENTRY(atomic_swap_ushort)
+	and	%o0, 0x2, %o4		! %o4 = byte offset, left-to-right
+	xor	%o4, 0x2, %g1		! %g1 = byte offset, right-to-left
+	sll	%o4, 3, %o4		! %o4 = bit offset, left-to-right
+	sll	%g1, 3, %g1		! %g1 = bit offset, right-to-left
+	sethi	%hi(0xffff0000), %o3	! %o3 = mask
+	srl	%o3, %o4, %o3		! %o3 = shifted to bit offset
+	sll	%o1, %g1, %o1		! %o1 = shifted to bit offset
+	and	%o1, %o3, %o1		! %o1 = single short value
+	andn	%o0, 0x2, %o0		! %o0 = word address
+	! if low-order bit is 1, we will properly get an alignment fault here
+	ld	[%o0], %o2		! read old value
+1:
+	andn	%o2, %o3, %o5		! clear target bits
+	or	%o5, %o1, %o5		! insert the new value
+	cas	[%o0], %o2, %o5
+	cmp	%o2, %o5
+	bne,a,pn %icc, 1b
+	  mov	%o5, %o2		! %o2 = old value
+	and	%o5, %o3, %o5
+	retl
+	srl	%o5, %g1, %o0		! %o0 = old value
+	SET_SIZE(atomic_swap_ushort)
+	SET_SIZE(atomic_swap_16)
+
+	ENTRY(atomic_swap_32)
+	ALTENTRY(atomic_swap_uint)
+	ALTENTRY(atomic_swap_ptr)
+	ALTENTRY(atomic_swap_ulong)
+	ld	[%o0], %o2
+1:
+	mov	%o1, %o3
+	cas	[%o0], %o2, %o3
+	cmp	%o2, %o3
+	bne,a,pn %icc, 1b
+	  mov	%o3, %o2
+	retl
+	mov	%o3, %o0
+	SET_SIZE(atomic_swap_ulong)
+	SET_SIZE(atomic_swap_ptr)
+	SET_SIZE(atomic_swap_uint)
+	SET_SIZE(atomic_swap_32)
+
+	ENTRY(atomic_swap_64)
+	sllx	%o1, 32, %o1		! upper 32 in %o1, lower in %o2
+	srl	%o2, 0, %o2
+	add	%o1, %o2, %o1		! convert 2 32-bit args into 1 64-bit
+	ldx	[%o0], %o2
+1:
+	mov	%o1, %o3
+	casx	[%o0], %o2, %o3
+	cmp	%o2, %o3
+	bne,a,pn %xcc, 1b
+	  mov	%o3, %o2
+	srl	%o3, 0, %o1		! return lower 32-bits in %o1
+	retl
+	srlx	%o3, 32, %o0		! return upper 32-bits in %o0
+	SET_SIZE(atomic_swap_64)
+
+	ENTRY(atomic_set_long_excl)
+	mov	1, %o3
+	slln	%o3, %o1, %o3
+	ldn	[%o0], %o2
+1:
+	andcc	%o2, %o3, %g0		! test if the bit is set
+	bnz,a,pn %ncc, 2f		! if so, then fail out
+	  mov	-1, %o0
+	or	%o2, %o3, %o4		! set the bit, and try to commit it
+	casn	[%o0], %o2, %o4
+	cmp	%o2, %o4
+	bne,a,pn %ncc, 1b		! failed to commit, try again
+	  mov	%o4, %o2
+	mov	%g0, %o0
+2:
+	retl
+	nop
+	SET_SIZE(atomic_set_long_excl)
+
+	ENTRY(atomic_clear_long_excl)
+	mov	1, %o3
+	slln	%o3, %o1, %o3
+	ldn	[%o0], %o2
+1:
+	andncc	%o3, %o2, %g0		! test if the bit is clear
+	bnz,a,pn %ncc, 2f		! if so, then fail out
+	  mov	-1, %o0
+	andn	%o2, %o3, %o4		! clear the bit, and try to commit it
+	casn	[%o0], %o2, %o4
+	cmp	%o2, %o4
+	bne,a,pn %ncc, 1b		! failed to commit, try again
+	  mov	%o4, %o2
+	mov	%g0, %o0
+2:
+	retl
+	nop
+	SET_SIZE(atomic_clear_long_excl)
+
+#if !defined(_KERNEL)
+
+	/*
+	 * Spitfires and Blackbirds have a problem with membars in the
+	 * delay slot (SF_ERRATA_51).  For safety's sake, we assume
+	 * that the whole world needs the workaround.
+	 */
+	ENTRY(membar_enter)
+	membar	#StoreLoad|#StoreStore
+	retl
+	nop
+	SET_SIZE(membar_enter)
+
+	ENTRY(membar_exit)
+	membar	#LoadStore|#StoreStore
+	retl
+	nop
+	SET_SIZE(membar_exit)
+
+	ENTRY(membar_producer)
+	membar	#StoreStore
+	retl
+	nop
+	SET_SIZE(membar_producer)
+
+	ENTRY(membar_consumer)
+	membar	#LoadLoad
+	retl
+	nop
+	SET_SIZE(membar_consumer)
+
+#endif	/* !_KERNEL */