diff options
Diffstat (limited to 'sys/powerpc/include/atomic.h')
| -rw-r--r-- | sys/powerpc/include/atomic.h | 1183 |
1 files changed, 1183 insertions, 0 deletions
diff --git a/sys/powerpc/include/atomic.h b/sys/powerpc/include/atomic.h new file mode 100644 index 000000000000..b2d7549e5bd0 --- /dev/null +++ b/sys/powerpc/include/atomic.h @@ -0,0 +1,1183 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2008 Marcel Moolenaar + * Copyright (c) 2001 Benno Rice + * Copyright (c) 2001 David E. O'Brien + * Copyright (c) 1998 Doug Rabson + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef _MACHINE_ATOMIC_H_ +#define _MACHINE_ATOMIC_H_ + +#include <sys/atomic_common.h> + +#ifndef __powerpc64__ +#include <sys/_atomic64e.h> +#endif + +/* + * The __ATOMIC_REL/ACQ() macros provide memory barriers only in conjunction + * with the atomic lXarx/stXcx. sequences below. They are not exposed outside + * of this file. See also Appendix B.2 of Book II of the architecture manual. + * + * Note that not all Book-E processors accept the light-weight sync variant. + * In particular, early models of E500 cores are known to wedge. Bank on all + * 64-bit capable CPUs to accept lwsync properly and pressimize 32-bit CPUs + * to use the heavier-weight sync. + */ + +#ifdef __powerpc64__ +#define mb() __asm __volatile("sync" : : : "memory") +#define rmb() __asm __volatile("lwsync" : : : "memory") +#define wmb() __asm __volatile("lwsync" : : : "memory") +#define __ATOMIC_REL() __asm __volatile("lwsync" : : : "memory") +#define __ATOMIC_ACQ() __asm __volatile("isync" : : : "memory") +#else +#define mb() __asm __volatile("sync" : : : "memory") +#define rmb() __asm __volatile("sync" : : : "memory") +#define wmb() __asm __volatile("sync" : : : "memory") +#define __ATOMIC_REL() __asm __volatile("sync" : : : "memory") +#define __ATOMIC_ACQ() __asm __volatile("isync" : : : "memory") +#endif + +static __inline void +powerpc_lwsync(void) +{ + +#ifdef __powerpc64__ + __asm __volatile("lwsync" : : : "memory"); +#else + __asm __volatile("sync" : : : "memory"); +#endif +} + +/* + * atomic_add(p, v) + * { *p += v; } + */ + +#define __atomic_add_int(p, v, t) \ + __asm __volatile( \ + "1: lwarx %0, 0, %2\n" \ + " add %0, %3, %0\n" \ + " stwcx. %0, 0, %2\n" \ + " bne- 1b\n" \ + : "=&r" (t), "=m" (*p) \ + : "r" (p), "r" (v), "m" (*p) \ + : "cr0", "memory") \ + /* __atomic_add_int */ + +#ifdef __powerpc64__ +#define __atomic_add_long(p, v, t) \ + __asm __volatile( \ + "1: ldarx %0, 0, %2\n" \ + " add %0, %3, %0\n" \ + " stdcx. %0, 0, %2\n" \ + " bne- 1b\n" \ + : "=&r" (t), "=m" (*p) \ + : "r" (p), "r" (v), "m" (*p) \ + : "cr0", "memory") \ + /* __atomic_add_long */ +#else +#define __atomic_add_long(p, v, t) \ + __asm __volatile( \ + "1: lwarx %0, 0, %2\n" \ + " add %0, %3, %0\n" \ + " stwcx. %0, 0, %2\n" \ + " bne- 1b\n" \ + : "=&r" (t), "=m" (*p) \ + : "r" (p), "r" (v), "m" (*p) \ + : "cr0", "memory") \ + /* __atomic_add_long */ +#endif + +#define _ATOMIC_ADD(type) \ + static __inline void \ + atomic_add_##type(volatile u_##type *p, u_##type v) { \ + u_##type t; \ + __atomic_add_##type(p, v, t); \ + } \ + \ + static __inline void \ + atomic_add_acq_##type(volatile u_##type *p, u_##type v) { \ + u_##type t; \ + __atomic_add_##type(p, v, t); \ + __ATOMIC_ACQ(); \ + } \ + \ + static __inline void \ + atomic_add_rel_##type(volatile u_##type *p, u_##type v) { \ + u_##type t; \ + __ATOMIC_REL(); \ + __atomic_add_##type(p, v, t); \ + } \ + /* _ATOMIC_ADD */ + +_ATOMIC_ADD(int) +_ATOMIC_ADD(long) + +#define atomic_add_32 atomic_add_int +#define atomic_add_acq_32 atomic_add_acq_int +#define atomic_add_rel_32 atomic_add_rel_int + +#ifdef __powerpc64__ +#define atomic_add_64 atomic_add_long +#define atomic_add_acq_64 atomic_add_acq_long +#define atomic_add_rel_64 atomic_add_rel_long + +#define atomic_add_ptr atomic_add_long +#define atomic_add_acq_ptr atomic_add_acq_long +#define atomic_add_rel_ptr atomic_add_rel_long +#else +#define atomic_add_ptr atomic_add_int +#define atomic_add_acq_ptr atomic_add_acq_int +#define atomic_add_rel_ptr atomic_add_rel_int +#endif +#undef _ATOMIC_ADD +#undef __atomic_add_long +#undef __atomic_add_int + +/* + * atomic_clear(p, v) + * { *p &= ~v; } + */ + +#define __atomic_clear_int(p, v, t) \ + __asm __volatile( \ + "1: lwarx %0, 0, %2\n" \ + " andc %0, %0, %3\n" \ + " stwcx. %0, 0, %2\n" \ + " bne- 1b\n" \ + : "=&r" (t), "=m" (*p) \ + : "r" (p), "r" (v), "m" (*p) \ + : "cr0", "memory") \ + /* __atomic_clear_int */ + +#ifdef __powerpc64__ +#define __atomic_clear_long(p, v, t) \ + __asm __volatile( \ + "1: ldarx %0, 0, %2\n" \ + " andc %0, %0, %3\n" \ + " stdcx. %0, 0, %2\n" \ + " bne- 1b\n" \ + : "=&r" (t), "=m" (*p) \ + : "r" (p), "r" (v), "m" (*p) \ + : "cr0", "memory") \ + /* __atomic_clear_long */ +#else +#define __atomic_clear_long(p, v, t) \ + __asm __volatile( \ + "1: lwarx %0, 0, %2\n" \ + " andc %0, %0, %3\n" \ + " stwcx. %0, 0, %2\n" \ + " bne- 1b\n" \ + : "=&r" (t), "=m" (*p) \ + : "r" (p), "r" (v), "m" (*p) \ + : "cr0", "memory") \ + /* __atomic_clear_long */ +#endif + +#define _ATOMIC_CLEAR(type) \ + static __inline void \ + atomic_clear_##type(volatile u_##type *p, u_##type v) { \ + u_##type t; \ + __atomic_clear_##type(p, v, t); \ + } \ + \ + static __inline void \ + atomic_clear_acq_##type(volatile u_##type *p, u_##type v) { \ + u_##type t; \ + __atomic_clear_##type(p, v, t); \ + __ATOMIC_ACQ(); \ + } \ + \ + static __inline void \ + atomic_clear_rel_##type(volatile u_##type *p, u_##type v) { \ + u_##type t; \ + __ATOMIC_REL(); \ + __atomic_clear_##type(p, v, t); \ + } \ + /* _ATOMIC_CLEAR */ + +_ATOMIC_CLEAR(int) +_ATOMIC_CLEAR(long) + +#define atomic_clear_32 atomic_clear_int +#define atomic_clear_acq_32 atomic_clear_acq_int +#define atomic_clear_rel_32 atomic_clear_rel_int + +#ifdef __powerpc64__ +#define atomic_clear_64 atomic_clear_long +#define atomic_clear_acq_64 atomic_clear_acq_long +#define atomic_clear_rel_64 atomic_clear_rel_long + +#define atomic_clear_ptr atomic_clear_long +#define atomic_clear_acq_ptr atomic_clear_acq_long +#define atomic_clear_rel_ptr atomic_clear_rel_long +#else +#define atomic_clear_ptr atomic_clear_int +#define atomic_clear_acq_ptr atomic_clear_acq_int +#define atomic_clear_rel_ptr atomic_clear_rel_int +#endif +#undef _ATOMIC_CLEAR +#undef __atomic_clear_long +#undef __atomic_clear_int + +/* + * atomic_cmpset(p, o, n) + */ +/* TODO -- see below */ + +/* + * atomic_load_acq(p) + */ +/* TODO -- see below */ + +/* + * atomic_readandclear(p) + */ +/* TODO -- see below */ + +/* + * atomic_set(p, v) + * { *p |= v; } + */ + +#define __atomic_set_int(p, v, t) \ + __asm __volatile( \ + "1: lwarx %0, 0, %2\n" \ + " or %0, %3, %0\n" \ + " stwcx. %0, 0, %2\n" \ + " bne- 1b\n" \ + : "=&r" (t), "=m" (*p) \ + : "r" (p), "r" (v), "m" (*p) \ + : "cr0", "memory") \ + /* __atomic_set_int */ + +#ifdef __powerpc64__ +#define __atomic_set_long(p, v, t) \ + __asm __volatile( \ + "1: ldarx %0, 0, %2\n" \ + " or %0, %3, %0\n" \ + " stdcx. %0, 0, %2\n" \ + " bne- 1b\n" \ + : "=&r" (t), "=m" (*p) \ + : "r" (p), "r" (v), "m" (*p) \ + : "cr0", "memory") \ + /* __atomic_set_long */ +#else +#define __atomic_set_long(p, v, t) \ + __asm __volatile( \ + "1: lwarx %0, 0, %2\n" \ + " or %0, %3, %0\n" \ + " stwcx. %0, 0, %2\n" \ + " bne- 1b\n" \ + : "=&r" (t), "=m" (*p) \ + : "r" (p), "r" (v), "m" (*p) \ + : "cr0", "memory") \ + /* __atomic_set_long */ +#endif + +#define _ATOMIC_SET(type) \ + static __inline void \ + atomic_set_##type(volatile u_##type *p, u_##type v) { \ + u_##type t; \ + __atomic_set_##type(p, v, t); \ + } \ + \ + static __inline void \ + atomic_set_acq_##type(volatile u_##type *p, u_##type v) { \ + u_##type t; \ + __atomic_set_##type(p, v, t); \ + __ATOMIC_ACQ(); \ + } \ + \ + static __inline void \ + atomic_set_rel_##type(volatile u_##type *p, u_##type v) { \ + u_##type t; \ + __ATOMIC_REL(); \ + __atomic_set_##type(p, v, t); \ + } \ + /* _ATOMIC_SET */ + +_ATOMIC_SET(int) +_ATOMIC_SET(long) + +#define atomic_set_32 atomic_set_int +#define atomic_set_acq_32 atomic_set_acq_int +#define atomic_set_rel_32 atomic_set_rel_int + +#ifdef __powerpc64__ +#define atomic_set_64 atomic_set_long +#define atomic_set_acq_64 atomic_set_acq_long +#define atomic_set_rel_64 atomic_set_rel_long + +#define atomic_set_ptr atomic_set_long +#define atomic_set_acq_ptr atomic_set_acq_long +#define atomic_set_rel_ptr atomic_set_rel_long +#else +#define atomic_set_ptr atomic_set_int +#define atomic_set_acq_ptr atomic_set_acq_int +#define atomic_set_rel_ptr atomic_set_rel_int +#endif +#undef _ATOMIC_SET +#undef __atomic_set_long +#undef __atomic_set_int + +/* + * atomic_subtract(p, v) + * { *p -= v; } + */ + +#define __atomic_subtract_int(p, v, t) \ + __asm __volatile( \ + "1: lwarx %0, 0, %2\n" \ + " subf %0, %3, %0\n" \ + " stwcx. %0, 0, %2\n" \ + " bne- 1b\n" \ + : "=&r" (t), "=m" (*p) \ + : "r" (p), "r" (v), "m" (*p) \ + : "cr0", "memory") \ + /* __atomic_subtract_int */ + +#ifdef __powerpc64__ +#define __atomic_subtract_long(p, v, t) \ + __asm __volatile( \ + "1: ldarx %0, 0, %2\n" \ + " subf %0, %3, %0\n" \ + " stdcx. %0, 0, %2\n" \ + " bne- 1b\n" \ + : "=&r" (t), "=m" (*p) \ + : "r" (p), "r" (v), "m" (*p) \ + : "cr0", "memory") \ + /* __atomic_subtract_long */ +#else +#define __atomic_subtract_long(p, v, t) \ + __asm __volatile( \ + "1: lwarx %0, 0, %2\n" \ + " subf %0, %3, %0\n" \ + " stwcx. %0, 0, %2\n" \ + " bne- 1b\n" \ + : "=&r" (t), "=m" (*p) \ + : "r" (p), "r" (v), "m" (*p) \ + : "cr0", "memory") \ + /* __atomic_subtract_long */ +#endif + +#define _ATOMIC_SUBTRACT(type) \ + static __inline void \ + atomic_subtract_##type(volatile u_##type *p, u_##type v) { \ + u_##type t; \ + __atomic_subtract_##type(p, v, t); \ + } \ + \ + static __inline void \ + atomic_subtract_acq_##type(volatile u_##type *p, u_##type v) { \ + u_##type t; \ + __atomic_subtract_##type(p, v, t); \ + __ATOMIC_ACQ(); \ + } \ + \ + static __inline void \ + atomic_subtract_rel_##type(volatile u_##type *p, u_##type v) { \ + u_##type t; \ + __ATOMIC_REL(); \ + __atomic_subtract_##type(p, v, t); \ + } \ + /* _ATOMIC_SUBTRACT */ + +_ATOMIC_SUBTRACT(int) +_ATOMIC_SUBTRACT(long) + +#define atomic_subtract_32 atomic_subtract_int +#define atomic_subtract_acq_32 atomic_subtract_acq_int +#define atomic_subtract_rel_32 atomic_subtract_rel_int + +#ifdef __powerpc64__ +#define atomic_subtract_64 atomic_subtract_long +#define atomic_subtract_acq_64 atomic_subract_acq_long +#define atomic_subtract_rel_64 atomic_subtract_rel_long + +#define atomic_subtract_ptr atomic_subtract_long +#define atomic_subtract_acq_ptr atomic_subtract_acq_long +#define atomic_subtract_rel_ptr atomic_subtract_rel_long +#else +#define atomic_subtract_ptr atomic_subtract_int +#define atomic_subtract_acq_ptr atomic_subtract_acq_int +#define atomic_subtract_rel_ptr atomic_subtract_rel_int +#endif +#undef _ATOMIC_SUBTRACT +#undef __atomic_subtract_long +#undef __atomic_subtract_int + +/* + * atomic_store_rel(p, v) + */ +/* TODO -- see below */ + +/* + * Old/original implementations that still need revisiting. + */ + +static __inline u_int +atomic_readandclear_int(volatile u_int *addr) +{ + u_int result,temp; + + __asm __volatile ( + "\tsync\n" /* drain writes */ + "1:\tlwarx %0, 0, %3\n\t" /* load old value */ + "li %1, 0\n\t" /* load new value */ + "stwcx. %1, 0, %3\n\t" /* attempt to store */ + "bne- 1b\n\t" /* spin if failed */ + : "=&r"(result), "=&r"(temp), "=m" (*addr) + : "r" (addr), "m" (*addr) + : "cr0", "memory"); + + return (result); +} + +#ifdef __powerpc64__ +static __inline u_long +atomic_readandclear_long(volatile u_long *addr) +{ + u_long result,temp; + + __asm __volatile ( + "\tsync\n" /* drain writes */ + "1:\tldarx %0, 0, %3\n\t" /* load old value */ + "li %1, 0\n\t" /* load new value */ + "stdcx. %1, 0, %3\n\t" /* attempt to store */ + "bne- 1b\n\t" /* spin if failed */ + : "=&r"(result), "=&r"(temp), "=m" (*addr) + : "r" (addr), "m" (*addr) + : "cr0", "memory"); + + return (result); +} +#endif + +#define atomic_readandclear_32 atomic_readandclear_int + +#ifdef __powerpc64__ +#define atomic_readandclear_64 atomic_readandclear_long + +#define atomic_readandclear_ptr atomic_readandclear_long +#else +static __inline u_long +atomic_readandclear_long(volatile u_long *addr) +{ + + return ((u_long)atomic_readandclear_int((volatile u_int *)addr)); +} + +#define atomic_readandclear_ptr atomic_readandclear_int +#endif + +/* + * We assume that a = b will do atomic loads and stores. + */ +#define ATOMIC_STORE_LOAD(TYPE) \ +static __inline u_##TYPE \ +atomic_load_acq_##TYPE(const volatile u_##TYPE *p) \ +{ \ + u_##TYPE v; \ + \ + v = *p; \ + powerpc_lwsync(); \ + return (v); \ +} \ + \ +static __inline void \ +atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ +{ \ + \ + powerpc_lwsync(); \ + *p = v; \ +} + +ATOMIC_STORE_LOAD(int) + +#define atomic_load_acq_32 atomic_load_acq_int +#define atomic_store_rel_32 atomic_store_rel_int + +#ifdef __powerpc64__ +ATOMIC_STORE_LOAD(long) + +#define atomic_load_acq_64 atomic_load_acq_long +#define atomic_store_rel_64 atomic_store_rel_long + +#define atomic_load_acq_ptr atomic_load_acq_long +#define atomic_store_rel_ptr atomic_store_rel_long +#else +static __inline u_long +atomic_load_acq_long(const volatile u_long *addr) +{ + + return ((u_long)atomic_load_acq_int((const volatile u_int *)addr)); +} + +static __inline void +atomic_store_rel_long(volatile u_long *addr, u_long val) +{ + + atomic_store_rel_int((volatile u_int *)addr, (u_int)val); +} + +#define atomic_load_acq_ptr atomic_load_acq_int +#define atomic_store_rel_ptr atomic_store_rel_int +#endif +#undef ATOMIC_STORE_LOAD + +/* + * Atomically compare the value stored at *p with cmpval and if the + * two values are equal, update the value of *p with newval. Returns + * zero if the compare failed, nonzero otherwise. + */ +#ifdef ISA_206_ATOMICS +static __inline int +atomic_cmpset_char(volatile u_char *p, u_char cmpval, u_char newval) +{ + int ret; + + __asm __volatile ( + "1:\tlbarx %0, 0, %2\n\t" /* load old value */ + "cmplw %3, %0\n\t" /* compare */ + "bne- 2f\n\t" /* exit if not equal */ + "stbcx. %4, 0, %2\n\t" /* attempt to store */ + "bne- 1b\n\t" /* spin if failed */ + "li %0, 1\n\t" /* success - retval = 1 */ + "b 3f\n\t" /* we've succeeded */ + "2:\n\t" + "stbcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ + "li %0, 0\n\t" /* failure - retval = 0 */ + "3:\n\t" + : "=&r" (ret), "=m" (*p) + : "r" (p), "r" (cmpval), "r" (newval), "m" (*p) + : "cr0", "memory"); + + return (ret); +} + +static __inline int +atomic_cmpset_short(volatile u_short *p, u_short cmpval, u_short newval) +{ + int ret; + + __asm __volatile ( + "1:\tlharx %0, 0, %2\n\t" /* load old value */ + "cmplw %3, %0\n\t" /* compare */ + "bne- 2f\n\t" /* exit if not equal */ + "sthcx. %4, 0, %2\n\t" /* attempt to store */ + "bne- 1b\n\t" /* spin if failed */ + "li %0, 1\n\t" /* success - retval = 1 */ + "b 3f\n\t" /* we've succeeded */ + "2:\n\t" + "sthcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ + "li %0, 0\n\t" /* failure - retval = 0 */ + "3:\n\t" + : "=&r" (ret), "=m" (*p) + : "r" (p), "r" (cmpval), "r" (newval), "m" (*p) + : "cr0", "memory"); + + return (ret); +} +#else +static __inline int +atomic_cmpset_masked(uint32_t *p, uint32_t cmpval, uint32_t newval, + uint32_t mask) +{ + int ret; + uint32_t tmp; + + __asm __volatile ( + "1:\tlwarx %2, 0, %3\n\t" /* load old value */ + "and %0, %2, %7\n\t" + "cmplw %4, %0\n\t" /* compare */ + "bne- 2f\n\t" /* exit if not equal */ + "andc %2, %2, %7\n\t" + "or %2, %2, %5\n\t" + "stwcx. %2, 0, %3\n\t" /* attempt to store */ + "bne- 1b\n\t" /* spin if failed */ + "li %0, 1\n\t" /* success - retval = 1 */ + "b 3f\n\t" /* we've succeeded */ + "2:\n\t" + "stwcx. %2, 0, %3\n\t" /* clear reservation (74xx) */ + "li %0, 0\n\t" /* failure - retval = 0 */ + "3:\n\t" + : "=&r" (ret), "=m" (*p), "+&r" (tmp) + : "r" (p), "r" (cmpval), "r" (newval), "m" (*p), + "r" (mask) + : "cr0", "memory"); + + return (ret); +} + +#define _atomic_cmpset_masked_word(a,o,v,m) atomic_cmpset_masked(a, o, v, m) +#endif + +static __inline int +atomic_cmpset_int(volatile u_int* p, u_int cmpval, u_int newval) +{ + int ret; + + __asm __volatile ( + "1:\tlwarx %0, 0, %2\n\t" /* load old value */ + "cmplw %3, %0\n\t" /* compare */ + "bne- 2f\n\t" /* exit if not equal */ + "stwcx. %4, 0, %2\n\t" /* attempt to store */ + "bne- 1b\n\t" /* spin if failed */ + "li %0, 1\n\t" /* success - retval = 1 */ + "b 3f\n\t" /* we've succeeded */ + "2:\n\t" + "stwcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ + "li %0, 0\n\t" /* failure - retval = 0 */ + "3:\n\t" + : "=&r" (ret), "=m" (*p) + : "r" (p), "r" (cmpval), "r" (newval), "m" (*p) + : "cr0", "memory"); + + return (ret); +} +static __inline int +atomic_cmpset_long(volatile u_long* p, u_long cmpval, u_long newval) +{ + int ret; + + __asm __volatile ( + #ifdef __powerpc64__ + "1:\tldarx %0, 0, %2\n\t" /* load old value */ + "cmpld %3, %0\n\t" /* compare */ + "bne- 2f\n\t" /* exit if not equal */ + "stdcx. %4, 0, %2\n\t" /* attempt to store */ + #else + "1:\tlwarx %0, 0, %2\n\t" /* load old value */ + "cmplw %3, %0\n\t" /* compare */ + "bne- 2f\n\t" /* exit if not equal */ + "stwcx. %4, 0, %2\n\t" /* attempt to store */ + #endif + "bne- 1b\n\t" /* spin if failed */ + "li %0, 1\n\t" /* success - retval = 1 */ + "b 3f\n\t" /* we've succeeded */ + "2:\n\t" + #ifdef __powerpc64__ + "stdcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ + #else + "stwcx. %0, 0, %2\n\t" /* clear reservation (74xx) */ + #endif + "li %0, 0\n\t" /* failure - retval = 0 */ + "3:\n\t" + : "=&r" (ret), "=m" (*p) + : "r" (p), "r" (cmpval), "r" (newval), "m" (*p) + : "cr0", "memory"); + + return (ret); +} + +#define ATOMIC_CMPSET_ACQ_REL(type) \ + static __inline int \ + atomic_cmpset_acq_##type(volatile u_##type *p, \ + u_##type cmpval, u_##type newval)\ + {\ + u_##type retval; \ + retval = atomic_cmpset_##type(p, cmpval, newval);\ + __ATOMIC_ACQ();\ + return (retval);\ + }\ + static __inline int \ + atomic_cmpset_rel_##type(volatile u_##type *p, \ + u_##type cmpval, u_##type newval)\ + {\ + __ATOMIC_REL();\ + return (atomic_cmpset_##type(p, cmpval, newval));\ + }\ + struct hack + +ATOMIC_CMPSET_ACQ_REL(int); +ATOMIC_CMPSET_ACQ_REL(long); + +#ifdef ISA_206_ATOMICS +#define atomic_cmpset_8 atomic_cmpset_char +#endif +#define atomic_cmpset_acq_8 atomic_cmpset_acq_char +#define atomic_cmpset_rel_8 atomic_cmpset_rel_char + +#ifdef ISA_206_ATOMICS +#define atomic_cmpset_16 atomic_cmpset_short +#endif +#define atomic_cmpset_acq_16 atomic_cmpset_acq_short +#define atomic_cmpset_rel_16 atomic_cmpset_rel_short + +#define atomic_cmpset_32 atomic_cmpset_int +#define atomic_cmpset_acq_32 atomic_cmpset_acq_int +#define atomic_cmpset_rel_32 atomic_cmpset_rel_int + +#ifdef __powerpc64__ +#define atomic_cmpset_64 atomic_cmpset_long +#define atomic_cmpset_acq_64 atomic_cmpset_acq_long +#define atomic_cmpset_rel_64 atomic_cmpset_rel_long + +#define atomic_cmpset_ptr atomic_cmpset_long +#define atomic_cmpset_acq_ptr atomic_cmpset_acq_long +#define atomic_cmpset_rel_ptr atomic_cmpset_rel_long +#else +#define atomic_cmpset_ptr atomic_cmpset_int +#define atomic_cmpset_acq_ptr atomic_cmpset_acq_int +#define atomic_cmpset_rel_ptr atomic_cmpset_rel_int +#endif + +/* + * Atomically compare the value stored at *p with *cmpval and if the + * two values are equal, update the value of *p with newval. Returns + * zero if the compare failed and sets *cmpval to the read value from *p, + * nonzero otherwise. + */ +#ifdef ISA_206_ATOMICS +static __inline int +atomic_fcmpset_char(volatile u_char *p, u_char *cmpval, u_char newval) +{ + int ret; + + __asm __volatile ( + "lbarx %0, 0, %3\n\t" /* load old value */ + "cmplw %4, %0\n\t" /* compare */ + "bne- 1f\n\t" /* exit if not equal */ + "stbcx. %5, 0, %3\n\t" /* attempt to store */ + "bne- 1f\n\t" /* exit if failed */ + "li %0, 1\n\t" /* success - retval = 1 */ + "b 2f\n\t" /* we've succeeded */ + "1:\n\t" + "stbcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ + "stbx %0, 0, %7\n\t" + "li %0, 0\n\t" /* failure - retval = 0 */ + "2:\n\t" + : "=&r" (ret), "=m" (*p), "=m" (*cmpval) + : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval) + : "cr0", "memory"); + + return (ret); +} + +static __inline int +atomic_fcmpset_short(volatile u_short *p, u_short *cmpval, u_short newval) +{ + int ret; + + __asm __volatile ( + "lharx %0, 0, %3\n\t" /* load old value */ + "cmplw %4, %0\n\t" /* compare */ + "bne- 1f\n\t" /* exit if not equal */ + "sthcx. %5, 0, %3\n\t" /* attempt to store */ + "bne- 1f\n\t" /* exit if failed */ + "li %0, 1\n\t" /* success - retval = 1 */ + "b 2f\n\t" /* we've succeeded */ + "1:\n\t" + "sthcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ + "sthx %0, 0, %7\n\t" + "li %0, 0\n\t" /* failure - retval = 0 */ + "2:\n\t" + : "=&r" (ret), "=m" (*p), "=m" (*cmpval) + : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval) + : "cr0", "memory"); + + return (ret); +} +#endif /* ISA_206_ATOMICS */ + +static __inline int +atomic_fcmpset_int(volatile u_int *p, u_int *cmpval, u_int newval) +{ + int ret; + + __asm __volatile ( + "lwarx %0, 0, %3\n\t" /* load old value */ + "cmplw %4, %0\n\t" /* compare */ + "bne- 1f\n\t" /* exit if not equal */ + "stwcx. %5, 0, %3\n\t" /* attempt to store */ + "bne- 1f\n\t" /* exit if failed */ + "li %0, 1\n\t" /* success - retval = 1 */ + "b 2f\n\t" /* we've succeeded */ + "1:\n\t" + "stwcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ + "stwx %0, 0, %7\n\t" + "li %0, 0\n\t" /* failure - retval = 0 */ + "2:\n\t" + : "=&r" (ret), "=m" (*p), "=m" (*cmpval) + : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval) + : "cr0", "memory"); + + return (ret); +} +static __inline int +atomic_fcmpset_long(volatile u_long *p, u_long *cmpval, u_long newval) +{ + int ret; + + __asm __volatile ( + #ifdef __powerpc64__ + "ldarx %0, 0, %3\n\t" /* load old value */ + "cmpld %4, %0\n\t" /* compare */ + "bne- 1f\n\t" /* exit if not equal */ + "stdcx. %5, 0, %3\n\t" /* attempt to store */ + #else + "lwarx %0, 0, %3\n\t" /* load old value */ + "cmplw %4, %0\n\t" /* compare */ + "bne- 1f\n\t" /* exit if not equal */ + "stwcx. %5, 0, %3\n\t" /* attempt to store */ + #endif + "bne- 1f\n\t" /* exit if failed */ + "li %0, 1\n\t" /* success - retval = 1 */ + "b 2f\n\t" /* we've succeeded */ + "1:\n\t" + #ifdef __powerpc64__ + "stdcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ + "stdx %0, 0, %7\n\t" + #else + "stwcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ + "stwx %0, 0, %7\n\t" + #endif + "li %0, 0\n\t" /* failure - retval = 0 */ + "2:\n\t" + : "=&r" (ret), "=m" (*p), "=m" (*cmpval) + : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval) + : "cr0", "memory"); + + return (ret); +} + +#define ATOMIC_FCMPSET_ACQ_REL(type) \ + static __inline int \ + atomic_fcmpset_acq_##type(volatile u_##type *p, \ + u_##type *cmpval, u_##type newval)\ + {\ + u_##type retval; \ + retval = atomic_fcmpset_##type(p, cmpval, newval);\ + __ATOMIC_ACQ();\ + return (retval);\ + }\ + static __inline int \ + atomic_fcmpset_rel_##type(volatile u_##type *p, \ + u_##type *cmpval, u_##type newval)\ + {\ + __ATOMIC_REL();\ + return (atomic_fcmpset_##type(p, cmpval, newval));\ + }\ + struct hack + +ATOMIC_FCMPSET_ACQ_REL(int); +ATOMIC_FCMPSET_ACQ_REL(long); + +#ifdef ISA_206_ATOMICS +#define atomic_fcmpset_8 atomic_fcmpset_char +#endif +#define atomic_fcmpset_acq_8 atomic_fcmpset_acq_char +#define atomic_fcmpset_rel_8 atomic_fcmpset_rel_char + +#ifdef ISA_206_ATOMICS +#define atomic_fcmpset_16 atomic_fcmpset_short +#endif +#define atomic_fcmpset_acq_16 atomic_fcmpset_acq_short +#define atomic_fcmpset_rel_16 atomic_fcmpset_rel_short + +#define atomic_fcmpset_32 atomic_fcmpset_int +#define atomic_fcmpset_acq_32 atomic_fcmpset_acq_int +#define atomic_fcmpset_rel_32 atomic_fcmpset_rel_int + +#ifdef __powerpc64__ +#define atomic_fcmpset_64 atomic_fcmpset_long +#define atomic_fcmpset_acq_64 atomic_fcmpset_acq_long +#define atomic_fcmpset_rel_64 atomic_fcmpset_rel_long + +#define atomic_fcmpset_ptr atomic_fcmpset_long +#define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_long +#define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_long +#else +#define atomic_fcmpset_ptr atomic_fcmpset_int +#define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_int +#define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_int +#endif + +static __inline u_int +atomic_fetchadd_int(volatile u_int *p, u_int v) +{ + u_int value; + + do { + value = *p; + } while (!atomic_cmpset_int(p, value, value + v)); + return (value); +} + +static __inline u_long +atomic_fetchadd_long(volatile u_long *p, u_long v) +{ + u_long value; + + do { + value = *p; + } while (!atomic_cmpset_long(p, value, value + v)); + return (value); +} + +static __inline u_int +atomic_swap_32(volatile u_int *p, u_int v) +{ + u_int prev; + + __asm __volatile( + "1: lwarx %0,0,%2\n" + " stwcx. %3,0,%2\n" + " bne- 1b\n" + : "=&r" (prev), "+m" (*(volatile u_int *)p) + : "r" (p), "r" (v) + : "cr0", "memory"); + + return (prev); +} + +#ifdef __powerpc64__ +static __inline u_long +atomic_swap_64(volatile u_long *p, u_long v) +{ + u_long prev; + + __asm __volatile( + "1: ldarx %0,0,%2\n" + " stdcx. %3,0,%2\n" + " bne- 1b\n" + : "=&r" (prev), "+m" (*(volatile u_long *)p) + : "r" (p), "r" (v) + : "cr0", "memory"); + + return (prev); +} +#endif + +#define atomic_fetchadd_32 atomic_fetchadd_int +#define atomic_swap_int atomic_swap_32 + +#ifdef __powerpc64__ +#define atomic_fetchadd_64 atomic_fetchadd_long +#define atomic_swap_long atomic_swap_64 +#define atomic_swap_ptr atomic_swap_64 +#else +#define atomic_swap_long(p,v) atomic_swap_32((volatile u_int *)(p), v) +#define atomic_swap_ptr(p,v) atomic_swap_32((volatile u_int *)(p), v) +#endif + +static __inline int +atomic_testandset_int(volatile u_int *p, u_int v) +{ + u_int m = (1u << (v & 0x1f)); + u_int res; + u_int tmp; + + __asm __volatile( + "1: lwarx %0,0,%3\n" + " and %1,%0,%4\n" + " or %0,%0,%4\n" + " stwcx. %0,0,%3\n" + " bne- 1b\n" + : "=&r"(tmp), "=&r"(res), "+m"(*p) + : "r"(p), "r"(m) + : "cr0", "memory"); + + return (res != 0); +} + +static __inline int +atomic_testandclear_int(volatile u_int *p, u_int v) +{ + u_int m = (1u << (v & 0x1f)); + u_int res; + u_int tmp; + + __asm __volatile( + "1: lwarx %0,0,%3\n" + " and %1,%0,%4\n" + " andc %0,%0,%4\n" + " stwcx. %0,0,%3\n" + " bne- 1b\n" + : "=&r"(tmp), "=&r"(res), "+m"(*p) + : "r"(p), "r"(m) + : "cr0", "memory"); + + return (res != 0); +} + +#ifdef __powerpc64__ +static __inline int +atomic_testandset_long(volatile u_long *p, u_int v) +{ + u_long m = (1ul << (v & 0x3f)); + u_long res; + u_long tmp; + + __asm __volatile( + "1: ldarx %0,0,%3\n" + " and %1,%0,%4\n" + " or %0,%0,%4\n" + " stdcx. %0,0,%3\n" + " bne- 1b\n" + : "=&r"(tmp), "=&r"(res), "+m"(*(volatile u_long *)p) + : "r"(p), "r"(m) + : "cr0", "memory"); + + return (res != 0); +} + +static __inline int +atomic_testandclear_long(volatile u_long *p, u_int v) +{ + u_long m = (1ul << (v & 0x3f)); + u_long res; + u_long tmp; + + __asm __volatile( + "1: ldarx %0,0,%3\n" + " and %1,%0,%4\n" + " andc %0,%0,%4\n" + " stdcx. %0,0,%3\n" + " bne- 1b\n" + : "=&r"(tmp), "=&r"(res), "+m"(*p) + : "r"(p), "r"(m) + : "cr0", "memory"); + + return (res != 0); +} +#else +static __inline int +atomic_testandset_long(volatile u_long *p, u_int v) +{ + return (atomic_testandset_int((volatile u_int *)p, v)); +} + +static __inline int +atomic_testandclear_long(volatile u_long *p, u_int v) +{ + return (atomic_testandclear_int((volatile u_int *)p, v)); +} +#endif + +#define atomic_testandclear_32 atomic_testandclear_int +#define atomic_testandset_32 atomic_testandset_int + +static __inline int +atomic_testandset_acq_long(volatile u_long *p, u_int v) +{ + u_int a = atomic_testandset_long(p, v); + __ATOMIC_ACQ(); + return (a); +} + +#ifdef __powerpc64__ +#define atomic_testandclear_ptr atomic_testandclear_long +#define atomic_testandset_ptr atomic_testandset_long +#else +#define atomic_testandclear_ptr(p,v) \ + atomic_testandclear_32((volatile u_int *)(p), v) +#define atomic_testandset_ptr(p,v) \ + atomic_testandset_32((volatile u_int *)(p), v) +#endif + +static __inline void +atomic_thread_fence_acq(void) +{ + + powerpc_lwsync(); +} + +static __inline void +atomic_thread_fence_rel(void) +{ + + powerpc_lwsync(); +} + +static __inline void +atomic_thread_fence_acq_rel(void) +{ + + powerpc_lwsync(); +} + +static __inline void +atomic_thread_fence_seq_cst(void) +{ + + __asm __volatile("sync" : : : "memory"); +} + +#ifndef ISA_206_ATOMICS +#include <sys/_atomic_subword.h> +#define atomic_cmpset_char atomic_cmpset_8 +#define atomic_cmpset_short atomic_cmpset_16 +#define atomic_fcmpset_char atomic_fcmpset_8 +#define atomic_fcmpset_short atomic_fcmpset_16 +#define atomic_set_short atomic_set_16 +#define atomic_clear_short atomic_clear_16 +#else + +static __inline void +atomic_set_short(volatile u_short *p, u_short bit) +{ + u_short v; + + v = atomic_load_short(p); + for (;;) { + if (atomic_fcmpset_16(p, &v, v | bit)) + break; + } +} + +static __inline void +atomic_clear_short(volatile u_short *p, u_short bit) +{ + u_short v; + + v = atomic_load_short(p); + for (;;) { + if (atomic_fcmpset_16(p, &v, v & ~bit)) + break; + } +} + +#define atomic_set_16 atomic_set_short +#define atomic_clear_16 atomic_clear_short + +#endif /* ISA_206_ATOMICS */ + +/* These need sys/_atomic_subword.h on non-ISA-2.06-atomic platforms. */ +ATOMIC_CMPSET_ACQ_REL(char); +ATOMIC_CMPSET_ACQ_REL(short); + +ATOMIC_FCMPSET_ACQ_REL(char); +ATOMIC_FCMPSET_ACQ_REL(short); + +#undef __ATOMIC_REL +#undef __ATOMIC_ACQ + +#endif /* ! _MACHINE_ATOMIC_H_ */ |