aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--include/ck_pr.h49
-rw-r--r--include/ck_queue.h14
-rw-r--r--include/gcc/x86/ck_pr.h10
-rw-r--r--include/gcc/x86_64/ck_pr.h10
4 files changed, 67 insertions, 16 deletions
diff --git a/include/ck_pr.h b/include/ck_pr.h
index 4fdbdffa1110..2de6e13ec3c9 100644
--- a/include/ck_pr.h
+++ b/include/ck_pr.h
@@ -615,8 +615,8 @@ CK_PR_BTX_S(bts, 16, uint16_t, |,)
}
#define CK_PR_UNARY_Z(K, S, M, T, P, C, Z) \
- CK_CC_INLINE static void \
- ck_pr_##K##_##S##_zero(M *target, bool *zero) \
+ CK_CC_INLINE static bool \
+ ck_pr_##K##_##S##_is_zero(M *target) \
{ \
T previous; \
C punt; \
@@ -627,12 +627,21 @@ CK_PR_BTX_S(bts, 16, uint16_t, |,)
(C)(previous P 1), \
&previous) == false) \
ck_pr_stall(); \
- *zero = previous == (T)Z; \
+ return previous == (T)Z; \
+ }
+
+#define CK_PR_UNARY_Z_STUB(K, S, M) \
+ CK_CC_INLINE static void \
+ ck_pr_##K##_##S##_zero(M *target, bool *zero) \
+ { \
+ *zero = ck_pr_##K##_##S##_is_zero(target); \
return; \
}
#define CK_PR_UNARY_S(K, X, S, M) CK_PR_UNARY(K, X, S, M, M)
-#define CK_PR_UNARY_Z_S(K, S, M, P, Z) CK_PR_UNARY_Z(K, S, M, M, P, M, Z)
+#define CK_PR_UNARY_Z_S(K, S, M, P, Z) \
+ CK_PR_UNARY_Z(K, S, M, M, P, M, Z) \
+ CK_PR_UNARY_Z_STUB(K, S, M)
#if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE)
@@ -644,6 +653,8 @@ CK_PR_UNARY_S(inc, add, char, char)
#ifndef CK_F_PR_INC_CHAR_ZERO
#define CK_F_PR_INC_CHAR_ZERO
CK_PR_UNARY_Z_S(inc, char, char, +, -1)
+#else
+CK_PR_UNARY_Z_STUB(inc, char, char)
#endif /* CK_F_PR_INC_CHAR_ZERO */
#ifndef CK_F_PR_DEC_CHAR
@@ -654,6 +665,8 @@ CK_PR_UNARY_S(dec, sub, char, char)
#ifndef CK_F_PR_DEC_CHAR_ZERO
#define CK_F_PR_DEC_CHAR_ZERO
CK_PR_UNARY_Z_S(dec, char, char, -, 1)
+#else
+CK_PR_UNARY_Z_STUB(dec, char, char)
#endif /* CK_F_PR_DEC_CHAR_ZERO */
#endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */
@@ -668,6 +681,8 @@ CK_PR_UNARY_S(inc, add, int, int)
#ifndef CK_F_PR_INC_INT_ZERO
#define CK_F_PR_INC_INT_ZERO
CK_PR_UNARY_Z_S(inc, int, int, +, -1)
+#else
+CK_PR_UNARY_Z_STUB(inc, int, int)
#endif /* CK_F_PR_INC_INT_ZERO */
#ifndef CK_F_PR_DEC_INT
@@ -678,6 +693,8 @@ CK_PR_UNARY_S(dec, sub, int, int)
#ifndef CK_F_PR_DEC_INT_ZERO
#define CK_F_PR_DEC_INT_ZERO
CK_PR_UNARY_Z_S(dec, int, int, -, 1)
+#else
+CK_PR_UNARY_Z_STUB(dec, int, int)
#endif /* CK_F_PR_DEC_INT_ZERO */
#endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */
@@ -707,6 +724,8 @@ CK_PR_UNARY_S(inc, add, uint, unsigned int)
#ifndef CK_F_PR_INC_UINT_ZERO
#define CK_F_PR_INC_UINT_ZERO
CK_PR_UNARY_Z_S(inc, uint, unsigned int, +, UINT_MAX)
+#else
+CK_PR_UNARY_Z_STUB(inc, uint, unsigned int)
#endif /* CK_F_PR_INC_UINT_ZERO */
#ifndef CK_F_PR_DEC_UINT
@@ -717,6 +736,8 @@ CK_PR_UNARY_S(dec, sub, uint, unsigned int)
#ifndef CK_F_PR_DEC_UINT_ZERO
#define CK_F_PR_DEC_UINT_ZERO
CK_PR_UNARY_Z_S(dec, uint, unsigned int, -, 1)
+#else
+CK_PR_UNARY_Z_STUB(dec, uint, unsigned int)
#endif /* CK_F_PR_DEC_UINT_ZERO */
#endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */
@@ -731,6 +752,8 @@ CK_PR_UNARY(inc, add, ptr, void, uintptr_t)
#ifndef CK_F_PR_INC_PTR_ZERO
#define CK_F_PR_INC_PTR_ZERO
CK_PR_UNARY_Z(inc, ptr, void, uintptr_t, +, void *, UINT_MAX)
+#else
+CK_PR_UNARY_Z_STUB(inc, ptr, void)
#endif /* CK_F_PR_INC_PTR_ZERO */
#ifndef CK_F_PR_DEC_PTR
@@ -741,6 +764,8 @@ CK_PR_UNARY(dec, sub, ptr, void, uintptr_t)
#ifndef CK_F_PR_DEC_PTR_ZERO
#define CK_F_PR_DEC_PTR_ZERO
CK_PR_UNARY_Z(dec, ptr, void, uintptr_t, -, void *, 1)
+#else
+CK_PR_UNARY_Z_STUB(dec, ptr, void)
#endif /* CK_F_PR_DEC_PTR_ZERO */
#endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */
@@ -755,6 +780,8 @@ CK_PR_UNARY_S(inc, add, 64, uint64_t)
#ifndef CK_F_PR_INC_64_ZERO
#define CK_F_PR_INC_64_ZERO
CK_PR_UNARY_Z_S(inc, 64, uint64_t, +, UINT64_MAX)
+#else
+CK_PR_UNARY_Z_STUB(inc, 64, uint64_t)
#endif /* CK_F_PR_INC_64_ZERO */
#ifndef CK_F_PR_DEC_64
@@ -765,6 +792,8 @@ CK_PR_UNARY_S(dec, sub, 64, uint64_t)
#ifndef CK_F_PR_DEC_64_ZERO
#define CK_F_PR_DEC_64_ZERO
CK_PR_UNARY_Z_S(dec, 64, uint64_t, -, 1)
+#else
+CK_PR_UNARY_Z_STUB(dec, 64, uint64_t)
#endif /* CK_F_PR_DEC_64_ZERO */
#endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */
@@ -779,6 +808,8 @@ CK_PR_UNARY_S(inc, add, 32, uint32_t)
#ifndef CK_F_PR_INC_32_ZERO
#define CK_F_PR_INC_32_ZERO
CK_PR_UNARY_Z_S(inc, 32, uint32_t, +, UINT32_MAX)
+#else
+CK_PR_UNARY_Z_STUB(inc, 32, uint32_t)
#endif /* CK_F_PR_INC_32_ZERO */
#ifndef CK_F_PR_DEC_32
@@ -789,6 +820,8 @@ CK_PR_UNARY_S(dec, sub, 32, uint32_t)
#ifndef CK_F_PR_DEC_32_ZERO
#define CK_F_PR_DEC_32_ZERO
CK_PR_UNARY_Z_S(dec, 32, uint32_t, -, 1)
+#else
+CK_PR_UNARY_Z_STUB(dec, 32, uint32_t)
#endif /* CK_F_PR_DEC_32_ZERO */
#endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */
@@ -803,6 +836,8 @@ CK_PR_UNARY_S(inc, add, 16, uint16_t)
#ifndef CK_F_PR_INC_16_ZERO
#define CK_F_PR_INC_16_ZERO
CK_PR_UNARY_Z_S(inc, 16, uint16_t, +, UINT16_MAX)
+#else
+CK_PR_UNARY_Z_STUB(inc, 16, uint16_t)
#endif /* CK_F_PR_INC_16_ZERO */
#ifndef CK_F_PR_DEC_16
@@ -813,6 +848,8 @@ CK_PR_UNARY_S(dec, sub, 16, uint16_t)
#ifndef CK_F_PR_DEC_16_ZERO
#define CK_F_PR_DEC_16_ZERO
CK_PR_UNARY_Z_S(dec, 16, uint16_t, -, 1)
+#else
+CK_PR_UNARY_Z_STUB(dec, 16, uint16_t)
#endif /* CK_F_PR_DEC_16_ZERO */
#endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */
@@ -827,6 +864,8 @@ CK_PR_UNARY_S(inc, add, 8, uint8_t)
#ifndef CK_F_PR_INC_8_ZERO
#define CK_F_PR_INC_8_ZERO
CK_PR_UNARY_Z_S(inc, 8, uint8_t, +, UINT8_MAX)
+#else
+CK_PR_UNARY_Z_STUB(inc, 8, uint8_t)
#endif /* CK_F_PR_INC_8_ZERO */
#ifndef CK_F_PR_DEC_8
@@ -837,6 +876,8 @@ CK_PR_UNARY_S(dec, sub, 8, uint8_t)
#ifndef CK_F_PR_DEC_8_ZERO
#define CK_F_PR_DEC_8_ZERO
CK_PR_UNARY_Z_S(dec, 8, uint8_t, -, 1)
+#else
+CK_PR_UNARY_Z_STUB(dec, 8, uint8_t)
#endif /* CK_F_PR_DEC_8_ZERO */
#endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */
diff --git a/include/ck_queue.h b/include/ck_queue.h
index faf96a1771b2..3f503aa6c3e5 100644
--- a/include/ck_queue.h
+++ b/include/ck_queue.h
@@ -180,8 +180,14 @@ struct { \
ck_pr_store_ptr(&(head)->cslh_first, elm); \
} while (0)
+#define CK_SLIST_INSERT_PREVPTR(prevp, slistelm, elm, field) do { \
+ (elm)->field.csle_next = (slistelm); \
+ ck_pr_fence_store(); \
+ ck_pr_store_ptr(prevp, elm); \
+} while (0)
+
#define CK_SLIST_REMOVE_AFTER(elm, field) do { \
- ck_pr_store_ptr(&(elm)->field.csle_next, \
+ ck_pr_store_ptr(&(elm)->field.csle_next, \
(elm)->field.csle_next->field.csle_next); \
} while (0)
@@ -190,7 +196,7 @@ struct { \
CK_SLIST_REMOVE_HEAD((head), field); \
} else { \
struct type *curelm = (head)->cslh_first; \
- while (curelm->field.csle_next != (elm)) \
+ while (curelm->field.csle_next != (elm)) \
curelm = curelm->field.csle_next; \
CK_SLIST_REMOVE_AFTER(curelm, field); \
} \
@@ -201,6 +207,10 @@ struct { \
(head)->cslh_first->field.csle_next); \
} while (0)
+#define CK_SLIST_REMOVE_PREVPTR(prevp, elm, field) do { \
+ ck_pr_store_ptr(prevptr, (elm)->field.csle_next); \
+} while (0)
+
#define CK_SLIST_MOVE(head1, head2, field) do { \
ck_pr_store_ptr(&(head1)->cslh_first, (head2)->cslh_first); \
} while (0)
diff --git a/include/gcc/x86/ck_pr.h b/include/gcc/x86/ck_pr.h
index 3e36376fdd95..e678e830e0b4 100644
--- a/include/gcc/x86/ck_pr.h
+++ b/include/gcc/x86/ck_pr.h
@@ -233,18 +233,18 @@ CK_PR_FAA_S(8, uint8_t, "xaddb")
}
#define CK_PR_UNARY_V(K, S, T, C, I) \
- CK_CC_INLINE static void \
- ck_pr_##K##_##S##_zero(T *target, bool *r) \
+ CK_CC_INLINE static bool \
+ ck_pr_##K##_##S##_is_zero(T *target) \
{ \
+ bool ret; \
__asm__ __volatile__(CK_PR_LOCK_PREFIX I " %0; setz %1" \
: "+m" (*(C *)target), \
- "=m" (*r) \
+ "=rm" (ret) \
: \
: "memory", "cc"); \
- return; \
+ return ret; \
}
-
#define CK_PR_UNARY_S(K, S, T, I) CK_PR_UNARY(K, S, T, T, I)
#define CK_PR_GENERATE(K) \
diff --git a/include/gcc/x86_64/ck_pr.h b/include/gcc/x86_64/ck_pr.h
index 4de1332910ce..fb2804e8d8e5 100644
--- a/include/gcc/x86_64/ck_pr.h
+++ b/include/gcc/x86_64/ck_pr.h
@@ -332,18 +332,18 @@ CK_PR_FAA_S(8, uint8_t, "xaddb")
}
#define CK_PR_UNARY_V(K, S, T, C, I) \
- CK_CC_INLINE static void \
- ck_pr_##K##_##S##_zero(T *target, bool *r) \
+ CK_CC_INLINE static bool \
+ ck_pr_##K##_##S##_is_zero(T *target) \
{ \
+ bool ret; \
__asm__ __volatile__(CK_PR_LOCK_PREFIX I " %0; setz %1" \
: "+m" (*(C *)target), \
- "=m" (*r) \
+ "=rm" (ret) \
: \
: "memory", "cc"); \
- return; \
+ return ret; \
}
-
#define CK_PR_UNARY_S(K, S, T, I) CK_PR_UNARY(K, S, T, T, I)
#define CK_PR_GENERATE(K) \
generated by cgit v1.2.3 (git 2.25.1) at 2024-06-17 03:02:34 +0000