atomic_prv_x86.h
1 /***************************************************************************
2  Copyright (C) 2002-2015 Kentaro Kitagawa
3  kitagawa@phys.s.u-tokyo.ac.jp
4 
5  This program is free software; you can redistribute it and/or
6  modify it under the terms of the GNU Library General Public
7  License as published by the Free Software Foundation; either
8  version 2 of the License, or (at your option) any later version.
9 
10  You should have received a copy of the GNU Library General
11  Public License and a list of authors along with this program;
12  see the files COPYING and AUTHORS.
13  ***************************************************************************/
14 #ifndef ATOMIC_PRV_X86_H_
15 #define ATOMIC_PRV_X86_H_
16 
17 #include <type_traits>
18 #include <inttypes.h>
19 
20 #include "atomic_prv_mfence_x86.h"
21 
22 #define HAVE_CAS_2
23 
24 typedef intptr_t int_cas2;
25 typedef uintptr_t uint_cas2;
26 typedef int_cas2 int_cas_max;
27 typedef uint_cas2 uint_cas_max;
28 
29 #if !defined __LP64__ && !defined __LLP64__
30  #ifdef __SSE2__
31  #define HAVE_ATOMIC_RW64
32  #endif
33 #endif
34 
35 #ifdef HAVE_ATOMIC_RW64
36 //! \param x must be aligned to 8bytes.
37 template <typename T>
38 inline void atomicWrite64(const T &x, T *target) noexcept {
39  static_assert(sizeof(T) == 8, "");
40  asm (
41  " movq %0, %%xmm0;"
42  " movq %%xmm0, %1;"
43  :
44  : "m" (x), "m" (*target)
45  : "memory", "%xmm0");
46 }
47 
48 //! \param x must be aligned to 8bytes.
49 template <typename T>
50 inline void atomicRead64(T *x, const T &target) noexcept {
51  static_assert(__alignof__(T) >= 8, "");
52  static_assert(sizeof(T) == 8, "");
53  asm (
54  " movq %0, %%xmm0;"
55  " movq %%xmm0, %1;"
56  :
57  : "m" (target), "m" (*x)
58  : "memory", "%xmm0");
59 }
60 #endif
61 
62 //! Compare-And-Swap 2 long words.
63 //! \param oldv0 compared with \p target[0].
64 //! \param oldv1 compared with \p target[1].
65 //! \param newv0 new value to \p target[0].
66 //! \param newv1 new value to \p target[1].
67 template <typename T>
68 inline bool atomicCompareAndSet2(
69  T oldv0, T oldv1,
70  T newv0, T newv1, T *target ) noexcept {
71  unsigned char ret;
72 #if defined __LP64__ || defined __LLP64__
73  asm volatile (
74  " lock; cmpxchg16b (%%rsi);"
75  " sete %0;" // ret = zflag ? 1 : 0
76  : "=q" (ret), "=d" (oldv1), "=a" (oldv0)
77  : "1" (oldv1), "2" (oldv0),
78  "c" (newv1), "b" (newv0),
79  "S" (target)
80  : "memory");
81 #else
82  asm volatile (
83  //gcc with -fPIC cannot handle EBX correctly.
84  " push %%ebx;"
85  " mov %6, %%ebx;"
86  " lock; cmpxchg8b (%%esi);"
87  " pop %%ebx;"
88  " sete %0;" // ret = zflag ? 1 : 0
89  : "=q" (ret), "=d" (oldv1), "=a" (oldv0)
90  : "1" (oldv1), "2" (oldv0),
91  "c" (newv1), "D" (newv0),
92  "S" (target)
93  : "memory");
94 #endif
95  return ret;
96 }
97 template <typename T, typename X>
98 inline typename std::enable_if<
99 std::is_pod<T>::value && (sizeof(T) == sizeof(int_cas2) * 2) && (sizeof(X) == sizeof(int_cas2) * 2), bool>::type
100 atomicCompareAndSet(
101  T oldv,
102  T newv, X *target ) noexcept {
103  union {
104  T x;
105  uint_cas2 w[2];
106  } newv_, oldv_;
107  newv_.x = newv;
108  oldv_.x = oldv;
109 
110  return atomicCompareAndSet2(oldv_.w[0], oldv_.w[1], newv_.w[0], newv_.w[1], (uint_cas2*)(target));
111 }
112 
113 //! \return true if old == *target and new value is assigned
114 template <typename T>
115 inline typename std::enable_if<
116 std::is_pod<T>::value && (sizeof(T) <= sizeof(int_cas_max)), bool>::type
117 atomicCompareAndSet(T oldv, T newv, T *target ) noexcept {
118  unsigned char ret;
119  asm volatile (
120  " lock; cmpxchg %2,%3;"
121  " sete %0" // ret = zflag ? 1 : 0
122  : "=q" (ret), "=a" (oldv)
123  : "q" (newv), "m" ( *target), "1" (oldv)
124  : "memory");
125  return ret;
126 }
127 template <typename T>
128 inline T atomicSwap(T v, T *target ) noexcept {
129  asm volatile (
130  "xchg %0,%1" //lock prefix is not needed.
131  : "=q" (v)
132  : "m" ( *target), "0" (v)
133  : "memory" );
134  return v;
135 }
136 template <typename T>
137 inline void atomicAdd(T *target, T x ) noexcept {
138  asm (
139  "lock; add %1,%0"
140  :
141  : "m" ( *target), "q" (x)
142  : "memory" );
143 }
144 //! \return true if new value is zero.
145 template <typename T>
146 inline bool atomicAddAndTest(T *target, T x ) noexcept {
147  unsigned char ret;
148  asm volatile (
149  "lock; add %2,%1;"
150  " sete %0" // ret = zflag ? 1 : 0
151  : "=q" (ret)
152  : "m" ( *target), "q" (x)
153  : "memory" );
154  return ret;
155 }
156 template <typename T>
157 inline
158 typename std::enable_if<(4 > sizeof(T)), void>::type
159 atomicInc(T *target ) noexcept {
160  asm (
161  "lock; inc%z0 %0"
162  :
163  : "m" ( *target)
164  : "memory" );
165 }
166 template <typename T>
167 inline
168 typename std::enable_if<(4 == sizeof(T)), void>::type //hack for buggy %z.
169 atomicInc(T *target ) noexcept {
170  asm (
171  "lock; incl %0"
172  :
173  : "m" ( *target)
174  : "memory" );
175 }
176 template <typename T>
177 inline
178 typename std::enable_if<(8 == sizeof(T)), void>::type //hack for buggy %z.
179 atomicInc(T *target ) noexcept {
180  asm (
181  "lock; incq %0"
182  :
183  : "m" ( *target)
184  : "memory" );
185 }
186 
187 template <typename T>
188 inline
189 typename std::enable_if<(4 > sizeof(T)), void>::type
190 atomicDec(T *target ) noexcept {
191  asm (
192  "lock; dec%z0 %0"
193  :
194  : "m" ( *target)
195  : "memory" );
196 }
197 template <typename T>
198 inline
199 typename std::enable_if<(4 == sizeof(T)), void>::type //hack for buggy %z.
200 atomicDec(T *target ) noexcept {
201  asm (
202  "lock; decl %0"
203  :
204  : "m" ( *target)
205  : "memory" );
206 }
207 template <typename T>
208 inline
209 typename std::enable_if<(8 == sizeof(T)), void>::type //hack for buggy %z.
210 atomicDec(T *target ) noexcept {
211  asm (
212  "lock; decq %0"
213  :
214  : "m" ( *target)
215  : "memory" );
216 }
217 //! \return zero flag.
218 template <typename T>
219 inline
220 typename std::enable_if<(4 > sizeof(T)), bool>::type
221 atomicDecAndTest(T *target ) noexcept {
222  unsigned char ret;
223  asm volatile (
224  "lock; dec%z1 %1;"
225  " sete %0" // ret = zflag ? 1 : 0
226  : "=q" (ret)
227  : "m" ( *target)
228  : "memory" );
229  return ret;
230 }
231 template <typename T>
232 inline
233 typename std::enable_if<(4 == sizeof(T)), bool>::type //hack for buggy %z.
234 atomicDecAndTest(T *target ) noexcept {
235  unsigned char ret;
236  asm volatile (
237  "lock; decl %1;"
238  " sete %0" // ret = zflag ? 1 : 0
239  : "=q" (ret)
240  : "m" ( *target)
241  : "memory" );
242  return ret;
243 }
244 template <typename T>
245 inline
246 typename std::enable_if<(8 == sizeof(T)), bool>::type //hack for buggy %z.
247 atomicDecAndTest(T *target ) noexcept {
248  unsigned char ret;
249  asm volatile (
250  "lock; decq %1;"
251  " sete %0" // ret = zflag ? 1 : 0
252  : "=q" (ret)
253  : "m" ( *target)
254  : "memory" );
255  return ret;
256 }
257 
258 #endif /*ATOMIC_PRV_X86_H_*/
Generated for KAME4 by  doxygen 1.8.3