1 #ifndef _ASM_X86_BITOPS_H
2 #define _ASM_X86_BITOPS_H
5 * Copyright 1992, Linus Torvalds.
8 #ifndef _LINUX_BITOPS_H
9 #error only <linux/bitops.h> can be included directly
12 #include <linux/compiler.h>
13 #include <asm/alternative.h>
16 * These have to be done with inline assembly: that way the bit-setting
17 * is guaranteed to be atomic. All bit operations return 0 if the bit
18 * was cleared before the operation and != 0 if it was not.
20 * bit 0 is the LSB of addr; bit 32 is the LSB of (addr+1).
23 #if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 1)
24 /* Technically wrong, but this avoids compilation errors on some gcc
26 #define ADDR "=m" (*(volatile long *)addr)
27 #define BIT_ADDR "=m" (((volatile int *)addr)[nr >> 5])
29 #define ADDR "+m" (*(volatile long *) addr)
30 #define BIT_ADDR "+m" (((volatile int *)addr)[nr >> 5])
32 #define BASE_ADDR "m" (*(volatile int *)addr)
35 * set_bit - Atomically set a bit in memory
37 * @addr: the address to start counting from
39 * This function is atomic and may not be reordered. See __set_bit()
40 * if you do not require the atomic guarantees.
42 * Note: there are no guarantees that this function will not be reordered
43 * on non x86 architectures, so if you are writing portable code,
44 * make sure not to rely on its reordering guarantees.
46 * Note that @nr may be almost arbitrarily large; this function is not
47 * restricted to acting on a single-word quantity.
49 static inline void set_bit(int nr, volatile void *addr)
51 asm volatile(LOCK_PREFIX "bts %1,%0" : ADDR : "Ir" (nr) : "memory");
55 * __set_bit - Set a bit in memory
57 * @addr: the address to start counting from
59 * Unlike set_bit(), this function is non-atomic and may be reordered.
60 * If it's called on the same region of memory simultaneously, the effect
61 * may be that only one operation succeeds.
63 static inline void __set_bit(int nr, volatile void *addr)
65 asm volatile("bts %1,%0"
67 : "Ir" (nr) : "memory");
71 * clear_bit - Clears a bit in memory
73 * @addr: Address to start counting from
75 * clear_bit() is atomic and may not be reordered. However, it does
76 * not contain a memory barrier, so if it is used for locking purposes,
77 * you should call smp_mb__before_clear_bit() and/or smp_mb__after_clear_bit()
78 * in order to ensure changes are visible on other processors.
80 static inline void clear_bit(int nr, volatile void *addr)
82 asm volatile(LOCK_PREFIX "btr %1,%2" : BIT_ADDR : "Ir" (nr), BASE_ADDR);
86 * clear_bit_unlock - Clears a bit in memory
88 * @addr: Address to start counting from
90 * clear_bit() is atomic and implies release semantics before the memory
91 * operation. It can be used for an unlock.
93 static inline void clear_bit_unlock(unsigned nr, volatile void *addr)
99 static inline void __clear_bit(int nr, volatile void *addr)
101 asm volatile("btr %1,%2" : BIT_ADDR : "Ir" (nr), BASE_ADDR);
105 * __clear_bit_unlock - Clears a bit in memory
107 * @addr: Address to start counting from
109 * __clear_bit() is non-atomic and implies release semantics before the memory
110 * operation. It can be used for an unlock if no other CPUs can concurrently
111 * modify other bits in the word.
113 * No memory barrier is required here, because x86 cannot reorder stores past
114 * older loads. Same principle as spin_unlock.
116 static inline void __clear_bit_unlock(unsigned nr, volatile void *addr)
119 __clear_bit(nr, addr);
122 #define smp_mb__before_clear_bit() barrier()
123 #define smp_mb__after_clear_bit() barrier()
126 * __change_bit - Toggle a bit in memory
127 * @nr: the bit to change
128 * @addr: the address to start counting from
130 * Unlike change_bit(), this function is non-atomic and may be reordered.
131 * If it's called on the same region of memory simultaneously, the effect
132 * may be that only one operation succeeds.
134 static inline void __change_bit(int nr, volatile void *addr)
136 asm volatile("btc %1,%2" : BIT_ADDR : "Ir" (nr), BASE_ADDR);
140 * change_bit - Toggle a bit in memory
142 * @addr: Address to start counting from
144 * change_bit() is atomic and may not be reordered.
145 * Note that @nr may be almost arbitrarily large; this function is not
146 * restricted to acting on a single-word quantity.
148 static inline void change_bit(int nr, volatile void *addr)
150 asm volatile(LOCK_PREFIX "btc %1,%2" : BIT_ADDR : "Ir" (nr), BASE_ADDR);
154 * test_and_set_bit - Set a bit and return its old value
156 * @addr: Address to count from
158 * This operation is atomic and cannot be reordered.
159 * It also implies a memory barrier.
161 static inline int test_and_set_bit(int nr, volatile void *addr)
165 asm volatile(LOCK_PREFIX "bts %2,%1\n\t"
166 "sbb %0,%0" : "=r" (oldbit), ADDR : "Ir" (nr) : "memory");
172 * test_and_set_bit_lock - Set a bit and return its old value for lock
174 * @addr: Address to count from
176 * This is the same as test_and_set_bit on x86.
178 static inline int test_and_set_bit_lock(int nr, volatile void *addr)
180 return test_and_set_bit(nr, addr);
184 * __test_and_set_bit - Set a bit and return its old value
186 * @addr: Address to count from
188 * This operation is non-atomic and can be reordered.
189 * If two examples of this operation race, one can appear to succeed
190 * but actually fail. You must protect multiple accesses with a lock.
192 static inline int __test_and_set_bit(int nr, volatile void *addr)
196 asm volatile("bts %2,%3\n\t"
198 : "=r" (oldbit), BIT_ADDR : "Ir" (nr), BASE_ADDR);
203 * test_and_clear_bit - Clear a bit and return its old value
205 * @addr: Address to count from
207 * This operation is atomic and cannot be reordered.
208 * It also implies a memory barrier.
210 static inline int test_and_clear_bit(int nr, volatile void *addr)
214 asm volatile(LOCK_PREFIX "btr %2,%1\n\t"
216 : "=r" (oldbit), ADDR : "Ir" (nr) : "memory");
222 * __test_and_clear_bit - Clear a bit and return its old value
224 * @addr: Address to count from
226 * This operation is non-atomic and can be reordered.
227 * If two examples of this operation race, one can appear to succeed
228 * but actually fail. You must protect multiple accesses with a lock.
230 static inline int __test_and_clear_bit(int nr, volatile void *addr)
234 asm volatile("btr %2,%3\n\t"
236 : "=r" (oldbit), BIT_ADDR : "Ir" (nr), BASE_ADDR);
240 /* WARNING: non atomic and it can be reordered! */
241 static inline int __test_and_change_bit(int nr, volatile void *addr)
245 asm volatile("btc %2,%3\n\t"
247 : "=r" (oldbit), BIT_ADDR : "Ir" (nr), BASE_ADDR);
253 * test_and_change_bit - Change a bit and return its old value
255 * @addr: Address to count from
257 * This operation is atomic and cannot be reordered.
258 * It also implies a memory barrier.
260 static inline int test_and_change_bit(int nr, volatile void *addr)
264 asm volatile(LOCK_PREFIX "btc %2,%1\n\t"
266 : "=r" (oldbit), ADDR : "Ir" (nr) : "memory");
271 static inline int constant_test_bit(int nr, const volatile void *addr)
273 return ((1UL << (nr % BITS_PER_LONG)) &
274 (((unsigned long *)addr)[nr / BITS_PER_LONG])) != 0;
277 static inline int variable_test_bit(int nr, volatile const void *addr)
281 asm volatile("bt %2,%3\n\t"
284 : "m" (((volatile const int *)addr)[nr >> 5]),
285 "Ir" (nr), BASE_ADDR);
290 #if 0 /* Fool kernel-doc since it doesn't do macros yet */
292 * test_bit - Determine whether a bit is set
293 * @nr: bit number to test
294 * @addr: Address to start counting from
296 static int test_bit(int nr, const volatile unsigned long *addr);
299 #define test_bit(nr,addr) \
300 (__builtin_constant_p(nr) ? \
301 constant_test_bit((nr),(addr)) : \
302 variable_test_bit((nr),(addr)))
307 * __ffs - find first set bit in word
308 * @word: The word to search
310 * Undefined if no bit exists, so code should check against 0 first.
312 static inline unsigned long __ffs(unsigned long word)
321 * ffz - find first zero bit in word
322 * @word: The word to search
324 * Undefined if no zero exists, so code should check against ~0UL first.
326 static inline unsigned long ffz(unsigned long word)
335 * __fls: find last set bit in word
336 * @word: The word to search
338 * Undefined if no zero exists, so code should check against ~0UL first.
340 static inline unsigned long __fls(unsigned long word)
350 * ffs - find first set bit in word
351 * @x: the word to search
353 * This is defined the same way as the libc and compiler builtin ffs
354 * routines, therefore differs in spirit from the other bitops.
356 * ffs(value) returns 0 if value is 0 or the position of the first
357 * set bit if value is nonzero. The first (least significant) bit
360 static inline int ffs(int x)
363 #ifdef CONFIG_X86_CMOV
364 __asm__("bsfl %1,%0\n\t"
366 : "=r" (r) : "rm" (x), "r" (-1));
368 __asm__("bsfl %1,%0\n\t"
371 "1:" : "=r" (r) : "rm" (x));
377 * fls - find last set bit in word
378 * @x: the word to search
380 * This is defined in a similar way as the libc and compiler builtin
381 * ffs, but returns the position of the most significant set bit.
383 * fls(value) returns 0 if value is 0 or the position of the last
384 * set bit if value is nonzero. The last (most significant) bit is
387 static inline int fls(int x)
390 #ifdef CONFIG_X86_CMOV
391 __asm__("bsrl %1,%0\n\t"
393 : "=&r" (r) : "rm" (x), "rm" (-1));
395 __asm__("bsrl %1,%0\n\t"
398 "1:" : "=r" (r) : "rm" (x));
402 #endif /* __KERNEL__ */
407 # include "bitops_32.h"
409 # include "bitops_64.h"
412 #endif /* _ASM_X86_BITOPS_H */