smp.h revision 82bcd89b
1/*
2 * Copyright 2002-2005, Axel D��rfler, axeld@pinc-software.de.
3 * Distributed under the terms of the MIT License.
4 *
5 * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved.
6 * Distributed under the terms of the NewOS License.
7 */
8#ifndef KERNEL_SMP_H
9#define KERNEL_SMP_H
10
11
12#include <arch/atomic.h>
13#include <boot/kernel_args.h>
14#include <kernel.h>
15
16#include <KernelExport.h>
17
18#include <string.h>
19
20
21struct kernel_args;
22
23
24// intercpu messages
25enum {
26	SMP_MSG_INVALIDATE_PAGE_RANGE = 0,
27	SMP_MSG_INVALIDATE_PAGE_LIST,
28	SMP_MSG_USER_INVALIDATE_PAGES,
29	SMP_MSG_GLOBAL_INVALIDATE_PAGES,
30	SMP_MSG_CPU_HALT,
31	SMP_MSG_CALL_FUNCTION,
32	SMP_MSG_RESCHEDULE
33};
34
35enum {
36	SMP_MSG_FLAG_ASYNC		= 0x0,
37	SMP_MSG_FLAG_SYNC		= 0x1,
38	SMP_MSG_FLAG_FREE_ARG	= 0x2,
39};
40
41typedef void (*smp_call_func)(addr_t data1, int32 currentCPU, addr_t data2, addr_t data3);
42
43class CPUSet {
44public:
45	inline				CPUSet();
46
47	inline	void		ClearAll();
48	inline	void		SetAll();
49
50	inline	void		SetBit(int32 cpu);
51	inline	void		ClearBit(int32 cpu);
52
53	inline	void		SetBitAtomic(int32 cpu);
54	inline	void		ClearBitAtomic(int32 cpu);
55
56	inline	bool		GetBit(int32 cpu) const;
57
58	inline	bool		IsEmpty() const;
59
60private:
61	static	const int	kArraySize = ROUNDUP(SMP_MAX_CPUS, 32) / 32;
62
63			uint32		fBitmap[kArraySize];
64};
65
66
67#ifdef __cplusplus
68extern "C" {
69#endif
70
71bool try_acquire_spinlock(spinlock* lock);
72
73status_t smp_init(struct kernel_args *args);
74status_t smp_per_cpu_init(struct kernel_args *args, int32 cpu);
75status_t smp_init_post_generic_syscalls(void);
76bool smp_trap_non_boot_cpus(int32 cpu, uint32* rendezVous);
77void smp_wake_up_non_boot_cpus(void);
78void smp_cpu_rendezvous(uint32* var);
79void smp_send_ici(int32 targetCPU, int32 message, addr_t data, addr_t data2, addr_t data3,
80		void *data_ptr, uint32 flags);
81void smp_send_multicast_ici(CPUSet& cpuMask, int32 message, addr_t data,
82		addr_t data2, addr_t data3, void *data_ptr, uint32 flags);
83void smp_send_broadcast_ici(int32 message, addr_t data, addr_t data2, addr_t data3,
84		void *data_ptr, uint32 flags);
85void smp_send_broadcast_ici_interrupts_disabled(int32 currentCPU, int32 message,
86		addr_t data, addr_t data2, addr_t data3, void *data_ptr, uint32 flags);
87
88int32 smp_get_num_cpus(void);
89void smp_set_num_cpus(int32 numCPUs);
90int32 smp_get_current_cpu(void);
91
92int smp_intercpu_int_handler(int32 cpu);
93
94#ifdef __cplusplus
95}
96#endif
97
98
99inline
100CPUSet::CPUSet()
101{
102	memset(fBitmap, 0, sizeof(fBitmap));
103}
104
105
106inline void
107CPUSet::ClearAll()
108{
109	memset(fBitmap, 0, sizeof(fBitmap));
110}
111
112
113inline void
114CPUSet::SetAll()
115{
116	memset(fBitmap, ~uint8(0), sizeof(fBitmap));
117}
118
119
120inline void
121CPUSet::SetBit(int32 cpu)
122{
123	int32* element = (int32*)&fBitmap[cpu % kArraySize];
124	*element |= 1u << (cpu / kArraySize);
125}
126
127
128inline void
129CPUSet::ClearBit(int32 cpu)
130{
131	int32* element = (int32*)&fBitmap[cpu % kArraySize];
132	*element &= ~uint32(1u << (cpu / kArraySize));
133}
134
135
136inline void
137CPUSet::SetBitAtomic(int32 cpu)
138{
139	int32* element = (int32*)&fBitmap[cpu % kArraySize];
140	atomic_or(element, 1u << (cpu / kArraySize));
141}
142
143
144inline void
145CPUSet::ClearBitAtomic(int32 cpu)
146{
147	int32* element = (int32*)&fBitmap[cpu % kArraySize];
148	atomic_and(element, ~uint32(1u << (cpu / kArraySize)));
149}
150
151
152inline bool
153CPUSet::GetBit(int32 cpu) const
154{
155	int32* element = (int32*)&fBitmap[cpu % kArraySize];
156	return ((uint32)atomic_get(element) & (1u << (cpu / kArraySize))) != 0;
157}
158
159
160inline bool
161CPUSet::IsEmpty() const
162{
163	for (int i = 0; i < kArraySize; i++) {
164		if (fBitmap[i] != 0)
165			return false;
166	}
167
168	return true;
169}
170
171
172// Unless spinlock debug features are enabled, try to inline
173// {acquire,release}_spinlock().
174#if !DEBUG_SPINLOCKS && !B_DEBUG_SPINLOCK_CONTENTION
175
176
177static inline bool
178try_acquire_spinlock_inline(spinlock* lock)
179{
180	return atomic_get_and_set((int32*)lock, 1) == 0;
181}
182
183
184static inline void
185acquire_spinlock_inline(spinlock* lock)
186{
187	if (try_acquire_spinlock_inline(lock))
188		return;
189	acquire_spinlock(lock);
190}
191
192
193static inline void
194release_spinlock_inline(spinlock* lock)
195{
196	atomic_set((int32*)lock, 0);
197}
198
199
200#define try_acquire_spinlock(lock)	try_acquire_spinlock_inline(lock)
201#define acquire_spinlock(lock)		acquire_spinlock_inline(lock)
202#define release_spinlock(lock)		release_spinlock_inline(lock)
203
204
205static inline bool
206try_acquire_write_spinlock_inline(rw_spinlock* lock)
207{
208	return atomic_test_and_set(&lock->lock, 1u << 31, 0) == 0;
209}
210
211
212static inline void
213acquire_write_spinlock_inline(rw_spinlock* lock)
214{
215	if (try_acquire_write_spinlock(lock))
216		return;
217	acquire_write_spinlock(lock);
218}
219
220
221static inline void
222release_write_spinlock_inline(rw_spinlock* lock)
223{
224	atomic_set(&lock->lock, 0);
225}
226
227
228static inline bool
229try_acquire_read_spinlock_inline(rw_spinlock* lock)
230{
231	uint32 previous = atomic_add(&lock->lock, 1);
232	return (previous & (1u << 31)) == 0;
233}
234
235
236static inline void
237acquire_read_spinlock_inline(rw_spinlock* lock)
238{
239	if (try_acquire_read_spinlock(lock))
240		return;
241	acquire_read_spinlock(lock);
242}
243
244
245static inline void
246release_read_spinlock_inline(rw_spinlock* lock)
247{
248	atomic_add(&lock->lock, -1);
249}
250
251
252#define try_acquire_read_spinlock(lock)	try_acquire_read_spinlock_inline(lock)
253#define acquire_read_spinlock(lock)		acquire_read_spinlock_inline(lock)
254#define release_read_spinlock(lock)		release_read_spinlock_inline(lock)
255#define try_acquire_write_spinlock(lock) \
256	try_acquire_write_spinlock(lock)
257#define acquire_write_spinlock(lock)	acquire_write_spinlock_inline(lock)
258#define release_write_spinlock(lock)	release_write_spinlock_inline(lock)
259
260
261static inline bool
262try_acquire_write_seqlock_inline(seqlock* lock) {
263	bool succeed = try_acquire_spinlock(&lock->lock);
264	if (succeed)
265		atomic_add((int32*)&lock->count, 1);
266	return succeed;
267}
268
269
270static inline void
271acquire_write_seqlock_inline(seqlock* lock) {
272	acquire_spinlock(&lock->lock);
273	atomic_add((int32*)&lock->count, 1);
274}
275
276
277static inline void
278release_write_seqlock_inline(seqlock* lock) {
279	atomic_add((int32*)&lock->count, 1);
280	release_spinlock(&lock->lock);
281}
282
283
284static inline uint32
285acquire_read_seqlock_inline(seqlock* lock) {
286	return atomic_get((int32*)&lock->count);
287}
288
289
290static inline bool
291release_read_seqlock_inline(seqlock* lock, uint32 count) {
292	uint32 current = atomic_get((int32*)&lock->count);
293
294	return count % 2 == 0 && current == count;
295}
296
297
298#define try_acquire_write_seqlock(lock)	try_acquire_write_seqlock_inline(lock)
299#define acquire_write_seqlock(lock)		acquire_write_seqlock_inline(lock)
300#define release_write_seqlock(lock)		release_write_seqlock_inline(lock)
301#define acquire_read_seqlock(lock)		acquire_read_seqlock_inline(lock)
302#define release_read_seqlock(lock, count)	\
303	release_read_seqlock_inline(lock, count)
304
305
306#endif	// !DEBUG_SPINLOCKS && !B_DEBUG_SPINLOCK_CONTENTION
307
308
309#endif	/* KERNEL_SMP_H */
310