152a38012Sejakowatz/*
26cd505ceSAxel Dörfler * Copyright 2002-2005, Axel D��rfler, axeld@pinc-software.de.
36cd505ceSAxel Dörfler * Distributed under the terms of the MIT License.
46cd505ceSAxel Dörfler *
56cd505ceSAxel Dörfler * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved.
66cd505ceSAxel Dörfler * Distributed under the terms of the NewOS License.
76cd505ceSAxel Dörfler */
8564cba31SAxel Dörfler#ifndef KERNEL_SMP_H
9564cba31SAxel Dörfler#define KERNEL_SMP_H
10564cba31SAxel Dörfler
1152a38012Sejakowatz
128cf8e537SPawel Dziepak#include <arch/atomic.h>
137629d527SPawel Dziepak#include <boot/kernel_args.h>
147629d527SPawel Dziepak#include <kernel.h>
157629d527SPawel Dziepak
165cdacbaaSlillo#include <KernelExport.h>
1752a38012Sejakowatz
187629d527SPawel Dziepak#include <string.h>
197629d527SPawel Dziepak
207629d527SPawel Dziepak
21564cba31SAxel Dörflerstruct kernel_args;
22564cba31SAxel Dörfler
23564cba31SAxel Dörfler
2452a38012Sejakowatz// intercpu messages
2552a38012Sejakowatzenum {
26e0e9a3e6SAxel Dörfler	SMP_MSG_INVALIDATE_PAGE_RANGE = 0,
27e0e9a3e6SAxel Dörfler	SMP_MSG_INVALIDATE_PAGE_LIST,
28e0e9a3e6SAxel Dörfler	SMP_MSG_USER_INVALIDATE_PAGES,
29e0e9a3e6SAxel Dörfler	SMP_MSG_GLOBAL_INVALIDATE_PAGES,
3052a38012Sejakowatz	SMP_MSG_CPU_HALT,
31ef7bac18SAxel Dörfler	SMP_MSG_CALL_FUNCTION,
327ea42e7aSPawel Dziepak	SMP_MSG_RESCHEDULE
3352a38012Sejakowatz};
3452a38012Sejakowatz
3552a38012Sejakowatzenum {
36ef7bac18SAxel Dörfler	SMP_MSG_FLAG_ASYNC		= 0x0,
37ef7bac18SAxel Dörfler	SMP_MSG_FLAG_SYNC		= 0x1,
38ef7bac18SAxel Dörfler	SMP_MSG_FLAG_FREE_ARG	= 0x2,
3952a38012Sejakowatz};
4052a38012Sejakowatz
4111d3892dSAlex Smithtypedef void (*smp_call_func)(addr_t data1, int32 currentCPU, addr_t data2, addr_t data3);
42ef7bac18SAxel Dörfler
437629d527SPawel Dziepakclass CPUSet {
447629d527SPawel Dziepakpublic:
45cb66faefSPawel Dziepak	inline				CPUSet();
467629d527SPawel Dziepak
477629d527SPawel Dziepak	inline	void		ClearAll();
487629d527SPawel Dziepak	inline	void		SetAll();
497629d527SPawel Dziepak
507629d527SPawel Dziepak	inline	void		SetBit(int32 cpu);
517629d527SPawel Dziepak	inline	void		ClearBit(int32 cpu);
527629d527SPawel Dziepak
5382bcd89bSPawel Dziepak	inline	void		SetBitAtomic(int32 cpu);
5482bcd89bSPawel Dziepak	inline	void		ClearBitAtomic(int32 cpu);
5582bcd89bSPawel Dziepak
567629d527SPawel Dziepak	inline	bool		GetBit(int32 cpu) const;
577629d527SPawel Dziepak
587629d527SPawel Dziepak	inline	bool		IsEmpty() const;
597629d527SPawel Dziepak
607629d527SPawel Dziepakprivate:
617629d527SPawel Dziepak	static	const int	kArraySize = ROUNDUP(SMP_MAX_CPUS, 32) / 32;
627629d527SPawel Dziepak
637629d527SPawel Dziepak			uint32		fBitmap[kArraySize];
647629d527SPawel Dziepak};
657629d527SPawel Dziepak
66ef7bac18SAxel Dörfler
676cd505ceSAxel Dörfler#ifdef __cplusplus
686cd505ceSAxel Dörflerextern "C" {
696cd505ceSAxel Dörfler#endif
706cd505ceSAxel Dörfler
71dc3ba981SIngo Weinholdbool try_acquire_spinlock(spinlock* lock);
72dc3ba981SIngo Weinhold
734a2e872cSAxel Dörflerstatus_t smp_init(struct kernel_args *args);
744a2e872cSAxel Dörflerstatus_t smp_per_cpu_init(struct kernel_args *args, int32 cpu);
751c8de858SIngo Weinholdstatus_t smp_init_post_generic_syscalls(void);
767f987e49SIngo Weinholdbool smp_trap_non_boot_cpus(int32 cpu, uint32* rendezVous);
776cd505ceSAxel Dörflervoid smp_wake_up_non_boot_cpus(void);
787629d527SPawel Dziepakvoid smp_cpu_rendezvous(uint32* var);
7911d3892dSAlex Smithvoid smp_send_ici(int32 targetCPU, int32 message, addr_t data, addr_t data2, addr_t data3,
806cd505ceSAxel Dörfler		void *data_ptr, uint32 flags);
817629d527SPawel Dziepakvoid smp_send_multicast_ici(CPUSet& cpuMask, int32 message, addr_t data,
8211d3892dSAlex Smith		addr_t data2, addr_t data3, void *data_ptr, uint32 flags);
8311d3892dSAlex Smithvoid smp_send_broadcast_ici(int32 message, addr_t data, addr_t data2, addr_t data3,
846cd505ceSAxel Dörfler		void *data_ptr, uint32 flags);
85671a2442SIngo Weinholdvoid smp_send_broadcast_ici_interrupts_disabled(int32 currentCPU, int32 message,
8611d3892dSAlex Smith		addr_t data, addr_t data2, addr_t data3, void *data_ptr, uint32 flags);
876cd505ceSAxel Dörfler
886cd505ceSAxel Dörflerint32 smp_get_num_cpus(void);
896cd505ceSAxel Dörflervoid smp_set_num_cpus(int32 numCPUs);
906cd505ceSAxel Dörflerint32 smp_get_current_cpu(void);
9152a38012Sejakowatz
92671a2442SIngo Weinholdint smp_intercpu_int_handler(int32 cpu);
9352a38012Sejakowatz
946cd505ceSAxel Dörfler#ifdef __cplusplus
956cd505ceSAxel Dörfler}
966cd505ceSAxel Dörfler#endif
976cd505ceSAxel Dörfler
983fb2a94dSIngo Weinhold
99cb66faefSPawel Dziepakinline
100cb66faefSPawel DziepakCPUSet::CPUSet()
101cb66faefSPawel Dziepak{
102cb66faefSPawel Dziepak	memset(fBitmap, 0, sizeof(fBitmap));
103cb66faefSPawel Dziepak}
104cb66faefSPawel Dziepak
105cb66faefSPawel Dziepak
1067629d527SPawel Dziepakinline void
1077629d527SPawel DziepakCPUSet::ClearAll()
1087629d527SPawel Dziepak{
1097629d527SPawel Dziepak	memset(fBitmap, 0, sizeof(fBitmap));
1107629d527SPawel Dziepak}
1117629d527SPawel Dziepak
1127629d527SPawel Dziepak
1137629d527SPawel Dziepakinline void
1147629d527SPawel DziepakCPUSet::SetAll()
1157629d527SPawel Dziepak{
1167629d527SPawel Dziepak	memset(fBitmap, ~uint8(0), sizeof(fBitmap));
1177629d527SPawel Dziepak}
1187629d527SPawel Dziepak
1197629d527SPawel Dziepak
1207629d527SPawel Dziepakinline void
1217629d527SPawel DziepakCPUSet::SetBit(int32 cpu)
1227629d527SPawel Dziepak{
123f0fe9817SPawel Dziepak	int32* element = (int32*)&fBitmap[cpu % kArraySize];
12482bcd89bSPawel Dziepak	*element |= 1u << (cpu / kArraySize);
1257629d527SPawel Dziepak}
1267629d527SPawel Dziepak
1277629d527SPawel Dziepak
1287629d527SPawel Dziepakinline void
1297629d527SPawel DziepakCPUSet::ClearBit(int32 cpu)
13082bcd89bSPawel Dziepak{
13182bcd89bSPawel Dziepak	int32* element = (int32*)&fBitmap[cpu % kArraySize];
13282bcd89bSPawel Dziepak	*element &= ~uint32(1u << (cpu / kArraySize));
13382bcd89bSPawel Dziepak}
13482bcd89bSPawel Dziepak
13582bcd89bSPawel Dziepak
13682bcd89bSPawel Dziepakinline void
13782bcd89bSPawel DziepakCPUSet::SetBitAtomic(int32 cpu)
13882bcd89bSPawel Dziepak{
13982bcd89bSPawel Dziepak	int32* element = (int32*)&fBitmap[cpu % kArraySize];
14082bcd89bSPawel Dziepak	atomic_or(element, 1u << (cpu / kArraySize));
14182bcd89bSPawel Dziepak}
14282bcd89bSPawel Dziepak
14382bcd89bSPawel Dziepak
14482bcd89bSPawel Dziepakinline void
14582bcd89bSPawel DziepakCPUSet::ClearBitAtomic(int32 cpu)
1467629d527SPawel Dziepak{
147f0fe9817SPawel Dziepak	int32* element = (int32*)&fBitmap[cpu % kArraySize];
1487629d527SPawel Dziepak	atomic_and(element, ~uint32(1u << (cpu / kArraySize)));
1497629d527SPawel Dziepak}
1507629d527SPawel Dziepak
1517629d527SPawel Dziepak
1527629d527SPawel Dziepakinline bool
1537629d527SPawel DziepakCPUSet::GetBit(int32 cpu) const
1547629d527SPawel Dziepak{
1557629d527SPawel Dziepak	int32* element = (int32*)&fBitmap[cpu % kArraySize];
1567629d527SPawel Dziepak	return ((uint32)atomic_get(element) & (1u << (cpu / kArraySize))) != 0;
1577629d527SPawel Dziepak}
1587629d527SPawel Dziepak
1597629d527SPawel Dziepak
1607629d527SPawel Dziepakinline bool
1617629d527SPawel DziepakCPUSet::IsEmpty() const
1627629d527SPawel Dziepak{
1637629d527SPawel Dziepak	for (int i = 0; i < kArraySize; i++) {
1647629d527SPawel Dziepak		if (fBitmap[i] != 0)
1657629d527SPawel Dziepak			return false;
1667629d527SPawel Dziepak	}
1677629d527SPawel Dziepak
1687629d527SPawel Dziepak	return true;
1697629d527SPawel Dziepak}
1707629d527SPawel Dziepak
1717629d527SPawel Dziepak
1723fb2a94dSIngo Weinhold// Unless spinlock debug features are enabled, try to inline
1733fb2a94dSIngo Weinhold// {acquire,release}_spinlock().
1743fb2a94dSIngo Weinhold#if !DEBUG_SPINLOCKS && !B_DEBUG_SPINLOCK_CONTENTION
1753fb2a94dSIngo Weinhold
176dc3ba981SIngo Weinhold
177dc3ba981SIngo Weinholdstatic inline bool
178dc3ba981SIngo Weinholdtry_acquire_spinlock_inline(spinlock* lock)
179dc3ba981SIngo Weinhold{
180273f2f38SPawel Dziepak	return atomic_get_and_set((int32*)lock, 1) == 0;
181dc3ba981SIngo Weinhold}
182dc3ba981SIngo Weinhold
183dc3ba981SIngo Weinhold
1843fb2a94dSIngo Weinholdstatic inline void
1853fb2a94dSIngo Weinholdacquire_spinlock_inline(spinlock* lock)
1863fb2a94dSIngo Weinhold{
187dc3ba981SIngo Weinhold	if (try_acquire_spinlock_inline(lock))
1883fb2a94dSIngo Weinhold		return;
18907655104SIngo Weinhold	acquire_spinlock(lock);
1903fb2a94dSIngo Weinhold}
1913fb2a94dSIngo Weinhold
1923fb2a94dSIngo Weinhold
1933fb2a94dSIngo Weinholdstatic inline void
1943fb2a94dSIngo Weinholdrelease_spinlock_inline(spinlock* lock)
1953fb2a94dSIngo Weinhold{
196273f2f38SPawel Dziepak	atomic_set((int32*)lock, 0);
1973fb2a94dSIngo Weinhold}
1983fb2a94dSIngo Weinhold
1993fb2a94dSIngo Weinhold
200dc3ba981SIngo Weinhold#define try_acquire_spinlock(lock)	try_acquire_spinlock_inline(lock)
201dc3ba981SIngo Weinhold#define acquire_spinlock(lock)		acquire_spinlock_inline(lock)
202dc3ba981SIngo Weinhold#define release_spinlock(lock)		release_spinlock_inline(lock)
2033fb2a94dSIngo Weinhold
2043fb2a94dSIngo Weinhold
205defee266SPawel Dziepakstatic inline bool
206defee266SPawel Dziepaktry_acquire_write_spinlock_inline(rw_spinlock* lock)
207defee266SPawel Dziepak{
2088cf8e537SPawel Dziepak	return atomic_test_and_set(&lock->lock, 1u << 31, 0) == 0;
209defee266SPawel Dziepak}
210defee266SPawel Dziepak
211defee266SPawel Dziepak
212defee266SPawel Dziepakstatic inline void
213defee266SPawel Dziepakacquire_write_spinlock_inline(rw_spinlock* lock)
214defee266SPawel Dziepak{
215defee266SPawel Dziepak	if (try_acquire_write_spinlock(lock))
216defee266SPawel Dziepak		return;
217defee266SPawel Dziepak	acquire_write_spinlock(lock);
218defee266SPawel Dziepak}
219defee266SPawel Dziepak
220defee266SPawel Dziepak
221defee266SPawel Dziepakstatic inline void
222defee266SPawel Dziepakrelease_write_spinlock_inline(rw_spinlock* lock)
223defee266SPawel Dziepak{
224defee266SPawel Dziepak	atomic_set(&lock->lock, 0);
225defee266SPawel Dziepak}
226defee266SPawel Dziepak
227defee266SPawel Dziepak
228defee266SPawel Dziepakstatic inline bool
229defee266SPawel Dziepaktry_acquire_read_spinlock_inline(rw_spinlock* lock)
230defee266SPawel Dziepak{
231defee266SPawel Dziepak	uint32 previous = atomic_add(&lock->lock, 1);
2324ca31ac9SPawel Dziepak	return (previous & (1u << 31)) == 0;
233defee266SPawel Dziepak}
234defee266SPawel Dziepak
235defee266SPawel Dziepak
236defee266SPawel Dziepakstatic inline void
237defee266SPawel Dziepakacquire_read_spinlock_inline(rw_spinlock* lock)
238defee266SPawel Dziepak{
239defee266SPawel Dziepak	if (try_acquire_read_spinlock(lock))
240defee266SPawel Dziepak		return;
241defee266SPawel Dziepak	acquire_read_spinlock(lock);
242defee266SPawel Dziepak}
243defee266SPawel Dziepak
244defee266SPawel Dziepak
245defee266SPawel Dziepakstatic inline void
246defee266SPawel Dziepakrelease_read_spinlock_inline(rw_spinlock* lock)
247defee266SPawel Dziepak{
248defee266SPawel Dziepak	atomic_add(&lock->lock, -1);
249defee266SPawel Dziepak}
250defee266SPawel Dziepak
251defee266SPawel Dziepak
252defee266SPawel Dziepak#define try_acquire_read_spinlock(lock)	try_acquire_read_spinlock_inline(lock)
253defee266SPawel Dziepak#define acquire_read_spinlock(lock)		acquire_read_spinlock_inline(lock)
254defee266SPawel Dziepak#define release_read_spinlock(lock)		release_read_spinlock_inline(lock)
255defee266SPawel Dziepak#define try_acquire_write_spinlock(lock) \
256defee266SPawel Dziepak	try_acquire_write_spinlock(lock)
257defee266SPawel Dziepak#define acquire_write_spinlock(lock)	acquire_write_spinlock_inline(lock)
258defee266SPawel Dziepak#define release_write_spinlock(lock)	release_write_spinlock_inline(lock)
259defee266SPawel Dziepak
260defee266SPawel Dziepak
2614824f763SPawel Dziepakstatic inline bool
2624824f763SPawel Dziepaktry_acquire_write_seqlock_inline(seqlock* lock) {
2634824f763SPawel Dziepak	bool succeed = try_acquire_spinlock(&lock->lock);
2644824f763SPawel Dziepak	if (succeed)
265077c84ebSPawel Dziepak		atomic_add((int32*)&lock->count, 1);
2664824f763SPawel Dziepak	return succeed;
2674824f763SPawel Dziepak}
2684824f763SPawel Dziepak
2694824f763SPawel Dziepak
2704824f763SPawel Dziepakstatic inline void
2714824f763SPawel Dziepakacquire_write_seqlock_inline(seqlock* lock) {
2724824f763SPawel Dziepak	acquire_spinlock(&lock->lock);
273077c84ebSPawel Dziepak	atomic_add((int32*)&lock->count, 1);
2744824f763SPawel Dziepak}
2754824f763SPawel Dziepak
2764824f763SPawel Dziepak
2774824f763SPawel Dziepakstatic inline void
2784824f763SPawel Dziepakrelease_write_seqlock_inline(seqlock* lock) {
279077c84ebSPawel Dziepak	atomic_add((int32*)&lock->count, 1);
2804824f763SPawel Dziepak	release_spinlock(&lock->lock);
2814824f763SPawel Dziepak}
2824824f763SPawel Dziepak
2834824f763SPawel Dziepak
2844824f763SPawel Dziepakstatic inline uint32
2856b83d77fSAugustin Cavalieracquire_read_seqlock_inline(seqlock* lock)
2866b83d77fSAugustin Cavalier{
2876b83d77fSAugustin Cavalier	return (uint32)atomic_get((int32*)&lock->count);
2884824f763SPawel Dziepak}
2894824f763SPawel Dziepak
2904824f763SPawel Dziepak
2914824f763SPawel Dziepakstatic inline bool
2926b83d77fSAugustin Cavalierrelease_read_seqlock_inline(seqlock* lock, uint32 count)
2936b83d77fSAugustin Cavalier{
2946b83d77fSAugustin Cavalier	uint32 current = (uint32)atomic_get((int32*)&lock->count);
2954824f763SPawel Dziepak
2964824f763SPawel Dziepak	return count % 2 == 0 && current == count;
2974824f763SPawel Dziepak}
2984824f763SPawel Dziepak
2994824f763SPawel Dziepak
3004824f763SPawel Dziepak#define try_acquire_write_seqlock(lock)	try_acquire_write_seqlock_inline(lock)
3014824f763SPawel Dziepak#define acquire_write_seqlock(lock)		acquire_write_seqlock_inline(lock)
3024824f763SPawel Dziepak#define release_write_seqlock(lock)		release_write_seqlock_inline(lock)
3034824f763SPawel Dziepak#define acquire_read_seqlock(lock)		acquire_read_seqlock_inline(lock)
3044824f763SPawel Dziepak#define release_read_seqlock(lock, count)	\
3054824f763SPawel Dziepak	release_read_seqlock_inline(lock, count)
3064824f763SPawel Dziepak
3074824f763SPawel Dziepak
308024541a4SPawel Dziepak#endif	// !DEBUG_SPINLOCKS && !B_DEBUG_SPINLOCK_CONTENTION
309024541a4SPawel Dziepak
310024541a4SPawel Dziepak
311564cba31SAxel Dörfler#endif	/* KERNEL_SMP_H */
312