152a38012Sejakowatz/*
2e736a456SPawel Dziepak * Copyright 2013, Pawe�� Dziepak, pdziepak@quarnos.org.
3671a2442SIngo Weinhold * Copyright 2008-2009, Ingo Weinhold, ingo_weinhold@gmx.de.
491897430SAxel Dörfler * Copyright 2002-2010, Axel D��rfler, axeld@pinc-software.de.
5b82b7593SAxel Dörfler * Distributed under the terms of the MIT License.
6b82b7593SAxel Dörfler *
7b82b7593SAxel Dörfler * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved.
8b82b7593SAxel Dörfler * Distributed under the terms of the NewOS License.
9b82b7593SAxel Dörfler */
1041691b99SAxel Dörfler
116242fd59SAxel Dörfler
126242fd59SAxel Dörfler/*! Functionality for symetrical multi-processors */
136242fd59SAxel Dörfler
147605ddddSAxel Dörfler
1552a38012Sejakowatz#include <smp.h>
1652a38012Sejakowatz
17b82b7593SAxel Dörfler#include <stdlib.h>
1852a38012Sejakowatz#include <string.h>
1952a38012Sejakowatz
208cf8e537SPawel Dziepak#include <arch/atomic.h>
211c8de858SIngo Weinhold#include <arch/cpu.h>
221c8de858SIngo Weinhold#include <arch/debug.h>
231c8de858SIngo Weinhold#include <arch/int.h>
241c8de858SIngo Weinhold#include <arch/smp.h>
2545bd7bb3SIngo Weinhold#include <boot/kernel_args.h>
261c8de858SIngo Weinhold#include <cpu.h>
271c8de858SIngo Weinhold#include <generic_syscall.h>
281c8de858SIngo Weinhold#include <int.h>
291c8de858SIngo Weinhold#include <spinlock_contention.h>
301c8de858SIngo Weinhold#include <thread.h>
31e736a456SPawel Dziepak#include <util/atomic.h>
326242fd59SAxel Dörfler#if DEBUG_SPINLOCK_LATENCIES
336242fd59SAxel Dörfler#	include <safemode.h>
346242fd59SAxel Dörfler#endif
351c8de858SIngo Weinhold
3659dbd26fSIngo Weinhold#include "kernel_debug_config.h"
3759dbd26fSIngo Weinhold
381c8de858SIngo Weinhold
396cd505ceSAxel Dörfler//#define TRACE_SMP
406cd505ceSAxel Dörfler#ifdef TRACE_SMP
413514fd77SPawel Dziepak#	define TRACE(...) dprintf_no_syslog(__VA_ARGS__)
4230753466SAxel Dörfler#else
433514fd77SPawel Dziepak#	define TRACE(...) (void)0
4430753466SAxel Dörfler#endif
4542acb44cSbeveloper
466242fd59SAxel Dörfler
47dc3ba981SIngo Weinhold#undef try_acquire_spinlock
483fb2a94dSIngo Weinhold#undef acquire_spinlock
493fb2a94dSIngo Weinhold#undef release_spinlock
503fb2a94dSIngo Weinhold
51defee266SPawel Dziepak#undef try_acquire_read_spinlock
52defee266SPawel Dziepak#undef acquire_read_spinlock
53defee266SPawel Dziepak#undef release_read_spinlock
54defee266SPawel Dziepak#undef try_acquire_write_spinlock
55defee266SPawel Dziepak#undef acquire_write_spinlock
56defee266SPawel Dziepak#undef release_write_spinlock
57defee266SPawel Dziepak
584824f763SPawel Dziepak#undef try_acquire_write_seqlock
594824f763SPawel Dziepak#undef acquire_write_seqlock
604824f763SPawel Dziepak#undef release_write_seqlock
614824f763SPawel Dziepak#undef acquire_read_seqlock
624824f763SPawel Dziepak#undef release_read_seqlock
634824f763SPawel Dziepak
643fb2a94dSIngo Weinhold
6552a38012Sejakowatz#define MSG_POOL_SIZE (SMP_MAX_CPUS * 4)
6652a38012Sejakowatz
67880d0bdeSIngo Weinhold// These macros define the number of unsuccessful iterations in
68880d0bdeSIngo Weinhold// acquire_spinlock() and acquire_spinlock_nocheck() after which the functions
69880d0bdeSIngo Weinhold// panic(), assuming a deadlock.
70880d0bdeSIngo Weinhold#define SPINLOCK_DEADLOCK_COUNT				100000000
71880d0bdeSIngo Weinhold#define SPINLOCK_DEADLOCK_COUNT_NO_CHECK	2000000000
72880d0bdeSIngo Weinhold
73880d0bdeSIngo Weinhold
7452a38012Sejakowatzstruct smp_msg {
756cd505ceSAxel Dörfler	struct smp_msg	*next;
766cd505ceSAxel Dörfler	int32			message;
7711d3892dSAlex Smith	addr_t			data;
7811d3892dSAlex Smith	addr_t			data2;
7911d3892dSAlex Smith	addr_t			data3;
806cd505ceSAxel Dörfler	void			*data_ptr;
816cd505ceSAxel Dörfler	uint32			flags;
826cd505ceSAxel Dörfler	int32			ref_count;
833514fd77SPawel Dziepak	int32			done;
847629d527SPawel Dziepak	CPUSet			proc_bitmap;
8552a38012Sejakowatz};
8652a38012Sejakowatz
8791897430SAxel Dörflerenum mailbox_source {
8891897430SAxel Dörfler	MAILBOX_LOCAL,
8991897430SAxel Dörfler	MAILBOX_BCAST,
9091897430SAxel Dörfler};
9152a38012Sejakowatz
92077c84ebSPawel Dziepakstatic int32 sBootCPUSpin = 0;
93bd4454cbSIngo Weinhold
947629d527SPawel Dziepakstatic int32 sEarlyCPUCallCount;
957629d527SPawel Dziepakstatic CPUSet sEarlyCPUCallSet;
96bd4454cbSIngo Weinholdstatic void (*sEarlyCPUCallFunction)(void*, int);
97bd4454cbSIngo Weinholdvoid* sEarlyCPUCallCookie;
9852a38012Sejakowatz
9991897430SAxel Dörflerstatic struct smp_msg* sFreeMessages = NULL;
1003514fd77SPawel Dziepakstatic int32 sFreeMessageCount = 0;
10115374c5dSAxel Dörflerstatic spinlock sFreeMessageSpinlock = B_SPINLOCK_INITIALIZER;
10252a38012Sejakowatz
10391897430SAxel Dörflerstatic struct smp_msg* sCPUMessages[SMP_MAX_CPUS] = { NULL, };
10452a38012Sejakowatz
10591897430SAxel Dörflerstatic struct smp_msg* sBroadcastMessages = NULL;
10615374c5dSAxel Dörflerstatic spinlock sBroadcastMessageSpinlock = B_SPINLOCK_INITIALIZER;
1073514fd77SPawel Dziepakstatic int32 sBroadcastMessageCounter;
10852a38012Sejakowatz
1096cd505ceSAxel Dörflerstatic bool sICIEnabled = false;
1106cd505ceSAxel Dörflerstatic int32 sNumCPUs = 1;
11152a38012Sejakowatz
1126cd505ceSAxel Dörflerstatic int32 process_pending_ici(int32 currentCPU);
11352a38012Sejakowatz
11452a38012Sejakowatz
1156cd505ceSAxel Dörfler#if DEBUG_SPINLOCKS
116b82b7593SAxel Dörfler#define NUM_LAST_CALLERS	32
117b82b7593SAxel Dörfler
118b82b7593SAxel Dörflerstatic struct {
119b82b7593SAxel Dörfler	void		*caller;
120b82b7593SAxel Dörfler	spinlock	*lock;
121b82b7593SAxel Dörfler} sLastCaller[NUM_LAST_CALLERS];
12205fd6d79SIngo Weinhold
123077c84ebSPawel Dziepakstatic int32 sLastIndex = 0;
12405fd6d79SIngo Weinhold	// Is incremented atomically. Must be % NUM_LAST_CALLERS before being used
12505fd6d79SIngo Weinhold	// as index into sLastCaller. Note, that it has to be casted to uint32
12605fd6d79SIngo Weinhold	// before applying the modulo operation, since otherwise after overflowing
12705fd6d79SIngo Weinhold	// that would yield negative indices.
128b82b7593SAxel Dörfler
129b82b7593SAxel Dörfler
130b82b7593SAxel Dörflerstatic void
13191897430SAxel Dörflerpush_lock_caller(void* caller, spinlock* lock)
132b82b7593SAxel Dörfler{
13305fd6d79SIngo Weinhold	int32 index = (uint32)atomic_add(&sLastIndex, 1) % NUM_LAST_CALLERS;
134b82b7593SAxel Dörfler
1357ab39de9SIngo Weinhold	sLastCaller[index].caller = caller;
1367ab39de9SIngo Weinhold	sLastCaller[index].lock = lock;
137b82b7593SAxel Dörfler}
138b82b7593SAxel Dörfler
139b82b7593SAxel Dörfler
14091897430SAxel Dörflerstatic void*
14191897430SAxel Dörflerfind_lock_caller(spinlock* lock)
142b82b7593SAxel Dörfler{
143077c84ebSPawel Dziepak	int32 lastIndex = (uint32)atomic_get(&sLastIndex) % NUM_LAST_CALLERS;
144b82b7593SAxel Dörfler
1457ab39de9SIngo Weinhold	for (int32 i = 0; i < NUM_LAST_CALLERS; i++) {
1467ab39de9SIngo Weinhold		int32 index = (NUM_LAST_CALLERS + lastIndex - 1 - i) % NUM_LAST_CALLERS;
1479a63135fSAxel Dörfler		if (sLastCaller[index].lock == lock)
1489a63135fSAxel Dörfler			return sLastCaller[index].caller;
149b82b7593SAxel Dörfler	}
150b82b7593SAxel Dörfler
151b82b7593SAxel Dörfler	return NULL;
152b82b7593SAxel Dörfler}
1537ab39de9SIngo Weinhold
1547ab39de9SIngo Weinhold
1557ab39de9SIngo Weinholdint
1567ab39de9SIngo Weinholddump_spinlock(int argc, char** argv)
1577ab39de9SIngo Weinhold{
1587ab39de9SIngo Weinhold	if (argc != 2) {
1597ab39de9SIngo Weinhold		print_debugger_command_usage(argv[0]);
1607ab39de9SIngo Weinhold		return 0;
1617ab39de9SIngo Weinhold	}
1627ab39de9SIngo Weinhold
1637ab39de9SIngo Weinhold	uint64 address;
1647ab39de9SIngo Weinhold	if (!evaluate_debug_expression(argv[1], &address, false))
1657ab39de9SIngo Weinhold		return 0;
1667ab39de9SIngo Weinhold
1677ab39de9SIngo Weinhold	spinlock* lock = (spinlock*)(addr_t)address;
1687ab39de9SIngo Weinhold	kprintf("spinlock %p:\n", lock);
1697ab39de9SIngo Weinhold	bool locked = B_SPINLOCK_IS_LOCKED(lock);
1707ab39de9SIngo Weinhold	if (locked) {
1717ab39de9SIngo Weinhold		kprintf("  locked from %p\n", find_lock_caller(lock));
1727ab39de9SIngo Weinhold	} else
1737ab39de9SIngo Weinhold		kprintf("  not locked\n");
1747ab39de9SIngo Weinhold
1757ab39de9SIngo Weinhold	return 0;
1767ab39de9SIngo Weinhold}
1777ab39de9SIngo Weinhold
1787ab39de9SIngo Weinhold
179b82b7593SAxel Dörfler#endif	// DEBUG_SPINLOCKS
180b82b7593SAxel Dörfler
181b82b7593SAxel Dörfler
1826242fd59SAxel Dörfler#if DEBUG_SPINLOCK_LATENCIES
1836242fd59SAxel Dörfler
1846242fd59SAxel Dörfler
1856242fd59SAxel Dörfler#define NUM_LATENCY_LOCKS	4
1866242fd59SAxel Dörfler#define DEBUG_LATENCY		200
1876242fd59SAxel Dörfler
1886242fd59SAxel Dörfler
1896242fd59SAxel Dörflerstatic struct {
1906242fd59SAxel Dörfler	spinlock	*lock;
1916242fd59SAxel Dörfler	bigtime_t	timestamp;
1923e0e3be7SPawel Dziepak} sLatency[SMP_MAX_CPUS][NUM_LATENCY_LOCKS];
1936242fd59SAxel Dörfler
1943e0e3be7SPawel Dziepakstatic int32 sLatencyIndex[SMP_MAX_CPUS];
1956242fd59SAxel Dörflerstatic bool sEnableLatencyCheck;
1966242fd59SAxel Dörfler
1976242fd59SAxel Dörfler
1986242fd59SAxel Dörflerstatic void
1996242fd59SAxel Dörflerpush_latency(spinlock* lock)
2006242fd59SAxel Dörfler{
2016242fd59SAxel Dörfler	if (!sEnableLatencyCheck)
2026242fd59SAxel Dörfler		return;
2036242fd59SAxel Dörfler
2046242fd59SAxel Dörfler	int32 cpu = smp_get_current_cpu();
2056242fd59SAxel Dörfler	int32 index = (++sLatencyIndex[cpu]) % NUM_LATENCY_LOCKS;
2066242fd59SAxel Dörfler
2076242fd59SAxel Dörfler	sLatency[cpu][index].lock = lock;
2086242fd59SAxel Dörfler	sLatency[cpu][index].timestamp = system_time();
2096242fd59SAxel Dörfler}
2106242fd59SAxel Dörfler
2116242fd59SAxel Dörfler
2126242fd59SAxel Dörflerstatic void
2136242fd59SAxel Dörflertest_latency(spinlock* lock)
2146242fd59SAxel Dörfler{
2156242fd59SAxel Dörfler	if (!sEnableLatencyCheck)
2166242fd59SAxel Dörfler		return;
2176242fd59SAxel Dörfler
2186242fd59SAxel Dörfler	int32 cpu = smp_get_current_cpu();
2196242fd59SAxel Dörfler
2206242fd59SAxel Dörfler	for (int32 i = 0; i < NUM_LATENCY_LOCKS; i++) {
2216242fd59SAxel Dörfler		if (sLatency[cpu][i].lock == lock) {
2226242fd59SAxel Dörfler			bigtime_t diff = system_time() - sLatency[cpu][i].timestamp;
2236242fd59SAxel Dörfler			if (diff > DEBUG_LATENCY && diff < 500000) {
22441418981SMichael Lotz				panic("spinlock %p was held for %lld usecs (%d allowed)\n",
2256242fd59SAxel Dörfler					lock, diff, DEBUG_LATENCY);
2266242fd59SAxel Dörfler			}
2276242fd59SAxel Dörfler
2286242fd59SAxel Dörfler			sLatency[cpu][i].lock = NULL;
2296242fd59SAxel Dörfler		}
2306242fd59SAxel Dörfler	}
2316242fd59SAxel Dörfler}
2326242fd59SAxel Dörfler
2336242fd59SAxel Dörfler
2346242fd59SAxel Dörfler#endif	// DEBUG_SPINLOCK_LATENCIES
2356242fd59SAxel Dörfler
2366242fd59SAxel Dörfler
2377ab39de9SIngo Weinholdint
2387ab39de9SIngo Weinholddump_ici_messages(int argc, char** argv)
2397ab39de9SIngo Weinhold{
2407ab39de9SIngo Weinhold	// count broadcast messages
2417ab39de9SIngo Weinhold	int32 count = 0;
2427ab39de9SIngo Weinhold	int32 doneCount = 0;
2437ab39de9SIngo Weinhold	int32 unreferencedCount = 0;
2447ab39de9SIngo Weinhold	smp_msg* message = sBroadcastMessages;
2457ab39de9SIngo Weinhold	while (message != NULL) {
2467ab39de9SIngo Weinhold		count++;
2473514fd77SPawel Dziepak		if (message->done == 1)
2487ab39de9SIngo Weinhold			doneCount++;
2497ab39de9SIngo Weinhold		if (message->ref_count <= 0)
2507ab39de9SIngo Weinhold			unreferencedCount++;
2517ab39de9SIngo Weinhold		message = message->next;
2527ab39de9SIngo Weinhold	}
2537ab39de9SIngo Weinhold
2540e88a887SAlex Smith	kprintf("ICI broadcast messages: %" B_PRId32 ", first: %p\n", count,
2557ab39de9SIngo Weinhold		sBroadcastMessages);
2560e88a887SAlex Smith	kprintf("  done:         %" B_PRId32 "\n", doneCount);
2570e88a887SAlex Smith	kprintf("  unreferenced: %" B_PRId32 "\n", unreferencedCount);
2587ab39de9SIngo Weinhold
2597ab39de9SIngo Weinhold	// count per-CPU messages
2607ab39de9SIngo Weinhold	for (int32 i = 0; i < sNumCPUs; i++) {
2617ab39de9SIngo Weinhold		count = 0;
2627ab39de9SIngo Weinhold		message = sCPUMessages[i];
2637ab39de9SIngo Weinhold		while (message != NULL) {
2647ab39de9SIngo Weinhold			count++;
2657ab39de9SIngo Weinhold			message = message->next;
2667ab39de9SIngo Weinhold		}
2677ab39de9SIngo Weinhold
2680e88a887SAlex Smith		kprintf("CPU %" B_PRId32 " messages: %" B_PRId32 ", first: %p\n", i,
2690e88a887SAlex Smith			count, sCPUMessages[i]);
2707ab39de9SIngo Weinhold	}
2717ab39de9SIngo Weinhold
2727ab39de9SIngo Weinhold	return 0;
2737ab39de9SIngo Weinhold}
2747ab39de9SIngo Weinhold
2757ab39de9SIngo Weinhold
2767ab39de9SIngo Weinholdint
2777ab39de9SIngo Weinholddump_ici_message(int argc, char** argv)
2787ab39de9SIngo Weinhold{
2797ab39de9SIngo Weinhold	if (argc != 2) {
2807ab39de9SIngo Weinhold		print_debugger_command_usage(argv[0]);
2817ab39de9SIngo Weinhold		return 0;
2827ab39de9SIngo Weinhold	}
2837ab39de9SIngo Weinhold
2847ab39de9SIngo Weinhold	uint64 address;
2857ab39de9SIngo Weinhold	if (!evaluate_debug_expression(argv[1], &address, false))
2867ab39de9SIngo Weinhold		return 0;
2877ab39de9SIngo Weinhold
2887ab39de9SIngo Weinhold	smp_msg* message = (smp_msg*)(addr_t)address;
2897ab39de9SIngo Weinhold	kprintf("ICI message %p:\n", message);
2907ab39de9SIngo Weinhold	kprintf("  next:        %p\n", message->next);
2910e88a887SAlex Smith	kprintf("  message:     %" B_PRId32 "\n", message->message);
29211d3892dSAlex Smith	kprintf("  data:        0x%lx\n", message->data);
29311d3892dSAlex Smith	kprintf("  data2:       0x%lx\n", message->data2);
29411d3892dSAlex Smith	kprintf("  data3:       0x%lx\n", message->data3);
2957ab39de9SIngo Weinhold	kprintf("  data_ptr:    %p\n", message->data_ptr);
2960e88a887SAlex Smith	kprintf("  flags:       %" B_PRIx32 "\n", message->flags);
2970e88a887SAlex Smith	kprintf("  ref_count:   %" B_PRIx32 "\n", message->ref_count);
2983514fd77SPawel Dziepak	kprintf("  done:        %s\n", message->done == 1 ? "true" : "false");
2997629d527SPawel Dziepak
3007629d527SPawel Dziepak	kprintf("  proc_bitmap: ");
3017629d527SPawel Dziepak	for (int32 i = 0; i < sNumCPUs; i++) {
3027629d527SPawel Dziepak		if (message->proc_bitmap.GetBit(i))
3037629d527SPawel Dziepak			kprintf("%s%" B_PRId32, i != 0 ? ", " : "", i);
3047629d527SPawel Dziepak	}
3057629d527SPawel Dziepak	kprintf("\n");
3067ab39de9SIngo Weinhold
3077ab39de9SIngo Weinhold	return 0;
3087ab39de9SIngo Weinhold}
3097ab39de9SIngo Weinhold
3107ab39de9SIngo Weinhold
3117ab39de9SIngo Weinholdstatic inline void
3127ab39de9SIngo Weinholdprocess_all_pending_ici(int32 currentCPU)
3137ab39de9SIngo Weinhold{
3147ab39de9SIngo Weinhold	while (process_pending_ici(currentCPU) != B_ENTRY_NOT_FOUND)
3157ab39de9SIngo Weinhold		;
3167ab39de9SIngo Weinhold}
3177ab39de9SIngo Weinhold
3187ab39de9SIngo Weinhold
319dc3ba981SIngo Weinholdbool
320dc3ba981SIngo Weinholdtry_acquire_spinlock(spinlock* lock)
321dc3ba981SIngo Weinhold{
322dc3ba981SIngo Weinhold#if DEBUG_SPINLOCKS
323dc3ba981SIngo Weinhold	if (are_interrupts_enabled()) {
324dc3ba981SIngo Weinhold		panic("try_acquire_spinlock: attempt to acquire lock %p with "
325dc3ba981SIngo Weinhold			"interrupts enabled", lock);
326dc3ba981SIngo Weinhold	}
327dc3ba981SIngo Weinhold#endif
328dc3ba981SIngo Weinhold
329dc3ba981SIngo Weinhold#if B_DEBUG_SPINLOCK_CONTENTION
330dc3ba981SIngo Weinhold	if (atomic_add(&lock->lock, 1) != 0)
331dc3ba981SIngo Weinhold		return false;
332dc3ba981SIngo Weinhold#else
333273f2f38SPawel Dziepak	if (atomic_get_and_set((int32*)lock, 1) != 0)
334dc3ba981SIngo Weinhold		return false;
335dc3ba981SIngo Weinhold
336dc3ba981SIngo Weinhold#	if DEBUG_SPINLOCKS
337dc3ba981SIngo Weinhold	push_lock_caller(arch_debug_get_caller(), lock);
338dc3ba981SIngo Weinhold#	endif
339dc3ba981SIngo Weinhold#endif
340dc3ba981SIngo Weinhold
341dc3ba981SIngo Weinhold	return true;
342dc3ba981SIngo Weinhold}
343dc3ba981SIngo Weinhold
344dc3ba981SIngo Weinhold
345fcbbd36eSAxel Dörflervoid
34607655104SIngo Weinholdacquire_spinlock(spinlock* lock)
34752a38012Sejakowatz{
348880d0bdeSIngo Weinhold#if DEBUG_SPINLOCKS
349880d0bdeSIngo Weinhold	if (are_interrupts_enabled()) {
350880d0bdeSIngo Weinhold		panic("acquire_spinlock: attempt to acquire lock %p with interrupts "
351880d0bdeSIngo Weinhold			"enabled", lock);
352880d0bdeSIngo Weinhold	}
353880d0bdeSIngo Weinhold#endif
354880d0bdeSIngo Weinhold
3556cd505ceSAxel Dörfler	if (sNumCPUs > 1) {
3566cd505ceSAxel Dörfler		int currentCPU = smp_get_current_cpu();
3571c8de858SIngo Weinhold#if B_DEBUG_SPINLOCK_CONTENTION
3581c8de858SIngo Weinhold		while (atomic_add(&lock->lock, 1) != 0)
3597ab39de9SIngo Weinhold			process_all_pending_ici(currentCPU);
3601c8de858SIngo Weinhold#else
361fcbbd36eSAxel Dörfler		while (1) {
362880d0bdeSIngo Weinhold			uint32 count = 0;
3633ed7ce75SPawel Dziepak			while (lock->lock != 0) {
364880d0bdeSIngo Weinhold				if (++count == SPINLOCK_DEADLOCK_COUNT) {
3659a633145SMichael Lotz#	if DEBUG_SPINLOCKS
366880d0bdeSIngo Weinhold					panic("acquire_spinlock(): Failed to acquire spinlock %p "
36741418981SMichael Lotz						"for a long time (last caller: %p, value: %" B_PRIx32
36841418981SMichael Lotz						")", lock, find_lock_caller(lock), lock->lock);
3699a633145SMichael Lotz#	else
3709a633145SMichael Lotz					panic("acquire_spinlock(): Failed to acquire spinlock %p "
3719a633145SMichael Lotz						"for a long time (value: %" B_PRIx32 ")", lock,
3729a633145SMichael Lotz						lock->lock);
3739a633145SMichael Lotz#	endif
374880d0bdeSIngo Weinhold					count = 0;
375880d0bdeSIngo Weinhold				}
376880d0bdeSIngo Weinhold
3777ab39de9SIngo Weinhold				process_all_pending_ici(currentCPU);
378e736a456SPawel Dziepak				cpu_wait(&lock->lock, 0);
379d8dd7430Sbeveloper			}
380e736a456SPawel Dziepak			if (atomic_get_and_set(&lock->lock, 1) == 0)
38152a38012Sejakowatz				break;
38252a38012Sejakowatz		}
3837ab39de9SIngo Weinhold
3846242fd59SAxel Dörfler#	if DEBUG_SPINLOCKS
3857ab39de9SIngo Weinhold		push_lock_caller(arch_debug_get_caller(), lock);
3866242fd59SAxel Dörfler#	endif
3871c8de858SIngo Weinhold#endif
38842acb44cSbeveloper	} else {
389b82b7593SAxel Dörfler#if DEBUG_SPINLOCKS
39041418981SMichael Lotz		int32 oldValue = atomic_get_and_set(&lock->lock, 1);
391028aae6cSAxel Dörfler		if (oldValue != 0) {
392880d0bdeSIngo Weinhold			panic("acquire_spinlock: attempt to acquire lock %p twice on "
39341418981SMichael Lotz				"non-SMP system (last caller: %p, value %" B_PRIx32 ")", lock,
394880d0bdeSIngo Weinhold				find_lock_caller(lock), oldValue);
395b82b7593SAxel Dörfler		}
396b82b7593SAxel Dörfler
3979a63135fSAxel Dörfler		push_lock_caller(arch_debug_get_caller(), lock);
398b82b7593SAxel Dörfler#endif
39952a38012Sejakowatz	}
4006242fd59SAxel Dörfler#if DEBUG_SPINLOCK_LATENCIES
4016242fd59SAxel Dörfler	push_latency(lock);
4026242fd59SAxel Dörfler#endif
40352a38012Sejakowatz}
40415374c5dSAxel Dörfler
405fcbbd36eSAxel Dörfler
406fcbbd36eSAxel Dörflerstatic void
4075cdacbaaSlilloacquire_spinlock_nocheck(spinlock *lock)
40852a38012Sejakowatz{
409b82b7593SAxel Dörfler#if DEBUG_SPINLOCKS
410880d0bdeSIngo Weinhold	if (are_interrupts_enabled()) {
411880d0bdeSIngo Weinhold		panic("acquire_spinlock_nocheck: attempt to acquire lock %p with "
412880d0bdeSIngo Weinhold			"interrupts enabled", lock);
413880d0bdeSIngo Weinhold	}
414b82b7593SAxel Dörfler#endif
415880d0bdeSIngo Weinhold
416880d0bdeSIngo Weinhold	if (sNumCPUs > 1) {
4171c8de858SIngo Weinhold#if B_DEBUG_SPINLOCK_CONTENTION
4181c8de858SIngo Weinhold		while (atomic_add(&lock->lock, 1) != 0) {
4191c8de858SIngo Weinhold		}
4201c8de858SIngo Weinhold#else
421fcbbd36eSAxel Dörfler		while (1) {
422880d0bdeSIngo Weinhold			uint32 count = 0;
4233ed7ce75SPawel Dziepak			while (lock->lock != 0) {
424880d0bdeSIngo Weinhold				if (++count == SPINLOCK_DEADLOCK_COUNT_NO_CHECK) {
4259a633145SMichael Lotz#	if DEBUG_SPINLOCKS
42641418981SMichael Lotz					panic("acquire_spinlock_nocheck(): Failed to acquire "
42741418981SMichael Lotz						"spinlock %p for a long time (last caller: %p, value: %"
42841418981SMichael Lotz						B_PRIx32 ")", lock, find_lock_caller(lock), lock->lock);
4299a633145SMichael Lotz#	else
4309a633145SMichael Lotz					panic("acquire_spinlock_nocheck(): Failed to acquire "
4319a633145SMichael Lotz						"spinlock %p for a long time (value: %" B_PRIx32 ")",
4329a633145SMichael Lotz						lock, lock->lock);
4339a633145SMichael Lotz#	endif
434880d0bdeSIngo Weinhold					count = 0;
435880d0bdeSIngo Weinhold				}
436880d0bdeSIngo Weinhold
437e736a456SPawel Dziepak				cpu_wait(&lock->lock, 0);
438880d0bdeSIngo Weinhold			}
439880d0bdeSIngo Weinhold
440e736a456SPawel Dziepak			if (atomic_get_and_set(&lock->lock, 1) == 0)
44152a38012Sejakowatz				break;
44252a38012Sejakowatz		}
443eac94f5dSMichael Lotz
444eac94f5dSMichael Lotz#	if DEBUG_SPINLOCKS
445eac94f5dSMichael Lotz		push_lock_caller(arch_debug_get_caller(), lock);
446eac94f5dSMichael Lotz#	endif
4471c8de858SIngo Weinhold#endif
44842acb44cSbeveloper	} else {
449b82b7593SAxel Dörfler#if DEBUG_SPINLOCKS
45041418981SMichael Lotz		int32 oldValue = atomic_get_and_set(&lock->lock, 1);
45141418981SMichael Lotz		if (oldValue != 0) {
452880d0bdeSIngo Weinhold			panic("acquire_spinlock_nocheck: attempt to acquire lock %p twice "
45341418981SMichael Lotz				"on non-SMP system (last caller: %p, value %" B_PRIx32 ")",
45441418981SMichael Lotz				lock, find_lock_caller(lock), oldValue);
455880d0bdeSIngo Weinhold		}
456eac94f5dSMichael Lotz
457eac94f5dSMichael Lotz		push_lock_caller(arch_debug_get_caller(), lock);
458b82b7593SAxel Dörfler#endif
45952a38012Sejakowatz	}
46052a38012Sejakowatz}
46152a38012Sejakowatz
462fcbbd36eSAxel Dörfler
463671a2442SIngo Weinhold/*!	Equivalent to acquire_spinlock(), save for currentCPU parameter. */
464671a2442SIngo Weinholdstatic void
465671a2442SIngo Weinholdacquire_spinlock_cpu(int32 currentCPU, spinlock *lock)
466671a2442SIngo Weinhold{
467671a2442SIngo Weinhold#if DEBUG_SPINLOCKS
468671a2442SIngo Weinhold	if (are_interrupts_enabled()) {
469671a2442SIngo Weinhold		panic("acquire_spinlock_cpu: attempt to acquire lock %p with "
470671a2442SIngo Weinhold			"interrupts enabled", lock);
471671a2442SIngo Weinhold	}
472671a2442SIngo Weinhold#endif
473671a2442SIngo Weinhold
474671a2442SIngo Weinhold	if (sNumCPUs > 1) {
475671a2442SIngo Weinhold#if B_DEBUG_SPINLOCK_CONTENTION
476671a2442SIngo Weinhold		while (atomic_add(&lock->lock, 1) != 0)
477671a2442SIngo Weinhold			process_all_pending_ici(currentCPU);
478671a2442SIngo Weinhold#else
479671a2442SIngo Weinhold		while (1) {
480671a2442SIngo Weinhold			uint32 count = 0;
4813ed7ce75SPawel Dziepak			while (lock->lock != 0) {
482671a2442SIngo Weinhold				if (++count == SPINLOCK_DEADLOCK_COUNT) {
4839a633145SMichael Lotz#	if DEBUG_SPINLOCKS
484671a2442SIngo Weinhold					panic("acquire_spinlock_cpu(): Failed to acquire spinlock "
48541418981SMichael Lotz						"%p for a long time (last caller: %p, value: %" B_PRIx32
48641418981SMichael Lotz						")", lock, find_lock_caller(lock), lock->lock);
4879a633145SMichael Lotz#	else
4889a633145SMichael Lotz					panic("acquire_spinlock_cpu(): Failed to acquire spinlock "
4899a633145SMichael Lotz						"%p for a long time (value: %" B_PRIx32 ")", lock,
4909a633145SMichael Lotz						lock->lock);
4919a633145SMichael Lotz#	endif
492671a2442SIngo Weinhold					count = 0;
493671a2442SIngo Weinhold				}
494