1/*
2 * Copyright 2018, J��r��me Duval, jerome.duval@gmail.com.
3 * Copyright 2002-2010, Axel D��rfler, axeld@pinc-software.de.
4 * Copyright 2013, Pawe�� Dziepak, pdziepak@quarnos.org.
5 * Copyright 2012, Alex Smith, alex@alex-smith.me.uk.
6 * Distributed under the terms of the MIT License.
7 *
8 * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved.
9 * Distributed under the terms of the NewOS License.
10 */
11
12
13#include <cpu.h>
14
15#include <string.h>
16#include <stdlib.h>
17#include <stdio.h>
18
19#include <algorithm>
20
21#include <ACPI.h>
22
23#include <boot_device.h>
24#include <commpage.h>
25#include <debug.h>
26#include <elf.h>
27#include <safemode.h>
28#include <smp.h>
29#include <util/BitUtils.h>
30#include <vm/vm.h>
31#include <vm/vm_types.h>
32#include <vm/VMAddressSpace.h>
33
34#include <arch_system_info.h>
35#include <arch/x86/apic.h>
36#include <boot/kernel_args.h>
37
38#include "paging/X86PagingStructures.h"
39#include "paging/X86VMTranslationMap.h"
40
41
42#define DUMP_FEATURE_STRING 1
43#define DUMP_CPU_TOPOLOGY	1
44
45
46/* cpu vendor info */
47struct cpu_vendor_info {
48	const char *vendor;
49	const char *ident_string[2];
50};
51
52static const struct cpu_vendor_info vendor_info[VENDOR_NUM] = {
53	{ "Intel", { "GenuineIntel" } },
54	{ "AMD", { "AuthenticAMD" } },
55	{ "Cyrix", { "CyrixInstead" } },
56	{ "UMC", { "UMC UMC UMC" } },
57	{ "NexGen", { "NexGenDriven" } },
58	{ "Centaur", { "CentaurHauls" } },
59	{ "Rise", { "RiseRiseRise" } },
60	{ "Transmeta", { "GenuineTMx86", "TransmetaCPU" } },
61	{ "NSC", { "Geode by NSC" } },
62};
63
64#define K8_SMIONCMPHALT			(1ULL << 27)
65#define K8_C1EONCMPHALT			(1ULL << 28)
66
67#define K8_CMPHALT				(K8_SMIONCMPHALT | K8_C1EONCMPHALT)
68
69struct set_mtrr_parameter {
70	int32	index;
71	uint64	base;
72	uint64	length;
73	uint8	type;
74};
75
76struct set_mtrrs_parameter {
77	const x86_mtrr_info*	infos;
78	uint32					count;
79	uint8					defaultType;
80};
81
82
83#ifdef __x86_64__
84extern addr_t _stac;
85extern addr_t _clac;
86#endif
87
88extern "C" void x86_reboot(void);
89	// from arch.S
90
91void (*gCpuIdleFunc)(void);
92#ifndef __x86_64__
93void (*gX86SwapFPUFunc)(void* oldState, const void* newState) = x86_noop_swap;
94bool gHasSSE = false;
95#endif
96
97static uint32 sCpuRendezvous;
98static uint32 sCpuRendezvous2;
99static uint32 sCpuRendezvous3;
100static vint32 sTSCSyncRendezvous;
101
102/* Some specials for the double fault handler */
103static uint8* sDoubleFaultStacks;
104static const size_t kDoubleFaultStackSize = 4096;	// size per CPU
105
106static x86_cpu_module_info* sCpuModule;
107
108
109/* CPU topology information */
110static uint32 (*sGetCPUTopologyID)(int currentCPU);
111static uint32 sHierarchyMask[CPU_TOPOLOGY_LEVELS];
112static uint32 sHierarchyShift[CPU_TOPOLOGY_LEVELS];
113
114/* Cache topology information */
115static uint32 sCacheSharingMask[CPU_MAX_CACHE_LEVEL];
116
117
118static status_t
119acpi_shutdown(bool rebootSystem)
120{
121	if (debug_debugger_running() || !are_interrupts_enabled())
122		return B_ERROR;
123
124	acpi_module_info* acpi;
125	if (get_module(B_ACPI_MODULE_NAME, (module_info**)&acpi) != B_OK)
126		return B_NOT_SUPPORTED;
127
128	status_t status;
129	if (rebootSystem) {
130		status = acpi->reboot();
131	} else {
132		status = acpi->prepare_sleep_state(ACPI_POWER_STATE_OFF, NULL, 0);
133		if (status == B_OK) {
134			//cpu_status state = disable_interrupts();
135			status = acpi->enter_sleep_state(ACPI_POWER_STATE_OFF);
136			//restore_interrupts(state);
137		}
138	}
139
140	put_module(B_ACPI_MODULE_NAME);
141	return status;
142}
143
144
145/*!	Disable CPU caches, and invalidate them. */
146static void
147disable_caches()
148{
149	x86_write_cr0((x86_read_cr0() | CR0_CACHE_DISABLE)
150		& ~CR0_NOT_WRITE_THROUGH);
151	wbinvd();
152	arch_cpu_global_TLB_invalidate();
153}
154
155
156/*!	Invalidate CPU caches, and enable them. */
157static void
158enable_caches()
159{
160	wbinvd();
161	arch_cpu_global_TLB_invalidate();
162	x86_write_cr0(x86_read_cr0()
163		& ~(CR0_CACHE_DISABLE | CR0_NOT_WRITE_THROUGH));
164}
165
166
167static void
168set_mtrr(void* _parameter, int cpu)
169{
170	struct set_mtrr_parameter* parameter
171		= (struct set_mtrr_parameter*)_parameter;
172
173	// wait until all CPUs have arrived here
174	smp_cpu_rendezvous(&sCpuRendezvous);
175
176	// One CPU has to reset sCpuRendezvous3 -- it is needed to prevent the CPU
177	// that initiated the call_all_cpus() from doing that again and clearing
178	// sCpuRendezvous2 before the last CPU has actually left the loop in
179	// smp_cpu_rendezvous();
180	if (cpu == 0)
181		atomic_set((int32*)&sCpuRendezvous3, 0);
182
183	disable_caches();
184
185	sCpuModule->set_mtrr(parameter->index, parameter->base, parameter->length,
186		parameter->type);
187
188	enable_caches();
189
190	// wait until all CPUs have arrived here
191	smp_cpu_rendezvous(&sCpuRendezvous2);
192	smp_cpu_rendezvous(&sCpuRendezvous3);
193}
194
195
196static void
197set_mtrrs(void* _parameter, int cpu)
198{
199	set_mtrrs_parameter* parameter = (set_mtrrs_parameter*)_parameter;
200
201	// wait until all CPUs have arrived here
202	smp_cpu_rendezvous(&sCpuRendezvous);
203
204	// One CPU has to reset sCpuRendezvous3 -- it is needed to prevent the CPU
205	// that initiated the call_all_cpus() from doing that again and clearing
206	// sCpuRendezvous2 before the last CPU has actually left the loop in
207	// smp_cpu_rendezvous();
208	if (cpu == 0)
209		atomic_set((int32*)&sCpuRendezvous3, 0);
210
211	disable_caches();
212
213	sCpuModule->set_mtrrs(parameter->defaultType, parameter->infos,
214		parameter->count);
215
216	enable_caches();
217
218	// wait until all CPUs have arrived here
219	smp_cpu_rendezvous(&sCpuRendezvous2);
220	smp_cpu_rendezvous(&sCpuRendezvous3);
221}
222
223
224static void
225init_mtrrs(void* _unused, int cpu)
226{
227	// wait until all CPUs have arrived here
228	smp_cpu_rendezvous(&sCpuRendezvous);
229
230	// One CPU has to reset sCpuRendezvous3 -- it is needed to prevent the CPU
231	// that initiated the call_all_cpus() from doing that again and clearing
232	// sCpuRendezvous2 before the last CPU has actually left the loop in
233	// smp_cpu_rendezvous();
234	if (cpu == 0)
235		atomic_set((int32*)&sCpuRendezvous3, 0);
236
237	disable_caches();
238
239	sCpuModule->init_mtrrs();
240
241	enable_caches();
242
243	// wait until all CPUs have arrived here
244	smp_cpu_rendezvous(&sCpuRendezvous2);
245	smp_cpu_rendezvous(&sCpuRendezvous3);
246}
247
248
249uint32
250x86_count_mtrrs(void)
251{
252	if (sCpuModule == NULL)
253		return 0;
254
255	return sCpuModule->count_mtrrs();
256}
257
258
259void
260x86_set_mtrr(uint32 index, uint64 base, uint64 length, uint8 type)
261{
262	struct set_mtrr_parameter parameter;
263	parameter.index = index;
264	parameter.base = base;
265	parameter.length = length;
266	parameter.type = type;
267
268	sCpuRendezvous = sCpuRendezvous2 = 0;
269	call_all_cpus(&set_mtrr, &parameter);
270}
271
272
273status_t
274x86_get_mtrr(uint32 index, uint64* _base, uint64* _length, uint8* _type)
275{
276	// the MTRRs are identical on all CPUs, so it doesn't matter
277	// on which CPU this runs
278	return sCpuModule->get_mtrr(index, _base, _length, _type);
279}
280
281
282void
283x86_set_mtrrs(uint8 defaultType, const x86_mtrr_info* infos, uint32 count)
284{
285	if (sCpuModule == NULL)
286		return;
287
288	struct set_mtrrs_parameter parameter;
289	parameter.defaultType = defaultType;
290	parameter.infos = infos;
291	parameter.count = count;
292
293	sCpuRendezvous = sCpuRendezvous2 = 0;
294	call_all_cpus(&set_mtrrs, &parameter);
295}
296
297
298void
299x86_init_fpu(void)
300{
301	// All x86_64 CPUs support SSE, don't need to bother checking for it.
302#ifndef __x86_64__
303	if (!x86_check_feature(IA32_FEATURE_FPU, FEATURE_COMMON)) {
304		// No FPU... time to install one in your 386?
305		dprintf("%s: Warning: CPU has no reported FPU.\n", __func__);
306		gX86SwapFPUFunc = x86_noop_swap;
307		return;
308	}
309
310	if (!x86_check_feature(IA32_FEATURE_SSE, FEATURE_COMMON)
311		|| !x86_check_feature(IA32_FEATURE_FXSR, FEATURE_COMMON)) {
312		dprintf("%s: CPU has no SSE... just enabling FPU.\n", __func__);
313		// we don't have proper SSE support, just enable FPU
314		x86_write_cr0(x86_read_cr0() & ~(CR0_FPU_EMULATION | CR0_MONITOR_FPU));
315		gX86SwapFPUFunc = x86_fnsave_swap;
316		return;
317	}
318#endif
319
320	dprintf("%s: CPU has SSE... enabling FXSR and XMM.\n", __func__);
321#ifndef __x86_64__
322	// enable OS support for SSE
323	x86_write_cr4(x86_read_cr4() | CR4_OS_FXSR | CR4_OS_XMM_EXCEPTION);
324	x86_write_cr0(x86_read_cr0() & ~(CR0_FPU_EMULATION | CR0_MONITOR_FPU));
325
326	gX86SwapFPUFunc = x86_fxsave_swap;
327	gHasSSE = true;
328#endif
329}
330
331
332#if DUMP_FEATURE_STRING
333static void
334dump_feature_string(int currentCPU, cpu_ent* cpu)
335{
336	char features[512];
337	features[0] = 0;
338
339	if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_FPU)
340		strlcat(features, "fpu ", sizeof(features));
341	if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_VME)
342		strlcat(features, "vme ", sizeof(features));
343	if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_DE)
344		strlcat(features, "de ", sizeof(features));
345	if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PSE)
346		strlcat(features, "pse ", sizeof(features));
347	if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_TSC)
348		strlcat(features, "tsc ", sizeof(features));
349	if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MSR)
350		strlcat(features, "msr ", sizeof(features));
351	if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PAE)
352		strlcat(features, "pae ", sizeof(features));
353	if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MCE)
354		strlcat(features, "mce ", sizeof(features));
355	if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_CX8)
356		strlcat(features, "cx8 ", sizeof(features));
357	if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_APIC)
358		strlcat(features, "apic ", sizeof(features));
359	if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SEP)
360		strlcat(features, "sep ", sizeof(features));
361	if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MTRR)
362		strlcat(features, "mtrr ", sizeof(features));
363	if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PGE)
364		strlcat(features, "pge ", sizeof(features));
365	if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MCA)
366		strlcat(features, "mca ", sizeof(features));
367	if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_CMOV)
368		strlcat(features, "cmov ", sizeof(features));
369	if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PAT)
370		strlcat(features, "pat ", sizeof(features));
371	if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PSE36)
372		strlcat(features, "pse36 ", sizeof(features));
373	if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PSN)
374		strlcat(features, "psn ", sizeof(features));
375	if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_CLFSH)
376		strlcat(features, "clfsh ", sizeof(features));
377	if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_DS)
378		strlcat(features, "ds ", sizeof(features));
379	if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_ACPI)
380		strlcat(features, "acpi ", sizeof(features));
381	if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MMX)
382		strlcat(features, "mmx ", sizeof(features));
383	if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_FXSR)
384		strlcat(features, "fxsr ", sizeof(features));
385	if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SSE)
386		strlcat(features, "sse ", sizeof(features));
387	if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SSE2)
388		strlcat(features, "sse2 ", sizeof(features));
389	if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SS)
390		strlcat(features, "ss ", sizeof(features));
391	if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_HTT)
392		strlcat(features, "htt ", sizeof(features));
393	if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_TM)
394		strlcat(features, "tm ", sizeof(features));
395	if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PBE)
396		strlcat(features, "pbe ", sizeof(features));
397	if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SSE3)
398		strlcat(features, "sse3 ", sizeof(features));
399	if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_PCLMULQDQ)
400		strlcat(features, "pclmulqdq ", sizeof(features));
401	if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_DTES64)
402		strlcat(features, "dtes64 ", sizeof(features));
403	if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_MONITOR)
404		strlcat(features, "monitor ", sizeof(features));
405	if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_DSCPL)
406		strlcat(features, "dscpl ", sizeof(features));
407	if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_VMX)
408		strlcat(features, "vmx ", sizeof(features));
409	if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SMX)
410		strlcat(features, "smx ", sizeof(features));
411	if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_EST)
412		strlcat(features, "est ", sizeof(features));
413	if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_TM2)
414		strlcat(features, "tm2 ", sizeof(features));
415	if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SSSE3)
416		strlcat(features, "ssse3 ", sizeof(features));
417	if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_CNXTID)
418		strlcat(features, "cnxtid ", sizeof(features));
419	if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_FMA)
420		strlcat(features, "fma ", sizeof(features));
421	if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_CX16)
422		strlcat(features, "cx16 ", sizeof(features));
423	if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_XTPR)
424		strlcat(features, "xtpr ", sizeof(features));
425	if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_PDCM)
426		strlcat(features, "pdcm ", sizeof(features));
427	if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_PCID)
428		strlcat(features, "pcid ", sizeof(features));
429	if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_DCA)
430		strlcat(features, "dca ", sizeof(features));
431	if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SSE4_1)
432		strlcat(features, "sse4_1 ", sizeof(features));
433	if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SSE4_2)
434		strlcat(features, "sse4_2 ", sizeof(features));
435	if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_X2APIC)
436		strlcat(features, "x2apic ", sizeof(features));
437	if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_MOVBE)
438		strlcat(features, "movbe ", sizeof(features));
439	if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_POPCNT)
440		strlcat(features, "popcnt ", sizeof(features));
441	if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_TSCDEADLINE)
442		strlcat(features, "tscdeadline ", sizeof(features));
443	if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_AES)
444		strlcat(features, "aes ", sizeof(features));
445	if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_XSAVE)
446		strlcat(features, "xsave ", sizeof(features));
447	if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_OSXSAVE)
448		strlcat(features, "osxsave ", sizeof(features));
449	if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_AVX)
450		strlcat(features, "avx ", sizeof(features));
451	if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_F16C)
452		strlcat(features, "f16c ", sizeof(features));
453	if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_RDRND)
454		strlcat(features, "rdrnd ", sizeof(features));
455	if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_HYPERVISOR)
456		strlcat(features, "hypervisor ", sizeof(features));
457	if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_SYSCALL)
458		strlcat(features, "syscall ", sizeof(features));
459	if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_NX)
460		strlcat(features, "nx ", sizeof(features));
461	if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_MMXEXT)
462		strlcat(features, "mmxext ", sizeof(features));
463	if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_FFXSR)
464		strlcat(features, "ffxsr ", sizeof(features));
465	if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_LONG)
466		strlcat(features, "long ", sizeof(features));
467	if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_3DNOWEXT)
468		strlcat(features, "3dnowext ", sizeof(features));
469	if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_3DNOW)
470		strlcat(features, "3dnow ", sizeof(features));
471	if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_DTS)
472		strlcat(features, "dts ", sizeof(features));
473	if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_ITB)
474		strlcat(features, "itb ", sizeof(features));
475	if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_ARAT)
476		strlcat(features, "arat ", sizeof(features));
477	if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_PLN)
478		strlcat(features, "pln ", sizeof(features));
479	if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_ECMD)
480		strlcat(features, "ecmd ", sizeof(features));
481	if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_PTM)
482		strlcat(features, "ptm ", sizeof(features));
483	if (cpu->arch.feature[FEATURE_6_ECX] & IA32_FEATURE_APERFMPERF)
484		strlcat(features, "aperfmperf ", sizeof(features));
485	if (cpu->arch.feature[FEATURE_6_ECX] & IA32_FEATURE_EPB)
486		strlcat(features, "epb ", sizeof(features));
487	if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_TSC_ADJUST)
488		strlcat(features, "tsc_adjust ", sizeof(features));
489	if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_SGX)
490		strlcat(features, "sgx ", sizeof(features));
491	if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_BMI1)
492		strlcat(features, "bmi1 ", sizeof(features));
493	if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_HLE)
494		strlcat(features, "hle ", sizeof(features));
495	if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_AVX2)
496		strlcat(features, "avx2 ", sizeof(features));
497	if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_SMEP)
498		strlcat(features, "smep ", sizeof(features));
499	if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_BMI2)
500		strlcat(features, "bmi2 ", sizeof(features));
501	if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_ERMS)
502		strlcat(features, "erms ", sizeof(features));
503	if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_INVPCID)
504		strlcat(features, "invpcid ", sizeof(features));
505	if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_RTM)
506		strlcat(features, "rtm ", sizeof(features));
507	if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_CQM)
508		strlcat(features, "cqm ", sizeof(features));
509	if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_MPX)
510		strlcat(features, "mpx ", sizeof(features));
511	if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_RDT_A)
512		strlcat(features, "rdt_a ", sizeof(features));
513	if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_AVX512F)
514		strlcat(features, "avx512f ", sizeof(features));
515	if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_AVX512DQ)
516		strlcat(features, "avx512dq ", sizeof(features));
517	if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_RDSEED)
518		strlcat(features, "rdseed ", sizeof(features));
519	if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_ADX)
520		strlcat(features, "adx ", sizeof(features));
521	if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_SMAP)
522		strlcat(features, "smap ", sizeof(features));
523	if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_AVX512IFMA)
524		strlcat(features, "avx512ifma ", sizeof(features));
525	if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_PCOMMIT)
526		strlcat(features, "pcommit ", sizeof(features));
527	if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_CLFLUSHOPT)
528		strlcat(features, "cflushopt ", sizeof(features));
529	if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_CLWB)
530		strlcat(features, "clwb ", sizeof(features));
531	if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_INTEL_PT)
532		strlcat(features, "intel_pt ", sizeof(features));
533	if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_AVX512PF)
534		strlcat(features, "avx512pf ", sizeof(features));
535	if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_AVX512ER)
536		strlcat(features, "avx512er ", sizeof(features));
537	if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_AVX512CD)
538		strlcat(features, "avx512cd ", sizeof(features));
539	if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_SHA_NI)
540		strlcat(features, "sha_ni ", sizeof(features));
541	if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_AVX512BW)
542		strlcat(features, "avx512bw ", sizeof(features));
543	if (cpu->arch.feature[FEATURE_7_EBX] & IA32_FEATURE_AVX512VI)
544		strlcat(features, "avx512vi ", sizeof(features));
545	if (cpu->arch.feature[FEATURE_7_EDX] & IA32_FEATURE_IBRS)
546		strlcat(features, "ibrs ", sizeof(features));
547	if (cpu->arch.feature[FEATURE_7_EDX] & IA32_FEATURE_STIBP)
548		strlcat(features, "stibp ", sizeof(features));
549	if (cpu->arch.feature[FEATURE_7_EDX] & IA32_FEATURE_L1D_FLUSH)
550		strlcat(features, "l1d_flush ", sizeof(features));
551	if (cpu->arch.feature[FEATURE_7_EDX] & IA32_FEATURE_ARCH_CAPABILITIES)
552		strlcat(features, "msr_arch ", sizeof(features));
553	if (cpu->arch.feature[FEATURE_7_EDX] & IA32_FEATURE_SSBD)
554		strlcat(features, "ssbd ", sizeof(features));
555	if (cpu->arch.feature[FEATURE_EXT_8_EBX] & IA32_FEATURE_AMD_EXT_IBPB)
556		strlcat(features, "ibpb ", sizeof(features));
557	dprintf("CPU %d: features: %s\n", currentCPU, features);
558}
559#endif	// DUMP_FEATURE_STRING
560
561
562static void
563compute_cpu_hierarchy_masks(int maxLogicalID, int maxCoreID)
564{
565	ASSERT(maxLogicalID >= maxCoreID);
566	const int kMaxSMTID = maxLogicalID / maxCoreID;
567
568	sHierarchyMask[CPU_TOPOLOGY_SMT] = kMaxSMTID - 1;
569	sHierarchyShift[CPU_TOPOLOGY_SMT] = 0;
570
571	sHierarchyMask[CPU_TOPOLOGY_CORE] = (maxCoreID - 1) * kMaxSMTID;
572	sHierarchyShift[CPU_TOPOLOGY_CORE]
573		= count_set_bits(sHierarchyMask[CPU_TOPOLOGY_SMT]);
574
575	const uint32 kSinglePackageMask = sHierarchyMask[CPU_TOPOLOGY_SMT]
576		| sHierarchyMask[CPU_TOPOLOGY_CORE];
577	sHierarchyMask[CPU_TOPOLOGY_PACKAGE] = ~kSinglePackageMask;
578	sHierarchyShift[CPU_TOPOLOGY_PACKAGE] = count_set_bits(kSinglePackageMask);
579}
580
581
582static uint32
583get_cpu_legacy_initial_apic_id(int /* currentCPU */)
584{
585	cpuid_info cpuid;
586	get_current_cpuid(&cpuid, 1, 0);
587	return cpuid.regs.ebx >> 24;
588}
589
590
591static inline status_t
592detect_amd_cpu_topology(uint32 maxBasicLeaf, uint32 maxExtendedLeaf)
593{
594	sGetCPUTopologyID = get_cpu_legacy_initial_apic_id;
595
596	cpuid_info cpuid;
597	get_current_cpuid(&cpuid, 1, 0);
598	int maxLogicalID = next_power_of_2((cpuid.regs.ebx >> 16) & 0xff);
599
600	int maxCoreID = 1;
601	if (maxExtendedLeaf >= 0x80000008) {
602		get_current_cpuid(&cpuid, 0x80000008, 0);
603		maxCoreID = (cpuid.regs.ecx >> 12) & 0xf;
604		if (maxCoreID != 0)
605			maxCoreID = 1 << maxCoreID;
606		else
607			maxCoreID = next_power_of_2((cpuid.regs.edx & 0xf) + 1);
608	}
609
610	if (maxExtendedLeaf >= 0x80000001) {
611		get_current_cpuid(&cpuid, 0x80000001, 0);
612		if (x86_check_feature(IA32_FEATURE_AMD_EXT_CMPLEGACY,
613				FEATURE_EXT_AMD_ECX))
614			maxCoreID = maxLogicalID;
615	}
616
617	compute_cpu_hierarchy_masks(maxLogicalID, maxCoreID);
618
619	return B_OK;
620}
621
622
623static void
624detect_amd_cache_topology(uint32 maxExtendedLeaf)
625{
626	if (!x86_check_feature(IA32_FEATURE_AMD_EXT_TOPOLOGY, FEATURE_EXT_AMD_ECX))
627		return;
628
629	if (maxExtendedLeaf < 0x8000001d)
630		return;
631
632	uint8 hierarchyLevels[CPU_MAX_CACHE_LEVEL];
633	int maxCacheLevel = 0;
634
635	int currentLevel = 0;
636	int cacheType;
637	do {
638		cpuid_info cpuid;
639		get_current_cpuid(&cpuid, 0x8000001d, currentLevel);
640
641		cacheType = cpuid.regs.eax & 0x1f;
642		if (cacheType == 0)
643			break;
644
645		int cacheLevel = (cpuid.regs.eax >> 5) & 0x7;
646		int coresCount = next_power_of_2(((cpuid.regs.eax >> 14) & 0x3f) + 1);
647		hierarchyLevels[cacheLevel - 1]
648			= coresCount * (sHierarchyMask[CPU_TOPOLOGY_SMT] + 1);
649		maxCacheLevel = std::max(maxCacheLevel, cacheLevel);
650
651		currentLevel++;
652	} while (true);
653
654	for (int i = 0; i < maxCacheLevel; i++)
655		sCacheSharingMask[i] = ~uint32(hierarchyLevels[i] - 1);
656	gCPUCacheLevelCount = maxCacheLevel;
657}
658
659
660static uint32
661get_intel_cpu_initial_x2apic_id(int /* currentCPU */)
662{
663	cpuid_info cpuid;
664	get_current_cpuid(&cpuid, 11, 0);
665	return cpuid.regs.edx;
666}
667
668
669static inline status_t
670detect_intel_cpu_topology_x2apic(uint32 maxBasicLeaf)
671{
672	if (maxBasicLeaf < 11)
673		return B_UNSUPPORTED;
674
675	uint8 hierarchyLevels[CPU_TOPOLOGY_LEVELS] = { 0 };
676
677	int currentLevel = 0;
678	int levelType;
679	unsigned int levelsSet = 0;
680
681	do {
682		cpuid_info cpuid;
683		get_current_cpuid(&cpuid, 11, currentLevel);
684		if (currentLevel == 0 && cpuid.regs.ebx == 0)
685			return B_UNSUPPORTED;
686
687		levelType = (cpuid.regs.ecx >> 8) & 0xff;
688		int levelValue = cpuid.regs.eax & 0x1f;
689
690		switch (levelType) {
691			case 1:	// SMT
692				hierarchyLevels[CPU_TOPOLOGY_SMT] = levelValue;
693				levelsSet |= 1;
694				break;
695			case 2:	// core
696				hierarchyLevels[CPU_TOPOLOGY_CORE] = levelValue;
697				levelsSet |= 2;
698				break;
699		}
700
701		currentLevel++;
702	} while (levelType != 0 && levelsSet != 3);
703
704	sGetCPUTopologyID = get_intel_cpu_initial_x2apic_id;
705
706	for (int i = 1; i < CPU_TOPOLOGY_LEVELS; i++) {
707		if ((levelsSet & (1u << i)) != 0)
708			continue;
709		hierarchyLevels[i] = hierarchyLevels[i - 1];
710	}
711
712	for (int i = 0; i < CPU_TOPOLOGY_LEVELS; i++) {
713		uint32 mask = ~uint32(0);
714		if (i < CPU_TOPOLOGY_LEVELS - 1)
715			mask = (1u << hierarchyLevels[i]) - 1;
716		if (i > 0)
717			mask &= ~sHierarchyMask[i - 1];
718		sHierarchyMask[i] = mask;
719		sHierarchyShift[i] = i > 0 ? hierarchyLevels[i - 1] : 0;
720	}
721
722	return B_OK;
723}
724
725
726static inline status_t
727detect_intel_cpu_topology_legacy(uint32 maxBasicLeaf)
728{
729	sGetCPUTopologyID = get_cpu_legacy_initial_apic_id;
730
731	cpuid_info cpuid;
732
733	get_current_cpuid(&cpuid, 1, 0);
734	int maxLogicalID = next_power_of_2((cpuid.regs.ebx >> 16) & 0xff);
735
736	int maxCoreID = 1;
737	if (maxBasicLeaf >= 4) {
738		get_current_cpuid(&cpuid, 4, 0);
739		maxCoreID = next_power_of_2((cpuid.regs.eax >> 26) + 1);
740	}
741
742	compute_cpu_hierarchy_masks(maxLogicalID, maxCoreID);
743
744	return B_OK;
745}
746
747
748static void
749detect_intel_cache_topology(uint32 maxBasicLeaf)
750{
751	if (maxBasicLeaf < 4)
752		return;
753
754	uint8 hierarchyLevels[CPU_MAX_CACHE_LEVEL];
755	int maxCacheLevel = 0;
756
757	int currentLevel = 0;
758	int cacheType;
759	do {
760		cpuid_info cpuid;
761		get_current_cpuid(&cpuid, 4, currentLevel);
762
763		cacheType = cpuid.regs.eax & 0x1f;
764		if (cacheType == 0)
765			break;
766
767		int cacheLevel = (cpuid.regs.eax >> 5) & 0x7;
768		hierarchyLevels[cacheLevel - 1]
769			= next_power_of_2(((cpuid.regs.eax >> 14) & 0x3f) + 1);
770		maxCacheLevel = std::max(maxCacheLevel, cacheLevel);
771
772		currentLevel++;
773	} while (true);
774
775	for (int i = 0; i < maxCacheLevel; i++)
776		sCacheSharingMask[i] = ~uint32(hierarchyLevels[i] - 1);
777
778	gCPUCacheLevelCount = maxCacheLevel;
779}
780
781
782static uint32
783get_simple_cpu_topology_id(int currentCPU)
784{
785	return currentCPU;
786}
787
788
789static inline int
790get_topology_level_id(uint32 id, cpu_topology_level level)
791{
792	ASSERT(level < CPU_TOPOLOGY_LEVELS);
793	return (id & sHierarchyMask[level]) >> sHierarchyShift[level];
794}
795
796
797static void
798detect_cpu_topology(int currentCPU, cpu_ent* cpu, uint32 maxBasicLeaf,
799	uint32 maxExtendedLeaf)
800{
801	if (currentCPU == 0) {
802		memset(sCacheSharingMask, 0xff, sizeof(sCacheSharingMask));
803
804		status_t result = B_UNSUPPORTED;
805		if (x86_check_feature(IA32_FEATURE_HTT, FEATURE_COMMON)) {
806			if (cpu->arch.vendor == VENDOR_AMD) {
807				result = detect_amd_cpu_topology(maxBasicLeaf, maxExtendedLeaf);
808
809				if (result == B_OK)
810					detect_amd_cache_topology(maxExtendedLeaf);
811			}
812
813			if (cpu->arch.vendor == VENDOR_INTEL) {
814				result = detect_intel_cpu_topology_x2apic(maxBasicLeaf);
815				if (result != B_OK)
816					result = detect_intel_cpu_topology_legacy(maxBasicLeaf);
817
818				if (result == B_OK)
819					detect_intel_cache_topology(maxBasicLeaf);
820			}
821		}
822
823		if (result != B_OK) {
824			dprintf("No CPU topology information available.\n");
825
826			sGetCPUTopologyID = get_simple_cpu_topology_id;
827
828			sHierarchyMask[CPU_TOPOLOGY_PACKAGE] = ~uint32(0);
829		}
830	}
831
832	ASSERT(sGetCPUTopologyID != NULL);
833	int topologyID = sGetCPUTopologyID(currentCPU);
834	cpu->topology_id[CPU_TOPOLOGY_SMT]
835		= get_topology_level_id(topologyID, CPU_TOPOLOGY_SMT);
836	cpu->topology_id[CPU_TOPOLOGY_CORE]
837		= get_topology_level_id(topologyID, CPU_TOPOLOGY_CORE);
838	cpu->topology_id[CPU_TOPOLOGY_PACKAGE]
839		= get_topology_level_id(topologyID, CPU_TOPOLOGY_PACKAGE);
840
841	unsigned int i;
842	for (i = 0; i < gCPUCacheLevelCount; i++)
843		cpu->cache_id[i] = topologyID & sCacheSharingMask[i];
844	for (; i < CPU_MAX_CACHE_LEVEL; i++)
845		cpu->cache_id[i] = -1;
846
847#if DUMP_CPU_TOPOLOGY
848	dprintf("CPU %d: apic id %d, package %d, core %d, smt %d\n", currentCPU,
849		topologyID, cpu->topology_id[CPU_TOPOLOGY_PACKAGE],
850		cpu->topology_id[CPU_TOPOLOGY_CORE],
851		cpu->topology_id[CPU_TOPOLOGY_SMT]);
852
853	if (gCPUCacheLevelCount > 0) {
854		char cacheLevels[256];
855		unsigned int offset = 0;
856		for (i = 0; i < gCPUCacheLevelCount; i++) {
857			offset += snprintf(cacheLevels + offset,
858					sizeof(cacheLevels) - offset,
859					" L%d id %d%s", i + 1, cpu->cache_id[i],
860					i < gCPUCacheLevelCount - 1 ? "," : "");
861
862			if (offset >= sizeof(cacheLevels))
863				break;
864		}
865
866		dprintf("CPU %d: cache sharing:%s\n", currentCPU, cacheLevels);
867	}
868#endif
869}
870
871
872static void
873detect_cpu(int currentCPU)
874{
875	cpu_ent* cpu = get_cpu_struct();
876	char vendorString[17];
877	cpuid_info cpuid;
878
879	// clear out the cpu info data
880	cpu->arch.vendor = VENDOR_UNKNOWN;
881	cpu->arch.vendor_name = "UNKNOWN VENDOR";
882	cpu->arch.feature[FEATURE_COMMON] = 0;
883	cpu->arch.feature[FEATURE_EXT] = 0;
884	cpu->arch.feature[FEATURE_EXT_AMD] = 0;
885	cpu->arch.feature[FEATURE_7_EBX] = 0;
886	cpu->arch.feature[FEATURE_7_ECX] = 0;
887	cpu->arch.feature[FEATURE_7_EDX] = 0;
888	cpu->arch.model_name[0] = 0;
889
890	// print some fun data
891	get_current_cpuid(&cpuid, 0, 0);
892	uint32 maxBasicLeaf = cpuid.eax_0.max_eax;
893
894	// build the vendor string
895	memset(vendorString, 0, sizeof(vendorString));
896	memcpy(vendorString, cpuid.eax_0.vendor_id, sizeof(cpuid.eax_0.vendor_id));
897
898	// get the family, model, stepping
899	get_current_cpuid(&cpuid, 1, 0);
900	cpu->arch.type = cpuid.eax_1.type;
901	cpu->arch.family = cpuid.eax_1.family;
902	cpu->arch.extended_family = cpuid.eax_1.extended_family;
903	cpu->arch.model = cpuid.eax_1.model;
904	cpu->arch.extended_model = cpuid.eax_1.extended_model;
905	cpu->arch.stepping = cpuid.eax_1.stepping;
906	dprintf("CPU %d: type %d family %d extended_family %d model %d "
907		"extended_model %d stepping %d, string '%s'\n",
908		currentCPU, cpu->arch.type, cpu->arch.family,
909		cpu->arch.extended_family, cpu->arch.model,
910		cpu->arch.extended_model, cpu->arch.stepping, vendorString);
911
912	// figure out what vendor we have here
913
914	for (int32 i = 0; i < VENDOR_NUM; i++) {
915		if (vendor_info[i].ident_string[0]
916			&& !strcmp(vendorString, vendor_info[i].ident_string[0])) {
917			cpu->arch.vendor = (x86_vendors)i;
918			cpu->arch.vendor_name = vendor_info[i].vendor;
919			break;
920		}
921		if (vendor_info[i].ident_string[1]
922			&& !strcmp(vendorString, vendor_info[i].ident_string[1])) {
923			cpu->arch.vendor = (x86_vendors)i;
924			cpu->arch.vendor_name = vendor_info[i].vendor;
925			break;
926		}
927	}
928
929	// see if we can get the model name
930	get_current_cpuid(&cpuid, 0x80000000, 0);
931	uint32 maxExtendedLeaf = cpuid.eax_0.max_eax;
932	if (maxExtendedLeaf >= 0x80000004) {
933		// build the model string (need to swap ecx/edx data before copying)
934		unsigned int temp;
935		memset(cpu->arch.model_name, 0, sizeof(cpu->arch.model_name));
936
937		get_current_cpuid(&cpuid, 0x80000002, 0);
938		temp = cpuid.regs.edx;
939		cpuid.regs.edx = cpuid.regs.ecx;
940		cpuid.regs.ecx = temp;
941		memcpy(cpu->arch.model_name, cpuid.as_chars, sizeof(cpuid.as_chars));
942
943		get_current_cpuid(&cpuid, 0x80000003, 0);
944		temp = cpuid.regs.edx;
945		cpuid.regs.edx = cpuid.regs.ecx;
946		cpuid.regs.ecx = temp;
947		memcpy(cpu->arch.model_name + 16, cpuid.as_chars,
948			sizeof(cpuid.as_chars));
949
950		get_current_cpuid(&cpuid, 0x80000004, 0);
951		temp = cpuid.regs.edx;
952		cpuid.regs.edx = cpuid.regs.ecx;
953		cpuid.regs.ecx = temp;
954		memcpy(cpu->arch.model_name + 32, cpuid.as_chars,
955			sizeof(cpuid.as_chars));
956
957		// some cpus return a right-justified string
958		int32 i = 0;
959		while (cpu->arch.model_name[i] == ' ')
960			i++;
961		if (i > 0) {
962			memmove(cpu->arch.model_name, &cpu->arch.model_name[i],
963				strlen(&cpu->arch.model_name[i]) + 1);
964		}
965
966		dprintf("CPU %d: vendor '%s' model name '%s'\n",
967			currentCPU, cpu->arch.vendor_name, cpu->arch.model_name);
968	} else {
969		strlcpy(cpu->arch.model_name, "unknown", sizeof(cpu->arch.model_name));
970	}
971
972	// load feature bits
973	get_current_cpuid(&cpuid, 1, 0);
974	cpu->arch.feature[FEATURE_COMMON] = cpuid.eax_1.features; // edx
975	cpu->arch.feature[FEATURE_EXT] = cpuid.eax_1.extended_features; // ecx
976
977	if (maxExtendedLeaf >= 0x80000001) {
978		get_current_cpuid(&cpuid, 0x80000001, 0);
979		if (cpu->arch.vendor == VENDOR_AMD)
980			cpu->arch.feature[FEATURE_EXT_AMD_ECX] = cpuid.regs.ecx; // ecx
981		cpu->arch.feature[FEATURE_EXT_AMD] = cpuid.regs.edx; // edx
982		if (cpu->arch.vendor != VENDOR_AMD)
983			cpu->arch.feature[FEATURE_EXT_AMD] &= IA32_FEATURES_INTEL_EXT;
984	}
985
986	if (maxBasicLeaf >= 5) {
987		get_current_cpuid(&cpuid, 5, 0);
988		cpu->arch.feature[FEATURE_5_ECX] = cpuid.regs.ecx;
989	}
990
991	if (maxBasicLeaf >= 6) {
992		get_current_cpuid(&cpuid, 6, 0);
993		cpu->arch.feature[FEATURE_6_EAX] = cpuid.regs.eax;
994		cpu->arch.feature[FEATURE_6_ECX] = cpuid.regs.ecx;
995	}
996
997	if (maxBasicLeaf >= 7) {
998		get_current_cpuid(&cpuid, 7, 0);
999		cpu->arch.feature[FEATURE_7_EBX] = cpuid.regs.ebx;
1000		cpu->arch.feature[FEATURE_7_ECX] = cpuid.regs.ecx;
1001		cpu->arch.feature[FEATURE_7_EDX] = cpuid.regs.edx;
1002	}
1003
1004	if (maxExtendedLeaf >= 0x80000007) {
1005		get_current_cpuid(&cpuid, 0x80000007, 0);
1006		cpu->arch.feature[FEATURE_EXT_7_EDX] = cpuid.regs.edx;
1007	}
1008
1009	if (maxExtendedLeaf >= 0x80000008) {
1010		get_current_cpuid(&cpuid, 0x80000008, 0);
1011			cpu->arch.feature[FEATURE_EXT_8_EBX] = cpuid.regs.ebx;
1012	}
1013
1014	detect_cpu_topology(currentCPU, cpu, maxBasicLeaf, maxExtendedLeaf);
1015
1016#if DUMP_FEATURE_STRING
1017	dump_feature_string(currentCPU, cpu);
1018#endif
1019}
1020
1021
1022bool
1023x86_check_feature(uint32 feature, enum x86_feature_type type)
1024{
1025	cpu_ent* cpu = get_cpu_struct();
1026
1027#if 0
1028	int i;
1029	dprintf("x86_check_feature: feature 0x%x, type %d\n", feature, type);
1030	for (i = 0; i < FEATURE_NUM; i++) {
1031		dprintf("features %d: 0x%x\n", i, cpu->arch.feature[i]);
1032	}
1033#endif
1034
1035	return (cpu->arch.feature[type] & feature) != 0;
1036}
1037
1038
1039void*
1040x86_get_double_fault_stack(int32 cpu, size_t* _size)
1041{
1042	*_size = kDoubleFaultStackSize;
1043	return sDoubleFaultStacks + kDoubleFaultStackSize * cpu;
1044}
1045
1046
1047/*!	Returns the index of the current CPU. Can only be called from the double
1048	fault handler.
1049*/
1050int32
1051x86_double_fault_get_cpu(void)
1052{
1053	addr_t stack = x86_get_stack_frame();
1054	return (stack - (addr_t)sDoubleFaultStacks) / kDoubleFaultStackSize;
1055}
1056
1057
1058//	#pragma mark -
1059
1060
1061status_t
1062arch_cpu_preboot_init_percpu(kernel_args* args, int cpu)
1063{
1064	// On SMP system we want to synchronize the CPUs' TSCs, so system_time()
1065	// will return consistent values.
1066	if (smp_get_num_cpus() > 1) {
1067		// let the first CPU prepare the rendezvous point
1068		if (cpu == 0)
1069			sTSCSyncRendezvous = smp_get_num_cpus() - 1;
1070
1071		// One CPU after the other will drop out of this loop and be caught by
1072		// the loop below, until the last CPU (0) gets there. Save for +/- a few
1073		// cycles the CPUs should pass the second loop at the same time.
1074		while (sTSCSyncRendezvous != cpu) {
1075		}
1076
1077		sTSCSyncRendezvous = cpu - 1;
1078
1079		while (sTSCSyncRendezvous != -1) {
1080		}
1081
1082		// reset TSC to 0
1083		x86_write_msr(IA32_MSR_TSC, 0);
1084	}
1085
1086	x86_descriptors_preboot_init_percpu(args, cpu);
1087
1088	return B_OK;
1089}
1090
1091
1092static void
1093halt_idle(void)
1094{
1095	asm("hlt");
1096}
1097
1098
1099static void
1100amdc1e_noarat_idle(void)
1101{
1102	uint64 msr = x86_read_msr(K8_MSR_IPM);
1103	if (msr & K8_CMPHALT)
1104		x86_write_msr(K8_MSR_IPM, msr & ~K8_CMPHALT);
1105	halt_idle();
1106}
1107
1108
1109static bool
1110detect_amdc1e_noarat()
1111{
1112	cpu_ent* cpu = get_cpu_struct();
1113
1114	if (cpu->arch.vendor != VENDOR_AMD)
1115		return false;
1116
1117	// Family 0x12 and higher processors support ARAT
1118	// Family lower than 0xf processors doesn't support C1E
1119	// Family 0xf with model <= 0x40 procssors doesn't support C1E
1120	uint32 family = cpu->arch.family + cpu->arch.extended_family;
1121	uint32 model = (cpu->arch.extended_model << 4) | cpu->arch.model;
1122	return (family < 0x12 && family > 0xf) || (family == 0xf && model > 0x40);
1123}
1124
1125
1126status_t
1127arch_cpu_init_percpu(kernel_args* args, int cpu)
1128{
1129	detect_cpu(cpu);
1130
1131	if (!gCpuIdleFunc) {
1132		if (detect_amdc1e_noarat())
1133			gCpuIdleFunc = amdc1e_noarat_idle;
1134		else
1135			gCpuIdleFunc = halt_idle;
1136	}
1137
1138	return __x86_patch_errata_percpu(cpu);
1139}
1140
1141
1142status_t
1143arch_cpu_init(kernel_args* args)
1144{
1145	// init the TSC -> system_time() conversion factors
1146
1147	uint32 conversionFactor = args->arch_args.system_time_cv_factor;
1148	uint64 conversionFactorNsecs = (uint64)conversionFactor * 1000;
1149
1150#ifdef __x86_64__
1151	// The x86_64 system_time() implementation uses 64-bit multiplication and
1152	// therefore shifting is not necessary for low frequencies (it's also not
1153	// too likely that there'll be any x86_64 CPUs clocked under 1GHz).
1154	__x86_setup_system_time((uint64)conversionFactor << 32,
1155		conversionFactorNsecs);
1156#else
1157	if (conversionFactorNsecs >> 32 != 0) {
1158		// the TSC frequency is < 1 GHz, which forces us to shift the factor
1159		__x86_setup_system_time(conversionFactor, conversionFactorNsecs >> 16,
1160			true);
1161	} else {
1162		// the TSC frequency is >= 1 GHz
1163		__x86_setup_system_time(conversionFactor, conversionFactorNsecs, false);
1164	}
1165#endif
1166
1167	// Initialize descriptor tables.
1168	x86_descriptors_init(args);
1169
1170	return B_OK;
1171}
1172
1173
1174#ifdef __x86_64__
1175static void
1176enable_smap(void* dummy, int cpu)
1177{
1178	x86_write_cr4(x86_read_cr4() | IA32_CR4_SMAP);
1179}
1180
1181
1182static void
1183enable_smep(void* dummy, int cpu)
1184{
1185	x86_write_cr4(x86_read_cr4() | IA32_CR4_SMEP);
1186}
1187#endif
1188
1189
1190status_t
1191arch_cpu_init_post_vm(kernel_args* args)
1192{
1193	uint32 i;
1194
1195	// allocate an area for the double fault stacks
1196	virtual_address_restrictions virtualRestrictions = {};
1197	virtualRestrictions.address_specification = B_ANY_KERNEL_ADDRESS;
1198	physical_address_restrictions physicalRestrictions = {};
1199	create_area_etc(B_SYSTEM_TEAM, "double fault stacks",
1200		kDoubleFaultStackSize * smp_get_num_cpus(), B_FULL_LOCK,
1201		B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, CREATE_AREA_DONT_WAIT, 0,
1202		&virtualRestrictions, &physicalRestrictions,
1203		(void**)&sDoubleFaultStacks);
1204
1205	X86PagingStructures* kernelPagingStructures
1206		= static_cast<X86VMTranslationMap*>(
1207			VMAddressSpace::Kernel()->TranslationMap())->PagingStructures();
1208
1209	// Set active translation map on each CPU.
1210	for (i = 0; i < args->num_cpus; i++) {
1211		gCPU[i].arch.active_paging_structures = kernelPagingStructures;
1212		kernelPagingStructures->AddReference();
1213	}
1214
1215	if (!apic_available())
1216		x86_init_fpu();
1217	// else fpu gets set up in smp code
1218
1219#ifdef __x86_64__
1220	// if available enable SMEP (Supervisor Memory Execution Protection)
1221	if (x86_check_feature(IA32_FEATURE_SMEP, FEATURE_7_EBX)) {
1222		if (!get_safemode_boolean(B_SAFEMODE_DISABLE_SMEP_SMAP, false)) {
1223			dprintf("enable SMEP\n");
1224			call_all_cpus_sync(&enable_smep, NULL);
1225		} else
1226			dprintf("SMEP disabled per safemode setting\n");
1227	}
1228
1229	// if available enable SMAP (Supervisor Memory Access Protection)
1230	if (x86_check_feature(IA32_FEATURE_SMAP, FEATURE_7_EBX)) {
1231		if (!get_safemode_boolean(B_SAFEMODE_DISABLE_SMEP_SMAP, false)) {
1232			dprintf("enable SMAP\n");
1233			call_all_cpus_sync(&enable_smap, NULL);
1234
1235			arch_altcodepatch_replace(ALTCODEPATCH_TAG_STAC, &_stac, 3);
1236			arch_altcodepatch_replace(ALTCODEPATCH_TAG_CLAC, &_clac, 3);
1237		} else
1238			dprintf("SMAP disabled per safemode setting\n");
1239	}
1240#endif
1241
1242	return B_OK;
1243}
1244
1245
1246status_t
1247arch_cpu_init_post_modules(kernel_args* args)
1248{
1249	// initialize CPU module
1250
1251	void* cookie = open_module_list("cpu");
1252
1253	while (true) {
1254		char name[B_FILE_NAME_LENGTH];
1255		size_t nameLength = sizeof(name);
1256
1257		if (read_next_module_name(cookie, name, &nameLength) != B_OK
1258			|| get_module(name, (module_info**)&sCpuModule) == B_OK)
1259			break;
1260	}
1261
1262	close_module_list(cookie);
1263
1264	// initialize MTRRs if available
1265	if (x86_count_mtrrs() > 0) {
1266		sCpuRendezvous = sCpuRendezvous2 = 0;
1267		call_all_cpus(&init_mtrrs, NULL);
1268	}
1269
1270	size_t threadExitLen = (addr_t)x86_end_userspace_thread_exit
1271		- (addr_t)x86_userspace_thread_exit;
1272	addr_t threadExitPosition = fill_commpage_entry(
1273		COMMPAGE_ENTRY_X86_THREAD_EXIT, (const void*)x86_userspace_thread_exit,
1274		threadExitLen);
1275
1276	// add the functions to the commpage image
1277	image_id image = get_commpage_image();
1278
1279	elf_add_memory_image_symbol(image, "commpage_thread_exit",
1280		threadExitPosition, threadExitLen, B_SYMBOL_TYPE_TEXT);
1281
1282	return B_OK;
1283}
1284
1285
1286void
1287arch_cpu_user_TLB_invalidate(void)
1288{
1289	x86_write_cr3(x86_read_cr3());
1290}
1291
1292
1293void
1294arch_cpu_global_TLB_invalidate(void)
1295{
1296	uint32 flags = x86_read_cr4();
1297
1298	if (flags & IA32_CR4_GLOBAL_PAGES) {
1299		// disable and reenable the global pages to flush all TLBs regardless
1300		// of the global page bit
1301		x86_write_cr4(flags & ~IA32_CR4_GLOBAL_PAGES);
1302		x86_write_cr4(flags | IA32_CR4_GLOBAL_PAGES);
1303	} else {
1304		cpu_status state = disable_interrupts();
1305		arch_cpu_user_TLB_invalidate();
1306		restore_interrupts(state);
1307	}
1308}
1309
1310
1311void
1312arch_cpu_invalidate_TLB_range(addr_t start, addr_t end)
1313{
1314	int32 num_pages = end / B_PAGE_SIZE - start / B_PAGE_SIZE;
1315	while (num_pages-- >= 0) {
1316		invalidate_TLB(start);
1317		start += B_PAGE_SIZE;
1318	}
1319}
1320
1321
1322void
1323arch_cpu_invalidate_TLB_list(addr_t pages[], int num_pages)
1324{
1325	int i;
1326	for (i = 0; i < num_pages; i++) {
1327		invalidate_TLB(pages[i]);
1328	}
1329}
1330
1331
1332status_t
1333arch_cpu_shutdown(bool rebootSystem)
1334{
1335	if (acpi_shutdown(rebootSystem) == B_OK)
1336		return B_OK;
1337
1338	if (!rebootSystem) {
1339#ifndef __x86_64__
1340		return apm_shutdown();
1341#else
1342		return B_NOT_SUPPORTED;
1343#endif
1344	}
1345
1346	cpu_status state = disable_interrupts();
1347
1348	// try to reset the system using the keyboard controller
1349	out8(0xfe, 0x64);
1350
1351	// Give some time to the controller to do its job (0.5s)
1352	snooze(500000);
1353
1354	// if that didn't help, try it this way
1355	x86_reboot();
1356
1357	restore_interrupts(state);
1358	return B_ERROR;
1359}
1360
1361
1362void
1363arch_cpu_sync_icache(void* address, size_t length)
1364{
1365	// instruction cache is always consistent on x86
1366}
1367
1368