1/*
2 * Copyright 2016 Haiku, Inc. All rights reserved.
3 * Copyright 2014, Jessica Hamilton, jessica.l.hamilton@gmail.com.
4 * Copyright 2014, Henry Harrington, henry.harrington@gmail.com.
5 * Distributed under the terms of the MIT License.
6 */
7
8
9#include <algorithm>
10
11#include <boot/addr_range.h>
12#include <boot/platform.h>
13#include <boot/stage2.h>
14#include <kernel/arch/x86/arch_kernel.h>
15#include <kernel/kernel.h>
16
17#include "efi_platform.h"
18#include "mmu.h"
19
20
21struct allocated_memory_region {
22	allocated_memory_region *next;
23	uint64_t vaddr;
24	uint64_t paddr;
25	size_t size;
26	bool released;
27};
28
29
30static uint64_t next_virtual_address = KERNEL_LOAD_BASE_64_BIT + 32 * 1024 * 1024;
31static allocated_memory_region *allocated_memory_regions = NULL;
32
33
34static uint64_t mmu_allocate_page()
35{
36	EFI_PHYSICAL_ADDRESS addr;
37	EFI_STATUS s = kBootServices->AllocatePages(AllocateAnyPages, EfiLoaderData, 1, &addr);
38	if (s != EFI_SUCCESS)
39		panic("Unabled to allocate memory: %li", s);
40
41	return addr;
42}
43
44
45uint64_t
46mmu_generate_post_efi_page_tables(UINTN memory_map_size,
47	EFI_MEMORY_DESCRIPTOR *memory_map, UINTN descriptor_size,
48	UINTN descriptor_version)
49{
50	// Generate page tables, matching bios_ia32/long.cpp.
51	uint64_t *pml4;
52	uint64_t *pdpt;
53	uint64_t *pageDir;
54	uint64_t *pageTable;
55
56	// Allocate the top level PML4.
57	pml4 = NULL;
58	if (platform_allocate_region((void**)&pml4, B_PAGE_SIZE, 0, false) != B_OK)
59		panic("Failed to allocate PML4.");
60	gKernelArgs.arch_args.phys_pgdir = (uint32_t)(addr_t)pml4;
61	memset(pml4, 0, B_PAGE_SIZE);
62	platform_bootloader_address_to_kernel_address(pml4, &gKernelArgs.arch_args.vir_pgdir);
63
64	// Store the virtual memory usage information.
65	gKernelArgs.virtual_allocated_range[0].start = KERNEL_LOAD_BASE_64_BIT;
66	gKernelArgs.virtual_allocated_range[0].size = next_virtual_address - KERNEL_LOAD_BASE_64_BIT;
67	gKernelArgs.num_virtual_allocated_ranges = 1;
68	gKernelArgs.arch_args.virtual_end = ROUNDUP(KERNEL_LOAD_BASE_64_BIT
69		+ gKernelArgs.virtual_allocated_range[0].size, 0x200000);
70
71	// Find the highest physical memory address. We map all physical memory
72	// into the kernel address space, so we want to make sure we map everything
73	// we have available.
74	uint64 maxAddress = 0;
75	for (UINTN i = 0; i < memory_map_size / descriptor_size; ++i) {
76		EFI_MEMORY_DESCRIPTOR *entry = (EFI_MEMORY_DESCRIPTOR *)((addr_t)memory_map + i * descriptor_size);
77		maxAddress = std::max(maxAddress,
78				      entry->PhysicalStart + entry->NumberOfPages * 4096);
79	}
80
81	// Want to map at least 4GB, there may be stuff other than usable RAM that
82	// could be in the first 4GB of physical address space.
83	maxAddress = std::max(maxAddress, (uint64)0x100000000ll);
84	maxAddress = ROUNDUP(maxAddress, 0x40000000);
85
86	// Currently only use 1 PDPT (512GB). This will need to change if someone
87	// wants to use Haiku on a box with more than 512GB of RAM but that's
88	// probably not going to happen any time soon.
89	if (maxAddress / 0x40000000 > 512)
90		panic("Can't currently support more than 512GB of RAM!");
91
92	// Create page tables for the physical map area. Also map this PDPT
93	// temporarily at the bottom of the address space so that we are identity
94	// mapped.
95
96	pdpt = (uint64*)mmu_allocate_page();
97	memset(pdpt, 0, B_PAGE_SIZE);
98	pml4[510] = (addr_t)pdpt | kTableMappingFlags;
99	pml4[0] = (addr_t)pdpt | kTableMappingFlags;
100
101	for (uint64 i = 0; i < maxAddress; i += 0x40000000) {
102		pageDir = (uint64*)mmu_allocate_page();
103		memset(pageDir, 0, B_PAGE_SIZE);
104		pdpt[i / 0x40000000] = (addr_t)pageDir | kTableMappingFlags;
105
106		for (uint64 j = 0; j < 0x40000000; j += 0x200000) {
107			pageDir[j / 0x200000] = (i + j) | kLargePageMappingFlags;
108		}
109	}
110
111	// Allocate tables for the kernel mappings.
112
113	pdpt = (uint64*)mmu_allocate_page();
114	memset(pdpt, 0, B_PAGE_SIZE);
115	pml4[511] = (addr_t)pdpt | kTableMappingFlags;
116
117	pageDir = (uint64*)mmu_allocate_page();
118	memset(pageDir, 0, B_PAGE_SIZE);
119	pdpt[510] = (addr_t)pageDir | kTableMappingFlags;
120
121	// We can now allocate page tables and duplicate the mappings across from
122	// the 32-bit address space to them.
123	pageTable = NULL; // shush, compiler.
124	for (uint32 i = 0; i < gKernelArgs.virtual_allocated_range[0].size
125			/ B_PAGE_SIZE; i++) {
126		if ((i % 512) == 0) {
127			pageTable = (uint64*)mmu_allocate_page();
128			memset(pageTable, 0, B_PAGE_SIZE);
129			pageDir[i / 512] = (addr_t)pageTable | kTableMappingFlags;
130		}
131
132		// Get the physical address to map.
133		void *phys;
134		if (platform_kernel_address_to_bootloader_address(KERNEL_LOAD_BASE_64_BIT + (i * B_PAGE_SIZE),
135								  &phys) != B_OK)
136			continue;
137
138		pageTable[i % 512] = (addr_t)phys | kPageMappingFlags;
139	}
140
141	return (uint64)pml4;
142}
143
144
145// Called after EFI boot services exit.
146// Currently assumes that the memory map is sane... Sorted and no overlapping
147// regions.
148void
149mmu_post_efi_setup(UINTN memory_map_size, EFI_MEMORY_DESCRIPTOR *memory_map, UINTN descriptor_size, UINTN descriptor_version)
150{
151	// Add physical memory to the kernel args and update virtual addresses for EFI regions..
152	addr_t addr = (addr_t)memory_map;
153	gKernelArgs.num_physical_memory_ranges = 0;
154	for (UINTN i = 0; i < memory_map_size / descriptor_size; ++i) {
155		EFI_MEMORY_DESCRIPTOR *entry = (EFI_MEMORY_DESCRIPTOR *)(addr + i * descriptor_size);
156		switch (entry->Type) {
157		case EfiLoaderCode:
158		case EfiLoaderData:
159		case EfiBootServicesCode:
160		case EfiBootServicesData:
161		case EfiConventionalMemory: {
162			// Usable memory.
163			// Ignore memory below 1MB and above 512GB.
164			uint64_t base = entry->PhysicalStart;
165			uint64_t end = entry->PhysicalStart + entry->NumberOfPages * 4096;
166			if (base < 0x100000)
167				base = 0x100000;
168			if (end > (512ull * 1024 * 1024 * 1024))
169				end = 512ull * 1024 * 1024 * 1024;
170			if (base >= end)
171				break;
172			uint64_t size = end - base;
173
174			insert_physical_memory_range(base, size);
175			// LoaderData memory is bootloader allocated memory, possibly
176			// containing the kernel or loaded drivers.
177			if (entry->Type == EfiLoaderData)
178				insert_physical_allocated_range(base, size);
179			break;
180		}
181		case EfiACPIReclaimMemory:
182			// ACPI reclaim -- physical memory we could actually use later
183			gKernelArgs.ignored_physical_memory += entry->NumberOfPages * 4096;
184			break;
185		case EfiRuntimeServicesCode:
186		case EfiRuntimeServicesData:
187			entry->VirtualStart = entry->PhysicalStart;
188			break;
189		}
190	}
191
192	// Sort the address ranges.
193	sort_address_ranges(gKernelArgs.physical_memory_range,
194		gKernelArgs.num_physical_memory_ranges);
195	sort_address_ranges(gKernelArgs.physical_allocated_range,
196		gKernelArgs.num_physical_allocated_ranges);
197	sort_address_ranges(gKernelArgs.virtual_allocated_range,
198		gKernelArgs.num_virtual_allocated_ranges);
199
200	// Switch EFI to virtual mode, using the kernel pmap.
201	// Something involving ConvertPointer might need to be done after this?
202	// http://wiki.phoenix.com/wiki/index.php/EFI_RUNTIME_SERVICES#SetVirtualAddressMap.28.29
203	kRuntimeServices->SetVirtualAddressMap(memory_map_size, descriptor_size, descriptor_version, memory_map);
204
205	// Important.  Make sure supervisor threads can fault on read only pages...
206	asm("mov %%rax, %%cr0" : : "a" ((1 << 31) | (1 << 16) | (1 << 5) | 1));
207}
208
209
210// Platform allocator.
211// The bootloader assumes that bootloader address space == kernel address space.
212// This is not true until just before the kernel is booted, so an ugly hack is
213// used to cover the difference. platform_allocate_region allocates addresses
214// in bootloader space, but can convert them to kernel space. The ELF loader
215// accesses kernel memory via Mao(), and much later in the boot process,
216// addresses in the kernel argument struct are converted from bootloader
217// addresses to kernel addresses.
218
219extern "C" status_t
220platform_allocate_region(void **_address, size_t size, uint8 /* protection */, bool exactAddress)
221{
222	// We don't have any control over the page tables, give up right away if an
223	// exactAddress is wanted.
224	if (exactAddress)
225		return B_NO_MEMORY;
226
227	EFI_PHYSICAL_ADDRESS addr;
228	size_t aligned_size = ROUNDUP(size, B_PAGE_SIZE);
229	allocated_memory_region *region = new(std::nothrow) allocated_memory_region;
230
231	if (region == NULL)
232		return B_NO_MEMORY;
233
234	EFI_STATUS status = kBootServices->AllocatePages(AllocateAnyPages,
235		EfiLoaderData, aligned_size / B_PAGE_SIZE, &addr);
236	if (status != EFI_SUCCESS) {
237		delete region;
238		return B_NO_MEMORY;
239	}
240
241	// Addresses above 512GB not supported.
242	// Memory map regions above 512GB can be ignored, but if EFI returns pages
243	// above that there's nothing that can be done to fix it.
244	if (addr + size > (512ull * 1024 * 1024 * 1024))
245		panic("Can't currently support more than 512GB of RAM!");
246
247	region->next = allocated_memory_regions;
248	allocated_memory_regions = region;
249	region->vaddr = 0;
250	region->paddr = addr;
251	region->size = size;
252	region->released = false;
253
254	if (*_address != NULL) {
255		region->vaddr = (uint64_t)*_address;
256	}
257
258	//dprintf("Allocated region %#lx (requested %p) %#lx %lu\n", region->vaddr, *_address, region->paddr, region->size);
259
260	*_address = (void *)region->paddr;
261
262	return B_OK;
263}
264
265
266/*!
267	Neither \a virtualAddress nor \a size need to be aligned, but the function
268	will map all pages the range intersects with.
269	If physicalAddress is not page-aligned, the returned virtual address will
270	have the same "misalignment".
271*/
272extern "C" addr_t
273mmu_map_physical_memory(addr_t physicalAddress, size_t size, uint32 flags)
274{
275	addr_t pageOffset = physicalAddress & (B_PAGE_SIZE - 1);
276
277	physicalAddress -= pageOffset;
278	size += pageOffset;
279
280	if (insert_physical_allocated_range(physicalAddress, ROUNDUP(size, B_PAGE_SIZE)) != B_OK)
281		return B_NO_MEMORY;
282
283	return physicalAddress + pageOffset;
284}
285
286
287extern "C" void
288mmu_free(void *virtualAddress, size_t size)
289{
290	addr_t physicalAddress = (addr_t)virtualAddress;
291	addr_t pageOffset = physicalAddress & (B_PAGE_SIZE - 1);
292
293	physicalAddress -= pageOffset;
294	size += pageOffset;
295
296	size_t aligned_size = ROUNDUP(size, B_PAGE_SIZE);
297
298	for (allocated_memory_region *region = allocated_memory_regions; region; region = region->next) {
299		if (region->paddr == physicalAddress && region->size == aligned_size) {
300			region->released = true;
301			return;
302		}
303	}
304}
305
306
307static allocated_memory_region *
308get_region(void *address, size_t size)
309{
310	for (allocated_memory_region *region = allocated_memory_regions; region; region = region->next) {
311		if (region->paddr == (uint64_t)address && region->size == size) {
312			return region;
313		}
314	}
315	return 0;
316}
317
318
319static void
320convert_physical_ranges() {
321	addr_range *range = gKernelArgs.physical_allocated_range;
322	uint32 num_ranges = gKernelArgs.num_physical_allocated_ranges;
323
324	for (uint32 i = 0; i < num_ranges; ++i) {
325		allocated_memory_region *region = new(std::nothrow) allocated_memory_region;
326
327		if (!region)
328			panic("Couldn't add allocated region");
329
330		// Addresses above 512GB not supported.
331		// Memory map regions above 512GB can be ignored, but if EFI returns pages above
332		// that there's nothing that can be done to fix it.
333		if (range[i].start + range[i].size > (512ull * 1024 * 1024 * 1024))
334			panic("Can't currently support more than 512GB of RAM!");
335
336		region->next = allocated_memory_regions;
337		allocated_memory_regions = region;
338		region->vaddr = 0;
339		region->paddr = range[i].start;
340		region->size = range[i].size;
341		region->released = false;
342
343		// Clear out the allocated range
344		range[i].start = 0;
345		range[i].size = 0;
346		gKernelArgs.num_physical_allocated_ranges--;
347	}
348}
349
350
351extern "C" status_t
352platform_bootloader_address_to_kernel_address(void *address, uint64_t *_result)
353{
354	// Convert any physical ranges prior to looking up address
355	convert_physical_ranges();
356
357	uint64_t addr = (uint64_t)address;
358
359	for (allocated_memory_region *region = allocated_memory_regions; region; region = region->next) {
360		if (region->paddr <= addr && addr < region->paddr + region->size) {
361			// Lazily allocate virtual memory.
362			if (region->vaddr == 0) {
363				region->vaddr = next_virtual_address;
364				next_virtual_address += ROUNDUP(region->size, B_PAGE_SIZE);
365			}
366			*_result = region->vaddr + (addr - region->paddr);
367			//dprintf("Converted bootloader address %p in region %#lx-%#lx to %#lx\n",
368			//	address, region->paddr, region->paddr + region->size, *_result);
369			return B_OK;
370		}
371	}
372
373	return B_ERROR;
374}
375
376
377extern "C" status_t
378platform_kernel_address_to_bootloader_address(uint64_t address, void **_result)
379{
380	for (allocated_memory_region *region = allocated_memory_regions; region; region = region->next) {
381		if (region->vaddr != 0 && region->vaddr <= address && address < region->vaddr + region->size) {
382			*_result = (void *)(region->paddr + (address - region->vaddr));
383			//dprintf("Converted kernel address %#lx in region %#lx-%#lx to %p\n",
384			//	address, region->vaddr, region->vaddr + region->size, *_result);
385			return B_OK;
386		}
387	}
388
389	return B_ERROR;
390}
391
392
393extern "C" status_t
394platform_free_region(void *address, size_t size)
395{
396	//dprintf("Release region %p %lu\n", address, size);
397	allocated_memory_region *region = get_region(address, size);
398	if (!region)
399		panic("Unknown region??");
400
401	kBootServices->FreePages((EFI_PHYSICAL_ADDRESS)address, ROUNDUP(size, B_PAGE_SIZE) / B_PAGE_SIZE);
402
403	return B_OK;
404}
405