1/*
2 * Copyright 2004-2009, Axel D��rfler, axeld@pinc-software.de.
3 * Distributed under the terms of the MIT License.
4 */
5
6
7#include "vnode_store.h"
8
9#include <unistd.h>
10#include <stdlib.h>
11#include <string.h>
12
13#include <KernelExport.h>
14#include <fs_cache.h>
15
16#include <condition_variable.h>
17#include <file_cache.h>
18#include <generic_syscall.h>
19#include <low_resource_manager.h>
20#include <thread.h>
21#include <util/AutoLock.h>
22#include <util/kernel_cpp.h>
23#include <vfs.h>
24#include <vm/vm.h>
25#include <vm/vm_page.h>
26#include <vm/VMCache.h>
27
28#include "IORequest.h"
29
30
31//#define TRACE_FILE_CACHE
32#ifdef TRACE_FILE_CACHE
33#	define TRACE(x) dprintf x
34#else
35#	define TRACE(x) ;
36#endif
37
38// maximum number of iovecs per request
39#define MAX_IO_VECS			32	// 128 kB
40
41#define BYPASS_IO_SIZE		65536
42#define LAST_ACCESSES		3
43
44struct file_cache_ref {
45	VMCache			*cache;
46	struct vnode	*vnode;
47	off_t			last_access[LAST_ACCESSES];
48		// TODO: it would probably be enough to only store the least
49		//	significant 31 bits, and make this uint32 (one bit for
50		//	write vs. read)
51	int32			last_access_index;
52	uint16			disabled_count;
53
54	inline void SetLastAccess(int32 index, off_t access, bool isWrite)
55	{
56		// we remember writes as negative offsets
57		last_access[index] = isWrite ? -access : access;
58	}
59
60	inline off_t LastAccess(int32 index, bool isWrite) const
61	{
62		return isWrite ? -last_access[index] : last_access[index];
63	}
64
65	inline uint32 LastAccessPageOffset(int32 index, bool isWrite)
66	{
67		return LastAccess(index, isWrite) >> PAGE_SHIFT;
68	}
69};
70
71class PrecacheIO : public AsyncIOCallback {
72public:
73								PrecacheIO(file_cache_ref* ref, off_t offset,
74									generic_size_t size);
75								~PrecacheIO();
76
77			status_t			Prepare(vm_page_reservation* reservation);
78			void				ReadAsync();
79
80	virtual	void				IOFinished(status_t status,
81									bool partialTransfer,
82									generic_size_t bytesTransferred);
83
84private:
85			file_cache_ref*		fRef;
86			VMCache*			fCache;
87			vm_page**			fPages;
88			size_t				fPageCount;
89			ConditionVariable*	fBusyConditions;
90			generic_io_vec*		fVecs;
91			off_t				fOffset;
92			uint32				fVecCount;
93			generic_size_t		fSize;
94#if DEBUG_PAGE_ACCESS
95			thread_id			fAllocatingThread;
96#endif
97};
98
99typedef status_t (*cache_func)(file_cache_ref* ref, void* cookie, off_t offset,
100	int32 pageOffset, addr_t buffer, size_t bufferSize, bool useBuffer,
101	vm_page_reservation* reservation, size_t reservePages);
102
103static void add_to_iovec(generic_io_vec* vecs, uint32 &index, uint32 max,
104	generic_addr_t address, generic_size_t size);
105
106
107static struct cache_module_info* sCacheModule;
108
109
110static const uint32 kZeroVecCount = 32;
111static const size_t kZeroVecSize = kZeroVecCount * B_PAGE_SIZE;
112static phys_addr_t sZeroPage;	// physical address
113static generic_io_vec sZeroVecs[kZeroVecCount];
114
115
116//	#pragma mark -
117
118
119PrecacheIO::PrecacheIO(file_cache_ref* ref, off_t offset, generic_size_t size)
120	:
121	fRef(ref),
122	fCache(ref->cache),
123	fPages(NULL),
124	fVecs(NULL),
125	fOffset(offset),
126	fVecCount(0),
127	fSize(size)
128{
129	fPageCount = (size + B_PAGE_SIZE - 1) / B_PAGE_SIZE;
130	fCache->AcquireRefLocked();
131}
132
133
134PrecacheIO::~PrecacheIO()
135{
136	delete[] fPages;
137	delete[] fVecs;
138	fCache->ReleaseRefLocked();
139}
140
141
142status_t
143PrecacheIO::Prepare(vm_page_reservation* reservation)
144{
145	if (fPageCount == 0)
146		return B_BAD_VALUE;
147
148	fPages = new(std::nothrow) vm_page*[fPageCount];
149	if (fPages == NULL)
150		return B_NO_MEMORY;
151
152	fVecs = new(std::nothrow) generic_io_vec[fPageCount];
153	if (fVecs == NULL)
154		return B_NO_MEMORY;
155
156	// allocate pages for the cache and mark them busy
157	uint32 i = 0;
158	for (generic_size_t pos = 0; pos < fSize; pos += B_PAGE_SIZE) {
159		vm_page* page = vm_page_allocate_page(reservation,
160			PAGE_STATE_CACHED | VM_PAGE_ALLOC_BUSY);
161
162		fCache->InsertPage(page, fOffset + pos);
163
164		add_to_iovec(fVecs, fVecCount, fPageCount,
165			page->physical_page_number * B_PAGE_SIZE, B_PAGE_SIZE);
166		fPages[i++] = page;
167	}
168
169#if DEBUG_PAGE_ACCESS
170	fAllocatingThread = find_thread(NULL);
171#endif
172
173	return B_OK;
174}
175
176
177void
178PrecacheIO::ReadAsync()
179{
180	// This object is going to be deleted after the I/O request has been
181	// fulfilled
182	vfs_asynchronous_read_pages(fRef->vnode, NULL, fOffset, fVecs, fVecCount,
183		fSize, B_PHYSICAL_IO_REQUEST, this);
184}
185
186
187void
188PrecacheIO::IOFinished(status_t status, bool partialTransfer,
189	generic_size_t bytesTransferred)
190{
191	AutoLocker<VMCache> locker(fCache);
192
193	// Make successfully loaded pages accessible again (partially
194	// transferred pages are considered failed)
195	phys_size_t pagesTransferred
196		= (bytesTransferred + B_PAGE_SIZE - 1) / B_PAGE_SIZE;
197
198	if (fOffset + (off_t)bytesTransferred > fCache->virtual_end)
199		bytesTransferred = fCache->virtual_end - fOffset;
200
201	for (uint32 i = 0; i < pagesTransferred; i++) {
202		if (i == pagesTransferred - 1
203			&& (bytesTransferred % B_PAGE_SIZE) != 0) {
204			// clear partial page
205			size_t bytesTouched = bytesTransferred % B_PAGE_SIZE;
206			vm_memset_physical(
207				((phys_addr_t)fPages[i]->physical_page_number << PAGE_SHIFT)
208					+ bytesTouched,
209				0, B_PAGE_SIZE - bytesTouched);
210		}
211
212		DEBUG_PAGE_ACCESS_TRANSFER(fPages[i], fAllocatingThread);
213
214		fCache->MarkPageUnbusy(fPages[i]);
215
216		DEBUG_PAGE_ACCESS_END(fPages[i]);
217	}
218
219	// Free pages after failed I/O
220	for (uint32 i = pagesTransferred; i < fPageCount; i++) {
221		DEBUG_PAGE_ACCESS_TRANSFER(fPages[i], fAllocatingThread);
222		fCache->NotifyPageEvents(fPages[i], PAGE_EVENT_NOT_BUSY);
223		fCache->RemovePage(fPages[i]);
224		vm_page_set_state(fPages[i], PAGE_STATE_FREE);
225	}
226
227	delete this;
228}
229
230
231//	#pragma mark -
232
233
234static void
235add_to_iovec(generic_io_vec* vecs, uint32 &index, uint32 max,
236	generic_addr_t address, generic_size_t size)
237{
238	if (index > 0 && vecs[index - 1].base + vecs[index - 1].length == address) {
239		// the iovec can be combined with the previous one
240		vecs[index - 1].length += size;
241		return;
242	}
243
244	if (index == max)
245		panic("no more space for iovecs!");
246
247	// we need to start a new iovec
248	vecs[index].base = address;
249	vecs[index].length = size;
250	index++;
251}
252
253
254static inline bool
255access_is_sequential(file_cache_ref* ref)
256{
257	return ref->last_access[ref->last_access_index] != 0;
258}
259
260
261static inline void
262push_access(file_cache_ref* ref, off_t offset, generic_size_t bytes,
263	bool isWrite)
264{
265	TRACE(("%p: push %Ld, %ld, %s\n", ref, offset, bytes,
266		isWrite ? "write" : "read"));
267
268	int32 index = ref->last_access_index;
269	int32 previous = index - 1;
270	if (previous < 0)
271		previous = LAST_ACCESSES - 1;
272
273	if (offset != ref->LastAccess(previous, isWrite))
274		ref->last_access[previous] = 0;
275
276	ref->SetLastAccess(index, offset + bytes, isWrite);
277
278	if (++index >= LAST_ACCESSES)
279		index = 0;
280	ref->last_access_index = index;
281}
282
283
284static void
285reserve_pages(file_cache_ref* ref, vm_page_reservation* reservation,
286	size_t reservePages, bool isWrite)
287{
288	if (low_resource_state(B_KERNEL_RESOURCE_PAGES) != B_NO_LOW_RESOURCE) {
289		VMCache* cache = ref->cache;
290		cache->Lock();
291
292		if (cache->consumers.IsEmpty() && cache->areas == NULL
293			&& access_is_sequential(ref)) {
294			// we are not mapped, and we're accessed sequentially
295
296			if (isWrite) {
297				// Just write some pages back, and actually wait until they
298				// have been written back in order to relieve the page pressure
299				// a bit.
300				int32 index = ref->last_access_index;
301				int32 previous = index - 1;
302				if (previous < 0)
303					previous = LAST_ACCESSES - 1;
304
305				vm_page_write_modified_page_range(cache,
306					ref->LastAccessPageOffset(previous, true),
307					ref->LastAccessPageOffset(index, true));
308			} else {
309				// free some pages from our cache
310				// TODO: start with oldest
311				uint32 left = reservePages;
312				vm_page* page;
313				for (VMCachePagesTree::Iterator it = cache->pages.GetIterator();
314						(page = it.Next()) != NULL && left > 0;) {
315					if (page->State() == PAGE_STATE_CACHED && !page->busy) {
316						DEBUG_PAGE_ACCESS_START(page);
317						ASSERT(!page->IsMapped());
318						ASSERT(!page->modified);
319						cache->RemovePage(page);
320						vm_page_set_state(page, PAGE_STATE_FREE);
321						left--;
322					}
323				}
324			}
325		}
326		cache->Unlock();
327	}
328
329	vm_page_reserve_pages(reservation, reservePages, VM_PRIORITY_USER);
330}
331
332
333static inline status_t
334read_pages_and_clear_partial(file_cache_ref* ref, void* cookie, off_t offset,
335	const generic_io_vec* vecs, size_t count, uint32 flags,
336	generic_size_t* _numBytes)
337{
338	generic_size_t bytesUntouched = *_numBytes;
339
340	status_t status = vfs_read_pages(ref->vnode, cookie, offset, vecs, count,
341		flags, _numBytes);
342
343	generic_size_t bytesEnd = *_numBytes;
344
345	if (offset + (off_t)bytesEnd > ref->cache->virtual_end)
346		bytesEnd = ref->cache->virtual_end - offset;
347
348	if (status == B_OK && bytesEnd < bytesUntouched) {
349		// Clear out any leftovers that were not touched by the above read.
350		// We're doing this here so that not every file system/device has to
351		// implement this.
352		bytesUntouched -= bytesEnd;
353
354		for (int32 i = count; i-- > 0 && bytesUntouched != 0; ) {
355			generic_size_t length = min_c(bytesUntouched, vecs[i].length);
356			vm_memset_physical(vecs[i].base + vecs[i].length - length, 0,
357				length);
358
359			bytesUntouched -= length;
360		}
361	}
362
363	return status;
364}
365
366
367/*!	Reads the requested amount of data into the cache, and allocates
368	pages needed to fulfill that request. This function is called by cache_io().
369	It can only handle a certain amount of bytes, and the caller must make
370	sure that it matches that criterion.
371	The cache_ref lock must be held when calling this function; during
372	operation it will unlock the cache, though.
373*/
374static status_t
375read_into_cache(file_cache_ref* ref, void* cookie, off_t offset,
376	int32 pageOffset, addr_t buffer, size_t bufferSize, bool useBuffer,
377	vm_page_reservation* reservation, size_t reservePages)
378{
379	TRACE(("read_into_cache(offset = %Ld, pageOffset = %ld, buffer = %#lx, "
380		"bufferSize = %lu\n", offset, pageOffset, buffer, bufferSize));
381
382	VMCache* cache = ref->cache;
383
384	// TODO: We're using way too much stack! Rather allocate a sufficiently
385	// large chunk on the heap.
386	generic_io_vec vecs[MAX_IO_VECS];
387	uint32 vecCount = 0;
388
389	generic_size_t numBytes = PAGE_ALIGN(pageOffset + bufferSize);
390	vm_page* pages[MAX_IO_VECS];
391	int32 pageIndex = 0;
392
393	// allocate pages for the cache and mark them busy
394	for (generic_size_t pos = 0; pos < numBytes; pos += B_PAGE_SIZE) {
395		vm_page* page = pages[pageIndex++] = vm_page_allocate_page(
396			reservation, PAGE_STATE_CACHED | VM_PAGE_ALLOC_BUSY);
397
398		cache->InsertPage(page, offset + pos);
399
400		add_to_iovec(vecs, vecCount, MAX_IO_VECS,
401			page->physical_page_number * B_PAGE_SIZE, B_PAGE_SIZE);
402			// TODO: check if the array is large enough (currently panics)!
403	}
404
405	push_access(ref, offset, bufferSize, false);
406	cache->Unlock();
407	vm_page_unreserve_pages(reservation);
408
409	// read file into reserved pages
410	status_t status = read_pages_and_clear_partial(ref, cookie, offset, vecs,
411		vecCount, B_PHYSICAL_IO_REQUEST, &numBytes);
412	if (status != B_OK) {
413		// reading failed, free allocated pages
414
415		dprintf("file_cache: read pages failed: %s\n", strerror(status));
416
417		cache->Lock();
418
419		for (int32 i = 0; i < pageIndex; i++) {
420			cache->NotifyPageEvents(pages[i], PAGE_EVENT_NOT_BUSY);
421			cache->RemovePage(pages[i]);
422			vm_page_set_state(pages[i], PAGE_STATE_FREE);
423		}
424
425		return status;
426	}
427
428	// copy the pages if needed and unmap them again
429
430	for (int32 i = 0; i < pageIndex; i++) {
431		if (useBuffer && bufferSize != 0) {
432			size_t bytes = min_c(bufferSize, (size_t)B_PAGE_SIZE - pageOffset);
433
434			vm_memcpy_from_physical((void*)buffer,
435				pages[i]->physical_page_number * B_PAGE_SIZE + pageOffset,
436				bytes, IS_USER_ADDRESS(buffer));
437
438			buffer += bytes;
439			bufferSize -= bytes;
440			pageOffset = 0;
441		}
442	}
443
444	reserve_pages(ref, reservation, reservePages, false);
445	cache->Lock();
446
447	// make the pages accessible in the cache
448	for (int32 i = pageIndex; i-- > 0;) {
449		DEBUG_PAGE_ACCESS_END(pages[i]);
450
451		cache->MarkPageUnbusy(pages[i]);
452	}
453
454	return B_OK;
455}
456
457
458static status_t
459read_from_file(file_cache_ref* ref, void* cookie, off_t offset,
460	int32 pageOffset, addr_t buffer, size_t bufferSize, bool useBuffer,
461	vm_page_reservation* reservation, size_t reservePages)
462{
463	TRACE(("read_from_file(offset = %Ld, pageOffset = %ld, buffer = %#lx, "
464		"bufferSize = %lu\n", offset, pageOffset, buffer, bufferSize));
465
466	if (!useBuffer)
467		return B_OK;
468
469	generic_io_vec vec;
470	vec.base = buffer;
471	vec.length = bufferSize;
472
473	push_access(ref, offset, bufferSize, false);
474	ref->cache->Unlock();
475	vm_page_unreserve_pages(reservation);
476
477	generic_size_t toRead = bufferSize;
478	status_t status = vfs_read_pages(ref->vnode, cookie, offset + pageOffset,
479		&vec, 1, 0, &toRead);
480
481	if (status == B_OK)
482		reserve_pages(ref, reservation, reservePages, false);
483
484	ref->cache->Lock();
485
486	return status;
487}
488
489
490/*!	Like read_into_cache() but writes data into the cache.
491	To preserve data consistency, it might also read pages into the cache,
492	though, if only a partial page gets written.
493	The same restrictions apply.
494*/
495static status_t
496write_to_cache(file_cache_ref* ref, void* cookie, off_t offset,
497	int32 pageOffset, addr_t buffer, size_t bufferSize, bool useBuffer,
498	vm_page_reservation* reservation, size_t reservePages)
499{
500	// TODO: We're using way too much stack! Rather allocate a sufficiently
501	// large chunk on the heap.
502	generic_io_vec vecs[MAX_IO_VECS];
503	uint32 vecCount = 0;
504	generic_size_t numBytes = PAGE_ALIGN(pageOffset + bufferSize);
505	vm_page* pages[MAX_IO_VECS];
506	int32 pageIndex = 0;
507	status_t status = B_OK;
508
509	// ToDo: this should be settable somewhere
510	bool writeThrough = false;
511
512	// allocate pages for the cache and mark them busy
513	for (generic_size_t pos = 0; pos < numBytes; pos += B_PAGE_SIZE) {
514		// TODO: if space is becoming tight, and this cache is already grown
515		//	big - shouldn't we better steal the pages directly in that case?
516		//	(a working set like approach for the file cache)
517		// TODO: the pages we allocate here should have been reserved upfront
518		//	in cache_io()
519		vm_page* page = pages[pageIndex++] = vm_page_allocate_page(
520			reservation,
521			(writeThrough ? PAGE_STATE_CACHED : PAGE_STATE_MODIFIED)
522				| VM_PAGE_ALLOC_BUSY);
523
524		page->modified = !writeThrough;
525
526		ref->cache->InsertPage(page, offset + pos);
527
528		add_to_iovec(vecs, vecCount, MAX_IO_VECS,
529			page->physical_page_number * B_PAGE_SIZE, B_PAGE_SIZE);
530	}
531
532	push_access(ref, offset, bufferSize, true);
533	ref->cache->Unlock();
534	vm_page_unreserve_pages(reservation);
535
536	// copy contents (and read in partially written pages first)
537
538	if (pageOffset != 0) {
539		// This is only a partial write, so we have to read the rest of the page
540		// from the file to have consistent data in the cache
541		generic_io_vec readVec = { vecs[0].base, B_PAGE_SIZE };
542		generic_size_t bytesRead = B_PAGE_SIZE;
543
544		status = vfs_read_pages(ref->vnode, cookie, offset, &readVec, 1,
545			B_PHYSICAL_IO_REQUEST, &bytesRead);
546		// ToDo: handle errors for real!
547		if (status < B_OK)
548			panic("1. vfs_read_pages() failed: %s!\n", strerror(status));
549	}
550
551	size_t lastPageOffset = (pageOffset + bufferSize) % B_PAGE_SIZE;
552	if (lastPageOffset != 0) {
553		// get the last page in the I/O vectors
554		generic_addr_t last = vecs[vecCount - 1].base
555			+ vecs[vecCount - 1].length - B_PAGE_SIZE;
556
557		if ((off_t)(offset + pageOffset + bufferSize) == ref->cache->virtual_end) {
558			// the space in the page after this write action needs to be cleaned
559			vm_memset_physical(last + lastPageOffset, 0,
560				B_PAGE_SIZE - lastPageOffset);
561		} else {
562			// the end of this write does not happen on a page boundary, so we
563			// need to fetch the last page before we can update it
564			generic_io_vec readVec = { last, B_PAGE_SIZE };
565			generic_size_t bytesRead = B_PAGE_SIZE;
566
567			status = vfs_read_pages(ref->vnode, cookie,
568				PAGE_ALIGN(offset + pageOffset + bufferSize) - B_PAGE_SIZE,
569				&readVec, 1, B_PHYSICAL_IO_REQUEST, &bytesRead);
570			// ToDo: handle errors for real!
571			if (status < B_OK)
572				panic("vfs_read_pages() failed: %s!\n", strerror(status));
573
574			if (bytesRead < B_PAGE_SIZE) {
575				// the space beyond the file size needs to be cleaned
576				vm_memset_physical(last + bytesRead, 0,
577					B_PAGE_SIZE - bytesRead);
578			}
579		}
580	}
581
582	for (uint32 i = 0; i < vecCount; i++) {
583		generic_addr_t base = vecs[i].base;
584		generic_size_t bytes = min_c((generic_size_t)bufferSize,
585			generic_size_t(vecs[i].length - pageOffset));
586
587		if (useBuffer) {
588			// copy data from user buffer
589			vm_memcpy_to_physical(base + pageOffset, (void*)buffer, bytes,
590				IS_USER_ADDRESS(buffer));
591		} else {
592			// clear buffer instead
593			vm_memset_physical(base + pageOffset, 0, bytes);
594		}
595
596		bufferSize -= bytes;
597		if (bufferSize == 0)
598			break;
599
600		buffer += bytes;
601		pageOffset = 0;
602	}
603
604	if (writeThrough) {
605		// write cached pages back to the file if we were asked to do that
606		status_t status = vfs_write_pages(ref->vnode, cookie, offset, vecs,
607			vecCount, B_PHYSICAL_IO_REQUEST, &numBytes);
608		if (status < B_OK) {
609			// ToDo: remove allocated pages, ...?
610			panic("file_cache: remove allocated pages! write pages failed: %s\n",
611				strerror(status));
612		}
613	}
614
615	if (status == B_OK)
616		reserve_pages(ref, reservation, reservePages, true);
617
618	ref->cache->Lock();
619
620	// make the pages accessible in the cache
621	for (int32 i = pageIndex; i-- > 0;) {
622		ref->cache->MarkPageUnbusy(pages[i]);
623
624		DEBUG_PAGE_ACCESS_END(pages[i]);
625	}
626
627	return status;
628}
629
630
631static status_t
632write_to_file(file_cache_ref* ref, void* cookie, off_t offset, int32 pageOffset,
633	addr_t buffer, size_t bufferSize, bool useBuffer,
634	vm_page_reservation* reservation, size_t reservePages)
635{
636	push_access(ref, offset, bufferSize, true);
637	ref->cache->Unlock();
638	vm_page_unreserve_pages(reservation);
639
640	status_t status = B_OK;
641
642	if (!useBuffer) {
643		while (bufferSize > 0) {
644			generic_size_t written = min_c(bufferSize, kZeroVecSize);
645			status = vfs_write_pages(ref->vnode, cookie, offset + pageOffset,
646				sZeroVecs, kZeroVecCount, B_PHYSICAL_IO_REQUEST, &written);
647			if (status != B_OK)
648				return status;
649			if (written == 0)
650				return B_ERROR;
651
652			bufferSize -= written;
653			pageOffset += written;
654		}
655	} else {
656		generic_io_vec vec;
657		vec.base = buffer;
658		vec.length = bufferSize;
659		generic_size_t toWrite = bufferSize;
660		status = vfs_write_pages(ref->vnode, cookie, offset + pageOffset,
661			&vec, 1, 0, &toWrite);
662	}
663
664	if (status == B_OK)
665		reserve_pages(ref, reservation, reservePages, true);
666
667	ref->cache->Lock();
668
669	return status;
670}
671
672
673static inline status_t
674satisfy_cache_io(file_cache_ref* ref, void* cookie, cache_func function,
675	off_t offset, addr_t buffer, bool useBuffer, int32 &pageOffset,
676	size_t bytesLeft, size_t &reservePages, off_t &lastOffset,
677	addr_t &lastBuffer, int32 &lastPageOffset, size_t &lastLeft,
678	size_t &lastReservedPages, vm_page_reservation* reservation)
679{
680	if (lastBuffer == buffer)
681		return B_OK;
682
683	size_t requestSize = buffer - lastBuffer;
684	reservePages = min_c(MAX_IO_VECS, (lastLeft - requestSize
685		+ lastPageOffset + B_PAGE_SIZE - 1) >> PAGE_SHIFT);
686
687	status_t status = function(ref, cookie, lastOffset, lastPageOffset,
688		lastBuffer, requestSize, useBuffer, reservation, reservePages);
689	if (status == B_OK) {
690		lastReservedPages = reservePages;
691		lastBuffer = buffer;
692		lastLeft = bytesLeft;
693		lastOffset = offset;
694		lastPageOffset = 0;
695		pageOffset = 0;
696	}
697	return status;
698}
699
700
701static status_t
702cache_io(void* _cacheRef, void* cookie, off_t offset, addr_t buffer,
703	size_t* _size, bool doWrite)
704{
705	if (_cacheRef == NULL)
706		panic("cache_io() called with NULL ref!\n");
707
708	file_cache_ref* ref = (file_cache_ref*)_cacheRef;
709	VMCache* cache = ref->cache;
710	bool useBuffer = buffer != 0;
711
712	TRACE(("cache_io(ref = %p, offset = %Ld, buffer = %p, size = %lu, %s)\n",
713		ref, offset, (void*)buffer, *_size, doWrite ? "write" : "read"));
714
715	int32 pageOffset = offset & (B_PAGE_SIZE - 1);
716	size_t size = *_size;
717	offset -= pageOffset;
718
719	// "offset" and "lastOffset" are always aligned to B_PAGE_SIZE,
720	// the "last*" variables always point to the end of the last
721	// satisfied request part
722
723	const uint32 kMaxChunkSize = MAX_IO_VECS * B_PAGE_SIZE;
724	size_t bytesLeft = size, lastLeft = size;
725	int32 lastPageOffset = pageOffset;
726	addr_t lastBuffer = buffer;
727	off_t lastOffset = offset;
728	size_t lastReservedPages = min_c(MAX_IO_VECS, (pageOffset + bytesLeft
729		+ B_PAGE_SIZE - 1) >> PAGE_SHIFT);
730	size_t reservePages = 0;
731	size_t pagesProcessed = 0;
732	cache_func function = NULL;
733
734	vm_page_reservation reservation;
735	reserve_pages(ref, &reservation, lastReservedPages, doWrite);
736
737	AutoLocker<VMCache> locker(cache);
738
739	while (bytesLeft > 0) {
740		// Periodically reevaluate the low memory situation and select the
741		// read/write hook accordingly
742		if (pagesProcessed % 32 == 0) {
743			if (size >= BYPASS_IO_SIZE
744				&& low_resource_state(B_KERNEL_RESOURCE_PAGES)
745					!= B_NO_LOW_RESOURCE) {
746				// In low memory situations we bypass the cache beyond a
747				// certain I/O size.
748				function = doWrite ? write_to_file : read_from_file;
749			} else
750				function = doWrite ? write_to_cache : read_into_cache;
751		}
752
753		// check if this page is already in memory
754		vm_page* page = cache->LookupPage(offset);
755		if (page != NULL) {
756			// The page may be busy - since we need to unlock the cache sometime
757			// in the near future, we need to satisfy the request of the pages
758			// we didn't get yet (to make sure no one else interferes in the
759			// meantime).
760			status_t status = satisfy_cache_io(ref, cookie, function, offset,
761				buffer, useBuffer, pageOffset, bytesLeft, reservePages,
762				lastOffset, lastBuffer, lastPageOffset, lastLeft,
763				lastReservedPages, &reservation);
764			if (status != B_OK)
765				return status;
766
767			// Since satisfy_cache_io() unlocks the cache, we need to look up
768			// the page again.
769			page = cache->LookupPage(offset);
770			if (page != NULL && page->busy) {
771				cache->WaitForPageEvents(page, PAGE_EVENT_NOT_BUSY, true);
772				continue;
773			}
774		}
775
776		size_t bytesInPage = min_c(size_t(B_PAGE_SIZE - pageOffset), bytesLeft);
777
778		TRACE(("lookup page from offset %Ld: %p, size = %lu, pageOffset "
779			"= %lu\n", offset, page, bytesLeft, pageOffset));
780
781		if (page != NULL) {
782			if (doWrite || useBuffer) {
783				// Since the following user_mem{cpy,set}() might cause a page
784				// fault, which in turn might cause pages to be reserved, we
785				// need to unlock the cache temporarily to avoid a potential
786				// deadlock. To make sure that our page doesn't go away, we mark
787				// it busy for the time.
788				page->busy = true;
789				locker.Unlock();
790
791				// copy the contents of the page already in memory
792				phys_addr_t pageAddress
793					= (phys_addr_t)page->physical_page_number * B_PAGE_SIZE
794						+ pageOffset;
795				bool userBuffer = IS_USER_ADDRESS(buffer);
796				if (doWrite) {
797					if (useBuffer) {
798						vm_memcpy_to_physical(pageAddress, (void*)buffer,
799							bytesInPage, userBuffer);
800					} else {
801						vm_memset_physical(pageAddress, 0, bytesInPage);
802					}
803				} else if (useBuffer) {
804					vm_memcpy_from_physical((void*)buffer, pageAddress,
805						bytesInPage, userBuffer);
806				}
807
808				locker.Lock();
809
810				if (doWrite) {
811					DEBUG_PAGE_ACCESS_START(page);
812
813					page->modified = true;
814
815					if (page->State() != PAGE_STATE_MODIFIED)
816						vm_page_set_state(page, PAGE_STATE_MODIFIED);
817
818					DEBUG_PAGE_ACCESS_END(page);
819				}
820
821				cache->MarkPageUnbusy(page);
822			}
823
824			// If it is cached only, requeue the page, so the respective queue
825			// roughly remains LRU first sorted.
826			if (page->State() == PAGE_STATE_CACHED
827					|| page->State() == PAGE_STATE_MODIFIED) {
828				DEBUG_PAGE_ACCESS_START(page);
829				vm_page_requeue(page, true);
830				DEBUG_PAGE_ACCESS_END(page);
831			}
832
833			if (bytesLeft <= bytesInPage) {
834				// we've read the last page, so we're done!
835				locker.Unlock();
836				vm_page_unreserve_pages(&reservation);
837				return B_OK;
838			}
839
840			// prepare a potential gap request
841			lastBuffer = buffer + bytesInPage;
842			lastLeft = bytesLeft - bytesInPage;
843			lastOffset = offset + B_PAGE_SIZE;
844			lastPageOffset = 0;
845		}
846
847		if (bytesLeft <= bytesInPage)
848			break;
849
850		buffer += bytesInPage;
851		bytesLeft -= bytesInPage;
852		pageOffset = 0;
853		offset += B_PAGE_SIZE;
854		pagesProcessed++;
855
856		if (buffer - lastBuffer + lastPageOffset >= kMaxChunkSize) {
857			status_t status = satisfy_cache_io(ref, cookie, function, offset,
858				buffer, useBuffer, pageOffset, bytesLeft, reservePages,
859				lastOffset, lastBuffer, lastPageOffset, lastLeft,
860				lastReservedPages, &reservation);
861			if (status != B_OK)
862				return status;
863		}
864	}
865
866	// fill the last remaining bytes of the request (either write or read)
867
868	return function(ref, cookie, lastOffset, lastPageOffset, lastBuffer,
869		lastLeft, useBuffer, &reservation, 0);
870}
871
872
873static status_t
874file_cache_control(const char* subsystem, uint32 function, void* buffer,
875	size_t bufferSize)
876{
877	switch (function) {
878		case CACHE_CLEAR:
879			// ToDo: clear the cache
880			dprintf("cache_control: clear cache!\n");
881			return B_OK;
882
883		case CACHE_SET_MODULE:
884		{
885			cache_module_info* module = sCacheModule;
886
887			// unset previous module
888
889			if (sCacheModule != NULL) {
890				sCacheModule = NULL;
891				snooze(100000);	// 0.1 secs
892				put_module(module->info.name);
893			}
894
895			// get new module, if any
896
897			if (buffer == NULL)
898				return B_OK;
899
900			char name[B_FILE_NAME_LENGTH];
901			if (!IS_USER_ADDRESS(buffer)
902				|| user_strlcpy(name, (char*)buffer,
903						B_FILE_NAME_LENGTH) < B_OK)
904				return B_BAD_ADDRESS;
905
906			if (strncmp(name, CACHE_MODULES_NAME, strlen(CACHE_MODULES_NAME)))
907				return B_BAD_VALUE;
908
909			dprintf("cache_control: set module %s!\n", name);
910
911			status_t status = get_module(name, (module_info**)&module);
912			if (status == B_OK)
913				sCacheModule = module;
914
915			return status;
916		}
917	}
918
919	return B_BAD_HANDLER;
920}
921
922
923//	#pragma mark - private kernel API
924
925
926extern "C" void
927cache_prefetch_vnode(struct vnode* vnode, off_t offset, size_t size)
928{
929	if (size == 0)
930		return;
931
932	VMCache* cache;
933	if (vfs_get_vnode_cache(vnode, &cache, false) != B_OK)
934		return;
935	if (cache->type != CACHE_TYPE_VNODE)
936		return;
937
938	file_cache_ref* ref = ((VMVnodeCache*)cache)->FileCacheRef();
939	off_t fileSize = cache->virtual_end;
940
941	if ((off_t)(offset + size) > fileSize)
942		size = fileSize - offset;
943
944	// "offset" and "size" are always aligned to B_PAGE_SIZE,
945	offset = ROUNDDOWN(offset, B_PAGE_SIZE);
946	size = ROUNDUP(size, B_PAGE_SIZE);
947
948	size_t reservePages = size / B_PAGE_SIZE;
949
950	// Don't do anything if we don't have the resources left, or the cache
951	// already contains more than 2/3 of its pages
952	if (offset >= fileSize || vm_page_num_unused_pages() < 2 * reservePages
953		|| 3 * cache->page_count > 2 * fileSize / B_PAGE_SIZE) {
954		cache->ReleaseRef();
955		return;
956	}
957
958	size_t bytesToRead = 0;
959	off_t lastOffset = offset;
960
961	vm_page_reservation reservation;
962	vm_page_reserve_pages(&reservation, reservePages, VM_PRIORITY_USER);
963
964	cache->Lock();
965
966	while (true) {
967		// check if this page is already in memory
968		if (size > 0) {
969			vm_page* page = cache->LookupPage(offset);
970
971			offset += B_PAGE_SIZE;
972			size -= B_PAGE_SIZE;
973
974			if (page == NULL) {
975				bytesToRead += B_PAGE_SIZE;
976				continue;
977			}
978		}
979		if (bytesToRead != 0) {
980			// read the part before the current page (or the end of the request)
981			PrecacheIO* io = new(std::nothrow) PrecacheIO(ref, lastOffset,
982				bytesToRead);
983			if (io == NULL || io->Prepare(&reservation) != B_OK) {
984				delete io;
985				break;
986			}
987
988			// we must not have the cache locked during I/O
989			cache->Unlock();
990			io->ReadAsync();
991			cache->Lock();
992
993			bytesToRead = 0;
994		}
995
996		if (size == 0) {
997			// we have reached the end of the request
998			break;
999		}
1000
1001		lastOffset = offset;
1002	}
1003
1004	cache->ReleaseRefAndUnlock();
1005	vm_page_unreserve_pages(&reservation);
1006}
1007
1008
1009extern "C" void
1010cache_prefetch(dev_t mountID, ino_t vnodeID, off_t offset, size_t size)
1011{
1012	// ToDo: schedule prefetch
1013
1014	TRACE(("cache_prefetch(vnode %ld:%Ld)\n", mountID, vnodeID));
1015
1016	// get the vnode for the object, this also grabs a ref to it
1017	struct vnode* vnode;
1018	if (vfs_get_vnode(mountID, vnodeID, true, &vnode) != B_OK)
1019		return;
1020
1021	cache_prefetch_vnode(vnode, offset, size);
1022	vfs_put_vnode(vnode);
1023}
1024
1025
1026extern "C" void
1027cache_node_opened(struct vnode* vnode, int32 fdType, VMCache* cache,
1028	dev_t mountID, ino_t parentID, ino_t vnodeID, const char* name)
1029{
1030	if (sCacheModule == NULL || sCacheModule->node_opened == NULL)
1031		return;
1032
1033	off_t size = -1;
1034	if (cache != NULL && cache->type == CACHE_TYPE_VNODE) {
1035		file_cache_ref* ref = ((VMVnodeCache*)cache)->FileCacheRef();
1036		if (ref != NULL)
1037			size = cache->virtual_end;
1038	}
1039
1040	sCacheModule->node_opened(vnode, fdType, mountID, parentID, vnodeID, name,
1041		size);
1042}
1043
1044
1045extern "C" void
1046cache_node_closed(struct vnode* vnode, int32 fdType, VMCache* cache,
1047	dev_t mountID, ino_t vnodeID)
1048{
1049	if (sCacheModule == NULL || sCacheModule->node_closed == NULL)
1050		return;
1051
1052	int32 accessType = 0;
1053	if (cache != NULL && cache->type == CACHE_TYPE_VNODE) {
1054		// ToDo: set accessType
1055	}
1056
1057	sCacheModule->node_closed(vnode, fdType, mountID, vnodeID, accessType);
1058}
1059
1060
1061extern "C" void
1062cache_node_launched(size_t argCount, char*  const* args)
1063{
1064	if (sCacheModule == NULL || sCacheModule->node_launched == NULL)
1065		return;
1066
1067	sCacheModule->node_launched(argCount, args);
1068}
1069
1070
1071extern "C" status_t
1072file_cache_init_post_boot_device(void)
1073{
1074	// ToDo: get cache module out of driver settings
1075
1076	if (get_module("file_cache/launch_speedup/v1",
1077			(module_info**)&sCacheModule) == B_OK) {
1078		dprintf("** opened launch speedup: %" B_PRId64 "\n", system_time());
1079	}
1080	return B_OK;
1081}
1082
1083
1084extern "C" status_t
1085file_cache_init(void)
1086{
1087	// allocate a clean page we can use for writing zeroes
1088	vm_page_reservation reservation;
1089	vm_page_reserve_pages(&reservation, 1, VM_PRIORITY_SYSTEM);
1090	vm_page* page = vm_page_allocate_page(&reservation,
1091		PAGE_STATE_WIRED | VM_PAGE_ALLOC_CLEAR);
1092	vm_page_unreserve_pages(&reservation);
1093
1094	sZeroPage = (phys_addr_t)page->physical_page_number * B_PAGE_SIZE;
1095
1096	for (uint32 i = 0; i < kZeroVecCount; i++) {
1097		sZeroVecs[i].base = sZeroPage;
1098		sZeroVecs[i].length = B_PAGE_SIZE;
1099	}
1100
1101	register_generic_syscall(CACHE_SYSCALLS, file_cache_control, 1, 0);
1102	return B_OK;
1103}
1104
1105
1106//	#pragma mark - public FS API
1107
1108
1109extern "C" void*
1110file_cache_create(dev_t mountID, ino_t vnodeID, off_t size)
1111{
1112	TRACE(("file_cache_create(mountID = %ld, vnodeID = %Ld, size = %Ld)\n",
1113		mountID, vnodeID, size));
1114
1115	file_cache_ref* ref = new file_cache_ref;
1116	if (ref == NULL)
1117		return NULL;
1118
1119	memset(ref->last_access, 0, sizeof(ref->last_access));
1120	ref->last_access_index = 0;
1121	ref->disabled_count = 0;
1122
1123	// TODO: delay VMCache creation until data is
1124	//	requested/written for the first time? Listing lots of
1125	//	files in Tracker (and elsewhere) could be slowed down.
1126	//	Since the file_cache_ref itself doesn't have a lock,
1127	//	we would need to "rent" one during construction, possibly
1128	//	the vnode lock, maybe a dedicated one.
1129	//	As there shouldn't be too much contention, we could also
1130	//	use atomic_test_and_set(), and free the resources again
1131	//	when that fails...
1132
1133	// Get the vnode for the object
1134	// (note, this does not grab a reference to the node)
1135	if (vfs_lookup_vnode(mountID, vnodeID, &ref->vnode) != B_OK)
1136		goto err1;
1137
1138	// Gets (usually creates) the cache for the node
1139	if (vfs_get_vnode_cache(ref->vnode, &ref->cache, true) != B_OK)
1140		goto err1;
1141
1142	ref->cache->virtual_end = size;
1143	((VMVnodeCache*)ref->cache)->SetFileCacheRef(ref);
1144	return ref;
1145
1146err1:
1147	delete ref;
1148	return NULL;
1149}
1150
1151
1152extern "C" void
1153file_cache_delete(void* _cacheRef)
1154{
1155	file_cache_ref* ref = (file_cache_ref*)_cacheRef;
1156
1157	if (ref == NULL)
1158		return;
1159
1160	TRACE(("file_cache_delete(ref = %p)\n", ref));
1161
1162	ref->cache->ReleaseRef();
1163	delete ref;
1164}
1165
1166
1167extern "C" void
1168file_cache_enable(void* _cacheRef)
1169{
1170	file_cache_ref* ref = (file_cache_ref*)_cacheRef;
1171
1172	AutoLocker<VMCache> _(ref->cache);
1173
1174	if (ref->disabled_count == 0) {
1175		panic("Unbalanced file_cache_enable()!");
1176		return;
1177	}
1178
1179	ref->disabled_count--;
1180}
1181
1182
1183extern "C" status_t
1184file_cache_disable(void* _cacheRef)
1185{
1186	// TODO: This function only removes all pages from the cache and prevents
1187	// that the file cache functions add any new ones until re-enabled. The
1188	// VM (on page fault) can still add pages, if the file is mmap()ed. We
1189	// should mark the cache to prevent shared mappings of the file and fix
1190	// the page fault code to deal correctly with private mappings (i.e. only
1191	// insert pages in consumer caches).
1192
1193	file_cache_ref* ref = (file_cache_ref*)_cacheRef;
1194
1195	AutoLocker<VMCache> _(ref->cache);
1196
1197	// If already disabled, there's nothing to do for us.
1198	if (ref->disabled_count > 0) {
1199		ref->disabled_count++;
1200		return B_OK;
1201	}
1202
1203	// The file cache is not yet disabled. We need to evict all cached pages.
1204	status_t error = ref->cache->FlushAndRemoveAllPages();
1205	if (error != B_OK)
1206		return error;
1207
1208	ref->disabled_count++;
1209	return B_OK;
1210}
1211
1212
1213extern "C" bool
1214file_cache_is_enabled(void* _cacheRef)
1215{
1216	file_cache_ref* ref = (file_cache_ref*)_cacheRef;
1217	AutoLocker<VMCache> _(ref->cache);
1218
1219	return ref->disabled_count == 0;
1220}
1221
1222
1223extern "C" status_t
1224file_cache_set_size(void* _cacheRef, off_t newSize)
1225{
1226	file_cache_ref* ref = (file_cache_ref*)_cacheRef;
1227
1228	TRACE(("file_cache_set_size(ref = %p, size = %Ld)\n", ref, newSize));
1229
1230	if (ref == NULL)
1231		return B_OK;
1232
1233	VMCache* cache = ref->cache;
1234	AutoLocker<VMCache> _(cache);
1235
1236	off_t oldSize = cache->virtual_end;
1237	status_t status = cache->Resize(newSize, VM_PRIORITY_USER);
1238		// Note, the priority doesn't really matter, since this cache doesn't
1239		// reserve any memory.
1240	if (status == B_OK && newSize < oldSize) {
1241		// We may have a new partial page at the end of the cache that must be
1242		// cleared.
1243		uint32 partialBytes = newSize % B_PAGE_SIZE;
1244		if (partialBytes != 0) {
1245			vm_page* page = cache->LookupPage(newSize - partialBytes);
1246			if (page != NULL) {
1247				vm_memset_physical(page->physical_page_number * B_PAGE_SIZE
1248					+ partialBytes, 0, B_PAGE_SIZE - partialBytes);
1249			}
1250		}
1251	}
1252
1253	return status;
1254}
1255
1256
1257extern "C" status_t
1258file_cache_sync(void* _cacheRef)
1259{
1260	file_cache_ref* ref = (file_cache_ref*)_cacheRef;
1261	if (ref == NULL)
1262		return B_BAD_VALUE;
1263
1264	return ref->cache->WriteModified();
1265}
1266
1267
1268extern "C" status_t
1269file_cache_read(void* _cacheRef, void* cookie, off_t offset, void* buffer,
1270	size_t* _size)
1271{
1272	file_cache_ref* ref = (file_cache_ref*)_cacheRef;
1273
1274	TRACE(("file_cache_read(ref = %p, offset = %Ld, buffer = %p, size = %lu)\n",
1275		ref, offset, buffer, *_size));
1276
1277	// Bounds checking. We do this here so it applies to uncached I/O.
1278	if (offset < 0)
1279		return B_BAD_VALUE;
1280	const off_t fileSize = ref->cache->virtual_end;
1281	if (offset >= fileSize || *_size == 0) {
1282		*_size = 0;
1283		return B_OK;
1284	}
1285	if ((off_t)(offset + *_size) > fileSize)
1286		*_size = fileSize - offset;
1287
1288	if (ref->disabled_count > 0) {
1289		// Caching is disabled -- read directly from the file.
1290		generic_io_vec vec;
1291		vec.base = (addr_t)buffer;
1292		generic_size_t size = vec.length = *_size;
1293		status_t error = vfs_read_pages(ref->vnode, cookie, offset, &vec, 1, 0,
1294			&size);
1295		*_size = size;
1296		return error;
1297	}
1298
1299	return cache_io(ref, cookie, offset, (addr_t)buffer, _size, false);
1300}
1301
1302
1303extern "C" status_t
1304file_cache_write(void* _cacheRef, void* cookie, off_t offset,
1305	const void* buffer, size_t* _size)
1306{
1307	file_cache_ref* ref = (file_cache_ref*)_cacheRef;
1308
1309	// We don't do bounds checking here, as we are relying on the
1310	// file system which called us to already have done that and made
1311	// adjustments as necessary, unlike in read().
1312
1313	if (ref->disabled_count > 0) {
1314		// Caching is disabled -- write directly to the file.
1315
1316		if (buffer != NULL) {
1317			generic_io_vec vec;
1318			vec.base = (addr_t)buffer;
1319			generic_size_t size = vec.length = *_size;
1320
1321			status_t error = vfs_write_pages(ref->vnode, cookie, offset, &vec,
1322				1, 0, &size);
1323			*_size = size;
1324			return error;
1325		}
1326
1327		// NULL buffer -- use a dummy buffer to write zeroes
1328		size_t size = *_size;
1329		while (size > 0) {
1330			size_t toWrite = min_c(size, kZeroVecSize);
1331			generic_size_t written = toWrite;
1332			status_t error = vfs_write_pages(ref->vnode, cookie, offset,
1333				sZeroVecs, kZeroVecCount, B_PHYSICAL_IO_REQUEST, &written);
1334			if (error != B_OK)
1335				return error;
1336			if (written == 0)
1337				break;
1338
1339			offset += written;
1340			size -= written;
1341		}
1342
1343		*_size -= size;
1344		return B_OK;
1345	}
1346
1347	status_t status = cache_io(ref, cookie, offset,
1348		(addr_t)const_cast<void*>(buffer), _size, true);
1349
1350	TRACE(("file_cache_write(ref = %p, offset = %Ld, buffer = %p, size = %lu)"
1351		" = %ld\n", ref, offset, buffer, *_size, status));
1352
1353	return status;
1354}
1355