1/*
2 * Copyright 2008-2011, Ingo Weinhold, ingo_weinhold@gmx.de.
3 * Copyright 2008-2017, Axel D��rfler, axeld@pinc-software.de.
4 * Distributed under the terms of the MIT License.
5 */
6
7
8#include "IORequest.h"
9
10#include <string.h>
11
12#include <arch/debug.h>
13#include <debug.h>
14#include <heap.h>
15#include <kernel.h>
16#include <team.h>
17#include <thread.h>
18#include <util/AutoLock.h>
19#include <vm/vm.h>
20#include <vm/VMAddressSpace.h>
21
22#include "dma_resources.h"
23
24
25//#define TRACE_IO_REQUEST
26#ifdef TRACE_IO_REQUEST
27#	define TRACE(x...) dprintf(x)
28#else
29#	define TRACE(x...) ;
30#endif
31
32
33// partial I/O operation phases
34enum {
35	PHASE_READ_BEGIN	= 0,
36	PHASE_READ_END		= 1,
37	PHASE_DO_ALL		= 2
38};
39
40
41struct virtual_vec_cookie {
42	uint32			vec_index;
43	generic_size_t	vec_offset;
44	area_id			mapped_area;
45	void*			physical_page_handle;
46	addr_t			virtual_address;
47
48	virtual_vec_cookie()
49		:
50		vec_index(0),
51		vec_offset(0),
52		mapped_area(-1),
53		physical_page_handle(NULL),
54		virtual_address((addr_t)-1)
55	{
56	}
57
58	void PutPhysicalPageIfNeeded()
59	{
60		if (virtual_address != (addr_t)-1) {
61			vm_put_physical_page(virtual_address, physical_page_handle);
62			virtual_address = (addr_t)-1;
63		}
64	}
65};
66
67
68// #pragma mark -
69
70
71IORequestChunk::IORequestChunk()
72	:
73	fParent(NULL),
74	fStatus(1)
75{
76}
77
78
79IORequestChunk::~IORequestChunk()
80{
81}
82
83
84//	#pragma mark -
85
86
87IOBuffer*
88IOBuffer::Create(uint32 count, bool vip)
89{
90	size_t size = sizeof(IOBuffer) + sizeof(generic_io_vec) * (count - 1);
91	IOBuffer* buffer
92		= (IOBuffer*)(malloc_etc(size, vip ? HEAP_PRIORITY_VIP : 0));
93	if (buffer == NULL)
94		return NULL;
95
96	buffer->fCapacity = count;
97	buffer->fVecCount = 0;
98	buffer->fUser = false;
99	buffer->fPhysical = false;
100	buffer->fVIP = vip;
101	buffer->fMemoryLocked = false;
102
103	return buffer;
104}
105
106
107void
108IOBuffer::Delete()
109{
110	free_etc(this, fVIP ? HEAP_PRIORITY_VIP : 0);
111}
112
113
114void
115IOBuffer::SetVecs(generic_size_t firstVecOffset, const generic_io_vec* vecs,
116	uint32 count, generic_size_t length, uint32 flags)
117{
118	memcpy(fVecs, vecs, sizeof(generic_io_vec) * count);
119
120	if (count > 0 && firstVecOffset > 0) {
121		fVecs[0].base += firstVecOffset;
122		fVecs[0].length -= firstVecOffset;
123	}
124
125	fVecCount = count;
126	fLength = length;
127	fPhysical = (flags & B_PHYSICAL_IO_REQUEST) != 0;
128	fUser = !fPhysical && IS_USER_ADDRESS(vecs[0].base);
129}
130
131
132status_t
133IOBuffer::GetNextVirtualVec(void*& _cookie, iovec& vector)
134{
135	virtual_vec_cookie* cookie = (virtual_vec_cookie*)_cookie;
136	if (cookie == NULL) {
137		cookie = new(malloc_flags(fVIP ? HEAP_PRIORITY_VIP : 0))
138			virtual_vec_cookie;
139		if (cookie == NULL)
140			return B_NO_MEMORY;
141
142		_cookie = cookie;
143	}
144
145	// recycle a potential previously mapped page
146	cookie->PutPhysicalPageIfNeeded();
147
148	if (cookie->vec_index >= fVecCount)
149		return B_BAD_INDEX;
150
151	if (!fPhysical) {
152		vector.iov_base = (void*)(addr_t)fVecs[cookie->vec_index].base;
153		vector.iov_len = fVecs[cookie->vec_index++].length;
154		return B_OK;
155	}
156
157	if (cookie->vec_index == 0
158		&& (fVecCount > 1 || fVecs[0].length > B_PAGE_SIZE)) {
159		void* mappedAddress;
160		addr_t mappedSize;
161
162// TODO: This is a potential violation of the VIP requirement, since
163// vm_map_physical_memory_vecs() allocates memory without special flags!
164		cookie->mapped_area = vm_map_physical_memory_vecs(
165			VMAddressSpace::KernelID(), "io buffer mapped physical vecs",
166			&mappedAddress, B_ANY_KERNEL_ADDRESS, &mappedSize,
167			B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, fVecs, fVecCount);
168
169		if (cookie->mapped_area >= 0) {
170			vector.iov_base = mappedAddress;
171			vector.iov_len = mappedSize;
172			return B_OK;
173		} else
174			ktrace_printf("failed to map area: %s\n", strerror(cookie->mapped_area));
175	}
176
177	// fallback to page wise mapping
178	generic_io_vec& currentVec = fVecs[cookie->vec_index];
179	generic_addr_t address = currentVec.base + cookie->vec_offset;
180	size_t pageOffset = address % B_PAGE_SIZE;
181
182// TODO: This is a potential violation of the VIP requirement, since
183// vm_get_physical_page() may allocate memory without special flags!
184	status_t result = vm_get_physical_page(address - pageOffset,
185		&cookie->virtual_address, &cookie->physical_page_handle);
186	if (result != B_OK)
187		return result;
188
189	generic_size_t length = min_c(currentVec.length - cookie->vec_offset,
190		B_PAGE_SIZE - pageOffset);
191
192	vector.iov_base = (void*)(cookie->virtual_address + pageOffset);
193	vector.iov_len = length;
194
195	cookie->vec_offset += length;
196	if (cookie->vec_offset >= currentVec.length) {
197		cookie->vec_index++;
198		cookie->vec_offset = 0;
199	}
200
201	return B_OK;
202}
203
204
205void
206IOBuffer::FreeVirtualVecCookie(void* _cookie)
207{
208	virtual_vec_cookie* cookie = (virtual_vec_cookie*)_cookie;
209	if (cookie->mapped_area >= 0)
210		delete_area(cookie->mapped_area);
211
212	cookie->PutPhysicalPageIfNeeded();
213
214	free_etc(cookie, fVIP ? HEAP_PRIORITY_VIP : 0);
215}
216
217
218status_t
219IOBuffer::LockMemory(team_id team, bool isWrite)
220{
221	if (fMemoryLocked) {
222		panic("memory already locked!");
223		return B_BAD_VALUE;
224	}
225
226	for (uint32 i = 0; i < fVecCount; i++) {
227		status_t status = lock_memory_etc(team, (void*)(addr_t)fVecs[i].base,
228			fVecs[i].length, isWrite ? 0 : B_READ_DEVICE);
229		if (status != B_OK) {
230			_UnlockMemory(team, i, isWrite);
231			return status;
232		}
233	}
234
235	fMemoryLocked = true;
236	return B_OK;
237}
238
239
240void
241IOBuffer::_UnlockMemory(team_id team, size_t count, bool isWrite)
242{
243	for (uint32 i = 0; i < count; i++) {
244		unlock_memory_etc(team, (void*)(addr_t)fVecs[i].base, fVecs[i].length,
245			isWrite ? 0 : B_READ_DEVICE);
246	}
247}
248
249
250void
251IOBuffer::UnlockMemory(team_id team, bool isWrite)
252{
253	if (!fMemoryLocked) {
254		panic("memory not locked");
255		return;
256	}
257
258	_UnlockMemory(team, fVecCount, isWrite);
259	fMemoryLocked = false;
260}
261
262
263void
264IOBuffer::Dump() const
265{
266	kprintf("IOBuffer at %p\n", this);
267
268	kprintf("  origin:     %s\n", fUser ? "user" : "kernel");
269	kprintf("  kind:       %s\n", fPhysical ? "physical" : "virtual");
270	kprintf("  length:     %" B_PRIuGENADDR "\n", fLength);
271	kprintf("  capacity:   %" B_PRIuSIZE "\n", fCapacity);
272	kprintf("  vecs:       %" B_PRIuSIZE "\n", fVecCount);
273
274	for (uint32 i = 0; i < fVecCount; i++) {
275		kprintf("    [%" B_PRIu32 "] %#" B_PRIxGENADDR ", %" B_PRIuGENADDR "\n",
276			i, fVecs[i].base, fVecs[i].length);
277	}
278}
279
280
281// #pragma mark -
282
283
284bool
285IOOperation::Finish()
286{
287	TRACE("IOOperation::Finish()\n");
288	if (fStatus == B_OK) {
289		if (fParent->IsWrite()) {
290			TRACE("  is write\n");
291			if (fPhase == PHASE_READ_BEGIN) {
292				TRACE("  phase read begin\n");
293				// repair phase adjusted vec
294				fDMABuffer->VecAt(fSavedVecIndex).length = fSavedVecLength;
295
296				// partial write: copy partial begin to bounce buffer
297				bool skipReadEndPhase;
298				status_t error = _CopyPartialBegin(true, skipReadEndPhase);
299				if (error == B_OK) {
300					// We're done with the first phase only (read in begin).
301					// Get ready for next phase...
302					fPhase = HasPartialEnd() && !skipReadEndPhase
303						? PHASE_READ_END : PHASE_DO_ALL;
304					_PrepareVecs();
305					ResetStatus();
306						// TODO: Is there a race condition, if the request is
307						// aborted at the same time?
308					return false;
309				}
310
311				SetStatus(error);
312			} else if (fPhase == PHASE_READ_END) {
313				TRACE("  phase read end\n");
314				// repair phase adjusted vec
315				generic_io_vec& vec = fDMABuffer->VecAt(fSavedVecIndex);
316				vec.base += vec.length - fSavedVecLength;
317				vec.length = fSavedVecLength;
318
319				// partial write: copy partial end to bounce buffer
320				status_t error = _CopyPartialEnd(true);
321				if (error == B_OK) {
322					// We're done with the second phase only (read in end).
323					// Get ready for next phase...
324					fPhase = PHASE_DO_ALL;
325					ResetStatus();
326						// TODO: Is there a race condition, if the request is
327						// aborted at the same time?
328					return false;
329				}
330
331				SetStatus(error);
332			}
333		}
334	}
335
336	if (fParent->IsRead() && UsesBounceBuffer()) {
337		TRACE("  read with bounce buffer\n");
338		// copy the bounce buffer segments to the final location
339		uint8* bounceBuffer = (uint8*)fDMABuffer->BounceBufferAddress();
340		phys_addr_t bounceBufferStart
341			= fDMABuffer->PhysicalBounceBufferAddress();
342		phys_addr_t bounceBufferEnd = bounceBufferStart
343			+ fDMABuffer->BounceBufferSize();
344
345		const generic_io_vec* vecs = fDMABuffer->Vecs();
346		uint32 vecCount = fDMABuffer->VecCount();
347
348		status_t error = B_OK;
349
350		// We iterate through the vecs we have read, moving offset (the device
351		// offset) as we go. If [offset, offset + vec.length) intersects with
352		// [startOffset, endOffset) we copy to the final location.
353		off_t offset = fOffset;
354		const off_t startOffset = fOriginalOffset;
355		const off_t endOffset = fOriginalOffset + fOriginalLength;
356
357		for (uint32 i = 0; error == B_OK && i < vecCount; i++) {
358			const generic_io_vec& vec = vecs[i];
359			generic_addr_t base = vec.base;
360			generic_size_t length = vec.length;
361
362			if (offset < startOffset) {
363				// If the complete vector is before the start offset, skip it.
364				if (offset + (off_t)length <= startOffset) {
365					offset += length;
366					continue;
367				}
368
369				// The vector starts before the start offset, but intersects
370				// with it. Skip the part we aren't interested in.
371				generic_size_t diff = startOffset - offset;
372				offset += diff;
373				base += diff;
374				length -= diff;
375			}
376
377			if (offset + (off_t)length > endOffset) {
378				// If we're already beyond the end offset, we're done.
379				if (offset >= endOffset)
380					break;
381
382				// The vector extends beyond the end offset -- cut it.
383				length = endOffset - offset;
384			}
385
386			if (base >= bounceBufferStart && base < bounceBufferEnd) {
387				error = fParent->CopyData(
388					bounceBuffer + (base - bounceBufferStart), offset, length);
389			}
390
391			offset += length;
392		}
393
394		if (error != B_OK)
395			SetStatus(error);
396	}
397
398	return true;
399}
400
401
402/*!	Note: SetPartial() must be called first!
403*/
404status_t
405IOOperation::Prepare(IORequest* request)
406{
407	if (fParent != NULL)
408		fParent->RemoveOperation(this);
409
410	fParent = request;
411
412	fTransferredBytes = 0;
413
414	// set initial phase
415	fPhase = PHASE_DO_ALL;
416	if (fParent->IsWrite()) {
417		// Copy data to bounce buffer segments, save the partial begin/end vec,
418		// which will be copied after their respective read phase.
419		if (UsesBounceBuffer()) {
420			TRACE("  write with bounce buffer\n");
421			uint8* bounceBuffer = (uint8*)fDMABuffer->BounceBufferAddress();
422			phys_addr_t bounceBufferStart
423				= fDMABuffer->PhysicalBounceBufferAddress();
424			phys_addr_t bounceBufferEnd = bounceBufferStart
425				+ fDMABuffer->BounceBufferSize();
426
427			const generic_io_vec* vecs = fDMABuffer->Vecs();
428			uint32 vecCount = fDMABuffer->VecCount();
429			generic_size_t vecOffset = 0;
430			uint32 i = 0;
431
432			off_t offset = fOffset;
433			off_t endOffset = fOffset + fLength;
434
435			if (HasPartialBegin()) {
436				// skip first block
437				generic_size_t toSkip = fBlockSize;
438				while (toSkip > 0) {
439					if (vecs[i].length <= toSkip) {
440						toSkip -= vecs[i].length;
441						i++;
442					} else {
443						vecOffset = toSkip;
444						break;
445					}
446				}
447
448				offset += fBlockSize;
449			}
450
451			if (HasPartialEnd()) {
452				// skip last block
453				generic_size_t toSkip = fBlockSize;
454				while (toSkip > 0) {
455					if (vecs[vecCount - 1].length <= toSkip) {
456						toSkip -= vecs[vecCount - 1].length;
457						vecCount--;
458					} else
459						break;
460				}
461
462				endOffset -= fBlockSize;
463			}
464
465			for (; i < vecCount; i++) {
466				const generic_io_vec& vec = vecs[i];
467				generic_addr_t base = vec.base + vecOffset;
468				generic_size_t length = vec.length - vecOffset;
469				vecOffset = 0;
470
471				if (base >= bounceBufferStart && base < bounceBufferEnd) {
472					if (offset + (off_t)length > endOffset)
473						length = endOffset - offset;
474					status_t error = fParent->CopyData(offset,
475						bounceBuffer + (base - bounceBufferStart), length);
476					if (error != B_OK)
477						return error;
478				}
479
480				offset += length;
481			}
482		}
483
484		if (HasPartialBegin())
485			fPhase = PHASE_READ_BEGIN;
486		else if (HasPartialEnd())
487			fPhase = PHASE_READ_END;
488
489		_PrepareVecs();
490	}
491
492	ResetStatus();
493
494	if (fParent != NULL)
495		fParent->AddOperation(this);
496
497	return B_OK;
498}
499
500
501void
502IOOperation::SetOriginalRange(off_t offset, generic_size_t length)
503{
504	fOriginalOffset = fOffset = offset;
505	fOriginalLength = fLength = length;
506}
507
508
509void
510IOOperation::SetRange(off_t offset, generic_size_t length)
511{
512	fOffset = offset;
513	fLength = length;
514}
515
516
517off_t
518IOOperation::Offset() const
519{
520	return fPhase == PHASE_READ_END ? fOffset + fLength - fBlockSize : fOffset;
521}
522
523
524generic_size_t
525IOOperation::Length() const
526{
527	return fPhase == PHASE_DO_ALL ? fLength : fBlockSize;
528}
529
530
531generic_io_vec*
532IOOperation::Vecs() const
533{
534	switch (fPhase) {
535		case PHASE_READ_END:
536			return fDMABuffer->Vecs() + fSavedVecIndex;
537		case PHASE_READ_BEGIN:
538		case PHASE_DO_ALL:
539		default:
540			return fDMABuffer->Vecs();
541	}
542}
543
544
545uint32
546IOOperation::VecCount() const
547{
548	switch (fPhase) {
549		case PHASE_READ_BEGIN:
550			return fSavedVecIndex + 1;
551		case PHASE_READ_END:
552			return fDMABuffer->VecCount() - fSavedVecIndex;
553		case PHASE_DO_ALL:
554		default:
555			return fDMABuffer->VecCount();
556	}
557}
558
559
560void
561IOOperation::SetPartial(bool partialBegin, bool partialEnd)
562{
563	TRACE("partial begin %d, end %d\n", partialBegin, partialEnd);
564	fPartialBegin = partialBegin;
565	fPartialEnd = partialEnd;
566}
567
568
569bool
570IOOperation::IsWrite() const
571{
572	return fParent->IsWrite() && fPhase == PHASE_DO_ALL;
573}
574
575
576bool
577IOOperation::IsRead() const
578{
579	return fParent->IsRead();
580}
581
582
583void
584IOOperation::_PrepareVecs()
585{
586	// we need to prepare the vecs for consumption by the drivers
587	if (fPhase == PHASE_READ_BEGIN) {
588		generic_io_vec* vecs = fDMABuffer->Vecs();
589		uint32 vecCount = fDMABuffer->VecCount();
590		generic_size_t vecLength = fBlockSize;
591		for (uint32 i = 0; i < vecCount; i++) {
592			generic_io_vec& vec = vecs[i];
593			if (vec.length >= vecLength) {
594				fSavedVecIndex = i;
595				fSavedVecLength = vec.length;
596				vec.length = vecLength;
597				break;
598			}
599			vecLength -= vec.length;
600		}
601	} else if (fPhase == PHASE_READ_END) {
602		generic_io_vec* vecs = fDMABuffer->Vecs();
603		uint32 vecCount = fDMABuffer->VecCount();
604		generic_size_t vecLength = fBlockSize;
605		for (int32 i = vecCount - 1; i >= 0; i--) {
606			generic_io_vec& vec = vecs[i];
607			if (vec.length >= vecLength) {
608				fSavedVecIndex = i;
609				fSavedVecLength = vec.length;
610				vec.base += vec.length - vecLength;
611				vec.length = vecLength;
612				break;
613			}
614			vecLength -= vec.length;
615		}
616	}
617}
618
619
620status_t
621IOOperation::_CopyPartialBegin(bool isWrite, bool& singleBlockOnly)
622{
623	generic_size_t relativeOffset = OriginalOffset() - fOffset;
624	generic_size_t length = fBlockSize - relativeOffset;
625
626	singleBlockOnly = length >= OriginalLength();
627	if (singleBlockOnly)
628		length = OriginalLength();
629
630	TRACE("_CopyPartialBegin(%s, single only %d)\n",
631		isWrite ? "write" : "read", singleBlockOnly);
632
633	if (isWrite) {
634		return fParent->CopyData(OriginalOffset(),
635			(uint8*)fDMABuffer->BounceBufferAddress() + relativeOffset, length);
636	} else {
637		return fParent->CopyData(
638			(uint8*)fDMABuffer->BounceBufferAddress() + relativeOffset,
639			OriginalOffset(), length);
640	}
641}
642
643
644status_t
645IOOperation::_CopyPartialEnd(bool isWrite)
646{
647	TRACE("_CopyPartialEnd(%s)\n", isWrite ? "write" : "read");
648
649	const generic_io_vec& lastVec
650		= fDMABuffer->VecAt(fDMABuffer->VecCount() - 1);
651	off_t lastVecPos = fOffset + fLength - fBlockSize;
652	uint8* base = (uint8*)fDMABuffer->BounceBufferAddress()
653		+ (lastVec.base + lastVec.length - fBlockSize
654		- fDMABuffer->PhysicalBounceBufferAddress());
655		// NOTE: this won't work if we don't use the bounce buffer contiguously
656		// (because of boundary alignments).
657	generic_size_t length = OriginalOffset() + OriginalLength() - lastVecPos;
658
659	if (isWrite)
660		return fParent->CopyData(lastVecPos, base, length);
661
662	return fParent->CopyData(base, lastVecPos, length);
663}
664
665
666void
667IOOperation::Dump() const
668{
669	kprintf("io_operation at %p\n", this);
670
671	kprintf("  parent:           %p\n", fParent);
672	kprintf("  status:           %s\n", strerror(fStatus));
673	kprintf("  dma buffer:       %p\n", fDMABuffer);
674	kprintf("  offset:           %-8" B_PRIdOFF " (original: %" B_PRIdOFF ")\n",
675		fOffset, fOriginalOffset);
676	kprintf("  length:           %-8" B_PRIuGENADDR " (original: %"
677		B_PRIuGENADDR ")\n", fLength, fOriginalLength);
678	kprintf("  transferred:      %" B_PRIuGENADDR "\n", fTransferredBytes);
679	kprintf("  block size:       %" B_PRIuGENADDR "\n", fBlockSize);
680	kprintf("  saved vec index:  %u\n", fSavedVecIndex);
681	kprintf("  saved vec length: %u\n", fSavedVecLength);
682	kprintf("  r/w:              %s\n", IsWrite() ? "write" : "read");
683	kprintf("  phase:            %s\n", fPhase == PHASE_READ_BEGIN
684		? "read begin" : fPhase == PHASE_READ_END ? "read end"
685		: fPhase == PHASE_DO_ALL ? "do all" : "unknown");
686	kprintf("  partial begin:    %s\n", fPartialBegin ? "yes" : "no");
687	kprintf("  partial end:      %s\n", fPartialEnd ? "yes" : "no");
688	kprintf("  bounce buffer:    %s\n", fUsesBounceBuffer ? "yes" : "no");
689
690	set_debug_variable("_parent", (addr_t)fParent);
691	set_debug_variable("_buffer", (addr_t)fDMABuffer);
692}
693
694
695// #pragma mark -
696
697
698IORequest::IORequest()
699	:
700	fIsNotified(false),
701	fFinishedCallback(NULL),
702	fFinishedCookie(NULL),
703	fIterationCallback(NULL),
704	fIterationCookie(NULL)
705{
706	mutex_init(&fLock, "I/O request lock");
707	fFinishedCondition.Init(this, "I/O request finished");
708}
709
710
711IORequest::~IORequest()
712{
713	mutex_lock(&fLock);
714	DeleteSubRequests();
715	if (fBuffer != NULL)
716		fBuffer->Delete();
717	mutex_destroy(&fLock);
718}
719
720
721/* static */ IORequest*
722IORequest::Create(bool vip)
723{
724	return vip
725		? new(malloc_flags(HEAP_PRIORITY_VIP)) IORequest
726		: new(std::nothrow) IORequest;
727}
728
729
730status_t
731IORequest::Init(off_t offset, generic_addr_t buffer, generic_size_t length,
732	bool write, uint32 flags)
733{
734	ASSERT(offset >= 0);
735
736	generic_io_vec vec;
737	vec.base = buffer;
738	vec.length = length;
739	return Init(offset, &vec, 1, length, write, flags);
740}
741
742
743status_t
744IORequest::Init(off_t offset, generic_size_t firstVecOffset,
745	const generic_io_vec* vecs, size_t count, generic_size_t length, bool write,
746	uint32 flags)
747{
748	ASSERT(offset >= 0);
749
750	fBuffer = IOBuffer::Create(count, (flags & B_VIP_IO_REQUEST) != 0);
751	if (fBuffer == NULL)
752		return B_NO_MEMORY;
753
754	fBuffer->SetVecs(firstVecOffset, vecs, count, length, flags);
755
756	fOwner = NULL;
757	fOffset = offset;
758	fLength = length;
759	fRelativeParentOffset = 0;
760	fTransferSize = 0;
761	fFlags = flags;
762	Thread* thread = thread_get_current_thread();
763	fTeam = thread->team->id;
764	fThread = thread->id;
765	fIsWrite = write;
766	fPartialTransfer = false;
767	fSuppressChildNotifications = false;
768
769	// these are for iteration
770	fVecIndex = 0;
771	fVecOffset = 0;
772	fRemainingBytes = length;
773
774	fPendingChildren = 0;
775
776	fStatus = 1;
777
778	return B_OK;
779}
780
781
782status_t
783IORequest::CreateSubRequest(off_t parentOffset, off_t offset,
784	generic_size_t length, IORequest*& _subRequest)
785{
786	ASSERT(parentOffset >= fOffset && length <= fLength
787		&& parentOffset - fOffset <= (off_t)(fLength - length));
788
789	// find start vec
790	generic_size_t vecOffset = parentOffset - fOffset;
791	generic_io_vec* vecs = fBuffer->Vecs();
792	int32 vecCount = fBuffer->VecCount();
793	int32 startVec = 0;
794	for (; startVec < vecCount; startVec++) {
795		const generic_io_vec& vec = vecs[startVec];
796		if (vecOffset < vec.length)
797			break;
798
799		vecOffset -= vec.length;
800	}
801
802	// count vecs
803	generic_size_t currentVecOffset = vecOffset;
804	int32 endVec = startVec;
805	generic_size_t remainingLength = length;
806	for (; endVec < vecCount; endVec++) {
807		const generic_io_vec& vec = vecs[endVec];
808		if (vec.length - currentVecOffset >= remainingLength)
809			break;
810
811		remainingLength -= vec.length - currentVecOffset;
812		currentVecOffset = 0;
813	}
814
815	// create subrequest
816	IORequest* subRequest = Create((fFlags & B_VIP_IO_REQUEST) != 0);
817	if (subRequest == NULL)
818		return B_NO_MEMORY;
819
820	status_t error = subRequest->Init(offset, vecOffset, vecs + startVec,
821		endVec - startVec + 1, length, fIsWrite, fFlags & ~B_DELETE_IO_REQUEST);
822	if (error != B_OK) {
823		delete subRequest;
824		return error;
825	}
826
827	subRequest->fRelativeParentOffset = parentOffset - fOffset;
828	subRequest->fTeam = fTeam;
829	subRequest->fThread = fThread;
830
831	_subRequest = subRequest;
832	subRequest->SetParent(this);
833
834	MutexLocker _(fLock);
835
836	fChildren.Add(subRequest);
837	fPendingChildren++;
838	TRACE("IORequest::CreateSubRequest(): request: %p, subrequest: %p\n", this,
839		subRequest);
840
841	return B_OK;
842}
843
844
845void
846IORequest::DeleteSubRequests()
847{
848	while (IORequestChunk* chunk = fChildren.RemoveHead())
849		delete chunk;
850	fPendingChildren = 0;
851}
852
853
854void
855IORequest::SetFinishedCallback(io_request_finished_callback callback,
856	void* cookie)
857{
858	fFinishedCallback = callback;
859	fFinishedCookie = cookie;
860}
861
862
863void
864IORequest::SetIterationCallback(io_request_iterate_callback callback,
865	void* cookie)
866{
867	fIterationCallback = callback;
868	fIterationCookie = cookie;
869}
870
871
872io_request_finished_callback
873IORequest::FinishedCallback(void** _cookie) const
874{
875	if (_cookie != NULL)
876		*_cookie = fFinishedCookie;
877	return fFinishedCallback;
878}
879
880
881status_t
882IORequest::Wait(uint32 flags, bigtime_t timeout)
883{
884	MutexLocker locker(fLock);
885
886	if (IsFinished() && fIsNotified)
887		return Status();
888
889	ConditionVariableEntry entry;
890	fFinishedCondition.Add(&entry);
891
892	locker.Unlock();
893
894	status_t error = entry.Wait(flags, timeout);
895	if (error != B_OK)
896		return error;
897
898	return Status();
899}
900
901
902void
903IORequest::NotifyFinished()
904{
905	TRACE("IORequest::NotifyFinished(): request: %p\n", this);
906
907	MutexLocker locker(fLock);
908
909	if (fStatus == B_OK && !fPartialTransfer && RemainingBytes() > 0) {
910		// The request is not really done yet. If it has an iteration callback,
911		// call it.
912		if (fIterationCallback != NULL) {
913			ResetStatus();
914			locker.Unlock();
915			bool partialTransfer = false;
916			status_t error = fIterationCallback(fIterationCookie, this,
917				&partialTransfer);
918			if (error == B_OK && !partialTransfer)
919				return;
920
921			// Iteration failed, which means we're responsible for notifying the
922			// requests finished.
923			locker.Lock();
924			fStatus = error;
925			fPartialTransfer = true;
926		}
927	}
928
929	ASSERT(!fIsNotified);
930	ASSERT(fPendingChildren == 0);
931	ASSERT(fChildren.IsEmpty()
932		|| dynamic_cast<IOOperation*>(fChildren.Head()) == NULL);
933
934	// unlock the memory
935	if (fBuffer->IsMemoryLocked())
936		fBuffer->UnlockMemory(fTeam, fIsWrite);
937
938	// Cache the callbacks before we unblock waiters and unlock. Any of the
939	// following could delete this request, so we don't want to touch it
940	// once we have started telling others that it is done.
941	IORequest* parent = fParent;
942	io_request_finished_callback finishedCallback = fFinishedCallback;
943	void* finishedCookie = fFinishedCookie;
944	status_t status = fStatus;
945	generic_size_t lastTransferredOffset
946		= fRelativeParentOffset + fTransferSize;
947	bool partialTransfer = status != B_OK || fPartialTransfer;
948	bool deleteRequest = (fFlags & B_DELETE_IO_REQUEST) != 0;
949
950	// unblock waiters
951	fIsNotified = true;
952	fFinishedCondition.NotifyAll();
953
954	locker.Unlock();
955
956	// notify callback
957	if (finishedCallback != NULL) {
958		finishedCallback(finishedCookie, this, status, partialTransfer,
959			lastTransferredOffset);
960	}
961
962	// notify parent
963	if (parent != NULL) {
964		parent->SubRequestFinished(this, status, partialTransfer,
965			lastTransferredOffset);
966	}
967
968	if (deleteRequest)
969		delete this;
970}
971
972
973/*!	Returns whether this request or any of it's ancestors has a finished or
974	notification callback. Used to decide whether NotifyFinished() can be called
975	synchronously.
976*/
977bool
978IORequest::HasCallbacks() const
979{
980	if (fFinishedCallback != NULL || fIterationCallback != NULL)
981		return true;
982
983	return fParent != NULL && fParent->HasCallbacks();
984}
985
986
987void
988IORequest::SetStatusAndNotify(status_t status)
989{
990	MutexLocker locker(fLock);
991
992	if (fStatus != 1)
993		return;
994
995	fStatus = status;
996
997	locker.Unlock();
998
999	NotifyFinished();
1000}
1001
1002
1003void
1004IORequest::OperationFinished(IOOperation* operation, status_t status,
1005	bool partialTransfer, generic_size_t transferEndOffset)
1006{
1007	TRACE("IORequest::OperationFinished(%p, %#" B_PRIx32 "): request: %p\n",
1008		operation, status, this);
1009
1010	MutexLocker locker(fLock);
1011
1012	fChildren.Remove(operation);
1013	operation->SetParent(NULL);
1014
1015	if (status != B_OK || partialTransfer) {
1016		if (fTransferSize > transferEndOffset)
1017			fTransferSize = transferEndOffset;
1018		fPartialTransfer = true;
1019	}
1020
1021	if (status != B_OK && fStatus == 1)
1022		fStatus = status;
1023
1024	if (--fPendingChildren > 0)
1025		return;
1026
1027	// last child finished
1028
1029	// set status, if not done yet
1030	if (fStatus == 1)
1031		fStatus = B_OK;
1032}
1033
1034
1035void
1036IORequest::SubRequestFinished(IORequest* request, status_t status,
1037	bool partialTransfer, generic_size_t transferEndOffset)
1038{
1039	TRACE("IORequest::SubrequestFinished(%p, %#" B_PRIx32 ", %d, %"
1040		B_PRIuGENADDR "): request: %p\n", request, status, partialTransfer, transferEndOffset, this);
1041
1042	MutexLocker locker(fLock);
1043
1044	if (status != B_OK || partialTransfer) {
1045		if (fTransferSize > transferEndOffset)
1046			fTransferSize = transferEndOffset;
1047		fPartialTransfer = true;
1048	}
1049
1050	if (status != B_OK && fStatus == 1)
1051		fStatus = status;
1052
1053	if (--fPendingChildren > 0 || fSuppressChildNotifications)
1054		return;
1055
1056	// last child finished
1057
1058	// set status, if not done yet
1059	if (fStatus == 1)
1060		fStatus = B_OK;
1061
1062	locker.Unlock();
1063
1064	NotifyFinished();
1065}
1066
1067
1068void
1069IORequest::SetUnfinished()
1070{
1071	MutexLocker _(fLock);
1072	ResetStatus();
1073}
1074
1075
1076void
1077IORequest::SetTransferredBytes(bool partialTransfer,
1078	generic_size_t transferredBytes)
1079{
1080	TRACE("%p->IORequest::SetTransferredBytes(%d, %" B_PRIuGENADDR ")\n", this,
1081		partialTransfer, transferredBytes);
1082
1083	MutexLocker _(fLock);
1084
1085	fPartialTransfer = partialTransfer;
1086	fTransferSize = transferredBytes;
1087}
1088
1089
1090void
1091IORequest::SetSuppressChildNotifications(bool suppress)
1092{
1093	fSuppressChildNotifications = suppress;
1094}
1095
1096
1097void
1098IORequest::Advance(generic_size_t bySize)
1099{
1100	TRACE("IORequest::Advance(%" B_PRIuGENADDR "): remaining: %" B_PRIuGENADDR
1101		" -> %" B_PRIuGENADDR "\n", bySize, fRemainingBytes,
1102		fRemainingBytes - bySize);
1103	fRemainingBytes -= bySize;
1104	fTransferSize += bySize;
1105
1106	generic_io_vec* vecs = fBuffer->Vecs();
1107	uint32 vecCount = fBuffer->VecCount();
1108	while (fVecIndex < vecCount
1109			&& vecs[fVecIndex].length - fVecOffset <= bySize) {
1110		bySize -= vecs[fVecIndex].length - fVecOffset;
1111		fVecOffset = 0;
1112		fVecIndex++;
1113	}
1114
1115	fVecOffset += bySize;
1116}
1117
1118
1119IORequest*
1120IORequest::FirstSubRequest()
1121{
1122	return dynamic_cast<IORequest*>(fChildren.Head());
1123}
1124
1125
1126IORequest*
1127IORequest::NextSubRequest(IORequest* previous)
1128{
1129	if (previous == NULL)
1130		return NULL;
1131	return dynamic_cast<IORequest*>(fChildren.GetNext(previous));
1132}
1133
1134
1135void
1136IORequest::AddOperation(IOOperation* operation)
1137{
1138	MutexLocker locker(fLock);
1139	TRACE("IORequest::AddOperation(%p): request: %p\n", operation, this);
1140	fChildren.Add(operation);
1141	fPendingChildren++;
1142}
1143
1144
1145void
1146IORequest::RemoveOperation(IOOperation* operation)
1147{
1148	MutexLocker locker(fLock);
1149	fChildren.Remove(operation);
1150	operation->SetParent(NULL);
1151}
1152
1153
1154status_t
1155IORequest::CopyData(off_t offset, void* buffer, size_t size)
1156{
1157	return _CopyData(buffer, offset, size, true);
1158}
1159
1160
1161status_t
1162IORequest::CopyData(const void* buffer, off_t offset, size_t size)
1163{
1164	return _CopyData((void*)buffer, offset, size, false);
1165}
1166
1167
1168status_t
1169IORequest::ClearData(off_t offset, generic_size_t size)
1170{
1171	if (size == 0)
1172		return B_OK;
1173
1174	if (offset < fOffset || offset + (off_t)size > fOffset + (off_t)fLength) {
1175		panic("IORequest::ClearData(): invalid range: (%" B_PRIdOFF
1176			", %" B_PRIuGENADDR ")", offset, size);
1177		return B_BAD_VALUE;
1178	}
1179
1180	// If we can, we directly copy from/to the virtual buffer. The memory is
1181	// locked in this case.
1182	status_t (*clearFunction)(generic_addr_t, generic_size_t, team_id);
1183	if (fBuffer->IsPhysical()) {
1184		clearFunction = &IORequest::_ClearDataPhysical;
1185	} else {
1186		clearFunction = fBuffer->IsUser() && fTeam != team_get_current_team_id()
1187			? &IORequest::_ClearDataUser : &IORequest::_ClearDataSimple;
1188	}
1189
1190	// skip bytes if requested
1191	generic_io_vec* vecs = fBuffer->Vecs();
1192	generic_size_t skipBytes = offset - fOffset;
1193	generic_size_t vecOffset = 0;
1194	while (skipBytes > 0) {
1195		if (vecs[0].length > skipBytes) {
1196			vecOffset = skipBytes;
1197			break;
1198		}
1199
1200		skipBytes -= vecs[0].length;
1201		vecs++;
1202	}
1203
1204	// clear vector-wise
1205	while (size > 0) {
1206		generic_size_t toClear = min_c(size, vecs[0].length - vecOffset);
1207		status_t error = clearFunction(vecs[0].base + vecOffset, toClear,
1208			fTeam);
1209		if (error != B_OK)
1210			return error;
1211
1212		size -= toClear;
1213		vecs++;
1214		vecOffset = 0;
1215	}
1216
1217	return B_OK;
1218
1219}
1220
1221
1222status_t
1223IORequest::_CopyData(void* _buffer, off_t offset, size_t size, bool copyIn)
1224{
1225	if (size == 0)
1226		return B_OK;
1227
1228	uint8* buffer = (uint8*)_buffer;
1229
1230	if (offset < fOffset || offset + (off_t)size > fOffset + (off_t)fLength) {
1231		panic("IORequest::_CopyData(): invalid range: (%" B_PRIdOFF ", %lu)",
1232			offset, size);
1233		return B_BAD_VALUE;
1234	}
1235
1236	// If we can, we directly copy from/to the virtual buffer. The memory is
1237	// locked in this case.
1238	status_t (*copyFunction)(void*, generic_addr_t, size_t, team_id, bool);
1239	if (fBuffer->IsPhysical()) {
1240		copyFunction = &IORequest::_CopyPhysical;
1241	} else {
1242		copyFunction = fBuffer->IsUser() && fTeam != team_get_current_team_id()
1243			? &IORequest::_CopyUser : &IORequest::_CopySimple;
1244	}
1245
1246	// skip bytes if requested
1247	generic_io_vec* vecs = fBuffer->Vecs();
1248	generic_size_t skipBytes = offset - fOffset;
1249	generic_size_t vecOffset = 0;
1250	while (skipBytes > 0) {
1251		if (vecs[0].length > skipBytes) {
1252			vecOffset = skipBytes;
1253			break;
1254		}
1255
1256		skipBytes -= vecs[0].length;
1257		vecs++;
1258	}
1259
1260	// copy vector-wise
1261	while (size > 0) {
1262		generic_size_t toCopy = min_c(size, vecs[0].length - vecOffset);
1263		status_t error = copyFunction(buffer, vecs[0].base + vecOffset, toCopy,
1264			fTeam, copyIn);
1265		if (error != B_OK)
1266			return error;
1267
1268		buffer += toCopy;
1269		size -= toCopy;
1270		vecs++;
1271		vecOffset = 0;
1272	}
1273
1274	return B_OK;
1275}
1276
1277
1278/* static */ status_t
1279IORequest::_CopySimple(void* bounceBuffer, generic_addr_t external, size_t size,
1280	team_id team, bool copyIn)
1281{
1282	TRACE("  IORequest::_CopySimple(%p, %#" B_PRIxGENADDR ", %lu, %d)\n",
1283		bounceBuffer, external, size, copyIn);
1284	if (IS_USER_ADDRESS(external)) {
1285		status_t status = B_OK;
1286		if (copyIn)
1287			status = user_memcpy(bounceBuffer, (void*)(addr_t)external, size);
1288		else
1289			status = user_memcpy((void*)(addr_t)external, bounceBuffer, size);
1290		if (status < B_OK)
1291			return status;
1292		return B_OK;
1293	}
1294	if (copyIn)
1295		memcpy(bounceBuffer, (void*)(addr_t)external, size);
1296	else
1297		memcpy((void*)(addr_t)external, bounceBuffer, size);
1298	return B_OK;
1299}
1300
1301
1302/* static */ status_t
1303IORequest::_CopyPhysical(void* bounceBuffer, generic_addr_t external,
1304	size_t size, team_id team, bool copyIn)
1305{
1306	if (copyIn)
1307		return vm_memcpy_from_physical(bounceBuffer, external, size, false);
1308
1309	return vm_memcpy_to_physical(external, bounceBuffer, size, false);
1310}
1311
1312
1313/* static */ status_t
1314IORequest::_CopyUser(void* _bounceBuffer, generic_addr_t _external, size_t size,
1315	team_id team, bool copyIn)
1316{
1317	uint8* bounceBuffer = (uint8*)_bounceBuffer;
1318	uint8* external = (uint8*)(addr_t)_external;
1319
1320	while (size > 0) {
1321		static const int32 kEntryCount = 8;
1322		physical_entry entries[kEntryCount];
1323
1324		uint32 count = kEntryCount;
1325		status_t error = get_memory_map_etc(team, external, size, entries,
1326			&count);
1327		if (error != B_OK && error != B_BUFFER_OVERFLOW) {
1328			panic("IORequest::_CopyUser(): Failed to get physical memory for "
1329				"user memory %p\n", external);
1330			return B_BAD_ADDRESS;
1331		}
1332
1333		for (uint32 i = 0; i < count; i++) {
1334			const physical_entry& entry = entries[i];
1335			error = _CopyPhysical(bounceBuffer, entry.address, entry.size, team,
1336				copyIn);
1337			if (error != B_OK)
1338				return error;
1339
1340			size -= entry.size;
1341			bounceBuffer += entry.size;
1342			external += entry.size;
1343		}
1344	}
1345
1346	return B_OK;
1347}
1348
1349
1350/*static*/ status_t
1351IORequest::_ClearDataSimple(generic_addr_t external, generic_size_t size,
1352	team_id team)
1353{
1354	memset((void*)(addr_t)external, 0, (size_t)size);
1355	return B_OK;
1356}
1357
1358
1359/*static*/ status_t
1360IORequest::_ClearDataPhysical(generic_addr_t external, generic_size_t size,
1361	team_id team)
1362{
1363	return vm_memset_physical((phys_addr_t)external, 0, (phys_size_t)size);
1364}
1365
1366
1367/*static*/ status_t
1368IORequest::_ClearDataUser(generic_addr_t _external, generic_size_t size,
1369	team_id team)
1370{
1371	uint8* external = (uint8*)(addr_t)_external;
1372
1373	while (size > 0) {
1374		static const int32 kEntryCount = 8;
1375		physical_entry entries[kEntryCount];
1376
1377		uint32 count = kEntryCount;
1378		status_t error = get_memory_map_etc(team, external, size, entries,
1379			&count);
1380		if (error != B_OK && error != B_BUFFER_OVERFLOW) {
1381			panic("IORequest::_ClearDataUser(): Failed to get physical memory "
1382				"for user memory %p\n", external);
1383			return B_BAD_ADDRESS;
1384		}
1385
1386		for (uint32 i = 0; i < count; i++) {
1387			const physical_entry& entry = entries[i];
1388			error = _ClearDataPhysical(entry.address, entry.size, team);
1389			if (error != B_OK)
1390				return error;
1391
1392			size -= entry.size;
1393			external += entry.size;
1394		}
1395	}
1396
1397	return B_OK;
1398}
1399
1400
1401void
1402IORequest::Dump() const
1403{
1404	kprintf("io_request at %p\n", this);
1405
1406	kprintf("  owner:             %p\n", fOwner);
1407	kprintf("  parent:            %p\n", fParent);
1408	kprintf("  status:            %s\n", strerror(fStatus));
1409	kprintf("  mutex:             %p\n", &fLock);
1410	kprintf("  IOBuffer:          %p\n", fBuffer);
1411	kprintf("  offset:            %" B_PRIdOFF "\n", fOffset);
1412	kprintf("  length:            %" B_PRIuGENADDR "\n", fLength);
1413	kprintf("  transfer size:     %" B_PRIuGENADDR "\n", fTransferSize);
1414	kprintf("  relative offset:   %" B_PRIuGENADDR "\n", fRelativeParentOffset);
1415	kprintf("  pending children:  %" B_PRId32 "\n", fPendingChildren);
1416	kprintf("  flags:             %#" B_PRIx32 "\n", fFlags);
1417	kprintf("  team:              %" B_PRId32 "\n", fTeam);
1418	kprintf("  thread:            %" B_PRId32 "\n", fThread);
1419	kprintf("  r/w:               %s\n", fIsWrite ? "write" : "read");
1420	kprintf("  partial transfer:  %s\n", fPartialTransfer ? "yes" : "no");
1421	kprintf("  finished cvar:     %p\n", &fFinishedCondition);
1422	kprintf("  iteration:\n");
1423	kprintf("    vec index:       %" B_PRIu32 "\n", fVecIndex);
1424	kprintf("    vec offset:      %" B_PRIuGENADDR "\n", fVecOffset);
1425	kprintf("    remaining bytes: %" B_PRIuGENADDR "\n", fRemainingBytes);
1426	kprintf("  callbacks:\n");
1427	kprintf("    finished %p, cookie %p\n", fFinishedCallback, fFinishedCookie);
1428	kprintf("    iteration %p, cookie %p\n", fIterationCallback,
1429		fIterationCookie);
1430	kprintf("  children:\n");
1431
1432	IORequestChunkList::ConstIterator iterator = fChildren.GetIterator();
1433	while (iterator.HasNext()) {
1434		kprintf("    %p\n", iterator.Next());
1435	}
1436
1437	set_debug_variable("_parent", (addr_t)fParent);
1438	set_debug_variable("_mutex", (addr_t)&fLock);
1439	set_debug_variable("_buffer", (addr_t)fBuffer);
1440	set_debug_variable("_cvar", (addr_t)&fFinishedCondition);
1441}
1442