1/*
2 * Copyright 2006-2011, Haiku Inc. All rights reserved.
3 * Distributed under the terms of the MIT License.
4 *
5 * Authors:
6 *		Michael Lotz <mmlr@mlotz.ch>
7 *		J��r��me Duval <korli@users.berlios.de>
8 */
9
10
11#include <driver_settings.h>
12#include <module.h>
13#include <PCI.h>
14#include <PCI_x86.h>
15#include <USB3.h>
16#include <KernelExport.h>
17
18#include "ehci.h"
19
20#define USB_MODULE_NAME	"ehci"
21
22pci_module_info *EHCI::sPCIModule = NULL;
23pci_x86_module_info *EHCI::sPCIx86Module = NULL;
24
25
26static int32
27ehci_std_ops(int32 op, ...)
28{
29	switch (op) {
30		case B_MODULE_INIT:
31			TRACE_MODULE("ehci init module\n");
32			return B_OK;
33		case B_MODULE_UNINIT:
34			TRACE_MODULE("ehci uninit module\n");
35			return B_OK;
36	}
37
38	return EINVAL;
39}
40
41
42usb_host_controller_info ehci_module = {
43	{
44		"busses/usb/ehci",
45		0,
46		ehci_std_ops
47	},
48	NULL,
49	EHCI::AddTo
50};
51
52
53module_info *modules[] = {
54	(module_info *)&ehci_module,
55	NULL
56};
57
58
59//
60// #pragma mark -
61//
62
63
64#ifdef TRACE_USB
65
66void
67print_descriptor_chain(ehci_qtd *descriptor)
68{
69	while (descriptor) {
70		dprintf(" %08" B_PRIx32 " n%08" B_PRIx32 " a%08" B_PRIx32 " t%08"
71			B_PRIx32 " %08" B_PRIx32 " %08" B_PRIx32 " %08" B_PRIx32 " %08"
72			B_PRIx32 " %08" B_PRIx32 " s%" B_PRIuSIZE "\n",
73			descriptor->this_phy, descriptor->next_phy,
74			descriptor->alt_next_phy, descriptor->token,
75			descriptor->buffer_phy[0], descriptor->buffer_phy[1],
76			descriptor->buffer_phy[2], descriptor->buffer_phy[3],
77			descriptor->buffer_phy[4], descriptor->buffer_size);
78
79		if (descriptor->next_phy & EHCI_ITEM_TERMINATE)
80			break;
81
82		descriptor = descriptor->next_log;
83	}
84}
85
86
87void
88print_queue(ehci_qh *queueHead)
89{
90	dprintf("queue:    t%08" B_PRIx32 " n%08" B_PRIx32 " ch%08" B_PRIx32
91		" ca%08" B_PRIx32 " cu%08" B_PRIx32 "\n",
92		queueHead->this_phy, queueHead->next_phy, queueHead->endpoint_chars,
93		queueHead->endpoint_caps, queueHead->current_qtd_phy);
94	dprintf("overlay:  n%08" B_PRIx32 " a%08" B_PRIx32 " t%08" B_PRIx32
95		" %08" B_PRIx32 " %08" B_PRIx32 " %08" B_PRIx32 " %08" B_PRIx32
96		" %08" B_PRIx32 "\n", queueHead->overlay.next_phy,
97		queueHead->overlay.alt_next_phy, queueHead->overlay.token,
98		queueHead->overlay.buffer_phy[0], queueHead->overlay.buffer_phy[1],
99		queueHead->overlay.buffer_phy[2], queueHead->overlay.buffer_phy[3],
100		queueHead->overlay.buffer_phy[4]);
101	print_descriptor_chain(queueHead->element_log);
102}
103
104
105#endif // TRACE_USB
106
107
108//
109// #pragma mark -
110//
111
112
113status_t
114EHCI::AddTo(Stack *stack)
115{
116	if (!sPCIModule) {
117		status_t status = get_module(B_PCI_MODULE_NAME,
118			(module_info **)&sPCIModule);
119		if (status != B_OK) {
120			TRACE_MODULE_ERROR("getting pci module failed! 0x%08" B_PRIx32
121				"\n", status);
122			return status;
123		}
124	}
125
126	TRACE_MODULE("searching devices\n");
127	bool found = false;
128	pci_info *item = new(std::nothrow) pci_info;
129	if (!item) {
130		sPCIModule = NULL;
131		put_module(B_PCI_MODULE_NAME);
132		return B_NO_MEMORY;
133	}
134
135	// Try to get the PCI x86 module as well so we can enable possible MSIs.
136	if (sPCIx86Module == NULL && get_module(B_PCI_X86_MODULE_NAME,
137			(module_info **)&sPCIx86Module) != B_OK) {
138		// If it isn't there, that's not critical though.
139		TRACE_MODULE_ERROR("failed to get pci x86 module\n");
140		sPCIx86Module = NULL;
141	}
142
143	for (int32 i = 0; sPCIModule->get_nth_pci_info(i, item) >= B_OK; i++) {
144		if (item->class_base == PCI_serial_bus && item->class_sub == PCI_usb
145			&& item->class_api == PCI_usb_ehci) {
146			TRACE_MODULE("found device at PCI:%d:%d:%d\n",
147				item->bus, item->device, item->function);
148			EHCI *bus = new(std::nothrow) EHCI(item, stack);
149			if (bus == NULL) {
150				delete item;
151				put_module(B_PCI_MODULE_NAME);
152				if (sPCIx86Module != NULL)
153					put_module(B_PCI_X86_MODULE_NAME);
154				return B_NO_MEMORY;
155			}
156
157			// The bus will put the PCI modules when it is destroyed, so get
158			// them again to increase their reference count.
159			get_module(B_PCI_MODULE_NAME, (module_info **)&sPCIModule);
160			if (sPCIx86Module != NULL)
161				get_module(B_PCI_X86_MODULE_NAME, (module_info **)&sPCIx86Module);
162
163			if (bus->InitCheck() != B_OK) {
164				TRACE_MODULE_ERROR("bus failed init check\n");
165				delete bus;
166				continue;
167			}
168
169			// the bus took it away
170			item = new(std::nothrow) pci_info;
171
172			if (bus->Start() != B_OK) {
173				delete bus;
174				continue;
175			}
176			found = true;
177		}
178	}
179
180	// The modules will have been gotten again if we successfully
181	// initialized a bus, so we should put them here.
182	put_module(B_PCI_MODULE_NAME);
183	if (sPCIx86Module != NULL)
184		put_module(B_PCI_X86_MODULE_NAME);
185
186	if (!found)
187		TRACE_MODULE_ERROR("no devices found\n");
188	delete item;
189	return found ? B_OK : ENODEV;
190}
191
192
193EHCI::EHCI(pci_info *info, Stack *stack)
194	:	BusManager(stack),
195		fCapabilityRegisters(NULL),
196		fOperationalRegisters(NULL),
197		fRegisterArea(-1),
198		fPCIInfo(info),
199		fStack(stack),
200		fEnabledInterrupts(0),
201		fThreshold(0),
202		fPeriodicFrameListArea(-1),
203		fPeriodicFrameList(NULL),
204		fInterruptEntries(NULL),
205		fItdEntries(NULL),
206		fSitdEntries(NULL),
207		fAsyncQueueHead(NULL),
208		fAsyncAdvanceSem(-1),
209		fFirstTransfer(NULL),
210		fLastTransfer(NULL),
211		fFinishTransfersSem(-1),
212		fFinishThread(-1),
213		fProcessingPipe(NULL),
214		fFreeListHead(NULL),
215		fCleanupSem(-1),
216		fCleanupThread(-1),
217		fStopThreads(false),
218		fNextStartingFrame(-1),
219		fFrameBandwidth(NULL),
220		fFirstIsochronousTransfer(NULL),
221		fLastIsochronousTransfer(NULL),
222		fFinishIsochronousTransfersSem(-1),
223		fFinishIsochronousThread(-1),
224		fRootHub(NULL),
225		fRootHubAddress(0),
226		fPortCount(0),
227		fPortResetChange(0),
228		fPortSuspendChange(0),
229		fInterruptPollThread(-1),
230		fIRQ(0),
231		fUseMSI(false)
232{
233	// Create a lock for the isochronous transfer list
234	mutex_init(&fIsochronousLock, "EHCI isochronous lock");
235
236	if (BusManager::InitCheck() != B_OK) {
237		TRACE_ERROR("bus manager failed to init\n");
238		return;
239	}
240
241	TRACE("constructing new EHCI host controller driver\n");
242	fInitOK = false;
243
244	// ATI/AMD SB600/SB700 periodic list cache workaround
245	// Logic kindly borrowed from NetBSD PR 40056
246	if (fPCIInfo->vendor_id == AMD_SBX00_VENDOR) {
247		bool applyWorkaround = false;
248
249		if (fPCIInfo->device_id == AMD_SB600_EHCI_CONTROLLER) {
250			// always apply on SB600
251			applyWorkaround = true;
252		} else if (fPCIInfo->device_id == AMD_SB700_SB800_EHCI_CONTROLLER) {
253			// only apply on certain chipsets, determined by SMBus revision
254			pci_info smbus;
255			int32 index = 0;
256			while (sPCIModule->get_nth_pci_info(index++, &smbus) == B_OK) {
257				if (smbus.vendor_id == AMD_SBX00_VENDOR
258					&& smbus.device_id == AMD_SBX00_SMBUS_CONTROLLER) {
259
260					// Only applies to chipsets < SB710 (rev A14)
261					if (smbus.revision == 0x3a || smbus.revision == 0x3b)
262						applyWorkaround = true;
263
264					break;
265				}
266			}
267		}
268
269		if (applyWorkaround) {
270			// According to AMD errata of SB700 and SB600 register documentation
271			// this disables the Periodic List Cache on SB600 and the Advanced
272			// Periodic List Cache on early SB700. Both the BSDs and Linux use
273			// this workaround.
274
275			TRACE_ALWAYS("disabling SB600/SB700 periodic list cache\n");
276			uint32 workaround = sPCIModule->read_pci_config(fPCIInfo->bus,
277				fPCIInfo->device, fPCIInfo->function,
278				AMD_SBX00_EHCI_MISC_REGISTER, 4);
279
280			sPCIModule->write_pci_config(fPCIInfo->bus, fPCIInfo->device,
281				fPCIInfo->function, AMD_SBX00_EHCI_MISC_REGISTER, 4,
282				workaround | AMD_SBX00_EHCI_MISC_DISABLE_PERIODIC_LIST_CACHE);
283		}
284	}
285
286	// enable busmaster and memory mapped access
287	uint16 command = sPCIModule->read_pci_config(fPCIInfo->bus,
288		fPCIInfo->device, fPCIInfo->function, PCI_command, 2);
289	command &= ~PCI_command_io;
290	command |= PCI_command_master | PCI_command_memory;
291
292	sPCIModule->write_pci_config(fPCIInfo->bus, fPCIInfo->device,
293		fPCIInfo->function, PCI_command, 2, command);
294
295	// map the registers
296	uint32 offset = fPCIInfo->u.h0.base_registers[0] & (B_PAGE_SIZE - 1);
297	phys_addr_t physicalAddress = fPCIInfo->u.h0.base_registers[0] - offset;
298	size_t mapSize = (fPCIInfo->u.h0.base_register_sizes[0] + offset
299		+ B_PAGE_SIZE - 1) & ~(B_PAGE_SIZE - 1);
300
301	TRACE("map physical memory 0x%08" B_PRIx32 " (base: 0x%08" B_PRIxPHYSADDR
302		"; offset: %" B_PRIx32 "); size: %" B_PRIu32 "\n",
303		fPCIInfo->u.h0.base_registers[0], physicalAddress, offset,
304		fPCIInfo->u.h0.base_register_sizes[0]);
305
306	fRegisterArea = map_physical_memory("EHCI memory mapped registers",
307		physicalAddress, mapSize, B_ANY_KERNEL_BLOCK_ADDRESS,
308		B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA,
309		(void **)&fCapabilityRegisters);
310	if (fRegisterArea < 0) {
311		TRACE_ERROR("failed to map register memory\n");
312		return;
313	}
314
315	fCapabilityRegisters += offset;
316	fOperationalRegisters = fCapabilityRegisters + ReadCapReg8(EHCI_CAPLENGTH);
317	TRACE("mapped capability registers: 0x%p\n", fCapabilityRegisters);
318	TRACE("mapped operational registers: 0x%p\n", fOperationalRegisters);
319
320	TRACE("structural parameters: 0x%08" B_PRIx32 "\n",
321		ReadCapReg32(EHCI_HCSPARAMS));
322	TRACE("capability parameters: 0x%08" B_PRIx32 "\n",
323		ReadCapReg32(EHCI_HCCPARAMS));
324
325	if (EHCI_HCCPARAMS_FRAME_CACHE(ReadCapReg32(EHCI_HCCPARAMS)))
326		fThreshold = 2 + 8;
327	else
328		fThreshold = 2 + EHCI_HCCPARAMS_IPT(ReadCapReg32(EHCI_HCCPARAMS));
329
330	// read port count from capability register
331	fPortCount = ReadCapReg32(EHCI_HCSPARAMS) & 0x0f;
332
333	uint32 extendedCapPointer = ReadCapReg32(EHCI_HCCPARAMS) >> EHCI_ECP_SHIFT;
334	extendedCapPointer &= EHCI_ECP_MASK;
335	if (extendedCapPointer > 0) {
336		TRACE("extended capabilities register at %" B_PRIu32 "\n",
337			extendedCapPointer);
338
339		uint32 legacySupport = sPCIModule->read_pci_config(fPCIInfo->bus,
340			fPCIInfo->device, fPCIInfo->function, extendedCapPointer, 4);
341		if ((legacySupport & EHCI_LEGSUP_CAPID_MASK) == EHCI_LEGSUP_CAPID) {
342			if ((legacySupport & EHCI_LEGSUP_BIOSOWNED) != 0) {
343				TRACE_ALWAYS("the host controller is bios owned, claiming"
344					" ownership\n");
345
346				sPCIModule->write_pci_config(fPCIInfo->bus, fPCIInfo->device,
347					fPCIInfo->function, extendedCapPointer + 3, 1, 1);
348
349				for (int32 i = 0; i < 20; i++) {
350					legacySupport = sPCIModule->read_pci_config(fPCIInfo->bus,
351						fPCIInfo->device, fPCIInfo->function,
352						extendedCapPointer, 4);
353
354					if ((legacySupport & EHCI_LEGSUP_BIOSOWNED) == 0)
355						break;
356
357					TRACE_ALWAYS("controller is still bios owned, waiting\n");
358					snooze(50000);
359				}
360			}
361
362			if (legacySupport & EHCI_LEGSUP_BIOSOWNED) {
363				TRACE_ERROR("bios won't give up control over the host "
364					"controller (ignoring)\n");
365			} else if (legacySupport & EHCI_LEGSUP_OSOWNED) {
366				TRACE_ALWAYS(
367					"successfully took ownership of the host controller\n");
368			}
369
370			// Force off the BIOS owned flag, and clear all SMIs. Some BIOSes
371			// do indicate a successful handover but do not remove their SMIs
372			// and then freeze the system when interrupts are generated.
373			sPCIModule->write_pci_config(fPCIInfo->bus, fPCIInfo->device,
374				fPCIInfo->function, extendedCapPointer + 2, 1, 0);
375			sPCIModule->write_pci_config(fPCIInfo->bus, fPCIInfo->device,
376				fPCIInfo->function, extendedCapPointer + 4, 4, 0);
377		} else {
378			TRACE_ALWAYS(
379				"extended capability is not a legacy support register\n");
380		}
381	} else {
382		TRACE_ALWAYS("no extended capabilities register\n");
383	}
384
385	// disable interrupts
386	WriteOpReg(EHCI_USBINTR, 0);
387
388	// reset the host controller
389	if (ControllerReset() != B_OK) {
390		TRACE_ERROR("host controller failed to reset\n");
391		return;
392	}
393
394	// reset the segment register
395	WriteOpReg(EHCI_CTRDSSEGMENT, 0);
396
397	// create semaphores the finisher thread will wait for
398	fAsyncAdvanceSem = create_sem(0, "EHCI Async Advance");
399	fFinishTransfersSem = create_sem(0, "EHCI Finish Transfers");
400	fCleanupSem = create_sem(0, "EHCI Cleanup");
401	if (fFinishTransfersSem < 0 || fAsyncAdvanceSem < 0 || fCleanupSem < 0) {
402		TRACE_ERROR("failed to create semaphores\n");
403		return;
404	}
405
406	// create finisher service thread
407	fFinishThread = spawn_kernel_thread(FinishThread, "ehci finish thread",
408		B_NORMAL_PRIORITY, (void *)this);
409	resume_thread(fFinishThread);
410
411	// Create semaphore the isochronous finisher thread will wait for
412	fFinishIsochronousTransfersSem = create_sem(0,
413		"EHCI Isochronous Finish Transfers");
414	if (fFinishIsochronousTransfersSem < 0) {
415		TRACE_ERROR("failed to create isochronous finisher semaphore\n");
416		return;
417	}
418
419	// Create the isochronous finisher service thread
420	fFinishIsochronousThread = spawn_kernel_thread(FinishIsochronousThread,
421		"ehci isochronous finish thread", B_URGENT_DISPLAY_PRIORITY,
422		(void *)this);
423	resume_thread(fFinishIsochronousThread);
424
425	// create cleanup service thread
426	fCleanupThread = spawn_kernel_thread(CleanupThread, "ehci cleanup thread",
427		B_NORMAL_PRIORITY, (void *)this);
428	resume_thread(fCleanupThread);
429
430	// set up interrupts or interrupt polling now that the controller is ready
431	bool polling = false;
432	void *settings = load_driver_settings(B_SAFEMODE_DRIVER_SETTINGS);
433	if (settings != NULL) {
434		polling = get_driver_boolean_parameter(settings, "ehci_polling", false,
435			false);
436		unload_driver_settings(settings);
437	}
438
439	if (polling) {
440		// create and run the polling thread
441		TRACE_ALWAYS("enabling ehci polling\n");
442		fInterruptPollThread = spawn_kernel_thread(InterruptPollThread,
443			"ehci interrupt poll thread", B_NORMAL_PRIORITY, (void *)this);
444		resume_thread(fInterruptPollThread);
445	} else {
446		// Find the right interrupt vector, using MSIs if available.
447		fIRQ = fPCIInfo->u.h0.interrupt_line;
448		if (sPCIx86Module != NULL && sPCIx86Module->get_msi_count(
449				fPCIInfo->bus, fPCIInfo->device, fPCIInfo->function) >= 1) {
450			uint8 msiVector = 0;
451			if (sPCIx86Module->configure_msi(fPCIInfo->bus, fPCIInfo->device,
452					fPCIInfo->function, 1, &msiVector) == B_OK
453				&& sPCIx86Module->enable_msi(fPCIInfo->bus, fPCIInfo->device,
454					fPCIInfo->function) == B_OK) {
455				TRACE_ALWAYS("using message signaled interrupts\n");
456				fIRQ = msiVector;
457				fUseMSI = true;
458			}
459		}
460
461		if (fIRQ == 0 || fIRQ == 0xFF) {
462			TRACE_MODULE_ERROR("device PCI:%d:%d:%d was assigned an invalid IRQ\n",
463				fPCIInfo->bus, fPCIInfo->device, fPCIInfo->function);
464			return;
465		}
466
467		// install the interrupt handler and enable interrupts
468		install_io_interrupt_handler(fIRQ, InterruptHandler,
469			(void *)this, 0);
470	}
471
472	// ensure that interrupts are en-/disabled on the PCI device
473	command = sPCIModule->read_pci_config(fPCIInfo->bus, fPCIInfo->device,
474		fPCIInfo->function, PCI_command, 2);
475	if ((polling || fUseMSI) == ((command & PCI_command_int_disable) == 0)) {
476		if (polling || fUseMSI)
477			command &= ~PCI_command_int_disable;
478		else
479			command |= PCI_command_int_disable;
480
481		sPCIModule->write_pci_config(fPCIInfo->bus, fPCIInfo->device,
482			fPCIInfo->function, PCI_command, 2, command);
483	}
484
485	fEnabledInterrupts = EHCI_USBINTR_HOSTSYSERR | EHCI_USBINTR_USBERRINT
486		| EHCI_USBINTR_USBINT | EHCI_USBINTR_INTONAA;
487	WriteOpReg(EHCI_USBINTR, fEnabledInterrupts);
488
489	// structures don't span page boundaries
490	size_t itdListSize = EHCI_VFRAMELIST_ENTRIES_COUNT
491		/ (B_PAGE_SIZE / sizeof(itd_entry)) * B_PAGE_SIZE;
492	size_t sitdListSize = EHCI_VFRAMELIST_ENTRIES_COUNT
493		/ (B_PAGE_SIZE / sizeof(sitd_entry)) * B_PAGE_SIZE;
494	size_t frameListSize = B_PAGE_SIZE + B_PAGE_SIZE + itdListSize
495		+ sitdListSize;
496
497	// allocate the periodic frame list
498	fPeriodicFrameListArea = fStack->AllocateArea((void **)&fPeriodicFrameList,
499		&physicalAddress, frameListSize, "USB EHCI Periodic Framelist");
500	if (fPeriodicFrameListArea < 0) {
501		TRACE_ERROR("unable to allocate periodic framelist\n");
502		return;
503	}
504
505	if ((physicalAddress & 0xfff) != 0) {
506		panic("EHCI_PERIODICLISTBASE not aligned on 4k: 0x%" B_PRIxPHYSADDR
507			"\n", physicalAddress);
508	}
509
510	// set the periodic frame list base on the controller
511	WriteOpReg(EHCI_PERIODICLISTBASE, (uint32)physicalAddress);
512
513	// create the interrupt entries to support different polling intervals
514	TRACE("creating interrupt entries\n");
515	uint32_t physicalBase = physicalAddress + B_PAGE_SIZE;
516	uint8 *logicalBase = (uint8 *)fPeriodicFrameList + B_PAGE_SIZE;
517	memset(logicalBase, 0, B_PAGE_SIZE);
518
519	fInterruptEntries = (interrupt_entry *)logicalBase;
520	for (int32 i = 0; i < EHCI_INTERRUPT_ENTRIES_COUNT; i++) {
521		ehci_qh *queueHead = &fInterruptEntries[i].queue_head;
522		queueHead->this_phy = physicalBase | EHCI_ITEM_TYPE_QH;
523		queueHead->current_qtd_phy = 0;
524		queueHead->overlay.next_phy = EHCI_ITEM_TERMINATE;
525		queueHead->overlay.alt_next_phy = EHCI_ITEM_TERMINATE;
526		queueHead->overlay.token = EHCI_QTD_STATUS_HALTED;
527
528		// set dummy endpoint information
529		queueHead->endpoint_chars = EHCI_QH_CHARS_EPS_HIGH
530			| (3 << EHCI_QH_CHARS_RL_SHIFT) | (64 << EHCI_QH_CHARS_MPL_SHIFT)
531			| EHCI_QH_CHARS_TOGGLE;
532		queueHead->endpoint_caps = (1 << EHCI_QH_CAPS_MULT_SHIFT)
533			| (0xff << EHCI_QH_CAPS_ISM_SHIFT);
534
535		physicalBase += sizeof(interrupt_entry);
536		if ((physicalBase & 0x1f) != 0) {
537			panic("physical base for interrupt entry %" B_PRId32
538				" not aligned on 32, interrupt entry structure size %lu\n",
539					i, sizeof(interrupt_entry));
540		}
541	}
542
543	// create the itd and sitd entries
544	TRACE("build up iso entries\n");
545	uint32_t itdPhysicalBase = physicalAddress + B_PAGE_SIZE + B_PAGE_SIZE;
546	itd_entry* itds = (itd_entry *)((uint8 *)fPeriodicFrameList + B_PAGE_SIZE
547		+ B_PAGE_SIZE);
548	memset(itds, 0, itdListSize);
549
550	uint32_t sitdPhysicalBase = itdPhysicalBase + itdListSize;
551	sitd_entry* sitds = (sitd_entry *)((uint8 *)fPeriodicFrameList + B_PAGE_SIZE
552		+ B_PAGE_SIZE + itdListSize);
553	memset(sitds, 0, sitdListSize);
554
555	fItdEntries = new(std::nothrow) ehci_itd *[EHCI_VFRAMELIST_ENTRIES_COUNT];
556	fSitdEntries = new(std::nothrow) ehci_sitd *[EHCI_VFRAMELIST_ENTRIES_COUNT];
557
558	dprintf("sitd entry size %lu, itd entry size %lu\n", sizeof(sitd_entry),
559		sizeof(itd_entry));
560	for (int32 i = 0; i < EHCI_VFRAMELIST_ENTRIES_COUNT; i++) {
561		ehci_sitd *sitd = &sitds[i].sitd;
562		sitd->this_phy = sitdPhysicalBase | EHCI_ITEM_TYPE_SITD;
563		sitd->back_phy = EHCI_ITEM_TERMINATE;
564		fSitdEntries[i] = sitd;
565		TRACE("sitd entry %" B_PRId32 " %p 0x%" B_PRIx32 "\n", i, sitd,
566			sitd->this_phy);
567
568		ehci_itd *itd = &itds[i].itd;
569		itd->this_phy = itdPhysicalBase | EHCI_ITEM_TYPE_ITD;
570		itd->next_phy = sitd->this_phy;
571		fItdEntries[i] = itd;
572		TRACE("itd entry %" B_PRId32 " %p 0x%" B_PRIx32 "\n", i, itd,
573			itd->this_phy);
574
575		sitdPhysicalBase += sizeof(sitd_entry);
576		itdPhysicalBase += sizeof(itd_entry);
577		if ((sitdPhysicalBase & 0x10) != 0 || (itdPhysicalBase & 0x10) != 0)
578			panic("physical base for entry %" B_PRId32 " not aligned on 32\n",
579				i);
580	}
581
582	// build flat interrupt tree
583	TRACE("build up interrupt links\n");
584	uint32 interval = EHCI_VFRAMELIST_ENTRIES_COUNT;
585	uint32 intervalIndex = EHCI_INTERRUPT_ENTRIES_COUNT - 1;
586	while (interval > 1) {
587		for (uint32 insertIndex = interval / 2;
588			insertIndex < EHCI_VFRAMELIST_ENTRIES_COUNT;
589			insertIndex += interval) {
590			fSitdEntries[insertIndex]->next_phy
591				= fInterruptEntries[intervalIndex].queue_head.this_phy;
592		}
593
594		intervalIndex--;
595		interval /= 2;
596	}
597
598	// setup the empty slot in the list and linking of all -> first
599	ehci_qh *firstLogical = &fInterruptEntries[0].queue_head;
600	fSitdEntries[0]->next_phy = firstLogical->this_phy;
601	for (int32 i = 1; i < EHCI_INTERRUPT_ENTRIES_COUNT; i++) {
602		fInterruptEntries[i].queue_head.next_phy = firstLogical->this_phy;
603		fInterruptEntries[i].queue_head.next_log = firstLogical;
604		fInterruptEntries[i].queue_head.prev_log = NULL;
605	}
606
607	// terminate the first entry
608	firstLogical->next_phy = EHCI_ITEM_TERMINATE;
609	firstLogical->next_log = NULL;
610	firstLogical->prev_log = NULL;
611
612	for (int32 i = 0; i < EHCI_FRAMELIST_ENTRIES_COUNT; i++) {
613		fPeriodicFrameList[i]
614			= fItdEntries[i & (EHCI_VFRAMELIST_ENTRIES_COUNT - 1)]->this_phy;
615		TRACE("periodic entry %" B_PRId32 " linked to 0x%" B_PRIx32 "\n", i,
616			fPeriodicFrameList[i]);
617	}
618
619	// Create the array that will keep bandwidth information
620	fFrameBandwidth = new(std::nothrow) uint16[EHCI_VFRAMELIST_ENTRIES_COUNT];
621	for (int32 i = 0; i < EHCI_VFRAMELIST_ENTRIES_COUNT; i++) {
622		fFrameBandwidth[i] = MAX_AVAILABLE_BANDWIDTH;
623	}
624
625	// allocate a queue head that will always stay in the async frame list
626	fAsyncQueueHead = CreateQueueHead();
627	if (!fAsyncQueueHead) {
628		TRACE_ERROR("unable to allocate stray async queue head\n");
629		return;
630	}
631
632	fAsyncQueueHead->next_phy = fAsyncQueueHead->this_phy;
633	fAsyncQueueHead->next_log = fAsyncQueueHead;
634	fAsyncQueueHead->prev_log = fAsyncQueueHead;
635	fAsyncQueueHead->endpoint_chars = EHCI_QH_CHARS_EPS_HIGH
636		| EHCI_QH_CHARS_RECHEAD;
637	fAsyncQueueHead->endpoint_caps = 1 << EHCI_QH_CAPS_MULT_SHIFT;
638	fAsyncQueueHead->overlay.next_phy = EHCI_ITEM_TERMINATE;
639
640	WriteOpReg(EHCI_ASYNCLISTADDR, (uint32)fAsyncQueueHead->this_phy);
641	TRACE("set the async list addr to 0x%08" B_PRIx32 "\n",
642		ReadOpReg(EHCI_ASYNCLISTADDR));
643
644	fInitOK = true;
645	TRACE("EHCI host controller driver constructed\n");
646}
647
648
649EHCI::~EHCI()
650{
651	TRACE("tear down EHCI host controller driver\n");
652
653	WriteOpReg(EHCI_USBCMD, 0);
654	WriteOpReg(EHCI_CONFIGFLAG, 0);
655	CancelAllPendingTransfers();
656
657	int32 result = 0;
658	fStopThreads = true;
659	delete_sem(fAsyncAdvanceSem);
660	delete_sem(fFinishTransfersSem);
661	delete_sem(fFinishIsochronousTransfersSem);
662	delete_sem(fCleanupSem);
663	wait_for_thread(fFinishThread, &result);
664	wait_for_thread(fCleanupThread, &result);
665	wait_for_thread(fFinishIsochronousThread, &result);
666
667	if (fInterruptPollThread >= 0)
668		wait_for_thread(fInterruptPollThread, &result);
669	else
670		remove_io_interrupt_handler(fIRQ, InterruptHandler, (void *)this);
671
672	LockIsochronous();
673	isochronous_transfer_data *isoTransfer = fFirstIsochronousTransfer;
674	while (isoTransfer) {
675		isochronous_transfer_data *next = isoTransfer->link;
676		delete isoTransfer;
677		isoTransfer = next;
678	}
679	mutex_destroy(&fIsochronousLock);
680
681	delete fRootHub;
682	delete [] fFrameBandwidth;
683	delete [] fItdEntries;
684	delete [] fSitdEntries;
685	delete_area(fPeriodicFrameListArea);
686	delete_area(fRegisterArea);
687
688	if (fUseMSI && sPCIx86Module != NULL) {
689		sPCIx86Module->disable_msi(fPCIInfo->bus,
690			fPCIInfo->device, fPCIInfo->function);
691		sPCIx86Module->unconfigure_msi(fPCIInfo->bus,
692			fPCIInfo->device, fPCIInfo->function);
693	}
694
695	put_module(B_PCI_MODULE_NAME);
696	if (sPCIx86Module != NULL)
697		put_module(B_PCI_X86_MODULE_NAME);
698}
699
700
701status_t
702EHCI::Start()
703{
704	TRACE("starting EHCI host controller\n");
705	TRACE("usbcmd: 0x%08" B_PRIx32 "; usbsts: 0x%08" B_PRIx32 "\n",
706		ReadOpReg(EHCI_USBCMD), ReadOpReg(EHCI_USBSTS));
707
708	bool hasPerPortChangeEvent = (ReadCapReg32(EHCI_HCCPARAMS)
709		& EHCI_HCCPARAMS_PPCEC) != 0;
710
711	uint32 config = ReadOpReg(EHCI_USBCMD);
712	config &= ~((EHCI_USBCMD_ITC_MASK << EHCI_USBCMD_ITC_SHIFT)
713		| EHCI_USBCMD_PPCEE);
714	uint32 frameListSize = (config >> EHCI_USBCMD_FLS_SHIFT)
715		& EHCI_USBCMD_FLS_MASK;
716
717	WriteOpReg(EHCI_USBCMD, config | EHCI_USBCMD_RUNSTOP
718		| (hasPerPortChangeEvent ? EHCI_USBCMD_PPCEE : 0)
719		| EHCI_USBCMD_ASENABLE | EHCI_USBCMD_PSENABLE
720		| (frameListSize << EHCI_USBCMD_FLS_SHIFT)
721		| (1 << EHCI_USBCMD_ITC_SHIFT));
722
723	switch (frameListSize) {
724		case 0:
725			TRACE("frame list size 1024\n");
726			break;
727		case 1:
728			TRACE("frame list size 512\n");
729			break;
730		case 2:
731			TRACE("frame list size 256\n");
732			break;
733		default:
734			TRACE_ALWAYS("unknown frame list size\n");
735	}
736
737	bool running = false;
738	for (int32 i = 0; i < 10; i++) {
739		uint32 status = ReadOpReg(EHCI_USBSTS);
740		TRACE("try %" B_PRId32 ": status 0x%08" B_PRIx32 "\n", i, status);
741
742		if (status & EHCI_USBSTS_HCHALTED) {
743			snooze(10000);
744		} else {
745			running = true;
746			break;
747		}
748	}
749
750	if (!running) {
751		TRACE_ERROR("host controller didn't start\n");
752		return B_ERROR;
753	}
754
755	// route all ports to us
756	WriteOpReg(EHCI_CONFIGFLAG, EHCI_CONFIGFLAG_FLAG);
757	snooze(10000);
758
759	fRootHubAddress = AllocateAddress();
760	fRootHub = new(std::nothrow) EHCIRootHub(RootObject(), fRootHubAddress);
761	if (!fRootHub) {
762		TRACE_ERROR("no memory to allocate root hub\n");
763		return B_NO_MEMORY;
764	}
765
766	if (fRootHub->InitCheck() != B_OK) {
767		TRACE_ERROR("root hub failed init check\n");
768		return fRootHub->InitCheck();
769	}
770
771	SetRootHub(fRootHub);
772
773	TRACE_ALWAYS("successfully started the controller\n");
774	return BusManager::Start();
775}
776
777
778status_t
779EHCI::StartDebugTransfer(Transfer *transfer)
780{
781	static transfer_data transferData;
782
783	transferData.queue_head = CreateQueueHead();
784	if (transferData.queue_head == NULL)
785		return B_NO_MEMORY;
786
787	Pipe *pipe = transfer->TransferPipe();
788	status_t result = InitQueueHead(transferData.queue_head, pipe);
789	if (result != B_OK) {
790		FreeQueueHead(transferData.queue_head);
791		return result;
792	}
793
794	if ((pipe->Type() & USB_OBJECT_CONTROL_PIPE) != 0) {
795		result = FillQueueWithRequest(transfer, transferData.queue_head,
796			&transferData.data_descriptor, &transferData.incoming);
797	} else {
798		result = FillQueueWithData(transfer, transferData.queue_head,
799			&transferData.data_descriptor, &transferData.incoming);
800	}
801
802	if (result != B_OK) {
803		FreeQueueHead(transferData.queue_head);
804		return result;
805	}
806
807	if ((pipe->Type() & USB_OBJECT_INTERRUPT_PIPE) != 0)
808		LinkPeriodicDebugQueueHead(transferData.queue_head, pipe);
809	else
810		LinkAsyncDebugQueueHead(transferData.queue_head);
811
812	// we abuse the callback cookie to hold our transfer data
813	transfer->SetCallback(NULL, &transferData);
814	return B_OK;
815}
816
817
818void
819EHCI::LinkAsyncDebugQueueHead(ehci_qh *queueHead)
820{
821	ehci_qh *prevHead = fAsyncQueueHead->prev_log;
822	queueHead->next_phy = fAsyncQueueHead->this_phy;
823	queueHead->next_log = fAsyncQueueHead;
824	queueHead->prev_log = prevHead;
825	fAsyncQueueHead->prev_log = queueHead;
826	prevHead->next_log = queueHead;
827	prevHead->next_phy = queueHead->this_phy;
828}
829
830
831void
832EHCI::LinkPeriodicDebugQueueHead(ehci_qh *queueHead, Pipe *pipe)
833{
834	if (pipe->Speed() == USB_SPEED_HIGHSPEED)
835		queueHead->endpoint_caps |= (0xff << EHCI_QH_CAPS_ISM_SHIFT);
836	else {
837		queueHead->endpoint_caps |= (0x01 << EHCI_QH_CAPS_ISM_SHIFT);
838		queueHead->endpoint_caps |= (0x1c << EHCI_QH_CAPS_SCM_SHIFT);
839	}
840
841	ehci_qh *interruptQueue = &fInterruptEntries[0].queue_head;
842	queueHead->next_phy = interruptQueue->next_phy;
843	queueHead->next_log = interruptQueue->next_log;
844	queueHead->prev_log = interruptQueue;
845	if (interruptQueue->next_log)
846		interruptQueue->next_log->prev_log = queueHead;
847	interruptQueue->next_log = queueHead;
848	interruptQueue->next_phy = queueHead->this_phy;
849}
850
851
852status_t
853EHCI::CheckDebugTransfer(Transfer *transfer)
854{
855	bool transferOK = false;
856	bool transferError = false;
857	transfer_data *transferData = (transfer_data *)transfer->CallbackCookie();
858	ehci_qtd *descriptor = transferData->queue_head->element_log;
859
860	while (descriptor) {
861		uint32 status = descriptor->token;
862		if ((status & EHCI_QTD_STATUS_ACTIVE) != 0) {
863			// still in progress
864			break;
865		}
866
867		if ((status & EHCI_QTD_STATUS_ERRMASK) != 0) {
868			transferError = true;
869			break;
870		}
871
872		if ((descriptor->next_phy & EHCI_ITEM_TERMINATE) != 0) {
873			// we arrived at the last (stray) descriptor, we're done
874			transferOK = true;
875			break;
876		}
877
878		if (((status >> EHCI_QTD_PID_SHIFT) & EHCI_QTD_PID_MASK)
879				== EHCI_QTD_PID_IN
880			&& ((status >> EHCI_QTD_BYTES_SHIFT) & EHCI_QTD_BYTES_MASK) != 0) {
881			// a short packet condition existed on this descriptor
882			if (descriptor->alt_next_log != NULL) {
883				descriptor = descriptor->alt_next_log;
884				continue;
885			}
886
887			transferOK = true;
888			break;
889		}
890
891		descriptor = descriptor->next_log;
892	}
893
894	if (!transferOK && !transferError) {
895		spin(75);
896		return B_DEV_PENDING;
897	}
898
899	if (transferOK) {
900		bool nextDataToggle = false;
901		if (transferData->data_descriptor != NULL && transferData->incoming) {
902			// data to read out
903			iovec *vector = transfer->Vector();
904			size_t vectorCount = transfer->VectorCount();
905
906			ReadDescriptorChain(transferData->data_descriptor,
907				vector, vectorCount, &nextDataToggle);
908		} else if (transferData->data_descriptor != NULL)
909			ReadActualLength(transferData->data_descriptor, &nextDataToggle);
910
911		transfer->TransferPipe()->SetDataToggle(nextDataToggle);
912	}
913
914	CleanupDebugTransfer(transfer);
915	return transferOK ? B_OK : B_IO_ERROR;
916}
917
918
919void
920EHCI::CancelDebugTransfer(Transfer *transfer)
921{
922	transfer_data *transferData = (transfer_data *)transfer->CallbackCookie();
923
924	// clear the active bit so the descriptors are canceled
925	ehci_qtd *descriptor = transferData->queue_head->element_log;
926	while (descriptor != NULL) {
927		descriptor->token &= ~EHCI_QTD_STATUS_ACTIVE;
928		descriptor = descriptor->next_log;
929	}
930
931	transfer->Finished(B_CANCELED, 0);
932	CleanupDebugTransfer(transfer);
933}
934
935
936void
937EHCI::CleanupDebugTransfer(Transfer *transfer)
938{
939	transfer_data *transferData = (transfer_data *)transfer->CallbackCookie();
940	ehci_qh *queueHead = transferData->queue_head;
941	ehci_qh *prevHead = queueHead->prev_log;
942	if (prevHead != NULL) {
943		prevHead->next_phy = queueHead->next_phy;
944		prevHead->next_log = queueHead->next_log;
945	}
946
947	ehci_qh *nextHead = queueHead->next_log;
948	if (nextHead != NULL)
949		nextHead->prev_log = queueHead->prev_log;
950
951	queueHead->next_phy = fAsyncQueueHead->this_phy;
952	queueHead->prev_log = NULL;
953	queueHead->next_log = NULL;
954
955	// wait for async advance to ensure the controller does not access this
956	// queue head anymore.
957	spin(125);
958
959	FreeQueueHead(queueHead);
960}
961
962
963status_t
964EHCI::SubmitTransfer(Transfer *transfer)
965{
966	// short circuit the root hub
967	if (transfer->TransferPipe()->DeviceAddress() == fRootHubAddress)
968		return fRootHub->ProcessTransfer(this, transfer);
969
970	Pipe *pipe = transfer->TransferPipe();
971	if ((pipe->Type() & USB_OBJECT_ISO_PIPE) != 0)
972		return SubmitIsochronous(transfer);
973
974	ehci_qh *queueHead = CreateQueueHead();
975	if (!queueHead) {
976		TRACE_ERROR("failed to allocate queue head\n");
977		return B_NO_MEMORY;
978	}
979
980	status_t result = InitQueueHead(queueHead, pipe);
981	if (result != B_OK) {
982		TRACE_ERROR("failed to init queue head\n");
983		FreeQueueHead(queueHead);
984		return result;
985	}
986
987	bool directionIn;
988	ehci_qtd *dataDescriptor;
989	if ((pipe->Type() & USB_OBJECT_CONTROL_PIPE) != 0) {
990		result = FillQueueWithRequest(transfer, queueHead, &dataDescriptor,
991			&directionIn);
992	} else {
993		result = FillQueueWithData(transfer, queueHead, &dataDescriptor,
994			&directionIn);
995	}
996
997	if (result != B_OK) {
998		TRACE_ERROR("failed to fill transfer queue with data\n");
999		FreeQueueHead(queueHead);
1000		return result;
1001	}
1002
1003	result = AddPendingTransfer(transfer, queueHead, dataDescriptor,
1004		directionIn);
1005	if (result != B_OK) {
1006		TRACE_ERROR("failed to add pending transfer\n");
1007		FreeQueueHead(queueHead);
1008		return result;
1009	}
1010
1011#ifdef TRACE_USB
1012	TRACE("linking queue\n");
1013	print_queue(queueHead);
1014#endif
1015
1016	if ((pipe->Type() & USB_OBJECT_INTERRUPT_PIPE) != 0)
1017		result = LinkInterruptQueueHead(queueHead, pipe);
1018	else
1019		result = LinkQueueHead(queueHead);
1020
1021	if (result != B_OK) {
1022		TRACE_ERROR("failed to link queue head\n");
1023		FreeQueueHead(queueHead);
1024		return result;
1025	}
1026
1027	return B_OK;
1028}
1029
1030
1031status_t
1032EHCI::SubmitIsochronous(Transfer *transfer)
1033{
1034	Pipe *pipe = transfer->TransferPipe();
1035	bool directionIn = (pipe->Direction() == Pipe::In);
1036	usb_isochronous_data *isochronousData = transfer->IsochronousData();
1037	size_t packetSize = transfer->DataLength();
1038#ifdef TRACE_USB
1039	size_t restSize = packetSize % isochronousData->packet_count;
1040#endif
1041	packetSize /= isochronousData->packet_count;
1042	uint16 currentFrame;
1043
1044	if (packetSize > pipe->MaxPacketSize()) {
1045		TRACE_ERROR(
1046			"isochronous packetSize is bigger than pipe MaxPacketSize\n");
1047		return B_BAD_VALUE;
1048	}
1049
1050	// Ignore the fact that the last descriptor might need less bandwidth.
1051	// The overhead is not worthy.
1052	uint16 bandwidth = transfer->Bandwidth() / isochronousData->packet_count;
1053
1054	TRACE("isochronous transfer descriptor bandwidth %d\n", bandwidth);
1055
1056	// The following holds the list of transfer descriptor of the
1057	// isochronous request. It is used to quickly remove all the isochronous
1058	// descriptors from the frame list, as descriptors are not link to each
1059	// other in a queue like for every other transfer.
1060	ehci_itd **isoRequest
1061		= new(std::nothrow) ehci_itd *[isochronousData->packet_count];
1062	if (isoRequest == NULL) {
1063		TRACE("failed to create isoRequest array!\n");
1064		return B_NO_MEMORY;
1065	}
1066
1067	TRACE("isochronous submitted size=%" B_PRIuSIZE " bytes, TDs=%" B_PRIu32
1068		", maxPacketSize=%" B_PRIuSIZE ", packetSize=%" B_PRIuSIZE
1069		", restSize=%" B_PRIuSIZE "\n", transfer->DataLength(),
1070		isochronousData->packet_count, pipe->MaxPacketSize(), packetSize,
1071		restSize);
1072
1073	// Find the entry where to start inserting the first Isochronous descriptor
1074	if ((isochronousData->flags & USB_ISO_ASAP) != 0 ||
1075		isochronousData->starting_frame_number == NULL) {
1076
1077		if (fFirstIsochronousTransfer != NULL && fNextStartingFrame != -1)
1078			currentFrame = fNextStartingFrame;
1079		else {
1080			uint32 threshold = fThreshold;
1081			TRACE("threshold: %" B_PRIu32 "\n", threshold);
1082
1083			// find the first available frame with enough bandwidth.
1084			// This should always be the case, as defining the starting frame
1085			// number in the driver makes no sense for many reason, one of which
1086			// is that frame numbers value are host controller specific, and the
1087			// driver does not know which host controller is running.
1088			currentFrame = ((ReadOpReg(EHCI_FRINDEX) + threshold) / 8)
1089				& (EHCI_FRAMELIST_ENTRIES_COUNT - 1);
1090		}
1091
1092		// Make sure that:
1093		// 1. We are at least 5ms ahead the controller
1094		// 2. We stay in the range 0-127
1095		// 3. There is enough bandwidth in the first entry
1096		currentFrame &= EHCI_VFRAMELIST_ENTRIES_COUNT - 1;
1097	} else {
1098		// Find out if the frame number specified has enough bandwidth,
1099		// otherwise find the first next available frame with enough bandwidth
1100		currentFrame = *isochronousData->starting_frame_number;
1101	}
1102
1103	TRACE("isochronous starting frame=%d\n", currentFrame);
1104
1105	uint16 itdIndex = 0;
1106	size_t dataLength = transfer->DataLength();
1107	void* bufferLog;
1108	phys_addr_t bufferPhy;
1109	if (fStack->AllocateChunk(&bufferLog, &bufferPhy, dataLength) != B_OK) {
1110		TRACE_ERROR("unable to allocate itd buffer\n");
1111		delete[] isoRequest;
1112		return B_NO_MEMORY;
1113	}
1114
1115	memset(bufferLog, 0, dataLength);
1116
1117	phys_addr_t currentPhy = bufferPhy;
1118	uint32 frameCount = 0;
1119	while (dataLength > 0) {
1120		ehci_itd* itd = CreateItdDescriptor();
1121		isoRequest[itdIndex++] = itd;
1122		uint16 pg = 0;
1123		itd->buffer_phy[pg] = currentPhy & 0xfffff000;
1124		uint32 offset = currentPhy & 0xfff;
1125		TRACE("isochronous created itd, filling it with phy %" B_PRIxPHYSADDR
1126			"\n", currentPhy);
1127		for (int32 i = 0; i < 8 && dataLength > 0; i++) {
1128			size_t length = min_c(dataLength, packetSize);
1129			itd->token[i] = (EHCI_ITD_STATUS_ACTIVE << EHCI_ITD_STATUS_SHIFT)
1130				| (length << EHCI_ITD_TLENGTH_SHIFT) | (pg << EHCI_ITD_PG_SHIFT)
1131				| (offset << EHCI_ITD_TOFFSET_SHIFT);
1132			itd->last_token = i;
1133			TRACE("isochronous filled slot %" B_PRId32 " 0x%" B_PRIx32 "\n", i,
1134				itd->token[i]);
1135			dataLength -= length;
1136			offset += length;
1137			if (dataLength > 0 && offset > 0xfff) {
1138				offset -= B_PAGE_SIZE;
1139				currentPhy += B_PAGE_SIZE;
1140				itd->buffer_phy[pg + 1] = currentPhy & 0xfffff000;
1141				pg++;
1142			}
1143			if (dataLength <= 0)
1144				itd->token[i] |= EHCI_ITD_IOC;
1145		}
1146
1147		currentPhy += (offset & 0xfff) - (currentPhy & 0xfff);
1148
1149		itd->buffer_phy[0]
1150			|= (pipe->EndpointAddress() << EHCI_ITD_ENDPOINT_SHIFT)
1151				| (pipe->DeviceAddress() << EHCI_ITD_ADDRESS_SHIFT);
1152		itd->buffer_phy[1]
1153			|= (pipe->MaxPacketSize() & EHCI_ITD_MAXPACKETSIZE_MASK)
1154				| (directionIn << EHCI_ITD_DIR_SHIFT);
1155		itd->buffer_phy[2]
1156			|= ((((pipe->MaxPacketSize() >> EHCI_ITD_MAXPACKETSIZE_LENGTH) + 1)
1157				& EHCI_ITD_MUL_MASK) << EHCI_ITD_MUL_SHIFT);
1158
1159		TRACE("isochronous filled itd buffer_phy[0,1,2] 0x%" B_PRIx32 ", 0x%"
1160			B_PRIx32 " 0x%" B_PRIx32 "\n",
1161			itd->buffer_phy[0], itd->buffer_phy[1], itd->buffer_phy[2]);
1162
1163		if (!LockIsochronous())
1164			continue;
1165		LinkITDescriptors(itd, &fItdEntries[currentFrame]);
1166		UnlockIsochronous();
1167		fFrameBandwidth[currentFrame] -= bandwidth;
1168		currentFrame = (currentFrame + 1) & (EHCI_VFRAMELIST_ENTRIES_COUNT - 1);
1169		frameCount++;
1170	}
1171
1172	TRACE("isochronous filled itds count %d\n", itdIndex);
1173
1174	// Add transfer to the list
1175	status_t result = AddPendingIsochronousTransfer(transfer, isoRequest,
1176		itdIndex - 1, directionIn, bufferPhy, bufferLog,
1177		transfer->DataLength());
1178	if (result != B_OK) {
1179		TRACE_ERROR("failed to add pending isochronous transfer\n");
1180		for (uint32 i = 0; i < itdIndex; i++)
1181			FreeDescriptor(isoRequest[i]);
1182		delete[] isoRequest;
1183		return result;
1184	}
1185
1186	TRACE("appended isochronous transfer by starting at frame number %d\n",
1187		currentFrame);
1188	fNextStartingFrame = currentFrame + 1;
1189
1190	// Wake up the isochronous finisher thread
1191	release_sem_etc(fFinishIsochronousTransfersSem, 1 /*frameCount*/,
1192		B_DO_NOT_RESCHEDULE);
1193
1194	return B_OK;
1195}
1196
1197
1198isochronous_transfer_data *
1199EHCI::FindIsochronousTransfer(ehci_itd *itd)
1200{
1201	// Simply check every last descriptor of the isochronous transfer list
1202	isochronous_transfer_data *transfer = fFirstIsochronousTransfer;
1203	if (transfer) {
1204		while (transfer->descriptors[transfer->last_to_process]
1205			!= itd) {
1206			transfer = transfer->link;
1207			if (!transfer)
1208				break;
1209		}
1210	}
1211	return transfer;
1212}
1213
1214
1215status_t
1216EHCI::NotifyPipeChange(Pipe *pipe, usb_change change)
1217{
1218	TRACE("pipe change %d for pipe %p\n", change, pipe);
1219	switch (change) {
1220		case USB_CHANGE_CREATED:
1221		case USB_CHANGE_DESTROYED: {
1222			// ToDo: we should create and keep a single queue head
1223			// for all transfers to/from this pipe
1224			break;
1225		}
1226
1227		case USB_CHANGE_PIPE_POLICY_CHANGED: {
1228			// ToDo: for isochronous pipes we might need to adapt to new
1229			// pipe policy settings here
1230			break;
1231		}
1232	}
1233
1234	return B_OK;
1235}
1236
1237
1238status_t
1239EHCI::GetPortStatus(uint8 index, usb_port_status *status)
1240{
1241	if (index >= fPortCount)
1242		return B_BAD_INDEX;
1243
1244	status->status = status->change = 0;
1245	uint32 portStatus = ReadOpReg(EHCI_PORTSC + index * sizeof(uint32));
1246
1247	// build the status
1248	if (portStatus & EHCI_PORTSC_CONNSTATUS)
1249		status->status |= PORT_STATUS_CONNECTION;
1250	if (portStatus & EHCI_PORTSC_ENABLE)
1251		status->status |= PORT_STATUS_ENABLE;
1252	if (portStatus & EHCI_PORTSC_ENABLE)
1253		status->status |= PORT_STATUS_HIGH_SPEED;
1254	if (portStatus & EHCI_PORTSC_OCACTIVE)
1255		status->status |= PORT_STATUS_OVER_CURRENT;
1256	if (portStatus & EHCI_PORTSC_PORTRESET)
1257		status->status |= PORT_STATUS_RESET;
1258	if (portStatus & EHCI_PORTSC_PORTPOWER)
1259		status->status |= PORT_STATUS_POWER;
1260	if (portStatus & EHCI_PORTSC_SUSPEND)
1261		status->status |= PORT_STATUS_SUSPEND;
1262	if (portStatus & EHCI_PORTSC_DMINUS)
1263		status->status |= PORT_STATUS_LOW_SPEED;
1264
1265	// build the change
1266	if (portStatus & EHCI_PORTSC_CONNCHANGE)
1267		status->change |= PORT_STATUS_CONNECTION;
1268	if (portStatus & EHCI_PORTSC_ENABLECHANGE)
1269		status->change |= PORT_STATUS_ENABLE;
1270	if (portStatus & EHCI_PORTSC_OCCHANGE)
1271		status->change |= PORT_STATUS_OVER_CURRENT;
1272
1273	// there are no bits to indicate suspend and reset change
1274	if (fPortResetChange & (1 << index))
1275		status->change |= PORT_STATUS_RESET;
1276	if (fPortSuspendChange & (1 << index))
1277		status->change |= PORT_STATUS_SUSPEND;
1278
1279	return B_OK;
1280}
1281
1282
1283status_t
1284EHCI::SetPortFeature(uint8 index, uint16 feature)
1285{
1286	if (index >= fPortCount)
1287		return B_BAD_INDEX;
1288
1289	uint32 portRegister = EHCI_PORTSC + index * sizeof(uint32);
1290	uint32 portStatus = ReadOpReg(portRegister) & EHCI_PORTSC_DATAMASK;
1291
1292	switch (feature) {
1293		case PORT_SUSPEND:
1294			return SuspendPort(index);
1295
1296		case PORT_RESET:
1297			return ResetPort(index);
1298
1299		case PORT_POWER:
1300			WriteOpReg(portRegister, portStatus | EHCI_PORTSC_PORTPOWER);
1301			return B_OK;
1302	}
1303
1304	return B_BAD_VALUE;
1305}
1306
1307
1308status_t
1309EHCI::ClearPortFeature(uint8 index, uint16 feature)
1310{
1311	if (index >= fPortCount)
1312		return B_BAD_INDEX;
1313
1314	uint32 portRegister = EHCI_PORTSC + index * sizeof(uint32);
1315	uint32 portStatus = ReadOpReg(portRegister) & EHCI_PORTSC_DATAMASK;
1316
1317	switch (feature) {
1318		case PORT_ENABLE:
1319			WriteOpReg(portRegister, portStatus & ~EHCI_PORTSC_ENABLE);
1320			return B_OK;
1321
1322		case PORT_POWER:
1323			WriteOpReg(portRegister, portStatus & ~EHCI_PORTSC_PORTPOWER);
1324			return B_OK;
1325
1326		case C_PORT_CONNECTION:
1327			WriteOpReg(portRegister, portStatus | EHCI_PORTSC_CONNCHANGE);
1328			return B_OK;
1329
1330		case C_PORT_ENABLE:
1331			WriteOpReg(portRegister, portStatus | EHCI_PORTSC_ENABLECHANGE);
1332			return B_OK;
1333
1334		case C_PORT_OVER_CURRENT:
1335			WriteOpReg(portRegister, portStatus | EHCI_PORTSC_OCCHANGE);
1336			return B_OK;
1337
1338		case C_PORT_RESET:
1339			fPortResetChange &= ~(1 << index);
1340			return B_OK;
1341
1342		case C_PORT_SUSPEND:
1343			fPortSuspendChange &= ~(1 << index);
1344			return B_OK;
1345	}
1346
1347	return B_BAD_VALUE;
1348}
1349
1350
1351status_t
1352EHCI::ResetPort(uint8 index)
1353{
1354	TRACE("reset port %d\n", index);
1355	uint32 portRegister = EHCI_PORTSC + index * sizeof(uint32);
1356	uint32 portStatus = ReadOpReg(portRegister) & EHCI_PORTSC_DATAMASK;
1357
1358	if (portStatus & EHCI_PORTSC_DMINUS) {
1359		TRACE_ALWAYS("lowspeed device connected, giving up port ownership\n");
1360		// there is a lowspeed device connected.
1361		// we give the ownership to a companion controller.
1362		WriteOpReg(portRegister, portStatus | EHCI_PORTSC_PORTOWNER);
1363		fPortResetChange |= (1 << index);
1364		return B_OK;
1365	}
1366
1367	// enable reset signaling
1368	WriteOpReg(portRegister, (portStatus & ~EHCI_PORTSC_ENABLE)
1369		| EHCI_PORTSC_PORTRESET);
1370	snooze(50000);
1371
1372	// disable reset signaling
1373	portStatus = ReadOpReg(portRegister) & EHCI_PORTSC_DATAMASK;
1374	WriteOpReg(portRegister, portStatus & ~EHCI_PORTSC_PORTRESET);
1375	snooze(2000);
1376
1377	portStatus = ReadOpReg(portRegister) & EHCI_PORTSC_DATAMASK;
1378	if (portStatus & EHCI_PORTSC_PORTRESET) {
1379		TRACE_ERROR("port reset won't complete\n");
1380		return B_ERROR;
1381	}
1382
1383	if ((portStatus & EHCI_PORTSC_ENABLE) == 0) {
1384		TRACE_ALWAYS("fullspeed device connected, giving up port ownership\n");
1385		// the port was not enabled, this means that no high speed device is
1386		// attached to this port. we give up ownership to a companion controler
1387		WriteOpReg(portRegister, portStatus | EHCI_PORTSC_PORTOWNER);
1388	}
1389
1390	fPortResetChange |= (1 << index);
1391	return B_OK;
1392}
1393
1394
1395status_t
1396EHCI::SuspendPort(uint8 index)
1397{
1398	uint32 portRegister = EHCI_PORTSC + index * sizeof(uint32);
1399	uint32 portStatus = ReadOpReg(portRegister) & EHCI_PORTSC_DATAMASK;
1400	WriteOpReg(portRegister, portStatus | EHCI_PORTSC_SUSPEND);
1401	fPortSuspendChange |= (1 << index);
1402	return B_OK;
1403}
1404
1405
1406status_t
1407EHCI::ControllerReset()
1408{
1409	// halt the controller first
1410	WriteOpReg(EHCI_USBCMD, 0);
1411	snooze(10000);
1412
1413	// then reset it
1414	WriteOpReg(EHCI_USBCMD, EHCI_USBCMD_HCRESET);
1415
1416	int32 tries = 5;
1417	while (ReadOpReg(EHCI_USBCMD) & EHCI_USBCMD_HCRESET) {
1418		snooze(10000);
1419		if (tries-- < 0)
1420			return B_ERROR;
1421	}
1422
1423	return B_OK;
1424}
1425
1426
1427status_t
1428EHCI::LightReset()
1429{
1430	return B_ERROR;
1431}
1432
1433
1434int32
1435EHCI::InterruptHandler(void *data)
1436{
1437	return ((EHCI *)data)->Interrupt();
1438}
1439
1440
1441int32
1442EHCI::Interrupt()
1443{
1444	static spinlock lock = B_SPINLOCK_INITIALIZER;
1445	acquire_spinlock(&lock);
1446
1447	// check if any interrupt was generated
1448	uint32 status = ReadOpReg(EHCI_USBSTS) & EHCI_USBSTS_INTMASK;
1449	if ((status & fEnabledInterrupts) == 0) {
1450		if (status != 0) {
1451			TRACE("discarding not enabled interrupts 0x%08" B_PRIx32 "\n",
1452				status);
1453			WriteOpReg(EHCI_USBSTS, status);
1454		}
1455
1456		release_spinlock(&lock);
1457		return B_UNHANDLED_INTERRUPT;
1458	}
1459
1460	bool asyncAdvance = false;
1461	bool finishTransfers = false;
1462	int32 result = B_HANDLED_INTERRUPT;
1463
1464	if (status & EHCI_USBSTS_USBINT) {
1465		TRACE("transfer finished\n");
1466		result = B_INVOKE_SCHEDULER;
1467		finishTransfers = true;
1468	}
1469
1470	if (status & EHCI_USBSTS_USBERRINT) {
1471		TRACE("transfer error\n");
1472		result = B_INVOKE_SCHEDULER;
1473		finishTransfers = true;
1474	}
1475
1476	if (status & EHCI_USBSTS_FLROLLOVER)
1477		TRACE("frame list rollover\n");
1478
1479	if (status & EHCI_USBSTS_PORTCHANGE)
1480		TRACE("port change detected\n");
1481
1482	if (status & EHCI_USBSTS_INTONAA) {
1483		TRACE("interrupt on async advance\n");
1484		asyncAdvance = true;
1485		result = B_INVOKE_SCHEDULER;
1486	}
1487
1488	if (status & EHCI_USBSTS_HOSTSYSERR)
1489		TRACE_ERROR("host system error!\n");
1490
1491	WriteOpReg(EHCI_USBSTS, status);
1492	release_spinlock(&lock);
1493
1494	if (asyncAdvance)
1495		release_sem_etc(fAsyncAdvanceSem, 1, B_DO_NOT_RESCHEDULE);
1496	if (finishTransfers)
1497		release_sem_etc(fFinishTransfersSem, 1, B_DO_NOT_RESCHEDULE);
1498
1499	return result;
1500}
1501
1502
1503int32
1504EHCI::InterruptPollThread(void *data)
1505{
1506	EHCI *ehci = (EHCI *)data;
1507
1508	while (!ehci->fStopThreads) {
1509		// TODO: this could be handled much better by only polling when there
1510		// are actual transfers going on...
1511		snooze(1000);
1512
1513		cpu_status status = disable_interrupts();
1514		ehci->Interrupt();
1515		restore_interrupts(status);
1516	}
1517
1518	return 0;
1519}
1520
1521
1522status_t
1523EHCI::AddPendingTransfer(Transfer *transfer, ehci_qh *queueHead,
1524	ehci_qtd *dataDescriptor, bool directionIn)
1525{
1526	transfer_data *data = new(std::nothrow) transfer_data;
1527	if (!data)
1528		return B_NO_MEMORY;
1529
1530	status_t result = transfer->InitKernelAccess();
1531	if (result != B_OK) {
1532		delete data;
1533		return result;
1534	}
1535
1536	data->transfer = transfer;
1537	data->queue_head = queueHead;
1538	data->data_descriptor = dataDescriptor;
1539	data->incoming = directionIn;
1540	data->canceled = false;
1541	data->link = NULL;
1542
1543	if (!Lock()) {
1544		delete data;
1545		return B_ERROR;
1546	}
1547
1548	if (fLastTransfer)
1549		fLastTransfer->link = data;
1550	else
1551		fFirstTransfer = data;
1552
1553	fLastTransfer = data;
1554	Unlock();
1555
1556	return B_OK;
1557}
1558
1559
1560status_t
1561EHCI::AddPendingIsochronousTransfer(Transfer *transfer, ehci_itd **isoRequest,
1562	uint32 lastIndex, bool directionIn, addr_t bufferPhy, void* bufferLog,
1563	size_t bufferSize)
1564{
1565	if (!transfer || !isoRequest)
1566		return B_BAD_VALUE;
1567
1568	isochronous_transfer_data *data
1569		= new(std::nothrow) isochronous_transfer_data;
1570	if (!data)
1571		return B_NO_MEMORY;
1572
1573	status_t result = transfer->InitKernelAccess();
1574	if (result != B_OK) {
1575		delete data;
1576		return result;
1577	}
1578
1579	data->transfer = transfer;
1580	data->descriptors = isoRequest;
1581	data->last_to_process = lastIndex;
1582	data->incoming = directionIn;
1583	data->is_active = true;
1584	data->link = NULL;
1585	data->buffer_phy = bufferPhy;
1586	data->buffer_log = bufferLog;
1587	data->buffer_size = bufferSize;
1588
1589	// Put in the isochronous transfer list
1590	if (!LockIsochronous()) {
1591		delete data;
1592		return B_ERROR;
1593	}
1594
1595	if (fLastIsochronousTransfer)
1596		fLastIsochronousTransfer->link = data;
1597	else if (!fFirstIsochronousTransfer)
1598		fFirstIsochronousTransfer = data;
1599
1600	fLastIsochronousTransfer = data;
1601	UnlockIsochronous();
1602	return B_OK;
1603}
1604
1605
1606status_t
1607EHCI::CancelQueuedTransfers(Pipe *pipe, bool force)
1608{
1609	if ((pipe->Type() & USB_OBJECT_ISO_PIPE) != 0)
1610		return CancelQueuedIsochronousTransfers(pipe, force);
1611
1612	if (!Lock())
1613		return B_ERROR;
1614
1615	struct transfer_entry {
1616		Transfer *			transfer;
1617		transfer_entry *	next;
1618	};
1619
1620	transfer_entry *list = NULL;
1621	transfer_data *current = fFirstTransfer;
1622	while (current) {
1623		if (current->transfer && current->transfer->TransferPipe() == pipe) {
1624			// clear the active bit so the descriptors are canceled
1625			ehci_qtd *descriptor = current->queue_head->element_log;
1626			while (descriptor) {
1627				descriptor->token &= ~EHCI_QTD_STATUS_ACTIVE;
1628				descriptor = descriptor->next_log;
1629			}
1630
1631			if (!force) {
1632				// if the transfer is canceled by force, the one causing the
1633				// cancel is probably not the one who initiated the transfer
1634				// and the callback is likely not safe anymore
1635				transfer_entry *entry
1636					= (transfer_entry *)malloc(sizeof(transfer_entry));
1637				if (entry != NULL) {
1638					entry->transfer = current->transfer;
1639					current->transfer = NULL;
1640					entry->next = list;
1641					list = entry;
1642				}
1643			}
1644
1645			current->canceled = true;
1646		}
1647
1648		current = current->link;
1649	}
1650
1651	Unlock();
1652
1653	while (list != NULL) {
1654		transfer_entry *next = list->next;
1655		list->transfer->Finished(B_CANCELED, 0);
1656		delete list->transfer;
1657		free(list);
1658		list = next;
1659	}
1660
1661	// wait for any transfers that might have made it before canceling
1662	while (fProcessingPipe == pipe)
1663		snooze(1000);
1664
1665	// notify the finisher so it can clean up the canceled transfers
1666	release_sem_etc(fFinishTransfersSem, 1, B_DO_NOT_RESCHEDULE);
1667	return B_OK;
1668}
1669
1670
1671status_t
1672EHCI::CancelQueuedIsochronousTransfers(Pipe *pipe, bool force)
1673{
1674	isochronous_transfer_data *current = fFirstIsochronousTransfer;
1675
1676	while (current) {
1677		if (current->transfer->TransferPipe() == pipe) {
1678			// TODO implement
1679
1680			// TODO: Use the force paramater in order to avoid calling
1681			// invalid callbacks
1682			current->is_active = false;
1683		}
1684
1685		current = current->link;
1686	}
1687
1688	TRACE_ERROR("no isochronous transfer found!\n");
1689	return B_ERROR;
1690}
1691
1692
1693status_t
1694EHCI::CancelAllPendingTransfers()
1695{
1696	if (!Lock())
1697		return B_ERROR;
1698
1699	transfer_data *transfer = fFirstTransfer;
1700	while (transfer) {
1701		transfer->transfer->Finished(B_CANCELED, 0);
1702		delete transfer->transfer;
1703
1704		transfer_data *next = transfer->link;
1705		delete transfer;
1706		transfer = next;
1707	}
1708
1709	fFirstTransfer = NULL;
1710	fLastTransfer = NULL;
1711	Unlock();
1712	return B_OK;
1713}
1714
1715
1716int32
1717EHCI::FinishThread(void *data)
1718{
1719	((EHCI *)data)->FinishTransfers();
1720	return B_OK;
1721}
1722
1723
1724void
1725EHCI::FinishTransfers()
1726{
1727	while (!fStopThreads) {
1728		if (acquire_sem(fFinishTransfersSem) != B_OK)
1729			continue;
1730
1731		// eat up sems that have been released by multiple interrupts
1732		int32 semCount = 0;
1733		get_sem_count(fFinishTransfersSem, &semCount);
1734		if (semCount > 0) {
1735			acquire_sem_etc(fFinishTransfersSem, semCount, B_RELATIVE_TIMEOUT,
1736				0);
1737		}
1738
1739		if (!Lock())
1740			continue;
1741
1742		TRACE("finishing transfers\n");
1743		transfer_data *lastTransfer = NULL;
1744		transfer_data *transfer = fFirstTransfer;
1745		Unlock();
1746
1747		while (transfer) {
1748			bool transferDone = false;
1749			ehci_qtd *descriptor = transfer->queue_head->element_log;
1750			status_t callbackStatus = B_OK;
1751
1752			while (descriptor) {
1753				uint32 status = descriptor->token;
1754				if (status & EHCI_QTD_STATUS_ACTIVE) {
1755					// still in progress
1756					TRACE("qtd (0x%08" B_PRIx32 ") still active\n",
1757						descriptor->this_phy);
1758					break;
1759				}
1760
1761				if (status & EHCI_QTD_STATUS_ERRMASK) {
1762					// a transfer error occured
1763					TRACE_ERROR("qtd (0x%" B_PRIx32 ") error: 0x%08" B_PRIx32
1764						"\n", descriptor->this_phy, status);
1765
1766					uint8 errorCount = status >> EHCI_QTD_ERRCOUNT_SHIFT;
1767					errorCount &= EHCI_QTD_ERRCOUNT_MASK;
1768					if (errorCount == 0) {
1769						// the error counter counted down to zero, report why
1770						int32 reasons = 0;
1771						if (status & EHCI_QTD_STATUS_BUFFER) {
1772							callbackStatus = transfer->incoming
1773								? B_DEV_DATA_OVERRUN : B_DEV_DATA_UNDERRUN;
1774							reasons++;
1775						}
1776						if (status & EHCI_QTD_STATUS_TERROR) {
1777							callbackStatus = B_DEV_CRC_ERROR;
1778							reasons++;
1779						}
1780						if ((transfer->queue_head->endpoint_chars
1781								& EHCI_QH_CHARS_EPS_HIGH) == 0) {
1782							// For full-/lowspeed endpoints the unused ping
1783							// state bit is used as another error bit, it is
1784							// unspecific however.
1785							if ((status & EHCI_QTD_STATUS_LS_ERR) != 0) {
1786								callbackStatus = B_DEV_STALLED;
1787								reasons++;
1788							}
1789						}
1790
1791						if (reasons > 1)
1792							callbackStatus = B_DEV_MULTIPLE_ERRORS;
1793						else if (reasons == 0) {
1794							TRACE_ERROR("error counter counted down to zero "
1795								"but none of the error bits are set\n");
1796							callbackStatus = B_DEV_STALLED;
1797						}
1798					} else if (status & EHCI_QTD_STATUS_BABBLE) {
1799						// there is a babble condition
1800						callbackStatus = transfer->incoming
1801							? B_DEV_FIFO_OVERRUN : B_DEV_FIFO_UNDERRUN;
1802					} else {
1803						// if the error counter didn't count down to zero
1804						// and there was no babble, then this halt was caused
1805						// by a stall handshake
1806						callbackStatus = B_DEV_STALLED;
1807					}
1808
1809					transferDone = true;
1810					break;
1811				}
1812
1813				if (descriptor->next_phy & EHCI_ITEM_TERMINATE) {
1814					// we arrived at the last (stray) descriptor, we're done
1815					TRACE("qtd (0x%08" B_PRIx32 ") done\n",
1816						descriptor->this_phy);
1817					callbackStatus = B_OK;
1818					transferDone = true;
1819					break;
1820				}
1821
1822				if (((status >> EHCI_QTD_PID_SHIFT) & EHCI_QTD_PID_MASK)
1823						== EHCI_QTD_PID_IN
1824					&& ((status >> EHCI_QTD_BYTES_SHIFT) & EHCI_QTD_BYTES_MASK)
1825						!= 0) {
1826					// a short packet condition existed on this descriptor,
1827					// follow the alternate next pointer if set
1828					if (descriptor->alt_next_log != NULL) {
1829						descriptor = descriptor->alt_next_log;
1830						continue;
1831					}
1832
1833					// no alternate next, transfer is done
1834					callbackStatus = B_OK;
1835					transferDone = true;
1836					break;
1837				}
1838
1839				descriptor = descriptor->next_log;
1840			}
1841
1842			if (!transferDone) {
1843				lastTransfer = transfer;
1844				transfer = transfer->link;
1845				continue;
1846			}
1847
1848			// remove the transfer from the list first so we are sure
1849			// it doesn't get canceled while we still process it
1850			transfer_data *next = transfer->link;
1851			if (Lock()) {
1852				if (lastTransfer)
1853					lastTransfer->link = transfer->link;
1854
1855				if (transfer == fFirstTransfer)
1856					fFirstTransfer = transfer->link;
1857				if (transfer == fLastTransfer)
1858					fLastTransfer = lastTransfer;
1859
1860				// store the currently processing pipe here so we can wait
1861				// in cancel if we are processing something on the target pipe
1862				if (!transfer->canceled)
1863					fProcessingPipe = transfer->transfer->TransferPipe();
1864
1865				transfer->link = NULL;
1866				Unlock();
1867			}
1868
1869			// if canceled the callback has already been called
1870			if (!transfer->canceled) {
1871				size_t actualLength = 0;
1872
1873				if (callbackStatus == B_OK) {
1874					bool nextDataToggle = false;
1875					if (transfer->data_descriptor && transfer->incoming) {
1876						// data to read out
1877						iovec *vector = transfer->transfer->Vector();
1878						size_t vectorCount = transfer->transfer->VectorCount();
1879						transfer->transfer->PrepareKernelAccess();
1880						actualLength = ReadDescriptorChain(
1881							transfer->data_descriptor,
1882							vector, vectorCount,
1883							&nextDataToggle);
1884					} else if (transfer->data_descriptor) {
1885						// calculate transfered length
1886						actualLength = ReadActualLength(
1887							transfer->data_descriptor, &nextDataToggle);
1888					}
1889
1890					transfer->transfer->TransferPipe()->SetDataToggle(
1891						nextDataToggle);
1892
1893					if (transfer->transfer->IsFragmented()) {
1894						// this transfer may still have data left
1895						transfer->transfer->AdvanceByFragment(actualLength);
1896						if (transfer->transfer->VectorLength() > 0) {
1897							FreeDescriptorChain(transfer->data_descriptor);
1898							transfer->transfer->PrepareKernelAccess();
1899							status_t result = FillQueueWithData(
1900								transfer->transfer,
1901								transfer->queue_head,
1902								&transfer->data_descriptor, NULL);
1903
1904							if (result == B_OK && Lock()) {
1905								// reappend the transfer
1906								if (fLastTransfer)
1907									fLastTransfer->link = transfer;
1908								if (!fFirstTransfer)
1909									fFirstTransfer = transfer;
1910
1911								fLastTransfer = transfer;
1912								Unlock();
1913
1914								transfer = next;
1915								continue;
1916							}
1917						}
1918
1919						// the transfer is done, but we already set the
1920						// actualLength with AdvanceByFragment()
1921						actualLength = 0;
1922					}
1923				}
1924
1925				transfer->transfer->Finished(callbackStatus, actualLength);
1926				fProcessingPipe = NULL;
1927			}
1928
1929			// unlink hardware queue and delete the transfer
1930			UnlinkQueueHead(transfer->queue_head, &fFreeListHead);
1931			delete transfer->transfer;
1932			delete transfer;
1933			transfer = next;
1934			release_sem(fCleanupSem);
1935		}
1936	}
1937}
1938
1939
1940int32
1941EHCI::CleanupThread(void *data)
1942{
1943	((EHCI *)data)->Cleanup();
1944	return B_OK;
1945}
1946
1947
1948void
1949EHCI::Cleanup()
1950{
1951	ehci_qh *lastFreeListHead = NULL;
1952
1953	while (!fStopThreads) {
1954		if (acquire_sem(fCleanupSem) != B_OK)
1955			continue;
1956
1957		ehci_qh *freeListHead = fFreeListHead;
1958		if (freeListHead == lastFreeListHead)
1959			continue;
1960
1961		// set the doorbell and wait for the host controller to notify us
1962		WriteOpReg(EHCI_USBCMD, ReadOpReg(EHCI_USBCMD) | EHCI_USBCMD_INTONAAD);
1963		if (acquire_sem(fAsyncAdvanceSem) != B_OK)
1964			continue;
1965
1966		ehci_qh *current = freeListHead;
1967		while (current != lastFreeListHead) {
1968			ehci_qh *next = current->next_log;
1969			FreeQueueHead(current);
1970			current = next;
1971		}
1972
1973		lastFreeListHead = freeListHead;
1974	}
1975}
1976
1977
1978int32
1979EHCI::FinishIsochronousThread(void *data)
1980{
1981	((EHCI *)data)->FinishIsochronousTransfers();
1982	return B_OK;
1983}
1984
1985
1986void
1987EHCI::FinishIsochronousTransfers()
1988{
1989	/* This thread stays one position behind the controller and processes every
1990	* isochronous descriptor. Once it finds the last isochronous descriptor
1991	* of a transfer, it processes the entire transfer.
1992	*/
1993	while (!fStopThreads) {
1994		// Go to sleep if there are no isochronous transfers to process
1995		if (acquire_sem(fFinishIsochronousTransfersSem) != B_OK)
1996			return;
1997
1998		bool transferDone = false;
1999
2000		uint32 frame = (ReadOpReg(EHCI_FRINDEX) / 8 )
2001			& (EHCI_FRAMELIST_ENTRIES_COUNT - 1);
2002		uint32 currentFrame = (frame + EHCI_VFRAMELIST_ENTRIES_COUNT - 5)
2003			& (EHCI_VFRAMELIST_ENTRIES_COUNT - 1);
2004		uint32 loop = 0;
2005
2006		// Process the frame list until one transfer is processed
2007		while (!transferDone && loop++ < EHCI_VFRAMELIST_ENTRIES_COUNT) {
2008			// wait 1ms in order to be sure to be one position behind
2009			// the controller
2010			while (currentFrame == (((ReadOpReg(EHCI_FRINDEX) / 8)
2011				& (EHCI_VFRAMELIST_ENTRIES_COUNT - 1)))) {
2012				snooze(1000);
2013			}
2014
2015			ehci_itd *itd = fItdEntries[currentFrame];
2016
2017			TRACE("FinishIsochronousTransfers itd %p phy 0x%" B_PRIx32
2018				" prev (%p/0x%" B_PRIx32 ") at frame %" B_PRId32 "\n", itd,
2019				itd->this_phy, itd->prev, itd->prev != NULL
2020					? itd->prev->this_phy : 0, currentFrame);
2021
2022			if (!LockIsochronous())
2023				continue;
2024
2025			// Process the frame till it has isochronous descriptors in it.
2026			while (!(itd->next_phy & EHCI_ITEM_TERMINATE) && itd->prev != NULL) {
2027				TRACE("FinishIsochronousTransfers checking itd %p last_token"
2028					" %" B_PRId32 "\n", itd, itd->last_token);
2029				TRACE("FinishIsochronousTransfers tokens 0x%" B_PRIx32 " 0x%"
2030					B_PRIx32 " 0x%" B_PRIx32 " 0x%" B_PRIx32 " 0x%" B_PRIx32
2031					" 0x%" B_PRIx32 " 0x%" B_PRIx32 " 0x%" B_PRIx32 "\n",
2032					itd->token[0], itd->token[1], itd->token[2], itd->token[3],
2033					itd->token[4], itd->token[5], itd->token[6], itd->token[7]);
2034				if (((itd->token[itd->last_token] >> EHCI_ITD_STATUS_SHIFT)
2035					& EHCI_ITD_STATUS_ACTIVE) == EHCI_ITD_STATUS_ACTIVE) {
2036					TRACE("FinishIsochronousTransfers unprocessed active itd\n");
2037				}
2038				UnlinkITDescriptors(itd, &fItdEntries[currentFrame]);
2039
2040				// Process the transfer if we found the last descriptor
2041				isochronous_transfer_data *transfer
2042					= FindIsochronousTransfer(itd);
2043					// Process the descriptors only if it is still active and
2044					// belongs to an inbound transfer. If the transfer is not
2045					// active, it means the request has been removed, so simply
2046					// remove the descriptors.
2047				if (transfer && transfer->is_active) {
2048					TRACE("FinishIsochronousTransfers active transfer\n");
2049					size_t actualLength = 0;
2050					if (((itd->buffer_phy[1] >> EHCI_ITD_DIR_SHIFT) & 1) != 0) {
2051						transfer->transfer->PrepareKernelAccess();
2052						actualLength = ReadIsochronousDescriptorChain(transfer);
2053					}
2054
2055					// Remove the transfer
2056					if (transfer == fFirstIsochronousTransfer) {
2057						fFirstIsochronousTransfer = transfer->link;
2058						if (transfer == fLastIsochronousTransfer)
2059							fLastIsochronousTransfer = NULL;
2060					} else {
2061						isochronous_transfer_data *temp
2062							= fFirstIsochronousTransfer;
2063						while (temp != NULL && transfer != temp->link)
2064							temp = temp->link;
2065
2066						if (transfer == fLastIsochronousTransfer)
2067							fLastIsochronousTransfer = temp;
2068						if (temp != NULL && temp->link != NULL)
2069							temp->link = temp->link->link;
2070					}
2071					transfer->link = NULL;
2072
2073					transfer->transfer->Finished(B_OK, actualLength);
2074
2075					itd = itd->prev;
2076
2077					for (uint32 i = 0; i <= transfer->last_to_process; i++)
2078						FreeDescriptor(transfer->descriptors[i]);
2079
2080					TRACE("FinishIsochronousTransfers descriptors freed\n");
2081
2082					delete [] transfer->descriptors;
2083					delete transfer->transfer;
2084					fStack->FreeChunk(transfer->buffer_log,
2085						(phys_addr_t)transfer->buffer_phy,
2086						transfer->buffer_size);
2087					delete transfer;
2088					transferDone = true;
2089				} else {
2090					TRACE("FinishIsochronousTransfers not end of transfer\n");
2091					itd = itd->prev;
2092				}
2093			}
2094
2095			UnlockIsochronous();
2096
2097			TRACE("FinishIsochronousTransfers next frame\n");
2098
2099			// Make sure to reset the frame bandwidth
2100			fFrameBandwidth[currentFrame] = MAX_AVAILABLE_BANDWIDTH;
2101			currentFrame = (currentFrame + 1) % EHCI_VFRAMELIST_ENTRIES_COUNT;
2102		}
2103	}
2104}
2105
2106
2107ehci_qh *
2108EHCI::CreateQueueHead()
2109{
2110	ehci_qh *result;
2111	phys_addr_t physicalAddress;
2112	if (fStack->AllocateChunk((void **)&result, &physicalAddress,
2113			sizeof(ehci_qh)) != B_OK) {
2114		TRACE_ERROR("failed to allocate queue head\n");
2115		return NULL;
2116	}
2117
2118	result->this_phy = (addr_t)physicalAddress | EHCI_ITEM_TYPE_QH;
2119	result->next_phy = EHCI_ITEM_TERMINATE;
2120	result->next_log = NULL;
2121	result->prev_log = NULL;
2122
2123	ehci_qtd *descriptor = CreateDescriptor(0, 0);
2124	if (!descriptor) {
2125		TRACE_ERROR("failed to allocate initial qtd for queue head\n");
2126		fStack->FreeChunk(result, physicalAddress, sizeof(ehci_qh));
2127		return NULL;
2128	}
2129
2130	descriptor->token &= ~EHCI_QTD_STATUS_ACTIVE;
2131	result->stray_log = descriptor;
2132	result->element_log = descriptor;
2133	result->current_qtd_phy = 0;
2134	result->overlay.next_phy = descriptor->this_phy;
2135	result->overlay.alt_next_phy = EHCI_ITEM_TERMINATE;
2136	result->overlay.token = 0;
2137	for (int32 i = 0; i < 5; i++) {
2138		result->overlay.buffer_phy[i] = 0;
2139		result->overlay.ext_buffer_phy[i] = 0;
2140	}
2141
2142	return result;
2143}
2144
2145
2146status_t
2147EHCI::InitQueueHead(ehci_qh *queueHead, Pipe *pipe)
2148{
2149	switch (pipe->Speed()) {
2150		case USB_SPEED_LOWSPEED:
2151			queueHead->endpoint_chars = EHCI_QH_CHARS_EPS_LOW;
2152			break;
2153		case USB_SPEED_FULLSPEED:
2154			queueHead->endpoint_chars = EHCI_QH_CHARS_EPS_FULL;
2155			break;
2156		case USB_SPEED_HIGHSPEED:
2157			queueHead->endpoint_chars = EHCI_QH_CHARS_EPS_HIGH;
2158			break;
2159		default:
2160			TRACE_ERROR("unknown pipe speed\n");
2161			return B_ERROR;
2162	}
2163
2164	queueHead->endpoint_chars |= (3 << EHCI_QH_CHARS_RL_SHIFT)
2165		| (pipe->MaxPacketSize() << EHCI_QH_CHARS_MPL_SHIFT)
2166		| (pipe->EndpointAddress() << EHCI_QH_CHARS_EPT_SHIFT)
2167		| (pipe->DeviceAddress() << EHCI_QH_CHARS_DEV_SHIFT)
2168		| EHCI_QH_CHARS_TOGGLE;
2169
2170	queueHead->endpoint_caps = (1 << EHCI_QH_CAPS_MULT_SHIFT);
2171	if (pipe->Speed() != USB_SPEED_HIGHSPEED) {
2172		if ((pipe->Type() & USB_OBJECT_CONTROL_PIPE) != 0)
2173			queueHead->endpoint_chars |= EHCI_QH_CHARS_CONTROL;
2174
2175		queueHead->endpoint_caps |= (pipe->HubPort() << EHCI_QH_CAPS_PORT_SHIFT)
2176			| (pipe->HubAddress() << EHCI_QH_CAPS_HUB_SHIFT);
2177	}
2178
2179	return B_OK;
2180}
2181
2182
2183void
2184EHCI::FreeQueueHead(ehci_qh *queueHead)
2185{
2186	if (!queueHead)
2187		return;
2188
2189	FreeDescriptorChain(queueHead->element_log);
2190	FreeDescriptor(queueHead->stray_log);
2191	fStack->FreeChunk(queueHead, (phys_addr_t)queueHead->this_phy,
2192		sizeof(ehci_qh));
2193}
2194
2195
2196status_t
2197EHCI::LinkQueueHead(ehci_qh *queueHead)
2198{
2199	if (!Lock())
2200		return B_ERROR;
2201
2202	ehci_qh *prevHead = fAsyncQueueHead->prev_log;
2203	queueHead->next_phy = fAsyncQueueHead->this_phy;
2204	queueHead->next_log = fAsyncQueueHead;
2205	queueHead->prev_log = prevHead;
2206	fAsyncQueueHead->prev_log = queueHead;
2207	prevHead->next_log = queueHead;
2208	prevHead->next_phy = queueHead->this_phy;
2209
2210	Unlock();
2211	return B_OK;
2212}
2213
2214
2215status_t
2216EHCI::LinkInterruptQueueHead(ehci_qh *queueHead, Pipe *pipe)
2217{
2218	uint8 interval = pipe->Interval();
2219	if (pipe->Speed() == USB_SPEED_HIGHSPEED) {
2220		// Allow interrupts to be scheduled on each possible micro frame.
2221		queueHead->endpoint_caps |= (0xff << EHCI_QH_CAPS_ISM_SHIFT);
2222	} else {
2223		// As we do not yet support FSTNs to correctly reference low/full
2224		// speed interrupt transfers, we simply put them into the 1 interval
2225		// queue. This way we ensure that we reach them on every micro frame
2226		// and can do the corresponding start/complete split transactions.
2227		// ToDo: use FSTNs to correctly link non high speed interrupt transfers
2228		interval = 1;
2229
2230		// For now we also force start splits to be in micro frame 0 and
2231		// complete splits to be in micro frame 2, 3 and 4.
2232		queueHead->endpoint_caps |= (0x01 << EHCI_QH_CAPS_ISM_SHIFT);
2233		queueHead->endpoint_caps |= (0x1c << EHCI_QH_CAPS_SCM_SHIFT);
2234	}
2235
2236	// this should not happen
2237	if (interval < 1)
2238		interval = 1;
2239
2240	// this may happen as intervals can go up to 16; we limit the value to
2241	// EHCI_INTERRUPT_ENTRIES_COUNT as you cannot support intervals above
2242	// that with a frame list of just EHCI_VFRAMELIST_ENTRIES_COUNT entries...
2243	if (interval > EHCI_INTERRUPT_ENTRIES_COUNT)
2244		interval = EHCI_INTERRUPT_ENTRIES_COUNT;
2245
2246	if (!Lock())
2247		return B_ERROR;
2248
2249	ehci_qh *interruptQueue = &fInterruptEntries[interval - 1].queue_head;
2250	queueHead->next_phy = interruptQueue->next_phy;
2251	queueHead->next_log = interruptQueue->next_log;
2252	queueHead->prev_log = interruptQueue;
2253	if (interruptQueue->next_log)
2254		interruptQueue->next_log->prev_log = queueHead;
2255	interruptQueue->next_log = queueHead;
2256	interruptQueue->next_phy = queueHead->this_phy;
2257
2258	Unlock();
2259	return B_OK;
2260}
2261
2262
2263status_t
2264EHCI::UnlinkQueueHead(ehci_qh *queueHead, ehci_qh **freeListHead)
2265{
2266	if (!Lock())
2267		return B_ERROR;
2268
2269	ehci_qh *prevHead = queueHead->prev_log;
2270	ehci_qh *nextHead = queueHead->next_log;
2271	if (prevHead) {
2272		prevHead->next_phy = queueHead->next_phy;
2273		prevHead->next_log = queueHead->next_log;
2274	}
2275
2276	if (nextHead)
2277		nextHead->prev_log = queueHead->prev_log;
2278
2279	queueHead->next_phy = fAsyncQueueHead->this_phy;
2280	queueHead->prev_log = NULL;
2281
2282	queueHead->next_log = *freeListHead;
2283	*freeListHead = queueHead;
2284
2285	Unlock();
2286	return B_OK;
2287}
2288
2289
2290status_t
2291EHCI::FillQueueWithRequest(Transfer *transfer, ehci_qh *queueHead,
2292	ehci_qtd **_dataDescriptor, bool *_directionIn)
2293{
2294	Pipe *pipe = transfer->TransferPipe();
2295	usb_request_data *requestData = transfer->RequestData();
2296	bool directionIn = (requestData->RequestType & USB_REQTYPE_DEVICE_IN) > 0;
2297
2298	ehci_qtd *setupDescriptor = CreateDescriptor(sizeof(usb_request_data),
2299		EHCI_QTD_PID_SETUP);
2300	ehci_qtd *statusDescriptor = CreateDescriptor(0,
2301		directionIn ? EHCI_QTD_PID_OUT : EHCI_QTD_PID_IN);
2302
2303	if (!setupDescriptor || !statusDescriptor) {
2304		TRACE_ERROR("failed to allocate descriptors\n");
2305		FreeDescriptor(setupDescriptor);
2306		FreeDescriptor(statusDescriptor);
2307		return B_NO_MEMORY;
2308	}
2309
2310	iovec vector;
2311	vector.iov_base = requestData;
2312	vector.iov_len = sizeof(usb_request_data);
2313	WriteDescriptorChain(setupDescriptor, &vector, 1);
2314
2315	ehci_qtd *strayDescriptor = queueHead->stray_log;
2316	statusDescriptor->token |= EHCI_QTD_IOC | EHCI_QTD_DATA_TOGGLE;
2317
2318	ehci_qtd *dataDescriptor = NULL;
2319	if (transfer->VectorCount() > 0) {
2320		ehci_qtd *lastDescriptor = NULL;
2321		status_t result = CreateDescriptorChain(pipe, &dataDescriptor,
2322			&lastDescriptor, statusDescriptor, transfer->VectorLength(),
2323			directionIn ? EHCI_QTD_PID_IN : EHCI_QTD_PID_OUT);
2324
2325		if (result != B_OK) {
2326			FreeDescriptor(setupDescriptor);
2327			FreeDescriptor(statusDescriptor);
2328			return result;
2329		}
2330
2331		if (!directionIn) {
2332			WriteDescriptorChain(dataDescriptor, transfer->Vector(),
2333				transfer->VectorCount());
2334		}
2335
2336		LinkDescriptors(setupDescriptor, dataDescriptor, strayDescriptor);
2337		LinkDescriptors(lastDescriptor, statusDescriptor, statusDescriptor);
2338	} else {
2339		// no data: link setup and status descriptors directly
2340		LinkDescriptors(setupDescriptor, statusDescriptor, strayDescriptor);
2341	}
2342
2343	queueHead->element_log = setupDescriptor;
2344	queueHead->overlay.next_phy = setupDescriptor->this_phy;
2345	queueHead->overlay.alt_next_phy = EHCI_ITEM_TERMINATE;
2346
2347	*_dataDescriptor = dataDescriptor;
2348	*_directionIn = directionIn;
2349	return B_OK;
2350}
2351
2352
2353status_t
2354EHCI::FillQueueWithData(Transfer *transfer, ehci_qh *queueHead,
2355	ehci_qtd **_dataDescriptor, bool *_directionIn)
2356{
2357	Pipe *pipe = transfer->TransferPipe();
2358	bool directionIn = (pipe->Direction() == Pipe::In);
2359
2360	ehci_qtd *firstDescriptor = NULL;
2361	ehci_qtd *lastDescriptor = NULL;
2362	ehci_qtd *strayDescriptor = queueHead->stray_log;
2363	status_t result = CreateDescriptorChain(pipe, &firstDescriptor,
2364		&lastDescriptor, strayDescriptor, transfer->VectorLength(),
2365		directionIn ? EHCI_QTD_PID_IN : EHCI_QTD_PID_OUT);
2366
2367	if (result != B_OK)
2368		return result;
2369
2370	lastDescriptor->token |= EHCI_QTD_IOC;
2371	if (!directionIn) {
2372		WriteDescriptorChain(firstDescriptor, transfer->Vector(),
2373			transfer->VectorCount());
2374	}
2375
2376	queueHead->element_log = firstDescriptor;
2377	queueHead->overlay.next_phy = firstDescriptor->this_phy;
2378	queueHead->overlay.alt_next_phy = EHCI_ITEM_TERMINATE;
2379
2380	*_dataDescriptor = firstDescriptor;
2381	if (_directionIn)
2382		*_directionIn = directionIn;
2383	return B_OK;
2384}
2385
2386
2387ehci_qtd *
2388EHCI::CreateDescriptor(size_t bufferSize, uint8 pid)
2389{
2390	ehci_qtd *result;
2391	phys_addr_t physicalAddress;
2392	if (fStack->AllocateChunk((void **)&result, &physicalAddress,
2393			sizeof(ehci_qtd)) != B_OK) {
2394		TRACE_ERROR("failed to allocate a qtd\n");
2395		return NULL;
2396	}
2397
2398	result->this_phy = (addr_t)physicalAddress;
2399	result->next_phy = EHCI_ITEM_TERMINATE;
2400	result->next_log = NULL;
2401	result->alt_next_phy = EHCI_ITEM_TERMINATE;
2402	result->alt_next_log = NULL;
2403	result->buffer_size = bufferSize;
2404	result->token = bufferSize << EHCI_QTD_BYTES_SHIFT;
2405	result->token |= 3 << EHCI_QTD_ERRCOUNT_SHIFT;
2406	result->token |= pid << EHCI_QTD_PID_SHIFT;
2407	result->token |= EHCI_QTD_STATUS_ACTIVE;
2408	if (bufferSize == 0) {
2409		result->buffer_log = NULL;
2410		for (int32 i = 0; i < 5; i++) {
2411			result->buffer_phy[i] = 0;
2412			result->ext_buffer_phy[i] = 0;
2413		}
2414
2415		return result;
2416	}
2417
2418	if (fStack->AllocateChunk(&result->buffer_log, &physicalAddress,
2419			bufferSize) != B_OK) {
2420		TRACE_ERROR("unable to allocate qtd buffer\n");
2421		fStack->FreeChunk(result, (phys_addr_t)result->this_phy,
2422			sizeof(ehci_qtd));
2423		return NULL;
2424	}
2425
2426	addr_t physicalBase = (addr_t)physicalAddress;
2427	result->buffer_phy[0] = physicalBase;
2428	result->ext_buffer_phy[0] = 0;
2429	for (int32 i = 1; i < 5; i++) {
2430		physicalBase += B_PAGE_SIZE;
2431		result->buffer_phy[i] = physicalBase & EHCI_QTD_PAGE_MASK;
2432		result->ext_buffer_phy[i] = 0;
2433	}
2434
2435	return result;
2436}
2437
2438
2439status_t
2440EHCI::CreateDescriptorChain(Pipe *pipe, ehci_qtd **_firstDescriptor,
2441	ehci_qtd **_lastDescriptor, ehci_qtd *strayDescriptor, size_t bufferSize,
2442	uint8 pid)
2443{
2444	size_t packetSize = B_PAGE_SIZE * 4;
2445	int32 descriptorCount = (bufferSize + packetSize - 1) / packetSize;
2446
2447	bool dataToggle = pipe->DataToggle();
2448	ehci_qtd *firstDescriptor = NULL;
2449	ehci_qtd *lastDescriptor = *_firstDescriptor;
2450	for (int32 i = 0; i < descriptorCount; i++) {
2451		ehci_qtd *descriptor = CreateDescriptor(min_c(packetSize, bufferSize),
2452			pid);
2453
2454		if (!descriptor) {
2455			FreeDescriptorChain(firstDescriptor);
2456			return B_NO_MEMORY;
2457		}
2458
2459		if (dataToggle)
2460			descriptor->token |= EHCI_QTD_DATA_TOGGLE;
2461
2462		if (lastDescriptor)
2463			LinkDescriptors(lastDescriptor, descriptor, strayDescriptor);
2464
2465		bufferSize -= packetSize;
2466		lastDescriptor = descriptor;
2467		if (!firstDescriptor)
2468			firstDescriptor = descriptor;
2469	}
2470
2471	*_firstDescriptor = firstDescriptor;
2472	*_lastDescriptor = lastDescriptor;
2473	return B_OK;
2474}
2475
2476
2477void
2478EHCI::FreeDescriptor(ehci_qtd *descriptor)
2479{
2480	if (!descriptor)
2481		return;
2482
2483	if (descriptor->buffer_log) {
2484		fStack->FreeChunk(descriptor->buffer_log,
2485			(phys_addr_t)descriptor->buffer_phy[0], descriptor->buffer_size);
2486	}
2487
2488	fStack->FreeChunk(descriptor, (phys_addr_t)descriptor->this_phy,
2489		sizeof(ehci_qtd));
2490}
2491
2492
2493void
2494EHCI::FreeDescriptorChain(ehci_qtd *topDescriptor)
2495{
2496	ehci_qtd *current = topDescriptor;
2497	ehci_qtd *next = NULL;
2498
2499	while (current) {
2500		next = current->next_log;
2501		FreeDescriptor(current);
2502		current = next;
2503	}
2504}
2505
2506
2507ehci_itd *
2508EHCI::CreateItdDescriptor()
2509{
2510	ehci_itd *result;
2511	phys_addr_t physicalAddress;
2512	if (fStack->AllocateChunk((void **)&result, &physicalAddress,
2513			sizeof(ehci_itd)) != B_OK) {
2514		TRACE_ERROR("failed to allocate a itd\n");
2515		return NULL;
2516	}
2517
2518	memset(result, 0, sizeof(ehci_itd));
2519	result->this_phy = (addr_t)physicalAddress;
2520	result->next_phy = EHCI_ITEM_TERMINATE;
2521
2522	return result;
2523}
2524
2525
2526ehci_sitd *
2527EHCI::CreateSitdDescriptor()
2528{
2529	ehci_sitd *result;
2530	phys_addr_t physicalAddress;
2531	if (fStack->AllocateChunk((void **)&result, &physicalAddress,
2532			sizeof(ehci_sitd)) != B_OK) {
2533		TRACE_ERROR("failed to allocate a sitd\n");
2534		return NULL;
2535	}
2536
2537	memset(result, 0, sizeof(ehci_sitd));
2538	result->this_phy = (addr_t)physicalAddress | EHCI_ITEM_TYPE_SITD;
2539	result->next_phy = EHCI_ITEM_TERMINATE;
2540
2541	return result;
2542}
2543
2544
2545void
2546EHCI::FreeDescriptor(ehci_itd *descriptor)
2547{
2548	if (!descriptor)
2549		return;
2550
2551	fStack->FreeChunk(descriptor, (phys_addr_t)descriptor->this_phy,
2552		sizeof(ehci_itd));
2553}
2554
2555
2556void
2557EHCI::FreeDescriptor(ehci_sitd *descriptor)
2558{
2559	if (!descriptor)
2560		return;
2561
2562	fStack->FreeChunk(descriptor, (phys_addr_t)descriptor->this_phy,
2563		sizeof(ehci_sitd));
2564}
2565
2566
2567void
2568EHCI::LinkDescriptors(ehci_qtd *first, ehci_qtd *last, ehci_qtd *alt)
2569{
2570	first->next_phy = last->this_phy;
2571	first->next_log = last;
2572
2573	if (alt) {
2574		first->alt_next_phy = alt->this_phy;
2575		first->alt_next_log = alt;
2576	} else {
2577		first->alt_next_phy = EHCI_ITEM_TERMINATE;
2578		first->alt_next_log = NULL;
2579	}
2580}
2581
2582
2583void
2584EHCI::LinkITDescriptors(ehci_itd *itd, ehci_itd **_last)
2585{
2586	ehci_itd *last = *_last;
2587	itd->next_phy = last->next_phy;
2588	itd->next = NULL;
2589	itd->prev = last;
2590	last->next = itd;
2591	last->next_phy = itd->this_phy;
2592	*_last = itd;
2593}
2594
2595
2596void
2597EHCI::LinkSITDescriptors(ehci_sitd *sitd, ehci_sitd **_last)
2598{
2599	ehci_sitd *last = *_last;
2600	sitd->next_phy = last->next_phy;
2601	sitd->next = NULL;
2602	sitd->prev = last;
2603	last->next = sitd;
2604	last->next_phy = sitd->this_phy;
2605	*_last = sitd;
2606}
2607
2608
2609void
2610EHCI::UnlinkITDescriptors(ehci_itd *itd, ehci_itd **last)
2611{
2612	itd->prev->next_phy = itd->next_phy;
2613	itd->prev->next = itd->next;
2614	if (itd->next != NULL)
2615		itd->next->prev = itd->prev;
2616	if (itd == *last)
2617		*last = itd->prev;
2618}
2619
2620
2621void
2622EHCI::UnlinkSITDescriptors(ehci_sitd *sitd, ehci_sitd **last)
2623{
2624	sitd->prev->next_phy = sitd->next_phy;
2625	sitd->prev->next = sitd->next;
2626	if (sitd->next != NULL)
2627		sitd->next->prev = sitd->prev;
2628	if (sitd == *last)
2629		*last = sitd->prev;
2630}
2631
2632
2633size_t
2634EHCI::WriteDescriptorChain(ehci_qtd *topDescriptor, iovec *vector,
2635	size_t vectorCount)
2636{
2637	ehci_qtd *current = topDescriptor;
2638	size_t actualLength = 0;
2639	size_t vectorIndex = 0;
2640	size_t vectorOffset = 0;
2641	size_t bufferOffset = 0;
2642
2643	while (current) {
2644		if (!current->buffer_log)
2645			break;
2646
2647		while (true) {
2648			size_t length = min_c(current->buffer_size - bufferOffset,
2649				vector[vectorIndex].iov_len - vectorOffset);
2650
2651			memcpy((uint8 *)current->buffer_log + bufferOffset,
2652				(uint8 *)vector[vectorIndex].iov_base + vectorOffset, length);
2653
2654			actualLength += length;
2655			vectorOffset += length;
2656			bufferOffset += length;
2657
2658			if (vectorOffset >= vector[vectorIndex].iov_len) {
2659				if (++vectorIndex >= vectorCount) {
2660					TRACE("wrote descriptor chain (%ld bytes, no more vectors)"
2661						"\n", actualLength);
2662					return actualLength;
2663				}
2664
2665				vectorOffset = 0;
2666			}
2667
2668			if (bufferOffset >= current->buffer_size) {
2669				bufferOffset = 0;
2670				break;
2671			}
2672		}
2673
2674		if (current->next_phy & EHCI_ITEM_TERMINATE)
2675			break;
2676
2677		current = current->next_log;
2678	}
2679
2680	TRACE("wrote descriptor chain (%ld bytes)\n", actualLength);
2681	return actualLength;
2682}
2683
2684
2685size_t
2686EHCI::ReadDescriptorChain(ehci_qtd *topDescriptor, iovec *vector,
2687	size_t vectorCount, bool *nextDataToggle)
2688{
2689	uint32 dataToggle = 0;
2690	ehci_qtd *current = topDescriptor;
2691	size_t actualLength = 0;
2692	size_t vectorIndex = 0;
2693	size_t vectorOffset = 0;
2694	size_t bufferOffset = 0;
2695
2696	while (current && (current->token & EHCI_QTD_STATUS_ACTIVE) == 0) {
2697		if (!current->buffer_log)
2698			break;
2699
2700		dataToggle = current->token & EHCI_QTD_DATA_TOGGLE;
2701		size_t bufferSize = current->buffer_size;
2702		bufferSize -= (current->token >> EHCI_QTD_BYTES_SHIFT)
2703			& EHCI_QTD_BYTES_MASK;
2704
2705		while (true) {
2706			size_t length = min_c(bufferSize - bufferOffset,
2707				vector[vectorIndex].iov_len - vectorOffset);
2708
2709			memcpy((uint8 *)vector[vectorIndex].iov_base + vectorOffset,
2710				(uint8 *)current->buffer_log + bufferOffset, length);
2711
2712			actualLength += length;
2713			vectorOffset += length;
2714			bufferOffset += length;
2715
2716			if (vectorOffset >= vector[vectorIndex].iov_len) {
2717				if (++vectorIndex >= vectorCount) {
2718					TRACE("read descriptor chain (%ld bytes, no more vectors)"
2719						"\n", actualLength);
2720					*nextDataToggle = dataToggle > 0 ? true : false;
2721					return actualLength;
2722				}
2723
2724				vectorOffset = 0;
2725			}
2726
2727			if (bufferOffset >= bufferSize) {
2728				bufferOffset = 0;
2729				break;
2730			}
2731		}
2732
2733		if (current->next_phy & EHCI_ITEM_TERMINATE)
2734			break;
2735
2736		current = current->next_log;
2737	}
2738
2739	TRACE("read descriptor chain (%ld bytes)\n", actualLength);
2740	*nextDataToggle = dataToggle > 0 ? true : false;
2741	return actualLength;
2742}
2743
2744
2745size_t
2746EHCI::ReadActualLength(ehci_qtd *topDescriptor, bool *nextDataToggle)
2747{
2748	size_t actualLength = 0;
2749	ehci_qtd *current = topDescriptor;
2750	uint32 dataToggle = 0;
2751
2752	while (current && (current->token & EHCI_QTD_STATUS_ACTIVE) == 0) {
2753		dataToggle = current->token & EHCI_QTD_DATA_TOGGLE;
2754		size_t length = current->buffer_size;
2755		length -= (current->token >> EHCI_QTD_BYTES_SHIFT)
2756			& EHCI_QTD_BYTES_MASK;
2757		actualLength += length;
2758
2759		if (current->next_phy & EHCI_ITEM_TERMINATE)
2760			break;
2761
2762		current = current->next_log;
2763	}
2764
2765	TRACE("read actual length (%ld bytes)\n", actualLength);
2766	*nextDataToggle = dataToggle > 0 ? true : false;
2767	return actualLength;
2768}
2769
2770
2771size_t
2772EHCI::WriteIsochronousDescriptorChain(isochronous_transfer_data *transfer,
2773	uint32 packetCount,	iovec *vector)
2774{
2775	// TODO implement
2776	return 0;
2777}
2778
2779
2780size_t
2781EHCI::ReadIsochronousDescriptorChain(isochronous_transfer_data *transfer)
2782{
2783	iovec *vector = transfer->transfer->Vector();
2784	size_t vectorCount = transfer->transfer->VectorCount();
2785	size_t vectorOffset = 0;
2786	size_t vectorIndex = 0;
2787	usb_isochronous_data *isochronousData
2788		= transfer->transfer->IsochronousData();
2789	uint32 packet = 0;
2790	size_t totalLength = 0;
2791	size_t bufferOffset = 0;
2792
2793	size_t packetSize = transfer->transfer->DataLength();
2794	packetSize /= isochronousData->packet_count;
2795
2796	for (uint32 i = 0; i <= transfer->last_to_process; i++) {
2797		ehci_itd *itd = transfer->descriptors[i];
2798		for (uint32 j = 0; j <= itd->last_token
2799			&& packet < isochronousData->packet_count; j++) {
2800
2801			size_t bufferSize = (itd->token[j] >> EHCI_ITD_TLENGTH_SHIFT)
2802				& EHCI_ITD_TLENGTH_MASK;
2803			if (((itd->token[j] >> EHCI_ITD_STATUS_SHIFT)
2804				& EHCI_ITD_STATUS_MASK) != 0) {
2805				bufferSize = 0;
2806			}
2807			isochronousData->packet_descriptors[packet].actual_length
2808				= bufferSize;
2809
2810			if (bufferSize > 0)
2811				isochronousData->packet_descriptors[packet].status = B_OK;
2812			else
2813				isochronousData->packet_descriptors[packet].status = B_ERROR;
2814
2815			totalLength += bufferSize;
2816
2817			size_t offset = bufferOffset;
2818			size_t skipSize = packetSize - bufferSize;
2819			while (bufferSize > 0) {
2820				size_t length = min_c(bufferSize,
2821					vector[vectorIndex].iov_len - vectorOffset);
2822				memcpy((uint8 *)vector[vectorIndex].iov_base + vectorOffset,
2823					(uint8 *)transfer->buffer_log + bufferOffset, length);
2824				offset += length;
2825				vectorOffset += length;
2826				bufferSize -= length;
2827
2828				if (vectorOffset >= vector[vectorIndex].iov_len) {
2829					if (++vectorIndex >= vectorCount) {
2830						TRACE("read isodescriptor chain (%ld bytes, no more "
2831							"vectors)\n", totalLength);
2832						return totalLength;
2833					}
2834
2835					vectorOffset = 0;
2836				}
2837			}
2838
2839			// skip to next packet offset
2840			while (skipSize > 0) {
2841				size_t length = min_c(skipSize,
2842					vector[vectorIndex].iov_len - vectorOffset);
2843				vectorOffset += length;
2844				skipSize -= length;
2845				if (vectorOffset >= vector[vectorIndex].iov_len) {
2846					if (++vectorIndex >= vectorCount) {
2847						TRACE("read isodescriptor chain (%ld bytes, no more "
2848							"vectors)\n", totalLength);
2849						return totalLength;
2850					}
2851
2852					vectorOffset = 0;
2853				}
2854			}
2855
2856			bufferOffset += packetSize;
2857			if (bufferOffset >= transfer->buffer_size)
2858				return totalLength;
2859
2860			packet++;
2861		}
2862	}
2863
2864	TRACE("ReadIsochronousDescriptorChain packet count %" B_PRId32 "\n",
2865		packet);
2866
2867	return totalLength;
2868}
2869
2870
2871bool
2872EHCI::LockIsochronous()
2873{
2874	return (mutex_lock(&fIsochronousLock) == B_OK);
2875}
2876
2877
2878void
2879EHCI::UnlockIsochronous()
2880{
2881	mutex_unlock(&fIsochronousLock);
2882}
2883
2884
2885inline void
2886EHCI::WriteOpReg(uint32 reg, uint32 value)
2887{
2888	*(volatile uint32 *)(fOperationalRegisters + reg) = value;
2889}
2890
2891
2892inline uint32
2893EHCI::ReadOpReg(uint32 reg)
2894{
2895	return *(volatile uint32 *)(fOperationalRegisters + reg);
2896}
2897
2898
2899inline uint8
2900EHCI::ReadCapReg8(uint32 reg)
2901{
2902	return *(volatile uint8 *)(fCapabilityRegisters + reg);
2903}
2904
2905
2906inline uint16
2907EHCI::ReadCapReg16(uint32 reg)
2908{
2909	return *(volatile uint16 *)(fCapabilityRegisters + reg);
2910}
2911
2912
2913inline uint32
2914EHCI::ReadCapReg32(uint32 reg)
2915{
2916	return *(volatile uint32 *)(fCapabilityRegisters + reg);
2917}
2918