1/*-
2 * Copyright (c) 1997, 1998 Justin T. Gibbs.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions, and the following disclaimer,
10 *    without modification, immediately at the beginning of the file.
11 * 2. The name of the author may not be used to endorse or promote products
12 *    derived from this software without specific prior written permission.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
18 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26#include <sys/cdefs.h>
27__FBSDID("$FreeBSD: src/sys/i386/i386/busdma_machdep.c,v 1.74.2.4 2006/10/21 16:26:53 hrs Exp $");
28
29#include <sys/param.h>
30#include <sys/systm.h>
31#include <sys/malloc.h>
32#include <sys/bus.h>
33#include <sys/kernel.h>
34#include <sys/ktr.h>
35#include <sys/lock.h>
36#include <sys/mutex.h>
37#include <sys/mbuf.h>
38#include <sys/uio.h>
39#include <sys/sysctl.h>
40
41#include <vm/vm.h>
42
43#include <machine/atomic.h>
44#include <machine/bus.h>
45
46#ifdef __x86_64__
47#	define MAX_BPAGES 8192
48#else
49#	define MAX_BPAGES 512
50#endif
51
52/* -hugo */
53#define malloc(a, b, c)						kernel_malloc(a, b, c)
54#define free(a, b)							kernel_free(a, b)
55#define contigmalloc(a, b, c, d, e, f, g)	kernel_contigmalloc(a, b, c, d, e, f, g)
56#define contigfree(a, b, c)					kernel_contigfree(a, b, c)
57void busdma_swi(void);
58void init_bounce_pages(void);
59void uninit_bounce_pages(void);
60/* </> */
61
62struct bounce_zone;
63
64struct bus_dma_tag {
65	bus_dma_tag_t	  parent;
66	bus_size_t	  alignment;
67	bus_addr_t	  boundary;
68	bus_addr_t	  lowaddr;
69	bus_addr_t	  highaddr;
70	bus_dma_filter_t *filter;
71	void		 *filterarg;
72	bus_size_t	  maxsize;
73	u_int		  nsegments;
74	bus_size_t	  maxsegsz;
75	int		  flags;
76	int		  ref_count;
77	int		  map_count;
78	bus_dma_lock_t	 *lockfunc;
79	void		 *lockfuncarg;
80	bus_dma_segment_t *segments;
81	struct bounce_zone *bounce_zone;
82};
83
84struct bounce_page {
85	vm_offset_t	vaddr;		/* kva of bounce buffer */
86	bus_addr_t	busaddr;	/* Physical address */
87	vm_offset_t	datavaddr;	/* kva of client data */
88	bus_size_t	datacount;	/* client data count */
89	STAILQ_ENTRY(bounce_page) links;
90};
91
92int busdma_swi_pending;
93
94struct bounce_zone {
95	STAILQ_ENTRY(bounce_zone) links;
96	STAILQ_HEAD(bp_list, bounce_page) bounce_page_list;
97	int		total_bpages;
98	int		free_bpages;
99	int		reserved_bpages;
100	int		active_bpages;
101	int		total_bounced;
102	int		total_deferred;
103	bus_size_t	alignment;
104	bus_addr_t	boundary;
105	bus_addr_t	lowaddr;
106	char		zoneid[8];
107	char		lowaddrid[20];
108	struct sysctl_ctx_list sysctl_tree;
109	struct sysctl_oid *sysctl_tree_top;
110};
111
112static struct mtx bounce_lock;
113static int total_bpages;
114static int busdma_zonecount;
115static STAILQ_HEAD(, bounce_zone) bounce_zone_list;
116
117SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD, 0, "Busdma parameters");
118SYSCTL_INT(_hw_busdma, OID_AUTO, total_bpages, CTLFLAG_RD, &total_bpages, 0,
119	   "Total bounce pages");
120
121struct bus_dmamap {
122	struct bp_list	       bpages;
123	int		       pagesneeded;
124	int		       pagesreserved;
125	bus_dma_tag_t	       dmat;
126	void		      *buf;		/* unmapped buffer pointer */
127	bus_size_t	       buflen;		/* unmapped buffer length */
128	bus_dmamap_callback_t *callback;
129	void		      *callback_arg;
130	STAILQ_ENTRY(bus_dmamap) links;
131};
132
133static STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist;
134static STAILQ_HEAD(, bus_dmamap) bounce_map_callbacklist;
135static struct bus_dmamap nobounce_dmamap;
136
137static int alloc_bounce_zone(bus_dma_tag_t dmat);
138static int alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages);
139static int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map,
140				int commit);
141static bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map,
142				   vm_offset_t vaddr, bus_size_t size);
143static void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage);
144static __inline int run_filter(bus_dma_tag_t dmat, bus_addr_t paddr);
145
146/*
147 * Return true if a match is made.
148 *
149 * To find a match walk the chain of bus_dma_tag_t's looking for 'paddr'.
150 *
151 * If paddr is within the bounds of the dma tag then call the filter callback
152 * to check for a match, if there is no filter callback then assume a match.
153 */
154static __inline
155int
156run_filter(bus_dma_tag_t dmat, bus_addr_t paddr)
157{
158	int retval;
159
160	retval = 0;
161
162	do {
163		if (((paddr > dmat->lowaddr && paddr <= dmat->highaddr)
164		 || ((paddr & (dmat->alignment - 1)) != 0))
165		 && (dmat->filter == NULL
166		  || (*dmat->filter)(dmat->filterarg, paddr) != 0))
167			retval = 1;
168
169		dmat = dmat->parent;
170	} while (retval == 0 && dmat != NULL);
171	return (retval);
172}
173
174/*
175 * Convenience function for manipulating driver locks from busdma (during
176 * busdma_swi, for example).  Drivers that don't provide their own locks
177 * should specify &Giant to dmat->lockfuncarg.  Drivers that use their own
178 * non-mutex locking scheme don't have to use this at all.
179 */
180void
181busdma_lock_mutex(void *arg, bus_dma_lock_op_t op)
182{
183	struct mtx *dmtx;
184
185	dmtx = (struct mtx *)arg;
186	switch (op) {
187	case BUS_DMA_LOCK:
188		mtx_lock(dmtx);
189		break;
190	case BUS_DMA_UNLOCK:
191		mtx_unlock(dmtx);
192		break;
193	default:
194		panic("Unknown operation 0x%x for busdma_lock_mutex!", op);
195	}
196}
197
198/*
199 * dflt_lock should never get called.  It gets put into the dma tag when
200 * lockfunc == NULL, which is only valid if the maps that are associated
201 * with the tag are meant to never be defered.
202 * XXX Should have a way to identify which driver is responsible here.
203 */
204static
205void
206dflt_lock(void *arg, bus_dma_lock_op_t op)
207{
208	panic("driver error: busdma dflt_lock called");
209}
210
211#define BUS_DMA_COULD_BOUNCE	BUS_DMA_BUS3
212#define BUS_DMA_MIN_ALLOC_COMP	BUS_DMA_BUS4
213
214/*
215 * Allocate a device specific dma_tag.
216 */
217int
218bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment,
219		   bus_size_t boundary, bus_addr_t lowaddr,
220		   bus_addr_t highaddr, bus_dma_filter_t *filter,
221		   void *filterarg, bus_size_t maxsize, int nsegments,
222		   bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc,
223		   void *lockfuncarg, bus_dma_tag_t *dmat)
224{
225	bus_dma_tag_t newtag;
226	int error = 0;
227
228	/* Basic sanity checking */
229	if (boundary != 0 && boundary < maxsegsz)
230		maxsegsz = boundary;
231
232	/* Return a NULL tag on failure */
233	*dmat = NULL;
234
235	newtag = (bus_dma_tag_t)malloc(sizeof(*newtag), M_DEVBUF,
236	    M_ZERO | M_NOWAIT);
237	if (newtag == NULL) {
238		CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d",
239		    __func__, newtag, 0, error);
240		return (ENOMEM);
241	}
242
243	newtag->parent = parent;
244	newtag->alignment = alignment;
245	newtag->boundary = boundary;
246	newtag->lowaddr = trunc_page((vm_paddr_t)lowaddr) + (PAGE_SIZE - 1);
247	newtag->highaddr = trunc_page((vm_paddr_t)highaddr) +
248	    (PAGE_SIZE - 1);
249	newtag->filter = filter;
250	newtag->filterarg = filterarg;
251	newtag->maxsize = maxsize;
252	newtag->nsegments = nsegments;
253	newtag->maxsegsz = maxsegsz;
254	newtag->flags = flags;
255	newtag->ref_count = 1; /* Count ourself */
256	newtag->map_count = 0;
257	if (lockfunc != NULL) {
258		newtag->lockfunc = lockfunc;
259		newtag->lockfuncarg = lockfuncarg;
260	} else {
261		newtag->lockfunc = dflt_lock;
262		newtag->lockfuncarg = NULL;
263	}
264	newtag->segments = NULL;
265
266	/* Take into account any restrictions imposed by our parent tag */
267	if (parent != NULL) {
268		newtag->lowaddr = MIN(parent->lowaddr, newtag->lowaddr);
269		newtag->highaddr = MAX(parent->highaddr, newtag->highaddr);
270		if (newtag->boundary == 0)
271			newtag->boundary = parent->boundary;
272		else if (parent->boundary != 0)
273			newtag->boundary = MIN(parent->boundary,
274					       newtag->boundary);
275		if (newtag->filter == NULL) {
276			/*
277			 * Short circuit looking at our parent directly
278			 * since we have encapsulated all of its information
279			 */
280			newtag->filter = parent->filter;
281			newtag->filterarg = parent->filterarg;
282			newtag->parent = parent->parent;
283		}
284		if (newtag->parent != NULL)
285			atomic_add_int(&parent->ref_count, 1);
286	}
287
288	if (newtag->lowaddr < ptoa((vm_paddr_t)Maxmem)
289	 || newtag->alignment > 1)
290		newtag->flags |= BUS_DMA_COULD_BOUNCE;
291
292	if (((newtag->flags & BUS_DMA_COULD_BOUNCE) != 0) &&
293	    (flags & BUS_DMA_ALLOCNOW) != 0) {
294		struct bounce_zone *bz;
295
296		/* Must bounce */
297
298		if ((error = alloc_bounce_zone(newtag)) != 0) {
299			free(newtag, M_DEVBUF);
300			return (error);
301		}
302		bz = newtag->bounce_zone;
303
304		if (ptoa(bz->total_bpages) < maxsize) {
305			int pages;
306
307			pages = atop(maxsize) - bz->total_bpages;
308
309			/* Add pages to our bounce pool */
310			if (alloc_bounce_pages(newtag, pages) < pages)
311				error = ENOMEM;
312		}
313		/* Performed initial allocation */
314		newtag->flags |= BUS_DMA_MIN_ALLOC_COMP;
315	}
316
317	if (error != 0) {
318		free(newtag, M_DEVBUF);
319	} else {
320		*dmat = newtag;
321	}
322	CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d",
323	    __func__, newtag, (newtag != NULL ? newtag->flags : 0), error);
324	return (error);
325}
326
327int
328bus_dma_tag_destroy(bus_dma_tag_t dmat)
329{
330	bus_dma_tag_t dmat_copy;
331	int error;
332
333	error = 0;
334	dmat_copy = dmat;
335
336	if (dmat != NULL) {
337
338		if (dmat->map_count != 0) {
339			error = EBUSY;
340			goto out;
341		}
342
343		while (dmat != NULL) {
344			bus_dma_tag_t parent;
345
346			parent = dmat->parent;
347			atomic_subtract_int(&dmat->ref_count, 1);
348			if (dmat->ref_count == 0) {
349				if (dmat->segments != NULL)
350					free(dmat->segments, M_DEVBUF);
351				free(dmat, M_DEVBUF);
352				/*
353				 * Last reference count, so
354				 * release our reference
355				 * count on our parent.
356				 */
357				dmat = parent;
358			} else
359				dmat = NULL;
360		}
361	}
362out:
363	CTR3(KTR_BUSDMA, "%s tag %p error %d", __func__, dmat_copy, error);
364	return (error);
365}
366
367/*
368 * Allocate a handle for mapping from kva/uva/physical
369 * address space into bus device space.
370 */
371int
372bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp)
373{
374	int error;
375
376	error = 0;
377
378	if (dmat->segments == NULL) {
379		dmat->segments = (bus_dma_segment_t *)malloc(
380		    sizeof(bus_dma_segment_t) * dmat->nsegments, M_DEVBUF,
381		    M_NOWAIT);
382		if (dmat->segments == NULL) {
383			CTR3(KTR_BUSDMA, "%s: tag %p error %d",
384			    __func__, dmat, ENOMEM);
385			return (ENOMEM);
386		}
387	}
388
389	/*
390	 * Bouncing might be required if the driver asks for an active
391	 * exclusion region, a data alignment that is stricter than 1, and/or
392	 * an active address boundary.
393	 */
394	if (dmat->flags & BUS_DMA_COULD_BOUNCE) {
395
396		/* Must bounce */
397		struct bounce_zone *bz;
398		int maxpages;
399
400		if (dmat->bounce_zone == NULL) {
401			if ((error = alloc_bounce_zone(dmat)) != 0)
402				return (error);
403		}
404		bz = dmat->bounce_zone;
405
406		*mapp = (bus_dmamap_t)malloc(sizeof(**mapp), M_DEVBUF,
407					     M_NOWAIT | M_ZERO);
408		if (*mapp == NULL) {
409			CTR3(KTR_BUSDMA, "%s: tag %p error %d",
410			    __func__, dmat, ENOMEM);
411			return (ENOMEM);
412		}
413
414		/* Initialize the new map */
415		STAILQ_INIT(&((*mapp)->bpages));
416
417		/*
418		 * Attempt to add pages to our pool on a per-instance
419		 * basis up to a sane limit.
420		 */
421		if (dmat->alignment > 1)
422			maxpages = MAX_BPAGES;
423		else
424			maxpages = MIN(MAX_BPAGES, Maxmem -atop(dmat->lowaddr));
425		if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0
426		 || (dmat->map_count > 0 && bz->total_bpages < maxpages)) {
427			int pages;
428
429			pages = MAX(atop(dmat->maxsize), 1);
430			pages = MIN(maxpages - bz->total_bpages, pages);
431			pages = MAX(pages, 1);
432			if (alloc_bounce_pages(dmat, pages) < pages)
433				error = ENOMEM;
434
435			if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0) {
436				if (error == 0)
437					dmat->flags |= BUS_DMA_MIN_ALLOC_COMP;
438			} else {
439				error = 0;
440			}
441		}
442	} else {
443		*mapp = NULL;
444	}
445	if (error == 0)
446		dmat->map_count++;
447	CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
448	    __func__, dmat, dmat->flags, error);
449	return (error);
450}
451
452/*
453 * Destroy a handle for mapping from kva/uva/physical
454 * address space into bus device space.
455 */
456int
457bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map)
458{
459	if (map != NULL && map != &nobounce_dmamap) {
460		if (STAILQ_FIRST(&map->bpages) != NULL) {
461			CTR3(KTR_BUSDMA, "%s: tag %p error %d",
462			    __func__, dmat, EBUSY);
463			return (EBUSY);
464		}
465		free(map, M_DEVBUF);
466	}
467	dmat->map_count--;
468	CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat);
469	return (0);
470}
471
472/*
473 * Allocate a piece of memory that can be efficiently mapped into
474 * bus device space based on the constraints lited in the dma tag.
475 * A dmamap to for use with dmamap_load is also allocated.
476 */
477int
478bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags,
479		 bus_dmamap_t *mapp)
480{
481	int mflags;
482
483	if (flags & BUS_DMA_NOWAIT)
484		mflags = M_NOWAIT;
485	else
486		mflags = M_WAITOK;
487	if (flags & BUS_DMA_ZERO)
488		mflags |= M_ZERO;
489
490	/* If we succeed, no mapping/bouncing will be required */
491	*mapp = NULL;
492
493	if (dmat->segments == NULL) {
494		dmat->segments = (bus_dma_segment_t *)malloc(
495		    sizeof(bus_dma_segment_t) * dmat->nsegments, M_DEVBUF,
496		    M_NOWAIT);
497		if (dmat->segments == NULL) {
498			CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
499			    __func__, dmat, dmat->flags, ENOMEM);
500			return (ENOMEM);
501		}
502	}
503
504	/*
505	 * XXX:
506	 * (dmat->alignment < dmat->maxsize) is just a quick hack; the exact
507	 * alignment guarantees of malloc need to be nailed down, and the
508	 * code below should be rewritten to take that into account.
509	 *
510	 * In the meantime, we'll warn the user if malloc gets it wrong.
511	 */
512	if ((dmat->maxsize <= PAGE_SIZE) &&
513	   (dmat->alignment < dmat->maxsize) &&
514	    dmat->lowaddr >= ptoa((vm_paddr_t)Maxmem)) {
515		*vaddr = malloc(dmat->maxsize, M_DEVBUF, mflags);
516	} else {
517		/*
518		 * XXX Use Contigmalloc until it is merged into this facility
519		 *     and handles multi-seg allocations.  Nobody is doing
520		 *     multi-seg allocations yet though.
521		 * XXX Certain AGP hardware does.
522		 */
523		*vaddr = contigmalloc(dmat->maxsize, M_DEVBUF, mflags,
524		    0ul, dmat->lowaddr, dmat->alignment? dmat->alignment : 1ul,
525		    dmat->boundary);
526	}
527	if (*vaddr == NULL) {
528		CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
529		    __func__, dmat, dmat->flags, ENOMEM);
530		return (ENOMEM);
531	} else if (vtophys(*vaddr) & (dmat->alignment - 1)) {
532		printf("bus_dmamem_alloc failed to align memory: wanted %#x, got %#x\n",
533			dmat->alignment, vtophys(vaddr));
534	}
535	CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
536	    __func__, dmat, dmat->flags, 0);
537	return (0);
538}
539
540/*
541 * Free a piece of memory and it's allociated dmamap, that was allocated
542 * via bus_dmamem_alloc.  Make the same choice for free/contigfree.
543 */
544void
545bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map)
546{
547	/*
548	 * dmamem does not need to be bounced, so the map should be
549	 * NULL
550	 */
551	if (map != NULL)
552		panic("bus_dmamem_free: Invalid map freed\n");
553	if ((dmat->maxsize <= PAGE_SIZE) &&
554	   (dmat->alignment < dmat->maxsize) &&
555	    dmat->lowaddr >= ptoa((vm_paddr_t)Maxmem))
556		free(vaddr, M_DEVBUF);
557	else {
558		contigfree(vaddr, dmat->maxsize, M_DEVBUF);
559	}
560	CTR3(KTR_BUSDMA, "%s: tag %p flags 0x%x", __func__, dmat, dmat->flags);
561}
562
563/*
564 * Utility function to load a linear buffer.  lastaddrp holds state
565 * between invocations (for multiple-buffer loads).  segp contains
566 * the starting segment on entrace, and the ending segment on exit.
567 * first indicates if this is the first invocation of this function.
568 */
569static __inline int
570_bus_dmamap_load_buffer(bus_dma_tag_t dmat,
571    			bus_dmamap_t map,
572			void *buf, bus_size_t buflen,
573			pmap_t pmap,
574			int flags,
575			bus_addr_t *lastaddrp,
576			bus_dma_segment_t *segs,
577			int *segp,
578			int first)
579{
580	bus_size_t sgsize;
581	bus_addr_t curaddr, lastaddr, baddr, bmask;
582	vm_offset_t vaddr;
583	bus_addr_t paddr;
584	int needbounce = 0;
585	int seg;
586
587	if (map == NULL)
588		map = &nobounce_dmamap;
589
590	if ((map != &nobounce_dmamap && map->pagesneeded == 0)
591	 && ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0)) {
592		vm_offset_t	vendaddr;
593
594		CTR4(KTR_BUSDMA, "lowaddr= %d Maxmem= %d, boundary= %d, "
595		    "alignment= %d", dmat->lowaddr, ptoa((vm_paddr_t)Maxmem),
596		    dmat->boundary, dmat->alignment);
597		CTR3(KTR_BUSDMA, "map= %p, nobouncemap= %p, pagesneeded= %d",
598		    map, &nobounce_dmamap, map->pagesneeded);
599		/*
600		 * Count the number of bounce pages
601		 * needed in order to complete this transfer
602		 */
603		vaddr = trunc_page((vm_offset_t)buf);
604		vendaddr = (vm_offset_t)buf + buflen;
605
606		while (vaddr < vendaddr) {
607			paddr = pmap_kextract(vaddr);
608			if (run_filter(dmat, paddr) != 0) {
609				needbounce = 1;
610				map->pagesneeded++;
611			}
612			vaddr += PAGE_SIZE;
613		}
614		CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded);
615	}
616
617	/* Reserve Necessary Bounce Pages */
618	if (map->pagesneeded != 0) {
619		mtx_lock(&bounce_lock);
620		if (flags & BUS_DMA_NOWAIT) {
621			if (reserve_bounce_pages(dmat, map, 0) != 0) {
622				mtx_unlock(&bounce_lock);
623				return (ENOMEM);
624			}
625		} else {
626			if (reserve_bounce_pages(dmat, map, 1) != 0) {
627				/* Queue us for resources */
628				map->dmat = dmat;
629				map->buf = buf;
630				map->buflen = buflen;
631				STAILQ_INSERT_TAIL(&bounce_map_waitinglist,
632				    map, links);
633				mtx_unlock(&bounce_lock);
634				return (EINPROGRESS);
635			}
636		}
637		mtx_unlock(&bounce_lock);
638	}
639
640	vaddr = (vm_offset_t)buf;
641	lastaddr = *lastaddrp;
642	bmask = ~(dmat->boundary - 1);
643
644	for (seg = *segp; buflen > 0 ; ) {
645		/*
646		 * Get the physical address for this segment.
647		 */
648		if (pmap)
649			curaddr = (bus_addr_t)pmap_extract(pmap, vaddr);
650		else
651			curaddr = pmap_kextract(vaddr);
652
653		/*
654		 * Compute the segment size, and adjust counts.
655		 */
656		sgsize = PAGE_SIZE - ((u_long)curaddr & PAGE_MASK);
657		if (buflen < sgsize)
658			sgsize = buflen;
659
660		/*
661		 * Make sure we don't cross any boundaries.
662		 */
663		if (dmat->boundary > 0) {
664			baddr = (curaddr + dmat->boundary) & bmask;
665			if (sgsize > (baddr - curaddr))
666				sgsize = (baddr - curaddr);
667		}
668
669		if (map->pagesneeded != 0 && run_filter(dmat, curaddr))
670			curaddr = add_bounce_page(dmat, map, vaddr, sgsize);
671
672		/*
673		 * Insert chunk into a segment, coalescing with
674		 * previous segment if possible.
675		 */
676		if (first) {
677			segs[seg].ds_addr = curaddr;
678			segs[seg].ds_len = sgsize;
679			first = 0;
680		} else {
681			if (needbounce == 0 && curaddr == lastaddr &&
682			    (segs[seg].ds_len + sgsize) <= dmat->maxsegsz &&
683			    (dmat->boundary == 0 ||
684			     (segs[seg].ds_addr & bmask) == (curaddr & bmask)))
685				segs[seg].ds_len += sgsize;
686			else {
687				if (++seg >= dmat->nsegments)
688					break;
689				segs[seg].ds_addr = curaddr;
690				segs[seg].ds_len = sgsize;
691			}
692		}
693
694		lastaddr = curaddr + sgsize;
695		vaddr += sgsize;
696		buflen -= sgsize;
697	}
698
699	*segp = seg;
700	*lastaddrp = lastaddr;
701
702	/*
703	 * Did we fit?
704	 */
705	return (buflen != 0 ? EFBIG : 0); /* XXX better return value here? */
706}
707
708/*
709 * Map the buffer buf into bus space using the dmamap map.
710 */
711int
712bus_dmamap_load(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf,
713		bus_size_t buflen, bus_dmamap_callback_t *callback,
714		void *callback_arg, int flags)
715{
716	bus_addr_t		lastaddr = 0;
717	int			error, nsegs = 0;
718
719	if (map != NULL) {
720		flags |= BUS_DMA_WAITOK;
721		map->callback = callback;
722		map->callback_arg = callback_arg;
723	}
724
725	error = _bus_dmamap_load_buffer(dmat, map, buf, buflen, NULL, flags,
726	     &lastaddr, dmat->segments, &nsegs, 1);
727
728	CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d",
729	    __func__, dmat, dmat->flags, error, nsegs + 1);
730
731	if (error == EINPROGRESS) {
732		return (error);
733	}
734
735	if (error)
736		(*callback)(callback_arg, dmat->segments, 0, error);
737	else
738		(*callback)(callback_arg, dmat->segments, nsegs + 1, 0);
739
740	/*
741	 * Return ENOMEM to the caller so that it can pass it up the stack.
742	 * This error only happens when NOWAIT is set, so deferal is disabled.
743	 */
744	if (error == ENOMEM)
745		return (error);
746
747	return (0);
748}
749
750/*
751 * Like _bus_dmamap_load(), but for mbufs.
752 */
753int
754bus_dmamap_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map,
755		     struct mbuf *m0,
756		     bus_dmamap_callback2_t *callback, void *callback_arg,
757		     int flags)
758{
759	int nsegs, error;
760
761	M_ASSERTPKTHDR(m0);
762
763	flags |= BUS_DMA_NOWAIT;
764	nsegs = 0;
765	error = 0;
766	if (m0->m_pkthdr.len <= dmat->maxsize) {
767		int first = 1;
768		bus_addr_t lastaddr = 0;
769		struct mbuf *m;
770
771		for (m = m0; m != NULL && error == 0; m = m->m_next) {
772			if (m->m_len > 0) {
773				error = _bus_dmamap_load_buffer(dmat, map,
774						m->m_data, m->m_len,
775						NULL, flags, &lastaddr,
776						dmat->segments, &nsegs, first);
777				first = 0;
778			}
779		}
780	} else {
781		error = EINVAL;
782	}
783
784	if (error) {
785		/* force "no valid mappings" in callback */
786		(*callback)(callback_arg, dmat->segments, 0, 0, error);
787	} else {
788		(*callback)(callback_arg, dmat->segments,
789			    nsegs+1, m0->m_pkthdr.len, error);
790	}
791	CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d",
792	    __func__, dmat, dmat->flags, error, nsegs + 1);
793	return (error);
794}
795
796int
797bus_dmamap_load_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map,
798			struct mbuf *m0, bus_dma_segment_t *segs, int *nsegs,
799			int flags)
800{
801	int error;
802
803	M_ASSERTPKTHDR(m0);
804
805	flags |= BUS_DMA_NOWAIT;
806	*nsegs = 0;
807	error = 0;
808	if (m0->m_pkthdr.len <= dmat->maxsize) {
809		int first = 1;
810		bus_addr_t lastaddr = 0;
811		struct mbuf *m;
812
813		for (m = m0; m != NULL && error == 0; m = m->m_next) {
814			if (m->m_len > 0) {
815				error = _bus_dmamap_load_buffer(dmat, map,
816						m->m_data, m->m_len,
817						NULL, flags, &lastaddr,
818						segs, nsegs, first);
819				first = 0;
820			}
821		}
822	} else {
823		error = EINVAL;
824	}
825
826	/* XXX FIXME: Having to increment nsegs is really annoying */
827	++*nsegs;
828	CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d",
829	    __func__, dmat, dmat->flags, error, *nsegs);
830	return (error);
831}
832
833#if 0
834/*
835 * Like _bus_dmamap_load(), but for uios.
836 */
837int
838bus_dmamap_load_uio(bus_dma_tag_t dmat, bus_dmamap_t map,
839		    struct uio *uio,
840		    bus_dmamap_callback2_t *callback, void *callback_arg,
841		    int flags)
842{
843	bus_addr_t lastaddr;
844	int nsegs, error, first, i;
845	bus_size_t resid;
846	struct iovec *iov;
847	pmap_t pmap;
848
849	flags |= BUS_DMA_NOWAIT;
850	resid = uio->uio_resid;
851	iov = uio->uio_iov;
852
853	if (uio->uio_segflg == UIO_USERSPACE) {
854		KASSERT(uio->uio_td != NULL,
855			("bus_dmamap_load_uio: USERSPACE but no proc"));
856		pmap = vmspace_pmap(uio->uio_td->td_proc->p_vmspace);
857	} else
858		pmap = NULL;
859
860	nsegs = 0;
861	error = 0;
862	first = 1;
863	for (i = 0; i < uio->uio_iovcnt && resid != 0 && !error; i++) {
864		/*
865		 * Now at the first iovec to load.  Load each iovec
866		 * until we have exhausted the residual count.
867		 */
868		bus_size_t minlen =
869			resid < iov[i].iov_len ? resid : iov[i].iov_len;
870		caddr_t addr = (caddr_t) iov[i].iov_base;
871
872		if (minlen > 0) {
873			error = _bus_dmamap_load_buffer(dmat, map,
874					addr, minlen, pmap, flags, &lastaddr,
875					dmat->segments, &nsegs, first);
876			first = 0;
877
878			resid -= minlen;
879		}
880	}
881
882	if (error) {
883		/* force "no valid mappings" in callback */
884		(*callback)(callback_arg, dmat->segments, 0, 0, error);
885	} else {
886		(*callback)(callback_arg, dmat->segments,
887			    nsegs+1, uio->uio_resid, error);
888	}
889	CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d",
890	    __func__, dmat, dmat->flags, error, nsegs + 1);
891	return (error);
892}
893#endif
894
895/*
896 * Release the mapping held by map.
897 */
898void
899_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map)
900{
901	struct bounce_page *bpage;
902
903	while ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) {
904		STAILQ_REMOVE_HEAD(&map->bpages, links);
905		free_bounce_page(dmat, bpage);
906	}
907}
908
909void
910_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
911{
912	struct bounce_page *bpage;
913
914	if ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) {
915		/*
916		 * Handle data bouncing.  We might also
917		 * want to add support for invalidating
918		 * the caches on broken hardware
919		 */
920		dmat->bounce_zone->total_bounced++;
921		CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x op 0x%x "
922		    "performing bounce", __func__, op, dmat, dmat->flags);
923
924		if (op & BUS_DMASYNC_PREWRITE) {
925			while (bpage != NULL) {
926				bcopy((void *)bpage->datavaddr,
927				      (void *)bpage->vaddr,
928				      bpage->datacount);
929				bpage = STAILQ_NEXT(bpage, links);
930			}
931		}
932
933		if (op & BUS_DMASYNC_POSTREAD) {
934			while (bpage != NULL) {
935				bcopy((void *)bpage->vaddr,
936				      (void *)bpage->datavaddr,
937				      bpage->datacount);
938				bpage = STAILQ_NEXT(bpage, links);
939			}
940		}
941	}
942}
943
944void
945init_bounce_pages()
946{
947
948	total_bpages = 0;
949	STAILQ_INIT(&bounce_zone_list);
950	STAILQ_INIT(&bounce_map_waitinglist);
951	STAILQ_INIT(&bounce_map_callbacklist);
952	mtx_init(&bounce_lock, "bounce pages lock", NULL, MTX_DEF);
953}
954
955/* Haiku extension */
956void
957uninit_bounce_pages()
958{
959	/* XXX deep free */
960
961	mtx_destroy(&bounce_lock);
962}
963
964static struct sysctl_ctx_list *
965busdma_sysctl_tree(struct bounce_zone *bz)
966{
967	return (&bz->sysctl_tree);
968}
969
970static struct sysctl_oid *
971busdma_sysctl_tree_top(struct bounce_zone *bz)
972{
973	return (bz->sysctl_tree_top);
974}
975
976static int
977alloc_bounce_zone(bus_dma_tag_t dmat)
978{
979	struct bounce_zone *bz;
980
981	/* Check to see if we already have a suitable zone */
982	STAILQ_FOREACH(bz, &bounce_zone_list, links) {
983		if ((dmat->alignment <= bz->alignment)
984		 && (dmat->boundary <= bz->boundary)
985		 && (dmat->lowaddr >= bz->lowaddr)) {
986			dmat->bounce_zone = bz;
987			return (0);
988		}
989	}
990
991	if ((bz = (struct bounce_zone *)malloc(sizeof(*bz), M_DEVBUF,
992	    M_NOWAIT | M_ZERO)) == NULL)
993		return (ENOMEM);
994
995	STAILQ_INIT(&bz->bounce_page_list);
996	bz->free_bpages = 0;
997	bz->reserved_bpages = 0;
998	bz->active_bpages = 0;
999	bz->lowaddr = dmat->lowaddr;
1000	bz->alignment = dmat->alignment;
1001	bz->boundary = dmat->boundary;
1002	snprintf(bz->zoneid, 8, "zone%d", busdma_zonecount);
1003	busdma_zonecount++;
1004	snprintf(bz->lowaddrid, 18, "%llx", (uintmax_t)bz->lowaddr);
1005	STAILQ_INSERT_TAIL(&bounce_zone_list, bz, links);
1006	dmat->bounce_zone = bz;
1007
1008	sysctl_ctx_init(&bz->sysctl_tree);
1009	bz->sysctl_tree_top = SYSCTL_ADD_NODE(&bz->sysctl_tree,
1010	    SYSCTL_STATIC_CHILDREN(_hw_busdma), OID_AUTO, bz->zoneid,
1011	    CTLFLAG_RD, 0, "");
1012	if (bz->sysctl_tree_top == NULL) {
1013		sysctl_ctx_free(&bz->sysctl_tree);
1014		return (0);	/* XXX error code? */
1015	}
1016
1017	SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1018	    SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1019	    "total_bpages", CTLFLAG_RD, &bz->total_bpages, 0,
1020	    "Total bounce pages");
1021	SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1022	    SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1023	    "free_bpages", CTLFLAG_RD, &bz->free_bpages, 0,
1024	    "Free bounce pages");
1025	SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1026	    SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1027	    "reserved_bpages", CTLFLAG_RD, &bz->reserved_bpages, 0,
1028	    "Reserved bounce pages");
1029	SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1030	    SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1031	    "active_bpages", CTLFLAG_RD, &bz->active_bpages, 0,
1032	    "Active bounce pages");
1033	SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1034	    SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1035	    "total_bounced", CTLFLAG_RD, &bz->total_bounced, 0,
1036	    "Total bounce requests");
1037	SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1038	    SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1039	    "total_deferred", CTLFLAG_RD, &bz->total_deferred, 0,
1040	    "Total bounce requests that were deferred");
1041	SYSCTL_ADD_STRING(busdma_sysctl_tree(bz),
1042	    SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1043	    "lowaddr", CTLFLAG_RD, bz->lowaddrid, 0, "");
1044	SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1045	    SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1046	    "alignment", CTLFLAG_RD, &bz->alignment, 0, "");
1047	SYSCTL_ADD_INT(busdma_sysctl_tree(bz),
1048	    SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO,
1049	    "boundary", CTLFLAG_RD, &bz->boundary, 0, "");
1050
1051	return (0);
1052}
1053
1054static int
1055alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages)
1056{
1057	struct bounce_zone *bz;
1058	int count;
1059
1060	bz = dmat->bounce_zone;
1061	count = 0;
1062	while (numpages > 0) {
1063		struct bounce_page *bpage;
1064
1065		bpage = (struct bounce_page *)malloc(sizeof(*bpage), M_DEVBUF,
1066						     M_NOWAIT | M_ZERO);
1067
1068		if (bpage == NULL)
1069			break;
1070		bpage->vaddr = (vm_offset_t)contigmalloc(PAGE_SIZE, M_DEVBUF,
1071							 M_NOWAIT, 0ul,
1072							 bz->lowaddr,
1073							 PAGE_SIZE,
1074							 bz->boundary);
1075		if (bpage->vaddr == 0) {
1076			free(bpage, M_DEVBUF);
1077			break;
1078		}
1079		bpage->busaddr = pmap_kextract(bpage->vaddr);
1080		mtx_lock(&bounce_lock);
1081		STAILQ_INSERT_TAIL(&bz->bounce_page_list, bpage, links);
1082		total_bpages++;
1083		bz->total_bpages++;
1084		bz->free_bpages++;
1085		mtx_unlock(&bounce_lock);
1086		count++;
1087		numpages--;
1088	}
1089	return (count);
1090}
1091
1092static int
1093reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int commit)
1094{
1095	struct bounce_zone *bz;
1096	int pages;
1097
1098	mtx_assert(&bounce_lock, MA_OWNED);
1099	bz = dmat->bounce_zone;
1100	pages = MIN(bz->free_bpages, map->pagesneeded - map->pagesreserved);
1101	if (commit == 0 && map->pagesneeded > (map->pagesreserved + pages))
1102		return (map->pagesneeded - (map->pagesreserved + pages));
1103	bz->free_bpages -= pages;
1104	bz->reserved_bpages += pages;
1105	map->pagesreserved += pages;
1106	pages = map->pagesneeded - map->pagesreserved;
1107
1108	return (pages);
1109}
1110
1111static bus_addr_t
1112add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr,
1113		bus_size_t size)
1114{
1115	struct bounce_zone *bz;
1116	struct bounce_page *bpage;
1117
1118	KASSERT(dmat->bounce_zone != NULL, ("no bounce zone in dma tag"));
1119	KASSERT(map != NULL && map != &nobounce_dmamap,
1120	    ("add_bounce_page: bad map %p", map));
1121
1122	bz = dmat->bounce_zone;
1123	if (map->pagesneeded == 0)
1124		panic("add_bounce_page: map doesn't need any pages");
1125	map->pagesneeded--;
1126
1127	if (map->pagesreserved == 0)
1128		panic("add_bounce_page: map doesn't need any pages");
1129	map->pagesreserved--;
1130
1131	mtx_lock(&bounce_lock);
1132	bpage = STAILQ_FIRST(&bz->bounce_page_list);
1133	if (bpage == NULL)
1134		panic("add_bounce_page: free page list is empty");
1135
1136	STAILQ_REMOVE_HEAD(&bz->bounce_page_list, links);
1137	bz->reserved_bpages--;
1138	bz->active_bpages++;
1139	mtx_unlock(&bounce_lock);
1140
1141	bpage->datavaddr = vaddr;
1142	bpage->datacount = size;
1143	STAILQ_INSERT_TAIL(&(map->bpages), bpage, links);
1144	return (bpage->busaddr);
1145}
1146
1147static void
1148free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage)
1149{
1150	struct bus_dmamap *map;
1151	struct bounce_zone *bz;
1152
1153	bz = dmat->bounce_zone;
1154	bpage->datavaddr = 0;
1155	bpage->datacount = 0;
1156
1157	mtx_lock(&bounce_lock);
1158	STAILQ_INSERT_HEAD(&bz->bounce_page_list, bpage, links);
1159	bz->free_bpages++;
1160	bz->active_bpages--;
1161	if ((map = STAILQ_FIRST(&bounce_map_waitinglist)) != NULL) {
1162		if (reserve_bounce_pages(map->dmat, map, 1) == 0) {
1163			STAILQ_REMOVE_HEAD(&bounce_map_waitinglist, links);
1164			STAILQ_INSERT_TAIL(&bounce_map_callbacklist,
1165					   map, links);
1166			busdma_swi_pending = 1;
1167			bz->total_deferred++;
1168#if 0
1169			swi_sched(vm_ih, 0);
1170#endif
1171			busdma_swi();
1172		}
1173	}
1174	mtx_unlock(&bounce_lock);
1175}
1176
1177void
1178busdma_swi(void)
1179{
1180	bus_dma_tag_t dmat;
1181	struct bus_dmamap *map;
1182
1183	mtx_lock(&bounce_lock);
1184	while ((map = STAILQ_FIRST(&bounce_map_callbacklist)) != NULL) {
1185		STAILQ_REMOVE_HEAD(&bounce_map_callbacklist, links);
1186		mtx_unlock(&bounce_lock);
1187		dmat = map->dmat;
1188		(dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_LOCK);
1189		bus_dmamap_load(map->dmat, map, map->buf, map->buflen,
1190				map->callback, map->callback_arg, /*flags*/0);
1191		(dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_UNLOCK);
1192		mtx_lock(&bounce_lock);
1193	}
1194	mtx_unlock(&bounce_lock);
1195}
1196