/* $OpenBSD: viommu.c,v 1.1 2008/03/09 18:56:45 kettenis Exp $ */ /* $NetBSD: iommu.c,v 1.47 2002/02/08 20:03:45 eeh Exp $ */ /* * Coptright (c) 2008 Mark Kettenis * Copyright (c) 2003 Henric Jungheim * Copyright (c) 2001, 2002 Eduardo Horvath * Copyright (c) 1999, 2000 Matthew R. Green * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * UltraSPARC Hypervisor IOMMU support. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef DDB #include #include #include #endif #ifdef DEBUG #define IDB_BUSDMA 0x1 #define IDB_IOMMU 0x2 #define IDB_INFO 0x4 #define IDB_SYNC 0x8 #define IDB_XXX 0x10 #define IDB_PRINT_MAP 0x20 #define IDB_BREAK 0x40 extern int iommudebug; #define DPRINTF(l, s) do { if (iommudebug & l) printf s; } while (0) #else #define DPRINTF(l, s) #endif void viommu_enter(struct iommu_state *, struct strbuf_ctl *, vaddr_t, paddr_t, int); void viommu_remove(struct iommu_state *, struct strbuf_ctl *, vaddr_t); int viommu_dvmamap_load_seg(bus_dma_tag_t, struct iommu_state *, bus_dmamap_t, bus_dma_segment_t *, int, int, bus_size_t, bus_size_t); int viommu_dvmamap_load_mlist(bus_dma_tag_t, struct iommu_state *, bus_dmamap_t, struct pglist *, int, bus_size_t, bus_size_t); int viommu_dvmamap_append_range(bus_dma_tag_t, bus_dmamap_t, paddr_t, bus_size_t, int, bus_size_t); int iommu_iomap_insert_page(struct iommu_map_state *, paddr_t); vaddr_t iommu_iomap_translate(struct iommu_map_state *, paddr_t); int viommu_iomap_load_map(struct iommu_state *, struct iommu_map_state *, vaddr_t, int); int viommu_iomap_unload_map(struct iommu_state *, struct iommu_map_state *); struct iommu_map_state *viommu_iomap_create(int); void iommu_iomap_destroy(struct iommu_map_state *); void iommu_iomap_clear_pages(struct iommu_map_state *); void _viommu_dvmamap_sync(bus_dma_tag_t, bus_dma_tag_t, bus_dmamap_t, bus_addr_t, bus_size_t, int); /* * initialise the UltraSPARC IOMMU (Hypervisior): * - allocate and setup the iotsb. * - enable the IOMMU * - create a private DVMA map. */ void viommu_init(char *name, struct iommu_state *is, int tsbsize, u_int32_t iovabase) { /* * Setup the iommu. * * The sun4v iommu is accessed through the hypervisor so we will * deal with it here.. */ is->is_tsbsize = tsbsize; if (iovabase == (u_int32_t)-1) { is->is_dvmabase = IOTSB_VSTART(is->is_tsbsize); is->is_dvmaend = IOTSB_VEND; } else { is->is_dvmabase = iovabase; is->is_dvmaend = iovabase + IOTSB_VSIZE(tsbsize) - 1; } /* * Allocate a dvma map. */ printf("dvma map %x-%x", is->is_dvmabase, is->is_dvmaend); is->is_dvmamap = extent_create(name, is->is_dvmabase, (u_long)is->is_dvmaend + 1, M_DEVBUF, 0, 0, EX_NOWAIT); mtx_init(&is->is_mtx, IPL_HIGH); printf("\n"); } /* * Add an entry to the IOMMU table. */ void viommu_enter(struct iommu_state *is, struct strbuf_ctl *sb, vaddr_t va, paddr_t pa, int flags) { u_int64_t tsbid = IOTSBSLOT(va, is->is_tsbsize); paddr_t page_list[1], addr; u_int64_t attr, nmapped; int err; KASSERT(sb == NULL); #ifdef DIAGNOSTIC if (va < is->is_dvmabase || (va + PAGE_MASK) > is->is_dvmaend) panic("viommu_enter: va %#lx not in DVMA space", va); #endif attr = PCI_MAP_ATTR_READ | PCI_MAP_ATTR_WRITE; if (flags & BUS_DMA_READ) attr &= ~PCI_MAP_ATTR_READ; if (flags & BUS_DMA_WRITE) attr &= ~PCI_MAP_ATTR_WRITE; page_list[0] = trunc_page(pa); if (!pmap_extract(pmap_kernel(), (vaddr_t)page_list, &addr)) panic("viommu_enter: pmap_extract failed\n"); err = hv_pci_iommu_map(is->is_devhandle, tsbid, 1, attr, addr, &nmapped); if (err != H_EOK || nmapped != 1) panic("hv_pci_iommu_map: err=%d", err); } /* * Remove an entry from the IOMMU table. */ void viommu_remove(struct iommu_state *is, struct strbuf_ctl *sb, vaddr_t va) { u_int64_t tsbid = IOTSBSLOT(va, is->is_tsbsize); u_int64_t ndemapped; int err; KASSERT(sb == NULL); #ifdef DIAGNOSTIC if (va < is->is_dvmabase || (va + PAGE_MASK) > is->is_dvmaend) panic("iommu_remove: va 0x%lx not in DVMA space", (u_long)va); if (va != trunc_page(va)) { printf("iommu_remove: unaligned va: %lx\n", va); va = trunc_page(va); } #endif err = hv_pci_iommu_demap(is->is_devhandle, tsbid, 1, &ndemapped); if (err != H_EOK || ndemapped != 1) panic("hv_pci_iommu_unmap: err=%d", err); } /* * IOMMU DVMA operations, sun4v hypervisor version. */ #define BUS_DMA_FIND_PARENT(t, fn) \ if (t->_parent == NULL) \ panic("null bus_dma parent (" #fn ")"); \ for (t = t->_parent; t->fn == NULL; t = t->_parent) \ if (t->_parent == NULL) \ panic("no bus_dma " #fn " located"); int viommu_dvmamap_create(bus_dma_tag_t t, bus_dma_tag_t t0, struct iommu_state *is, bus_size_t size, int nsegments, bus_size_t maxsegsz, bus_size_t boundary, int flags, bus_dmamap_t *dmamap) { int ret; bus_dmamap_t map; struct iommu_map_state *ims; BUS_DMA_FIND_PARENT(t, _dmamap_create); ret = (*t->_dmamap_create)(t, t0, size, nsegments, maxsegsz, boundary, flags, &map); if (ret) return (ret); ims = viommu_iomap_create(atop(round_page(size))); if (ims == NULL) { bus_dmamap_destroy(t0, map); return (ENOMEM); } ims->ims_iommu = is; map->_dm_cookie = ims; *dmamap = map; return (0); } void viommu_dvmamap_destroy(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dmamap_t map) { /* * The specification (man page) requires a loaded * map to be unloaded before it is destroyed. */ if (map->dm_nsegs) bus_dmamap_unload(t0, map); if (map->_dm_cookie) iommu_iomap_destroy(map->_dm_cookie); map->_dm_cookie = NULL; BUS_DMA_FIND_PARENT(t, _dmamap_destroy); (*t->_dmamap_destroy)(t, t0, map); } /* * Load a contiguous kva buffer into a dmamap. The physical pages are * not assumed to be contiguous. Two passes are made through the buffer * and both call pmap_extract() for the same va->pa translations. It * is possible to run out of pa->dvma mappings; the code should be smart * enough to resize the iomap (when the "flags" permit allocation). It * is trivial to compute the number of entries required (round the length * up to the page size and then divide by the page size)... */ int viommu_dvmamap_load(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dmamap_t map, void *buf, bus_size_t buflen, struct proc *p, int flags) { int err = 0; bus_size_t sgsize; u_long dvmaddr, sgstart, sgend; bus_size_t align, boundary; struct iommu_state *is; struct iommu_map_state *ims = map->_dm_cookie; pmap_t pmap; #ifdef DIAGNOSTIC if (ims == NULL) panic("viommu_dvmamap_load: null map state"); if (ims->ims_iommu == NULL) panic("viommu_dvmamap_load: null iommu"); #endif is = ims->ims_iommu; if (map->dm_nsegs) { /* * Is it still in use? _bus_dmamap_load should have taken care * of this. */ #ifdef DIAGNOSTIC panic("iommu_dvmamap_load: map still in use"); #endif bus_dmamap_unload(t0, map); } /* * Make sure that on error condition we return "no valid mappings". */ map->dm_nsegs = 0; if (buflen < 1 || buflen > map->_dm_size) { DPRINTF(IDB_BUSDMA, ("iommu_dvmamap_load(): error %d > %d -- " "map size exceeded!\n", (int)buflen, (int)map->_dm_size)); return (EINVAL); } /* * A boundary presented to bus_dmamem_alloc() takes precedence * over boundary in the map. */ if ((boundary = (map->dm_segs[0]._ds_boundary)) == 0) boundary = map->_dm_boundary; align = MAX(map->dm_segs[0]._ds_align, PAGE_SIZE); pmap = p ? p->p_vmspace->vm_map.pmap : pmap = pmap_kernel(); /* Count up the total number of pages we need */ iommu_iomap_clear_pages(ims); { /* Scope */ bus_addr_t a, aend; bus_addr_t addr = (vaddr_t)buf; int seg_len = buflen; aend = round_page(addr + seg_len); for (a = trunc_page(addr); a < aend; a += PAGE_SIZE) { paddr_t pa; if (pmap_extract(pmap, a, &pa) == FALSE) { printf("iomap pmap error addr 0x%llx\n", a); iommu_iomap_clear_pages(ims); return (EFBIG); } err = iommu_iomap_insert_page(ims, pa); if (err) { printf("iomap insert error: %d for " "va 0x%llx pa 0x%lx " "(buf %p len %lld/%llx)\n", err, a, pa, buf, buflen, buflen); iommu_iomap_clear_pages(ims); return (EFBIG); } } } sgsize = ims->ims_map.ipm_pagecnt * PAGE_SIZE; mtx_enter(&is->is_mtx); if (flags & BUS_DMA_24BIT) { sgstart = MAX(is->is_dvmamap->ex_start, 0xff000000); sgend = MIN(is->is_dvmamap->ex_end, 0xffffffff); } else { sgstart = is->is_dvmamap->ex_start; sgend = is->is_dvmamap->ex_end; } /* * If our segment size is larger than the boundary we need to * split the transfer up into little pieces ourselves. */ err = extent_alloc_subregion(is->is_dvmamap, sgstart, sgend, sgsize, align, 0, (sgsize > boundary) ? 0 : boundary, EX_NOWAIT | EX_BOUNDZERO, (u_long *)&dvmaddr); mtx_leave(&is->is_mtx); #ifdef DEBUG if (err || (dvmaddr == (bus_addr_t)-1)) { printf("iommu_dvmamap_load(): extent_alloc(%d, %x) failed!\n", (int)sgsize, flags); #ifdef DDB if (iommudebug & IDB_BREAK) Debugger(); #endif } #endif if (err != 0) return (err); if (dvmaddr == (bus_addr_t)-1) return (ENOMEM); /* Set the active DVMA map */ map->_dm_dvmastart = dvmaddr; map->_dm_dvmasize = sgsize; map->dm_mapsize = buflen; if (viommu_iomap_load_map(is, ims, dvmaddr, flags)) return (EFBIG); { /* Scope */ bus_addr_t a, aend; bus_addr_t addr = (vaddr_t)buf; int seg_len = buflen; aend = round_page(addr + seg_len); for (a = trunc_page(addr); a < aend; a += PAGE_SIZE) { bus_addr_t pgstart; bus_addr_t pgend; paddr_t pa; int pglen; /* Yuck... Redoing the same pmap_extract... */ if (pmap_extract(pmap, a, &pa) == FALSE) { printf("iomap pmap error addr 0x%llx\n", a); iommu_iomap_clear_pages(ims); return (EFBIG); } pgstart = pa | (MAX(a, addr) & PAGE_MASK); pgend = pa | (MIN(a + PAGE_SIZE - 1, addr + seg_len - 1) & PAGE_MASK); pglen = pgend - pgstart + 1; if (pglen < 1) continue; err = viommu_dvmamap_append_range(t, map, pgstart, pglen, flags, boundary); if (err == EFBIG) return (err); if (err) { printf("iomap load seg page: %d for " "va 0x%llx pa %lx (%llx - %llx) " "for %d/0x%x\n", err, a, pa, pgstart, pgend, pglen, pglen); return (err); } } } return (err); } /* * Load a dvmamap from an array of segs or an mlist (if the first * "segs" entry's mlist is non-null). It calls iommu_dvmamap_load_segs() * or iommu_dvmamap_load_mlist() for part of the 2nd pass through the * mapping. This is ugly. A better solution would probably be to have * function pointers for implementing the traversal. That way, there * could be one core load routine for each of the three required algorithms * (buffer, seg, and mlist). That would also mean that the traversal * algorithm would then only need one implementation for each algorithm * instead of two (one for populating the iomap and one for populating * the dvma map). */ int viommu_dvmamap_load_raw(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dmamap_t map, bus_dma_segment_t *segs, int nsegs, bus_size_t size, int flags) { int i; int left; int err = 0; bus_size_t sgsize; bus_size_t boundary, align; u_long dvmaddr, sgstart, sgend; struct iommu_state *is; struct iommu_map_state *ims = map->_dm_cookie; #ifdef DIAGNOSTIC if (ims == NULL) panic("viommu_dvmamap_load_raw: null map state"); if (ims->ims_iommu == NULL) panic("viommu_dvmamap_load_raw: null iommu"); #endif is = ims->ims_iommu; if (map->dm_nsegs) { /* Already in use?? */ #ifdef DIAGNOSTIC panic("iommu_dvmamap_load_raw: map still in use"); #endif bus_dmamap_unload(t0, map); } /* * A boundary presented to bus_dmamem_alloc() takes precedence * over boundary in the map. */ if ((boundary = segs[0]._ds_boundary) == 0) boundary = map->_dm_boundary; align = MAX(segs[0]._ds_align, PAGE_SIZE); /* * Make sure that on error condition we return "no valid mappings". */ map->dm_nsegs = 0; iommu_iomap_clear_pages(ims); if (segs[0]._ds_mlist) { struct pglist *mlist = segs[0]._ds_mlist; struct vm_page *m; for (m = TAILQ_FIRST(mlist); m != NULL; m = TAILQ_NEXT(m,pageq)) { err = iommu_iomap_insert_page(ims, VM_PAGE_TO_PHYS(m)); if(err) { printf("iomap insert error: %d for " "pa 0x%lx\n", err, VM_PAGE_TO_PHYS(m)); iommu_iomap_clear_pages(ims); return (EFBIG); } } } else { /* Count up the total number of pages we need */ for (i = 0, left = size; left > 0 && i < nsegs; i++) { bus_addr_t a, aend; bus_size_t len = segs[i].ds_len; bus_addr_t addr = segs[i].ds_addr; int seg_len = MIN(left, len); if (len < 1) continue; aend = round_page(addr + seg_len); for (a = trunc_page(addr); a < aend; a += PAGE_SIZE) { err = iommu_iomap_insert_page(ims, a); if (err) { printf("iomap insert error: %d for " "pa 0x%llx\n", err, a); iommu_iomap_clear_pages(ims); return (EFBIG); } } left -= seg_len; } } sgsize = ims->ims_map.ipm_pagecnt * PAGE_SIZE; mtx_enter(&is->is_mtx); if (flags & BUS_DMA_24BIT) { sgstart = MAX(is->is_dvmamap->ex_start, 0xff000000); sgend = MIN(is->is_dvmamap->ex_end, 0xffffffff); } else { sgstart = is->is_dvmamap->ex_start; sgend = is->is_dvmamap->ex_end; } /* * If our segment size is larger than the boundary we need to * split the transfer up into little pieces ourselves. */ err = extent_alloc_subregion(is->is_dvmamap, sgstart, sgend, sgsize, align, 0, (sgsize > boundary) ? 0 : boundary, EX_NOWAIT | EX_BOUNDZERO, (u_long *)&dvmaddr); mtx_leave(&is->is_mtx); if (err != 0) return (err); #ifdef DEBUG if (dvmaddr == (bus_addr_t)-1) { printf("iommu_dvmamap_load_raw(): extent_alloc(%d, %x) " "failed!\n", (int)sgsize, flags); #ifdef DDB if (iommudebug & IDB_BREAK) Debugger(); #else panic(""); #endif } #endif if (dvmaddr == (bus_addr_t)-1) return (ENOMEM); /* Set the active DVMA map */ map->_dm_dvmastart = dvmaddr; map->_dm_dvmasize = sgsize; map->dm_mapsize = size; if (viommu_iomap_load_map(is, ims, dvmaddr, flags)) return (EFBIG); if (segs[0]._ds_mlist) err = viommu_dvmamap_load_mlist(t, is, map, segs[0]._ds_mlist, flags, size, boundary); else err = viommu_dvmamap_load_seg(t, is, map, segs, nsegs, flags, size, boundary); if (err) viommu_iomap_unload_map(is, ims); return (err); } /* * Insert a range of addresses into a loaded map respecting the specified * boundary and alignment restrictions. The range is specified by its * physical address and length. The range cannot cross a page boundary. * This code (along with most of the rest of the function in this file) * assumes that the IOMMU page size is equal to PAGE_SIZE. */ int viommu_dvmamap_append_range(bus_dma_tag_t t, bus_dmamap_t map, paddr_t pa, bus_size_t length, int flags, bus_size_t boundary) { struct iommu_map_state *ims = map->_dm_cookie; bus_addr_t sgstart, sgend, bd_mask; bus_dma_segment_t *seg = NULL; int i = map->dm_nsegs; #ifdef DEBUG if (ims == NULL) panic("iommu_dvmamap_append_range: null map state"); #endif sgstart = iommu_iomap_translate(ims, pa); sgend = sgstart + length - 1; #ifdef DIAGNOSTIC if (sgstart == NULL || sgstart > sgend) { printf("append range invalid mapping for %lx " "(0x%llx - 0x%llx)\n", pa, sgstart, sgend); map->dm_nsegs = 0; return (EINVAL); } #endif #ifdef DEBUG if (trunc_page(sgstart) != trunc_page(sgend)) { printf("append range crossing page boundary! " "pa %lx length %lld/0x%llx sgstart %llx sgend %llx\n", pa, length, length, sgstart, sgend); } #endif /* * We will attempt to merge this range with the previous entry * (if there is one). */ if (i > 0) { seg = &map->dm_segs[i - 1]; if (sgstart == seg->ds_addr + seg->ds_len) { length += seg->ds_len; sgstart = seg->ds_addr; sgend = sgstart + length - 1; } else seg = NULL; } if (seg == NULL) { seg = &map->dm_segs[i]; if (++i > map->_dm_segcnt) { map->dm_nsegs = 0; return (EFBIG); } } /* * At this point, "i" is the index of the *next* bus_dma_segment_t * (the segment count, aka map->dm_nsegs) and "seg" points to the * *current* entry. "length", "sgstart", and "sgend" reflect what * we intend to put in "*seg". No assumptions should be made about * the contents of "*seg". Only "boundary" issue can change this * and "boundary" is often zero, so explicitly test for that case * (the test is strictly an optimization). */ if (boundary != 0) { bd_mask = ~(boundary - 1); while ((sgstart & bd_mask) != (sgend & bd_mask)) { /* * We are crossing a boundary so fill in the current * segment with as much as possible, then grab a new * one. */ seg->ds_addr = sgstart; seg->ds_len = boundary - (sgstart & bd_mask); sgstart += seg->ds_len; /* sgend stays the same */ length -= seg->ds_len; seg = &map->dm_segs[i]; if (++i > map->_dm_segcnt) { map->dm_nsegs = 0; return (EFBIG); } } } seg->ds_addr = sgstart; seg->ds_len = length; map->dm_nsegs = i; return (0); } /* * Populate the iomap from a bus_dma_segment_t array. See note for * iommu_dvmamap_load() * regarding page entry exhaustion of the iomap. * This is less of a problem for load_seg, as the number of pages * is usually similar to the number of segments (nsegs). */ int viommu_dvmamap_load_seg(bus_dma_tag_t t, struct iommu_state *is, bus_dmamap_t map, bus_dma_segment_t *segs, int nsegs, int flags, bus_size_t size, bus_size_t boundary) { int i; int left; int seg; /* * This segs is made up of individual physical * segments, probably by _bus_dmamap_load_uio() or * _bus_dmamap_load_mbuf(). Ignore the mlist and * load each one individually. */ /* * Keep in mind that each segment could span * multiple pages and that these are not always * adjacent. The code is no longer adding dvma * aliases to the IOMMU. The STC will not cross * page boundaries anyway and a IOMMU table walk * vs. what may be a streamed PCI DMA to a ring * descriptor is probably a wash. It eases TLB * pressure and in the worst possible case, it is * only as bad a non-IOMMUed architecture. More * importantly, the code is not quite as hairy. * (It's bad enough as it is.) */ left = size; seg = 0; for (i = 0; left > 0 && i < nsegs; i++) { bus_addr_t a, aend; bus_size_t len = segs[i].ds_len; bus_addr_t addr = segs[i].ds_addr; int seg_len = MIN(left, len); if (len < 1) continue; aend = round_page(addr + seg_len); for (a = trunc_page(addr); a < aend; a += PAGE_SIZE) { bus_addr_t pgstart; bus_addr_t pgend; int pglen; int err; pgstart = MAX(a, addr); pgend = MIN(a + PAGE_SIZE - 1, addr + seg_len - 1); pglen = pgend - pgstart + 1; if (pglen < 1) continue; err = viommu_dvmamap_append_range(t, map, pgstart, pglen, flags, boundary); if (err == EFBIG) return (err); if (err) { printf("iomap load seg page: %d for " "pa 0x%llx (%llx - %llx for %d/%x\n", err, a, pgstart, pgend, pglen, pglen); return (err); } } left -= seg_len; } return (0); } /* * Populate the iomap from an mlist. See note for iommu_dvmamap_load() * regarding page entry exhaustion of the iomap. */ int viommu_dvmamap_load_mlist(bus_dma_tag_t t, struct iommu_state *is, bus_dmamap_t map, struct pglist *mlist, int flags, bus_size_t size, bus_size_t boundary) { struct vm_page *m; paddr_t pa; int err; /* * This was allocated with bus_dmamem_alloc. * The pages are on an `mlist'. */ for (m = TAILQ_FIRST(mlist); m != NULL; m = TAILQ_NEXT(m,pageq)) { pa = VM_PAGE_TO_PHYS(m); err = viommu_dvmamap_append_range(t, map, pa, PAGE_SIZE, flags, boundary); if (err == EFBIG) return (err); if (err) { printf("iomap load seg page: %d for pa 0x%lx " "(%lx - %lx for %d/%x\n", err, pa, pa, pa + PAGE_SIZE, PAGE_SIZE, PAGE_SIZE); return (err); } } return (0); } /* * Unload a dvmamap. */ void viommu_dvmamap_unload(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dmamap_t map) { struct iommu_state *is; struct iommu_map_state *ims = map->_dm_cookie; bus_addr_t dvmaddr = map->_dm_dvmastart; bus_size_t sgsize = map->_dm_dvmasize; int error; #ifdef DEBUG if (ims == NULL) panic("viommu_dvmamap_unload: null map state"); if (ims->ims_iommu == NULL) panic("viommu_dvmamap_unload: null iommu"); #endif /* DEBUG */ is = ims->ims_iommu; /* Remove the IOMMU entries */ viommu_iomap_unload_map(is, ims); /* Clear the iomap */ iommu_iomap_clear_pages(ims); bus_dmamap_unload(t->_parent, map); /* Mark the mappings as invalid. */ map->dm_mapsize = 0; map->dm_nsegs = 0; mtx_enter(&is->is_mtx); error = extent_free(is->is_dvmamap, dvmaddr, sgsize, EX_NOWAIT); map->_dm_dvmastart = 0; map->_dm_dvmasize = 0; mtx_leave(&is->is_mtx); if (error != 0) printf("warning: %qd of DVMA space lost\n", sgsize); } void viommu_dvmamap_sync(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dmamap_t map, bus_addr_t offset, bus_size_t len, int ops) { #ifdef DIAGNOSTIC struct iommu_map_state *ims = map->_dm_cookie; if (ims == NULL) panic("viommu_dvmamap_sync: null map state"); if (ims->ims_iommu == NULL) panic("viommu_dvmamap_sync: null iommu"); #endif if (len == 0) return; if (ops & BUS_DMASYNC_PREWRITE) membar(MemIssue); #if 0 if (ops & (BUS_DMASYNC_POSTREAD | BUS_DMASYNC_PREWRITE)) _viommu_dvmamap_sync(t, t0, map, offset, len, ops); #endif if (ops & BUS_DMASYNC_POSTREAD) membar(MemIssue); } int viommu_dvmamem_alloc(bus_dma_tag_t t, bus_dma_tag_t t0, bus_size_t size, bus_size_t alignment, bus_size_t boundary, bus_dma_segment_t *segs, int nsegs, int *rsegs, int flags) { DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_alloc: sz %llx align %llx " "bound %llx segp %p flags %d\n", (unsigned long long)size, (unsigned long long)alignment, (unsigned long long)boundary, segs, flags)); BUS_DMA_FIND_PARENT(t, _dmamem_alloc); return ((*t->_dmamem_alloc)(t, t0, size, alignment, boundary, segs, nsegs, rsegs, flags | BUS_DMA_DVMA)); } void viommu_dvmamem_free(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dma_segment_t *segs, int nsegs) { DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_free: segp %p nsegs %d\n", segs, nsegs)); BUS_DMA_FIND_PARENT(t, _dmamem_free); (*t->_dmamem_free)(t, t0, segs, nsegs); } /* * Map the DVMA mappings into the kernel pmap. * Check the flags to see whether we're streaming or coherent. */ int viommu_dvmamem_map(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dma_segment_t *segs, int nsegs, size_t size, caddr_t *kvap, int flags) { struct vm_page *m; vaddr_t va; bus_addr_t addr; struct pglist *mlist; bus_addr_t cbit = 0; DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_map: segp %p nsegs %d size %lx\n", segs, nsegs, size)); /* * Allocate some space in the kernel map, and then map these pages * into this space. */ size = round_page(size); va = uvm_km_valloc(kernel_map, size); if (va == 0) return (ENOMEM); *kvap = (caddr_t)va; /* * digest flags: */ #if 0 if (flags & BUS_DMA_COHERENT) /* Disable vcache */ cbit |= PMAP_NVC; #endif if (flags & BUS_DMA_NOCACHE) /* sideffects */ cbit |= PMAP_NC; /* * Now take this and map it into the CPU. */ mlist = segs[0]._ds_mlist; TAILQ_FOREACH(m, mlist, pageq) { #ifdef DIAGNOSTIC if (size == 0) panic("iommu_dvmamem_map: size botch"); #endif addr = VM_PAGE_TO_PHYS(m); DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_map: " "mapping va %lx at %llx\n", va, (unsigned long long)addr | cbit)); pmap_enter(pmap_kernel(), va, addr | cbit, VM_PROT_READ | VM_PROT_WRITE, VM_PROT_READ | VM_PROT_WRITE | PMAP_WIRED); va += PAGE_SIZE; size -= PAGE_SIZE; } pmap_update(pmap_kernel()); return (0); } /* * Unmap DVMA mappings from kernel */ void viommu_dvmamem_unmap(bus_dma_tag_t t, bus_dma_tag_t t0, caddr_t kva, size_t size) { DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_unmap: kvm %p size %lx\n", kva, size)); #ifdef DIAGNOSTIC if ((u_long)kva & PAGE_MASK) panic("iommu_dvmamem_unmap"); #endif size = round_page(size); pmap_remove(pmap_kernel(), (vaddr_t)kva, size); pmap_update(pmap_kernel()); uvm_km_free(kernel_map, (vaddr_t)kva, size); } /* * Create a new iomap. */ struct iommu_map_state * viommu_iomap_create(int n) { struct iommu_map_state *ims; /* Safety for heavily fragmented data, such as mbufs */ n += 4; if (n < 16) n = 16; ims = malloc(sizeof(*ims) + (n - 1) * sizeof(ims->ims_map.ipm_map[0]), M_DEVBUF, M_NOWAIT); if (ims == NULL) return (NULL); memset(ims, 0, sizeof *ims); /* Initialize the map. */ ims->ims_map.ipm_maxpage = n; SPLAY_INIT(&ims->ims_map.ipm_tree); return (ims); } /* * Locate the iomap by filling in the pa->va mapping and inserting it * into the IOMMU tables. */ int viommu_iomap_load_map(struct iommu_state *is, struct iommu_map_state *ims, vaddr_t vmaddr, int flags) { struct iommu_page_map *ipm = &ims->ims_map; struct iommu_page_entry *e; int i; for (i = 0, e = ipm->ipm_map; i < ipm->ipm_pagecnt; ++i, ++e) { e->ipe_va = vmaddr; viommu_enter(is, NULL, e->ipe_va, e->ipe_pa, flags); vmaddr += PAGE_SIZE; } return (0); } /* * Remove the iomap from the IOMMU. */ int viommu_iomap_unload_map(struct iommu_state *is, struct iommu_map_state *ims) { struct iommu_page_map *ipm = &ims->ims_map; struct iommu_page_entry *e; int i; for (i = 0, e = ipm->ipm_map; i < ipm->ipm_pagecnt; ++i, ++e) viommu_remove(is, NULL, e->ipe_va); return (0); }