diff options
Diffstat (limited to 'arch/sparc64/kernel/pci_sun4v.c')
-rw-r--r-- | arch/sparc64/kernel/pci_sun4v.c | 382 |
1 files changed, 150 insertions, 232 deletions
diff --git a/arch/sparc64/kernel/pci_sun4v.c b/arch/sparc64/kernel/pci_sun4v.c index 5ea2eab..ddca6c6 100644 --- a/arch/sparc64/kernel/pci_sun4v.c +++ b/arch/sparc64/kernel/pci_sun4v.c @@ -1,6 +1,6 @@ /* pci_sun4v.c: SUN4V specific PCI controller support. * - * Copyright (C) 2006, 2007 David S. Miller (davem@davemloft.net) + * Copyright (C) 2006, 2007, 2008 David S. Miller (davem@davemloft.net) */ #include <linux/kernel.h> @@ -89,6 +89,17 @@ static long iommu_batch_flush(struct iommu_batch *p) return 0; } +static inline void iommu_batch_new_entry(unsigned long entry) +{ + struct iommu_batch *p = &__get_cpu_var(iommu_batch); + + if (p->entry + p->npages == entry) + return; + if (p->entry != ~0UL) + iommu_batch_flush(p); + p->entry = entry; +} + /* Interrupts must be disabled. */ static inline long iommu_batch_add(u64 phys_page) { @@ -113,54 +124,6 @@ static inline long iommu_batch_end(void) return iommu_batch_flush(p); } -static long arena_alloc(struct iommu_arena *arena, unsigned long npages) -{ - unsigned long n, i, start, end, limit; - int pass; - - limit = arena->limit; - start = arena->hint; - pass = 0; - -again: - n = find_next_zero_bit(arena->map, limit, start); - end = n + npages; - if (unlikely(end >= limit)) { - if (likely(pass < 1)) { - limit = start; - start = 0; - pass++; - goto again; - } else { - /* Scanned the whole thing, give up. */ - return -1; - } - } - - for (i = n; i < end; i++) { - if (test_bit(i, arena->map)) { - start = i + 1; - goto again; - } - } - - for (i = n; i < end; i++) - __set_bit(i, arena->map); - - arena->hint = end; - - return n; -} - -static void arena_free(struct iommu_arena *arena, unsigned long base, - unsigned long npages) -{ - unsigned long i; - - for (i = base; i < (base + npages); i++) - __clear_bit(i, arena->map); -} - static void *dma_4v_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_addrp, gfp_t gfp) { @@ -185,11 +148,11 @@ static void *dma_4v_alloc_coherent(struct device *dev, size_t size, iommu = dev->archdata.iommu; spin_lock_irqsave(&iommu->lock, flags); - entry = arena_alloc(&iommu->arena, npages); + entry = iommu_range_alloc(dev, iommu, npages, NULL); spin_unlock_irqrestore(&iommu->lock, flags); - if (unlikely(entry < 0L)) - goto arena_alloc_fail; + if (unlikely(entry == DMA_ERROR_CODE)) + goto range_alloc_fail; *dma_addrp = (iommu->page_table_map_base + (entry << IO_PAGE_SHIFT)); @@ -219,10 +182,10 @@ static void *dma_4v_alloc_coherent(struct device *dev, size_t size, iommu_map_fail: /* Interrupts are disabled. */ spin_lock(&iommu->lock); - arena_free(&iommu->arena, entry, npages); + iommu_range_free(iommu, *dma_addrp, npages); spin_unlock_irqrestore(&iommu->lock, flags); -arena_alloc_fail: +range_alloc_fail: free_pages(first_page, order); return NULL; } @@ -243,7 +206,7 @@ static void dma_4v_free_coherent(struct device *dev, size_t size, void *cpu, spin_lock_irqsave(&iommu->lock, flags); - arena_free(&iommu->arena, entry, npages); + iommu_range_free(iommu, dvma, npages); do { unsigned long num; @@ -281,10 +244,10 @@ static dma_addr_t dma_4v_map_single(struct device *dev, void *ptr, size_t sz, npages >>= IO_PAGE_SHIFT; spin_lock_irqsave(&iommu->lock, flags); - entry = arena_alloc(&iommu->arena, npages); + entry = iommu_range_alloc(dev, iommu, npages, NULL); spin_unlock_irqrestore(&iommu->lock, flags); - if (unlikely(entry < 0L)) + if (unlikely(entry == DMA_ERROR_CODE)) goto bad; bus_addr = (iommu->page_table_map_base + @@ -319,7 +282,7 @@ bad: iommu_map_fail: /* Interrupts are disabled. */ spin_lock(&iommu->lock); - arena_free(&iommu->arena, entry, npages); + iommu_range_free(iommu, bus_addr, npages); spin_unlock_irqrestore(&iommu->lock, flags); return DMA_ERROR_CODE; @@ -350,9 +313,9 @@ static void dma_4v_unmap_single(struct device *dev, dma_addr_t bus_addr, spin_lock_irqsave(&iommu->lock, flags); - entry = (bus_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT; - arena_free(&iommu->arena, entry, npages); + iommu_range_free(iommu, bus_addr, npages); + entry = (bus_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT; do { unsigned long num; @@ -365,174 +328,134 @@ static void dma_4v_unmap_single(struct device *dev, dma_addr_t bus_addr, spin_unlock_irqrestore(&iommu->lock, flags); } -#define SG_ENT_PHYS_ADDRESS(SG) (__pa(sg_virt((SG)))) - -static long fill_sg(long entry, struct device *dev, - struct scatterlist *sg, - int nused, int nelems, unsigned long prot) +static int dma_4v_map_sg(struct device *dev, struct scatterlist *sglist, + int nelems, enum dma_data_direction direction) { - struct scatterlist *dma_sg = sg; - unsigned long flags; - int i; - - local_irq_save(flags); - - iommu_batch_start(dev, prot, entry); - - for (i = 0; i < nused; i++) { - unsigned long pteval = ~0UL; - u32 dma_npages; - - dma_npages = ((dma_sg->dma_address & (IO_PAGE_SIZE - 1UL)) + - dma_sg->dma_length + - ((IO_PAGE_SIZE - 1UL))) >> IO_PAGE_SHIFT; - do { - unsigned long offset; - signed int len; - - /* If we are here, we know we have at least one - * more page to map. So walk forward until we - * hit a page crossing, and begin creating new - * mappings from that spot. - */ - for (;;) { - unsigned long tmp; - - tmp = SG_ENT_PHYS_ADDRESS(sg); - len = sg->length; - if (((tmp ^ pteval) >> IO_PAGE_SHIFT) != 0UL) { - pteval = tmp & IO_PAGE_MASK; - offset = tmp & (IO_PAGE_SIZE - 1UL); - break; - } - if (((tmp ^ (tmp + len - 1UL)) >> IO_PAGE_SHIFT) != 0UL) { - pteval = (tmp + IO_PAGE_SIZE) & IO_PAGE_MASK; - offset = 0UL; - len -= (IO_PAGE_SIZE - (tmp & (IO_PAGE_SIZE - 1UL))); - break; - } - sg = sg_next(sg); - nelems--; - } - - pteval = (pteval & IOPTE_PAGE); - while (len > 0) { - long err; + struct scatterlist *s, *outs, *segstart; + unsigned long flags, handle, prot; + dma_addr_t dma_next = 0, dma_addr; + unsigned int max_seg_size; + int outcount, incount, i; + struct iommu *iommu; + long err; - err = iommu_batch_add(pteval); - if (unlikely(err < 0L)) - goto iommu_map_failed; + BUG_ON(direction == DMA_NONE); - pteval += IO_PAGE_SIZE; - len -= (IO_PAGE_SIZE - offset); - offset = 0; - dma_npages--; - } + iommu = dev->archdata.iommu; + if (nelems == 0 || !iommu) + return 0; + + prot = HV_PCI_MAP_ATTR_READ; + if (direction != DMA_TO_DEVICE) + prot |= HV_PCI_MAP_ATTR_WRITE; - pteval = (pteval & IOPTE_PAGE) + len; - sg = sg_next(sg); - nelems--; + outs = s = segstart = &sglist[0]; + outcount = 1; + incount = nelems; + handle = 0; - /* Skip over any tail mappings we've fully mapped, - * adjusting pteval along the way. Stop when we - * detect a page crossing event. - */ - while (nelems && - (pteval << (64 - IO_PAGE_SHIFT)) != 0UL && - (pteval == SG_ENT_PHYS_ADDRESS(sg)) && - ((pteval ^ - (SG_ENT_PHYS_ADDRESS(sg) + sg->length - 1UL)) >> IO_PAGE_SHIFT) == 0UL) { - pteval += sg->length; - sg = sg_next(sg); - nelems--; - } - if ((pteval << (64 - IO_PAGE_SHIFT)) == 0UL) - pteval = ~0UL; - } while (dma_npages != 0); - dma_sg = sg_next(dma_sg); - } + /* Init first segment length for backout at failure */ + outs->dma_length = 0; - if (unlikely(iommu_batch_end() < 0L)) - goto iommu_map_failed; + spin_lock_irqsave(&iommu->lock, flags); - local_irq_restore(flags); - return 0; + iommu_batch_start(dev, prot, ~0UL); -iommu_map_failed: - local_irq_restore(flags); - return -1L; -} + max_seg_size = dma_get_max_seg_size(dev); + for_each_sg(sglist, s, nelems, i) { + unsigned long paddr, npages, entry, slen; -static int dma_4v_map_sg(struct device *dev, struct scatterlist *sglist, - int nelems, enum dma_data_direction direction) -{ - struct iommu *iommu; - unsigned long flags, npages, prot; - u32 dma_base; - struct scatterlist *sgtmp; - long entry, err; - int used; - - /* Fast path single entry scatterlists. */ - if (nelems == 1) { - sglist->dma_address = - dma_4v_map_single(dev, sg_virt(sglist), - sglist->length, direction); - if (unlikely(sglist->dma_address == DMA_ERROR_CODE)) - return 0; - sglist->dma_length = sglist->length; - return 1; - } + slen = s->length; + /* Sanity check */ + if (slen == 0) { + dma_next = 0; + continue; + } + /* Allocate iommu entries for that segment */ + paddr = (unsigned long) SG_ENT_PHYS_ADDRESS(s); + npages = iommu_num_pages(paddr, slen); + entry = iommu_range_alloc(dev, iommu, npages, &handle); - iommu = dev->archdata.iommu; - - if (unlikely(direction == DMA_NONE)) - goto bad; + /* Handle failure */ + if (unlikely(entry == DMA_ERROR_CODE)) { + if (printk_ratelimit()) + printk(KERN_INFO "iommu_alloc failed, iommu %p paddr %lx" + " npages %lx\n", iommu, paddr, npages); + goto iommu_map_failed; + } - /* Step 1: Prepare scatter list. */ - npages = prepare_sg(dev, sglist, nelems); + iommu_batch_new_entry(entry); - /* Step 2: Allocate a cluster and context, if necessary. */ - spin_lock_irqsave(&iommu->lock, flags); - entry = arena_alloc(&iommu->arena, npages); - spin_unlock_irqrestore(&iommu->lock, flags); + /* Convert entry to a dma_addr_t */ + dma_addr = iommu->page_table_map_base + + (entry << IO_PAGE_SHIFT); + dma_addr |= (s->offset & ~IO_PAGE_MASK); - if (unlikely(entry < 0L)) - goto bad; + /* Insert into HW table */ + paddr &= IO_PAGE_MASK; + while (npages--) { + err = iommu_batch_add(paddr); + if (unlikely(err < 0L)) + goto iommu_map_failed; + paddr += IO_PAGE_SIZE; + } - dma_base = iommu->page_table_map_base + - (entry << IO_PAGE_SHIFT); + /* If we are in an open segment, try merging */ + if (segstart != s) { + /* We cannot merge if: + * - allocated dma_addr isn't contiguous to previous allocation + */ + if ((dma_addr != dma_next) || + (outs->dma_length + s->length > max_seg_size)) { + /* Can't merge: create a new segment */ + segstart = s; + outcount++; + outs = sg_next(outs); + } else { + outs->dma_length += s->length; + } + } - /* Step 3: Normalize DMA addresses. */ - used = nelems; + if (segstart == s) { + /* This is a new segment, fill entries */ + outs->dma_address = dma_addr; + outs->dma_length = slen; + } - sgtmp = sglist; - while (used && sgtmp->dma_length) { - sgtmp->dma_address += dma_base; - sgtmp = sg_next(sgtmp); - used--; + /* Calculate next page pointer for contiguous check */ + dma_next = dma_addr + slen; } - used = nelems - used; - /* Step 4: Create the mappings. */ - prot = HV_PCI_MAP_ATTR_READ; - if (direction != DMA_TO_DEVICE) - prot |= HV_PCI_MAP_ATTR_WRITE; + err = iommu_batch_end(); - err = fill_sg(entry, dev, sglist, used, nelems, prot); if (unlikely(err < 0L)) goto iommu_map_failed; - return used; + spin_unlock_irqrestore(&iommu->lock, flags); -bad: - if (printk_ratelimit()) - WARN_ON(1); - return 0; + if (outcount < incount) { + outs = sg_next(outs); + outs->dma_address = DMA_ERROR_CODE; + outs->dma_length = 0; + } + + return outcount; iommu_map_failed: - spin_lock_irqsave(&iommu->lock, flags); - arena_free(&iommu->arena, entry, npages); + for_each_sg(sglist, s, nelems, i) { + if (s->dma_length != 0) { + unsigned long vaddr, npages; + + vaddr = s->dma_address & IO_PAGE_MASK; + npages = iommu_num_pages(s->dma_address, s->dma_length); + iommu_range_free(iommu, vaddr, npages); + /* XXX demap? XXX */ + s->dma_address = DMA_ERROR_CODE; + s->dma_length = 0; + } + if (s == outs) + break; + } spin_unlock_irqrestore(&iommu->lock, flags); return 0; @@ -542,47 +465,42 @@ static void dma_4v_unmap_sg(struct device *dev, struct scatterlist *sglist, int nelems, enum dma_data_direction direction) { struct pci_pbm_info *pbm; + struct scatterlist *sg; struct iommu *iommu; - unsigned long flags, i, npages; - struct scatterlist *sg, *sgprv; - long entry; - u32 devhandle, bus_addr; + unsigned long flags; + u32 devhandle; - if (unlikely(direction == DMA_NONE)) { - if (printk_ratelimit()) - WARN_ON(1); - } + BUG_ON(direction == DMA_NONE); iommu = dev->archdata.iommu; pbm = dev->archdata.host_controller; devhandle = pbm->devhandle; - bus_addr = sglist->dma_address & IO_PAGE_MASK; - sgprv = NULL; - for_each_sg(sglist, sg, nelems, i) { - if (sg->dma_length == 0) - break; - - sgprv = sg; - } - - npages = (IO_PAGE_ALIGN(sgprv->dma_address + sgprv->dma_length) - - bus_addr) >> IO_PAGE_SHIFT; + spin_lock_irqsave(&iommu->lock, flags); - entry = ((bus_addr - iommu->page_table_map_base) >> IO_PAGE_SHIFT); + sg = sglist; + while (nelems--) { + dma_addr_t dma_handle = sg->dma_address; + unsigned int len = sg->dma_length; + unsigned long npages, entry; - spin_lock_irqsave(&iommu->lock, flags); + if (!len) + break; + npages = iommu_num_pages(dma_handle, len); + iommu_range_free(iommu, dma_handle, npages); - arena_free(&iommu->arena, entry, npages); + entry = ((dma_handle - iommu->page_table_map_base) >> IO_PAGE_SHIFT); + while (npages) { + unsigned long num; - do { - unsigned long num; + num = pci_sun4v_iommu_demap(devhandle, HV_PCI_TSBID(0, entry), + npages); + entry += num; + npages -= num; + } - num = pci_sun4v_iommu_demap(devhandle, HV_PCI_TSBID(0, entry), - npages); - entry += num; - npages -= num; - } while (npages != 0); + sg = sg_next(sg); + } spin_unlock_irqrestore(&iommu->lock, flags); } |