diff options
| -rw-r--r-- | Documentation/dmaengine/client.txt | 23 | ||||
| -rw-r--r-- | drivers/dma/acpi-dma.c | 5 | ||||
| -rw-r--r-- | drivers/dma/at_xdmac.c | 20 | ||||
| -rw-r--r-- | drivers/dma/dmaengine.c | 173 | ||||
| -rw-r--r-- | drivers/dma/dw/platform.c | 7 | ||||
| -rw-r--r-- | drivers/dma/edma.c | 10 | ||||
| -rw-r--r-- | drivers/dma/fsl-edma.c | 85 | ||||
| -rw-r--r-- | drivers/dma/hsu/hsu.c | 17 | ||||
| -rw-r--r-- | drivers/dma/hsu/hsu.h | 1 | ||||
| -rw-r--r-- | drivers/dma/img-mdc-dma.c | 78 | ||||
| -rw-r--r-- | drivers/dma/omap-dma.c | 4 | ||||
| -rw-r--r-- | drivers/dma/pxa_dma.c | 1 | ||||
| -rw-r--r-- | drivers/dma/sh/usb-dmac.c | 4 | ||||
| -rw-r--r-- | drivers/dma/virt-dma.c | 46 | ||||
| -rw-r--r-- | drivers/dma/virt-dma.h | 12 | ||||
| -rw-r--r-- | include/linux/dmaengine.h | 53 | ||||
| -rw-r--r-- | include/linux/omap-dma.h | 6 | ||||
| -rw-r--r-- | include/linux/platform_data/edma.h | 7 |
18 files changed, 417 insertions, 135 deletions
diff --git a/Documentation/dmaengine/client.txt b/Documentation/dmaengine/client.txt index d9f9f46..9e33189 100644 --- a/Documentation/dmaengine/client.txt +++ b/Documentation/dmaengine/client.txt @@ -22,25 +22,14 @@ The slave DMA usage consists of following steps: Channel allocation is slightly different in the slave DMA context, client drivers typically need a channel from a particular DMA controller only and even in some cases a specific channel is desired. - To request a channel dma_request_channel() API is used. + To request a channel dma_request_chan() API is used. Interface: - struct dma_chan *dma_request_channel(dma_cap_mask_t mask, - dma_filter_fn filter_fn, - void *filter_param); - where dma_filter_fn is defined as: - typedef bool (*dma_filter_fn)(struct dma_chan *chan, void *filter_param); - - The 'filter_fn' parameter is optional, but highly recommended for - slave and cyclic channels as they typically need to obtain a specific - DMA channel. - - When the optional 'filter_fn' parameter is NULL, dma_request_channel() - simply returns the first channel that satisfies the capability mask. - - Otherwise, the 'filter_fn' routine will be called once for each free - channel which has a capability in 'mask'. 'filter_fn' is expected to - return 'true' when the desired DMA channel is found. + struct dma_chan *dma_request_chan(struct device *dev, const char *name); + + Which will find and return the 'name' DMA channel associated with the 'dev' + device. The association is done via DT, ACPI or board file based + dma_slave_map matching table. A channel allocated via this interface is exclusive to the caller, until dma_release_channel() is called. diff --git a/drivers/dma/acpi-dma.c b/drivers/dma/acpi-dma.c index 16d0daa..eed6bda 100644 --- a/drivers/dma/acpi-dma.c +++ b/drivers/dma/acpi-dma.c @@ -15,6 +15,7 @@ #include <linux/device.h> #include <linux/err.h> #include <linux/module.h> +#include <linux/kernel.h> #include <linux/list.h> #include <linux/mutex.h> #include <linux/slab.h> @@ -72,7 +73,9 @@ static int acpi_dma_parse_resource_group(const struct acpi_csrt_group *grp, si = (const struct acpi_csrt_shared_info *)&grp[1]; /* Match device by MMIO and IRQ */ - if (si->mmio_base_low != mem || si->gsi_interrupt != irq) + if (si->mmio_base_low != lower_32_bits(mem) || + si->mmio_base_high != upper_32_bits(mem) || + si->gsi_interrupt != irq) return 0; dev_dbg(&adev->dev, "matches with %.4s%04X (rev %u)\n", diff --git a/drivers/dma/at_xdmac.c b/drivers/dma/at_xdmac.c index b5e132d..d0ae461 100644 --- a/drivers/dma/at_xdmac.c +++ b/drivers/dma/at_xdmac.c @@ -863,8 +863,12 @@ at_xdmac_interleaved_queue_desc(struct dma_chan *chan, * access. Hopefully we can access DDR through both ports (at least on * SAMA5D4x), so we can use the same interface for source and dest, * that solves the fact we don't know the direction. + * ERRATA: Even if useless for memory transfers, the PERID has to not + * match the one of another channel. If not, it could lead to spurious + * flag status. */ - u32 chan_cc = AT_XDMAC_CC_DIF(0) + u32 chan_cc = AT_XDMAC_CC_PERID(0x3f) + | AT_XDMAC_CC_DIF(0) | AT_XDMAC_CC_SIF(0) | AT_XDMAC_CC_MBSIZE_SIXTEEN | AT_XDMAC_CC_TYPE_MEM_TRAN; @@ -1039,8 +1043,12 @@ at_xdmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, * access DDR through both ports (at least on SAMA5D4x), so we can use * the same interface for source and dest, that solves the fact we * don't know the direction. + * ERRATA: Even if useless for memory transfers, the PERID has to not + * match the one of another channel. If not, it could lead to spurious + * flag status. */ - u32 chan_cc = AT_XDMAC_CC_DAM_INCREMENTED_AM + u32 chan_cc = AT_XDMAC_CC_PERID(0x3f) + | AT_XDMAC_CC_DAM_INCREMENTED_AM | AT_XDMAC_CC_SAM_INCREMENTED_AM | AT_XDMAC_CC_DIF(0) | AT_XDMAC_CC_SIF(0) @@ -1140,8 +1148,12 @@ static struct at_xdmac_desc *at_xdmac_memset_create_desc(struct dma_chan *chan, * access. Hopefully we can access DDR through both ports (at least on * SAMA5D4x), so we can use the same interface for source and dest, * that solves the fact we don't know the direction. + * ERRATA: Even if useless for memory transfers, the PERID has to not + * match the one of another channel. If not, it could lead to spurious + * flag status. */ - u32 chan_cc = AT_XDMAC_CC_DAM_UBS_AM + u32 chan_cc = AT_XDMAC_CC_PERID(0x3f) + | AT_XDMAC_CC_DAM_UBS_AM | AT_XDMAC_CC_SAM_INCREMENTED_AM | AT_XDMAC_CC_DIF(0) | AT_XDMAC_CC_SIF(0) @@ -1995,8 +2007,6 @@ static int at_xdmac_remove(struct platform_device *pdev) dma_async_device_unregister(&atxdmac->dma); clk_disable_unprepare(atxdmac->clk); - synchronize_irq(atxdmac->irq); - free_irq(atxdmac->irq, atxdmac->dma.dev); for (i = 0; i < atxdmac->dma.chancnt; i++) { diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c index d6fc82e..c50a247 100644 --- a/drivers/dma/dmaengine.c +++ b/drivers/dma/dmaengine.c @@ -43,6 +43,7 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#include <linux/platform_device.h> #include <linux/dma-mapping.h> #include <linux/init.h> #include <linux/module.h> @@ -496,6 +497,7 @@ int dma_get_slave_caps(struct dma_chan *chan, struct dma_slave_caps *caps) caps->dst_addr_widths = device->dst_addr_widths; caps->directions = device->directions; caps->residue_granularity = device->residue_granularity; + caps->descriptor_reuse = device->descriptor_reuse; /* * Some devices implement only pause (e.g. to get residuum) but no @@ -514,7 +516,7 @@ static struct dma_chan *private_candidate(const dma_cap_mask_t *mask, { struct dma_chan *chan; - if (!__dma_device_satisfies_mask(dev, mask)) { + if (mask && !__dma_device_satisfies_mask(dev, mask)) { pr_debug("%s: wrong capabilities\n", __func__); return NULL; } @@ -545,6 +547,42 @@ static struct dma_chan *private_candidate(const dma_cap_mask_t *mask, return NULL; } +static struct dma_chan *find_candidate(struct dma_device *device, + const dma_cap_mask_t *mask, + dma_filter_fn fn, void *fn_param) +{ + struct dma_chan *chan = private_candidate(mask, device, fn, fn_param); + int err; + + if (chan) { + /* Found a suitable channel, try to grab, prep, and return it. + * We first set DMA_PRIVATE to disable balance_ref_count as this + * channel will not be published in the general-purpose + * allocator + */ + dma_cap_set(DMA_PRIVATE, device->cap_mask); + device->privatecnt++; + err = dma_chan_get(chan); + + if (err) { + if (err == -ENODEV) { + pr_debug("%s: %s module removed\n", __func__, + dma_chan_name(chan)); + list_del_rcu(&device->global_node); + } else + pr_debug("%s: failed to get %s: (%d)\n", + __func__, dma_chan_name(chan), err); + + if (--device->privatecnt == 0) + dma_cap_clear(DMA_PRIVATE, device->cap_mask); + + chan = ERR_PTR(err); + } + } + + return chan ? chan : ERR_PTR(-EPROBE_DEFER); +} + /** * dma_get_slave_channel - try to get specific channel exclusively * @chan: target channel @@ -583,7 +621,6 @@ struct dma_chan *dma_get_any_slave_channel(struct dma_device *device) { dma_cap_mask_t mask; struct dma_chan *chan; - int err; dma_cap_zero(mask); dma_cap_set(DMA_SLAVE, mask); @@ -591,23 +628,11 @@ struct dma_chan *dma_get_any_slave_channel(struct dma_device *device) /* lock against __dma_request_channel */ mutex_lock(&dma_list_mutex); - chan = private_candidate(&mask, device, NULL, NULL); - if (chan) { - dma_cap_set(DMA_PRIVATE, device->cap_mask); - device->privatecnt++; - err = dma_chan_get(chan); - if (err) { - pr_debug("%s: failed to get %s: (%d)\n", - __func__, dma_chan_name(chan), err); - chan = NULL; - if (--device->privatecnt == 0) - dma_cap_clear(DMA_PRIVATE, device->cap_mask); - } - } + chan = find_candidate(device, &mask, NULL, NULL); mutex_unlock(&dma_list_mutex); - return chan; + return IS_ERR(chan) ? NULL : chan; } EXPORT_SYMBOL_GPL(dma_get_any_slave_channel); @@ -624,35 +649,15 @@ struct dma_chan *__dma_request_channel(const dma_cap_mask_t *mask, { struct dma_device *device, *_d; struct dma_chan *chan = NULL; - int err; /* Find a channel */ mutex_lock(&dma_list_mutex); list_for_each_entry_safe(device, _d, &dma_device_list, global_node) { - chan = private_candidate(mask, device, fn, fn_param); - if (chan) { - /* Found a suitable channel, try to grab, prep, and - * return it. We first set DMA_PRIVATE to disable - * balance_ref_count as this channel will not be - * published in the general-purpose allocator - */ - dma_cap_set(DMA_PRIVATE, device->cap_mask); - device->privatecnt++; - err = dma_chan_get(chan); + chan = find_candidate(device, mask, fn, fn_param); + if (!IS_ERR(chan)) + break; - if (err == -ENODEV) { - pr_debug("%s: %s module removed\n", - __func__, dma_chan_name(chan)); - list_del_rcu(&device->global_node); - } else if (err) - pr_debug("%s: failed to get %s: (%d)\n", - __func__, dma_chan_name(chan), err); - else - break; - if (--device->privatecnt == 0) - dma_cap_clear(DMA_PRIVATE, device->cap_mask); - chan = NULL; - } + chan = NULL; } mutex_unlock(&dma_list_mutex); @@ -665,27 +670,73 @@ struct dma_chan *__dma_request_channel(const dma_cap_mask_t *mask, } EXPORT_SYMBOL_GPL(__dma_request_channel); +static const struct dma_slave_map *dma_filter_match(struct dma_device *device, + const char *name, + struct device *dev) +{ + int i; + + if (!device->filter.mapcnt) + return NULL; + + for (i = 0; i < device->filter.mapcnt; i++) { + const struct dma_slave_map *map = &device->filter.map[i]; + + if (!strcmp(map->devname, dev_name(dev)) && + !strcmp(map->slave, name)) + return map; + } + + return NULL; +} + /** - * dma_request_slave_channel_reason - try to allocate an exclusive slave channel + * dma_request_chan - try to allocate an exclusive slave channel * @dev: pointer to client device structure * @name: slave channel name * * Returns pointer to appropriate DMA channel on success or an error pointer. */ -struct dma_chan *dma_request_slave_channel_reason(struct device *dev, - const char *name) +struct dma_chan *dma_request_chan(struct device *dev, const char *name) { + struct dma_device *d, *_d; + struct dma_chan *chan = NULL; + /* If device-tree is present get slave info from here */ if (dev->of_node) - return of_dma_request_slave_channel(dev->of_node, name); + chan = of_dma_request_slave_channel(dev->of_node, name); /* If device was enumerated by ACPI get slave info from here */ - if (ACPI_HANDLE(dev)) - return acpi_dma_request_slave_chan_by_name(dev, name); + if (has_acpi_companion(dev) && !chan) + chan = acpi_dma_request_slave_chan_by_name(dev, name); - return ERR_PTR(-ENODEV); + if (chan) { + /* Valid channel found or requester need to be deferred */ + if (!IS_ERR(chan) || PTR_ERR(chan) == -EPROBE_DEFER) + return chan; + } + + /* Try to find the channel via the DMA filter map(s) */ + mutex_lock(&dma_list_mutex); + list_for_each_entry_safe(d, _d, &dma_device_list, global_node) { + dma_cap_mask_t mask; + const struct dma_slave_map *map = dma_filter_match(d, name, dev); + + if (!map) + continue; + + dma_cap_zero(mask); + dma_cap_set(DMA_SLAVE, mask); + + chan = find_candidate(d, &mask, d->filter.fn, map->param); + if (!IS_ERR(chan)) + break; + } + mutex_unlock(&dma_list_mutex); + + return chan ? chan : ERR_PTR(-EPROBE_DEFER); } -EXPORT_SYMBOL_GPL(dma_request_slave_channel_reason); +EXPORT_SYMBOL_GPL(dma_request_chan); /** * dma_request_slave_channel - try to allocate an exclusive slave channel @@ -697,17 +748,35 @@ EXPORT_SYMBOL_GPL(dma_request_slave_channel_reason); struct dma_chan *dma_request_slave_channel(struct device *dev, const char *name) { - struct dma_chan *ch = dma_request_slave_channel_reason(dev, name); + struct dma_chan *ch = dma_request_chan(dev, name); if (IS_ERR(ch)) return NULL; - dma_cap_set(DMA_PRIVATE, ch->device->cap_mask); - ch->device->privatecnt++; - return ch; } EXPORT_SYMBOL_GPL(dma_request_slave_channel); +/** + * dma_request_chan_by_mask - allocate a channel satisfying certain capabilities + * @mask: capabilities that the channel must satisfy + * + * Returns pointer to appropriate DMA channel on success or an error pointer. + */ +struct dma_chan *dma_request_chan_by_mask(const dma_cap_mask_t *mask) +{ + struct dma_chan *chan; + + if (!mask) + return ERR_PTR(-ENODEV); + + chan = __dma_request_channel(mask, NULL, NULL); + if (!chan) + chan = ERR_PTR(-ENODEV); + + return chan; +} +EXPORT_SYMBOL_GPL(dma_request_chan_by_mask); + void dma_release_channel(struct dma_chan *chan) { mutex_lock(&dma_list_mutex); diff --git a/drivers/dma/dw/platform.c b/drivers/dma/dw/platform.c index 68a4815..5a417bb 100644 --- a/drivers/dma/dw/platform.c +++ b/drivers/dma/dw/platform.c @@ -103,18 +103,21 @@ dw_dma_parse_dt(struct platform_device *pdev) struct device_node *np = pdev->dev.of_node; struct dw_dma_platform_data *pdata; u32 tmp, arr[DW_DMA_MAX_NR_MASTERS]; + u32 nr_channels; if (!np) { dev_err(&pdev->dev, "Missing DT data\n"); return NULL; } + if (of_property_read_u32(np, "dma-channels", &nr_channels)) + return NULL; + pdata = devm_kzalloc(&pdev->dev, sizeof(*pdata), GFP_KERNEL); if (!pdata) return NULL; - if (of_property_read_u32(np, "dma-channels", &pdata->nr_channels)) - return NULL; + pdata->nr_channels = nr_channels; if (of_property_read_bool(np, "is_private")) pdata->is_private = true; diff --git a/drivers/dma/edma.c b/drivers/dma/edma.c index 6b03e4e..6b3e9d9 100644 --- a/drivers/dma/edma.c +++ b/drivers/dma/edma.c @@ -2297,6 +2297,10 @@ static int edma_probe(struct platform_device *pdev) edma_set_chmap(&ecc->slave_chans[i], ecc->dummy_slot); } + ecc->dma_slave.filter.map = info->slave_map; + ecc->dma_slave.filter.mapcnt = info->slavecnt; + ecc->dma_slave.filter.fn = edma_filter_fn; + ret = dma_async_device_register(&ecc->dma_slave); if (ret) { dev_err(dev, "slave ddev registration failed (%d)\n", ret); @@ -2404,7 +2408,13 @@ static struct platform_driver edma_driver = { }, }; +static int edma_tptc_probe(struct platform_device *pdev) +{ + return 0; +} + static struct platform_driver edma_tptc_driver = { + .probe = edma_tptc_probe, .driver = { .name = "edma3-tptc", .of_match_table = edma_tptc_of_ids, diff --git a/drivers/dma/fsl-edma.c b/drivers/dma/fsl-edma.c index 915eec3..be2e62b 100644 --- a/drivers/dma/fsl-edma.c +++ b/drivers/dma/fsl-edma.c @@ -116,6 +116,10 @@ BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \ BIT(DMA_SLAVE_BUSWIDTH_8_BYTES) +enum fsl_edma_pm_state { + RUNNING = 0, + SUSPENDED, +}; struct fsl_edma_hw_tcd { __le32 saddr; @@ -147,6 +151,9 @@ struct fsl_edma_slave_config { struct fsl_edma_chan { struct virt_dma_chan vchan; enum dma_status status; + enum fsl_edma_pm_state pm_state; + bool idle; + u32 slave_id; struct fsl_edma_engine *edma; struct fsl_edma_desc *edesc; struct fsl_edma_slave_config fsc; @@ -298,6 +305,7 @@ static int fsl_edma_terminate_all(struct dma_chan *chan) spin_lock_irqsave(&fsl_chan->vchan.lock, flags); fsl_edma_disable_request(fsl_chan); fsl_chan->edesc = NULL; + fsl_chan->idle = true; vchan_get_all_descriptors(&fsl_chan->vchan, &head); spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); vchan_dma_desc_free_list(&fsl_chan->vchan, &head); @@ -313,6 +321,7 @@ static int fsl_edma_pause(struct dma_chan *chan) if (fsl_chan->edesc) { fsl_edma_disable_request(fsl_chan); fsl_chan->status = DMA_PAUSED; + fsl_chan->idle = true; } spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); return 0; @@ -327,6 +336,7 @@ static int fsl_edma_resume(struct dma_chan *chan) if (fsl_chan->edesc) { fsl_edma_enable_request(fsl_chan); fsl_chan->status = DMA_IN_PROGRESS; + fsl_chan->idle = false; } spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); return 0; @@ -648,6 +658,7 @@ static void fsl_edma_xfer_desc(struct fsl_edma_chan *fsl_chan) fsl_edma_set_tcd_regs(fsl_chan, fsl_chan->edesc->tcd[0].vtcd); fsl_edma_enable_request(fsl_chan); fsl_chan->status = DMA_IN_PROGRESS; + fsl_chan->idle = false; } static irqreturn_t fsl_edma_tx_handler(int irq, void *dev_id) @@ -676,6 +687,7 @@ static irqreturn_t fsl_edma_tx_handler(int irq, void *dev_id) vchan_cookie_complete(&fsl_chan->edesc->vdesc); fsl_chan->edesc = NULL; fsl_chan->status = DMA_COMPLETE; + fsl_chan->idle = true; } else { vchan_cyclic_callback(&fsl_chan->edesc->vdesc); } @@ -704,6 +716,7 @@ static irqreturn_t fsl_edma_err_handler(int irq, void *dev_id) edma_writeb(fsl_edma, EDMA_CERR_CERR(ch), fsl_edma->membase + EDMA_CERR); fsl_edma->chans[ch].status = DMA_ERROR; + fsl_edma->chans[ch].idle = true; } } return IRQ_HANDLED; @@ -724,6 +737,12 @@ static void fsl_edma_issue_pending(struct dma_chan *chan) spin_lock_irqsave(&fsl_chan->vchan.lock, flags); + if (unlikely(fsl_chan->pm_state != RUNNING)) { + spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); + /* cannot submit due to suspend */ + return; + } + if (vchan_issue_pending(&fsl_chan->vchan) && !fsl_chan->edesc) fsl_edma_xfer_desc(fsl_chan); @@ -735,6 +754,7 @@ static struct dma_chan *fsl_edma_xlate(struct of_phandle_args *dma_spec, { struct fsl_edma_engine *fsl_edma = ofdma->of_dma_data; struct dma_chan *chan, *_chan; + struct fsl_edma_chan *fsl_chan; unsigned long chans_per_mux = fsl_edma->n_chans / DMAMUX_NR; if (dma_spec->args_count != 2) @@ -748,8 +768,10 @@ static struct dma_chan *fsl_edma_xlate(struct of_phandle_args *dma_spec, chan = dma_get_slave_channel(chan); if (chan) { chan->device->privatecnt++; - fsl_edma_chan_mux(to_fsl_edma_chan(chan), - dma_spec->args[1], true); + fsl_chan = to_fsl_edma_chan(chan); + fsl_chan->slave_id = dma_spec->args[1]; + fsl_edma_chan_mux(fsl_chan, fsl_chan->slave_id, + true); mutex_unlock(&fsl_edma->fsl_edma_mutex); return chan; } @@ -888,7 +910,9 @@ static int fsl_edma_probe(struct platform_device *pdev) struct fsl_edma_chan *fsl_chan = &fsl_edma->chans[i]; fsl_chan->edma = fsl_edma; - + fsl_chan->pm_state = RUNNING; + fsl_chan->slave_id = 0; + fsl_chan->idle = true; fsl_chan->vchan.desc_free = fsl_edma_free_desc; vchan_init(&fsl_chan->vchan, &fsl_edma->dma_dev); @@ -959,6 +983,60 @@ static int fsl_edma_remove(struct platform_device *pdev) return 0; } +static int fsl_edma_suspend_late(struct device *dev) +{ + struct fsl_edma_engine *fsl_edma = dev_get_drvdata(dev); + struct fsl_edma_chan *fsl_chan; + unsigned long flags; + int i; + + for (i = 0; i < fsl_edma->n_chans; i++) { + fsl_chan = &fsl_edma->chans[i]; + spin_lock_irqsave(&fsl_chan->vchan.lock, flags); + /* Make sure chan is idle or will force disable. */ + if (unlikely(!fsl_chan->idle)) { + dev_warn(dev, "WARN: There is non-idle channel."); + fsl_edma_disable_request(fsl_chan); + fsl_edma_chan_mux(fsl_chan, 0, false); + } + + fsl_chan->pm_state = SUSPENDED; + spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); + } + + return 0; +} + +static int fsl_edma_resume_early(struct device *dev) +{ + struct fsl_edma_engine *fsl_edma = dev_get_drvdata(dev); + struct fsl_edma_chan *fsl_chan; + int i; + + for (i = 0; i < fsl_edma->n_chans; i++) { + fsl_chan = &fsl_edma->chans[i]; + fsl_chan->pm_state = RUNNING; + edma_writew(fsl_edma, 0x0, fsl_edma->membase + EDMA_TCD_CSR(i)); + if (fsl_chan->slave_id != 0) + fsl_edma_chan_mux(fsl_chan, fsl_chan->slave_id, true); + } + + edma_writel(fsl_edma, EDMA_CR_ERGA | EDMA_CR_ERCA, + fsl_edma->membase + EDMA_CR); + + return 0; +} + +/* + * eDMA provides the service to others, so it should be suspend late + * and resume early. When eDMA suspend, all of the clients should stop + * the DMA data transmission and let the channel idle. + */ +static const struct dev_pm_ops fsl_edma_pm_ops = { + .suspend_late = fsl_edma_suspend_late, + .resume_early = fsl_edma_resume_early, +}; + static const struct of_device_id fsl_edma_dt_ids[] = { { .compatible = "fsl,vf610-edma", }, { /* sentinel */ } @@ -969,6 +1047,7 @@ static struct platform_driver fsl_edma_driver = { .driver = { .name = "fsl-edma", .of_match_table = fsl_edma_dt_ids, + .pm = &fsl_edma_pm_ops, }, .probe = fsl_edma_probe, .remove = fsl_edma_remove, diff --git a/drivers/dma/hsu/hsu.c b/drivers/dma/hsu/hsu.c index 823ad72..eef145e 100644 --- a/drivers/dma/hsu/hsu.c +++ b/drivers/dma/hsu/hsu.c @@ -228,6 +228,8 @@ static struct dma_async_tx_descriptor *hsu_dma_prep_slave_sg( for_each_sg(sgl, sg, sg_len, i) { desc->sg[i].addr = sg_dma_address(sg); desc->sg[i].len = sg_dma_len(sg); + + desc->length += sg_dma_len(sg); } desc->nents = sg_len; @@ -249,21 +251,10 @@ static void hsu_dma_issue_pending(struct dma_chan *chan) spin_unlock_irqrestore(&hsuc->vchan.lock, flags); } -static size_t hsu_dma_desc_size(struct hsu_dma_desc *desc) -{ - size_t bytes = 0; - unsigned int i; - - for (i = desc->active; i < desc->nents; i++) - bytes += desc->sg[i].len; - - return bytes; -} - static size_t hsu_dma_active_desc_size(struct hsu_dma_chan *hsuc) { struct hsu_dma_desc *desc = hsuc->desc; - size_t bytes = hsu_dma_desc_size(desc); + size_t bytes = desc->length; int i; i = desc->active % HSU_DMA_CHAN_NR_DESC; @@ -294,7 +285,7 @@ static enum dma_status hsu_dma_tx_status(struct dma_chan *chan, dma_set_residue(state, bytes); status = hsuc->desc->status; } else if (vdesc) { - bytes = hsu_dma_desc_size(to_hsu_dma_desc(vdesc)); + bytes = to_hsu_dma_desc(vdesc)->length; dma_set_residue(state, bytes); } spin_unlock_irqrestore(&hsuc->vchan.lock, flags); diff --git a/drivers/dma/hsu/hsu.h b/drivers/dma/hsu/hsu.h index f06579c..578a8ee 100644 --- a/drivers/dma/hsu/hsu.h +++ b/drivers/dma/hsu/hsu.h @@ -65,6 +65,7 @@ struct hsu_dma_desc { enum dma_transfer_direction direction; struct hsu_dma_sg *sg; unsigned int nents; + size_t length; unsigned int active; enum dma_status status; }; diff --git a/drivers/dma/img-mdc-dma.c b/drivers/dma/img-mdc-dma.c index 9ca5683..a4c53be 100644 --- a/drivers/dma/img-mdc-dma.c +++ b/drivers/dma/img-mdc-dma.c @@ -651,6 +651,48 @@ static enum dma_status mdc_tx_status(struct dma_chan *chan, return ret; } +static unsigned int mdc_get_new_events(struct mdc_chan *mchan) +{ + u32 val, processed, done1, done2; + unsigned int ret; + + val = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED); + processed = (val >> MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT) & + MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK; + /* + * CMDS_DONE may have incremented between reading CMDS_PROCESSED + * and clearing INT_ACTIVE. Re-read CMDS_PROCESSED to ensure we + * didn't miss a command completion. + */ + do { + val = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED); + + done1 = (val >> MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT) & + MDC_CMDS_PROCESSED_CMDS_DONE_MASK; + + val &= ~((MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK << + MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT) | + MDC_CMDS_PROCESSED_INT_ACTIVE); + + val |= done1 << MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT; + + mdc_chan_writel(mchan, val, MDC_CMDS_PROCESSED); + + val = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED); + + done2 = (val >> MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT) & + MDC_CMDS_PROCESSED_CMDS_DONE_MASK; + } while (done1 != done2); + + if (done1 >= processed) + ret = done1 - processed; + else + ret = ((MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK + 1) - + processed) + done1; + + return ret; +} + static int mdc_terminate_all(struct dma_chan *chan) { struct mdc_chan *mchan = to_mdc_chan(chan); @@ -667,6 +709,8 @@ static int mdc_terminate_all(struct dma_chan *chan) mchan->desc = NULL; vchan_get_all_descriptors(&mchan->vc, &head); + mdc_get_new_events(mchan); + spin_unlock_irqrestore(&mchan->vc.lock, flags); if (mdesc) @@ -703,35 +747,17 @@ static irqreturn_t mdc_chan_irq(int irq, void *dev_id) { struct mdc_chan *mchan = (struct mdc_chan *)dev_id; struct mdc_tx_desc *mdesc; - u32 val, processed, done1, done2; - unsigned int i; + unsigned int i, new_events; spin_lock(&mchan->vc.lock); - val = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED); - processed = (val >> MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT) & - MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK; - /* - * CMDS_DONE may have incremented between reading CMDS_PROCESSED - * and clearing INT_ACTIVE. Re-read CMDS_PROCESSED to ensure we - * didn't miss a command completion. - */ - do { - val = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED); - done1 = (val >> MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT) & - MDC_CMDS_PROCESSED_CMDS_DONE_MASK; - val &= ~((MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK << - MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT) | - MDC_CMDS_PROCESSED_INT_ACTIVE); - val |= done1 << MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT; - mdc_chan_writel(mchan, val, MDC_CMDS_PROCESSED); - val = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED); - done2 = (val >> MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT) & - MDC_CMDS_PROCESSED_CMDS_DONE_MASK; - } while (done1 != done2); - dev_dbg(mdma2dev(mchan->mdma), "IRQ on channel %d\n", mchan->chan_nr); + new_events = mdc_get_new_events(mchan); + + if (!new_events) + goto out; + mdesc = mchan->desc; if (!mdesc) { dev_warn(mdma2dev(mchan->mdma), @@ -740,8 +766,7 @@ static irqreturn_t mdc_chan_irq(int irq, void *dev_id) goto out; } - for (i = processed; i != done1; - i = (i + 1) % (MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK + 1)) { + for (i = 0; i < new_events; i++) { /* * The first interrupt in a transfer indicates that the * command list has been loaded, not that a command has @@ -979,7 +1004,6 @@ static int mdc_dma_remove(struct platform_device *pdev) vc.chan.device_node) { list_del(&mchan->vc.chan.device_node); - synchronize_irq(mchan->irq); devm_free_irq(&pdev->dev, mchan->irq, mchan); tasklet_kill(&mchan->vc.task); diff --git a/drivers/dma/omap-dma.c b/drivers/dma/omap-dma.c index 1dfc71c..48f77c2 100644 --- a/drivers/dma/omap-dma.c +++ b/drivers/dma/omap-dma.c @@ -1203,6 +1203,10 @@ static int omap_dma_probe(struct platform_device *pdev) return rc; } + od->ddev.filter.map = od->plat->slave_map; + od->ddev.filter.mapcnt = od->plat->slavecnt; + od->ddev.filter.fn = omap_dma_filter_fn; + rc = dma_async_device_register(&od->ddev); if (rc) { pr_warn("OMAP-DMA: failed to register slave DMA engine device: %d\n", diff --git a/drivers/dma/pxa_dma.c b/drivers/dma/pxa_dma.c index fc4156a..f2a0310 100644 --- a/drivers/dma/pxa_dma.c +++ b/drivers/dma/pxa_dma.c @@ -1414,6 +1414,7 @@ static int pxad_probe(struct platform_device *op) pdev->slave.dst_addr_widths = widths; pdev->slave.directions = BIT(DMA_MEM_TO_DEV) | BIT(DMA_DEV_TO_MEM); pdev->slave.residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; + pdev->slave.descriptor_reuse = true; pdev->slave.dev = &op->dev; ret = pxad_init_dmadev(op, pdev, dma_channels); diff --git a/drivers/dma/sh/usb-dmac.c b/drivers/dma/sh/usb-dmac.c index ebd8a5f..16fb330 100644 --- a/drivers/dma/sh/usb-dmac.c +++ b/drivers/dma/sh/usb-dmac.c @@ -448,7 +448,7 @@ usb_dmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, static int usb_dmac_chan_terminate_all(struct dma_chan *chan) { struct usb_dmac_chan *uchan = to_usb_dmac_chan(chan); - struct usb_dmac_desc *desc; + struct usb_dmac_desc *desc, *_desc; unsigned long flags; LIST_HEAD(head); LIST_HEAD(list); @@ -459,7 +459,7 @@ static int usb_dmac_chan_terminate_all(struct dma_chan *chan) if (uchan->desc) uchan->desc = NULL; list_splice_init(&uchan->desc_got, &list); - list_for_each_entry(desc, &list, node) + list_for_each_entry_safe(desc, _desc, &list, node) list_move_tail(&desc->node, &uchan->desc_freed); spin_unlock_irqrestore(&uchan->vc.lock, flags); vchan_dma_desc_free_list(&uchan->vc, &head); diff --git a/drivers/dma/virt-dma.c b/drivers/dma/virt-dma.c index 6f80432..a35c211 100644 --- a/drivers/dma/virt-dma.c +++ b/drivers/dma/virt-dma.c @@ -29,7 +29,7 @@ dma_cookie_t vchan_tx_submit(struct dma_async_tx_descriptor *tx) spin_lock_irqsave(&vc->lock, flags); cookie = dma_cookie_assign(tx); - list_add_tail(&vd->node, &vc->desc_submitted); + list_move_tail(&vd->node, &vc->desc_submitted); spin_unlock_irqrestore(&vc->lock, flags); dev_dbg(vc->chan.device->dev, "vchan %p: txd %p[%x]: submitted\n", @@ -39,6 +39,33 @@ dma_cookie_t vchan_tx_submit(struct dma_async_tx_descriptor *tx) } EXPORT_SYMBOL_GPL(vchan_tx_submit); +/** + * vchan_tx_desc_free - free a reusable descriptor + * @tx: the transfer + * + * This function frees a previously allocated reusable descriptor. The only + * other way is to clear the DMA_CTRL_REUSE flag and submit one last time the + * transfer. + * + * Returns 0 upon success + */ +int vchan_tx_desc_free(struct dma_async_tx_descriptor *tx) +{ + struct virt_dma_chan *vc = to_virt_chan(tx->chan); + struct virt_dma_desc *vd = to_virt_desc(tx); + unsigned long flags; + + spin_lock_irqsave(&vc->lock, flags); + list_del(&vd->node); + spin_unlock_irqrestore(&vc->lock, flags); + + dev_dbg(vc->chan.device->dev, "vchan %p: txd %p[%x]: freeing\n", + vc, vd, vd->tx.cookie); + vc->desc_free(vd); + return 0; +} +EXPORT_SYMBOL_GPL(vchan_tx_desc_free); + struct virt_dma_desc *vchan_find_desc(struct virt_dma_chan *vc, dma_cookie_t cookie) { @@ -83,8 +110,10 @@ static void vchan_complete(unsigned long arg) cb_data = vd->tx.callback_param; list_del(&vd->node); - - vc->desc_free(vd); + if (dmaengine_desc_test_reuse(&vd->tx)) + list_add(&vd->node, &vc->desc_allocated); + else + vc->desc_free(vd); if (cb) cb(cb_data); @@ -96,9 +125,13 @@ void vchan_dma_desc_free_list(struct virt_dma_chan *vc, struct list_head *head) while (!list_empty(head)) { struct virt_dma_desc *vd = list_first_entry(head, struct virt_dma_desc, node); - list_del(&vd->node); - dev_dbg(vc->chan.device->dev, "txd %p: freeing\n", vd); - vc->desc_free(vd); + if (dmaengine_desc_test_reuse(&vd->tx)) { + list_move_tail(&vd->node, &vc->desc_allocated); + } else { + dev_dbg(vc->chan.device->dev, "txd %p: freeing\n", vd); + list_del(&vd->node); + vc->desc_free(vd); + } } } EXPORT_SYMBOL_GPL(vchan_dma_desc_free_list); @@ -108,6 +141,7 @@ void vchan_init(struct virt_dma_chan *vc, struct dma_device *dmadev) dma_cookie_init(&vc->chan); spin_lock_init(&vc->lock); + INIT_LIST_HEAD(&vc->desc_allocated); INIT_LIST_HEAD(&vc->desc_submitted); INIT_LIST_HEAD(&vc->desc_issued); INIT_LIST_HEAD(&vc->desc_completed); diff --git a/drivers/dma/virt-dma.h b/drivers/dma/virt-dma.h index edbb575..d9731ca 100644 --- a/drivers/dma/virt-dma.h +++ b/drivers/dma/virt-dma.h @@ -29,6 +29,7 @@ struct virt_dma_chan { spinlock_t lock; /* protected by vc.lock */ + struct list_head desc_allocated; struct list_head desc_submitted; struct list_head desc_issued; struct list_head desc_completed; @@ -55,10 +56,17 @@ static inline struct dma_async_tx_descriptor *vchan_tx_prep(struct virt_dma_chan struct virt_dma_desc *vd, unsigned long tx_flags) { extern dma_cookie_t vchan_tx_submit(struct dma_async_tx_descriptor *); + extern int vchan_tx_desc_free(struct dma_async_tx_descriptor *); + unsigned long flags; dma_async_tx_descriptor_init(&vd->tx, &vc->chan); vd->tx.flags = tx_flags; vd->tx.tx_submit = vchan_tx_submit; + vd->tx.desc_free = vchan_tx_desc_free; + + spin_lock_irqsave(&vc->lock, flags); + list_add_tail(&vd->node, &vc->desc_allocated); + spin_unlock_irqrestore(&vc->lock, flags); return &vd->tx; } @@ -134,6 +142,7 @@ static inline struct virt_dma_desc *vchan_next_desc(struct virt_dma_chan *vc) static inline void vchan_get_all_descriptors(struct virt_dma_chan *vc, struct list_head *head) { + list_splice_tail_init(&vc->desc_allocated, head); list_splice_tail_init(&vc->desc_submitted, head); list_splice_tail_init(&vc->desc_issued, head); list_splice_tail_init(&vc->desc_completed, head); @@ -141,11 +150,14 @@ static inline void vchan_get_all_descriptors(struct virt_dma_chan *vc, static inline void vchan_free_chan_resources(struct virt_dma_chan *vc) { + struct virt_dma_desc *vd; unsigned long flags; LIST_HEAD(head); spin_lock_irqsave(&vc->lock, flags); vchan_get_all_descriptors(vc, &head); + list_for_each_entry(vd, &head, node) + dmaengine_desc_clear_reuse(&vd->tx); spin_unlock_irqrestore(&vc->lock, flags); vchan_dma_desc_free_list(vc, &head); diff --git a/include/linux/dmaengine.h b/include/linux/dmaengine.h index 2f69e1d..16a1cad 100644 --- a/include/linux/dmaengine.h +++ b/include/linux/dmaengine.h @@ -607,11 +607,38 @@ enum dmaengine_alignment { }; /** + * struct dma_slave_map - associates slave device and it's slave channel with + * parameter to be used by a filter function + * @devname: name of the device + * @slave: slave channel name + * @param: opaque parameter to pass to struct dma_filter.fn + */ +struct dma_slave_map { + const char *devname; + const char *slave; + void *param; +}; + +/** + * struct dma_filter - information for slave device/channel to filter_fn/param + * mapping + * @fn: filter function callback + * @mapcnt: number of slave device/channel in the map + * @map: array of channel to filter mapping data + */ +struct dma_filter { + dma_filter_fn fn; + int mapcnt; + const struct dma_slave_map *map; +}; + +/** * struct dma_device - info on the entity supplying DMA services * @chancnt: how many DMA channels are supported * @privatecnt: how many DMA channels are requested by dma_request_channel * @channels: the list of struct dma_chan * @global_node: list_head for global dma_device_list + * @filter: information for device/slave to filter function/param mapping * @cap_mask: one or more dma_capability flags * @max_xor: maximum number of xor sources, 0 if no capability * @max_pq: maximum number of PQ sources and PQ-continue capability @@ -661,6 +688,7 @@ enum dmaengine_alignment { * struct with auxiliary transfer status information, otherwise the call * will just return a simple status code * @device_issue_pending: push pending transactions to hardware + * @descriptor_reuse: a submitted transfer can be resubmitted after completion */ struct dma_device { @@ -668,6 +696,7 @@ struct dma_device { unsigned int privatecnt; struct list_head channels; struct list_head global_node; + struct dma_filter filter; dma_cap_mask_t cap_mask; unsigned short max_xor; unsigned short max_pq; @@ -683,6 +712,7 @@ struct dma_device { u32 src_addr_widths; u32 dst_addr_widths; u32 directions; + bool descriptor_reuse; enum dma_residue_granularity residue_granularity; int (*device_alloc_chan_resources)(struct dma_chan *chan); @@ -1232,9 +1262,11 @@ enum dma_status dma_wait_for_async_tx(struct dma_async_tx_descriptor *tx); void dma_issue_pending_all(void); struct dma_chan *__dma_request_channel(const dma_cap_mask_t *mask, dma_filter_fn fn, void *fn_param); -struct dma_chan *dma_request_slave_channel_reason(struct device *dev, - const char *name); struct dma_chan *dma_request_slave_channel(struct device *dev, const char *name); + +struct dma_chan *dma_request_chan(struct device *dev, const char *name); +struct dma_chan *dma_request_chan_by_mask(const dma_cap_mask_t *mask); + void dma_release_channel(struct dma_chan *chan); int dma_get_slave_caps(struct dma_chan *chan, struct dma_slave_caps *caps); #else @@ -1258,16 +1290,21 @@ static inline struct dma_chan *__dma_request_channel(const dma_cap_mask_t *mask, { return NULL; } -static inline struct dma_chan *dma_request_slave_channel_reason( - struct device *dev, const char *name) -{ - return ERR_PTR(-ENODEV); -} static inline struct dma_chan *dma_request_slave_channel(struct device *dev, const char *name) { return NULL; } +static inline struct dma_chan *dma_request_chan(struct device *dev, + const char *name) +{ + return ERR_PTR(-ENODEV); +} +static inline struct dma_chan *dma_request_chan_by_mask( + const dma_cap_mask_t *mask) +{ + return ERR_PTR(-ENODEV); +} static inline void dma_release_channel(struct dma_chan *chan) { } @@ -1278,6 +1315,8 @@ static inline int dma_get_slave_caps(struct dma_chan *chan, } #endif +#define dma_request_slave_channel_reason(dev, name) dma_request_chan(dev, name) + static inline int dmaengine_desc_set_reuse(struct dma_async_tx_descriptor *tx) { struct dma_slave_caps caps; diff --git a/include/linux/omap-dma.h b/include/linux/omap-dma.h index 88fa8af..1d99b61 100644 --- a/include/linux/omap-dma.h +++ b/include/linux/omap-dma.h @@ -267,6 +267,9 @@ struct omap_dma_reg { u8 type; }; +#define SDMA_FILTER_PARAM(hw_req) ((int[]) { (hw_req) }) +struct dma_slave_map; + /* System DMA platform data structure */ struct omap_system_dma_plat_info { const struct omap_dma_reg *reg_map; @@ -278,6 +281,9 @@ struct omap_system_dma_plat_info { void (*clear_dma)(int lch); void (*dma_write)(u32 val, int reg, int lch); u32 (*dma_read)(int reg, int lch); + + const struct dma_slave_map *slave_map; + int slavecnt; }; #ifdef CONFIG_ARCH_OMAP2PLUS diff --git a/include/linux/platform_data/edma.h b/include/linux/platform_data/edma.h index e2878ba..105700e 100644 --- a/include/linux/platform_data/edma.h +++ b/include/linux/platform_data/edma.h @@ -53,12 +53,16 @@ enum dma_event_q { #define EDMA_CTLR(i) ((i) >> 16) #define EDMA_CHAN_SLOT(i) ((i) & 0xffff) +#define EDMA_FILTER_PARAM(ctlr, chan) ((int[]) { EDMA_CTLR_CHAN(ctlr, chan) }) + struct edma_rsv_info { const s16 (*rsv_chans)[2]; const s16 (*rsv_slots)[2]; }; +struct dma_slave_map; + /* platform_data for EDMA driver */ struct edma_soc_info { /* @@ -76,6 +80,9 @@ struct edma_soc_info { s8 (*queue_priority_mapping)[2]; const s16 (*xbar_chans)[2]; + + const struct dma_slave_map *slave_map; + int slavecnt; }; #endif |
