* @base: Pointer to memory area when the pre_alloc_lli's are not large
* enough, IE bigger than the most common case, 1 dst and 1 src. NULL if
* pre_alloc_lli is used.
+ * @dma_addr: DMA address, if mapped
* @size: The size in bytes of the memory at base or the size of pre_alloc_lli.
* @pre_alloc_lli: Pre allocated area for the most common case of transfers,
* one buffer to one buffer.
struct d40_lli_pool {
void *base;
int size;
+ dma_addr_t dma_addr;
/* Space for dst and src, plus an extra for padding */
u8 pre_alloc_lli[3 * sizeof(struct d40_phy_lli)];
};
*/
struct d40_lcla_pool {
void *base;
+ dma_addr_t dma_addr;
void *base_unaligned;
int pages;
spinlock_t lock;
#define chan_err(d40c, format, arg...) \
d40_err(chan2dev(d40c), format, ## arg)
-static int d40_pool_lli_alloc(struct d40_desc *d40d,
- int lli_len, bool is_log)
+static int d40_pool_lli_alloc(struct d40_chan *d40c, struct d40_desc *d40d,
+ int lli_len)
{
+ bool is_log = chan_is_logical(d40c);
u32 align;
void *base;
}
if (is_log) {
- d40d->lli_log.src = PTR_ALIGN((struct d40_log_lli *) base,
- align);
+ d40d->lli_log.src = PTR_ALIGN(base, align);
d40d->lli_log.dst = d40d->lli_log.src + lli_len;
+
+ d40d->lli_pool.dma_addr = 0;
} else {
- d40d->lli_phy.src = PTR_ALIGN((struct d40_phy_lli *)base,
- align);
+ d40d->lli_phy.src = PTR_ALIGN(base, align);
d40d->lli_phy.dst = d40d->lli_phy.src + lli_len;
+
+ d40d->lli_pool.dma_addr = dma_map_single(d40c->base->dev,
+ d40d->lli_phy.src,
+ d40d->lli_pool.size,
+ DMA_TO_DEVICE);
+
+ if (dma_mapping_error(d40c->base->dev,
+ d40d->lli_pool.dma_addr)) {
+ kfree(d40d->lli_pool.base);
+ d40d->lli_pool.base = NULL;
+ d40d->lli_pool.dma_addr = 0;
+ return -ENOMEM;
+ }
}
return 0;
}
-static void d40_pool_lli_free(struct d40_desc *d40d)
+static void d40_pool_lli_free(struct d40_chan *d40c, struct d40_desc *d40d)
{
+ if (d40d->lli_pool.dma_addr)
+ dma_unmap_single(d40c->base->dev, d40d->lli_pool.dma_addr,
+ d40d->lli_pool.size, DMA_TO_DEVICE);
+
kfree(d40d->lli_pool.base);
d40d->lli_pool.base = NULL;
d40d->lli_pool.size = 0;
list_for_each_entry_safe(d, _d, &d40c->client, node)
if (async_tx_test_ack(&d->txd)) {
- d40_pool_lli_free(d);
+ d40_pool_lli_free(d40c, d);
d40_desc_remove(d);
desc = d;
memset(desc, 0, sizeof(*desc));
static void d40_desc_free(struct d40_chan *d40c, struct d40_desc *d40d)
{
+ d40_pool_lli_free(d40c, d40d);
d40_lcla_free_all(d40c, d40d);
kmem_cache_free(d40c->base->desc_slab, d40d);
}
d40d->lli_current++;
for (; d40d->lli_current < d40d->lli_len; d40d->lli_current++) {
- struct d40_log_lli *lcla;
+ unsigned int lcla_offset = d40c->phy_chan->num * 1024 +
+ 8 * curr_lcla * 2;
+ struct d40_lcla_pool *pool = &d40c->base->lcla_pool;
+ struct d40_log_lli *lcla = pool->base + lcla_offset;
if (d40d->lli_current + 1 < d40d->lli_len)
next_lcla = d40_lcla_alloc_one(d40c, d40d);
else
next_lcla = -EINVAL;
- lcla = d40c->base->lcla_pool.base +
- d40c->phy_chan->num * 1024 +
- 8 * curr_lcla * 2;
-
d40_log_lli_lcla_write(lcla,
&d40d->lli_log.dst[d40d->lli_current],
&d40d->lli_log.src[d40d->lli_current],
next_lcla);
- (void) dma_map_single(d40c->base->dev, lcla,
- 2 * sizeof(struct d40_log_lli),
- DMA_TO_DEVICE);
+ dma_sync_single_range_for_device(d40c->base->dev,
+ pool->dma_addr, lcla_offset,
+ 2 * sizeof(struct d40_log_lli),
+ DMA_TO_DEVICE);
curr_lcla = next_lcla;
callback_param = d40d->txd.callback_param;
if (async_tx_test_ack(&d40d->txd)) {
- d40_pool_lli_free(d40d);
+ d40_pool_lli_free(d40c, d40d);
d40_desc_remove(d40d);
d40_desc_free(d40c, d40d);
} else {
/* Release client owned descriptors */
if (!list_empty(&d40c->client))
list_for_each_entry_safe(d, _d, &d40c->client, node) {
- d40_pool_lli_free(d);
+ d40_pool_lli_free(d40c, d);
d40_desc_remove(d);
d40_desc_free(d40c, d);
}
return bytes_left;
}
+static int
+d40_prep_sg_log(struct d40_chan *chan, struct d40_desc *desc,
+ struct scatterlist *sg_src, struct scatterlist *sg_dst,
+ unsigned int sg_len, enum dma_data_direction direction,
+ dma_addr_t dev_addr)
+{
+ struct stedma40_chan_cfg *cfg = &chan->dma_cfg;
+ struct stedma40_half_channel_info *src_info = &cfg->src_info;
+ struct stedma40_half_channel_info *dst_info = &cfg->dst_info;
+
+ if (direction == DMA_NONE) {
+ /* memcpy */
+ (void) d40_log_sg_to_lli(sg_src, sg_len,
+ desc->lli_log.src,
+ chan->log_def.lcsp1,
+ src_info->data_width,
+ dst_info->data_width);
+
+ (void) d40_log_sg_to_lli(sg_dst, sg_len,
+ desc->lli_log.dst,
+ chan->log_def.lcsp3,
+ dst_info->data_width,
+ src_info->data_width);
+ } else {
+ unsigned int total_size;
+
+ total_size = d40_log_sg_to_dev(sg_src, sg_len,
+ &desc->lli_log,
+ &chan->log_def,
+ src_info->data_width,
+ dst_info->data_width,
+ direction, dev_addr);
+ if (total_size < 0)
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int
+d40_prep_sg_phy(struct d40_chan *chan, struct d40_desc *desc,
+ struct scatterlist *sg_src, struct scatterlist *sg_dst,
+ unsigned int sg_len, enum dma_data_direction direction,
+ dma_addr_t dev_addr)
+{
+ dma_addr_t src_dev_addr = direction == DMA_FROM_DEVICE ? dev_addr : 0;
+ dma_addr_t dst_dev_addr = direction == DMA_TO_DEVICE ? dev_addr : 0;
+ struct stedma40_chan_cfg *cfg = &chan->dma_cfg;
+ struct stedma40_half_channel_info *src_info = &cfg->src_info;
+ struct stedma40_half_channel_info *dst_info = &cfg->dst_info;
+ int ret;
+
+ ret = d40_phy_sg_to_lli(sg_src, sg_len, src_dev_addr,
+ desc->lli_phy.src,
+ virt_to_phys(desc->lli_phy.src),
+ chan->src_def_cfg,
+ src_info->data_width,
+ dst_info->data_width,
+ src_info->psize);
+
+ ret = d40_phy_sg_to_lli(sg_dst, sg_len, dst_dev_addr,
+ desc->lli_phy.dst,
+ virt_to_phys(desc->lli_phy.dst),
+ chan->dst_def_cfg,
+ dst_info->data_width,
+ src_info->data_width,
+ dst_info->psize);
+
+ dma_sync_single_for_device(chan->base->dev, desc->lli_pool.dma_addr,
+ desc->lli_pool.size, DMA_TO_DEVICE);
+
+ return ret < 0 ? ret : 0;
+}
+
+
+static struct d40_desc *
+d40_prep_desc(struct d40_chan *chan, struct scatterlist *sg,
+ unsigned int sg_len, unsigned long dma_flags)
+{
+ struct stedma40_chan_cfg *cfg = &chan->dma_cfg;
+ struct d40_desc *desc;
+ int ret;
+
+ desc = d40_desc_get(chan);
+ if (!desc)
+ return NULL;
+
+ desc->lli_len = d40_sg_2_dmalen(sg, sg_len, cfg->src_info.data_width,
+ cfg->dst_info.data_width);
+ if (desc->lli_len < 0) {
+ chan_err(chan, "Unaligned size\n");
+ goto err;
+ }
+
+ ret = d40_pool_lli_alloc(chan, desc, desc->lli_len);
+ if (ret < 0) {
+ chan_err(chan, "Could not allocate lli\n");
+ goto err;
+ }
+
+
+ desc->lli_current = 0;
+ desc->txd.flags = dma_flags;
+ desc->txd.tx_submit = d40_tx_submit;
+
+ dma_async_tx_descriptor_init(&desc->txd, &chan->chan);
+
+ return desc;
+
+err:
+ d40_desc_free(chan, desc);
+ return NULL;
+}
+
struct dma_async_tx_descriptor *stedma40_memcpy_sg(struct dma_chan *chan,
+static struct dma_async_tx_descriptor *stedma40_memcpy_sg(struct dma_chan *chan,
struct scatterlist *sgl_dst,
struct scatterlist *sgl_src,
unsigned int sgl_len,
unsigned long dma_flags)
{
- int res;
struct d40_desc *d40d;
struct d40_chan *d40c = container_of(chan, struct d40_chan,
chan);
}
spin_lock_irqsave(&d40c->lock, flags);
- d40d = d40_desc_get(d40c);
-
- if (d40d == NULL)
- goto err;
- d40d->lli_len = d40_sg_2_dmalen(sgl_dst, sgl_len,
- d40c->dma_cfg.src_info.data_width,
- d40c->dma_cfg.dst_info.data_width);
- if (d40d->lli_len < 0) {
- chan_err(d40c, "Unaligned size\n");
+ d40d = d40_prep_desc(d40c, sgl_dst, sgl_len, dma_flags);
+ if (!d40d)
goto err;
- }
-
- d40d->lli_current = 0;
- d40d->txd.flags = dma_flags;
if (chan_is_logical(d40c)) {
-
- if (d40_pool_lli_alloc(d40d, d40d->lli_len, true) < 0) {
- chan_err(d40c, "Out of memory\n");
- goto err;
- }
-
- (void) d40_log_sg_to_lli(sgl_src,
- sgl_len,
- d40d->lli_log.src,
- d40c->log_def.lcsp1,
- d40c->dma_cfg.src_info.data_width,
- d40c->dma_cfg.dst_info.data_width);
-
- (void) d40_log_sg_to_lli(sgl_dst,
- sgl_len,
- d40d->lli_log.dst,
- d40c->log_def.lcsp3,
- d40c->dma_cfg.dst_info.data_width,
- d40c->dma_cfg.src_info.data_width);
+ d40_prep_sg_log(d40c, d40d, sgl_src, sgl_dst,
+ sgl_len, DMA_NONE, 0);
} else {
- if (d40_pool_lli_alloc(d40d, d40d->lli_len, false) < 0) {
- chan_err(d40c, "Out of memory\n");
- goto err;
- }
-
- res = d40_phy_sg_to_lli(sgl_src,
- sgl_len,
- 0,
- d40d->lli_phy.src,
- virt_to_phys(d40d->lli_phy.src),
- d40c->src_def_cfg,
- d40c->dma_cfg.src_info.data_width,
- d40c->dma_cfg.dst_info.data_width,
- d40c->dma_cfg.src_info.psize);
-
- if (res < 0)
- goto err;
-
- res = d40_phy_sg_to_lli(sgl_dst,
- sgl_len,
- 0,
- d40d->lli_phy.dst,
- virt_to_phys(d40d->lli_phy.dst),
- d40c->dst_def_cfg,
- d40c->dma_cfg.dst_info.data_width,
- d40c->dma_cfg.src_info.data_width,
- d40c->dma_cfg.dst_info.psize);
-
- if (res < 0)
- goto err;
-
- (void) dma_map_single(d40c->base->dev, d40d->lli_phy.src,
- d40d->lli_pool.size, DMA_TO_DEVICE);
+ d40_prep_sg_phy(d40c, d40d, sgl_src, sgl_dst,
+ sgl_len, DMA_NONE, 0);
}
- dma_async_tx_descriptor_init(&d40d->txd, chan);
-
- d40d->txd.tx_submit = d40_tx_submit;
-
spin_unlock_irqrestore(&d40c->lock, flags);
return &d40d->txd;
spin_unlock_irqrestore(&d40c->lock, flags);
return NULL;
}
-EXPORT_SYMBOL(stedma40_memcpy_sg);
bool stedma40_filter(struct dma_chan *chan, void *data)
{
size_t size,
unsigned long dma_flags)
{
- struct d40_desc *d40d;
- struct d40_chan *d40c = container_of(chan, struct d40_chan,
- chan);
- unsigned long flags;
+ struct scatterlist dst_sg;
+ struct scatterlist src_sg;
- if (d40c->phy_chan == NULL) {
- chan_err(d40c, "Channel is not allocated.\n");
- return ERR_PTR(-EINVAL);
- }
+ sg_init_table(&dst_sg, 1);
+ sg_init_table(&src_sg, 1);
- spin_lock_irqsave(&d40c->lock, flags);
- d40d = d40_desc_get(d40c);
+ sg_dma_address(&dst_sg) = dst;
+ sg_dma_address(&src_sg) = src;
- if (d40d == NULL) {
- chan_err(d40c, "Descriptor is NULL\n");
- goto err;
- }
+ sg_dma_len(&dst_sg) = size;
+ sg_dma_len(&src_sg) = size;
- d40d->txd.flags = dma_flags;
- d40d->lli_len = d40_size_2_dmalen(size,
- d40c->dma_cfg.src_info.data_width,
- d40c->dma_cfg.dst_info.data_width);
- if (d40d->lli_len < 0) {
- chan_err(d40c, "Unaligned size\n");
- goto err;
- }
-
-
- dma_async_tx_descriptor_init(&d40d->txd, chan);
-
- d40d->txd.tx_submit = d40_tx_submit;
-
- if (chan_is_logical(d40c)) {
-
- if (d40_pool_lli_alloc(d40d, d40d->lli_len, true) < 0) {
- chan_err(d40c, "Out of memory\n");
- goto err;
- }
- d40d->lli_current = 0;
-
- if (d40_log_buf_to_lli(d40d->lli_log.src,
- src,
- size,
- d40c->log_def.lcsp1,
- d40c->dma_cfg.src_info.data_width,
- d40c->dma_cfg.dst_info.data_width,
- true) == NULL)
- goto err;
-
- if (d40_log_buf_to_lli(d40d->lli_log.dst,
- dst,
- size,
- d40c->log_def.lcsp3,
- d40c->dma_cfg.dst_info.data_width,
- d40c->dma_cfg.src_info.data_width,
- true) == NULL)
- goto err;
-
- } else {
-
- if (d40_pool_lli_alloc(d40d, d40d->lli_len, false) < 0) {
- chan_err(d40c, "Out of memory\n");
- goto err;
- }
-
- if (d40_phy_buf_to_lli(d40d->lli_phy.src,
- src,
- size,
- d40c->dma_cfg.src_info.psize,
- 0,
- d40c->src_def_cfg,
- true,
- d40c->dma_cfg.src_info.data_width,
- d40c->dma_cfg.dst_info.data_width,
- false) == NULL)
- goto err;
-
- if (d40_phy_buf_to_lli(d40d->lli_phy.dst,
- dst,
- size,
- d40c->dma_cfg.dst_info.psize,
- 0,
- d40c->dst_def_cfg,
- true,
- d40c->dma_cfg.dst_info.data_width,
- d40c->dma_cfg.src_info.data_width,
- false) == NULL)
- goto err;
-
- (void) dma_map_single(d40c->base->dev, d40d->lli_phy.src,
- d40d->lli_pool.size, DMA_TO_DEVICE);
- }
-
- spin_unlock_irqrestore(&d40c->lock, flags);
- return &d40d->txd;
-
-err:
- if (d40d)
- d40_desc_free(d40c, d40d);
- spin_unlock_irqrestore(&d40c->lock, flags);
- return NULL;
+ return stedma40_memcpy_sg(chan, &dst_sg, &src_sg, 1, dma_flags);
}
static struct dma_async_tx_descriptor *
return stedma40_memcpy_sg(chan, dst_sg, src_sg, dst_nents, dma_flags);
}
-static int d40_prep_slave_sg_log(struct d40_desc *d40d,
- struct d40_chan *d40c,
- struct scatterlist *sgl,
- unsigned int sg_len,
- enum dma_data_direction direction,
- unsigned long dma_flags)
+static dma_addr_t
+d40_get_dev_addr(struct d40_chan *chan, enum dma_data_direction direction)
{
- dma_addr_t dev_addr = 0;
- int total_size;
+ struct stedma40_platform_data *plat = chan->base->plat_data;
+ struct stedma40_chan_cfg *cfg = &chan->dma_cfg;
+ dma_addr_t addr;
- d40d->lli_len = d40_sg_2_dmalen(sgl, sg_len,
- d40c->dma_cfg.src_info.data_width,
- d40c->dma_cfg.dst_info.data_width);
- if (d40d->lli_len < 0) {
- chan_err(d40c, "Unaligned size\n");
- return -EINVAL;
- }
-
- if (d40_pool_lli_alloc(d40d, d40d->lli_len, true) < 0) {
- chan_err(d40c, "Out of memory\n");
- return -ENOMEM;
- }
-
- d40d->lli_current = 0;
+ if (chan->runtime_addr)
+ return chan->runtime_addr;
if (direction == DMA_FROM_DEVICE)
- if (d40c->runtime_addr)
- dev_addr = d40c->runtime_addr;
- else
- dev_addr = d40c->base->plat_data->dev_rx[d40c->dma_cfg.src_dev_type];
+ addr = plat->dev_rx[cfg->src_dev_type];
else if (direction == DMA_TO_DEVICE)
- if (d40c->runtime_addr)
- dev_addr = d40c->runtime_addr;
- else
- dev_addr = d40c->base->plat_data->dev_tx[d40c->dma_cfg.dst_dev_type];
-
- else
- return -EINVAL;
-
- total_size = d40_log_sg_to_dev(sgl, sg_len,
- &d40d->lli_log,
- &d40c->log_def,
- d40c->dma_cfg.src_info.data_width,
- d40c->dma_cfg.dst_info.data_width,
- direction,
- dev_addr);
-
- if (total_size < 0)
- return -EINVAL;
+ addr = plat->dev_tx[cfg->dst_dev_type];
- return 0;
-}
-
-static int d40_prep_slave_sg_phy(struct d40_desc *d40d,
- struct d40_chan *d40c,
- struct scatterlist *sgl,
- unsigned int sgl_len,
- enum dma_data_direction direction,
- unsigned long dma_flags)
-{
- dma_addr_t src_dev_addr;
- dma_addr_t dst_dev_addr;
- int res;
-
- d40d->lli_len = d40_sg_2_dmalen(sgl, sgl_len,
- d40c->dma_cfg.src_info.data_width,
- d40c->dma_cfg.dst_info.data_width);
- if (d40d->lli_len < 0) {
- chan_err(d40c, "Unaligned size\n");
- return -EINVAL;
- }
-
- if (d40_pool_lli_alloc(d40d, d40d->lli_len, false) < 0) {
- chan_err(d40c, "Out of memory\n");
- return -ENOMEM;
- }
-
- d40d->lli_current = 0;
-
- if (direction == DMA_FROM_DEVICE) {
- dst_dev_addr = 0;
- if (d40c->runtime_addr)
- src_dev_addr = d40c->runtime_addr;
- else
- src_dev_addr = d40c->base->plat_data->dev_rx[d40c->dma_cfg.src_dev_type];
- } else if (direction == DMA_TO_DEVICE) {
- if (d40c->runtime_addr)
- dst_dev_addr = d40c->runtime_addr;
- else
- dst_dev_addr = d40c->base->plat_data->dev_tx[d40c->dma_cfg.dst_dev_type];
- src_dev_addr = 0;
- } else
- return -EINVAL;
-
- res = d40_phy_sg_to_lli(sgl,
- sgl_len,
- src_dev_addr,
- d40d->lli_phy.src,
- virt_to_phys(d40d->lli_phy.src),
- d40c->src_def_cfg,
- d40c->dma_cfg.src_info.data_width,
- d40c->dma_cfg.dst_info.data_width,
- d40c->dma_cfg.src_info.psize);
- if (res < 0)
- return res;
-
- res = d40_phy_sg_to_lli(sgl,
- sgl_len,
- dst_dev_addr,
- d40d->lli_phy.dst,
- virt_to_phys(d40d->lli_phy.dst),
- d40c->dst_def_cfg,
- d40c->dma_cfg.dst_info.data_width,
- d40c->dma_cfg.src_info.data_width,
- d40c->dma_cfg.dst_info.psize);
- if (res < 0)
- return res;
-
- (void) dma_map_single(d40c->base->dev, d40d->lli_phy.src,
- d40d->lli_pool.size, DMA_TO_DEVICE);
- return 0;
+ return addr;
}
static struct dma_async_tx_descriptor *d40_prep_slave_sg(struct dma_chan *chan,
struct d40_desc *d40d;
struct d40_chan *d40c = container_of(chan, struct d40_chan,
chan);
+ dma_addr_t dev_addr;
unsigned long flags;
int err;
return ERR_PTR(-EINVAL);
}
+ if (direction != DMA_FROM_DEVICE && direction != DMA_TO_DEVICE)
+ return NULL;
+
spin_lock_irqsave(&d40c->lock, flags);
- d40d = d40_desc_get(d40c);
+ d40d = d40_prep_desc(d40c, sgl, sg_len, dma_flags);
if (d40d == NULL)
goto err;
+ dev_addr = d40_get_dev_addr(d40c, direction);
+
if (chan_is_logical(d40c))
- err = d40_prep_slave_sg_log(d40d, d40c, sgl, sg_len,
- direction, dma_flags);
+ err = d40_prep_sg_log(d40c, d40d, sgl, NULL,
+ sg_len, direction, dev_addr);
else
- err = d40_prep_slave_sg_phy(d40d, d40c, sgl, sg_len,
- direction, dma_flags);
+ err = d40_prep_sg_phy(d40c, d40d, sgl, NULL,
+ sg_len, direction, dev_addr);
+
if (err) {
chan_err(d40c, "Failed to prepare %s slave sg job: %d\n",
chan_is_logical(d40c) ? "log" : "phy", err);
goto err;
}
- d40d->txd.flags = dma_flags;
-
- dma_async_tx_descriptor_init(&d40d->txd, chan);
-
- d40d->txd.tx_submit = d40_tx_submit;
-
spin_unlock_irqrestore(&d40c->lock, flags);
return &d40d->txd;
static int __init d40_lcla_allocate(struct d40_base *base)
{
+ struct d40_lcla_pool *pool = &base->lcla_pool;
unsigned long *page_list;
int i, j;
int ret = 0;
LCLA_ALIGNMENT);
}
+ pool->dma_addr = dma_map_single(base->dev, pool->base,
+ SZ_1K * base->num_phy_chans,
+ DMA_TO_DEVICE);
+ if (dma_mapping_error(base->dev, pool->dma_addr)) {
+ pool->dma_addr = 0;
+ ret = -ENOMEM;
+ goto failure;
+ }
+
writel(virt_to_phys(base->lcla_pool.base),
base->virtbase + D40_DREG_LCLA);
failure:
kmem_cache_destroy(base->desc_slab);
if (base->virtbase)
iounmap(base->virtbase);
+
+ if (base->lcla_pool.dma_addr)
+ dma_unmap_single(base->dev, base->lcla_pool.dma_addr,
+ SZ_1K * base->num_phy_chans,
+ DMA_TO_DEVICE);
+
if (!base->lcla_pool.base_unaligned && base->lcla_pool.base)
free_pages((unsigned long)base->lcla_pool.base,
base->lcla_pool.pages);
projects / deliverable / linux.git / blobdiff