mirror of
https://github.com/AuxXxilium/linux_dsm_epyc7002.git
synced 2024-12-05 15:56:40 +07:00
546c054755
When dma_cookie_complete() is called in hidma_process_completed(), dma_cookie_status() will return DMA_COMPLETE in hidma_tx_status(). Then, hidma_txn_is_success() will be called to use channel cookie mchan->last_success to do additional DMA status check. Current code assigns mchan->last_success after dma_cookie_complete(). This causes a race condition of dma_cookie_status() returns DMA_COMPLETE before mchan->last_success is assigned correctly. The race will cause hidma_tx_status() return DMA_ERROR but the transaction is actually a success. Moreover, in async_tx case, it will cause a timeout panic in async_tx_quiesce(). Kernel panic - not syncing: async_tx_quiesce: DMA error waiting for transaction ... Call trace: [<ffff000008089994>] dump_backtrace+0x0/0x1f4 [<ffff000008089bac>] show_stack+0x24/0x2c [<ffff00000891e198>] dump_stack+0x84/0xa8 [<ffff0000080da544>] panic+0x12c/0x29c [<ffff0000045d0334>] async_tx_quiesce+0xa4/0xc8 [async_tx] [<ffff0000045d03c8>] async_trigger_callback+0x70/0x1c0 [async_tx] [<ffff0000048b7d74>] raid_run_ops+0x86c/0x1540 [raid456] [<ffff0000048bd084>] handle_stripe+0x5e8/0x1c7c [raid456] [<ffff0000048be9ec>] handle_active_stripes.isra.45+0x2d4/0x550 [raid456] [<ffff0000048beff4>] raid5d+0x38c/0x5d0 [raid456] [<ffff000008736538>] md_thread+0x108/0x168 [<ffff0000080fb1cc>] kthread+0x10c/0x138 [<ffff000008084d34>] ret_from_fork+0x10/0x18 Cc: Joey Zheng <yu.zheng@hxt-semitech.com> Reviewed-by: Sinan Kaya <okaya@kernel.org> Signed-off-by: Shunyong Yang <shunyong.yang@hxt-semitech.com> Signed-off-by: Vinod Koul <vkoul@kernel.org>
981 lines
25 KiB
C
981 lines
25 KiB
C
/*
|
|
* Qualcomm Technologies HIDMA DMA engine interface
|
|
*
|
|
* Copyright (c) 2015-2017, The Linux Foundation. All rights reserved.
|
|
*
|
|
* This program is free software; you can redistribute it and/or modify
|
|
* it under the terms of the GNU General Public License version 2 and
|
|
* only version 2 as published by the Free Software Foundation.
|
|
*
|
|
* This program is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU General Public License for more details.
|
|
*/
|
|
|
|
/*
|
|
* Copyright (C) Freescale Semicondutor, Inc. 2007, 2008.
|
|
* Copyright (C) Semihalf 2009
|
|
* Copyright (C) Ilya Yanok, Emcraft Systems 2010
|
|
* Copyright (C) Alexander Popov, Promcontroller 2014
|
|
*
|
|
* Written by Piotr Ziecik <kosmo@semihalf.com>. Hardware description
|
|
* (defines, structures and comments) was taken from MPC5121 DMA driver
|
|
* written by Hongjun Chen <hong-jun.chen@freescale.com>.
|
|
*
|
|
* Approved as OSADL project by a majority of OSADL members and funded
|
|
* by OSADL membership fees in 2009; for details see www.osadl.org.
|
|
*
|
|
* This program is free software; you can redistribute it and/or modify it
|
|
* under the terms of the GNU General Public License as published by the Free
|
|
* Software Foundation; either version 2 of the License, or (at your option)
|
|
* any later version.
|
|
*
|
|
* This program is distributed in the hope that it will be useful, but WITHOUT
|
|
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
|
* more details.
|
|
*
|
|
* The full GNU General Public License is included in this distribution in the
|
|
* file called COPYING.
|
|
*/
|
|
|
|
/* Linux Foundation elects GPLv2 license only. */
|
|
|
|
#include <linux/dmaengine.h>
|
|
#include <linux/dma-mapping.h>
|
|
#include <linux/list.h>
|
|
#include <linux/module.h>
|
|
#include <linux/platform_device.h>
|
|
#include <linux/slab.h>
|
|
#include <linux/spinlock.h>
|
|
#include <linux/of_dma.h>
|
|
#include <linux/of_device.h>
|
|
#include <linux/property.h>
|
|
#include <linux/delay.h>
|
|
#include <linux/acpi.h>
|
|
#include <linux/irq.h>
|
|
#include <linux/atomic.h>
|
|
#include <linux/pm_runtime.h>
|
|
#include <linux/msi.h>
|
|
|
|
#include "../dmaengine.h"
|
|
#include "hidma.h"
|
|
|
|
/*
|
|
* Default idle time is 2 seconds. This parameter can
|
|
* be overridden by changing the following
|
|
* /sys/bus/platform/devices/QCOM8061:<xy>/power/autosuspend_delay_ms
|
|
* during kernel boot.
|
|
*/
|
|
#define HIDMA_AUTOSUSPEND_TIMEOUT 2000
|
|
#define HIDMA_ERR_INFO_SW 0xFF
|
|
#define HIDMA_ERR_CODE_UNEXPECTED_TERMINATE 0x0
|
|
#define HIDMA_NR_DEFAULT_DESC 10
|
|
#define HIDMA_MSI_INTS 11
|
|
|
|
static inline struct hidma_dev *to_hidma_dev(struct dma_device *dmadev)
|
|
{
|
|
return container_of(dmadev, struct hidma_dev, ddev);
|
|
}
|
|
|
|
static inline
|
|
struct hidma_dev *to_hidma_dev_from_lldev(struct hidma_lldev **_lldevp)
|
|
{
|
|
return container_of(_lldevp, struct hidma_dev, lldev);
|
|
}
|
|
|
|
static inline struct hidma_chan *to_hidma_chan(struct dma_chan *dmach)
|
|
{
|
|
return container_of(dmach, struct hidma_chan, chan);
|
|
}
|
|
|
|
static inline
|
|
struct hidma_desc *to_hidma_desc(struct dma_async_tx_descriptor *t)
|
|
{
|
|
return container_of(t, struct hidma_desc, desc);
|
|
}
|
|
|
|
static void hidma_free(struct hidma_dev *dmadev)
|
|
{
|
|
INIT_LIST_HEAD(&dmadev->ddev.channels);
|
|
}
|
|
|
|
static unsigned int nr_desc_prm;
|
|
module_param(nr_desc_prm, uint, 0644);
|
|
MODULE_PARM_DESC(nr_desc_prm, "number of descriptors (default: 0)");
|
|
|
|
enum hidma_cap {
|
|
HIDMA_MSI_CAP = 1,
|
|
HIDMA_IDENTITY_CAP,
|
|
};
|
|
|
|
/* process completed descriptors */
|
|
static void hidma_process_completed(struct hidma_chan *mchan)
|
|
{
|
|
struct dma_device *ddev = mchan->chan.device;
|
|
struct hidma_dev *mdma = to_hidma_dev(ddev);
|
|
struct dma_async_tx_descriptor *desc;
|
|
dma_cookie_t last_cookie;
|
|
struct hidma_desc *mdesc;
|
|
struct hidma_desc *next;
|
|
unsigned long irqflags;
|
|
struct list_head list;
|
|
|
|
INIT_LIST_HEAD(&list);
|
|
|
|
/* Get all completed descriptors */
|
|
spin_lock_irqsave(&mchan->lock, irqflags);
|
|
list_splice_tail_init(&mchan->completed, &list);
|
|
spin_unlock_irqrestore(&mchan->lock, irqflags);
|
|
|
|
/* Execute callbacks and run dependencies */
|
|
list_for_each_entry_safe(mdesc, next, &list, node) {
|
|
enum dma_status llstat;
|
|
struct dmaengine_desc_callback cb;
|
|
struct dmaengine_result result;
|
|
|
|
desc = &mdesc->desc;
|
|
last_cookie = desc->cookie;
|
|
|
|
llstat = hidma_ll_status(mdma->lldev, mdesc->tre_ch);
|
|
|
|
spin_lock_irqsave(&mchan->lock, irqflags);
|
|
if (llstat == DMA_COMPLETE) {
|
|
mchan->last_success = last_cookie;
|
|
result.result = DMA_TRANS_NOERROR;
|
|
} else {
|
|
result.result = DMA_TRANS_ABORTED;
|
|
}
|
|
|
|
dma_cookie_complete(desc);
|
|
spin_unlock_irqrestore(&mchan->lock, irqflags);
|
|
|
|
dmaengine_desc_get_callback(desc, &cb);
|
|
|
|
dma_run_dependencies(desc);
|
|
|
|
spin_lock_irqsave(&mchan->lock, irqflags);
|
|
list_move(&mdesc->node, &mchan->free);
|
|
spin_unlock_irqrestore(&mchan->lock, irqflags);
|
|
|
|
dmaengine_desc_callback_invoke(&cb, &result);
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Called once for each submitted descriptor.
|
|
* PM is locked once for each descriptor that is currently
|
|
* in execution.
|
|
*/
|
|
static void hidma_callback(void *data)
|
|
{
|
|
struct hidma_desc *mdesc = data;
|
|
struct hidma_chan *mchan = to_hidma_chan(mdesc->desc.chan);
|
|
struct dma_device *ddev = mchan->chan.device;
|
|
struct hidma_dev *dmadev = to_hidma_dev(ddev);
|
|
unsigned long irqflags;
|
|
bool queued = false;
|
|
|
|
spin_lock_irqsave(&mchan->lock, irqflags);
|
|
if (mdesc->node.next) {
|
|
/* Delete from the active list, add to completed list */
|
|
list_move_tail(&mdesc->node, &mchan->completed);
|
|
queued = true;
|
|
|
|
/* calculate the next running descriptor */
|
|
mchan->running = list_first_entry(&mchan->active,
|
|
struct hidma_desc, node);
|
|
}
|
|
spin_unlock_irqrestore(&mchan->lock, irqflags);
|
|
|
|
hidma_process_completed(mchan);
|
|
|
|
if (queued) {
|
|
pm_runtime_mark_last_busy(dmadev->ddev.dev);
|
|
pm_runtime_put_autosuspend(dmadev->ddev.dev);
|
|
}
|
|
}
|
|
|
|
static int hidma_chan_init(struct hidma_dev *dmadev, u32 dma_sig)
|
|
{
|
|
struct hidma_chan *mchan;
|
|
struct dma_device *ddev;
|
|
|
|
mchan = devm_kzalloc(dmadev->ddev.dev, sizeof(*mchan), GFP_KERNEL);
|
|
if (!mchan)
|
|
return -ENOMEM;
|
|
|
|
ddev = &dmadev->ddev;
|
|
mchan->dma_sig = dma_sig;
|
|
mchan->dmadev = dmadev;
|
|
mchan->chan.device = ddev;
|
|
dma_cookie_init(&mchan->chan);
|
|
|
|
INIT_LIST_HEAD(&mchan->free);
|
|
INIT_LIST_HEAD(&mchan->prepared);
|
|
INIT_LIST_HEAD(&mchan->active);
|
|
INIT_LIST_HEAD(&mchan->completed);
|
|
INIT_LIST_HEAD(&mchan->queued);
|
|
|
|
spin_lock_init(&mchan->lock);
|
|
list_add_tail(&mchan->chan.device_node, &ddev->channels);
|
|
dmadev->ddev.chancnt++;
|
|
return 0;
|
|
}
|
|
|
|
static void hidma_issue_task(unsigned long arg)
|
|
{
|
|
struct hidma_dev *dmadev = (struct hidma_dev *)arg;
|
|
|
|
pm_runtime_get_sync(dmadev->ddev.dev);
|
|
hidma_ll_start(dmadev->lldev);
|
|
}
|
|
|
|
static void hidma_issue_pending(struct dma_chan *dmach)
|
|
{
|
|
struct hidma_chan *mchan = to_hidma_chan(dmach);
|
|
struct hidma_dev *dmadev = mchan->dmadev;
|
|
unsigned long flags;
|
|
struct hidma_desc *qdesc, *next;
|
|
int status;
|
|
|
|
spin_lock_irqsave(&mchan->lock, flags);
|
|
list_for_each_entry_safe(qdesc, next, &mchan->queued, node) {
|
|
hidma_ll_queue_request(dmadev->lldev, qdesc->tre_ch);
|
|
list_move_tail(&qdesc->node, &mchan->active);
|
|
}
|
|
|
|
if (!mchan->running) {
|
|
struct hidma_desc *desc = list_first_entry(&mchan->active,
|
|
struct hidma_desc,
|
|
node);
|
|
mchan->running = desc;
|
|
}
|
|
spin_unlock_irqrestore(&mchan->lock, flags);
|
|
|
|
/* PM will be released in hidma_callback function. */
|
|
status = pm_runtime_get(dmadev->ddev.dev);
|
|
if (status < 0)
|
|
tasklet_schedule(&dmadev->task);
|
|
else
|
|
hidma_ll_start(dmadev->lldev);
|
|
}
|
|
|
|
static inline bool hidma_txn_is_success(dma_cookie_t cookie,
|
|
dma_cookie_t last_success, dma_cookie_t last_used)
|
|
{
|
|
if (last_success <= last_used) {
|
|
if ((cookie <= last_success) || (cookie > last_used))
|
|
return true;
|
|
} else {
|
|
if ((cookie <= last_success) && (cookie > last_used))
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
static enum dma_status hidma_tx_status(struct dma_chan *dmach,
|
|
dma_cookie_t cookie,
|
|
struct dma_tx_state *txstate)
|
|
{
|
|
struct hidma_chan *mchan = to_hidma_chan(dmach);
|
|
enum dma_status ret;
|
|
|
|
ret = dma_cookie_status(dmach, cookie, txstate);
|
|
if (ret == DMA_COMPLETE) {
|
|
bool is_success;
|
|
|
|
is_success = hidma_txn_is_success(cookie, mchan->last_success,
|
|
dmach->cookie);
|
|
return is_success ? ret : DMA_ERROR;
|
|
}
|
|
|
|
if (mchan->paused && (ret == DMA_IN_PROGRESS)) {
|
|
unsigned long flags;
|
|
dma_cookie_t runcookie;
|
|
|
|
spin_lock_irqsave(&mchan->lock, flags);
|
|
if (mchan->running)
|
|
runcookie = mchan->running->desc.cookie;
|
|
else
|
|
runcookie = -EINVAL;
|
|
|
|
if (runcookie == cookie)
|
|
ret = DMA_PAUSED;
|
|
|
|
spin_unlock_irqrestore(&mchan->lock, flags);
|
|
}
|
|
|
|
return ret;
|
|
}
|
|
|
|
/*
|
|
* Submit descriptor to hardware.
|
|
* Lock the PM for each descriptor we are sending.
|
|
*/
|
|
static dma_cookie_t hidma_tx_submit(struct dma_async_tx_descriptor *txd)
|
|
{
|
|
struct hidma_chan *mchan = to_hidma_chan(txd->chan);
|
|
struct hidma_dev *dmadev = mchan->dmadev;
|
|
struct hidma_desc *mdesc;
|
|
unsigned long irqflags;
|
|
dma_cookie_t cookie;
|
|
|
|
pm_runtime_get_sync(dmadev->ddev.dev);
|
|
if (!hidma_ll_isenabled(dmadev->lldev)) {
|
|
pm_runtime_mark_last_busy(dmadev->ddev.dev);
|
|
pm_runtime_put_autosuspend(dmadev->ddev.dev);
|
|
return -ENODEV;
|
|
}
|
|
pm_runtime_mark_last_busy(dmadev->ddev.dev);
|
|
pm_runtime_put_autosuspend(dmadev->ddev.dev);
|
|
|
|
mdesc = container_of(txd, struct hidma_desc, desc);
|
|
spin_lock_irqsave(&mchan->lock, irqflags);
|
|
|
|
/* Move descriptor to queued */
|
|
list_move_tail(&mdesc->node, &mchan->queued);
|
|
|
|
/* Update cookie */
|
|
cookie = dma_cookie_assign(txd);
|
|
|
|
spin_unlock_irqrestore(&mchan->lock, irqflags);
|
|
|
|
return cookie;
|
|
}
|
|
|
|
static int hidma_alloc_chan_resources(struct dma_chan *dmach)
|
|
{
|
|
struct hidma_chan *mchan = to_hidma_chan(dmach);
|
|
struct hidma_dev *dmadev = mchan->dmadev;
|
|
struct hidma_desc *mdesc, *tmp;
|
|
unsigned long irqflags;
|
|
LIST_HEAD(descs);
|
|
unsigned int i;
|
|
int rc = 0;
|
|
|
|
if (mchan->allocated)
|
|
return 0;
|
|
|
|
/* Alloc descriptors for this channel */
|
|
for (i = 0; i < dmadev->nr_descriptors; i++) {
|
|
mdesc = kzalloc(sizeof(struct hidma_desc), GFP_NOWAIT);
|
|
if (!mdesc) {
|
|
rc = -ENOMEM;
|
|
break;
|
|
}
|
|
dma_async_tx_descriptor_init(&mdesc->desc, dmach);
|
|
mdesc->desc.tx_submit = hidma_tx_submit;
|
|
|
|
rc = hidma_ll_request(dmadev->lldev, mchan->dma_sig,
|
|
"DMA engine", hidma_callback, mdesc,
|
|
&mdesc->tre_ch);
|
|
if (rc) {
|
|
dev_err(dmach->device->dev,
|
|
"channel alloc failed at %u\n", i);
|
|
kfree(mdesc);
|
|
break;
|
|
}
|
|
list_add_tail(&mdesc->node, &descs);
|
|
}
|
|
|
|
if (rc) {
|
|
/* return the allocated descriptors */
|
|
list_for_each_entry_safe(mdesc, tmp, &descs, node) {
|
|
hidma_ll_free(dmadev->lldev, mdesc->tre_ch);
|
|
kfree(mdesc);
|
|
}
|
|
return rc;
|
|
}
|
|
|
|
spin_lock_irqsave(&mchan->lock, irqflags);
|
|
list_splice_tail_init(&descs, &mchan->free);
|
|
mchan->allocated = true;
|
|
spin_unlock_irqrestore(&mchan->lock, irqflags);
|
|
return 1;
|
|
}
|
|
|
|
static struct dma_async_tx_descriptor *
|
|
hidma_prep_dma_memcpy(struct dma_chan *dmach, dma_addr_t dest, dma_addr_t src,
|
|
size_t len, unsigned long flags)
|
|
{
|
|
struct hidma_chan *mchan = to_hidma_chan(dmach);
|
|
struct hidma_desc *mdesc = NULL;
|
|
struct hidma_dev *mdma = mchan->dmadev;
|
|
unsigned long irqflags;
|
|
|
|
/* Get free descriptor */
|
|
spin_lock_irqsave(&mchan->lock, irqflags);
|
|
if (!list_empty(&mchan->free)) {
|
|
mdesc = list_first_entry(&mchan->free, struct hidma_desc, node);
|
|
list_del(&mdesc->node);
|
|
}
|
|
spin_unlock_irqrestore(&mchan->lock, irqflags);
|
|
|
|
if (!mdesc)
|
|
return NULL;
|
|
|
|
mdesc->desc.flags = flags;
|
|
hidma_ll_set_transfer_params(mdma->lldev, mdesc->tre_ch,
|
|
src, dest, len, flags,
|
|
HIDMA_TRE_MEMCPY);
|
|
|
|
/* Place descriptor in prepared list */
|
|
spin_lock_irqsave(&mchan->lock, irqflags);
|
|
list_add_tail(&mdesc->node, &mchan->prepared);
|
|
spin_unlock_irqrestore(&mchan->lock, irqflags);
|
|
|
|
return &mdesc->desc;
|
|
}
|
|
|
|
static struct dma_async_tx_descriptor *
|
|
hidma_prep_dma_memset(struct dma_chan *dmach, dma_addr_t dest, int value,
|
|
size_t len, unsigned long flags)
|
|
{
|
|
struct hidma_chan *mchan = to_hidma_chan(dmach);
|
|
struct hidma_desc *mdesc = NULL;
|
|
struct hidma_dev *mdma = mchan->dmadev;
|
|
unsigned long irqflags;
|
|
|
|
/* Get free descriptor */
|
|
spin_lock_irqsave(&mchan->lock, irqflags);
|
|
if (!list_empty(&mchan->free)) {
|
|
mdesc = list_first_entry(&mchan->free, struct hidma_desc, node);
|
|
list_del(&mdesc->node);
|
|
}
|
|
spin_unlock_irqrestore(&mchan->lock, irqflags);
|
|
|
|
if (!mdesc)
|
|
return NULL;
|
|
|
|
mdesc->desc.flags = flags;
|
|
hidma_ll_set_transfer_params(mdma->lldev, mdesc->tre_ch,
|
|
value, dest, len, flags,
|
|
HIDMA_TRE_MEMSET);
|
|
|
|
/* Place descriptor in prepared list */
|
|
spin_lock_irqsave(&mchan->lock, irqflags);
|
|
list_add_tail(&mdesc->node, &mchan->prepared);
|
|
spin_unlock_irqrestore(&mchan->lock, irqflags);
|
|
|
|
return &mdesc->desc;
|
|
}
|
|
|
|
static int hidma_terminate_channel(struct dma_chan *chan)
|
|
{
|
|
struct hidma_chan *mchan = to_hidma_chan(chan);
|
|
struct hidma_dev *dmadev = to_hidma_dev(mchan->chan.device);
|
|
struct hidma_desc *tmp, *mdesc;
|
|
unsigned long irqflags;
|
|
LIST_HEAD(list);
|
|
int rc;
|
|
|
|
pm_runtime_get_sync(dmadev->ddev.dev);
|
|
/* give completed requests a chance to finish */
|
|
hidma_process_completed(mchan);
|
|
|
|
spin_lock_irqsave(&mchan->lock, irqflags);
|
|
mchan->last_success = 0;
|
|
list_splice_init(&mchan->active, &list);
|
|
list_splice_init(&mchan->prepared, &list);
|
|
list_splice_init(&mchan->completed, &list);
|
|
list_splice_init(&mchan->queued, &list);
|
|
spin_unlock_irqrestore(&mchan->lock, irqflags);
|
|
|
|
/* this suspends the existing transfer */
|
|
rc = hidma_ll_disable(dmadev->lldev);
|
|
if (rc) {
|
|
dev_err(dmadev->ddev.dev, "channel did not pause\n");
|
|
goto out;
|
|
}
|
|
|
|
/* return all user requests */
|
|
list_for_each_entry_safe(mdesc, tmp, &list, node) {
|
|
struct dma_async_tx_descriptor *txd = &mdesc->desc;
|
|
|
|
dma_descriptor_unmap(txd);
|
|
dmaengine_desc_get_callback_invoke(txd, NULL);
|
|
dma_run_dependencies(txd);
|
|
|
|
/* move myself to free_list */
|
|
list_move(&mdesc->node, &mchan->free);
|
|
}
|
|
|
|
rc = hidma_ll_enable(dmadev->lldev);
|
|
out:
|
|
pm_runtime_mark_last_busy(dmadev->ddev.dev);
|
|
pm_runtime_put_autosuspend(dmadev->ddev.dev);
|
|
return rc;
|
|
}
|
|
|
|
static int hidma_terminate_all(struct dma_chan *chan)
|
|
{
|
|
struct hidma_chan *mchan = to_hidma_chan(chan);
|
|
struct hidma_dev *dmadev = to_hidma_dev(mchan->chan.device);
|
|
int rc;
|
|
|
|
rc = hidma_terminate_channel(chan);
|
|
if (rc)
|
|
return rc;
|
|
|
|
/* reinitialize the hardware */
|
|
pm_runtime_get_sync(dmadev->ddev.dev);
|
|
rc = hidma_ll_setup(dmadev->lldev);
|
|
pm_runtime_mark_last_busy(dmadev->ddev.dev);
|
|
pm_runtime_put_autosuspend(dmadev->ddev.dev);
|
|
return rc;
|
|
}
|
|
|
|
static void hidma_free_chan_resources(struct dma_chan *dmach)
|
|
{
|
|
struct hidma_chan *mchan = to_hidma_chan(dmach);
|
|
struct hidma_dev *mdma = mchan->dmadev;
|
|
struct hidma_desc *mdesc, *tmp;
|
|
unsigned long irqflags;
|
|
LIST_HEAD(descs);
|
|
|
|
/* terminate running transactions and free descriptors */
|
|
hidma_terminate_channel(dmach);
|
|
|
|
spin_lock_irqsave(&mchan->lock, irqflags);
|
|
|
|
/* Move data */
|
|
list_splice_tail_init(&mchan->free, &descs);
|
|
|
|
/* Free descriptors */
|
|
list_for_each_entry_safe(mdesc, tmp, &descs, node) {
|
|
hidma_ll_free(mdma->lldev, mdesc->tre_ch);
|
|
list_del(&mdesc->node);
|
|
kfree(mdesc);
|
|
}
|
|
|
|
mchan->allocated = 0;
|
|
spin_unlock_irqrestore(&mchan->lock, irqflags);
|
|
}
|
|
|
|
static int hidma_pause(struct dma_chan *chan)
|
|
{
|
|
struct hidma_chan *mchan;
|
|
struct hidma_dev *dmadev;
|
|
|
|
mchan = to_hidma_chan(chan);
|
|
dmadev = to_hidma_dev(mchan->chan.device);
|
|
if (!mchan->paused) {
|
|
pm_runtime_get_sync(dmadev->ddev.dev);
|
|
if (hidma_ll_disable(dmadev->lldev))
|
|
dev_warn(dmadev->ddev.dev, "channel did not stop\n");
|
|
mchan->paused = true;
|
|
pm_runtime_mark_last_busy(dmadev->ddev.dev);
|
|
pm_runtime_put_autosuspend(dmadev->ddev.dev);
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
static int hidma_resume(struct dma_chan *chan)
|
|
{
|
|
struct hidma_chan *mchan;
|
|
struct hidma_dev *dmadev;
|
|
int rc = 0;
|
|
|
|
mchan = to_hidma_chan(chan);
|
|
dmadev = to_hidma_dev(mchan->chan.device);
|
|
if (mchan->paused) {
|
|
pm_runtime_get_sync(dmadev->ddev.dev);
|
|
rc = hidma_ll_enable(dmadev->lldev);
|
|
if (!rc)
|
|
mchan->paused = false;
|
|
else
|
|
dev_err(dmadev->ddev.dev,
|
|
"failed to resume the channel");
|
|
pm_runtime_mark_last_busy(dmadev->ddev.dev);
|
|
pm_runtime_put_autosuspend(dmadev->ddev.dev);
|
|
}
|
|
return rc;
|
|
}
|
|
|
|
static irqreturn_t hidma_chirq_handler(int chirq, void *arg)
|
|
{
|
|
struct hidma_lldev *lldev = arg;
|
|
|
|
/*
|
|
* All interrupts are request driven.
|
|
* HW doesn't send an interrupt by itself.
|
|
*/
|
|
return hidma_ll_inthandler(chirq, lldev);
|
|
}
|
|
|
|
#ifdef CONFIG_GENERIC_MSI_IRQ_DOMAIN
|
|
static irqreturn_t hidma_chirq_handler_msi(int chirq, void *arg)
|
|
{
|
|
struct hidma_lldev **lldevp = arg;
|
|
struct hidma_dev *dmadev = to_hidma_dev_from_lldev(lldevp);
|
|
|
|
return hidma_ll_inthandler_msi(chirq, *lldevp,
|
|
1 << (chirq - dmadev->msi_virqbase));
|
|
}
|
|
#endif
|
|
|
|
static ssize_t hidma_show_values(struct device *dev,
|
|
struct device_attribute *attr, char *buf)
|
|
{
|
|
struct hidma_dev *mdev = dev_get_drvdata(dev);
|
|
|
|
buf[0] = 0;
|
|
|
|
if (strcmp(attr->attr.name, "chid") == 0)
|
|
sprintf(buf, "%d\n", mdev->chidx);
|
|
|
|
return strlen(buf);
|
|
}
|
|
|
|
static inline void hidma_sysfs_uninit(struct hidma_dev *dev)
|
|
{
|
|
device_remove_file(dev->ddev.dev, dev->chid_attrs);
|
|
}
|
|
|
|
static struct device_attribute*
|
|
hidma_create_sysfs_entry(struct hidma_dev *dev, char *name, int mode)
|
|
{
|
|
struct device_attribute *attrs;
|
|
char *name_copy;
|
|
|
|
attrs = devm_kmalloc(dev->ddev.dev, sizeof(struct device_attribute),
|
|
GFP_KERNEL);
|
|
if (!attrs)
|
|
return NULL;
|
|
|
|
name_copy = devm_kstrdup(dev->ddev.dev, name, GFP_KERNEL);
|
|
if (!name_copy)
|
|
return NULL;
|
|
|
|
attrs->attr.name = name_copy;
|
|
attrs->attr.mode = mode;
|
|
attrs->show = hidma_show_values;
|
|
sysfs_attr_init(&attrs->attr);
|
|
|
|
return attrs;
|
|
}
|
|
|
|
static int hidma_sysfs_init(struct hidma_dev *dev)
|
|
{
|
|
dev->chid_attrs = hidma_create_sysfs_entry(dev, "chid", S_IRUGO);
|
|
if (!dev->chid_attrs)
|
|
return -ENOMEM;
|
|
|
|
return device_create_file(dev->ddev.dev, dev->chid_attrs);
|
|
}
|
|
|
|
#ifdef CONFIG_GENERIC_MSI_IRQ_DOMAIN
|
|
static void hidma_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg)
|
|
{
|
|
struct device *dev = msi_desc_to_dev(desc);
|
|
struct hidma_dev *dmadev = dev_get_drvdata(dev);
|
|
|
|
if (!desc->platform.msi_index) {
|
|
writel(msg->address_lo, dmadev->dev_evca + 0x118);
|
|
writel(msg->address_hi, dmadev->dev_evca + 0x11C);
|
|
writel(msg->data, dmadev->dev_evca + 0x120);
|
|
}
|
|
}
|
|
#endif
|
|
|
|
static void hidma_free_msis(struct hidma_dev *dmadev)
|
|
{
|
|
#ifdef CONFIG_GENERIC_MSI_IRQ_DOMAIN
|
|
struct device *dev = dmadev->ddev.dev;
|
|
struct msi_desc *desc;
|
|
|
|
/* free allocated MSI interrupts above */
|
|
for_each_msi_entry(desc, dev)
|
|
devm_free_irq(dev, desc->irq, &dmadev->lldev);
|
|
|
|
platform_msi_domain_free_irqs(dev);
|
|
#endif
|
|
}
|
|
|
|
static int hidma_request_msi(struct hidma_dev *dmadev,
|
|
struct platform_device *pdev)
|
|
{
|
|
#ifdef CONFIG_GENERIC_MSI_IRQ_DOMAIN
|
|
int rc;
|
|
struct msi_desc *desc;
|
|
struct msi_desc *failed_desc = NULL;
|
|
|
|
rc = platform_msi_domain_alloc_irqs(&pdev->dev, HIDMA_MSI_INTS,
|
|
hidma_write_msi_msg);
|
|
if (rc)
|
|
return rc;
|
|
|
|
for_each_msi_entry(desc, &pdev->dev) {
|
|
if (!desc->platform.msi_index)
|
|
dmadev->msi_virqbase = desc->irq;
|
|
|
|
rc = devm_request_irq(&pdev->dev, desc->irq,
|
|
hidma_chirq_handler_msi,
|
|
0, "qcom-hidma-msi",
|
|
&dmadev->lldev);
|
|
if (rc) {
|
|
failed_desc = desc;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (rc) {
|
|
/* free allocated MSI interrupts above */
|
|
for_each_msi_entry(desc, &pdev->dev) {
|
|
if (desc == failed_desc)
|
|
break;
|
|
devm_free_irq(&pdev->dev, desc->irq,
|
|
&dmadev->lldev);
|
|
}
|
|
} else {
|
|
/* Add callback to free MSIs on teardown */
|
|
hidma_ll_setup_irq(dmadev->lldev, true);
|
|
|
|
}
|
|
if (rc)
|
|
dev_warn(&pdev->dev,
|
|
"failed to request MSI irq, falling back to wired IRQ\n");
|
|
return rc;
|
|
#else
|
|
return -EINVAL;
|
|
#endif
|
|
}
|
|
|
|
static bool hidma_test_capability(struct device *dev, enum hidma_cap test_cap)
|
|
{
|
|
enum hidma_cap cap;
|
|
|
|
cap = (enum hidma_cap) device_get_match_data(dev);
|
|
return cap ? ((cap & test_cap) > 0) : 0;
|
|
}
|
|
|
|
static int hidma_probe(struct platform_device *pdev)
|
|
{
|
|
struct hidma_dev *dmadev;
|
|
struct resource *trca_resource;
|
|
struct resource *evca_resource;
|
|
int chirq;
|
|
void __iomem *evca;
|
|
void __iomem *trca;
|
|
int rc;
|
|
bool msi;
|
|
|
|
pm_runtime_set_autosuspend_delay(&pdev->dev, HIDMA_AUTOSUSPEND_TIMEOUT);
|
|
pm_runtime_use_autosuspend(&pdev->dev);
|
|
pm_runtime_set_active(&pdev->dev);
|
|
pm_runtime_enable(&pdev->dev);
|
|
|
|
trca_resource = platform_get_resource(pdev, IORESOURCE_MEM, 0);
|
|
trca = devm_ioremap_resource(&pdev->dev, trca_resource);
|
|
if (IS_ERR(trca)) {
|
|
rc = -ENOMEM;
|
|
goto bailout;
|
|
}
|
|
|
|
evca_resource = platform_get_resource(pdev, IORESOURCE_MEM, 1);
|
|
evca = devm_ioremap_resource(&pdev->dev, evca_resource);
|
|
if (IS_ERR(evca)) {
|
|
rc = -ENOMEM;
|
|
goto bailout;
|
|
}
|
|
|
|
/*
|
|
* This driver only handles the channel IRQs.
|
|
* Common IRQ is handled by the management driver.
|
|
*/
|
|
chirq = platform_get_irq(pdev, 0);
|
|
if (chirq < 0) {
|
|
rc = -ENODEV;
|
|
goto bailout;
|
|
}
|
|
|
|
dmadev = devm_kzalloc(&pdev->dev, sizeof(*dmadev), GFP_KERNEL);
|
|
if (!dmadev) {
|
|
rc = -ENOMEM;
|
|
goto bailout;
|
|
}
|
|
|
|
INIT_LIST_HEAD(&dmadev->ddev.channels);
|
|
spin_lock_init(&dmadev->lock);
|
|
dmadev->ddev.dev = &pdev->dev;
|
|
pm_runtime_get_sync(dmadev->ddev.dev);
|
|
|
|
dma_cap_set(DMA_MEMCPY, dmadev->ddev.cap_mask);
|
|
dma_cap_set(DMA_MEMSET, dmadev->ddev.cap_mask);
|
|
if (WARN_ON(!pdev->dev.dma_mask)) {
|
|
rc = -ENXIO;
|
|
goto dmafree;
|
|
}
|
|
|
|
dmadev->dev_evca = evca;
|
|
dmadev->evca_resource = evca_resource;
|
|
dmadev->dev_trca = trca;
|
|
dmadev->trca_resource = trca_resource;
|
|
dmadev->ddev.device_prep_dma_memcpy = hidma_prep_dma_memcpy;
|
|
dmadev->ddev.device_prep_dma_memset = hidma_prep_dma_memset;
|
|
dmadev->ddev.device_alloc_chan_resources = hidma_alloc_chan_resources;
|
|
dmadev->ddev.device_free_chan_resources = hidma_free_chan_resources;
|
|
dmadev->ddev.device_tx_status = hidma_tx_status;
|
|
dmadev->ddev.device_issue_pending = hidma_issue_pending;
|
|
dmadev->ddev.device_pause = hidma_pause;
|
|
dmadev->ddev.device_resume = hidma_resume;
|
|
dmadev->ddev.device_terminate_all = hidma_terminate_all;
|
|
dmadev->ddev.copy_align = 8;
|
|
|
|
/*
|
|
* Determine the MSI capability of the platform. Old HW doesn't
|
|
* support MSI.
|
|
*/
|
|
msi = hidma_test_capability(&pdev->dev, HIDMA_MSI_CAP);
|
|
device_property_read_u32(&pdev->dev, "desc-count",
|
|
&dmadev->nr_descriptors);
|
|
|
|
if (nr_desc_prm) {
|
|
dev_info(&pdev->dev, "overriding number of descriptors as %d\n",
|
|
nr_desc_prm);
|
|
dmadev->nr_descriptors = nr_desc_prm;
|
|
}
|
|
|
|
if (!dmadev->nr_descriptors)
|
|
dmadev->nr_descriptors = HIDMA_NR_DEFAULT_DESC;
|
|
|
|
if (hidma_test_capability(&pdev->dev, HIDMA_IDENTITY_CAP))
|
|
dmadev->chidx = readl(dmadev->dev_trca + 0x40);
|
|
else
|
|
dmadev->chidx = readl(dmadev->dev_trca + 0x28);
|
|
|
|
/* Set DMA mask to 64 bits. */
|
|
rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
|
|
if (rc) {
|
|
dev_warn(&pdev->dev, "unable to set coherent mask to 64");
|
|
rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
|
|
if (rc)
|
|
goto dmafree;
|
|
}
|
|
|
|
dmadev->lldev = hidma_ll_init(dmadev->ddev.dev,
|
|
dmadev->nr_descriptors, dmadev->dev_trca,
|
|
dmadev->dev_evca, dmadev->chidx);
|
|
if (!dmadev->lldev) {
|
|
rc = -EPROBE_DEFER;
|
|
goto dmafree;
|
|
}
|
|
|
|
platform_set_drvdata(pdev, dmadev);
|
|
if (msi)
|
|
rc = hidma_request_msi(dmadev, pdev);
|
|
|
|
if (!msi || rc) {
|
|
hidma_ll_setup_irq(dmadev->lldev, false);
|
|
rc = devm_request_irq(&pdev->dev, chirq, hidma_chirq_handler,
|
|
0, "qcom-hidma", dmadev->lldev);
|
|
if (rc)
|
|
goto uninit;
|
|
}
|
|
|
|
INIT_LIST_HEAD(&dmadev->ddev.channels);
|
|
rc = hidma_chan_init(dmadev, 0);
|
|
if (rc)
|
|
goto uninit;
|
|
|
|
rc = dma_async_device_register(&dmadev->ddev);
|
|
if (rc)
|
|
goto uninit;
|
|
|
|
dmadev->irq = chirq;
|
|
tasklet_init(&dmadev->task, hidma_issue_task, (unsigned long)dmadev);
|
|
hidma_debug_init(dmadev);
|
|
hidma_sysfs_init(dmadev);
|
|
dev_info(&pdev->dev, "HI-DMA engine driver registration complete\n");
|
|
pm_runtime_mark_last_busy(dmadev->ddev.dev);
|
|
pm_runtime_put_autosuspend(dmadev->ddev.dev);
|
|
return 0;
|
|
|
|
uninit:
|
|
if (msi)
|
|
hidma_free_msis(dmadev);
|
|
|
|
hidma_debug_uninit(dmadev);
|
|
hidma_ll_uninit(dmadev->lldev);
|
|
dmafree:
|
|
if (dmadev)
|
|
hidma_free(dmadev);
|
|
bailout:
|
|
pm_runtime_put_sync(&pdev->dev);
|
|
pm_runtime_disable(&pdev->dev);
|
|
return rc;
|
|
}
|
|
|
|
static void hidma_shutdown(struct platform_device *pdev)
|
|
{
|
|
struct hidma_dev *dmadev = platform_get_drvdata(pdev);
|
|
|
|
dev_info(dmadev->ddev.dev, "HI-DMA engine shutdown\n");
|
|
|
|
pm_runtime_get_sync(dmadev->ddev.dev);
|
|
if (hidma_ll_disable(dmadev->lldev))
|
|
dev_warn(dmadev->ddev.dev, "channel did not stop\n");
|
|
pm_runtime_mark_last_busy(dmadev->ddev.dev);
|
|
pm_runtime_put_autosuspend(dmadev->ddev.dev);
|
|
|
|
}
|
|
|
|
static int hidma_remove(struct platform_device *pdev)
|
|
{
|
|
struct hidma_dev *dmadev = platform_get_drvdata(pdev);
|
|
|
|
pm_runtime_get_sync(dmadev->ddev.dev);
|
|
dma_async_device_unregister(&dmadev->ddev);
|
|
if (!dmadev->lldev->msi_support)
|
|
devm_free_irq(dmadev->ddev.dev, dmadev->irq, dmadev->lldev);
|
|
else
|
|
hidma_free_msis(dmadev);
|
|
|
|
tasklet_kill(&dmadev->task);
|
|
hidma_sysfs_uninit(dmadev);
|
|
hidma_debug_uninit(dmadev);
|
|
hidma_ll_uninit(dmadev->lldev);
|
|
hidma_free(dmadev);
|
|
|
|
dev_info(&pdev->dev, "HI-DMA engine removed\n");
|
|
pm_runtime_put_sync_suspend(&pdev->dev);
|
|
pm_runtime_disable(&pdev->dev);
|
|
|
|
return 0;
|
|
}
|
|
|
|
#if IS_ENABLED(CONFIG_ACPI)
|
|
static const struct acpi_device_id hidma_acpi_ids[] = {
|
|
{"QCOM8061"},
|
|
{"QCOM8062", HIDMA_MSI_CAP},
|
|
{"QCOM8063", (HIDMA_MSI_CAP | HIDMA_IDENTITY_CAP)},
|
|
{},
|
|
};
|
|
MODULE_DEVICE_TABLE(acpi, hidma_acpi_ids);
|
|
#endif
|
|
|
|
static const struct of_device_id hidma_match[] = {
|
|
{.compatible = "qcom,hidma-1.0",},
|
|
{.compatible = "qcom,hidma-1.1", .data = (void *)(HIDMA_MSI_CAP),},
|
|
{.compatible = "qcom,hidma-1.2",
|
|
.data = (void *)(HIDMA_MSI_CAP | HIDMA_IDENTITY_CAP),},
|
|
{},
|
|
};
|
|
MODULE_DEVICE_TABLE(of, hidma_match);
|
|
|
|
static struct platform_driver hidma_driver = {
|
|
.probe = hidma_probe,
|
|
.remove = hidma_remove,
|
|
.shutdown = hidma_shutdown,
|
|
.driver = {
|
|
.name = "hidma",
|
|
.of_match_table = hidma_match,
|
|
.acpi_match_table = ACPI_PTR(hidma_acpi_ids),
|
|
},
|
|
};
|
|
|
|
module_platform_driver(hidma_driver);
|
|
MODULE_LICENSE("GPL v2");
|