linux_dsm_epyc7002/drivers/dma/mediatek/mtk-cqdma.c

// SPDX-License-Identifier: GPL-2.0
// Copyright (c) 2018-2019 MediaTek Inc.

/*
 * Driver for MediaTek Command-Queue DMA Controller
 *
 * Author: Shun-Chih Yu <shun-chih.yu@mediatek.com>
 *
 */

#include <linux/bitops.h>
#include <linux/clk.h>
#include <linux/dmaengine.h>
#include <linux/dma-mapping.h>
#include <linux/err.h>
#include <linux/iopoll.h>
#include <linux/interrupt.h>
#include <linux/list.h>
#include <linux/module.h>
#include <linux/of.h>
#include <linux/of_device.h>
#include <linux/of_dma.h>
#include <linux/platform_device.h>
#include <linux/pm_runtime.h>
#include <linux/refcount.h>
#include <linux/slab.h>

#include "../virt-dma.h"

#define MTK_CQDMA_USEC_POLL		10
#define MTK_CQDMA_TIMEOUT_POLL		1000
#define MTK_CQDMA_DMA_BUSWIDTHS		BIT(DMA_SLAVE_BUSWIDTH_4_BYTES)
#define MTK_CQDMA_ALIGN_SIZE		1

/* The default number of virtual channel */
#define MTK_CQDMA_NR_VCHANS		32

/* The default number of physical channel */
#define MTK_CQDMA_NR_PCHANS		3

/* Registers for underlying dma manipulation */
#define MTK_CQDMA_INT_FLAG		0x0
#define MTK_CQDMA_INT_EN		0x4
#define MTK_CQDMA_EN			0x8
#define MTK_CQDMA_RESET			0xc
#define MTK_CQDMA_FLUSH			0x14
#define MTK_CQDMA_SRC			0x1c
#define MTK_CQDMA_DST			0x20
#define MTK_CQDMA_LEN1			0x24
#define MTK_CQDMA_LEN2			0x28
#define MTK_CQDMA_SRC2			0x60
#define MTK_CQDMA_DST2			0x64

/* Registers setting */
#define MTK_CQDMA_EN_BIT		BIT(0)
#define MTK_CQDMA_INT_FLAG_BIT		BIT(0)
#define MTK_CQDMA_INT_EN_BIT		BIT(0)
#define MTK_CQDMA_FLUSH_BIT		BIT(0)

#define MTK_CQDMA_WARM_RST_BIT		BIT(0)
#define MTK_CQDMA_HARD_RST_BIT		BIT(1)

#define MTK_CQDMA_MAX_LEN		GENMASK(27, 0)
#define MTK_CQDMA_ADDR_LIMIT		GENMASK(31, 0)
#define MTK_CQDMA_ADDR2_SHFIT		(32)

/**
 * struct mtk_cqdma_vdesc - The struct holding info describing virtual
 *                         descriptor (CVD)
 * @vd:                    An instance for struct virt_dma_desc
 * @len:                   The total data size device wants to move
 * @residue:               The remaining data size device will move
 * @dest:                  The destination address device wants to move to
 * @src:                   The source address device wants to move from
 * @ch:                    The pointer to the corresponding dma channel
 * @node:                  The lise_head struct to build link-list for VDs
 * @parent:                The pointer to the parent CVD
 */
struct mtk_cqdma_vdesc {
	struct virt_dma_desc vd;
	size_t len;
	size_t residue;
	dma_addr_t dest;
	dma_addr_t src;
	struct dma_chan *ch;

	struct list_head node;
	struct mtk_cqdma_vdesc *parent;
};

/**
 * struct mtk_cqdma_pchan - The struct holding info describing physical
 *                         channel (PC)
 * @queue:                 Queue for the PDs issued to this PC
 * @base:                  The mapped register I/O base of this PC
 * @irq:                   The IRQ that this PC are using
 * @refcnt:                Track how many VCs are using this PC
 * @tasklet:               Tasklet for this PC
 * @lock:                  Lock protect agaisting multiple VCs access PC
 */
struct mtk_cqdma_pchan {
	struct list_head queue;
	void __iomem *base;
	u32 irq;

	refcount_t refcnt;

	struct tasklet_struct tasklet;

	/* lock to protect PC */
	spinlock_t lock;
};

/**
 * struct mtk_cqdma_vchan - The struct holding info describing virtual
 *                         channel (VC)
 * @vc:                    An instance for struct virt_dma_chan
 * @pc:                    The pointer to the underlying PC
 * @issue_completion:	   The wait for all issued descriptors completited
 * @issue_synchronize:	   Bool indicating channel synchronization starts
 */
struct mtk_cqdma_vchan {
	struct virt_dma_chan vc;
	struct mtk_cqdma_pchan *pc;
	struct completion issue_completion;
	bool issue_synchronize;
};

/**
 * struct mtk_cqdma_device - The struct holding info describing CQDMA
 *                          device
 * @ddev:                   An instance for struct dma_device
 * @clk:                    The clock that device internal is using
 * @dma_requests:           The number of VCs the device supports to
 * @dma_channels:           The number of PCs the device supports to
 * @vc:                     The pointer to all available VCs
 * @pc:                     The pointer to all the underlying PCs
 */
struct mtk_cqdma_device {
	struct dma_device ddev;
	struct clk *clk;

	u32 dma_requests;
	u32 dma_channels;
	struct mtk_cqdma_vchan *vc;
	struct mtk_cqdma_pchan **pc;
};

static struct mtk_cqdma_device *to_cqdma_dev(struct dma_chan *chan)
{
	return container_of(chan->device, struct mtk_cqdma_device, ddev);
}

static struct mtk_cqdma_vchan *to_cqdma_vchan(struct dma_chan *chan)
{
	return container_of(chan, struct mtk_cqdma_vchan, vc.chan);
}

static struct mtk_cqdma_vdesc *to_cqdma_vdesc(struct virt_dma_desc *vd)
{
	return container_of(vd, struct mtk_cqdma_vdesc, vd);
}

static struct device *cqdma2dev(struct mtk_cqdma_device *cqdma)
{
	return cqdma->ddev.dev;
}

static u32 mtk_dma_read(struct mtk_cqdma_pchan *pc, u32 reg)
{
	return readl(pc->base + reg);
}

static void mtk_dma_write(struct mtk_cqdma_pchan *pc, u32 reg, u32 val)
{
	writel_relaxed(val, pc->base + reg);
}

static void mtk_dma_rmw(struct mtk_cqdma_pchan *pc, u32 reg,
			u32 mask, u32 set)
{
	u32 val;

	val = mtk_dma_read(pc, reg);
	val &= ~mask;
	val |= set;
	mtk_dma_write(pc, reg, val);
}

static void mtk_dma_set(struct mtk_cqdma_pchan *pc, u32 reg, u32 val)
{
	mtk_dma_rmw(pc, reg, 0, val);
}

static void mtk_dma_clr(struct mtk_cqdma_pchan *pc, u32 reg, u32 val)
{
	mtk_dma_rmw(pc, reg, val, 0);
}

static void mtk_cqdma_vdesc_free(struct virt_dma_desc *vd)
{
	kfree(to_cqdma_vdesc(vd));
}

static int mtk_cqdma_poll_engine_done(struct mtk_cqdma_pchan *pc, bool atomic)
{
	u32 status = 0;

	if (!atomic)
		return readl_poll_timeout(pc->base + MTK_CQDMA_EN,
					  status,
					  !(status & MTK_CQDMA_EN_BIT),
					  MTK_CQDMA_USEC_POLL,
					  MTK_CQDMA_TIMEOUT_POLL);

	return readl_poll_timeout_atomic(pc->base + MTK_CQDMA_EN,
					 status,
					 !(status & MTK_CQDMA_EN_BIT),
					 MTK_CQDMA_USEC_POLL,
					 MTK_CQDMA_TIMEOUT_POLL);
}

static int mtk_cqdma_hard_reset(struct mtk_cqdma_pchan *pc)
{
	mtk_dma_set(pc, MTK_CQDMA_RESET, MTK_CQDMA_HARD_RST_BIT);
	mtk_dma_clr(pc, MTK_CQDMA_RESET, MTK_CQDMA_HARD_RST_BIT);

	return mtk_cqdma_poll_engine_done(pc, true);
}

static void mtk_cqdma_start(struct mtk_cqdma_pchan *pc,
			    struct mtk_cqdma_vdesc *cvd)
{
	/* wait for the previous transaction done */
	if (mtk_cqdma_poll_engine_done(pc, true) < 0)
		dev_err(cqdma2dev(to_cqdma_dev(cvd->ch)), "cqdma wait transaction timeout\n");

	/* warm reset the dma engine for the new transaction */
	mtk_dma_set(pc, MTK_CQDMA_RESET, MTK_CQDMA_WARM_RST_BIT);
	if (mtk_cqdma_poll_engine_done(pc, true) < 0)
		dev_err(cqdma2dev(to_cqdma_dev(cvd->ch)), "cqdma warm reset timeout\n");

	/* setup the source */
	mtk_dma_set(pc, MTK_CQDMA_SRC, cvd->src & MTK_CQDMA_ADDR_LIMIT);
#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
	mtk_dma_set(pc, MTK_CQDMA_SRC2, cvd->src >> MTK_CQDMA_ADDR2_SHFIT);
#else
	mtk_dma_set(pc, MTK_CQDMA_SRC2, 0);
#endif

	/* setup the destination */
	mtk_dma_set(pc, MTK_CQDMA_DST, cvd->dest & MTK_CQDMA_ADDR_LIMIT);
#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
	mtk_dma_set(pc, MTK_CQDMA_DST2, cvd->dest >> MTK_CQDMA_ADDR2_SHFIT);
#else
	mtk_dma_set(pc, MTK_CQDMA_DST2, 0);
#endif

	/* setup the length */
	mtk_dma_set(pc, MTK_CQDMA_LEN1, cvd->len);

	/* start dma engine */
	mtk_dma_set(pc, MTK_CQDMA_EN, MTK_CQDMA_EN_BIT);
}

static void mtk_cqdma_issue_vchan_pending(struct mtk_cqdma_vchan *cvc)
{
	struct virt_dma_desc *vd, *vd2;
	struct mtk_cqdma_pchan *pc = cvc->pc;
	struct mtk_cqdma_vdesc *cvd;
	bool trigger_engine = false;

	lockdep_assert_held(&cvc->vc.lock);
	lockdep_assert_held(&pc->lock);

	list_for_each_entry_safe(vd, vd2, &cvc->vc.desc_issued, node) {
		/* need to trigger dma engine if PC's queue is empty */
		if (list_empty(&pc->queue))
			trigger_engine = true;

		cvd = to_cqdma_vdesc(vd);

		/* add VD into PC's queue */
		list_add_tail(&cvd->node, &pc->queue);

		/* start the dma engine */
		if (trigger_engine)
			mtk_cqdma_start(pc, cvd);

		/* remove VD from list desc_issued */
		list_del(&vd->node);
	}
}

/*
 * return true if this VC is active,
 * meaning that there are VDs under processing by the PC
 */
static bool mtk_cqdma_is_vchan_active(struct mtk_cqdma_vchan *cvc)
{
	struct mtk_cqdma_vdesc *cvd;

	list_for_each_entry(cvd, &cvc->pc->queue, node)
		if (cvc == to_cqdma_vchan(cvd->ch))
			return true;

	return false;
}

/*
 * return the pointer of the CVD that is just consumed by the PC
 */
static struct mtk_cqdma_vdesc
*mtk_cqdma_consume_work_queue(struct mtk_cqdma_pchan *pc)
{
	struct mtk_cqdma_vchan *cvc;
	struct mtk_cqdma_vdesc *cvd, *ret = NULL;

	/* consume a CVD from PC's queue */
	cvd = list_first_entry_or_null(&pc->queue,
				       struct mtk_cqdma_vdesc, node);
	if (unlikely(!cvd || !cvd->parent))
		return NULL;

	cvc = to_cqdma_vchan(cvd->ch);
	ret = cvd;

	/* update residue of the parent CVD */
	cvd->parent->residue -= cvd->len;

	/* delete CVD from PC's queue */
	list_del(&cvd->node);

	spin_lock(&cvc->vc.lock);

	/* check whether all the child CVDs completed */
	if (!cvd->parent->residue) {
		/* add the parent VD into list desc_completed */
		vchan_cookie_complete(&cvd->parent->vd);

		/* setup completion if this VC is under synchronization */
		if (cvc->issue_synchronize && !mtk_cqdma_is_vchan_active(cvc)) {
			complete(&cvc->issue_completion);
			cvc->issue_synchronize = false;
		}
	}

	spin_unlock(&cvc->vc.lock);

	/* start transaction for next CVD in the queue */
	cvd = list_first_entry_or_null(&pc->queue,
				       struct mtk_cqdma_vdesc, node);
	if (cvd)
		mtk_cqdma_start(pc, cvd);

	return ret;
}

static void mtk_cqdma_tasklet_cb(unsigned long data)
{
	struct mtk_cqdma_pchan *pc = (struct mtk_cqdma_pchan *)data;
	struct mtk_cqdma_vdesc *cvd = NULL;
	unsigned long flags;

	spin_lock_irqsave(&pc->lock, flags);
	/* consume the queue */
	cvd = mtk_cqdma_consume_work_queue(pc);
	spin_unlock_irqrestore(&pc->lock, flags);

	/* submit the next CVD */
	if (cvd) {
		dma_run_dependencies(&cvd->vd.tx);

		/*
		 * free child CVD after completion.
		 * the parent CVD would be freeed with desc_free by user.
		 */
		if (cvd->parent != cvd)
			kfree(cvd);
	}

	/* re-enable interrupt before leaving tasklet */
	enable_irq(pc->irq);
}

static irqreturn_t mtk_cqdma_irq(int irq, void *devid)
{
	struct mtk_cqdma_device *cqdma = devid;
	irqreturn_t ret = IRQ_NONE;
	bool schedule_tasklet = false;
	u32 i;

	/* clear interrupt flags for each PC */
	for (i = 0; i < cqdma->dma_channels; ++i, schedule_tasklet = false) {
		spin_lock(&cqdma->pc[i]->lock);
		if (mtk_dma_read(cqdma->pc[i],
				 MTK_CQDMA_INT_FLAG) & MTK_CQDMA_INT_FLAG_BIT) {
			/* clear interrupt */
			mtk_dma_clr(cqdma->pc[i], MTK_CQDMA_INT_FLAG,
				    MTK_CQDMA_INT_FLAG_BIT);

			schedule_tasklet = true;
			ret = IRQ_HANDLED;
		}
		spin_unlock(&cqdma->pc[i]->lock);

		if (schedule_tasklet) {
			/* disable interrupt */
			disable_irq_nosync(cqdma->pc[i]->irq);

			/* schedule the tasklet to handle the transactions */
			tasklet_schedule(&cqdma->pc[i]->tasklet);
		}
	}

	return ret;
}

static struct virt_dma_desc *mtk_cqdma_find_active_desc(struct dma_chan *c,
							dma_cookie_t cookie)
{
	struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c);
	struct virt_dma_desc *vd;
	unsigned long flags;

	spin_lock_irqsave(&cvc->pc->lock, flags);
	list_for_each_entry(vd, &cvc->pc->queue, node)
		if (vd->tx.cookie == cookie) {
			spin_unlock_irqrestore(&cvc->pc->lock, flags);
			return vd;
		}
	spin_unlock_irqrestore(&cvc->pc->lock, flags);

	list_for_each_entry(vd, &cvc->vc.desc_issued, node)
		if (vd->tx.cookie == cookie)
			return vd;

	return NULL;
}

static enum dma_status mtk_cqdma_tx_status(struct dma_chan *c,
					   dma_cookie_t cookie,
					   struct dma_tx_state *txstate)
{
	struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c);
	struct mtk_cqdma_vdesc *cvd;
	struct virt_dma_desc *vd;
	enum dma_status ret;
	unsigned long flags;
	size_t bytes = 0;

	ret = dma_cookie_status(c, cookie, txstate);
	if (ret == DMA_COMPLETE || !txstate)
		return ret;

	spin_lock_irqsave(&cvc->vc.lock, flags);
	vd = mtk_cqdma_find_active_desc(c, cookie);
	spin_unlock_irqrestore(&cvc->vc.lock, flags);

	if (vd) {
		cvd = to_cqdma_vdesc(vd);
		bytes = cvd->residue;
	}

	dma_set_residue(txstate, bytes);

	return ret;
}

static void mtk_cqdma_issue_pending(struct dma_chan *c)
{
	struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c);
	unsigned long pc_flags;
	unsigned long vc_flags;

	/* acquire PC's lock before VS's lock for lock dependency in tasklet */
	spin_lock_irqsave(&cvc->pc->lock, pc_flags);
	spin_lock_irqsave(&cvc->vc.lock, vc_flags);

	if (vchan_issue_pending(&cvc->vc))
		mtk_cqdma_issue_vchan_pending(cvc);

	spin_unlock_irqrestore(&cvc->vc.lock, vc_flags);
	spin_unlock_irqrestore(&cvc->pc->lock, pc_flags);
}

static struct dma_async_tx_descriptor *
mtk_cqdma_prep_dma_memcpy(struct dma_chan *c, dma_addr_t dest,
			  dma_addr_t src, size_t len, unsigned long flags)
{
	struct mtk_cqdma_vdesc **cvd;
	struct dma_async_tx_descriptor *tx = NULL, *prev_tx = NULL;
	size_t i, tlen, nr_vd;

	/*
	 * In the case that trsanction length is larger than the
	 * DMA engine supports, a single memcpy transaction needs
	 * to be separated into several DMA transactions.
	 * Each DMA transaction would be described by a CVD,
	 * and the first one is referred as the parent CVD,
	 * while the others are child CVDs.
	 * The parent CVD's tx descriptor is the only tx descriptor
	 * returned to the DMA user, and it should not be completed
	 * until all the child CVDs completed.
	 */
	nr_vd = DIV_ROUND_UP(len, MTK_CQDMA_MAX_LEN);
	cvd = kcalloc(nr_vd, sizeof(*cvd), GFP_NOWAIT);
	if (!cvd)
		return NULL;

	for (i = 0; i < nr_vd; ++i) {
		cvd[i] = kzalloc(sizeof(*cvd[i]), GFP_NOWAIT);
		if (!cvd[i]) {
			for (; i > 0; --i)
				kfree(cvd[i - 1]);
			return NULL;
		}

		/* setup dma channel */
		cvd[i]->ch = c;

		/* setup sourece, destination, and length */
		tlen = (len > MTK_CQDMA_MAX_LEN) ? MTK_CQDMA_MAX_LEN : len;
		cvd[i]->len = tlen;
		cvd[i]->src = src;
		cvd[i]->dest = dest;

		/* setup tx descriptor */
		tx = vchan_tx_prep(to_virt_chan(c), &cvd[i]->vd, flags);
		tx->next = NULL;

		if (!i) {
			cvd[0]->residue = len;
		} else {
			prev_tx->next = tx;
			cvd[i]->residue = tlen;
		}

		cvd[i]->parent = cvd[0];

		/* update the src, dest, len, prev_tx for the next CVD */
		src += tlen;
		dest += tlen;
		len -= tlen;
		prev_tx = tx;
	}

	return &cvd[0]->vd.tx;
}

static void mtk_cqdma_free_inactive_desc(struct dma_chan *c)
{
	struct virt_dma_chan *vc = to_virt_chan(c);
	unsigned long flags;
	LIST_HEAD(head);

	/*
	 * set desc_allocated, desc_submitted,
	 * and desc_issued as the candicates to be freed
	 */
	spin_lock_irqsave(&vc->lock, flags);
	list_splice_tail_init(&vc->desc_allocated, &head);
	list_splice_tail_init(&vc->desc_submitted, &head);
	list_splice_tail_init(&vc->desc_issued, &head);
	spin_unlock_irqrestore(&vc->lock, flags);

	/* free descriptor lists */
	vchan_dma_desc_free_list(vc, &head);
}

static void mtk_cqdma_free_active_desc(struct dma_chan *c)
{
	struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c);
	bool sync_needed = false;
	unsigned long pc_flags;
	unsigned long vc_flags;

	/* acquire PC's lock first due to lock dependency in dma ISR */
	spin_lock_irqsave(&cvc->pc->lock, pc_flags);
	spin_lock_irqsave(&cvc->vc.lock, vc_flags);

	/* synchronization is required if this VC is active */
	if (mtk_cqdma_is_vchan_active(cvc)) {
		cvc->issue_synchronize = true;
		sync_needed = true;
	}

	spin_unlock_irqrestore(&cvc->vc.lock, vc_flags);
	spin_unlock_irqrestore(&cvc->pc->lock, pc_flags);

	/* waiting for the completion of this VC */
	if (sync_needed)
		wait_for_completion(&cvc->issue_completion);

	/* free all descriptors in list desc_completed */
	vchan_synchronize(&cvc->vc);

	WARN_ONCE(!list_empty(&cvc->vc.desc_completed),
		  "Desc pending still in list desc_completed\n");
}

static int mtk_cqdma_terminate_all(struct dma_chan *c)
{
	/* free descriptors not processed yet by hardware */
	mtk_cqdma_free_inactive_desc(c);

	/* free descriptors being processed by hardware */
	mtk_cqdma_free_active_desc(c);

	return 0;
}

static int mtk_cqdma_alloc_chan_resources(struct dma_chan *c)
{
	struct mtk_cqdma_device *cqdma = to_cqdma_dev(c);
	struct mtk_cqdma_vchan *vc = to_cqdma_vchan(c);
	struct mtk_cqdma_pchan *pc = NULL;
	u32 i, min_refcnt = U32_MAX, refcnt;
	unsigned long flags;

	/* allocate PC with the minimun refcount */
	for (i = 0; i < cqdma->dma_channels; ++i) {
		refcnt = refcount_read(&cqdma->pc[i]->refcnt);
		if (refcnt < min_refcnt) {
			pc = cqdma->pc[i];
			min_refcnt = refcnt;
		}
	}

	if (!pc)
		return -ENOSPC;

	spin_lock_irqsave(&pc->lock, flags);

	if (!refcount_read(&pc->refcnt)) {
		/* allocate PC when the refcount is zero */
		mtk_cqdma_hard_reset(pc);

		/* enable interrupt for this PC */
		mtk_dma_set(pc, MTK_CQDMA_INT_EN, MTK_CQDMA_INT_EN_BIT);

		/*
		 * refcount_inc would complain increment on 0; use-after-free.
		 * Thus, we need to explicitly set it as 1 initially.
		 */
		refcount_set(&pc->refcnt, 1);
	} else {
		refcount_inc(&pc->refcnt);
	}

	spin_unlock_irqrestore(&pc->lock, flags);

	vc->pc = pc;

	return 0;
}

static void mtk_cqdma_free_chan_resources(struct dma_chan *c)
{
	struct mtk_cqdma_vchan *cvc = to_cqdma_vchan(c);
	unsigned long flags;

	/* free all descriptors in all lists on the VC */
	mtk_cqdma_terminate_all(c);

	spin_lock_irqsave(&cvc->pc->lock, flags);

	/* PC is not freed until there is no VC mapped to it */
	if (refcount_dec_and_test(&cvc->pc->refcnt)) {
		/* start the flush operation and stop the engine */
		mtk_dma_set(cvc->pc, MTK_CQDMA_FLUSH, MTK_CQDMA_FLUSH_BIT);

		/* wait for the completion of flush operation */
		if (mtk_cqdma_poll_engine_done(cvc->pc, true) < 0)
			dev_err(cqdma2dev(to_cqdma_dev(c)), "cqdma flush timeout\n");

		/* clear the flush bit and interrupt flag */
		mtk_dma_clr(cvc->pc, MTK_CQDMA_FLUSH, MTK_CQDMA_FLUSH_BIT);
		mtk_dma_clr(cvc->pc, MTK_CQDMA_INT_FLAG,
			    MTK_CQDMA_INT_FLAG_BIT);

		/* disable interrupt for this PC */
		mtk_dma_clr(cvc->pc, MTK_CQDMA_INT_EN, MTK_CQDMA_INT_EN_BIT);
	}

	spin_unlock_irqrestore(&cvc->pc->lock, flags);
}

static int mtk_cqdma_hw_init(struct mtk_cqdma_device *cqdma)
{
	unsigned long flags;
	int err;
	u32 i;

	pm_runtime_enable(cqdma2dev(cqdma));
	pm_runtime_get_sync(cqdma2dev(cqdma));

	err = clk_prepare_enable(cqdma->clk);

	if (err) {
		pm_runtime_put_sync(cqdma2dev(cqdma));
		pm_runtime_disable(cqdma2dev(cqdma));
		return err;
	}

	/* reset all PCs */
	for (i = 0; i < cqdma->dma_channels; ++i) {
		spin_lock_irqsave(&cqdma->pc[i]->lock, flags);
		if (mtk_cqdma_hard_reset(cqdma->pc[i]) < 0) {
			dev_err(cqdma2dev(cqdma), "cqdma hard reset timeout\n");
			spin_unlock_irqrestore(&cqdma->pc[i]->lock, flags);

			clk_disable_unprepare(cqdma->clk);
			pm_runtime_put_sync(cqdma2dev(cqdma));
			pm_runtime_disable(cqdma2dev(cqdma));
			return -EINVAL;
		}
		spin_unlock_irqrestore(&cqdma->pc[i]->lock, flags);
	}

	return 0;
}

static void mtk_cqdma_hw_deinit(struct mtk_cqdma_device *cqdma)
{
	unsigned long flags;
	u32 i;

	/* reset all PCs */
	for (i = 0; i < cqdma->dma_channels; ++i) {
		spin_lock_irqsave(&cqdma->pc[i]->lock, flags);
		if (mtk_cqdma_hard_reset(cqdma->pc[i]) < 0)
			dev_err(cqdma2dev(cqdma), "cqdma hard reset timeout\n");
		spin_unlock_irqrestore(&cqdma->pc[i]->lock, flags);
	}

	clk_disable_unprepare(cqdma->clk);

	pm_runtime_put_sync(cqdma2dev(cqdma));
	pm_runtime_disable(cqdma2dev(cqdma));
}

static const struct of_device_id mtk_cqdma_match[] = {
	{ .compatible = "mediatek,mt6765-cqdma" },
	{ /* sentinel */ }
};
MODULE_DEVICE_TABLE(of, mtk_cqdma_match);

static int mtk_cqdma_probe(struct platform_device *pdev)
{
	struct mtk_cqdma_device *cqdma;
	struct mtk_cqdma_vchan *vc;
	struct dma_device *dd;
	struct resource *res;
	int err;
	u32 i;

	cqdma = devm_kzalloc(&pdev->dev, sizeof(*cqdma), GFP_KERNEL);
	if (!cqdma)
		return -ENOMEM;

	dd = &cqdma->ddev;

	cqdma->clk = devm_clk_get(&pdev->dev, "cqdma");
	if (IS_ERR(cqdma->clk)) {
		dev_err(&pdev->dev, "No clock for %s\n",
			dev_name(&pdev->dev));
		return PTR_ERR(cqdma->clk);
	}

	dma_cap_set(DMA_MEMCPY, dd->cap_mask);

	dd->copy_align = MTK_CQDMA_ALIGN_SIZE;
	dd->device_alloc_chan_resources = mtk_cqdma_alloc_chan_resources;
	dd->device_free_chan_resources = mtk_cqdma_free_chan_resources;
	dd->device_tx_status = mtk_cqdma_tx_status;
	dd->device_issue_pending = mtk_cqdma_issue_pending;
	dd->device_prep_dma_memcpy = mtk_cqdma_prep_dma_memcpy;
	dd->device_terminate_all = mtk_cqdma_terminate_all;
	dd->src_addr_widths = MTK_CQDMA_DMA_BUSWIDTHS;
	dd->dst_addr_widths = MTK_CQDMA_DMA_BUSWIDTHS;
	dd->directions = BIT(DMA_MEM_TO_MEM);
	dd->residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;
	dd->dev = &pdev->dev;
	INIT_LIST_HEAD(&dd->channels);

	if (pdev->dev.of_node && of_property_read_u32(pdev->dev.of_node,
						      "dma-requests",
						      &cqdma->dma_requests)) {
		dev_info(&pdev->dev,
			 "Using %u as missing dma-requests property\n",
			 MTK_CQDMA_NR_VCHANS);

		cqdma->dma_requests = MTK_CQDMA_NR_VCHANS;
	}

	if (pdev->dev.of_node && of_property_read_u32(pdev->dev.of_node,
						      "dma-channels",
						      &cqdma->dma_channels)) {
		dev_info(&pdev->dev,
			 "Using %u as missing dma-channels property\n",
			 MTK_CQDMA_NR_PCHANS);

		cqdma->dma_channels = MTK_CQDMA_NR_PCHANS;
	}

	cqdma->pc = devm_kcalloc(&pdev->dev, cqdma->dma_channels,
				 sizeof(*cqdma->pc), GFP_KERNEL);
	if (!cqdma->pc)
		return -ENOMEM;

	/* initialization for PCs */
	for (i = 0; i < cqdma->dma_channels; ++i) {
		cqdma->pc[i] = devm_kcalloc(&pdev->dev, 1,
					    sizeof(**cqdma->pc), GFP_KERNEL);
		if (!cqdma->pc[i])
			return -ENOMEM;

		INIT_LIST_HEAD(&cqdma->pc[i]->queue);
		spin_lock_init(&cqdma->pc[i]->lock);
		refcount_set(&cqdma->pc[i]->refcnt, 0);
		cqdma->pc[i]->base = devm_platform_ioremap_resource(pdev, i);
		if (IS_ERR(cqdma->pc[i]->base))
			return PTR_ERR(cqdma->pc[i]->base);

		/* allocate IRQ resource */
		res = platform_get_resource(pdev, IORESOURCE_IRQ, i);
		if (!res) {
			dev_err(&pdev->dev, "No irq resource for %s\n",
				dev_name(&pdev->dev));
			return -EINVAL;
		}
		cqdma->pc[i]->irq = res->start;

		err = devm_request_irq(&pdev->dev, cqdma->pc[i]->irq,
				       mtk_cqdma_irq, 0, dev_name(&pdev->dev),
				       cqdma);
		if (err) {
			dev_err(&pdev->dev,
				"request_irq failed with err %d\n", err);
			return -EINVAL;
		}
	}

	/* allocate resource for VCs */
	cqdma->vc = devm_kcalloc(&pdev->dev, cqdma->dma_requests,
				 sizeof(*cqdma->vc), GFP_KERNEL);
	if (!cqdma->vc)
		return -ENOMEM;

	for (i = 0; i < cqdma->dma_requests; i++) {
		vc = &cqdma->vc[i];
		vc->vc.desc_free = mtk_cqdma_vdesc_free;
		vchan_init(&vc->vc, dd);
		init_completion(&vc->issue_completion);
	}

	err = dma_async_device_register(dd);
	if (err)
		return err;

	err = of_dma_controller_register(pdev->dev.of_node,
					 of_dma_xlate_by_chan_id, cqdma);
	if (err) {
		dev_err(&pdev->dev,
			"MediaTek CQDMA OF registration failed %d\n", err);
		goto err_unregister;
	}

	err = mtk_cqdma_hw_init(cqdma);
	if (err) {
		dev_err(&pdev->dev,
			"MediaTek CQDMA HW initialization failed %d\n", err);
		goto err_unregister;
	}

	platform_set_drvdata(pdev, cqdma);

	/* initialize tasklet for each PC */
	for (i = 0; i < cqdma->dma_channels; ++i)
		tasklet_init(&cqdma->pc[i]->tasklet, mtk_cqdma_tasklet_cb,
			     (unsigned long)cqdma->pc[i]);

	dev_info(&pdev->dev, "MediaTek CQDMA driver registered\n");

	return 0;

err_unregister:
	dma_async_device_unregister(dd);

	return err;
}

static int mtk_cqdma_remove(struct platform_device *pdev)
{
	struct mtk_cqdma_device *cqdma = platform_get_drvdata(pdev);
	struct mtk_cqdma_vchan *vc;
	unsigned long flags;
	int i;

	/* kill VC task */
	for (i = 0; i < cqdma->dma_requests; i++) {
		vc = &cqdma->vc[i];

		list_del(&vc->vc.chan.device_node);
		tasklet_kill(&vc->vc.task);
	}

	/* disable interrupt */
	for (i = 0; i < cqdma->dma_channels; i++) {
		spin_lock_irqsave(&cqdma->pc[i]->lock, flags);
		mtk_dma_clr(cqdma->pc[i], MTK_CQDMA_INT_EN,
			    MTK_CQDMA_INT_EN_BIT);
		spin_unlock_irqrestore(&cqdma->pc[i]->lock, flags);

		/* Waits for any pending IRQ handlers to complete */
		synchronize_irq(cqdma->pc[i]->irq);

		tasklet_kill(&cqdma->pc[i]->tasklet);
	}

	/* disable hardware */
	mtk_cqdma_hw_deinit(cqdma);

	dma_async_device_unregister(&cqdma->ddev);
	of_dma_controller_free(pdev->dev.of_node);

	return 0;
}

static struct platform_driver mtk_cqdma_driver = {
	.probe = mtk_cqdma_probe,
	.remove = mtk_cqdma_remove,
	.driver = {
		.name           = KBUILD_MODNAME,
		.of_match_table = mtk_cqdma_match,
	},
};
module_platform_driver(mtk_cqdma_driver);

MODULE_DESCRIPTION("MediaTek CQDMA Controller Driver");
MODULE_AUTHOR("Shun-Chih Yu <shun-chih.yu@mediatek.com>");
MODULE_LICENSE("GPL v2");