linux_dsm_epyc7002/drivers/ntb/test/ntb_tool.c

/*
 * This file is provided under a dual BSD/GPLv2 license.  When using or
 *   redistributing this file, you may do so under either license.
 *
 *   GPL LICENSE SUMMARY
 *
 *   Copyright (C) 2015 EMC Corporation. All Rights Reserved.
 *   Copyright (C) 2017 T-Platforms All Rights Reserved.
 *
 *   This program is free software; you can redistribute it and/or modify
 *   it under the terms of version 2 of the GNU General Public License as
 *   published by the Free Software Foundation.
 *
 *   This program is distributed in the hope that it will be useful, but
 *   WITHOUT ANY WARRANTY; without even the implied warranty of
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *   General Public License for more details.
 *
 *   BSD LICENSE
 *
 *   Copyright (C) 2015 EMC Corporation. All Rights Reserved.
 *   Copyright (C) 2017 T-Platforms All Rights Reserved.
 *
 *   Redistribution and use in source and binary forms, with or without
 *   modification, are permitted provided that the following conditions
 *   are met:
 *
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copy
 *       notice, this list of conditions and the following disclaimer in
 *       the documentation and/or other materials provided with the
 *       distribution.
 *     * Neither the name of Intel Corporation nor the names of its
 *       contributors may be used to endorse or promote products derived
 *       from this software without specific prior written permission.
 *
 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * PCIe NTB Debugging Tool Linux driver
 */

/*
 * How to use this tool, by example.
 *
 * Assuming $DBG_DIR is something like:
 * '/sys/kernel/debug/ntb_tool/0000:00:03.0'
 * Suppose aside from local device there is at least one remote device
 * connected to NTB with index 0.
 *-----------------------------------------------------------------------------
 * Eg: check local/peer device information.
 *
 * # Get local device port number
 * root@self# cat $DBG_DIR/port
 *
 * # Check local device functionality
 * root@self# ls $DBG_DIR
 * db            msg1          msg_sts     peer4/        port
 * db_event      msg2          peer0/      peer5/        spad0
 * db_mask       msg3          peer1/      peer_db       spad1
 * link          msg_event     peer2/      peer_db_mask  spad2
 * msg0          msg_mask      peer3/      peer_spad     spad3
 * # As one can see it supports:
 * # 1) four inbound message registers
 * # 2) four inbound scratchpads
 * # 3) up to six peer devices
 *
 * # Check peer device port number
 * root@self# cat $DBG_DIR/peer0/port
 *
 * # Check peer device(s) functionality to be used
 * root@self# ls $DBG_DIR/peer0
 * link             mw_trans0       mw_trans6        port
 * link_event       mw_trans1       mw_trans7        spad0
 * msg0             mw_trans2       peer_mw_trans0   spad1
 * msg1             mw_trans3       peer_mw_trans1   spad2
 * msg2             mw_trans4       peer_mw_trans2   spad3
 * msg3             mw_trans5       peer_mw_trans3
 * # As one can see we got:
 * # 1) four outbound message registers
 * # 2) four outbound scratchpads
 * # 3) eight inbound memory windows
 * # 4) four outbound memory windows
 *-----------------------------------------------------------------------------
 * Eg: NTB link tests
 *
 * # Set local link up/down
 * root@self# echo Y > $DBG_DIR/link
 * root@self# echo N > $DBG_DIR/link
 *
 * # Check if link with peer device is up/down:
 * root@self# cat $DBG_DIR/peer0/link
 *
 * # Block until the link is up/down
 * root@self# echo Y > $DBG_DIR/peer0/link_event
 * root@self# echo N > $DBG_DIR/peer0/link_event
 *-----------------------------------------------------------------------------
 * Eg: Doorbell registers tests (some functionality might be absent)
 *
 * # Set/clear/get local doorbell
 * root@self# echo 's 1' > $DBG_DIR/db
 * root@self# echo 'c 1' > $DBG_DIR/db
 * root@self# cat  $DBG_DIR/db
 *
 * # Set/clear/get local doorbell mask
 * root@self# echo 's 1' > $DBG_DIR/db_mask
 * root@self# echo 'c 1' > $DBG_DIR/db_mask
 * root@self# cat $DBG_DIR/db_mask
 *
 * # Ring/clear/get peer doorbell
 * root@peer# echo 's 1' > $DBG_DIR/peer_db
 * root@peer# echo 'c 1' > $DBG_DIR/peer_db
 * root@peer# cat $DBG_DIR/peer_db
 *
 * # Set/clear/get peer doorbell mask
 * root@self# echo 's 1' > $DBG_DIR/peer_db_mask
 * root@self# echo 'c 1' > $DBG_DIR/peer_db_mask
 * root@self# cat $DBG_DIR/peer_db_mask
 *
 * # Block until local doorbell is set with specified value
 * root@self# echo 1 > $DBG_DIR/db_event
 *-----------------------------------------------------------------------------
 * Eg: Message registers tests (functionality might be absent)
 *
 * # Set/clear/get in/out message registers status
 * root@self# echo 's 1' > $DBG_DIR/msg_sts
 * root@self# echo 'c 1' > $DBG_DIR/msg_sts
 * root@self# cat $DBG_DIR/msg_sts
 *
 * # Set/clear in/out message registers mask
 * root@self# echo 's 1' > $DBG_DIR/msg_mask
 * root@self# echo 'c 1' > $DBG_DIR/msg_mask
 *
 * # Get inbound message register #0 value and source of port index
 * root@self# cat  $DBG_DIR/msg0
 *
 * # Send some data to peer over outbound message register #0
 * root@self# echo 0x01020304 > $DBG_DIR/peer0/msg0
 *-----------------------------------------------------------------------------
 * Eg: Scratchpad registers tests (functionality might be absent)
 *
 * # Write/read to/from local scratchpad register #0
 * root@peer# echo 0x01020304 > $DBG_DIR/spad0
 * root@peer# cat $DBG_DIR/spad0
 *
 * # Write/read to/from peer scratchpad register #0
 * root@peer# echo 0x01020304 > $DBG_DIR/peer0/spad0
 * root@peer# cat $DBG_DIR/peer0/spad0
 *-----------------------------------------------------------------------------
 * Eg: Memory windows tests
 *
 * # Create inbound memory window buffer of specified size/get its base address
 * root@peer# echo 16384 > $DBG_DIR/peer0/mw_trans0
 * root@peer# cat $DBG_DIR/peer0/mw_trans0
 *
 * # Write/read data to/from inbound memory window
 * root@peer# echo Hello > $DBG_DIR/peer0/mw0
 * root@peer# head -c 7 $DBG_DIR/peer0/mw0
 *
 * # Map outbound memory window/check it settings (on peer device)
 * root@peer# echo 0xADD0BA5E:16384 > $DBG_DIR/peer0/peer_mw_trans0
 * root@peer# cat $DBG_DIR/peer0/peer_mw_trans0
 *
 * # Write/read data to/from outbound memory window (on peer device)
 * root@peer# echo olleH > $DBG_DIR/peer0/peer_mw0
 * root@peer# head -c 7 $DBG_DIR/peer0/peer_mw0
 */

#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/module.h>

#include <linux/debugfs.h>
#include <linux/dma-mapping.h>
#include <linux/pci.h>
#include <linux/slab.h>
#include <linux/uaccess.h>

#include <linux/ntb.h>

#define DRIVER_NAME		"ntb_tool"
#define DRIVER_VERSION		"2.0"

MODULE_LICENSE("Dual BSD/GPL");
MODULE_VERSION(DRIVER_VERSION);
MODULE_AUTHOR("Allen Hubbe <Allen.Hubbe@emc.com>");
MODULE_DESCRIPTION("PCIe NTB Debugging Tool");

/*
 * Inbound and outbound memory windows descriptor. Union members selection
 * depends on the MW type the structure describes. mm_base/dma_base are the
 * virtual and DMA address of an inbound MW. io_base/tr_base are the MMIO
 * mapped virtual and xlat addresses of an outbound MW respectively.
 */
struct tool_mw {
	int widx;
	int pidx;
	struct tool_ctx *tc;
	union {
		u8 *mm_base;
		u8 __iomem *io_base;
	};
	union {
		dma_addr_t dma_base;
		u64 tr_base;
	};
	resource_size_t size;
	struct dentry *dbgfs_file;
};

/*
 * Wrapper structure is used to distinguish the outbound MW peers reference
 * within the corresponding DebugFS directory IO operation.
 */
struct tool_mw_wrap {
	int pidx;
	struct tool_mw *mw;
};

struct tool_msg {
	int midx;
	int pidx;
	struct tool_ctx *tc;
};

struct tool_spad {
	int sidx;
	int pidx;
	struct tool_ctx *tc;
};

struct tool_peer {
	int pidx;
	struct tool_ctx *tc;
	int inmw_cnt;
	struct tool_mw *inmws;
	int outmw_cnt;
	struct tool_mw_wrap *outmws;
	int outmsg_cnt;
	struct tool_msg *outmsgs;
	int outspad_cnt;
	struct tool_spad *outspads;
	struct dentry *dbgfs_dir;
};

struct tool_ctx {
	struct ntb_dev *ntb;
	wait_queue_head_t link_wq;
	wait_queue_head_t db_wq;
	wait_queue_head_t msg_wq;
	int outmw_cnt;
	struct tool_mw *outmws;
	int peer_cnt;
	struct tool_peer *peers;
	int inmsg_cnt;
	struct tool_msg *inmsgs;
	int inspad_cnt;
	struct tool_spad *inspads;
	struct dentry *dbgfs_dir;
};

#define TOOL_FOPS_RDWR(__name, __read, __write) \
	const struct file_operations __name = {	\
		.owner = THIS_MODULE,		\
		.open = simple_open,		\
		.read = __read,			\
		.write = __write,		\
	}

#define TOOL_BUF_LEN 32

static struct dentry *tool_dbgfs_topdir;

/*==============================================================================
 *                               NTB events handlers
 *==============================================================================
 */

static void tool_link_event(void *ctx)
{
	struct tool_ctx *tc = ctx;
	enum ntb_speed speed;
	enum ntb_width width;
	int up;

	up = ntb_link_is_up(tc->ntb, &speed, &width);

	dev_dbg(&tc->ntb->dev, "link is %s speed %d width %d\n",
		up ? "up" : "down", speed, width);

	wake_up(&tc->link_wq);
}

static void tool_db_event(void *ctx, int vec)
{
	struct tool_ctx *tc = ctx;
	u64 db_bits, db_mask;

	db_mask = ntb_db_vector_mask(tc->ntb, vec);
	db_bits = ntb_db_read(tc->ntb);

	dev_dbg(&tc->ntb->dev, "doorbell vec %d mask %#llx bits %#llx\n",
		vec, db_mask, db_bits);

	wake_up(&tc->db_wq);
}

static void tool_msg_event(void *ctx)
{
	struct tool_ctx *tc = ctx;
	u64 msg_sts;

	msg_sts = ntb_msg_read_sts(tc->ntb);

	dev_dbg(&tc->ntb->dev, "message bits %#llx\n", msg_sts);

	wake_up(&tc->msg_wq);
}

static const struct ntb_ctx_ops tool_ops = {
	.link_event = tool_link_event,
	.db_event = tool_db_event,
	.msg_event = tool_msg_event
};

/*==============================================================================
 *                        Common read/write methods
 *==============================================================================
 */

static ssize_t tool_fn_read(struct tool_ctx *tc, char __user *ubuf,
			    size_t size, loff_t *offp,
			    u64 (*fn_read)(struct ntb_dev *))
{
	size_t buf_size;
	char buf[TOOL_BUF_LEN];
	ssize_t pos;

	if (!fn_read)
		return -EINVAL;

	buf_size = min(size, sizeof(buf));

	pos = scnprintf(buf, buf_size, "%#llx\n", fn_read(tc->ntb));

	return simple_read_from_buffer(ubuf, size, offp, buf, pos);
}

static ssize_t tool_fn_write(struct tool_ctx *tc,
			     const char __user *ubuf,
			     size_t size, loff_t *offp,
			     int (*fn_set)(struct ntb_dev *, u64),
			     int (*fn_clear)(struct ntb_dev *, u64))
{
	char *buf, cmd;
	ssize_t ret;
	u64 bits;
	int n;

	buf = kmalloc(size + 1, GFP_KERNEL);
	if (!buf)
		return -ENOMEM;

	ret = simple_write_to_buffer(buf, size, offp, ubuf, size);
	if (ret < 0) {
		kfree(buf);
		return ret;
	}

	buf[size] = 0;

	n = sscanf(buf, "%c %lli", &cmd, &bits);

	kfree(buf);

	if (n != 2) {
		ret = -EINVAL;
	} else if (cmd == 's') {
		if (!fn_set)
			ret = -EINVAL;
		else
			ret = fn_set(tc->ntb, bits);
	} else if (cmd == 'c') {
		if (!fn_clear)
			ret = -EINVAL;
		else
			ret = fn_clear(tc->ntb, bits);
	} else {
		ret = -EINVAL;
	}

	return ret ? : size;
}

/*==============================================================================
 *                            Port read/write methods
 *==============================================================================
 */

static ssize_t tool_port_read(struct file *filep, char __user *ubuf,
			      size_t size, loff_t *offp)
{
	struct tool_ctx *tc = filep->private_data;
	char buf[TOOL_BUF_LEN];
	int pos;

	pos = scnprintf(buf, sizeof(buf), "%d\n", ntb_port_number(tc->ntb));

	return simple_read_from_buffer(ubuf, size, offp, buf, pos);
}

static TOOL_FOPS_RDWR(tool_port_fops,
		      tool_port_read,
		      NULL);

static ssize_t tool_peer_port_read(struct file *filep, char __user *ubuf,
				   size_t size, loff_t *offp)
{
	struct tool_peer *peer = filep->private_data;
	struct tool_ctx *tc = peer->tc;
	char buf[TOOL_BUF_LEN];
	int pos;

	pos = scnprintf(buf, sizeof(buf), "%d\n",
		ntb_peer_port_number(tc->ntb, peer->pidx));

	return simple_read_from_buffer(ubuf, size, offp, buf, pos);
}

static TOOL_FOPS_RDWR(tool_peer_port_fops,
		      tool_peer_port_read,
		      NULL);

static int tool_init_peers(struct tool_ctx *tc)
{
	int pidx;

	tc->peer_cnt = ntb_peer_port_count(tc->ntb);
	tc->peers = devm_kcalloc(&tc->ntb->dev, tc->peer_cnt,
				 sizeof(*tc->peers), GFP_KERNEL);
	if (tc->peers == NULL)
		return -ENOMEM;

	for (pidx = 0; pidx < tc->peer_cnt; pidx++) {
		tc->peers[pidx].pidx = pidx;
		tc->peers[pidx].tc = tc;
	}

	return 0;
}

/*==============================================================================
 *                       Link state read/write methods
 *==============================================================================
 */

static ssize_t tool_link_write(struct file *filep, const char __user *ubuf,
			       size_t size, loff_t *offp)
{
	struct tool_ctx *tc = filep->private_data;
	bool val;
	int ret;

	ret = kstrtobool_from_user(ubuf, size, &val);
	if (ret)
		return ret;

	if (val)
		ret = ntb_link_enable(tc->ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO);
	else
		ret = ntb_link_disable(tc->ntb);

	if (ret)
		return ret;

	return size;
}

static TOOL_FOPS_RDWR(tool_link_fops,
		      NULL,
		      tool_link_write);

static ssize_t tool_peer_link_read(struct file *filep, char __user *ubuf,
				   size_t size, loff_t *offp)
{
	struct tool_peer *peer = filep->private_data;
	struct tool_ctx *tc = peer->tc;
	char buf[3];

	if (ntb_link_is_up(tc->ntb, NULL, NULL) & BIT(peer->pidx))
		buf[0] = 'Y';
	else
		buf[0] = 'N';
	buf[1] = '\n';
	buf[2] = '\0';

	return simple_read_from_buffer(ubuf, size, offp, buf, 3);
}

static TOOL_FOPS_RDWR(tool_peer_link_fops,
		      tool_peer_link_read,
		      NULL);

static ssize_t tool_peer_link_event_write(struct file *filep,
					  const char __user *ubuf,
					  size_t size, loff_t *offp)
{
	struct tool_peer *peer = filep->private_data;
	struct tool_ctx *tc = peer->tc;
	u64 link_msk;
	bool val;
	int ret;

	ret = kstrtobool_from_user(ubuf, size, &val);
	if (ret)
		return ret;

	link_msk = BIT_ULL_MASK(peer->pidx);

	if (wait_event_interruptible(tc->link_wq,
		!!(ntb_link_is_up(tc->ntb, NULL, NULL) & link_msk) == val))
		return -ERESTART;

	return size;
}

static TOOL_FOPS_RDWR(tool_peer_link_event_fops,
		      NULL,
		      tool_peer_link_event_write);

/*==============================================================================
 *                  Memory windows read/write/setting methods
 *==============================================================================
 */

static ssize_t tool_mw_read(struct file *filep, char __user *ubuf,
			    size_t size, loff_t *offp)
{
	struct tool_mw *inmw = filep->private_data;

	if (inmw->mm_base == NULL)
		return -ENXIO;

	return simple_read_from_buffer(ubuf, size, offp,
				       inmw->mm_base, inmw->size);
}

static ssize_t tool_mw_write(struct file *filep, const char __user *ubuf,
			     size_t size, loff_t *offp)
{
	struct tool_mw *inmw = filep->private_data;

	if (inmw->mm_base == NULL)
		return -ENXIO;

	return simple_write_to_buffer(inmw->mm_base, inmw->size, offp,
				      ubuf, size);
}

static TOOL_FOPS_RDWR(tool_mw_fops,
		      tool_mw_read,
		      tool_mw_write);

static int tool_setup_mw(struct tool_ctx *tc, int pidx, int widx,
			 size_t req_size)
{
	resource_size_t size, addr_align, size_align;
	struct tool_mw *inmw = &tc->peers[pidx].inmws[widx];
	char buf[TOOL_BUF_LEN];
	int ret;

	if (inmw->mm_base != NULL)
		return 0;

	ret = ntb_mw_get_align(tc->ntb, pidx, widx, &addr_align,
				&size_align, &size);
	if (ret)
		return ret;

	inmw->size = min_t(resource_size_t, req_size, size);
	inmw->size = round_up(inmw->size, addr_align);
	inmw->size = round_up(inmw->size, size_align);
	inmw->mm_base = dma_alloc_coherent(&tc->ntb->dev, inmw->size,
					   &inmw->dma_base, GFP_KERNEL);
	if (!inmw->mm_base)
		return -ENOMEM;

	if (!IS_ALIGNED(inmw->dma_base, addr_align)) {
		ret = -ENOMEM;
		goto err_free_dma;
	}

	ret = ntb_mw_set_trans(tc->ntb, pidx, widx, inmw->dma_base, inmw->size);
	if (ret)
		goto err_free_dma;

	snprintf(buf, sizeof(buf), "mw%d", widx);
	inmw->dbgfs_file = debugfs_create_file(buf, 0600,
					       tc->peers[pidx].dbgfs_dir, inmw,
					       &tool_mw_fops);

	return 0;

err_free_dma:
	dma_free_coherent(&tc->ntb->dev, inmw->size, inmw->mm_base,
			  inmw->dma_base);
	inmw->mm_base = NULL;
	inmw->dma_base = 0;
	inmw->size = 0;

	return ret;
}

static void tool_free_mw(struct tool_ctx *tc, int pidx, int widx)
{
	struct tool_mw *inmw = &tc->peers[pidx].inmws[widx];

	debugfs_remove(inmw->dbgfs_file);

	if (inmw->mm_base != NULL) {
		ntb_mw_clear_trans(tc->ntb, pidx, widx);
		dma_free_coherent(&tc->ntb->dev, inmw->size,
				  inmw->mm_base, inmw->dma_base);
	}

	inmw->mm_base = NULL;
	inmw->dma_base = 0;
	inmw->size = 0;
	inmw->dbgfs_file = NULL;
}

static ssize_t tool_mw_trans_read(struct file *filep, char __user *ubuf,
				  size_t size, loff_t *offp)
{
	struct tool_mw *inmw = filep->private_data;
	resource_size_t addr_align;
	resource_size_t size_align;
	resource_size_t size_max;
	ssize_t ret, off = 0;
	size_t buf_size;
	char *buf;

	buf_size = min_t(size_t, size, 512);

	buf = kmalloc(buf_size, GFP_KERNEL);
	if (!buf)
		return -ENOMEM;

	ret = ntb_mw_get_align(inmw->tc->ntb, inmw->pidx, inmw->widx,
			       &addr_align, &size_align, &size_max);
	if (ret)
		goto err;

	off += scnprintf(buf + off, buf_size - off,
			 "Inbound MW     \t%d\n",
			 inmw->widx);

	off += scnprintf(buf + off, buf_size - off,
			 "Port           \t%d (%d)\n",
			 ntb_peer_port_number(inmw->tc->ntb, inmw->pidx),
			 inmw->pidx);

	off += scnprintf(buf + off, buf_size - off,
			 "Window Address \t0x%pK\n", inmw->mm_base);

	off += scnprintf(buf + off, buf_size - off,
			 "DMA Address    \t%pad\n",
			 &inmw->dma_base);

	off += scnprintf(buf + off, buf_size - off,
			 "Window Size    \t%pa[p]\n",
			 &inmw->size);

	off += scnprintf(buf + off, buf_size - off,
			 "Alignment      \t%pa[p]\n",
			 &addr_align);

	off += scnprintf(buf + off, buf_size - off,
			 "Size Alignment \t%pa[p]\n",
			 &size_align);

	off += scnprintf(buf + off, buf_size - off,
			 "Size Max       \t%pa[p]\n",
			 &size_max);

	ret = simple_read_from_buffer(ubuf, size, offp, buf, off);

err:
	kfree(buf);

	return ret;
}

static ssize_t tool_mw_trans_write(struct file *filep, const char __user *ubuf,
				   size_t size, loff_t *offp)
{
	struct tool_mw *inmw = filep->private_data;
	unsigned int val;
	int ret;

	ret = kstrtouint_from_user(ubuf, size, 0, &val);
	if (ret)
		return ret;

	tool_free_mw(inmw->tc, inmw->pidx, inmw->widx);
	if (val) {
		ret = tool_setup_mw(inmw->tc, inmw->pidx, inmw->widx, val);
		if (ret)
			return ret;
	}

	return size;
}

static TOOL_FOPS_RDWR(tool_mw_trans_fops,
		      tool_mw_trans_read,
		      tool_mw_trans_write);

static ssize_t tool_peer_mw_read(struct file *filep, char __user *ubuf,
				 size_t size, loff_t *offp)
{
	struct tool_mw *outmw = filep->private_data;
	loff_t pos = *offp;
	ssize_t ret;
	void *buf;

	if (outmw->io_base == NULL)
		return -EIO;

	if (pos >= outmw->size || !size)
		return 0;

	if (size > outmw->size - pos)
		size = outmw->size - pos;

	buf = kmalloc(size, GFP_KERNEL);
	if (!buf)
		return -ENOMEM;

	memcpy_fromio(buf, outmw->io_base + pos, size);
	ret = copy_to_user(ubuf, buf, size);
	if (ret == size) {
		ret = -EFAULT;
		goto err_free;
	}

	size -= ret;
	*offp = pos + size;
	ret = size;

err_free:
	kfree(buf);

	return ret;
}

static ssize_t tool_peer_mw_write(struct file *filep, const char __user *ubuf,
				  size_t size, loff_t *offp)
{
	struct tool_mw *outmw = filep->private_data;
	ssize_t ret;
	loff_t pos = *offp;
	void *buf;

	if (outmw->io_base == NULL)
		return -EIO;

	if (pos >= outmw->size || !size)
		return 0;
	if (size > outmw->size - pos)
		size = outmw->size - pos;

	buf = kmalloc(size, GFP_KERNEL);
	if (!buf)
		return -ENOMEM;

	ret = copy_from_user(buf, ubuf, size);
	if (ret == size) {
		ret = -EFAULT;
		goto err_free;
	}

	size -= ret;
	*offp = pos + size;
	ret = size;

	memcpy_toio(outmw->io_base + pos, buf, size);

err_free:
	kfree(buf);

	return ret;
}

static TOOL_FOPS_RDWR(tool_peer_mw_fops,
		      tool_peer_mw_read,
		      tool_peer_mw_write);

static int tool_setup_peer_mw(struct tool_ctx *tc, int pidx, int widx,
			      u64 req_addr, size_t req_size)
{
	struct tool_mw *outmw = &tc->outmws[widx];
	resource_size_t map_size;
	phys_addr_t map_base;
	char buf[TOOL_BUF_LEN];
	int ret;

	if (outmw->io_base != NULL)
		return 0;

	ret = ntb_peer_mw_get_addr(tc->ntb, widx, &map_base, &map_size);
	if (ret)
		return ret;

	ret = ntb_peer_mw_set_trans(tc->ntb, pidx, widx, req_addr, req_size);
	if (ret)
		return ret;

	outmw->io_base = ioremap_wc(map_base, map_size);
	if (outmw->io_base == NULL) {
		ret = -EFAULT;
		goto err_clear_trans;
	}

	outmw->tr_base = req_addr;
	outmw->size = req_size;
	outmw->pidx = pidx;

	snprintf(buf, sizeof(buf), "peer_mw%d", widx);
	outmw->dbgfs_file = debugfs_create_file(buf, 0600,
					       tc->peers[pidx].dbgfs_dir, outmw,
					       &tool_peer_mw_fops);

	return 0;

err_clear_trans:
	ntb_peer_mw_clear_trans(tc->ntb, pidx, widx);

	return ret;
}

static void tool_free_peer_mw(struct tool_ctx *tc, int widx)
{
	struct tool_mw *outmw = &tc->outmws[widx];

	debugfs_remove(outmw->dbgfs_file);

	if (outmw->io_base != NULL) {
		iounmap(tc->outmws[widx].io_base);
		ntb_peer_mw_clear_trans(tc->ntb, outmw->pidx, widx);
	}

	outmw->io_base = NULL;
	outmw->tr_base = 0;
	outmw->size = 0;
	outmw->pidx = -1;
	outmw->dbgfs_file = NULL;
}

static ssize_t tool_peer_mw_trans_read(struct file *filep, char __user *ubuf,
					size_t size, loff_t *offp)
{
	struct tool_mw_wrap *outmw_wrap = filep->private_data;
	struct tool_mw *outmw = outmw_wrap->mw;
	resource_size_t map_size;
	phys_addr_t map_base;
	ssize_t off = 0;
	size_t buf_size;
	char *buf;
	int ret;

	ret = ntb_peer_mw_get_addr(outmw->tc->ntb, outmw->widx,
				  &map_base, &map_size);
	if (ret)
		return ret;

	buf_size = min_t(size_t, size, 512);

	buf = kmalloc(buf_size, GFP_KERNEL);
	if (!buf)
		return -ENOMEM;

	off += scnprintf(buf + off, buf_size - off,
			 "Outbound MW:        \t%d\n", outmw->widx);

	if (outmw->io_base != NULL) {
		off += scnprintf(buf + off, buf_size - off,
			"Port attached       \t%d (%d)\n",
			ntb_peer_port_number(outmw->tc->ntb, outmw->pidx),
			outmw->pidx);
	} else {
		off += scnprintf(buf + off, buf_size - off,
				 "Port attached       \t-1 (-1)\n");
	}

	off += scnprintf(buf + off, buf_size - off,
			 "Virtual address     \t0x%pK\n", outmw->io_base);

	off += scnprintf(buf + off, buf_size - off,
			 "Phys Address        \t%pa[p]\n", &map_base);

	off += scnprintf(buf + off, buf_size - off,
			 "Mapping Size        \t%pa[p]\n", &map_size);

	off += scnprintf(buf + off, buf_size - off,
			 "Translation Address \t0x%016llx\n", outmw->tr_base);

	off += scnprintf(buf + off, buf_size - off,
			 "Window Size         \t%pa[p]\n", &outmw->size);

	ret = simple_read_from_buffer(ubuf, size, offp, buf, off);
	kfree(buf);

	return ret;
}

static ssize_t tool_peer_mw_trans_write(struct file *filep,
					const char __user *ubuf,
					size_t size, loff_t *offp)
{
	struct tool_mw_wrap *outmw_wrap = filep->private_data;
	struct tool_mw *outmw = outmw_wrap->mw;
	size_t buf_size, wsize;
	char buf[TOOL_BUF_LEN];
	int ret, n;
	u64 addr;

	buf_size = min(size, (sizeof(buf) - 1));
	if (copy_from_user(buf, ubuf, buf_size))
		return -EFAULT;

	buf[buf_size] = '\0';

	n = sscanf(buf, "%lli:%zi", &addr, &wsize);
	if (n != 2)
		return -EINVAL;

	tool_free_peer_mw(outmw->tc, outmw->widx);
	if (wsize) {
		ret = tool_setup_peer_mw(outmw->tc, outmw_wrap->pidx,
					 outmw->widx, addr, wsize);
		if (ret)
			return ret;
	}

	return size;
}

static TOOL_FOPS_RDWR(tool_peer_mw_trans_fops,
		      tool_peer_mw_trans_read,
		      tool_peer_mw_trans_write);

static int tool_init_mws(struct tool_ctx *tc)
{
	int widx, pidx;

	/* Initialize outbound memory windows */
	tc->outmw_cnt = ntb_peer_mw_count(tc->ntb);
	tc->outmws = devm_kcalloc(&tc->ntb->dev, tc->outmw_cnt,
				  sizeof(*tc->outmws), GFP_KERNEL);
	if (tc->outmws == NULL)
		return -ENOMEM;

	for (widx = 0; widx < tc->outmw_cnt; widx++) {
		tc->outmws[widx].widx = widx;
		tc->outmws[widx].pidx = -1;
		tc->outmws[widx].tc = tc;
	}

	/* Initialize inbound memory windows and outbound MWs wrapper */
	for (pidx = 0; pidx < tc->peer_cnt; pidx++) {
		tc->peers[pidx].inmw_cnt = ntb_mw_count(tc->ntb, pidx);
		tc->peers[pidx].inmws =
			devm_kcalloc(&tc->ntb->dev, tc->peers[pidx].inmw_cnt,
				    sizeof(*tc->peers[pidx].inmws), GFP_KERNEL);
		if (tc->peers[pidx].inmws == NULL)
			return -ENOMEM;

		for (widx = 0; widx < tc->peers[pidx].inmw_cnt; widx++) {
			tc->peers[pidx].inmws[widx].widx = widx;
			tc->peers[pidx].inmws[widx].pidx = pidx;
			tc->peers[pidx].inmws[widx].tc = tc;
		}

		tc->peers[pidx].outmw_cnt = ntb_peer_mw_count(tc->ntb);
		tc->peers[pidx].outmws =
			devm_kcalloc(&tc->ntb->dev, tc->peers[pidx].outmw_cnt,
				   sizeof(*tc->peers[pidx].outmws), GFP_KERNEL);

		for (widx = 0; widx < tc->peers[pidx].outmw_cnt; widx++) {
			tc->peers[pidx].outmws[widx].pidx = pidx;
			tc->peers[pidx].outmws[widx].mw = &tc->outmws[widx];
		}
	}

	return 0;
}

static void tool_clear_mws(struct tool_ctx *tc)
{
	int widx, pidx;

	/* Free outbound memory windows */
	for (widx = 0; widx < tc->outmw_cnt; widx++)
		tool_free_peer_mw(tc, widx);

	/* Free outbound memory windows */
	for (pidx = 0; pidx < tc->peer_cnt; pidx++)
		for (widx = 0; widx < tc->peers[pidx].inmw_cnt; widx++)
			tool_free_mw(tc, pidx, widx);
}

/*==============================================================================
 *                       Doorbell read/write methods
 *==============================================================================
 */

static ssize_t tool_db_read(struct file *filep, char __user *ubuf,
			    size_t size, loff_t *offp)
{
	struct tool_ctx *tc = filep->private_data;

	return tool_fn_read(tc, ubuf, size, offp, tc->ntb->ops->db_read);
}

static ssize_t tool_db_write(struct file *filep, const char __user *ubuf,
			     size_t size, loff_t *offp)
{
	struct tool_ctx *tc = filep->private_data;

	return tool_fn_write(tc, ubuf, size, offp, tc->ntb->ops->db_set,
			     tc->ntb->ops->db_clear);
}

static TOOL_FOPS_RDWR(tool_db_fops,
		      tool_db_read,
		      tool_db_write);

static ssize_t tool_db_valid_mask_read(struct file *filep, char __user *ubuf,
				       size_t size, loff_t *offp)
{
	struct tool_ctx *tc = filep->private_data;

	return tool_fn_read(tc, ubuf, size, offp, tc->ntb->ops->db_valid_mask);
}

static TOOL_FOPS_RDWR(tool_db_valid_mask_fops,
		      tool_db_valid_mask_read,
		      NULL);

static ssize_t tool_db_mask_read(struct file *filep, char __user *ubuf,
				 size_t size, loff_t *offp)
{
	struct tool_ctx *tc = filep->private_data;

	return tool_fn_read(tc, ubuf, size, offp, tc->ntb->ops->db_read_mask);
}

static ssize_t tool_db_mask_write(struct file *filep, const char __user *ubuf,
			       size_t size, loff_t *offp)
{
	struct tool_ctx *tc = filep->private_data;

	return tool_fn_write(tc, ubuf, size, offp, tc->ntb->ops->db_set_mask,
			     tc->ntb->ops->db_clear_mask);
}

static TOOL_FOPS_RDWR(tool_db_mask_fops,
		      tool_db_mask_read,
		      tool_db_mask_write);

static ssize_t tool_peer_db_read(struct file *filep, char __user *ubuf,
				 size_t size, loff_t *offp)
{
	struct tool_ctx *tc = filep->private_data;

	return tool_fn_read(tc, ubuf, size, offp, tc->ntb->ops->peer_db_read);
}

static ssize_t tool_peer_db_write(struct file *filep, const char __user *ubuf,
				  size_t size, loff_t *offp)
{
	struct tool_ctx *tc = filep->private_data;

	return tool_fn_write(tc, ubuf, size, offp, tc->ntb->ops->peer_db_set,
			     tc->ntb->ops->peer_db_clear);
}

static TOOL_FOPS_RDWR(tool_peer_db_fops,
		      tool_peer_db_read,
		      tool_peer_db_write);

static ssize_t tool_peer_db_mask_read(struct file *filep, char __user *ubuf,
				   size_t size, loff_t *offp)
{
	struct tool_ctx *tc = filep->private_data;

	return tool_fn_read(tc, ubuf, size, offp,
			    tc->ntb->ops->peer_db_read_mask);
}

static ssize_t tool_peer_db_mask_write(struct file *filep,
				       const char __user *ubuf,
				       size_t size, loff_t *offp)
{
	struct tool_ctx *tc = filep->private_data;

	return tool_fn_write(tc, ubuf, size, offp,
			     tc->ntb->ops->peer_db_set_mask,
			     tc->ntb->ops->peer_db_clear_mask);
}

static TOOL_FOPS_RDWR(tool_peer_db_mask_fops,
		      tool_peer_db_mask_read,
		      tool_peer_db_mask_write);

static ssize_t tool_db_event_write(struct file *filep,
				   const char __user *ubuf,
				   size_t size, loff_t *offp)
{
	struct tool_ctx *tc = filep->private_data;
	u64 val;
	int ret;

	ret = kstrtou64_from_user(ubuf, size, 0, &val);
	if (ret)
		return ret;

	if (wait_event_interruptible(tc->db_wq, ntb_db_read(tc->ntb) == val))
		return -ERESTART;

	return size;
}

static TOOL_FOPS_RDWR(tool_db_event_fops,
		      NULL,
		      tool_db_event_write);

/*==============================================================================
 *                       Scratchpads read/write methods
 *==============================================================================
 */

static ssize_t tool_spad_read(struct file *filep, char __user *ubuf,
			      size_t size, loff_t *offp)
{
	struct tool_spad *spad = filep->private_data;
	char buf[TOOL_BUF_LEN];
	ssize_t pos;

	if (!spad->tc->ntb->ops->spad_read)
		return -EINVAL;

	pos = scnprintf(buf, sizeof(buf), "%#x\n",
		ntb_spad_read(spad->tc->ntb, spad->sidx));

	return simple_read_from_buffer(ubuf, size, offp, buf, pos);
}

static ssize_t tool_spad_write(struct file *filep, const char __user *ubuf,
			       size_t size, loff_t *offp)
{
	struct tool_spad *spad = filep->private_data;
	u32 val;
	int ret;

	if (!spad->tc->ntb->ops->spad_write) {
		dev_dbg(&spad->tc->ntb->dev, "no spad write fn\n");
		return -EINVAL;
	}

	ret = kstrtou32_from_user(ubuf, size, 0, &val);
	if (ret)
		return ret;

	ret = ntb_spad_write(spad->tc->ntb, spad->sidx, val);

	return ret ?: size;
}

static TOOL_FOPS_RDWR(tool_spad_fops,
		      tool_spad_read,
		      tool_spad_write);

static ssize_t tool_peer_spad_read(struct file *filep, char __user *ubuf,
				   size_t size, loff_t *offp)
{
	struct tool_spad *spad = filep->private_data;
	char buf[TOOL_BUF_LEN];
	ssize_t pos;

	if (!spad->tc->ntb->ops->peer_spad_read)
		return -EINVAL;

	pos = scnprintf(buf, sizeof(buf), "%#x\n",
		ntb_peer_spad_read(spad->tc->ntb, spad->pidx, spad->sidx));

	return simple_read_from_buffer(ubuf, size, offp, buf, pos);
}

static ssize_t tool_peer_spad_write(struct file *filep, const char __user *ubuf,
				    size_t size, loff_t *offp)
{
	struct tool_spad *spad = filep->private_data;
	u32 val;
	int ret;

	if (!spad->tc->ntb->ops->peer_spad_write) {
		dev_dbg(&spad->tc->ntb->dev, "no spad write fn\n");
		return -EINVAL;
	}

	ret = kstrtou32_from_user(ubuf, size, 0, &val);
	if (ret)
		return ret;

	ret = ntb_peer_spad_write(spad->tc->ntb, spad->pidx, spad->sidx, val);

	return ret ?: size;
}

static TOOL_FOPS_RDWR(tool_peer_spad_fops,
		      tool_peer_spad_read,
		      tool_peer_spad_write);

static int tool_init_spads(struct tool_ctx *tc)
{
	int sidx, pidx;

	/* Initialize inbound scratchpad structures */
	tc->inspad_cnt = ntb_spad_count(tc->ntb);
	tc->inspads = devm_kcalloc(&tc->ntb->dev, tc->inspad_cnt,
				   sizeof(*tc->inspads), GFP_KERNEL);
	if (tc->inspads == NULL)
		return -ENOMEM;

	for (sidx = 0; sidx < tc->inspad_cnt; sidx++) {
		tc->inspads[sidx].sidx = sidx;
		tc->inspads[sidx].pidx = -1;
		tc->inspads[sidx].tc = tc;
	}

	/* Initialize outbound scratchpad structures */
	for (pidx = 0; pidx < tc->peer_cnt; pidx++) {
		tc->peers[pidx].outspad_cnt = ntb_spad_count(tc->ntb);
		tc->peers[pidx].outspads =
			devm_kcalloc(&tc->ntb->dev, tc->peers[pidx].outspad_cnt,
				sizeof(*tc->peers[pidx].outspads), GFP_KERNEL);
		if (tc->peers[pidx].outspads == NULL)
			return -ENOMEM;

		for (sidx = 0; sidx < tc->peers[pidx].outspad_cnt; sidx++) {
			tc->peers[pidx].outspads[sidx].sidx = sidx;
			tc->peers[pidx].outspads[sidx].pidx = pidx;
			tc->peers[pidx].outspads[sidx].tc = tc;
		}
	}

	return 0;
}

/*==============================================================================
 *                       Messages read/write methods
 *==============================================================================
 */

static ssize_t tool_inmsg_read(struct file *filep, char __user *ubuf,
			       size_t size, loff_t *offp)
{
	struct tool_msg *msg = filep->private_data;
	char buf[TOOL_BUF_LEN];
	ssize_t pos;
	u32 data;
	int pidx;

	data = ntb_msg_read(msg->tc->ntb, &pidx, msg->midx);

	pos = scnprintf(buf, sizeof(buf), "0x%08x<-%d\n", data, pidx);

	return simple_read_from_buffer(ubuf, size, offp, buf, pos);
}

static TOOL_FOPS_RDWR(tool_inmsg_fops,
		      tool_inmsg_read,
		      NULL);

static ssize_t tool_outmsg_write(struct file *filep,
				 const char __user *ubuf,
				 size_t size, loff_t *offp)
{
	struct tool_msg *msg = filep->private_data;
	u32 val;
	int ret;

	ret = kstrtou32_from_user(ubuf, size, 0, &val);
	if (ret)
		return ret;

	ret = ntb_peer_msg_write(msg->tc->ntb, msg->pidx, msg->midx, val);

	return ret ? : size;
}

static TOOL_FOPS_RDWR(tool_outmsg_fops,
		      NULL,
		      tool_outmsg_write);

static ssize_t tool_msg_sts_read(struct file *filep, char __user *ubuf,
				 size_t size, loff_t *offp)
{
	struct tool_ctx *tc = filep->private_data;

	return tool_fn_read(tc, ubuf, size, offp, tc->ntb->ops->msg_read_sts);
}

static ssize_t tool_msg_sts_write(struct file *filep, const char __user *ubuf,
				  size_t size, loff_t *offp)
{
	struct tool_ctx *tc = filep->private_data;

	return tool_fn_write(tc, ubuf, size, offp, NULL,
			     tc->ntb->ops->msg_clear_sts);
}

static TOOL_FOPS_RDWR(tool_msg_sts_fops,
		      tool_msg_sts_read,
		      tool_msg_sts_write);

static ssize_t tool_msg_inbits_read(struct file *filep, char __user *ubuf,
				    size_t size, loff_t *offp)
{
	struct tool_ctx *tc = filep->private_data;

	return tool_fn_read(tc, ubuf, size, offp, tc->ntb->ops->msg_inbits);
}

static TOOL_FOPS_RDWR(tool_msg_inbits_fops,
		      tool_msg_inbits_read,
		      NULL);

static ssize_t tool_msg_outbits_read(struct file *filep, char __user *ubuf,
				     size_t size, loff_t *offp)
{
	struct tool_ctx *tc = filep->private_data;

	return tool_fn_read(tc, ubuf, size, offp, tc->ntb->ops->msg_outbits);
}

static TOOL_FOPS_RDWR(tool_msg_outbits_fops,
		      tool_msg_outbits_read,
		      NULL);

static ssize_t tool_msg_mask_write(struct file *filep, const char __user *ubuf,
				   size_t size, loff_t *offp)
{
	struct tool_ctx *tc = filep->private_data;

	return tool_fn_write(tc, ubuf, size, offp,
			     tc->ntb->ops->msg_set_mask,
			     tc->ntb->ops->msg_clear_mask);
}

static TOOL_FOPS_RDWR(tool_msg_mask_fops,
		      NULL,
		      tool_msg_mask_write);

static ssize_t tool_msg_event_write(struct file *filep,
				    const char __user *ubuf,
				    size_t size, loff_t *offp)
{
	struct tool_ctx *tc = filep->private_data;
	u64 val;
	int ret;

	ret = kstrtou64_from_user(ubuf, size, 0, &val);
	if (ret)
		return ret;

	if (wait_event_interruptible(tc->msg_wq,
		ntb_msg_read_sts(tc->ntb) == val))
		return -ERESTART;

	return size;
}

static TOOL_FOPS_RDWR(tool_msg_event_fops,
		      NULL,
		      tool_msg_event_write);

static int tool_init_msgs(struct tool_ctx *tc)
{
	int midx, pidx;

	/* Initialize inbound message structures */
	tc->inmsg_cnt = ntb_msg_count(tc->ntb);
	tc->inmsgs = devm_kcalloc(&tc->ntb->dev, tc->inmsg_cnt,
				   sizeof(*tc->inmsgs), GFP_KERNEL);
	if (tc->inmsgs == NULL)
		return -ENOMEM;

	for (midx = 0; midx < tc->inmsg_cnt; midx++) {
		tc->inmsgs[midx].midx = midx;
		tc->inmsgs[midx].pidx = -1;
		tc->inmsgs[midx].tc = tc;
	}

	/* Initialize outbound message structures */
	for (pidx = 0; pidx < tc->peer_cnt; pidx++) {
		tc->peers[pidx].outmsg_cnt = ntb_msg_count(tc->ntb);
		tc->peers[pidx].outmsgs =
			devm_kcalloc(&tc->ntb->dev, tc->peers[pidx].outmsg_cnt,
				sizeof(*tc->peers[pidx].outmsgs), GFP_KERNEL);
		if (tc->peers[pidx].outmsgs == NULL)
			return -ENOMEM;

		for (midx = 0; midx < tc->peers[pidx].outmsg_cnt; midx++) {
			tc->peers[pidx].outmsgs[midx].midx = midx;
			tc->peers[pidx].outmsgs[midx].pidx = pidx;
			tc->peers[pidx].outmsgs[midx].tc = tc;
		}
	}

	return 0;
}

/*==============================================================================
 *                          Initialization methods
 *==============================================================================
 */

static struct tool_ctx *tool_create_data(struct ntb_dev *ntb)
{
	struct tool_ctx *tc;

	tc = devm_kzalloc(&ntb->dev, sizeof(*tc), GFP_KERNEL);
	if (tc == NULL)
		return ERR_PTR(-ENOMEM);

	tc->ntb = ntb;
	init_waitqueue_head(&tc->link_wq);
	init_waitqueue_head(&tc->db_wq);
	init_waitqueue_head(&tc->msg_wq);

	if (ntb_db_is_unsafe(ntb))
		dev_dbg(&ntb->dev, "doorbell is unsafe\n");

	if (ntb_spad_is_unsafe(ntb))
		dev_dbg(&ntb->dev, "scratchpad is unsafe\n");

	return tc;
}

static void tool_clear_data(struct tool_ctx *tc)
{
	wake_up(&tc->link_wq);
	wake_up(&tc->db_wq);
	wake_up(&tc->msg_wq);
}

static int tool_init_ntb(struct tool_ctx *tc)
{
	return ntb_set_ctx(tc->ntb, tc, &tool_ops);
}

static void tool_clear_ntb(struct tool_ctx *tc)
{
	ntb_clear_ctx(tc->ntb);
	ntb_link_disable(tc->ntb);
}

static void tool_setup_dbgfs(struct tool_ctx *tc)
{
	int pidx, widx, sidx, midx;
	char buf[TOOL_BUF_LEN];

	/* This modules is useless without dbgfs... */
	if (!tool_dbgfs_topdir) {
		tc->dbgfs_dir = NULL;
		return;
	}

	tc->dbgfs_dir = debugfs_create_dir(dev_name(&tc->ntb->dev),
					   tool_dbgfs_topdir);
	if (!tc->dbgfs_dir)
		return;

	debugfs_create_file("port", 0600, tc->dbgfs_dir,
			    tc, &tool_port_fops);

	debugfs_create_file("link", 0600, tc->dbgfs_dir,
			    tc, &tool_link_fops);

	debugfs_create_file("db", 0600, tc->dbgfs_dir,
			    tc, &tool_db_fops);

	debugfs_create_file("db_valid_mask", 0600, tc->dbgfs_dir,
			    tc, &tool_db_valid_mask_fops);

	debugfs_create_file("db_mask", 0600, tc->dbgfs_dir,
			    tc, &tool_db_mask_fops);

	debugfs_create_file("db_event", 0600, tc->dbgfs_dir,
			    tc, &tool_db_event_fops);

	debugfs_create_file("peer_db", 0600, tc->dbgfs_dir,
			    tc, &tool_peer_db_fops);

	debugfs_create_file("peer_db_mask", 0600, tc->dbgfs_dir,
			    tc, &tool_peer_db_mask_fops);

	if (tc->inspad_cnt != 0) {
		for (sidx = 0; sidx < tc->inspad_cnt; sidx++) {
			snprintf(buf, sizeof(buf), "spad%d", sidx);

			debugfs_create_file(buf, 0600, tc->dbgfs_dir,
					   &tc->inspads[sidx], &tool_spad_fops);
		}
	}

	if (tc->inmsg_cnt != 0) {
		for (midx = 0; midx < tc->inmsg_cnt; midx++) {
			snprintf(buf, sizeof(buf), "msg%d", midx);
			debugfs_create_file(buf, 0600, tc->dbgfs_dir,
					   &tc->inmsgs[midx], &tool_inmsg_fops);
		}

		debugfs_create_file("msg_sts", 0600, tc->dbgfs_dir,
				    tc, &tool_msg_sts_fops);

		debugfs_create_file("msg_inbits", 0600, tc->dbgfs_dir,
				    tc, &tool_msg_inbits_fops);

		debugfs_create_file("msg_outbits", 0600, tc->dbgfs_dir,
				    tc, &tool_msg_outbits_fops);

		debugfs_create_file("msg_mask", 0600, tc->dbgfs_dir,
				    tc, &tool_msg_mask_fops);

		debugfs_create_file("msg_event", 0600, tc->dbgfs_dir,
				    tc, &tool_msg_event_fops);
	}

	for (pidx = 0; pidx < tc->peer_cnt; pidx++) {
		snprintf(buf, sizeof(buf), "peer%d", pidx);
		tc->peers[pidx].dbgfs_dir =
			debugfs_create_dir(buf, tc->dbgfs_dir);

		debugfs_create_file("port", 0600,
				    tc->peers[pidx].dbgfs_dir,
				    &tc->peers[pidx], &tool_peer_port_fops);

		debugfs_create_file("link", 0200,
				    tc->peers[pidx].dbgfs_dir,
				    &tc->peers[pidx], &tool_peer_link_fops);

		debugfs_create_file("link_event", 0200,
				  tc->peers[pidx].dbgfs_dir,
				  &tc->peers[pidx], &tool_peer_link_event_fops);

		for (widx = 0; widx < tc->peers[pidx].inmw_cnt; widx++) {
			snprintf(buf, sizeof(buf), "mw_trans%d", widx);
			debugfs_create_file(buf, 0600,
					    tc->peers[pidx].dbgfs_dir,
					    &tc->peers[pidx].inmws[widx],
					    &tool_mw_trans_fops);
		}

		for (widx = 0; widx < tc->peers[pidx].outmw_cnt; widx++) {
			snprintf(buf, sizeof(buf), "peer_mw_trans%d", widx);
			debugfs_create_file(buf, 0600,
					    tc->peers[pidx].dbgfs_dir,
					    &tc->peers[pidx].outmws[widx],
					    &tool_peer_mw_trans_fops);
		}

		for (sidx = 0; sidx < tc->peers[pidx].outspad_cnt; sidx++) {
			snprintf(buf, sizeof(buf), "spad%d", sidx);

			debugfs_create_file(buf, 0600,
					    tc->peers[pidx].dbgfs_dir,
					    &tc->peers[pidx].outspads[sidx],
					    &tool_peer_spad_fops);
		}

		for (midx = 0; midx < tc->peers[pidx].outmsg_cnt; midx++) {
			snprintf(buf, sizeof(buf), "msg%d", midx);
			debugfs_create_file(buf, 0600,
					    tc->peers[pidx].dbgfs_dir,
					    &tc->peers[pidx].outmsgs[midx],
					    &tool_outmsg_fops);
		}
	}
}

static void tool_clear_dbgfs(struct tool_ctx *tc)
{
	debugfs_remove_recursive(tc->dbgfs_dir);
}

static int tool_probe(struct ntb_client *self, struct ntb_dev *ntb)
{
	struct tool_ctx *tc;
	int ret;

	tc = tool_create_data(ntb);
	if (IS_ERR(tc))
		return PTR_ERR(tc);

	ret = tool_init_peers(tc);
	if (ret != 0)
		goto err_clear_data;

	ret = tool_init_mws(tc);
	if (ret != 0)
		goto err_clear_data;

	ret = tool_init_spads(tc);
	if (ret != 0)
		goto err_clear_mws;

	ret = tool_init_msgs(tc);
	if (ret != 0)
		goto err_clear_mws;

	ret = tool_init_ntb(tc);
	if (ret != 0)
		goto err_clear_mws;

	tool_setup_dbgfs(tc);

	return 0;

err_clear_mws:
	tool_clear_mws(tc);

err_clear_data:
	tool_clear_data(tc);

	return ret;
}

static void tool_remove(struct ntb_client *self, struct ntb_dev *ntb)
{
	struct tool_ctx *tc = ntb->ctx;

	tool_clear_dbgfs(tc);

	tool_clear_ntb(tc);

	tool_clear_mws(tc);

	tool_clear_data(tc);
}

static struct ntb_client tool_client = {
	.ops = {
		.probe = tool_probe,
		.remove = tool_remove,
	}
};

static int __init tool_init(void)
{
	int ret;

	if (debugfs_initialized())
		tool_dbgfs_topdir = debugfs_create_dir(KBUILD_MODNAME, NULL);

	ret = ntb_register_client(&tool_client);
	if (ret)
		debugfs_remove_recursive(tool_dbgfs_topdir);

	return ret;
}
module_init(tool_init);

static void __exit tool_exit(void)
{
	ntb_unregister_client(&tool_client);
	debugfs_remove_recursive(tool_dbgfs_topdir);
}
module_exit(tool_exit);