mirror of
https://github.com/AuxXxilium/linux_dsm_epyc7002.git
synced 2024-12-24 20:14:27 +07:00
b050f9f6dd
Often when running the qgroups sanity test, a crash or a hang happened. This is because the extent buffer the test uses for the root node doesn't have an header level explicitly set, making it have a random level value. This is a problem when it's not zero for the btrfs_search_slot() calls the test ends up doing, resulting in crashes or hangs such as the following: [ 6454.127192] Btrfs loaded, debug=on, assert=on, integrity-checker=on (...) [ 6454.127760] BTRFS: selftest: Running qgroup tests [ 6454.127964] BTRFS: selftest: Running test_test_no_shared_qgroup [ 6454.127966] BTRFS: selftest: Qgroup basic add [ 6480.152005] BUG: soft lockup - CPU#0 stuck for 23s! [modprobe:5383] [ 6480.152005] Modules linked in: btrfs(+) xor raid6_pq binfmt_misc nfsd auth_rpcgss oid_registry nfs_acl nfs lockd fscache sunrpc i2c_piix4 i2c_core pcspkr evbug psmouse serio_raw e1000 [last unloaded: btrfs] [ 6480.152005] irq event stamp: 188448 [ 6480.152005] hardirqs last enabled at (188447): [<ffffffff8168ef5c>] restore_args+0x0/0x30 [ 6480.152005] hardirqs last disabled at (188448): [<ffffffff81698e6a>] apic_timer_interrupt+0x6a/0x80 [ 6480.152005] softirqs last enabled at (188446): [<ffffffff810516cf>] __do_softirq+0x1cf/0x450 [ 6480.152005] softirqs last disabled at (188441): [<ffffffff81051c25>] irq_exit+0xb5/0xc0 [ 6480.152005] CPU: 0 PID: 5383 Comm: modprobe Not tainted 3.15.0-rc8-fdm-btrfs-next-33+ #4 [ 6480.152005] Hardware name: Bochs Bochs, BIOS Bochs 01/01/2011 [ 6480.152005] task: ffff8802146125a0 ti: ffff8800d0d00000 task.ti: ffff8800d0d00000 [ 6480.152005] RIP: 0010:[<ffffffff81349a63>] [<ffffffff81349a63>] __write_lock_failed+0x13/0x20 [ 6480.152005] RSP: 0018:ffff8800d0d038e8 EFLAGS: 00000287 [ 6480.152005] RAX: 0000000000000000 RBX: ffffffff8168ef5c RCX: 000005deb8525852 [ 6480.152005] RDX: 0000000000000000 RSI: 0000000000001d45 RDI: ffff8802105000b8 [ 6480.152005] RBP: ffff8800d0d038e8 R08: fffffe12710f63db R09: ffffffffa03196fb [ 6480.152005] R10: ffff8802146125a0 R11: ffff880214612e28 R12: ffff8800d0d03858 [ 6480.152005] R13: 0000000000000000 R14: ffff8800d0d00000 R15: ffff8802146125a0 [ 6480.152005] FS: 00007f14ff804700(0000) GS:ffff880215e00000(0000) knlGS:0000000000000000 [ 6480.152005] CS: 0010 DS: 0000 ES: 0000 CR0: 000000008005003b [ 6480.152005] CR2: 00007fff4df0dac8 CR3: 00000000d1796000 CR4: 00000000000006f0 [ 6480.152005] Stack: [ 6480.152005] ffff8800d0d03908 ffffffff810ae967 0000000000000001 ffff8802105000b8 [ 6480.152005] ffff8800d0d03938 ffffffff8168e57e ffffffffa0319c16 0000000000000007 [ 6480.152005] ffff880210500000 ffff880210500100 ffff8800d0d039b8 ffffffffa0319c16 [ 6480.152005] Call Trace: [ 6480.152005] [<ffffffff810ae967>] do_raw_write_lock+0x47/0xa0 [ 6480.152005] [<ffffffff8168e57e>] _raw_write_lock+0x5e/0x80 [ 6480.152005] [<ffffffffa0319c16>] ? btrfs_tree_lock+0x116/0x270 [btrfs] [ 6480.152005] [<ffffffffa0319c16>] btrfs_tree_lock+0x116/0x270 [btrfs] [ 6480.152005] [<ffffffffa02b2acb>] btrfs_lock_root_node+0x3b/0x50 [btrfs] [ 6480.152005] [<ffffffffa02b81a6>] btrfs_search_slot+0x916/0xa20 [btrfs] [ 6480.152005] [<ffffffff811a727f>] ? create_object+0x23f/0x300 [ 6480.152005] [<ffffffffa02b9958>] btrfs_insert_empty_items+0x78/0xd0 [btrfs] [ 6480.152005] [<ffffffffa036041a>] insert_normal_tree_ref.constprop.4+0xa2/0x19a [btrfs] [ 6480.152005] [<ffffffffa03605c3>] test_no_shared_qgroup+0xb1/0x1ca [btrfs] [ 6480.152005] [<ffffffff8108cad6>] ? local_clock+0x16/0x30 [ 6480.152005] [<ffffffffa035ef8e>] btrfs_test_qgroups+0x1ae/0x1d7 [btrfs] [ 6480.152005] [<ffffffffa03a69d2>] ? ftrace_define_fields_btrfs_space_reservation+0xfd/0xfd [btrfs] [ 6480.152005] [<ffffffffa03a6a86>] init_btrfs_fs+0xb4/0x153 [btrfs] [ 6480.152005] [<ffffffff81000352>] do_one_initcall+0x102/0x150 [ 6480.152005] [<ffffffff8103d223>] ? set_memory_nx+0x43/0x50 [ 6480.152005] [<ffffffff81682668>] ? set_section_ro_nx+0x6d/0x74 [ 6480.152005] [<ffffffff810d91cc>] load_module+0x1cdc/0x2630 (...) Therefore initialize the extent buffer as an empty leaf (level 0). Issue easy to reproduce when btrfs is built as a module via: $ for ((i = 1; i <= 1000000; i++)); do rmmod btrfs; modprobe btrfs; done Signed-off-by: Filipe David Borba Manana <fdmanana@gmail.com> Signed-off-by: Chris Mason <clm@fb.com>
471 lines
12 KiB
C
471 lines
12 KiB
C
/*
|
|
* Copyright (C) 2013 Facebook. All rights reserved.
|
|
*
|
|
* This program is free software; you can redistribute it and/or
|
|
* modify it under the terms of the GNU General Public
|
|
* License v2 as published by the Free Software Foundation.
|
|
*
|
|
* This program is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
* General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU General Public
|
|
* License along with this program; if not, write to the
|
|
* Free Software Foundation, Inc., 59 Temple Place - Suite 330,
|
|
* Boston, MA 021110-1307, USA.
|
|
*/
|
|
|
|
#include "btrfs-tests.h"
|
|
#include "../ctree.h"
|
|
#include "../transaction.h"
|
|
#include "../disk-io.h"
|
|
#include "../qgroup.h"
|
|
|
|
static void init_dummy_trans(struct btrfs_trans_handle *trans)
|
|
{
|
|
memset(trans, 0, sizeof(*trans));
|
|
trans->transid = 1;
|
|
INIT_LIST_HEAD(&trans->qgroup_ref_list);
|
|
trans->type = __TRANS_DUMMY;
|
|
}
|
|
|
|
static int insert_normal_tree_ref(struct btrfs_root *root, u64 bytenr,
|
|
u64 num_bytes, u64 parent, u64 root_objectid)
|
|
{
|
|
struct btrfs_trans_handle trans;
|
|
struct btrfs_extent_item *item;
|
|
struct btrfs_extent_inline_ref *iref;
|
|
struct btrfs_tree_block_info *block_info;
|
|
struct btrfs_path *path;
|
|
struct extent_buffer *leaf;
|
|
struct btrfs_key ins;
|
|
u32 size = sizeof(*item) + sizeof(*iref) + sizeof(*block_info);
|
|
int ret;
|
|
|
|
init_dummy_trans(&trans);
|
|
|
|
ins.objectid = bytenr;
|
|
ins.type = BTRFS_EXTENT_ITEM_KEY;
|
|
ins.offset = num_bytes;
|
|
|
|
path = btrfs_alloc_path();
|
|
if (!path) {
|
|
test_msg("Couldn't allocate path\n");
|
|
return -ENOMEM;
|
|
}
|
|
|
|
path->leave_spinning = 1;
|
|
ret = btrfs_insert_empty_item(&trans, root, path, &ins, size);
|
|
if (ret) {
|
|
test_msg("Couldn't insert ref %d\n", ret);
|
|
btrfs_free_path(path);
|
|
return ret;
|
|
}
|
|
|
|
leaf = path->nodes[0];
|
|
item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
|
|
btrfs_set_extent_refs(leaf, item, 1);
|
|
btrfs_set_extent_generation(leaf, item, 1);
|
|
btrfs_set_extent_flags(leaf, item, BTRFS_EXTENT_FLAG_TREE_BLOCK);
|
|
block_info = (struct btrfs_tree_block_info *)(item + 1);
|
|
btrfs_set_tree_block_level(leaf, block_info, 1);
|
|
iref = (struct btrfs_extent_inline_ref *)(block_info + 1);
|
|
if (parent > 0) {
|
|
btrfs_set_extent_inline_ref_type(leaf, iref,
|
|
BTRFS_SHARED_BLOCK_REF_KEY);
|
|
btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
|
|
} else {
|
|
btrfs_set_extent_inline_ref_type(leaf, iref, BTRFS_TREE_BLOCK_REF_KEY);
|
|
btrfs_set_extent_inline_ref_offset(leaf, iref, root_objectid);
|
|
}
|
|
btrfs_free_path(path);
|
|
return 0;
|
|
}
|
|
|
|
static int add_tree_ref(struct btrfs_root *root, u64 bytenr, u64 num_bytes,
|
|
u64 parent, u64 root_objectid)
|
|
{
|
|
struct btrfs_trans_handle trans;
|
|
struct btrfs_extent_item *item;
|
|
struct btrfs_path *path;
|
|
struct btrfs_key key;
|
|
u64 refs;
|
|
int ret;
|
|
|
|
init_dummy_trans(&trans);
|
|
|
|
key.objectid = bytenr;
|
|
key.type = BTRFS_EXTENT_ITEM_KEY;
|
|
key.offset = num_bytes;
|
|
|
|
path = btrfs_alloc_path();
|
|
if (!path) {
|
|
test_msg("Couldn't allocate path\n");
|
|
return -ENOMEM;
|
|
}
|
|
|
|
path->leave_spinning = 1;
|
|
ret = btrfs_search_slot(&trans, root, &key, path, 0, 1);
|
|
if (ret) {
|
|
test_msg("Couldn't find extent ref\n");
|
|
btrfs_free_path(path);
|
|
return ret;
|
|
}
|
|
|
|
item = btrfs_item_ptr(path->nodes[0], path->slots[0],
|
|
struct btrfs_extent_item);
|
|
refs = btrfs_extent_refs(path->nodes[0], item);
|
|
btrfs_set_extent_refs(path->nodes[0], item, refs + 1);
|
|
btrfs_release_path(path);
|
|
|
|
key.objectid = bytenr;
|
|
if (parent) {
|
|
key.type = BTRFS_SHARED_BLOCK_REF_KEY;
|
|
key.offset = parent;
|
|
} else {
|
|
key.type = BTRFS_TREE_BLOCK_REF_KEY;
|
|
key.offset = root_objectid;
|
|
}
|
|
|
|
ret = btrfs_insert_empty_item(&trans, root, path, &key, 0);
|
|
if (ret)
|
|
test_msg("Failed to insert backref\n");
|
|
btrfs_free_path(path);
|
|
return ret;
|
|
}
|
|
|
|
static int remove_extent_item(struct btrfs_root *root, u64 bytenr,
|
|
u64 num_bytes)
|
|
{
|
|
struct btrfs_trans_handle trans;
|
|
struct btrfs_key key;
|
|
struct btrfs_path *path;
|
|
int ret;
|
|
|
|
init_dummy_trans(&trans);
|
|
|
|
key.objectid = bytenr;
|
|
key.type = BTRFS_EXTENT_ITEM_KEY;
|
|
key.offset = num_bytes;
|
|
|
|
path = btrfs_alloc_path();
|
|
if (!path) {
|
|
test_msg("Couldn't allocate path\n");
|
|
return -ENOMEM;
|
|
}
|
|
path->leave_spinning = 1;
|
|
|
|
ret = btrfs_search_slot(&trans, root, &key, path, -1, 1);
|
|
if (ret) {
|
|
test_msg("Didn't find our key %d\n", ret);
|
|
btrfs_free_path(path);
|
|
return ret;
|
|
}
|
|
btrfs_del_item(&trans, root, path);
|
|
btrfs_free_path(path);
|
|
return 0;
|
|
}
|
|
|
|
static int remove_extent_ref(struct btrfs_root *root, u64 bytenr,
|
|
u64 num_bytes, u64 parent, u64 root_objectid)
|
|
{
|
|
struct btrfs_trans_handle trans;
|
|
struct btrfs_extent_item *item;
|
|
struct btrfs_path *path;
|
|
struct btrfs_key key;
|
|
u64 refs;
|
|
int ret;
|
|
|
|
init_dummy_trans(&trans);
|
|
|
|
key.objectid = bytenr;
|
|
key.type = BTRFS_EXTENT_ITEM_KEY;
|
|
key.offset = num_bytes;
|
|
|
|
path = btrfs_alloc_path();
|
|
if (!path) {
|
|
test_msg("Couldn't allocate path\n");
|
|
return -ENOMEM;
|
|
}
|
|
|
|
path->leave_spinning = 1;
|
|
ret = btrfs_search_slot(&trans, root, &key, path, 0, 1);
|
|
if (ret) {
|
|
test_msg("Couldn't find extent ref\n");
|
|
btrfs_free_path(path);
|
|
return ret;
|
|
}
|
|
|
|
item = btrfs_item_ptr(path->nodes[0], path->slots[0],
|
|
struct btrfs_extent_item);
|
|
refs = btrfs_extent_refs(path->nodes[0], item);
|
|
btrfs_set_extent_refs(path->nodes[0], item, refs - 1);
|
|
btrfs_release_path(path);
|
|
|
|
key.objectid = bytenr;
|
|
if (parent) {
|
|
key.type = BTRFS_SHARED_BLOCK_REF_KEY;
|
|
key.offset = parent;
|
|
} else {
|
|
key.type = BTRFS_TREE_BLOCK_REF_KEY;
|
|
key.offset = root_objectid;
|
|
}
|
|
|
|
ret = btrfs_search_slot(&trans, root, &key, path, -1, 1);
|
|
if (ret) {
|
|
test_msg("Couldn't find backref %d\n", ret);
|
|
btrfs_free_path(path);
|
|
return ret;
|
|
}
|
|
btrfs_del_item(&trans, root, path);
|
|
btrfs_free_path(path);
|
|
return ret;
|
|
}
|
|
|
|
static int test_no_shared_qgroup(struct btrfs_root *root)
|
|
{
|
|
struct btrfs_trans_handle trans;
|
|
struct btrfs_fs_info *fs_info = root->fs_info;
|
|
int ret;
|
|
|
|
init_dummy_trans(&trans);
|
|
|
|
test_msg("Qgroup basic add\n");
|
|
ret = btrfs_create_qgroup(NULL, fs_info, 5, NULL);
|
|
if (ret) {
|
|
test_msg("Couldn't create a qgroup %d\n", ret);
|
|
return ret;
|
|
}
|
|
|
|
ret = btrfs_qgroup_record_ref(&trans, fs_info, 5, 4096, 4096,
|
|
BTRFS_QGROUP_OPER_ADD_EXCL, 0);
|
|
if (ret) {
|
|
test_msg("Couldn't add space to a qgroup %d\n", ret);
|
|
return ret;
|
|
}
|
|
|
|
ret = insert_normal_tree_ref(root, 4096, 4096, 0, 5);
|
|
if (ret)
|
|
return ret;
|
|
|
|
ret = btrfs_delayed_qgroup_accounting(&trans, fs_info);
|
|
if (ret) {
|
|
test_msg("Delayed qgroup accounting failed %d\n", ret);
|
|
return ret;
|
|
}
|
|
|
|
if (btrfs_verify_qgroup_counts(fs_info, 5, 4096, 4096)) {
|
|
test_msg("Qgroup counts didn't match expected values\n");
|
|
return -EINVAL;
|
|
}
|
|
|
|
ret = remove_extent_item(root, 4096, 4096);
|
|
if (ret)
|
|
return -EINVAL;
|
|
|
|
ret = btrfs_qgroup_record_ref(&trans, fs_info, 5, 4096, 4096,
|
|
BTRFS_QGROUP_OPER_SUB_EXCL, 0);
|
|
if (ret) {
|
|
test_msg("Couldn't remove space from the qgroup %d\n", ret);
|
|
return -EINVAL;
|
|
}
|
|
|
|
ret = btrfs_delayed_qgroup_accounting(&trans, fs_info);
|
|
if (ret) {
|
|
test_msg("Qgroup accounting failed %d\n", ret);
|
|
return -EINVAL;
|
|
}
|
|
|
|
if (btrfs_verify_qgroup_counts(fs_info, 5, 0, 0)) {
|
|
test_msg("Qgroup counts didn't match expected values\n");
|
|
return -EINVAL;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* Add a ref for two different roots to make sure the shared value comes out
|
|
* right, also remove one of the roots and make sure the exclusive count is
|
|
* adjusted properly.
|
|
*/
|
|
static int test_multiple_refs(struct btrfs_root *root)
|
|
{
|
|
struct btrfs_trans_handle trans;
|
|
struct btrfs_fs_info *fs_info = root->fs_info;
|
|
int ret;
|
|
|
|
init_dummy_trans(&trans);
|
|
|
|
test_msg("Qgroup multiple refs test\n");
|
|
|
|
/* We have 5 created already from the previous test */
|
|
ret = btrfs_create_qgroup(NULL, fs_info, 256, NULL);
|
|
if (ret) {
|
|
test_msg("Couldn't create a qgroup %d\n", ret);
|
|
return ret;
|
|
}
|
|
|
|
ret = insert_normal_tree_ref(root, 4096, 4096, 0, 5);
|
|
if (ret)
|
|
return ret;
|
|
|
|
ret = btrfs_qgroup_record_ref(&trans, fs_info, 5, 4096, 4096,
|
|
BTRFS_QGROUP_OPER_ADD_EXCL, 0);
|
|
if (ret) {
|
|
test_msg("Couldn't add space to a qgroup %d\n", ret);
|
|
return ret;
|
|
}
|
|
|
|
ret = btrfs_delayed_qgroup_accounting(&trans, fs_info);
|
|
if (ret) {
|
|
test_msg("Delayed qgroup accounting failed %d\n", ret);
|
|
return ret;
|
|
}
|
|
|
|
if (btrfs_verify_qgroup_counts(fs_info, 5, 4096, 4096)) {
|
|
test_msg("Qgroup counts didn't match expected values\n");
|
|
return -EINVAL;
|
|
}
|
|
|
|
ret = add_tree_ref(root, 4096, 4096, 0, 256);
|
|
if (ret)
|
|
return ret;
|
|
|
|
ret = btrfs_qgroup_record_ref(&trans, fs_info, 256, 4096, 4096,
|
|
BTRFS_QGROUP_OPER_ADD_SHARED, 0);
|
|
if (ret) {
|
|
test_msg("Qgroup record ref failed %d\n", ret);
|
|
return ret;
|
|
}
|
|
|
|
ret = btrfs_delayed_qgroup_accounting(&trans, fs_info);
|
|
if (ret) {
|
|
test_msg("Qgroup accounting failed %d\n", ret);
|
|
return ret;
|
|
}
|
|
|
|
if (btrfs_verify_qgroup_counts(fs_info, 5, 4096, 0)) {
|
|
test_msg("Qgroup counts didn't match expected values\n");
|
|
return -EINVAL;
|
|
}
|
|
|
|
if (btrfs_verify_qgroup_counts(fs_info, 256, 4096, 0)) {
|
|
test_msg("Qgroup counts didn't match expected values\n");
|
|
return -EINVAL;
|
|
}
|
|
|
|
ret = remove_extent_ref(root, 4096, 4096, 0, 256);
|
|
if (ret)
|
|
return ret;
|
|
|
|
ret = btrfs_qgroup_record_ref(&trans, fs_info, 256, 4096, 4096,
|
|
BTRFS_QGROUP_OPER_SUB_SHARED, 0);
|
|
if (ret) {
|
|
test_msg("Qgroup record ref failed %d\n", ret);
|
|
return ret;
|
|
}
|
|
|
|
ret = btrfs_delayed_qgroup_accounting(&trans, fs_info);
|
|
if (ret) {
|
|
test_msg("Qgroup accounting failed %d\n", ret);
|
|
return ret;
|
|
}
|
|
|
|
if (btrfs_verify_qgroup_counts(fs_info, 256, 0, 0)) {
|
|
test_msg("Qgroup counts didn't match expected values\n");
|
|
return -EINVAL;
|
|
}
|
|
|
|
if (btrfs_verify_qgroup_counts(fs_info, 5, 4096, 4096)) {
|
|
test_msg("Qgroup counts didn't match expected values\n");
|
|
return -EINVAL;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
int btrfs_test_qgroups(void)
|
|
{
|
|
struct btrfs_root *root;
|
|
struct btrfs_root *tmp_root;
|
|
int ret = 0;
|
|
|
|
root = btrfs_alloc_dummy_root();
|
|
if (IS_ERR(root)) {
|
|
test_msg("Couldn't allocate root\n");
|
|
return PTR_ERR(root);
|
|
}
|
|
|
|
root->fs_info = btrfs_alloc_dummy_fs_info();
|
|
if (!root->fs_info) {
|
|
test_msg("Couldn't allocate dummy fs info\n");
|
|
ret = -ENOMEM;
|
|
goto out;
|
|
}
|
|
|
|
/*
|
|
* Can't use bytenr 0, some things freak out
|
|
* *cough*backref walking code*cough*
|
|
*/
|
|
root->node = alloc_test_extent_buffer(root->fs_info, 4096, 4096);
|
|
if (!root->node) {
|
|
test_msg("Couldn't allocate dummy buffer\n");
|
|
ret = -ENOMEM;
|
|
goto out;
|
|
}
|
|
btrfs_set_header_level(root->node, 0);
|
|
btrfs_set_header_nritems(root->node, 0);
|
|
root->alloc_bytenr += 8192;
|
|
|
|
tmp_root = btrfs_alloc_dummy_root();
|
|
if (IS_ERR(tmp_root)) {
|
|
test_msg("Couldn't allocate a fs root\n");
|
|
ret = PTR_ERR(tmp_root);
|
|
goto out;
|
|
}
|
|
|
|
tmp_root->root_key.objectid = 5;
|
|
root->fs_info->fs_root = tmp_root;
|
|
ret = btrfs_insert_fs_root(root->fs_info, tmp_root);
|
|
if (ret) {
|
|
test_msg("Couldn't insert fs root %d\n", ret);
|
|
goto out;
|
|
}
|
|
|
|
tmp_root = btrfs_alloc_dummy_root();
|
|
if (IS_ERR(tmp_root)) {
|
|
test_msg("Couldn't allocate a fs root\n");
|
|
ret = PTR_ERR(tmp_root);
|
|
goto out;
|
|
}
|
|
|
|
tmp_root->root_key.objectid = 256;
|
|
ret = btrfs_insert_fs_root(root->fs_info, tmp_root);
|
|
if (ret) {
|
|
test_msg("Couldn't insert fs root %d\n", ret);
|
|
goto out;
|
|
}
|
|
|
|
/* We are using this root as our extent root */
|
|
root->fs_info->extent_root = root;
|
|
|
|
/*
|
|
* Some of the paths we test assume we have a filled out fs_info, so we
|
|
* just need to addt he root in there so we don't panic.
|
|
*/
|
|
root->fs_info->tree_root = root;
|
|
root->fs_info->quota_root = root;
|
|
root->fs_info->quota_enabled = 1;
|
|
|
|
test_msg("Running qgroup tests\n");
|
|
ret = test_no_shared_qgroup(root);
|
|
if (ret)
|
|
goto out;
|
|
ret = test_multiple_refs(root);
|
|
out:
|
|
btrfs_free_dummy_root(root);
|
|
return ret;
|
|
}
|