linux_dsm_epyc7002/fs/btrfs/locking.c

/*
 * Copyright (C) 2008 Oracle.  All rights reserved.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public
 * License v2 as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public
 * License along with this program; if not, write to the
 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
 * Boston, MA 021110-1307, USA.
 */
#include <linux/sched.h>
#include <linux/gfp.h>
#include <linux/pagemap.h>
#include <linux/spinlock.h>
#include <linux/page-flags.h>
#include <asm/bug.h>
#include "ctree.h"
#include "extent_io.h"
#include "locking.h"

static inline void spin_nested(struct extent_buffer *eb)
{
	spin_lock(&eb->lock);
}

/*
 * Setting a lock to blocking will drop the spinlock and set the
 * flag that forces other procs who want the lock to wait.  After
 * this you can safely schedule with the lock held.
 */
void btrfs_set_lock_blocking(struct extent_buffer *eb)
{
	if (!test_bit(EXTENT_BUFFER_BLOCKING, &eb->bflags)) {
		set_bit(EXTENT_BUFFER_BLOCKING, &eb->bflags);
		spin_unlock(&eb->lock);
	}
	/* exit with the spin lock released and the bit set */
}

/*
 * clearing the blocking flag will take the spinlock again.
 * After this you can't safely schedule
 */
void btrfs_clear_lock_blocking(struct extent_buffer *eb)
{
	if (test_bit(EXTENT_BUFFER_BLOCKING, &eb->bflags)) {
		spin_nested(eb);
		clear_bit(EXTENT_BUFFER_BLOCKING, &eb->bflags);
		smp_mb__after_clear_bit();
	}
	/* exit with the spin lock held */
}

/*
 * unfortunately, many of the places that currently set a lock to blocking
 * don't end up blocking for very long, and often they don't block
 * at all.  For a dbench 50 run, if we don't spin on the blocking bit
 * at all, the context switch rate can jump up to 400,000/sec or more.
 *
 * So, we're still stuck with this crummy spin on the blocking bit,
 * at least until the most common causes of the short blocks
 * can be dealt with.
 */
static int btrfs_spin_on_block(struct extent_buffer *eb)
{
	int i;

	for (i = 0; i < 512; i++) {
		if (!test_bit(EXTENT_BUFFER_BLOCKING, &eb->bflags))
			return 1;
		if (need_resched())
			break;
		cpu_relax();
	}
	return 0;
}

/*
 * This is somewhat different from trylock.  It will take the
 * spinlock but if it finds the lock is set to blocking, it will
 * return without the lock held.
 *
 * returns 1 if it was able to take the lock and zero otherwise
 *
 * After this call, scheduling is not safe without first calling
 * btrfs_set_lock_blocking()
 */
int btrfs_try_spin_lock(struct extent_buffer *eb)
{
	int i;

	if (btrfs_spin_on_block(eb)) {
		spin_nested(eb);
		if (!test_bit(EXTENT_BUFFER_BLOCKING, &eb->bflags))
			return 1;
		spin_unlock(&eb->lock);
	}
	/* spin for a bit on the BLOCKING flag */
	for (i = 0; i < 2; i++) {
		cpu_relax();
		if (!btrfs_spin_on_block(eb))
			break;

		spin_nested(eb);
		if (!test_bit(EXTENT_BUFFER_BLOCKING, &eb->bflags))
			return 1;
		spin_unlock(&eb->lock);
	}
	return 0;
}

/*
 * the autoremove wake function will return 0 if it tried to wake up
 * a process that was already awake, which means that process won't
 * count as an exclusive wakeup.  The waitq code will continue waking
 * procs until it finds one that was actually sleeping.
 *
 * For btrfs, this isn't quite what we want.  We want a single proc
 * to be notified that the lock is ready for taking.  If that proc
 * already happen to be awake, great, it will loop around and try for
 * the lock.
 *
 * So, btrfs_wake_function always returns 1, even when the proc that we
 * tried to wake up was already awake.
 */
static int btrfs_wake_function(wait_queue_t *wait, unsigned mode,
			       int sync, void *key)
{
	autoremove_wake_function(wait, mode, sync, key);
	return 1;
}

/*
 * returns with the extent buffer spinlocked.
 *
 * This will spin and/or wait as required to take the lock, and then
 * return with the spinlock held.
 *
 * After this call, scheduling is not safe without first calling
 * btrfs_set_lock_blocking()
 */
int btrfs_tree_lock(struct extent_buffer *eb)
{
	DEFINE_WAIT(wait);
	wait.func = btrfs_wake_function;

	if (!btrfs_spin_on_block(eb))
		goto sleep;

	while(1) {
		spin_nested(eb);

		/* nobody is blocking, exit with the spinlock held */
		if (!test_bit(EXTENT_BUFFER_BLOCKING, &eb->bflags))
			return 0;

		/*
		 * we have the spinlock, but the real owner is blocking.
		 * wait for them
		 */
		spin_unlock(&eb->lock);

		/*
		 * spin for a bit, and if the blocking flag goes away,
		 * loop around
		 */
		cpu_relax();
		if (btrfs_spin_on_block(eb))
			continue;
sleep:
		prepare_to_wait_exclusive(&eb->lock_wq, &wait,
					  TASK_UNINTERRUPTIBLE);

		if (test_bit(EXTENT_BUFFER_BLOCKING, &eb->bflags))
			schedule();

		finish_wait(&eb->lock_wq, &wait);
	}
	return 0;
}

/*
 * Very quick trylock, this does not spin or schedule.  It returns
 * 1 with the spinlock held if it was able to take the lock, or it
 * returns zero if it was unable to take the lock.
 *
 * After this call, scheduling is not safe without first calling
 * btrfs_set_lock_blocking()
 */
int btrfs_try_tree_lock(struct extent_buffer *eb)
{
	if (spin_trylock(&eb->lock)) {
		if (test_bit(EXTENT_BUFFER_BLOCKING, &eb->bflags)) {
			/*
			 * we've got the spinlock, but the real owner is
			 * blocking.  Drop the spinlock and return failure
			 */
			spin_unlock(&eb->lock);
			return 0;
		}
		return 1;
	}
	/* someone else has the spinlock giveup */
	return 0;
}

int btrfs_tree_unlock(struct extent_buffer *eb)
{
	/*
	 * if we were a blocking owner, we don't have the spinlock held
	 * just clear the bit and look for waiters
	 */
	if (test_and_clear_bit(EXTENT_BUFFER_BLOCKING, &eb->bflags))
		smp_mb__after_clear_bit();
	else
		spin_unlock(&eb->lock);

	if (waitqueue_active(&eb->lock_wq))
		wake_up(&eb->lock_wq);
	return 0;
}

void btrfs_assert_tree_locked(struct extent_buffer *eb)
{
	if (!test_bit(EXTENT_BUFFER_BLOCKING, &eb->bflags))
		assert_spin_locked(&eb->lock);
}