mirror of
https://github.com/AuxXxilium/linux_dsm_epyc7002.git
synced 2024-12-05 09:46:43 +07:00
8a81252b77
Mateusz Guzik reported : Currently obtaining a new file descriptor results in locking fdtable twice - once in order to reserve a slot and second time to fill it. Holding the spinlock in __fd_install() is needed in case a resize is done, or to prevent a resize. Mateusz provided an RFC patch and a micro benchmark : http://people.redhat.com/~mguzik/pipebench.c A resize is an unlikely operation in a process lifetime, as table size is at least doubled at every resize. We can use RCU instead of the spinlock. __fd_install() must wait if a resize is in progress. The resize must block new __fd_install() callers from starting, and wait that ongoing install are finished (synchronize_sched()) resize should be attempted by a single thread to not waste resources. rcu_sched variant is used, as __fd_install() and expand_fdtable() run from process context. It gives us a ~30% speedup using pipebench on a dual Intel(R) Xeon(R) CPU E5-2696 v2 @ 2.50GHz Signed-off-by: Eric Dumazet <edumazet@google.com> Reported-by: Mateusz Guzik <mguzik@redhat.com> Acked-by: Mateusz Guzik <mguzik@redhat.com> Tested-by: Mateusz Guzik <mguzik@redhat.com> Signed-off-by: Al Viro <viro@zeniv.linux.org.uk>
122 lines
3.1 KiB
C
122 lines
3.1 KiB
C
/*
|
|
* descriptor table internals; you almost certainly want file.h instead.
|
|
*/
|
|
|
|
#ifndef __LINUX_FDTABLE_H
|
|
#define __LINUX_FDTABLE_H
|
|
|
|
#include <linux/posix_types.h>
|
|
#include <linux/compiler.h>
|
|
#include <linux/spinlock.h>
|
|
#include <linux/rcupdate.h>
|
|
#include <linux/types.h>
|
|
#include <linux/init.h>
|
|
#include <linux/fs.h>
|
|
|
|
#include <linux/atomic.h>
|
|
|
|
/*
|
|
* The default fd array needs to be at least BITS_PER_LONG,
|
|
* as this is the granularity returned by copy_fdset().
|
|
*/
|
|
#define NR_OPEN_DEFAULT BITS_PER_LONG
|
|
|
|
struct fdtable {
|
|
unsigned int max_fds;
|
|
struct file __rcu **fd; /* current fd array */
|
|
unsigned long *close_on_exec;
|
|
unsigned long *open_fds;
|
|
struct rcu_head rcu;
|
|
};
|
|
|
|
static inline bool close_on_exec(int fd, const struct fdtable *fdt)
|
|
{
|
|
return test_bit(fd, fdt->close_on_exec);
|
|
}
|
|
|
|
static inline bool fd_is_open(int fd, const struct fdtable *fdt)
|
|
{
|
|
return test_bit(fd, fdt->open_fds);
|
|
}
|
|
|
|
/*
|
|
* Open file table structure
|
|
*/
|
|
struct files_struct {
|
|
/*
|
|
* read mostly part
|
|
*/
|
|
atomic_t count;
|
|
bool resize_in_progress;
|
|
wait_queue_head_t resize_wait;
|
|
|
|
struct fdtable __rcu *fdt;
|
|
struct fdtable fdtab;
|
|
/*
|
|
* written part on a separate cache line in SMP
|
|
*/
|
|
spinlock_t file_lock ____cacheline_aligned_in_smp;
|
|
int next_fd;
|
|
unsigned long close_on_exec_init[1];
|
|
unsigned long open_fds_init[1];
|
|
struct file __rcu * fd_array[NR_OPEN_DEFAULT];
|
|
};
|
|
|
|
struct file_operations;
|
|
struct vfsmount;
|
|
struct dentry;
|
|
|
|
#define rcu_dereference_check_fdtable(files, fdtfd) \
|
|
rcu_dereference_check((fdtfd), lockdep_is_held(&(files)->file_lock))
|
|
|
|
#define files_fdtable(files) \
|
|
rcu_dereference_check_fdtable((files), (files)->fdt)
|
|
|
|
/*
|
|
* The caller must ensure that fd table isn't shared or hold rcu or file lock
|
|
*/
|
|
static inline struct file *__fcheck_files(struct files_struct *files, unsigned int fd)
|
|
{
|
|
struct fdtable *fdt = rcu_dereference_raw(files->fdt);
|
|
|
|
if (fd < fdt->max_fds)
|
|
return rcu_dereference_raw(fdt->fd[fd]);
|
|
return NULL;
|
|
}
|
|
|
|
static inline struct file *fcheck_files(struct files_struct *files, unsigned int fd)
|
|
{
|
|
rcu_lockdep_assert(rcu_read_lock_held() ||
|
|
lockdep_is_held(&files->file_lock),
|
|
"suspicious rcu_dereference_check() usage");
|
|
return __fcheck_files(files, fd);
|
|
}
|
|
|
|
/*
|
|
* Check whether the specified fd has an open file.
|
|
*/
|
|
#define fcheck(fd) fcheck_files(current->files, fd)
|
|
|
|
struct task_struct;
|
|
|
|
struct files_struct *get_files_struct(struct task_struct *);
|
|
void put_files_struct(struct files_struct *fs);
|
|
void reset_files_struct(struct files_struct *);
|
|
int unshare_files(struct files_struct **);
|
|
struct files_struct *dup_fd(struct files_struct *, int *);
|
|
void do_close_on_exec(struct files_struct *);
|
|
int iterate_fd(struct files_struct *, unsigned,
|
|
int (*)(const void *, struct file *, unsigned),
|
|
const void *);
|
|
|
|
extern int __alloc_fd(struct files_struct *files,
|
|
unsigned start, unsigned end, unsigned flags);
|
|
extern void __fd_install(struct files_struct *files,
|
|
unsigned int fd, struct file *file);
|
|
extern int __close_fd(struct files_struct *files,
|
|
unsigned int fd);
|
|
|
|
extern struct kmem_cache *files_cachep;
|
|
|
|
#endif /* __LINUX_FDTABLE_H */
|