mirror of
https://github.com/AuxXxilium/linux_dsm_epyc7002.git
synced 2024-12-16 21:57:31 +07:00
852c788f83
IO scheduling and cgroup are tied to the issuing task via io_context and cgroup of %current. Unfortunately, there are cases where IOs need to be routed via a different task which makes scheduling and cgroup limit enforcement applied completely incorrectly. For example, all bios delayed by blk-throttle end up being issued by a delayed work item and get assigned the io_context of the worker task which happens to serve the work item and dumped to the default block cgroup. This is double confusing as bios which aren't delayed end up in the correct cgroup and makes using blk-throttle and cfq propio together impossible. Any code which punts IO issuing to another task is affected which is getting more and more common (e.g. btrfs). As both io_context and cgroup are firmly tied to task including userland visible APIs to manipulate them, it makes a lot of sense to match up tasks to bios. This patch implements bio_associate_current() which associates the specified bio with %current. The bio will record the associated ioc and blkcg at that point and block layer will use the recorded ones regardless of which task actually ends up issuing the bio. bio release puts the associated ioc and blkcg. It grabs and remembers ioc and blkcg instead of the task itself because task may already be dead by the time the bio is issued making ioc and blkcg inaccessible and those are all block layer cares about. elevator_set_req_fn() is updated such that the bio elvdata is being allocated for is available to the elevator. This doesn't update block cgroup policies yet. Further patches will implement the support. -v2: #ifdef CONFIG_BLK_CGROUP added around bio->bi_ioc dereference in rq_ioc() to fix build breakage. Signed-off-by: Tejun Heo <tj@kernel.org> Cc: Vivek Goyal <vgoyal@redhat.com> Cc: Kent Overstreet <koverstreet@google.com> Signed-off-by: Jens Axboe <axboe@kernel.dk>
211 lines
6.9 KiB
C
211 lines
6.9 KiB
C
#ifndef _LINUX_ELEVATOR_H
|
|
#define _LINUX_ELEVATOR_H
|
|
|
|
#include <linux/percpu.h>
|
|
|
|
#ifdef CONFIG_BLOCK
|
|
|
|
struct io_cq;
|
|
|
|
typedef int (elevator_merge_fn) (struct request_queue *, struct request **,
|
|
struct bio *);
|
|
|
|
typedef void (elevator_merge_req_fn) (struct request_queue *, struct request *, struct request *);
|
|
|
|
typedef void (elevator_merged_fn) (struct request_queue *, struct request *, int);
|
|
|
|
typedef int (elevator_allow_merge_fn) (struct request_queue *, struct request *, struct bio *);
|
|
|
|
typedef void (elevator_bio_merged_fn) (struct request_queue *,
|
|
struct request *, struct bio *);
|
|
|
|
typedef int (elevator_dispatch_fn) (struct request_queue *, int);
|
|
|
|
typedef void (elevator_add_req_fn) (struct request_queue *, struct request *);
|
|
typedef struct request *(elevator_request_list_fn) (struct request_queue *, struct request *);
|
|
typedef void (elevator_completed_req_fn) (struct request_queue *, struct request *);
|
|
typedef int (elevator_may_queue_fn) (struct request_queue *, int);
|
|
|
|
typedef void (elevator_init_icq_fn) (struct io_cq *);
|
|
typedef void (elevator_exit_icq_fn) (struct io_cq *);
|
|
typedef int (elevator_set_req_fn) (struct request_queue *, struct request *,
|
|
struct bio *, gfp_t);
|
|
typedef void (elevator_put_req_fn) (struct request *);
|
|
typedef void (elevator_activate_req_fn) (struct request_queue *, struct request *);
|
|
typedef void (elevator_deactivate_req_fn) (struct request_queue *, struct request *);
|
|
|
|
typedef int (elevator_init_fn) (struct request_queue *);
|
|
typedef void (elevator_exit_fn) (struct elevator_queue *);
|
|
|
|
struct elevator_ops
|
|
{
|
|
elevator_merge_fn *elevator_merge_fn;
|
|
elevator_merged_fn *elevator_merged_fn;
|
|
elevator_merge_req_fn *elevator_merge_req_fn;
|
|
elevator_allow_merge_fn *elevator_allow_merge_fn;
|
|
elevator_bio_merged_fn *elevator_bio_merged_fn;
|
|
|
|
elevator_dispatch_fn *elevator_dispatch_fn;
|
|
elevator_add_req_fn *elevator_add_req_fn;
|
|
elevator_activate_req_fn *elevator_activate_req_fn;
|
|
elevator_deactivate_req_fn *elevator_deactivate_req_fn;
|
|
|
|
elevator_completed_req_fn *elevator_completed_req_fn;
|
|
|
|
elevator_request_list_fn *elevator_former_req_fn;
|
|
elevator_request_list_fn *elevator_latter_req_fn;
|
|
|
|
elevator_init_icq_fn *elevator_init_icq_fn; /* see iocontext.h */
|
|
elevator_exit_icq_fn *elevator_exit_icq_fn; /* ditto */
|
|
|
|
elevator_set_req_fn *elevator_set_req_fn;
|
|
elevator_put_req_fn *elevator_put_req_fn;
|
|
|
|
elevator_may_queue_fn *elevator_may_queue_fn;
|
|
|
|
elevator_init_fn *elevator_init_fn;
|
|
elevator_exit_fn *elevator_exit_fn;
|
|
};
|
|
|
|
#define ELV_NAME_MAX (16)
|
|
|
|
struct elv_fs_entry {
|
|
struct attribute attr;
|
|
ssize_t (*show)(struct elevator_queue *, char *);
|
|
ssize_t (*store)(struct elevator_queue *, const char *, size_t);
|
|
};
|
|
|
|
/*
|
|
* identifies an elevator type, such as AS or deadline
|
|
*/
|
|
struct elevator_type
|
|
{
|
|
/* managed by elevator core */
|
|
struct kmem_cache *icq_cache;
|
|
|
|
/* fields provided by elevator implementation */
|
|
struct elevator_ops ops;
|
|
size_t icq_size; /* see iocontext.h */
|
|
size_t icq_align; /* ditto */
|
|
struct elv_fs_entry *elevator_attrs;
|
|
char elevator_name[ELV_NAME_MAX];
|
|
struct module *elevator_owner;
|
|
|
|
/* managed by elevator core */
|
|
char icq_cache_name[ELV_NAME_MAX + 5]; /* elvname + "_io_cq" */
|
|
struct list_head list;
|
|
};
|
|
|
|
/*
|
|
* each queue has an elevator_queue associated with it
|
|
*/
|
|
struct elevator_queue
|
|
{
|
|
struct elevator_type *type;
|
|
void *elevator_data;
|
|
struct kobject kobj;
|
|
struct mutex sysfs_lock;
|
|
struct hlist_head *hash;
|
|
unsigned int registered:1;
|
|
};
|
|
|
|
/*
|
|
* block elevator interface
|
|
*/
|
|
extern void elv_dispatch_sort(struct request_queue *, struct request *);
|
|
extern void elv_dispatch_add_tail(struct request_queue *, struct request *);
|
|
extern void elv_add_request(struct request_queue *, struct request *, int);
|
|
extern void __elv_add_request(struct request_queue *, struct request *, int);
|
|
extern int elv_merge(struct request_queue *, struct request **, struct bio *);
|
|
extern void elv_merge_requests(struct request_queue *, struct request *,
|
|
struct request *);
|
|
extern void elv_merged_request(struct request_queue *, struct request *, int);
|
|
extern void elv_bio_merged(struct request_queue *q, struct request *,
|
|
struct bio *);
|
|
extern void elv_requeue_request(struct request_queue *, struct request *);
|
|
extern struct request *elv_former_request(struct request_queue *, struct request *);
|
|
extern struct request *elv_latter_request(struct request_queue *, struct request *);
|
|
extern int elv_register_queue(struct request_queue *q);
|
|
extern void elv_unregister_queue(struct request_queue *q);
|
|
extern int elv_may_queue(struct request_queue *, int);
|
|
extern void elv_abort_queue(struct request_queue *);
|
|
extern void elv_completed_request(struct request_queue *, struct request *);
|
|
extern int elv_set_request(struct request_queue *q, struct request *rq,
|
|
struct bio *bio, gfp_t gfp_mask);
|
|
extern void elv_put_request(struct request_queue *, struct request *);
|
|
extern void elv_drain_elevator(struct request_queue *);
|
|
|
|
/*
|
|
* io scheduler registration
|
|
*/
|
|
extern int elv_register(struct elevator_type *);
|
|
extern void elv_unregister(struct elevator_type *);
|
|
|
|
/*
|
|
* io scheduler sysfs switching
|
|
*/
|
|
extern ssize_t elv_iosched_show(struct request_queue *, char *);
|
|
extern ssize_t elv_iosched_store(struct request_queue *, const char *, size_t);
|
|
|
|
extern int elevator_init(struct request_queue *, char *);
|
|
extern void elevator_exit(struct elevator_queue *);
|
|
extern int elevator_change(struct request_queue *, const char *);
|
|
extern bool elv_rq_merge_ok(struct request *, struct bio *);
|
|
|
|
/*
|
|
* Helper functions.
|
|
*/
|
|
extern struct request *elv_rb_former_request(struct request_queue *, struct request *);
|
|
extern struct request *elv_rb_latter_request(struct request_queue *, struct request *);
|
|
|
|
/*
|
|
* rb support functions.
|
|
*/
|
|
extern void elv_rb_add(struct rb_root *, struct request *);
|
|
extern void elv_rb_del(struct rb_root *, struct request *);
|
|
extern struct request *elv_rb_find(struct rb_root *, sector_t);
|
|
|
|
/*
|
|
* Return values from elevator merger
|
|
*/
|
|
#define ELEVATOR_NO_MERGE 0
|
|
#define ELEVATOR_FRONT_MERGE 1
|
|
#define ELEVATOR_BACK_MERGE 2
|
|
|
|
/*
|
|
* Insertion selection
|
|
*/
|
|
#define ELEVATOR_INSERT_FRONT 1
|
|
#define ELEVATOR_INSERT_BACK 2
|
|
#define ELEVATOR_INSERT_SORT 3
|
|
#define ELEVATOR_INSERT_REQUEUE 4
|
|
#define ELEVATOR_INSERT_FLUSH 5
|
|
#define ELEVATOR_INSERT_SORT_MERGE 6
|
|
|
|
/*
|
|
* return values from elevator_may_queue_fn
|
|
*/
|
|
enum {
|
|
ELV_MQUEUE_MAY,
|
|
ELV_MQUEUE_NO,
|
|
ELV_MQUEUE_MUST,
|
|
};
|
|
|
|
#define rq_end_sector(rq) (blk_rq_pos(rq) + blk_rq_sectors(rq))
|
|
#define rb_entry_rq(node) rb_entry((node), struct request, rb_node)
|
|
|
|
/*
|
|
* Hack to reuse the csd.list list_head as the fifo time holder while
|
|
* the request is in the io scheduler. Saves an unsigned long in rq.
|
|
*/
|
|
#define rq_fifo_time(rq) ((unsigned long) (rq)->csd.list.next)
|
|
#define rq_set_fifo_time(rq,exp) ((rq)->csd.list.next = (void *) (exp))
|
|
#define rq_entry_fifo(ptr) list_entry((ptr), struct request, queuelist)
|
|
#define rq_fifo_clear(rq) do { \
|
|
list_del_init(&(rq)->queuelist); \
|
|
INIT_LIST_HEAD(&(rq)->csd.list); \
|
|
} while (0)
|
|
|
|
#endif /* CONFIG_BLOCK */
|
|
#endif
|