From 8922e16cf6269e668123acb1ae1fdc62b7a3a4fc Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Thu, 20 Oct 2005 16:23:44 +0200 Subject: [PATCH] 01/05 Implement generic dispatch queue Implements generic dispatch queue which can replace all dispatch queues implemented by each iosched. This reduces code duplication, eases enforcing semantics over dispatch queue, and simplifies specific ioscheds. Signed-off-by: Tejun Heo Signed-off-by: Jens Axboe --- include/linux/blkdev.h | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) (limited to 'include/linux/blkdev.h') diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index efdc9b5bc05..2c7b9154927 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -203,6 +203,7 @@ struct request { enum rq_flag_bits { __REQ_RW, /* not set, read. set, write */ __REQ_FAILFAST, /* no low level driver retries */ + __REQ_SORTED, /* elevator knows about this request */ __REQ_SOFTBARRIER, /* may not be passed by ioscheduler */ __REQ_HARDBARRIER, /* may not be passed by drive either */ __REQ_CMD, /* is a regular fs rw request */ @@ -235,6 +236,7 @@ enum rq_flag_bits { #define REQ_RW (1 << __REQ_RW) #define REQ_FAILFAST (1 << __REQ_FAILFAST) +#define REQ_SORTED (1 << __REQ_SORTED) #define REQ_SOFTBARRIER (1 << __REQ_SOFTBARRIER) #define REQ_HARDBARRIER (1 << __REQ_HARDBARRIER) #define REQ_CMD (1 << __REQ_CMD) @@ -332,6 +334,13 @@ struct request_queue prepare_flush_fn *prepare_flush_fn; end_flush_fn *end_flush_fn; + /* + * Dispatch queue sorting + */ + sector_t last_sector; + struct request *boundary_rq; + unsigned int max_back_kb; + /* * Auto-unplugging state */ @@ -454,6 +463,7 @@ enum { #define blk_pm_request(rq) \ ((rq)->flags & (REQ_PM_SUSPEND | REQ_PM_RESUME)) +#define blk_sorted_rq(rq) ((rq)->flags & REQ_SORTED) #define blk_barrier_rq(rq) ((rq)->flags & REQ_HARDBARRIER) #define blk_barrier_preflush(rq) ((rq)->flags & REQ_BAR_PREFLUSH) #define blk_barrier_postflush(rq) ((rq)->flags & REQ_BAR_POSTFLUSH) @@ -611,12 +621,7 @@ extern void end_request(struct request *req, int uptodate); static inline void blkdev_dequeue_request(struct request *req) { - BUG_ON(list_empty(&req->queuelist)); - - list_del_init(&req->queuelist); - - if (req->rl) - elv_remove_request(req->q, req); + elv_dequeue_request(req->q, req); } /* -- cgit v1.2.3 From 1b47f531e244e339b648dfff071c086f936e49e1 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 20 Oct 2005 16:37:00 +0200 Subject: [PATCH] generic dispatch fixes - Split elv_dispatch_insert() into two functions - Rename rq_last_sector() to rq_end_sector() Signed-off-by: Jens Axboe --- drivers/block/elevator.c | 33 ++++++++++++--------------------- include/linux/blkdev.h | 13 ++++++++++++- include/linux/elevator.h | 4 +++- 3 files changed, 27 insertions(+), 23 deletions(-) (limited to 'include/linux/blkdev.h') diff --git a/drivers/block/elevator.c b/drivers/block/elevator.c index a27555908d3..237e43d760c 100644 --- a/drivers/block/elevator.c +++ b/drivers/block/elevator.c @@ -40,11 +40,6 @@ static DEFINE_SPINLOCK(elv_list_lock); static LIST_HEAD(elv_list); -static inline sector_t rq_last_sector(struct request *rq) -{ - return rq->sector + rq->nr_sectors; -} - /* * can we safely merge with this request? */ @@ -148,7 +143,7 @@ static int elevator_attach(request_queue_t *q, struct elevator_type *e, INIT_LIST_HEAD(&q->queue_head); q->last_merge = NULL; q->elevator = eq; - q->last_sector = 0; + q->end_sector = 0; q->boundary_rq = NULL; q->max_back_kb = 0; @@ -233,29 +228,25 @@ void elevator_exit(elevator_t *e) kfree(e); } +void elv_dispatch_insert_tail(request_queue_t *q, struct request *rq) +{ +} + /* * Insert rq into dispatch queue of q. Queue lock must be held on * entry. If sort != 0, rq is sort-inserted; otherwise, rq will be * appended to the dispatch queue. To be used by specific elevators. */ -void elv_dispatch_insert(request_queue_t *q, struct request *rq, int sort) +void elv_dispatch_sort(request_queue_t *q, struct request *rq) { sector_t boundary; unsigned max_back; struct list_head *entry; - if (!sort) { - /* Specific elevator is performing sort. Step away. */ - q->last_sector = rq_last_sector(rq); - q->boundary_rq = rq; - list_add_tail(&rq->queuelist, &q->queue_head); - return; - } - - boundary = q->last_sector; + boundary = q->end_sector; max_back = q->max_back_kb * 2; boundary = boundary > max_back ? boundary - max_back : 0; - + list_for_each_prev(entry, &q->queue_head) { struct request *pos = list_entry_rq(entry); @@ -343,10 +334,10 @@ void __elv_add_request(request_queue_t *q, struct request *rq, int where, where = ELEVATOR_INSERT_BACK; /* - * this request is scheduling boundary, update last_sector + * this request is scheduling boundary, update end_sector */ if (blk_fs_request(rq)) { - q->last_sector = rq_last_sector(rq); + q->end_sector = rq_end_sector(rq); q->boundary_rq = rq; } } @@ -479,7 +470,7 @@ struct request *elv_next_request(request_queue_t *q) q->last_merge = NULL; if (!q->boundary_rq || q->boundary_rq == rq) { - q->last_sector = rq_last_sector(rq); + q->end_sector = rq_end_sector(rq); q->boundary_rq = NULL; } @@ -802,7 +793,7 @@ ssize_t elv_iosched_show(request_queue_t *q, char *name) return len; } -EXPORT_SYMBOL(elv_dispatch_insert); +EXPORT_SYMBOL(elv_dispatch_sort); EXPORT_SYMBOL(elv_add_request); EXPORT_SYMBOL(__elv_add_request); EXPORT_SYMBOL(elv_requeue_request); diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 2c7b9154927..a2e496f285a 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -337,7 +337,7 @@ struct request_queue /* * Dispatch queue sorting */ - sector_t last_sector; + sector_t end_sector; struct request *boundary_rq; unsigned int max_back_kb; @@ -624,6 +624,17 @@ static inline void blkdev_dequeue_request(struct request *req) elv_dequeue_request(req->q, req); } +/* + * This should be in elevator.h, but that requires pulling in rq and q + */ +static inline void elv_dispatch_add_tail(struct request_queue *q, + struct request *rq) +{ + q->end_sector = rq_end_sector(rq); + q->boundary_rq = rq; + list_add_tail(&rq->queuelist, &q->queue_head); +} + /* * Access functions for manipulating queue properties */ diff --git a/include/linux/elevator.h b/include/linux/elevator.h index 76f4f692074..f9b5ec38121 100644 --- a/include/linux/elevator.h +++ b/include/linux/elevator.h @@ -79,7 +79,7 @@ struct elevator_queue /* * block elevator interface */ -extern void elv_dispatch_insert(request_queue_t *, struct request *, int); +extern void elv_dispatch_sort(request_queue_t *, struct request *); extern void elv_add_request(request_queue_t *, struct request *, int, int); extern void __elv_add_request(request_queue_t *, struct request *, int, int); extern int elv_merge(request_queue_t *, struct request **, struct bio *); @@ -140,4 +140,6 @@ enum { ELV_MQUEUE_MUST, }; +#define rq_end_sector(rq) ((rq)->sector + (rq)->nr_sectors) + #endif -- cgit v1.2.3 From 06b86245c052963029bfd9020ca1f08ceb66f85a Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Thu, 20 Oct 2005 16:46:23 +0200 Subject: [PATCH] 03/05 move last_merge handlin into generic elevator code Currently, both generic elevator code and specific ioscheds participate in the management and usage of last_merge. This and the following patches move last_merge handling into generic elevator code. Signed-off-by: Tejun Heo Signed-off-by: Jens Axboe --- drivers/block/elevator.c | 43 ++++++++++++++++++------------------------- include/linux/blkdev.h | 3 +++ 2 files changed, 21 insertions(+), 25 deletions(-) (limited to 'include/linux/blkdev.h') diff --git a/drivers/block/elevator.c b/drivers/block/elevator.c index 29d6c8237ab..415144372c7 100644 --- a/drivers/block/elevator.c +++ b/drivers/block/elevator.c @@ -83,15 +83,6 @@ inline int elv_try_merge(struct request *__rq, struct bio *bio) } EXPORT_SYMBOL(elv_try_merge); -inline int elv_try_last_merge(request_queue_t *q, struct bio *bio) -{ - if (q->last_merge) - return elv_try_merge(q->last_merge, bio); - - return ELEVATOR_NO_MERGE; -} -EXPORT_SYMBOL(elv_try_last_merge); - static struct elevator_type *elevator_find(const char *name) { struct elevator_type *e = NULL; @@ -239,6 +230,9 @@ void elv_dispatch_sort(request_queue_t *q, struct request *rq) unsigned max_back; struct list_head *entry; + if (q->last_merge == rq) + q->last_merge = NULL; + boundary = q->end_sector; max_back = q->max_back_kb * 2; boundary = boundary > max_back ? boundary - max_back : 0; @@ -265,6 +259,15 @@ void elv_dispatch_sort(request_queue_t *q, struct request *rq) int elv_merge(request_queue_t *q, struct request **req, struct bio *bio) { elevator_t *e = q->elevator; + int ret; + + if (q->last_merge) { + ret = elv_try_merge(q->last_merge, bio); + if (ret != ELEVATOR_NO_MERGE) { + *req = q->last_merge; + return ret; + } + } if (e->ops->elevator_merge_fn) return e->ops->elevator_merge_fn(q, req, bio); @@ -278,6 +281,8 @@ void elv_merged_request(request_queue_t *q, struct request *rq) if (e->ops->elevator_merged_fn) e->ops->elevator_merged_fn(q, rq); + + q->last_merge = rq; } void elv_merge_requests(request_queue_t *q, struct request *rq, @@ -285,11 +290,10 @@ void elv_merge_requests(request_queue_t *q, struct request *rq, { elevator_t *e = q->elevator; - if (q->last_merge == next) - q->last_merge = NULL; - if (e->ops->elevator_merge_req_fn) e->ops->elevator_merge_req_fn(q, rq, next); + + q->last_merge = rq; } void elv_requeue_request(request_queue_t *q, struct request *rq) @@ -384,6 +388,8 @@ void __elv_add_request(request_queue_t *q, struct request *rq, int where, BUG_ON(!blk_fs_request(rq)); rq->flags |= REQ_SORTED; q->elevator->ops->elevator_add_req_fn(q, rq); + if (q->last_merge == NULL && rq_mergeable(rq)) + q->last_merge = rq; break; default: @@ -462,9 +468,6 @@ struct request *elv_next_request(request_queue_t *q) rq->flags |= REQ_STARTED; } - if (rq == q->last_merge) - q->last_merge = NULL; - if (!q->boundary_rq || q->boundary_rq == rq) { q->end_sector = rq_end_sector(rq); q->boundary_rq = NULL; @@ -518,16 +521,6 @@ void elv_dequeue_request(request_queue_t *q, struct request *rq) */ if (blk_account_rq(rq)) q->in_flight++; - - /* - * the main clearing point for q->last_merge is on retrieval of - * request by driver (it calls elv_next_request()), but it _can_ - * also happen here if a request is added to the queue but later - * deleted without ever being given to driver (merged with another - * request). - */ - if (rq == q->last_merge) - q->last_merge = NULL; } int elv_queue_empty(request_queue_t *q) diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index a2e496f285a..bd55b1df899 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -630,6 +630,9 @@ static inline void blkdev_dequeue_request(struct request *req) static inline void elv_dispatch_add_tail(struct request_queue *q, struct request *rq) { + if (q->last_merge == rq) + q->last_merge = NULL; + q->end_sector = rq_end_sector(rq); q->boundary_rq = rq; list_add_tail(&rq->queuelist, &q->queue_head); -- cgit v1.2.3 From cb19833dccb32f97cacbfff834b53523915f13f6 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 24 Oct 2005 08:35:58 +0200 Subject: [BLOCK] kill generic max_back_kb handling This patch kills max_back_kb handling from elv_dispatch_sort() and kills max_back_kb field from struct request_queue. Signed-off-by: Tejun Heo Signed-off-by: Jens Axboe --- drivers/block/elevator.c | 6 +----- include/linux/blkdev.h | 1 - 2 files changed, 1 insertion(+), 6 deletions(-) (limited to 'include/linux/blkdev.h') diff --git a/drivers/block/elevator.c b/drivers/block/elevator.c index 415144372c7..af2388e73f6 100644 --- a/drivers/block/elevator.c +++ b/drivers/block/elevator.c @@ -136,7 +136,6 @@ static int elevator_attach(request_queue_t *q, struct elevator_type *e, q->elevator = eq; q->end_sector = 0; q->boundary_rq = NULL; - q->max_back_kb = 0; if (eq->ops->elevator_init_fn) ret = eq->ops->elevator_init_fn(q, eq); @@ -227,16 +226,13 @@ void elevator_exit(elevator_t *e) void elv_dispatch_sort(request_queue_t *q, struct request *rq) { sector_t boundary; - unsigned max_back; struct list_head *entry; if (q->last_merge == rq) q->last_merge = NULL; boundary = q->end_sector; - max_back = q->max_back_kb * 2; - boundary = boundary > max_back ? boundary - max_back : 0; - + list_for_each_prev(entry, &q->queue_head) { struct request *pos = list_entry_rq(entry); diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index bd55b1df899..159dbcd2eb5 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -339,7 +339,6 @@ struct request_queue */ sector_t end_sector; struct request *boundary_rq; - unsigned int max_back_kb; /* * Auto-unplugging state -- cgit v1.2.3