From b973cb7e89fe3dcc2bc72c5b3aa7a3bfd9d0e6d5 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 2 Feb 2017 08:54:40 -0700 Subject: blk-merge: return the merged request When we attempt to merge request-to-request, we return a 0/1 if we ended up merging or not. Change that to return the pointer to the request that we freed. We will use this to move the freeing of that request out of the merge logic, so that callers can drop locks before freeing the request. There should be no functional changes in this patch. Signed-off-by: Jens Axboe Reviewed-by: Omar Sandoval --- block/blk-merge.c | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) (limited to 'block/blk-merge.c') diff --git a/block/blk-merge.c b/block/blk-merge.c index 6aa43de..3826fc3 100644 --- a/block/blk-merge.c +++ b/block/blk-merge.c @@ -659,31 +659,32 @@ static void blk_account_io_merge(struct request *req) } /* - * Has to be called with the request spinlock acquired + * For non-mq, this has to be called with the request spinlock acquired. + * For mq with scheduling, the appropriate queue wide lock should be held. */ -static int attempt_merge(struct request_queue *q, struct request *req, - struct request *next) +static struct request *attempt_merge(struct request_queue *q, + struct request *req, struct request *next) { if (!rq_mergeable(req) || !rq_mergeable(next)) - return 0; + return NULL; if (req_op(req) != req_op(next)) - return 0; + return NULL; /* * not contiguous */ if (blk_rq_pos(req) + blk_rq_sectors(req) != blk_rq_pos(next)) - return 0; + return NULL; if (rq_data_dir(req) != rq_data_dir(next) || req->rq_disk != next->rq_disk || req_no_special_merge(next)) - return 0; + return NULL; if (req_op(req) == REQ_OP_WRITE_SAME && !blk_write_same_mergeable(req->bio, next->bio)) - return 0; + return NULL; /* * If we are allowed to merge, then append bio list @@ -692,7 +693,7 @@ static int attempt_merge(struct request_queue *q, struct request *req, * counts here. */ if (!ll_merge_requests_fn(q, req, next)) - return 0; + return NULL; /* * If failfast settings disagree or any of the two is already @@ -735,27 +736,27 @@ static int attempt_merge(struct request_queue *q, struct request *req, /* owner-ship of bio passed from next to req */ next->bio = NULL; __blk_put_request(q, next); - return 1; + return next; } -int attempt_back_merge(struct request_queue *q, struct request *rq) +struct request *attempt_back_merge(struct request_queue *q, struct request *rq) { struct request *next = elv_latter_request(q, rq); if (next) return attempt_merge(q, rq, next); - return 0; + return NULL; } -int attempt_front_merge(struct request_queue *q, struct request *rq) +struct request *attempt_front_merge(struct request_queue *q, struct request *rq) { struct request *prev = elv_former_request(q, rq); if (prev) return attempt_merge(q, prev, rq); - return 0; + return NULL; } int blk_attempt_req_merge(struct request_queue *q, struct request *rq, @@ -767,7 +768,7 @@ int blk_attempt_req_merge(struct request_queue *q, struct request *rq, if (!e->type->ops.sq.elevator_allow_rq_merge_fn(q, rq, next)) return 0; - return attempt_merge(q, rq, next); + return attempt_merge(q, rq, next) != NULL; } bool blk_rq_merge_ok(struct request *rq, struct bio *bio) -- cgit v1.1 From e4d750c97794ea2bab793d4c518b1f4006364588 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Fri, 3 Feb 2017 09:48:28 -0700 Subject: block: free merged request in the caller If we end up doing a request-to-request merge when we have completed a bio-to-request merge, we free the request from deep down in that path. For blk-mq-sched, the merge path has to hold the appropriate lock, but we don't need it for freeing the request. And in fact holding the lock is problematic, since we are now calling the mq sched put_rq_private() hook with the lock held. Other call paths do not hold this lock. Fix this inconsistency by ensuring that the caller frees a merged request. Then we can do it outside of the lock, making it both more efficient and fixing the blk-mq-sched problem of invoking parts of the scheduler with an unknown lock state. Reported-by: Paolo Valente Signed-off-by: Jens Axboe Reviewed-by: Omar Sandoval --- block/blk-merge.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) (limited to 'block/blk-merge.c') diff --git a/block/blk-merge.c b/block/blk-merge.c index 3826fc3..a373416 100644 --- a/block/blk-merge.c +++ b/block/blk-merge.c @@ -733,9 +733,11 @@ static struct request *attempt_merge(struct request_queue *q, if (blk_rq_cpu_valid(next)) req->cpu = next->cpu; - /* owner-ship of bio passed from next to req */ + /* + * ownership of bio passed from next to req, return 'next' for + * the caller to free + */ next->bio = NULL; - __blk_put_request(q, next); return next; } @@ -763,12 +765,19 @@ int blk_attempt_req_merge(struct request_queue *q, struct request *rq, struct request *next) { struct elevator_queue *e = q->elevator; + struct request *free; if (!e->uses_mq && e->type->ops.sq.elevator_allow_rq_merge_fn) if (!e->type->ops.sq.elevator_allow_rq_merge_fn(q, rq, next)) return 0; - return attempt_merge(q, rq, next) != NULL; + free = attempt_merge(q, rq, next); + if (free) { + __blk_put_request(q, free); + return 1; + } + + return 0; } bool blk_rq_merge_ok(struct request *rq, struct bio *bio) -- cgit v1.1 From 6cf7677f1a94546e472658290b3b8bdbb16cc045 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 8 Feb 2017 14:46:47 +0100 Subject: block: move req_set_nomerge to blk.h This makes it available outside of blk-merge.c, and inlining such a trivial helper seems pretty useful to start with. Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- block/blk-merge.c | 7 ------- 1 file changed, 7 deletions(-) (limited to 'block/blk-merge.c') diff --git a/block/blk-merge.c b/block/blk-merge.c index a373416..c956d9e 100644 --- a/block/blk-merge.c +++ b/block/blk-merge.c @@ -482,13 +482,6 @@ int blk_rq_map_sg(struct request_queue *q, struct request *rq, } EXPORT_SYMBOL(blk_rq_map_sg); -static void req_set_nomerge(struct request_queue *q, struct request *req) -{ - req->cmd_flags |= REQ_NOMERGE; - if (req == q->last_merge) - q->last_merge = NULL; -} - static inline int ll_new_hw_segment(struct request_queue *q, struct request *req, struct bio *bio) -- cgit v1.1 From 34fe7c05400663e01e23cddd1fea68bb7a2b3d29 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 8 Feb 2017 14:46:48 +0100 Subject: block: enumify ELEVATOR_*_MERGE Switch these constants to an enum, and make let the compiler ensure that all callers of blk_try_merge and elv_merge handle all potential values. Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- block/blk-merge.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'block/blk-merge.c') diff --git a/block/blk-merge.c b/block/blk-merge.c index c956d9e..6cbd90a 100644 --- a/block/blk-merge.c +++ b/block/blk-merge.c @@ -801,7 +801,7 @@ bool blk_rq_merge_ok(struct request *rq, struct bio *bio) return true; } -int blk_try_merge(struct request *rq, struct bio *bio) +enum elv_merge blk_try_merge(struct request *rq, struct bio *bio) { if (blk_rq_pos(rq) + blk_rq_sectors(rq) == bio->bi_iter.bi_sector) return ELEVATOR_BACK_MERGE; -- cgit v1.1 From 1e739730c5b9ea80a2f25e9cf6e1025d47e3d8ed Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Wed, 8 Feb 2017 14:46:49 +0100 Subject: block: optionally merge discontiguous discard bios into a single request Add a new merge strategy that merges discard bios into a request until the maximum number of discard ranges (or the maximum discard size) is reached from the plug merging code. I/O scheduler merging is not wired up yet but might also be useful, although not for fast devices like NVMe which are the only user for now. Note that for now we don't support limiting the size of each discard range, but if needed that can be added later. Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- block/blk-merge.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'block/blk-merge.c') diff --git a/block/blk-merge.c b/block/blk-merge.c index 6cbd90a..2afa262 100644 --- a/block/blk-merge.c +++ b/block/blk-merge.c @@ -803,7 +803,10 @@ bool blk_rq_merge_ok(struct request *rq, struct bio *bio) enum elv_merge blk_try_merge(struct request *rq, struct bio *bio) { - if (blk_rq_pos(rq) + blk_rq_sectors(rq) == bio->bi_iter.bi_sector) + if (req_op(rq) == REQ_OP_DISCARD && + queue_max_discard_segments(rq->q) > 1) + return ELEVATOR_DISCARD_MERGE; + else if (blk_rq_pos(rq) + blk_rq_sectors(rq) == bio->bi_iter.bi_sector) return ELEVATOR_BACK_MERGE; else if (blk_rq_pos(rq) - bio_sectors(bio) == bio->bi_iter.bi_sector) return ELEVATOR_FRONT_MERGE; -- cgit v1.1