diff options
Diffstat (limited to 'block')
| -rw-r--r-- | block/blk-exec.c | 2 | ||||
| -rw-r--r-- | block/blk-flush.c | 16 | ||||
| -rw-r--r-- | block/blk-lib.c | 82 | ||||
| -rw-r--r-- | block/blk-settings.c | 9 | ||||
| -rw-r--r-- | block/blk-sysfs.c | 3 | ||||
| -rw-r--r-- | block/blk.h | 21 | ||||
| -rw-r--r-- | block/elevator.c | 7 |
7 files changed, 69 insertions, 71 deletions
diff --git a/block/blk-exec.c b/block/blk-exec.c index 81e3181..8a0e7ec 100644 --- a/block/blk-exec.c +++ b/block/blk-exec.c @@ -56,7 +56,7 @@ void blk_execute_rq_nowait(struct request_queue *q, struct gendisk *bd_disk, spin_lock_irq(q->queue_lock); __elv_add_request(q, rq, where); __blk_run_queue(q); - /* the queue is stopped so it won't be plugged+unplugged */ + /* the queue is stopped so it won't be run */ if (rq->cmd_type == REQ_TYPE_PM_RESUME) q->request_fn(q); spin_unlock_irq(q->queue_lock); diff --git a/block/blk-flush.c b/block/blk-flush.c index 6c9b5e1..bb21e4c 100644 --- a/block/blk-flush.c +++ b/block/blk-flush.c @@ -212,13 +212,19 @@ static void flush_end_io(struct request *flush_rq, int error) } /* - * Moving a request silently to empty queue_head may stall the - * queue. Kick the queue in those cases. This function is called - * from request completion path and calling directly into - * request_fn may confuse the driver. Always use kblockd. + * Kick the queue to avoid stall for two cases: + * 1. Moving a request silently to empty queue_head may stall the + * queue. + * 2. When flush request is running in non-queueable queue, the + * queue is hold. Restart the queue after flush request is finished + * to avoid stall. + * This function is called from request completion path and calling + * directly into request_fn may confuse the driver. Always use + * kblockd. */ - if (queued) + if (queued || q->flush_queue_delayed) blk_run_queue_async(q); + q->flush_queue_delayed = 0; } /** diff --git a/block/blk-lib.c b/block/blk-lib.c index 25de73e..78e627e 100644 --- a/block/blk-lib.c +++ b/block/blk-lib.c @@ -9,17 +9,20 @@ #include "blk.h" -static void blkdev_discard_end_io(struct bio *bio, int err) -{ - if (err) { - if (err == -EOPNOTSUPP) - set_bit(BIO_EOPNOTSUPP, &bio->bi_flags); - clear_bit(BIO_UPTODATE, &bio->bi_flags); - } +struct bio_batch { + atomic_t done; + unsigned long flags; + struct completion *wait; +}; - if (bio->bi_private) - complete(bio->bi_private); +static void bio_batch_end_io(struct bio *bio, int err) +{ + struct bio_batch *bb = bio->bi_private; + if (err && (err != -EOPNOTSUPP)) + clear_bit(BIO_UPTODATE, &bb->flags); + if (atomic_dec_and_test(&bb->done)) + complete(bb->wait); bio_put(bio); } @@ -41,6 +44,7 @@ int blkdev_issue_discard(struct block_device *bdev, sector_t sector, struct request_queue *q = bdev_get_queue(bdev); int type = REQ_WRITE | REQ_DISCARD; unsigned int max_discard_sectors; + struct bio_batch bb; struct bio *bio; int ret = 0; @@ -67,7 +71,11 @@ int blkdev_issue_discard(struct block_device *bdev, sector_t sector, type |= REQ_SECURE; } - while (nr_sects && !ret) { + atomic_set(&bb.done, 1); + bb.flags = 1 << BIO_UPTODATE; + bb.wait = &wait; + + while (nr_sects) { bio = bio_alloc(gfp_mask, 1); if (!bio) { ret = -ENOMEM; @@ -75,9 +83,9 @@ int blkdev_issue_discard(struct block_device *bdev, sector_t sector, } bio->bi_sector = sector; - bio->bi_end_io = blkdev_discard_end_io; + bio->bi_end_io = bio_batch_end_io; bio->bi_bdev = bdev; - bio->bi_private = &wait; + bio->bi_private = &bb; if (nr_sects > max_discard_sectors) { bio->bi_size = max_discard_sectors << 9; @@ -88,45 +96,21 @@ int blkdev_issue_discard(struct block_device *bdev, sector_t sector, nr_sects = 0; } - bio_get(bio); + atomic_inc(&bb.done); submit_bio(type, bio); + } + /* Wait for bios in-flight */ + if (!atomic_dec_and_test(&bb.done)) wait_for_completion(&wait); - if (bio_flagged(bio, BIO_EOPNOTSUPP)) - ret = -EOPNOTSUPP; - else if (!bio_flagged(bio, BIO_UPTODATE)) - ret = -EIO; - bio_put(bio); - } + if (!test_bit(BIO_UPTODATE, &bb.flags)) + ret = -EIO; return ret; } EXPORT_SYMBOL(blkdev_issue_discard); -struct bio_batch -{ - atomic_t done; - unsigned long flags; - struct completion *wait; -}; - -static void bio_batch_end_io(struct bio *bio, int err) -{ - struct bio_batch *bb = bio->bi_private; - - if (err) { - if (err == -EOPNOTSUPP) - set_bit(BIO_EOPNOTSUPP, &bb->flags); - else - clear_bit(BIO_UPTODATE, &bb->flags); - } - if (bb) - if (atomic_dec_and_test(&bb->done)) - complete(bb->wait); - bio_put(bio); -} - /** * blkdev_issue_zeroout - generate number of zero filed write bios * @bdev: blockdev to issue @@ -151,7 +135,6 @@ int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector, bb.flags = 1 << BIO_UPTODATE; bb.wait = &wait; -submit: ret = 0; while (nr_sects != 0) { bio = bio_alloc(gfp_mask, @@ -168,9 +151,6 @@ submit: while (nr_sects != 0) { sz = min((sector_t) PAGE_SIZE >> 9 , nr_sects); - if (sz == 0) - /* bio has maximum size possible */ - break; ret = bio_add_page(bio, ZERO_PAGE(0), sz << 9, 0); nr_sects -= ret >> 9; sector += ret >> 9; @@ -190,16 +170,6 @@ submit: /* One of bios in the batch was completed with error.*/ ret = -EIO; - if (ret) - goto out; - - if (test_bit(BIO_EOPNOTSUPP, &bb.flags)) { - ret = -EOPNOTSUPP; - goto out; - } - if (nr_sects != 0) - goto submit; -out: return ret; } EXPORT_SYMBOL(blkdev_issue_zeroout); diff --git a/block/blk-settings.c b/block/blk-settings.c index 1fa7692..fa1eb04 100644 --- a/block/blk-settings.c +++ b/block/blk-settings.c @@ -120,7 +120,7 @@ void blk_set_default_limits(struct queue_limits *lim) lim->discard_granularity = 0; lim->discard_alignment = 0; lim->discard_misaligned = 0; - lim->discard_zeroes_data = -1; + lim->discard_zeroes_data = 1; lim->logical_block_size = lim->physical_block_size = lim->io_min = 512; lim->bounce_pfn = (unsigned long)(BLK_BOUNCE_ANY >> PAGE_SHIFT); lim->alignment_offset = 0; @@ -166,6 +166,7 @@ void blk_queue_make_request(struct request_queue *q, make_request_fn *mfn) blk_set_default_limits(&q->limits); blk_queue_max_hw_sectors(q, BLK_SAFE_MAX_SECTORS); + q->limits.discard_zeroes_data = 0; /* * by default assume old behaviour and bounce for any highmem page @@ -790,6 +791,12 @@ void blk_queue_flush(struct request_queue *q, unsigned int flush) } EXPORT_SYMBOL_GPL(blk_queue_flush); +void blk_queue_flush_queueable(struct request_queue *q, bool queueable) +{ + q->flush_not_queueable = !queueable; +} +EXPORT_SYMBOL_GPL(blk_queue_flush_queueable); + static int __init blk_settings_init(void) { blk_max_low_pfn = max_low_pfn - 1; diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c index bd23631..d935bd8 100644 --- a/block/blk-sysfs.c +++ b/block/blk-sysfs.c @@ -152,7 +152,8 @@ static ssize_t queue_discard_granularity_show(struct request_queue *q, char *pag static ssize_t queue_discard_max_show(struct request_queue *q, char *page) { - return queue_var_show(q->limits.max_discard_sectors << 9, page); + return sprintf(page, "%llu\n", + (unsigned long long)q->limits.max_discard_sectors << 9); } static ssize_t queue_discard_zeroes_data_show(struct request_queue *q, char *page) diff --git a/block/blk.h b/block/blk.h index 6126346..1f798b5 100644 --- a/block/blk.h +++ b/block/blk.h @@ -61,7 +61,26 @@ static inline struct request *__elv_next_request(struct request_queue *q) rq = list_entry_rq(q->queue_head.next); return rq; } - + /* + * Flush request is running and flush request isn't queueable + * in the drive, we can hold the queue till flush request is + * finished. Even we don't do this, driver can't dispatch next + * requests and will requeue them. And this can improve + * throughput too. For example, we have request flush1, write1, + * flush 2. flush1 is dispatched, then queue is hold, write1 + * isn't inserted to queue. After flush1 is finished, flush2 + * will be dispatched. Since disk cache is already clean, + * flush2 will be finished very soon, so looks like flush2 is + * folded to flush1. + * Since the queue is hold, a flag is set to indicate the queue + * should be restarted later. Please see flush_end_io() for + * details. + */ + if (q->flush_pending_idx != q->flush_running_idx && + !queue_flush_queueable(q)) { + q->flush_queue_delayed = 1; + return NULL; + } if (!q->elevator->ops->elevator_dispatch_fn(q, 0)) return NULL; } diff --git a/block/elevator.c b/block/elevator.c index 45ca1e3..2a0b653 100644 --- a/block/elevator.c +++ b/block/elevator.c @@ -155,13 +155,8 @@ static struct elevator_type *elevator_get(const char *name) e = elevator_find(name); if (!e) { - char elv[ELV_NAME_MAX + strlen("-iosched")]; - spin_unlock(&elv_list_lock); - - snprintf(elv, sizeof(elv), "%s-iosched", name); - - request_module("%s", elv); + request_module("%s-iosched", name); spin_lock(&elv_list_lock); e = elevator_find(name); } |
