diff options
Diffstat (limited to 'block/blk-mq.c')
-rw-r--r-- | block/blk-mq.c | 54 |
1 files changed, 33 insertions, 21 deletions
diff --git a/block/blk-mq.c b/block/blk-mq.c index bf90684..5d4ce7e 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -31,6 +31,7 @@ #include <linux/blk-mq.h> #include "blk.h" #include "blk-mq.h" +#include "blk-mq-debugfs.h" #include "blk-mq-tag.h" #include "blk-stat.h" #include "blk-wbt.h" @@ -41,6 +42,7 @@ static LIST_HEAD(all_q_list); static void blk_mq_poll_stats_start(struct request_queue *q); static void blk_mq_poll_stats_fn(struct blk_stat_callback *cb); +static void __blk_mq_stop_hw_queues(struct request_queue *q, bool sync); static int blk_mq_poll_stats_bkt(const struct request *rq) { @@ -166,7 +168,7 @@ void blk_mq_quiesce_queue(struct request_queue *q) unsigned int i; bool rcu = false; - blk_mq_stop_hw_queues(q); + __blk_mq_stop_hw_queues(q, true); queue_for_each_hw_ctx(q, hctx, i) { if (hctx->flags & BLK_MQ_F_BLOCKING) @@ -1218,20 +1220,34 @@ bool blk_mq_queue_stopped(struct request_queue *q) } EXPORT_SYMBOL(blk_mq_queue_stopped); -void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx) +static void __blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx, bool sync) { - cancel_delayed_work_sync(&hctx->run_work); + if (sync) + cancel_delayed_work_sync(&hctx->run_work); + else + cancel_delayed_work(&hctx->run_work); + set_bit(BLK_MQ_S_STOPPED, &hctx->state); } + +void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx) +{ + __blk_mq_stop_hw_queue(hctx, false); +} EXPORT_SYMBOL(blk_mq_stop_hw_queue); -void blk_mq_stop_hw_queues(struct request_queue *q) +void __blk_mq_stop_hw_queues(struct request_queue *q, bool sync) { struct blk_mq_hw_ctx *hctx; int i; queue_for_each_hw_ctx(q, hctx, i) - blk_mq_stop_hw_queue(hctx); + __blk_mq_stop_hw_queue(hctx, sync); +} + +void blk_mq_stop_hw_queues(struct request_queue *q) +{ + __blk_mq_stop_hw_queues(q, false); } EXPORT_SYMBOL(blk_mq_stop_hw_queues); @@ -1655,8 +1671,7 @@ void blk_mq_free_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags, if (!rq) continue; - set->ops->exit_request(set->driver_data, rq, - hctx_idx, i); + set->ops->exit_request(set, rq, hctx_idx); tags->static_rqs[i] = NULL; } } @@ -1787,8 +1802,7 @@ int blk_mq_alloc_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags, tags->static_rqs[i] = rq; if (set->ops->init_request) { - if (set->ops->init_request(set->driver_data, - rq, hctx_idx, i, + if (set->ops->init_request(set, rq, hctx_idx, node)) { tags->static_rqs[i] = NULL; goto fail; @@ -1849,14 +1863,12 @@ static void blk_mq_exit_hctx(struct request_queue *q, struct blk_mq_tag_set *set, struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx) { - unsigned flush_start_tag = set->queue_depth; + blk_mq_debugfs_unregister_hctx(hctx); blk_mq_tag_idle(hctx); if (set->ops->exit_request) - set->ops->exit_request(set->driver_data, - hctx->fq->flush_rq, hctx_idx, - flush_start_tag + hctx_idx); + set->ops->exit_request(set, hctx->fq->flush_rq, hctx_idx); blk_mq_sched_exit_hctx(q, hctx, hctx_idx); @@ -1889,7 +1901,6 @@ static int blk_mq_init_hctx(struct request_queue *q, struct blk_mq_hw_ctx *hctx, unsigned hctx_idx) { int node; - unsigned flush_start_tag = set->queue_depth; node = hctx->numa_node; if (node == NUMA_NO_NODE) @@ -1933,14 +1944,15 @@ static int blk_mq_init_hctx(struct request_queue *q, goto sched_exit_hctx; if (set->ops->init_request && - set->ops->init_request(set->driver_data, - hctx->fq->flush_rq, hctx_idx, - flush_start_tag + hctx_idx, node)) + set->ops->init_request(set, hctx->fq->flush_rq, hctx_idx, + node)) goto free_fq; if (hctx->flags & BLK_MQ_F_BLOCKING) init_srcu_struct(&hctx->queue_rq_srcu); + blk_mq_debugfs_register_hctx(q, hctx); + return 0; free_fq: @@ -2329,15 +2341,15 @@ struct request_queue *blk_mq_init_allocated_queue(struct blk_mq_tag_set *set, blk_mq_init_cpu_queues(q, set->nr_hw_queues); - get_online_cpus(); mutex_lock(&all_q_mutex); + get_online_cpus(); list_add_tail(&q->all_q_node, &all_q_list); blk_mq_add_queue_tag_set(set, q); blk_mq_map_swqueue(q, cpu_online_mask); - mutex_unlock(&all_q_mutex); put_online_cpus(); + mutex_unlock(&all_q_mutex); if (!(set->flags & BLK_MQ_F_NO_SCHED)) { int ret; @@ -2378,6 +2390,7 @@ static void blk_mq_queue_reinit(struct request_queue *q, { WARN_ON_ONCE(!atomic_read(&q->mq_freeze_depth)); + blk_mq_debugfs_unregister_hctxs(q); blk_mq_sysfs_unregister(q); /* @@ -2389,6 +2402,7 @@ static void blk_mq_queue_reinit(struct request_queue *q, blk_mq_map_swqueue(q, online_mask); blk_mq_sysfs_register(q); + blk_mq_debugfs_register_hctxs(q); } /* @@ -2617,7 +2631,6 @@ int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr) return -EINVAL; blk_mq_freeze_queue(q); - blk_mq_quiesce_queue(q); ret = 0; queue_for_each_hw_ctx(q, hctx, i) { @@ -2643,7 +2656,6 @@ int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr) q->nr_requests = nr; blk_mq_unfreeze_queue(q); - blk_mq_start_stopped_hw_queues(q, true); return ret; } |