sieni
/
linux
mirror of git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

ublk: implement batch request completion via blk_mq_end_request_batch() 

Reduce overhead when completing multiple requests in batch I/O mode by
accumulating them in an io_comp_batch structure and completing them
together via blk_mq_end_request_batch(). This minimizes per-request
completion overhead and improves performance for high IOPS workloads.

The implementation adds an io_comp_batch pointer to struct ublk_io and
initializes it in __ublk_fetch(). For batch I/O, the pointer is set to
the batch structure in ublk_batch_commit_io(). The __ublk_complete_rq()
function uses io->iob to call blk_mq_add_to_batch() for batch mode.
After processing all batch I/Os, the completion callback is invoked in
ublk_handle_batch_commit_cmd() to complete all accumulated requests
efficiently.

So far just covers direct completion. For deferred completion(zero copy,
auto buffer reg), ublk_io_release() is often delayed in freeing buffer
consumer io_uring request's code path, so this patch often doesn't work,
also it is hard to pass the per-task 'struct io_comp_batch' for deferred
completion.

Reviewed-by: Caleb Sander Mateos <csander@purestorage.com>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Ming Lei 2026-01-16 22:18:46 +08:00 committed by Jens Axboe
parent 4d8fd7c559
commit 7aa78d4a3c
1 changed files with 17 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
drivers/block/ublk_drv.c
View File

@ -136,6 +136,7 @@ struct ublk_batch_io_data {
struct io_uring_cmd *cmd;
struct ublk_batch_io header;
unsigned int issue_flags;
struct io_comp_batch *iob;
};
/*
@ -691,7 +692,7 @@ static blk_status_t ublk_setup_iod_zoned(struct ublk_queue *ubq,
#endif
static inline void __ublk_complete_rq(struct request *req, struct ublk_io *io,
bool need_map);
bool need_map, struct io_comp_batch *iob);
static dev_t ublk_chr_devt;
static const struct class ublk_chr_class = {
@ -1001,7 +1002,7 @@ static inline void ublk_put_req_ref(struct ublk_io *io, struct request *req)
return;
/* ublk_need_map_io() and ublk_need_req_ref() are mutually exclusive */
__ublk_complete_rq(req, io, false);
__ublk_complete_rq(req, io, false, NULL);
}
static inline bool ublk_sub_req_ref(struct ublk_io *io)
@ -1388,7 +1389,7 @@ static void ublk_end_request(struct request *req, blk_status_t error)
/* todo: handle partial completion */
static inline void __ublk_complete_rq(struct request *req, struct ublk_io *io,
bool need_map)
bool need_map, struct io_comp_batch *iob)
{
unsigned int unmapped_bytes;
blk_status_t res = BLK_STS_OK;
@ -1442,8 +1443,11 @@ static inline void __ublk_complete_rq(struct request *req, struct ublk_io *io,
local_bh_enable();
if (requeue)
blk_mq_requeue_request(req, true);
else if (likely(!blk_should_fake_timeout(req->q)))
else if (likely(!blk_should_fake_timeout(req->q))) {
if (blk_mq_add_to_batch(req, iob, false, blk_mq_end_request_batch))
return;
__blk_mq_end_request(req, BLK_STS_OK);
}
return;
exit:
@ -2478,7 +2482,7 @@ static void __ublk_fail_req(struct ublk_device *ub, struct ublk_io *io,
blk_mq_requeue_request(req, false);
else {
io->res = -EIO;
__ublk_complete_rq(req, io, ublk_dev_need_map_io(ub));
__ublk_complete_rq(req, io, ublk_dev_need_map_io(ub), NULL);
}
}
@ -3214,7 +3218,7 @@ static int ublk_ch_uring_cmd_local(struct io_uring_cmd *cmd,
if (req_op(req) == REQ_OP_ZONE_APPEND)
req->__sector = addr;
if (compl)
__ublk_complete_rq(req, io, ublk_dev_need_map_io(ub));
__ublk_complete_rq(req, io, ublk_dev_need_map_io(ub), NULL);
if (ret)
goto out;
@ -3533,11 +3537,11 @@ static int ublk_batch_commit_io(struct ublk_queue *ubq,
if (req_op(req) == REQ_OP_ZONE_APPEND)
req->__sector = ublk_batch_zone_lba(uc, elem);
if (compl)
__ublk_complete_rq(req, io, ublk_dev_need_map_io(data->ub));
__ublk_complete_rq(req, io, ublk_dev_need_map_io(data->ub), data->iob);
return 0;
}
static int ublk_handle_batch_commit_cmd(const struct ublk_batch_io_data *data)
static int ublk_handle_batch_commit_cmd(struct ublk_batch_io_data *data)
{
const struct ublk_batch_io *uc = &data->header;
struct io_uring_cmd *cmd = data->cmd;
@ -3546,10 +3550,15 @@ static int ublk_handle_batch_commit_cmd(const struct ublk_batch_io_data *data)
.total = uc->nr_elem * uc->elem_bytes,
.elem_bytes = uc->elem_bytes,
};
DEFINE_IO_COMP_BATCH(iob);
int ret;
data->iob = &iob;
ret = ublk_walk_cmd_buf(&iter, data, ublk_batch_commit_io);
if (iob.complete)
iob.complete(&iob);
return iter.done == 0 ? ret : iter.done;
}