From 50c69de75b60f6e47dc9f2a2ee85c6f69648e489 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Wed, 16 Feb 2022 12:17:58 -0700 Subject: apple-nvme: defer cache flushes by a specified amount Cache flushes on the M1 nvme are really slow, taking 17-18 msec to complete. This can slow down workloads considerably, pure random writes end up being bound by the flush latency and hence run at 55-60 IOPS. Add a deferred flush work around to provide better performance, at a minimal risk. By default, flushes are delayed at most 1 second, but this is configurable. With this work-around, a pure random write workload runs at ~12K IOPS rather than 56 IOPS. Signed-off-by: Jens Axboe --- drivers/nvme/host/apple.c | 66 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index 745f4d7fb8ca..b181921460d0 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -197,8 +197,20 @@ struct apple_nvme { int irq; spinlock_t lock; + + /* + * Delayed cache flush handling state + */ + struct nvme_ns *flush_ns; + unsigned long flush_interval; + unsigned long last_flush; + struct delayed_work flush_dwork; }; +unsigned int flush_interval = 1000; +module_param(flush_interval, uint, 0644); +MODULE_PARM_DESC(flush_interval, "Grace period in msecs between flushes"); + static_assert(sizeof(struct nvme_command) == 64); static_assert(sizeof(struct apple_nvmmu_tcb) == 128); @@ -726,6 +738,24 @@ static int apple_nvme_remove_sq(struct apple_nvme *anv) return nvme_submit_sync_cmd(anv->ctrl.admin_q, &c, NULL, 0); } +static bool apple_nvme_delayed_flush(struct apple_nvme *anv, struct nvme_ns *ns, + struct request *req) +{ + if (!anv->flush_interval || req_op(req) != REQ_OP_FLUSH) + return false; + if (delayed_work_pending(&anv->flush_dwork)) + return true; + if (time_before(jiffies, anv->last_flush + anv->flush_interval)) { + kblockd_mod_delayed_work_on(WORK_CPU_UNBOUND, &anv->flush_dwork, + anv->flush_interval); + WARN_ON_ONCE(anv->flush_ns && anv->flush_ns != ns); + anv->flush_ns = ns; + return true; + } + anv->last_flush = jiffies; + return false; +} + static blk_status_t apple_nvme_queue_rq(struct blk_mq_hw_ctx *hctx, const struct blk_mq_queue_data *bd) { @@ -761,6 +791,12 @@ static blk_status_t apple_nvme_queue_rq(struct blk_mq_hw_ctx *hctx, } blk_mq_start_request(req); + + if (apple_nvme_delayed_flush(anv, ns, req)) { + blk_mq_complete_request(req); + return BLK_STS_OK; + } + apple_nvme_submit_cmd(q, cmnd); return BLK_STS_OK; @@ -1304,6 +1340,28 @@ static int apple_nvme_queue_alloc(struct apple_nvme *anv, return 0; } +static void apple_nvme_flush(struct work_struct *work) +{ + struct nvme_command c = { }; + struct apple_nvme *anv; + struct nvme_ns *ns; + int err; + + anv = container_of(work, struct apple_nvme, flush_dwork.work); + ns = anv->flush_ns; + if (WARN_ON_ONCE(!ns)) + return; + + c.common.opcode = nvme_cmd_flush; + c.common.nsid = cpu_to_le32(anv->flush_ns->head->ns_id); + err = nvme_submit_sync_cmd(ns->queue, &c, NULL, 0); + if (err) { + dev_err(anv->dev, "Deferred flush failed: %d\n", err); + } else { + anv->last_flush = jiffies; + } +} + static int apple_nvme_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -1406,6 +1464,13 @@ static int apple_nvme_probe(struct platform_device *pdev) if (IS_ERR(anv->ctrl.admin_q)) return -ENOMEM; + if (flush_interval) { + anv->flush_interval = msecs_to_jiffies(flush_interval); + anv->flush_ns = NULL; + anv->last_flush = jiffies - anv->flush_interval; + INIT_DELAYED_WORK(&anv->flush_dwork, apple_nvme_flush); + } + nvme_reset_ctrl(&anv->ctrl); async_schedule(apple_nvme_async_probe, anv); @@ -1433,6 +1498,7 @@ static void apple_nvme_shutdown(struct platform_device *pdev) { struct apple_nvme *anv = platform_get_drvdata(pdev); + cancel_delayed_work_sync(&anv->flush_dwork); apple_nvme_disable(anv, true); if (apple_rtkit_is_running(anv->rtk)) apple_rtkit_shutdown(anv->rtk); -- cgit v1.2.3