summaryrefslogtreecommitdiffstats
path: root/drivers/iommu/amd_iommu.c
diff options
context:
space:
mode:
authorTom Lendacky <thomas.lendacky@amd.com>2017-06-05 14:52:12 -0500
committerJoerg Roedel <jroedel@suse.de>2017-06-08 14:31:03 +0200
commitd334a5637dfb53f7d07017afc1e491903b482ef8 (patch)
tree9f53f94c1126dc3a3e4011a9ae6a04f441cc5c81 /drivers/iommu/amd_iommu.c
parente2f9d45fb452c60c9aaab55ce6f40601f9ec6516 (diff)
downloadop-kernel-dev-d334a5637dfb53f7d07017afc1e491903b482ef8.zip
op-kernel-dev-d334a5637dfb53f7d07017afc1e491903b482ef8.tar.gz
iommu/amd: Reduce amount of MMIO when submitting commands
As newer, higher speed devices are developed, perf data shows that the amount of MMIO that is performed when submitting commands to the IOMMU causes performance issues. Currently, the command submission path reads the command buffer head and tail pointers and then writes the tail pointer once the command is ready. The tail pointer is only ever updated by the driver so it can be tracked by the driver without having to read it from the hardware. The head pointer is updated by the hardware, but can be read opportunistically. Reading the head pointer only when it appears that there might not be room in the command buffer and then re-checking the available space reduces the number of times the head pointer has to be read. Signed-off-by: Tom Lendacky <thomas.lendacky@amd.com> Signed-off-by: Joerg Roedel <jroedel@suse.de>
Diffstat (limited to 'drivers/iommu/amd_iommu.c')
-rw-r--r--drivers/iommu/amd_iommu.c35
1 files changed, 22 insertions, 13 deletions
diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c
index d7748955..d81c895 100644
--- a/drivers/iommu/amd_iommu.c
+++ b/drivers/iommu/amd_iommu.c
@@ -874,19 +874,20 @@ static int wait_on_sem(volatile u64 *sem)
}
static void copy_cmd_to_buffer(struct amd_iommu *iommu,
- struct iommu_cmd *cmd,
- u32 tail)
+ struct iommu_cmd *cmd)
{
u8 *target;
- target = iommu->cmd_buf + tail;
- tail = (tail + sizeof(*cmd)) % CMD_BUFFER_SIZE;
+ target = iommu->cmd_buf + iommu->cmd_buf_tail;
+
+ iommu->cmd_buf_tail += sizeof(*cmd);
+ iommu->cmd_buf_tail %= CMD_BUFFER_SIZE;
/* Copy command to buffer */
memcpy(target, cmd, sizeof(*cmd));
/* Tell the IOMMU about it */
- writel(tail, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
+ writel(iommu->cmd_buf_tail, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
}
static void build_completion_wait(struct iommu_cmd *cmd, u64 address)
@@ -1044,23 +1045,31 @@ static int __iommu_queue_command_sync(struct amd_iommu *iommu,
struct iommu_cmd *cmd,
bool sync)
{
- u32 left, tail, head, next_tail;
+ bool read_head = true;
+ u32 left, next_tail;
+ next_tail = (iommu->cmd_buf_tail + sizeof(*cmd)) % CMD_BUFFER_SIZE;
again:
-
- head = readl(iommu->mmio_base + MMIO_CMD_HEAD_OFFSET);
- tail = readl(iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
- next_tail = (tail + sizeof(*cmd)) % CMD_BUFFER_SIZE;
- left = (head - next_tail) % CMD_BUFFER_SIZE;
+ left = (iommu->cmd_buf_head - next_tail) % CMD_BUFFER_SIZE;
if (left <= 0x20) {
struct iommu_cmd sync_cmd;
int ret;
+ if (read_head) {
+ /* Update head and recheck remaining space */
+ iommu->cmd_buf_head = readl(iommu->mmio_base +
+ MMIO_CMD_HEAD_OFFSET);
+ read_head = false;
+ goto again;
+ }
+
+ read_head = true;
+
iommu->cmd_sem = 0;
build_completion_wait(&sync_cmd, (u64)&iommu->cmd_sem);
- copy_cmd_to_buffer(iommu, &sync_cmd, tail);
+ copy_cmd_to_buffer(iommu, &sync_cmd);
if ((ret = wait_on_sem(&iommu->cmd_sem)) != 0)
return ret;
@@ -1068,7 +1077,7 @@ again:
goto again;
}
- copy_cmd_to_buffer(iommu, cmd, tail);
+ copy_cmd_to_buffer(iommu, cmd);
/* We need to sync now to make sure all commands are processed */
iommu->need_sync = sync;
OpenPOWER on IntegriCloud