intel-iommu: add context-cache to cache context-entry

Add context-cache to cache context-entry encountered on a page-walk. Each
VTDAddressSpace has a member of VTDContextCacheEntry which represents an entry
in the context-cache. Since devices with different bus_num and devfn have their
respective VTDAddressSpace, this will be a good way to reference the cached
entries.
Each VTDContextCacheEntry will have a context_cache_gen and the cached entry
is valid only when context_cache_gen equals IntelIOMMUState.context_cache_gen.

Signed-off-by: Le Tan <tamlokveer@gmail.com>
Reviewed-by: Michael S. Tsirkin <mst@redhat.com>
Signed-off-by: Michael S. Tsirkin <mst@redhat.com>

4 files changed, 199 insertions, 35 deletions

diff --git a/hw/i386/intel_iommu.c b/hw/i386/intel_iommu.c
index 60dec4f..c514310 100644
--- a/hw/i386/intel_iommu.c
+++ b/hw/i386/intel_iommu.c
@@ -27,6 +27,7 @@
 #ifdef DEBUG_INTEL_IOMMU
 enum {
     DEBUG_GENERAL, DEBUG_CSR, DEBUG_INV, DEBUG_MMU, DEBUG_FLOG,
+    DEBUG_CACHE,
 };
 #define VTD_DBGBIT(x)   (1 << DEBUG_##x)
 static int vtd_dbgflags = VTD_DBGBIT(GENERAL) | VTD_DBGBIT(CSR);
@@ -131,6 +132,33 @@ static uint64_t vtd_set_clear_mask_quad(IntelIOMMUState *s, hwaddr addr,
     return new_val;
 }
 
+/* Reset all the gen of VTDAddressSpace to zero and set the gen of
+ * IntelIOMMUState to 1.
+ */
+static void vtd_reset_context_cache(IntelIOMMUState *s)
+{
+    VTDAddressSpace **pvtd_as;
+    VTDAddressSpace *vtd_as;
+    uint32_t bus_it;
+    uint32_t devfn_it;
+
+    VTD_DPRINTF(CACHE, "global context_cache_gen=1");
+    for (bus_it = 0; bus_it < VTD_PCI_BUS_MAX; ++bus_it) {
+        pvtd_as = s->address_spaces[bus_it];
+        if (!pvtd_as) {
+            continue;
+        }
+        for (devfn_it = 0; devfn_it < VTD_PCI_DEVFN_MAX; ++devfn_it) {
+            vtd_as = pvtd_as[devfn_it];
+            if (!vtd_as) {
+                continue;
+            }
+            vtd_as->context_cache_entry.context_cache_gen = 0;
+        }
+    }
+    s->context_cache_gen = 1;
+}
+
 /* Given the reg addr of both the message data and address, generate an
  * interrupt via MSI.
  */
@@ -651,11 +679,13 @@ static inline bool vtd_is_interrupt_addr(hwaddr addr)
  * @is_write: The access is a write operation
  * @entry: IOMMUTLBEntry that contain the addr to be translated and result
  */
-static void vtd_do_iommu_translate(IntelIOMMUState *s, uint8_t bus_num,
+static void vtd_do_iommu_translate(VTDAddressSpace *vtd_as, uint8_t bus_num,
                                    uint8_t devfn, hwaddr addr, bool is_write,
                                    IOMMUTLBEntry *entry)
 {
+    IntelIOMMUState *s = vtd_as->iommu_state;
     VTDContextEntry ce;
+    VTDContextCacheEntry *cc_entry = &vtd_as->context_cache_entry;
     uint64_t slpte;
     uint32_t level;
     uint16_t source_id = vtd_make_source_id(bus_num, devfn);
@@ -686,18 +716,35 @@ static void vtd_do_iommu_translate(IntelIOMMUState *s, uint8_t bus_num,
             return;
         }
     }
-
-    ret_fr = vtd_dev_to_context_entry(s, bus_num, devfn, &ce);
-    is_fpd_set = ce.lo & VTD_CONTEXT_ENTRY_FPD;
-    if (ret_fr) {
-        ret_fr = -ret_fr;
-        if (is_fpd_set && vtd_is_qualified_fault(ret_fr)) {
-            VTD_DPRINTF(FLOG, "fault processing is disabled for DMA requests "
-                        "through this context-entry (with FPD Set)");
-        } else {
-            vtd_report_dmar_fault(s, source_id, addr, ret_fr, is_write);
+    /* Try to fetch context-entry from cache first */
+    if (cc_entry->context_cache_gen == s->context_cache_gen) {
+        VTD_DPRINTF(CACHE, "hit context-cache bus %d devfn %d "
+                    "(hi %"PRIx64 " lo %"PRIx64 " gen %"PRIu32 ")",
+                    bus_num, devfn, cc_entry->context_entry.hi,
+                    cc_entry->context_entry.lo, cc_entry->context_cache_gen);
+        ce = cc_entry->context_entry;
+        is_fpd_set = ce.lo & VTD_CONTEXT_ENTRY_FPD;
+    } else {
+        ret_fr = vtd_dev_to_context_entry(s, bus_num, devfn, &ce);
+        is_fpd_set = ce.lo & VTD_CONTEXT_ENTRY_FPD;
+        if (ret_fr) {
+            ret_fr = -ret_fr;
+            if (is_fpd_set && vtd_is_qualified_fault(ret_fr)) {
+                VTD_DPRINTF(FLOG, "fault processing is disabled for DMA "
+                            "requests through this context-entry "
+                            "(with FPD Set)");
+            } else {
+                vtd_report_dmar_fault(s, source_id, addr, ret_fr, is_write);
+            }
+            return;
         }
-        return;
+        /* Update context-cache */
+        VTD_DPRINTF(CACHE, "update context-cache bus %d devfn %d "
+                    "(hi %"PRIx64 " lo %"PRIx64 " gen %"PRIu32 "->%"PRIu32 ")",
+                    bus_num, devfn, ce.hi, ce.lo,
+                    cc_entry->context_cache_gen, s->context_cache_gen);
+        cc_entry->context_entry = ce;
+        cc_entry->context_cache_gen = s->context_cache_gen;
     }
 
     ret_fr = vtd_gpa_to_slpte(&ce, addr, is_write, &slpte, &level,
@@ -729,6 +776,57 @@ static void vtd_root_table_setup(IntelIOMMUState *s)
                 (s->root_extended ? "(extended)" : ""));
 }
 
+static void vtd_context_global_invalidate(IntelIOMMUState *s)
+{
+    s->context_cache_gen++;
+    if (s->context_cache_gen == VTD_CONTEXT_CACHE_GEN_MAX) {
+        vtd_reset_context_cache(s);
+    }
+}
+
+/* Do a context-cache device-selective invalidation.
+ * @func_mask: FM field after shifting
+ */
+static void vtd_context_device_invalidate(IntelIOMMUState *s,
+                                          uint16_t source_id,
+                                          uint16_t func_mask)
+{
+    uint16_t mask;
+    VTDAddressSpace **pvtd_as;
+    VTDAddressSpace *vtd_as;
+    uint16_t devfn;
+    uint16_t devfn_it;
+
+    switch (func_mask & 3) {
+    case 0:
+        mask = 0;   /* No bits in the SID field masked */
+        break;
+    case 1:
+        mask = 4;   /* Mask bit 2 in the SID field */
+        break;
+    case 2:
+        mask = 6;   /* Mask bit 2:1 in the SID field */
+        break;
+    case 3:
+        mask = 7;   /* Mask bit 2:0 in the SID field */
+        break;
+    }
+    VTD_DPRINTF(INV, "device-selective invalidation source 0x%"PRIx16
+                    " mask %"PRIu16, source_id, mask);
+    pvtd_as = s->address_spaces[VTD_SID_TO_BUS(source_id)];
+    if (pvtd_as) {
+        devfn = VTD_SID_TO_DEVFN(source_id);
+        for (devfn_it = 0; devfn_it < VTD_PCI_DEVFN_MAX; ++devfn_it) {
+            vtd_as = pvtd_as[devfn_it];
+            if (vtd_as && ((devfn_it & mask) == (devfn & mask))) {
+                VTD_DPRINTF(INV, "invalidate context-cahce of devfn 0x%"PRIx16,
+                            devfn_it);
+                vtd_as->context_cache_entry.context_cache_gen = 0;
+            }
+        }
+    }
+}
+
 /* Context-cache invalidation
  * Returns the Context Actual Invalidation Granularity.
  * @val: the content of the CCMD_REG
@@ -739,24 +837,23 @@ static uint64_t vtd_context_cache_invalidate(IntelIOMMUState *s, uint64_t val)
     uint64_t type = val & VTD_CCMD_CIRG_MASK;
 
     switch (type) {
+    case VTD_CCMD_DOMAIN_INVL:
+        VTD_DPRINTF(INV, "domain-selective invalidation domain 0x%"PRIx16,
+                    (uint16_t)VTD_CCMD_DID(val));
+        /* Fall through */
     case VTD_CCMD_GLOBAL_INVL:
-        VTD_DPRINTF(INV, "Global invalidation request");
+        VTD_DPRINTF(INV, "global invalidation");
         caig = VTD_CCMD_GLOBAL_INVL_A;
-        break;
-
-    case VTD_CCMD_DOMAIN_INVL:
-        VTD_DPRINTF(INV, "Domain-selective invalidation request");
-        caig = VTD_CCMD_DOMAIN_INVL_A;
+        vtd_context_global_invalidate(s);
         break;
 
     case VTD_CCMD_DEVICE_INVL:
-        VTD_DPRINTF(INV, "Domain-selective invalidation request");
         caig = VTD_CCMD_DEVICE_INVL_A;
+        vtd_context_device_invalidate(s, VTD_CCMD_SID(val), VTD_CCMD_FM(val));
         break;
 
     default:
-        VTD_DPRINTF(GENERAL,
-                    "error: wrong context-cache invalidation granularity");
+        VTD_DPRINTF(GENERAL, "error: invalid granularity");
         caig = 0;
     }
     return caig;
@@ -994,6 +1091,38 @@ static bool vtd_process_wait_desc(IntelIOMMUState *s, VTDInvDesc *inv_desc)
     return true;
 }
 
+static bool vtd_process_context_cache_desc(IntelIOMMUState *s,
+                                           VTDInvDesc *inv_desc)
+{
+    if ((inv_desc->lo & VTD_INV_DESC_CC_RSVD) || inv_desc->hi) {
+        VTD_DPRINTF(GENERAL, "error: non-zero reserved field in Context-cache "
+                    "Invalidate Descriptor");
+        return false;
+    }
+    switch (inv_desc->lo & VTD_INV_DESC_CC_G) {
+    case VTD_INV_DESC_CC_DOMAIN:
+        VTD_DPRINTF(INV, "domain-selective invalidation domain 0x%"PRIx16,
+                    (uint16_t)VTD_INV_DESC_CC_DID(inv_desc->lo));
+        /* Fall through */
+    case VTD_INV_DESC_CC_GLOBAL:
+        VTD_DPRINTF(INV, "global invalidation");
+        vtd_context_global_invalidate(s);
+        break;
+
+    case VTD_INV_DESC_CC_DEVICE:
+        vtd_context_device_invalidate(s, VTD_INV_DESC_CC_SID(inv_desc->lo),
+                                      VTD_INV_DESC_CC_FM(inv_desc->lo));
+        break;
+
+    default:
+        VTD_DPRINTF(GENERAL, "error: invalid granularity in Context-cache "
+                    "Invalidate Descriptor hi 0x%"PRIx64  " lo 0x%"PRIx64,
+                    inv_desc->hi, inv_desc->lo);
+        return false;
+    }
+    return true;
+}
+
 static bool vtd_process_inv_desc(IntelIOMMUState *s)
 {
     VTDInvDesc inv_desc;
@@ -1012,6 +1141,9 @@ static bool vtd_process_inv_desc(IntelIOMMUState *s)
     case VTD_INV_DESC_CC:
         VTD_DPRINTF(INV, "Context-cache Invalidate Descriptor hi 0x%"PRIx64
                     " lo 0x%"PRIx64, inv_desc.hi, inv_desc.lo);
+        if (!vtd_process_context_cache_desc(s, &inv_desc)) {
+            return false;
+        }
         break;
 
     case VTD_INV_DESC_IOTLB:
@@ -1453,8 +1585,6 @@ static IOMMUTLBEntry vtd_iommu_translate(MemoryRegion *iommu, hwaddr addr,
 {
     VTDAddressSpace *vtd_as = container_of(iommu, VTDAddressSpace, iommu);
     IntelIOMMUState *s = vtd_as->iommu_state;
-    uint8_t bus_num = vtd_as->bus_num;
-    uint8_t devfn = vtd_as->devfn;
     IOMMUTLBEntry ret = {
         .target_as = &address_space_memory,
         .iova = addr,
@@ -1472,13 +1602,13 @@ static IOMMUTLBEntry vtd_iommu_translate(MemoryRegion *iommu, hwaddr addr,
         return ret;
     }
 
-    vtd_do_iommu_translate(s, bus_num, devfn, addr, is_write, &ret);
-
+    vtd_do_iommu_translate(vtd_as, vtd_as->bus_num, vtd_as->devfn, addr,
+                           is_write, &ret);
     VTD_DPRINTF(MMU,
                 "bus %"PRIu8 " slot %"PRIu8 " func %"PRIu8 " devfn %"PRIu8
-                " gpa 0x%"PRIx64 " hpa 0x%"PRIx64, bus_num,
-                VTD_PCI_SLOT(devfn), VTD_PCI_FUNC(devfn), devfn, addr,
-                ret.translated_addr);
+                " gpa 0x%"PRIx64 " hpa 0x%"PRIx64, vtd_as->bus_num,
+                VTD_PCI_SLOT(vtd_as->devfn), VTD_PCI_FUNC(vtd_as->devfn),
+                vtd_as->devfn, addr, ret.translated_addr);
     return ret;
 }
 
@@ -1531,6 +1661,8 @@ static void vtd_init(IntelIOMMUState *s)
              VTD_CAP_SAGAW;
     s->ecap = VTD_ECAP_QI | VTD_ECAP_IRO;
 
+    vtd_reset_context_cache(s);
+
     /* Define registers with default values and bit semantics */
     vtd_define_long(s, DMAR_VER_REG, 0x10UL, 0, 0);
     vtd_define_quad(s, DMAR_CAP_REG, s->cap, 0, 0);
diff --git a/hw/i386/intel_iommu_internal.h b/hw/i386/intel_iommu_internal.h
index cbcc8d1..30c318d 100644
--- a/hw/i386/intel_iommu_internal.h
+++ b/hw/i386/intel_iommu_internal.h
@@ -154,6 +154,9 @@
 #define VTD_CCMD_DOMAIN_INVL_A      (2ULL << 59)
 #define VTD_CCMD_DEVICE_INVL_A      (3ULL << 59)
 #define VTD_CCMD_CAIG_MASK          (3ULL << 59)
+#define VTD_CCMD_DID(val)           ((val) & VTD_DOMAIN_ID_MASK)
+#define VTD_CCMD_SID(val)           (((val) >> 16) & 0xffffULL)
+#define VTD_CCMD_FM(val)            (((val) >> 32) & 3ULL)
 
 /* RTADDR_REG */
 #define VTD_RTADDR_RTT              (1ULL << 11)
@@ -169,6 +172,7 @@
 #define VTD_CAP_FRO                 (DMAR_FRCD_REG_OFFSET << 20)
 #define VTD_CAP_NFR                 ((DMAR_FRCD_REG_NR - 1) << 40)
 #define VTD_DOMAIN_ID_SHIFT         16  /* 16-bit domain id for 64K domains */
+#define VTD_DOMAIN_ID_MASK          ((1UL << VTD_DOMAIN_ID_SHIFT) - 1)
 #define VTD_CAP_ND                  (((VTD_DOMAIN_ID_SHIFT - 4) / 2) & 7ULL)
 #define VTD_MGAW                    39  /* Maximum Guest Address Width */
 #define VTD_CAP_MGAW                (((VTD_MGAW - 1) & 0x3fULL) << 16)
@@ -255,6 +259,8 @@ typedef enum VTDFaultReason {
     VTD_FR_MAX,                 /* Guard */
 } VTDFaultReason;
 
+#define VTD_CONTEXT_CACHE_GEN_MAX       0xffffffffUL
+
 /* Queued Invalidation Descriptor */
 struct VTDInvDesc {
     uint64_t lo;
@@ -277,6 +283,16 @@ typedef struct VTDInvDesc VTDInvDesc;
 #define VTD_INV_DESC_WAIT_RSVD_LO       0Xffffff80ULL
 #define VTD_INV_DESC_WAIT_RSVD_HI       3ULL
 
+/* Masks for Context-cache Invalidation Descriptor */
+#define VTD_INV_DESC_CC_G               (3ULL << 4)
+#define VTD_INV_DESC_CC_GLOBAL          (1ULL << 4)
+#define VTD_INV_DESC_CC_DOMAIN          (2ULL << 4)
+#define VTD_INV_DESC_CC_DEVICE          (3ULL << 4)
+#define VTD_INV_DESC_CC_DID(val)        (((val) >> 16) & VTD_DOMAIN_ID_MASK)
+#define VTD_INV_DESC_CC_SID(val)        (((val) >> 32) & 0xffffUL)
+#define VTD_INV_DESC_CC_FM(val)         (((val) >> 48) & 3UL)
+#define VTD_INV_DESC_CC_RSVD            0xfffc00000000ffc0ULL
+
 /* Pagesize of VTD paging structures, including root and context tables */
 #define VTD_PAGE_SHIFT              12
 #define VTD_PAGE_SIZE               (1ULL << VTD_PAGE_SHIFT)
@@ -301,13 +317,6 @@ typedef struct VTDRootEntry VTDRootEntry;
 #define VTD_ROOT_ENTRY_NR           (VTD_PAGE_SIZE / sizeof(VTDRootEntry))
 #define VTD_ROOT_ENTRY_RSVD         (0xffeULL | ~VTD_HAW_MASK)
 
-/* Context-Entry */
-struct VTDContextEntry {
-    uint64_t lo;
-    uint64_t hi;
-};
-typedef struct VTDContextEntry VTDContextEntry;
-
 /* Masks for struct VTDContextEntry */
 /* lo */
 #define VTD_CONTEXT_ENTRY_P         (1ULL << 0)
diff --git a/hw/pci-host/q35.c b/hw/pci-host/q35.c
index 057cab6..b20bad8 100644
--- a/hw/pci-host/q35.c
+++ b/hw/pci-host/q35.c
@@ -368,6 +368,7 @@ static AddressSpace *q35_host_dma_iommu(PCIBus *bus, void *opaque, int devfn)
         pvtd_as[devfn]->bus_num = (uint8_t)bus_num;
         pvtd_as[devfn]->devfn = (uint8_t)devfn;
         pvtd_as[devfn]->iommu_state = s;
+        pvtd_as[devfn]->context_cache_entry.context_cache_gen = 0;
         memory_region_init_iommu(&pvtd_as[devfn]->iommu, OBJECT(s),
                                  &s->iommu_ops, "intel_iommu", UINT64_MAX);
         address_space_init(&pvtd_as[devfn]->as,
diff --git a/include/hw/i386/intel_iommu.h b/include/hw/i386/intel_iommu.h
index fe1f1e9..d9a5215 100644
--- a/include/hw/i386/intel_iommu.h
+++ b/include/hw/i386/intel_iommu.h
@@ -37,20 +37,40 @@
 #define VTD_PCI_DEVFN_MAX           256
 #define VTD_PCI_SLOT(devfn)         (((devfn) >> 3) & 0x1f)
 #define VTD_PCI_FUNC(devfn)         ((devfn) & 0x07)
+#define VTD_SID_TO_BUS(sid)         (((sid) >> 8) && 0xff)
+#define VTD_SID_TO_DEVFN(sid)       ((sid) & 0xff)
 
 #define DMAR_REG_SIZE               0x230
 #define VTD_HOST_ADDRESS_WIDTH      39
 #define VTD_HAW_MASK                ((1ULL << VTD_HOST_ADDRESS_WIDTH) - 1)
 
+typedef struct VTDContextEntry VTDContextEntry;
+typedef struct VTDContextCacheEntry VTDContextCacheEntry;
 typedef struct IntelIOMMUState IntelIOMMUState;
 typedef struct VTDAddressSpace VTDAddressSpace;
 
+
+/* Context-Entry */
+struct VTDContextEntry {
+    uint64_t lo;
+    uint64_t hi;
+};
+
+struct VTDContextCacheEntry {
+    /* The cache entry is obsolete if
+     * context_cache_gen!=IntelIOMMUState.context_cache_gen
+     */
+    uint32_t context_cache_gen;
+    struct VTDContextEntry context_entry;
+};
+
 struct VTDAddressSpace {
     uint8_t bus_num;
     uint8_t devfn;
     AddressSpace as;
     MemoryRegion iommu;
     IntelIOMMUState *iommu_state;
+    VTDContextCacheEntry context_cache_entry;
 };
 
 /* The iommu (DMAR) device state struct */
@@ -82,6 +102,8 @@ struct IntelIOMMUState {
     uint64_t cap;                   /* The value of capability reg */
     uint64_t ecap;                  /* The value of extended capability reg */
 
+    uint32_t context_cache_gen;     /* Should be in [1,MAX] */
+
     MemoryRegionIOMMUOps iommu_ops;
     VTDAddressSpace **address_spaces[VTD_PCI_BUS_MAX];
 };