From c826a6e1064419f78855463cf29ce9e8b9d25bf4 Mon Sep 17 00:00:00 2001 From: Eric Anholt Date: Fri, 9 Oct 2015 20:25:07 -0700 Subject: drm/vc4: Add a BO cache. We need to allocate new BOs in the kernel as part of each frame, but the CMA allocator is way too slow for that. As an optimization, keep track of recently-freed BOs and reuse them, with a 1 second timeout to fully free them back to the system. This improves 3D performance by about 15%. Signed-off-by: Eric Anholt --- drivers/gpu/drm/vc4/vc4_bo.c | 336 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 330 insertions(+), 6 deletions(-) (limited to 'drivers/gpu/drm/vc4/vc4_bo.c') diff --git a/drivers/gpu/drm/vc4/vc4_bo.c b/drivers/gpu/drm/vc4/vc4_bo.c index ab9f510..18faa5b 100644 --- a/drivers/gpu/drm/vc4/vc4_bo.c +++ b/drivers/gpu/drm/vc4/vc4_bo.c @@ -12,19 +12,229 @@ * access to system memory with no MMU in between. To support it, we * use the GEM CMA helper functions to allocate contiguous ranges of * physical memory for our BOs. + * + * Since the CMA allocator is very slow, we keep a cache of recently + * freed BOs around so that the kernel's allocation of objects for 3D + * rendering can return quickly. */ #include "vc4_drv.h" -struct vc4_bo *vc4_bo_create(struct drm_device *dev, size_t size) +static void vc4_bo_stats_dump(struct vc4_dev *vc4) +{ + DRM_INFO("num bos allocated: %d\n", + vc4->bo_stats.num_allocated); + DRM_INFO("size bos allocated: %dkb\n", + vc4->bo_stats.size_allocated / 1024); + DRM_INFO("num bos used: %d\n", + vc4->bo_stats.num_allocated - vc4->bo_stats.num_cached); + DRM_INFO("size bos used: %dkb\n", + (vc4->bo_stats.size_allocated - + vc4->bo_stats.size_cached) / 1024); + DRM_INFO("num bos cached: %d\n", + vc4->bo_stats.num_cached); + DRM_INFO("size bos cached: %dkb\n", + vc4->bo_stats.size_cached / 1024); +} + +#ifdef CONFIG_DEBUG_FS +int vc4_bo_stats_debugfs(struct seq_file *m, void *unused) +{ + struct drm_info_node *node = (struct drm_info_node *)m->private; + struct drm_device *dev = node->minor->dev; + struct vc4_dev *vc4 = to_vc4_dev(dev); + struct vc4_bo_stats stats; + + /* Take a snapshot of the current stats with the lock held. */ + mutex_lock(&vc4->bo_lock); + stats = vc4->bo_stats; + mutex_unlock(&vc4->bo_lock); + + seq_printf(m, "num bos allocated: %d\n", + stats.num_allocated); + seq_printf(m, "size bos allocated: %dkb\n", + stats.size_allocated / 1024); + seq_printf(m, "num bos used: %d\n", + stats.num_allocated - stats.num_cached); + seq_printf(m, "size bos used: %dkb\n", + (stats.size_allocated - stats.size_cached) / 1024); + seq_printf(m, "num bos cached: %d\n", + stats.num_cached); + seq_printf(m, "size bos cached: %dkb\n", + stats.size_cached / 1024); + + return 0; +} +#endif + +static uint32_t bo_page_index(size_t size) +{ + return (size / PAGE_SIZE) - 1; +} + +/* Must be called with bo_lock held. */ +static void vc4_bo_destroy(struct vc4_bo *bo) { + struct drm_gem_object *obj = &bo->base.base; + struct vc4_dev *vc4 = to_vc4_dev(obj->dev); + + vc4->bo_stats.num_allocated--; + vc4->bo_stats.size_allocated -= obj->size; + drm_gem_cma_free_object(obj); +} + +/* Must be called with bo_lock held. */ +static void vc4_bo_remove_from_cache(struct vc4_bo *bo) +{ + struct drm_gem_object *obj = &bo->base.base; + struct vc4_dev *vc4 = to_vc4_dev(obj->dev); + + vc4->bo_stats.num_cached--; + vc4->bo_stats.size_cached -= obj->size; + + list_del(&bo->unref_head); + list_del(&bo->size_head); +} + +static struct list_head *vc4_get_cache_list_for_size(struct drm_device *dev, + size_t size) +{ + struct vc4_dev *vc4 = to_vc4_dev(dev); + uint32_t page_index = bo_page_index(size); + + if (vc4->bo_cache.size_list_size <= page_index) { + uint32_t new_size = max(vc4->bo_cache.size_list_size * 2, + page_index + 1); + struct list_head *new_list; + uint32_t i; + + new_list = kmalloc_array(new_size, sizeof(struct list_head), + GFP_KERNEL); + if (!new_list) + return NULL; + + /* Rebase the old cached BO lists to their new list + * head locations. + */ + for (i = 0; i < vc4->bo_cache.size_list_size; i++) { + struct list_head *old_list = + &vc4->bo_cache.size_list[i]; + + if (list_empty(old_list)) + INIT_LIST_HEAD(&new_list[i]); + else + list_replace(old_list, &new_list[i]); + } + /* And initialize the brand new BO list heads. */ + for (i = vc4->bo_cache.size_list_size; i < new_size; i++) + INIT_LIST_HEAD(&new_list[i]); + + kfree(vc4->bo_cache.size_list); + vc4->bo_cache.size_list = new_list; + vc4->bo_cache.size_list_size = new_size; + } + + return &vc4->bo_cache.size_list[page_index]; +} + +void vc4_bo_cache_purge(struct drm_device *dev) +{ + struct vc4_dev *vc4 = to_vc4_dev(dev); + + mutex_lock(&vc4->bo_lock); + while (!list_empty(&vc4->bo_cache.time_list)) { + struct vc4_bo *bo = list_last_entry(&vc4->bo_cache.time_list, + struct vc4_bo, unref_head); + vc4_bo_remove_from_cache(bo); + vc4_bo_destroy(bo); + } + mutex_unlock(&vc4->bo_lock); +} + +static struct vc4_bo *vc4_bo_get_from_cache(struct drm_device *dev, + uint32_t size) +{ + struct vc4_dev *vc4 = to_vc4_dev(dev); + uint32_t page_index = bo_page_index(size); + struct vc4_bo *bo = NULL; + + size = roundup(size, PAGE_SIZE); + + mutex_lock(&vc4->bo_lock); + if (page_index >= vc4->bo_cache.size_list_size) + goto out; + + if (list_empty(&vc4->bo_cache.size_list[page_index])) + goto out; + + bo = list_first_entry(&vc4->bo_cache.size_list[page_index], + struct vc4_bo, size_head); + vc4_bo_remove_from_cache(bo); + kref_init(&bo->base.base.refcount); + +out: + mutex_unlock(&vc4->bo_lock); + return bo; +} + +/** + * vc4_gem_create_object - Implementation of driver->gem_create_object. + * + * This lets the CMA helpers allocate object structs for us, and keep + * our BO stats correct. + */ +struct drm_gem_object *vc4_create_object(struct drm_device *dev, size_t size) +{ + struct vc4_dev *vc4 = to_vc4_dev(dev); + struct vc4_bo *bo; + + bo = kzalloc(sizeof(*bo), GFP_KERNEL); + if (!bo) + return ERR_PTR(-ENOMEM); + + mutex_lock(&vc4->bo_lock); + vc4->bo_stats.num_allocated++; + vc4->bo_stats.size_allocated += size; + mutex_unlock(&vc4->bo_lock); + + return &bo->base.base; +} + +struct vc4_bo *vc4_bo_create(struct drm_device *dev, size_t unaligned_size, + bool from_cache) +{ + size_t size = roundup(unaligned_size, PAGE_SIZE); + struct vc4_dev *vc4 = to_vc4_dev(dev); struct drm_gem_cma_object *cma_obj; - cma_obj = drm_gem_cma_create(dev, size); - if (IS_ERR(cma_obj)) + if (size == 0) return NULL; - else - return to_vc4_bo(&cma_obj->base); + + /* First, try to get a vc4_bo from the kernel BO cache. */ + if (from_cache) { + struct vc4_bo *bo = vc4_bo_get_from_cache(dev, size); + + if (bo) + return bo; + } + + cma_obj = drm_gem_cma_create(dev, size); + if (IS_ERR(cma_obj)) { + /* + * If we've run out of CMA memory, kill the cache of + * CMA allocations we've got laying around and try again. + */ + vc4_bo_cache_purge(dev); + + cma_obj = drm_gem_cma_create(dev, size); + if (IS_ERR(cma_obj)) { + DRM_ERROR("Failed to allocate from CMA:\n"); + vc4_bo_stats_dump(vc4); + return NULL; + } + } + + return to_vc4_bo(&cma_obj->base); } int vc4_dumb_create(struct drm_file *file_priv, @@ -41,7 +251,7 @@ int vc4_dumb_create(struct drm_file *file_priv, if (args->size < args->pitch * args->height) args->size = args->pitch * args->height; - bo = vc4_bo_create(dev, roundup(args->size, PAGE_SIZE)); + bo = vc4_bo_create(dev, args->size, false); if (!bo) return -ENOMEM; @@ -50,3 +260,117 @@ int vc4_dumb_create(struct drm_file *file_priv, return ret; } + +/* Must be called with bo_lock held. */ +static void vc4_bo_cache_free_old(struct drm_device *dev) +{ + struct vc4_dev *vc4 = to_vc4_dev(dev); + unsigned long expire_time = jiffies - msecs_to_jiffies(1000); + + while (!list_empty(&vc4->bo_cache.time_list)) { + struct vc4_bo *bo = list_last_entry(&vc4->bo_cache.time_list, + struct vc4_bo, unref_head); + if (time_before(expire_time, bo->free_time)) { + mod_timer(&vc4->bo_cache.time_timer, + round_jiffies_up(jiffies + + msecs_to_jiffies(1000))); + return; + } + + vc4_bo_remove_from_cache(bo); + vc4_bo_destroy(bo); + } +} + +/* Called on the last userspace/kernel unreference of the BO. Returns + * it to the BO cache if possible, otherwise frees it. + * + * Note that this is called with the struct_mutex held. + */ +void vc4_free_object(struct drm_gem_object *gem_bo) +{ + struct drm_device *dev = gem_bo->dev; + struct vc4_dev *vc4 = to_vc4_dev(dev); + struct vc4_bo *bo = to_vc4_bo(gem_bo); + struct list_head *cache_list; + + mutex_lock(&vc4->bo_lock); + /* If the object references someone else's memory, we can't cache it. + */ + if (gem_bo->import_attach) { + vc4_bo_destroy(bo); + goto out; + } + + /* Don't cache if it was publicly named. */ + if (gem_bo->name) { + vc4_bo_destroy(bo); + goto out; + } + + cache_list = vc4_get_cache_list_for_size(dev, gem_bo->size); + if (!cache_list) { + vc4_bo_destroy(bo); + goto out; + } + + bo->free_time = jiffies; + list_add(&bo->size_head, cache_list); + list_add(&bo->unref_head, &vc4->bo_cache.time_list); + + vc4->bo_stats.num_cached++; + vc4->bo_stats.size_cached += gem_bo->size; + + vc4_bo_cache_free_old(dev); + +out: + mutex_unlock(&vc4->bo_lock); +} + +static void vc4_bo_cache_time_work(struct work_struct *work) +{ + struct vc4_dev *vc4 = + container_of(work, struct vc4_dev, bo_cache.time_work); + struct drm_device *dev = vc4->dev; + + mutex_lock(&vc4->bo_lock); + vc4_bo_cache_free_old(dev); + mutex_unlock(&vc4->bo_lock); +} + +static void vc4_bo_cache_time_timer(unsigned long data) +{ + struct drm_device *dev = (struct drm_device *)data; + struct vc4_dev *vc4 = to_vc4_dev(dev); + + schedule_work(&vc4->bo_cache.time_work); +} + +void vc4_bo_cache_init(struct drm_device *dev) +{ + struct vc4_dev *vc4 = to_vc4_dev(dev); + + mutex_init(&vc4->bo_lock); + + INIT_LIST_HEAD(&vc4->bo_cache.time_list); + + INIT_WORK(&vc4->bo_cache.time_work, vc4_bo_cache_time_work); + setup_timer(&vc4->bo_cache.time_timer, + vc4_bo_cache_time_timer, + (unsigned long)dev); +} + +void vc4_bo_cache_destroy(struct drm_device *dev) +{ + struct vc4_dev *vc4 = to_vc4_dev(dev); + + del_timer(&vc4->bo_cache.time_timer); + cancel_work_sync(&vc4->bo_cache.time_work); + + vc4_bo_cache_purge(dev); + + if (vc4->bo_stats.num_allocated) { + DRM_ERROR("Destroying BO cache while BOs still allocated:\n"); + vc4_bo_stats_dump(vc4); + } +} -- cgit v1.1