resync to git master

This reverts a private patch which is causing issues with many Intel chipsets.
I will review that patch and see what we need to do to fix it up later, but
for the time being, we will just get these chips working again.

This update contains a lot of code cleanup and is post gem merge
(no, we don't have gem support).  It should prove much easier to read the
code now.  A lot of thanks goes to vehemens for that work.  I have adapted
the code to use cdevpriv for tracking per open file data.  That alleviates
the old ugly hack that we used to try and accomplish the task and helped to
clean up the open / close behavior a good bit.  This also replaces the hack
that was put in place a year or so ago to prevent radeons from locking up
with AIGLX enabled.  I have had a couple of radeon testers report that it
still works as expected, though I no longer have radeon hardware to test with
myself.  Other various fixes from the linux crew and Intel, many of
which are muddled in with the gem merge.

Approved by:	jhb (mentor)
Obtained from:	mesa/drm git master
MFC after:	2 weeks

1 files changed, 215 insertions, 148 deletions

diff --git a/sys/dev/drm/i915_dma.c b/sys/dev/drm/i915_dma.c
index 79e687e..39a0809 100644
--- a/sys/dev/drm/i915_dma.c
+++ b/sys/dev/drm/i915_dma.c
@@ -44,10 +44,14 @@ int i915_wait_ring(struct drm_device * dev, int n, const char *caller)
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	drm_i915_ring_buffer_t *ring = &(dev_priv->ring);
 	u32 last_head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
+	u32 acthd_reg = IS_I965G(dev) ? ACTHD_I965 : ACTHD;
+	u32 last_acthd = I915_READ(acthd_reg);
+	u32 acthd;
 	int i;
 
-	for (i = 0; i < 10000; i++) {
+	for (i = 0; i < 100000; i++) {
 		ring->head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
+		acthd = I915_READ(acthd_reg);
 		ring->space = ring->head - (ring->tail + 8);
 		if (ring->space < 0)
 			ring->space += ring->Size;
@@ -57,13 +61,89 @@ int i915_wait_ring(struct drm_device * dev, int n, const char *caller)
 		if (ring->head != last_head)
 			i = 0;
 
+		if (acthd != last_acthd)
+			i = 0;
+
 		last_head = ring->head;
-		DRM_UDELAY(1);
+		last_acthd = acthd;
+		DRM_UDELAY(10 * 1000);
 	}
 
 	return -EBUSY;
 }
 
+int i915_init_hardware_status(struct drm_device *dev)
+{
+	drm_i915_private_t *dev_priv = dev->dev_private;
+	drm_dma_handle_t *dmah;
+
+	/* Program Hardware Status Page */
+#ifdef __FreeBSD__
+	DRM_UNLOCK();
+#endif
+	dmah = drm_pci_alloc(dev, PAGE_SIZE, PAGE_SIZE, 0xffffffff);
+#ifdef __FreeBSD__
+	DRM_LOCK();
+#endif
+	if (!dmah) {
+		DRM_ERROR("Can not allocate hardware status page\n");
+		return -ENOMEM;
+	}
+
+	dev_priv->status_page_dmah = dmah;
+	dev_priv->hw_status_page = dmah->vaddr;
+	dev_priv->dma_status_page = dmah->busaddr;
+
+	memset(dev_priv->hw_status_page, 0, PAGE_SIZE);
+
+	I915_WRITE(0x02080, dev_priv->dma_status_page);
+	DRM_DEBUG("Enabled hardware status page\n");
+	return 0;
+}
+
+void i915_free_hardware_status(struct drm_device *dev)
+{
+	drm_i915_private_t *dev_priv = dev->dev_private;
+	if (dev_priv->status_page_dmah) {
+		drm_pci_free(dev, dev_priv->status_page_dmah);
+		dev_priv->status_page_dmah = NULL;
+		/* Need to rewrite hardware status page */
+		I915_WRITE(0x02080, 0x1ffff000);
+	}
+
+	if (dev_priv->status_gfx_addr) {
+		dev_priv->status_gfx_addr = 0;
+		drm_core_ioremapfree(&dev_priv->hws_map, dev);
+		I915_WRITE(0x02080, 0x1ffff000);
+	}
+}
+
+#if I915_RING_VALIDATE
+/**
+ * Validate the cached ring tail value
+ *
+ * If the X server writes to the ring and DRM doesn't
+ * reload the head and tail pointers, it will end up writing
+ * data to the wrong place in the ring, causing havoc.
+ */
+void i915_ring_validate(struct drm_device *dev, const char *func, int line)
+{
+	drm_i915_private_t *dev_priv = dev->dev_private;
+	drm_i915_ring_buffer_t *ring = &(dev_priv->ring);
+	u32	tail = I915_READ(PRB0_TAIL) & HEAD_ADDR;
+	u32	head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
+
+	if (tail != ring->tail) {
+		DRM_ERROR("%s:%d head sw %x, hw %x. tail sw %x hw %x\n",
+			  func, line,
+			  ring->head, head, ring->tail, tail);
+#ifdef __linux__
+		BUG_ON(1);
+#endif
+	}
+}
+#endif
+
 void i915_kernel_lost_context(struct drm_device * dev)
 {
 	drm_i915_private_t *dev_priv = dev->dev_private;
@@ -83,7 +163,7 @@ static int i915_dma_cleanup(struct drm_device * dev)
 	 * may not have been called from userspace and after dev_private
 	 * is freed, it's too late.
 	 */
-	if (dev->irq)
+	if (dev->irq_enabled)
 		drm_irq_uninstall(dev);
 
 	if (dev_priv->ring.virtual_start) {
@@ -93,18 +173,8 @@ static int i915_dma_cleanup(struct drm_device * dev)
 		dev_priv->ring.map.size = 0;
 	}
 
-	if (dev_priv->status_page_dmah) {
-		drm_pci_free(dev, dev_priv->status_page_dmah);
-		dev_priv->status_page_dmah = NULL;
-		/* Need to rewrite hardware status page */
-		I915_WRITE(0x02080, 0x1ffff000);
-	}
-
-	if (dev_priv->status_gfx_addr) {
-		dev_priv->status_gfx_addr = 0;
-		drm_core_ioremapfree(&dev_priv->hws_map, dev);
-		I915_WRITE(0x02080, 0x1ffff000);
-	}
+	if (I915_NEED_GFX_HWS(dev))
+		i915_free_hardware_status(dev);
 
 	return 0;
 }
@@ -185,14 +255,6 @@ static int i915_initialize(struct drm_device * dev,
 		return -EINVAL;
 	}
 
-	if (init->mmio_offset != 0)
-		dev_priv->mmio_map = drm_core_findmap(dev, init->mmio_offset);
-	if (!dev_priv->mmio_map) {
-		i915_dma_cleanup(dev);
-		DRM_ERROR("can not find mmio map!\n");
-		return -EINVAL;
-	}
-
 #ifdef I915_HAVE_BUFFER
 	dev_priv->max_validate_buffers = I915_MAX_VALIDATE_BUFFERS;
 #endif
@@ -206,27 +268,27 @@ static int i915_initialize(struct drm_device * dev,
 		dev_priv->sarea_priv = NULL;
 	}
 
-	dev_priv->ring.Start = init->ring_start;
-	dev_priv->ring.End = init->ring_end;
-	dev_priv->ring.Size = init->ring_size;
-	dev_priv->ring.tail_mask = dev_priv->ring.Size - 1;
+	if (init->ring_size != 0) {
+		dev_priv->ring.Size = init->ring_size;
+		dev_priv->ring.tail_mask = dev_priv->ring.Size - 1;
 
-	dev_priv->ring.map.offset = init->ring_start;
-	dev_priv->ring.map.size = init->ring_size;
-	dev_priv->ring.map.type = 0;
-	dev_priv->ring.map.flags = 0;
-	dev_priv->ring.map.mtrr = 0;
+		dev_priv->ring.map.offset = init->ring_start;
+		dev_priv->ring.map.size = init->ring_size;
+		dev_priv->ring.map.type = 0;
+		dev_priv->ring.map.flags = 0;
+		dev_priv->ring.map.mtrr = 0;
 
-	drm_core_ioremap(&dev_priv->ring.map, dev);
+		drm_core_ioremap(&dev_priv->ring.map, dev);
 
-	if (dev_priv->ring.map.handle == NULL) {
-		i915_dma_cleanup(dev);
-		DRM_ERROR("can not ioremap virtual address for"
-			  " ring buffer\n");
-		return -ENOMEM;
-	}
+		if (dev_priv->ring.map.handle == NULL) {
+			i915_dma_cleanup(dev);
+			DRM_ERROR("can not ioremap virtual address for"
+				  " ring buffer\n");
+			return -ENOMEM;
+		}
 
-	dev_priv->ring.virtual_start = dev_priv->ring.map.handle;
+		dev_priv->ring.virtual_start = dev_priv->ring.map.handle;
+	}
 
 	dev_priv->cpp = init->cpp;
 
@@ -236,9 +298,6 @@ static int i915_initialize(struct drm_device * dev,
 	/* We are using separate values as placeholders for mechanisms for
 	 * private backbuffer/depthbuffer usage.
 	 */
-	dev_priv->use_mi_batchbuffer_start = 0;
-	if (IS_I965G(dev)) /* 965 doesn't support older method */
-		dev_priv->use_mi_batchbuffer_start = 1;
 
 	/* Allow hardware batchbuffers unless told otherwise.
 	 */
@@ -248,30 +307,6 @@ static int i915_initialize(struct drm_device * dev,
 	 */
 	dev_priv->vblank_pipe = DRM_I915_VBLANK_PIPE_A;
 
-	/* Program Hardware Status Page */
-	if (!I915_NEED_GFX_HWS(dev)) {
-		drm_dma_handle_t *dmah;
-#ifdef __FreeBSD__
-		DRM_UNLOCK();
-#endif
-		dmah = drm_pci_alloc(dev, PAGE_SIZE, PAGE_SIZE, 0xffffffff);
-#ifdef __FreeBSD__
-		DRM_LOCK();
-#endif
-		if (!dmah) {
-			i915_dma_cleanup(dev);
-			DRM_ERROR("Can not allocate hardware status page\n");
-			return -ENOMEM;
-		}
-		dev_priv->status_page_dmah = dmah;
-		dev_priv->hw_status_page = dmah->vaddr;
-		dev_priv->dma_status_page = dmah->busaddr;
-
-		memset(dev_priv->hw_status_page, 0, PAGE_SIZE);
-
-		I915_WRITE(0x02080, dev_priv->dma_status_page);
-	}
-	DRM_DEBUG("Enabled hardware status page\n");
 #ifdef I915_HAVE_BUFFER
 	mutex_init(&dev_priv->cmdbuf_mutex);
 #endif
@@ -300,11 +335,6 @@ static int i915_dma_resume(struct drm_device * dev)
 		return -EINVAL;
 	}
 
-	if (!dev_priv->mmio_map) {
-		DRM_ERROR("can not find mmio map!\n");
-		return -EINVAL;
-	}
-
 	if (dev_priv->ring.map.handle == NULL) {
 		DRM_ERROR("can not ioremap virtual address for"
 			  " ring buffer\n");
@@ -443,15 +473,20 @@ static int i915_emit_cmds(struct drm_device *dev, int __user *buffer,
 	for (i = 0; i < dwords;) {
 		int cmd, sz;
 
-		cmd = buffer[i];
+		if (DRM_COPY_FROM_USER_UNCHECKED(&cmd, &buffer[i], sizeof(cmd)))
+			return -EINVAL;
+
 		if ((sz = validate_cmd(cmd)) == 0 || i + sz > dwords)
 			return -EINVAL;
 
 		OUT_RING(cmd);
 
 		while (++i, --sz) {
+			if (DRM_COPY_FROM_USER_UNCHECKED(&cmd, &buffer[i],
+							 sizeof(cmd))) {
+				return -EINVAL;
+			}
 			OUT_RING(cmd);
-			cmd = buffer[i];
 		}
 	}
 
@@ -463,15 +498,18 @@ static int i915_emit_cmds(struct drm_device *dev, int __user *buffer,
 	return 0;
 }
 
-static int i915_emit_box(struct drm_device * dev,
-			 struct drm_clip_rect __user * boxes,
-			 int i, int DR1, int DR4)
+int i915_emit_box(struct drm_device * dev,
+		  struct drm_clip_rect __user * boxes,
+		  int i, int DR1, int DR4)
 {
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	struct drm_clip_rect box;
 	RING_LOCALS;
 
-	box = boxes[i];
+	if (DRM_COPY_FROM_USER_UNCHECKED(&box, &boxes[i], sizeof(box))) {
+		return -EFAULT;
+	}
+
 	if (box.y2 <= box.y1 || box.x2 <= box.x1 || box.y2 <= 0 || box.x2 <= 0) {
 		DRM_ERROR("Bad box %d,%d..%d,%d\n",
 			  box.x1, box.y1, box.x2, box.y2);
@@ -518,7 +556,7 @@ void i915_emit_breadcrumb(struct drm_device *dev)
 
 	BEGIN_LP_RING(4);
 	OUT_RING(MI_STORE_DWORD_INDEX);
-	OUT_RING(20);
+	OUT_RING(5 << MI_STORE_DWORD_INDEX_SHIFT);
 	OUT_RING(dev_priv->counter);
 	OUT_RING(0);
 	ADVANCE_LP_RING();
@@ -611,7 +649,14 @@ int i915_dispatch_batchbuffer(struct drm_device * dev,
 				return ret;
 		}
 
-		if (dev_priv->use_mi_batchbuffer_start) {
+		if (IS_I830(dev) || IS_845G(dev)) {
+			BEGIN_LP_RING(4);
+			OUT_RING(MI_BATCH_BUFFER);
+			OUT_RING(batch->start | MI_BATCH_NON_SECURE);
+			OUT_RING(batch->start + batch->used - 4);
+			OUT_RING(0);
+			ADVANCE_LP_RING();
+		} else {
 			BEGIN_LP_RING(2);
 			if (IS_I965G(dev)) {
 				OUT_RING(MI_BATCH_BUFFER_START | (2 << 6) | MI_BATCH_NON_SECURE_I965);
@@ -621,14 +666,6 @@ int i915_dispatch_batchbuffer(struct drm_device * dev,
 				OUT_RING(batch->start | MI_BATCH_NON_SECURE);
 			}
 			ADVANCE_LP_RING();
-
-		} else {
-			BEGIN_LP_RING(4);
-			OUT_RING(MI_BATCH_BUFFER);
-			OUT_RING(batch->start | MI_BATCH_NON_SECURE);
-			OUT_RING(batch->start + batch->used - 4);
-			OUT_RING(0);
-			ADVANCE_LP_RING();
 		}
 	}
 
@@ -716,9 +753,19 @@ void i915_dispatch_flip(struct drm_device * dev, int planes, int sync)
 int i915_quiescent(struct drm_device *dev)
 {
 	drm_i915_private_t *dev_priv = dev->dev_private;
+	int ret;
 
 	i915_kernel_lost_context(dev);
-	return i915_wait_ring(dev, dev_priv->ring.Size - 8, __FUNCTION__);
+	ret = i915_wait_ring(dev, dev_priv->ring.Size - 8, __FUNCTION__);
+	if (ret)
+	{
+		i915_kernel_lost_context (dev);
+		DRM_ERROR ("not quiescent head %08x tail %08x space %08x\n",
+			   dev_priv->ring.head,
+			   dev_priv->ring.tail,
+			   dev_priv->ring.space);
+	}
+	return ret;
 }
 
 static int i915_flush_ioctl(struct drm_device *dev, void *data,
@@ -737,7 +784,6 @@ static int i915_batchbuffer(struct drm_device *dev, void *data,
 	drm_i915_sarea_t *sarea_priv = (drm_i915_sarea_t *)
 	    dev_priv->sarea_priv;
 	drm_i915_batchbuffer_t *batch = data;
-	size_t cliplen;
 	int ret;
 
 	if (!dev_priv->allow_batchbuffer) {
@@ -750,29 +796,14 @@ static int i915_batchbuffer(struct drm_device *dev, void *data,
 
 	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
-	DRM_UNLOCK();
-	cliplen = batch->num_cliprects * sizeof(drm_clip_rect_t);
 	if (batch->num_cliprects && DRM_VERIFYAREA_READ(batch->cliprects,
-	    cliplen)) {
-		DRM_LOCK();
+							batch->num_cliprects *
+							sizeof(struct drm_clip_rect)))
 		return -EFAULT;
-	}
-	if (batch->num_cliprects) {
-		ret = vslock(batch->cliprects, cliplen);
-		if (ret) {
-			DRM_ERROR("Fault wiring cliprects\n");
-			DRM_LOCK();
-			return -EFAULT;
-		}
-	}
-	DRM_LOCK();
+
 	ret = i915_dispatch_batchbuffer(dev, batch);
 
 	sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv);
-	DRM_UNLOCK();
-	if (batch->num_cliprects)
-	    vsunlock(batch->cliprects, cliplen);
-	DRM_LOCK();
 	return ret;
 }
 
@@ -783,7 +814,6 @@ static int i915_cmdbuffer(struct drm_device *dev, void *data,
 	drm_i915_sarea_t *sarea_priv = (drm_i915_sarea_t *)
 	    dev_priv->sarea_priv;
 	drm_i915_cmdbuffer_t *cmdbuf = data;
-	size_t cliplen;
 	int ret;
 
 	DRM_DEBUG("i915 cmdbuffer, buf %p sz %d cliprects %d\n",
@@ -791,42 +821,22 @@ static int i915_cmdbuffer(struct drm_device *dev, void *data,
 
 	LOCK_TEST_WITH_RETURN(dev, file_priv);
 
-	DRM_UNLOCK();
-	cliplen = cmdbuf->num_cliprects * sizeof(drm_clip_rect_t);
 	if (cmdbuf->num_cliprects &&
-	    DRM_VERIFYAREA_READ(cmdbuf->cliprects, cliplen)) {
+	    DRM_VERIFYAREA_READ(cmdbuf->cliprects,
+				cmdbuf->num_cliprects *
+				sizeof(struct drm_clip_rect))) {
 		DRM_ERROR("Fault accessing cliprects\n");
-		DRM_LOCK();
 		return -EFAULT;
 	}
-	if (cmdbuf->num_cliprects) {
-		ret = vslock(cmdbuf->cliprects, cliplen);
-		if (ret) {
-			DRM_ERROR("Fault wiring cliprects\n");
-			DRM_LOCK();
-			return -EFAULT;
-		}
-		ret = vslock(cmdbuf->buf, cmdbuf->sz);
-		if (ret) {
-			vsunlock(cmdbuf->cliprects, cliplen);
-			DRM_ERROR("Fault wiring cmds\n");
-			DRM_LOCK();
-			return -EFAULT;
-		}
-	}
-	DRM_LOCK();
+
 	ret = i915_dispatch_cmdbuffer(dev, cmdbuf);
-	if (ret == 0)
-		sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv);
-	else
+	if (ret) {
 		DRM_ERROR("i915_dispatch_cmdbuffer failed\n");
-	DRM_UNLOCK();
-	if (cmdbuf->num_cliprects) {
-		vsunlock(cmdbuf->buf, cmdbuf->sz);
-		vsunlock(cmdbuf->cliprects, cliplen);
+		return ret;
 	}
-	DRM_LOCK();
-	return (ret);
+
+	sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv);
+	return 0;
 }
 
 #if defined(DRM_DEBUG_CODE)
@@ -892,7 +902,7 @@ static int i915_getparam(struct drm_device *dev, void *data,
 
 	switch (param->param) {
 	case I915_PARAM_IRQ_ACTIVE:
-		value = dev->irq ? 1 : 0;
+		value = dev->irq_enabled ? 1 : 0;
 		break;
 	case I915_PARAM_ALLOW_BATCHBUFFER:
 		value = dev_priv->allow_batchbuffer ? 1 : 0;
@@ -903,6 +913,9 @@ static int i915_getparam(struct drm_device *dev, void *data,
 	case I915_PARAM_CHIPSET_ID:
 		value = dev->pci_device;
 		break;
+	case I915_PARAM_HAS_GEM:
+		value = 1;
+		break;
 	default:
 		DRM_ERROR("Unknown parameter %d\n", param->param);
 		return -EINVAL;
@@ -929,8 +942,6 @@ static int i915_setparam(struct drm_device *dev, void *data,
 
 	switch (param->param) {
 	case I915_SETPARAM_USE_MI_BATCHBUFFER_START:
-		if (!IS_I965G(dev))
-			dev_priv->use_mi_batchbuffer_start = param->value;
 		break;
 	case I915_SETPARAM_TEX_LRU_LOG_GRANULARITY:
 		dev_priv->tex_lru_log_granularity = param->value;
@@ -1064,6 +1075,7 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
 	memset(dev_priv, 0, sizeof(drm_i915_private_t));
 
 	dev->dev_private = (void *)dev_priv;
+	dev_priv->dev = dev;
 
 	/* Add register map (needed for suspend/resume) */
 	base = drm_get_resource_start(dev, mmio_bar);
@@ -1071,7 +1083,9 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
 
 	ret = drm_addmap(dev, base, size, _DRM_REGISTERS,
 		_DRM_KERNEL | _DRM_DRIVER, &dev_priv->mmio_map);
-
+#ifdef I915_HAVE_GEM
+	i915_gem_load(dev);
+#endif
 	DRM_SPININIT(&dev_priv->swaps_lock, "swap");
 	DRM_SPININIT(&dev_priv->user_irq_lock, "userirq");
 
@@ -1084,6 +1098,13 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
 #endif
 #endif
 
+	/* Init HWS */
+	if (!I915_NEED_GFX_HWS(dev)) {
+		ret = i915_init_hardware_status(dev);
+		if(ret)
+			return ret;
+	}
+
 	return ret;
 }
 
@@ -1091,8 +1112,9 @@ int i915_driver_unload(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
 
-	if (dev_priv->mmio_map)
-		drm_rmmap(dev, dev_priv->mmio_map);
+	i915_free_hardware_status(dev);
+
+    	drm_rmmap(dev, dev_priv->mmio_map);
 
 	DRM_SPINUNINIT(&dev_priv->swaps_lock);
 	DRM_SPINUNINIT(&dev_priv->user_irq_lock);
@@ -1129,12 +1151,13 @@ void i915_driver_lastclose(struct drm_device * dev)
 		dev_priv->val_bufs = NULL;
 	}
 #endif
-
-	if (drm_getsarea(dev) && dev_priv->sarea_priv) {
+#ifdef I915_HAVE_GEM
+	i915_gem_lastclose(dev);
+#endif
+	if (drm_getsarea(dev) && dev_priv->sarea_priv)
 		i915_do_cleanup_pageflip(dev);
+	if (dev_priv->sarea_priv)
 		dev_priv->sarea_priv = NULL;
-		dev_priv->sarea = NULL;
-	}
 	if (dev_priv->agp_heap)
 		i915_mem_takedown(&(dev_priv->agp_heap));
 #if defined(I915_HAVE_BUFFER)
@@ -1155,12 +1178,38 @@ void i915_driver_lastclose(struct drm_device * dev)
 	i915_dma_cleanup(dev);
 }
 
+int i915_driver_open(struct drm_device *dev, struct drm_file *file_priv)
+{
+	struct drm_i915_file_private *i915_file_priv;
+
+	DRM_DEBUG("\n");
+	i915_file_priv = (struct drm_i915_file_private *)
+	    drm_alloc(sizeof(*i915_file_priv), DRM_MEM_FILES);
+
+	if (!i915_file_priv)
+		return -ENOMEM;
+
+	file_priv->driver_priv = i915_file_priv;
+
+	i915_file_priv->mm.last_gem_seqno = 0;
+	i915_file_priv->mm.last_gem_throttle_seqno = 0;
+
+	return 0;
+}
+
 void i915_driver_preclose(struct drm_device * dev, struct drm_file *file_priv)
 {
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	i915_mem_release(dev, file_priv, dev_priv->agp_heap);
 }
 
+void i915_driver_postclose(struct drm_device *dev, struct drm_file *file_priv)
+{
+	struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
+
+	drm_free(i915_file_priv, sizeof(*i915_file_priv), DRM_MEM_FILES);
+}
+
 struct drm_ioctl_desc i915_ioctls[] = {
 	DRM_IOCTL_DEF(DRM_I915_INIT, i915_dma_init, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
 	DRM_IOCTL_DEF(DRM_I915_FLUSH, i915_flush_ioctl, DRM_AUTH),
@@ -1183,6 +1232,24 @@ struct drm_ioctl_desc i915_ioctls[] = {
 #ifdef I915_HAVE_BUFFER
 	DRM_IOCTL_DEF(DRM_I915_EXECBUFFER, i915_execbuffer, DRM_AUTH),
 #endif
+#ifdef I915_HAVE_GEM
+	DRM_IOCTL_DEF(DRM_I915_GEM_INIT, i915_gem_init_ioctl, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_I915_GEM_EXECBUFFER, i915_gem_execbuffer, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_I915_GEM_PIN, i915_gem_pin_ioctl, DRM_AUTH|DRM_ROOT_ONLY),
+	DRM_IOCTL_DEF(DRM_I915_GEM_UNPIN, i915_gem_unpin_ioctl, DRM_AUTH|DRM_ROOT_ONLY),
+	DRM_IOCTL_DEF(DRM_I915_GEM_BUSY, i915_gem_busy_ioctl, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_I915_GEM_THROTTLE, i915_gem_throttle_ioctl, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_I915_GEM_ENTERVT, i915_gem_entervt_ioctl, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_I915_GEM_LEAVEVT, i915_gem_leavevt_ioctl, DRM_AUTH),
+	DRM_IOCTL_DEF(DRM_I915_GEM_CREATE, i915_gem_create_ioctl, 0),
+	DRM_IOCTL_DEF(DRM_I915_GEM_PREAD, i915_gem_pread_ioctl, 0),
+	DRM_IOCTL_DEF(DRM_I915_GEM_PWRITE, i915_gem_pwrite_ioctl, 0),
+	DRM_IOCTL_DEF(DRM_I915_GEM_MMAP, i915_gem_mmap_ioctl, 0),
+	DRM_IOCTL_DEF(DRM_I915_GEM_SET_DOMAIN, i915_gem_set_domain_ioctl, 0),
+	DRM_IOCTL_DEF(DRM_I915_GEM_SW_FINISH, i915_gem_sw_finish_ioctl, 0),
+	DRM_IOCTL_DEF(DRM_I915_GEM_SET_TILING, i915_gem_set_tiling, 0),
+	DRM_IOCTL_DEF(DRM_I915_GEM_GET_TILING, i915_gem_get_tiling, 0),
+#endif
 };
 
 int i915_max_ioctl = DRM_ARRAY_SIZE(i915_ioctls);